[Backport] crypto: fix ecdsa malleability (#509) (#511)

Resolves babylonlabs-io/pm#194

CHANGELOG.md

@@ -44,6 +44,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ### Bug fixes
 
+- [#509](https://github.com/babylonlabs-io/babylon/pull/509) crypto: fix ECDSA malleability
 - [#486](https://github.com/babylonlabs-io/babylon/pull/486) crypto: blinding base mult of nonce
 
 ## v1.0.0-rc5

crypto/ecdsa/ecdsa.go

@@ -45,6 +45,13 @@ func Verify(pk *btcec.PublicKey, msg string, sigBytes []byte) error {
 	if err != nil {
 		return err
 	}
+	var s btcec.ModNScalar
+	if overflow := s.SetByteSlice(sigBytes[33:65]); overflow {
+		return fmt.Errorf("invalid signature: S >= group order")
+	}
+	if s.IsOverHalfOrder() {
+		return fmt.Errorf("invalid signature: S >= group order/2")
+	}
 	pkBytes := schnorr.SerializePubKey(pk)
 	recoveredPKBytes := schnorr.SerializePubKey(recoveredPK)
 	if !bytes.Equal(pkBytes, recoveredPKBytes) {

crypto/ecdsa/ecdsa_test.go

@@ -35,3 +35,26 @@ func TestECDSA(t *testing.T) {
 	err = ecdsa.Verify(pk, testMsg, sig)
 	require.NoError(t, err)
 }
+
+func TestECDSAMalleability(t *testing.T) {
+	// decode SK and PK
+	skBytes, err := hex.DecodeString(skHex)
+	require.NoError(t, err)
+	sk, pk := btcec.PrivKeyFromBytes(skBytes)
+	require.NotNil(t, sk)
+	require.NotNil(t, pk)
+	// sign
+	sig := ecdsa.Sign(sk, testMsg)
+	// verify
+	err = ecdsa.Verify(pk, testMsg, sig)
+	require.NoError(t, err)
+	// Modify signature
+	sig[0] = ((sig[0] - 27) ^ 1) + 27
+	var s btcec.ModNScalar
+	s.SetByteSlice(sig[33:65])
+	s.Negate()
+	s.PutBytesUnchecked(sig[33:65])
+	// Verify modified signature
+	err = ecdsa.Verify(pk, testMsg, sig)
+	require.Error(t, err)
+}