Ensure proper error handling for invalid public key types in authentication flows

Author: rusher

auth_caching_sha2.go

@@ -112,7 +112,12 @@ func (p *CachingSha2PasswordPlugin) continuationAuth(packet []byte, authData []b
 						if err != nil {
 							return nil, fmt.Errorf("failed to parse public key: %w", err)
 						}
-						pubKey = pkix.(*rsa.PublicKey)
+
+						var ok bool
+						pubKey, ok = pkix.(*rsa.PublicKey)
+						if !ok {
+							return nil, fmt.Errorf("server sent an invalid public key type: %T", pkix)
+						}
 					}
 
 					// Encrypt and send password
@@ -142,7 +147,7 @@ func (p *CachingSha2PasswordPlugin) continuationAuth(packet []byte, authData []b
 //
 // The algorithm is:
 // 1. SHA256(password)
-// 2. SHA256(SHA256(SHA256(password)))
+// 2. SHA256(SHA256(password))
 // 3. XOR(SHA256(password), SHA256(SHA256(SHA256(password)), scramble))
 //
 // This provides a way to verify the password without storing it in cleartext.

auth_sha256.go

@@ -68,6 +68,9 @@ func (p *Sha256PasswordPlugin) InitAuth(authData []byte, cfg *Config) ([]byte, e
 // 2. Error packet - Authentication failed
 // 3. More data packet - Contains the server's public key for password encryption
 func (p *Sha256PasswordPlugin) continuationAuth(packet []byte, authData []byte, mc *mysqlConn) ([]byte, error) {
+	if len(packet) == 0 {
+		return nil, fmt.Errorf("%w: empty auth response packet", ErrMalformPkt)
+	}
 
 	switch packet[0] {
 	case iOK, iERR, iEOF:
@@ -86,8 +89,13 @@ func (p *Sha256PasswordPlugin) continuationAuth(packet []byte, authData []byte,
 			return nil, fmt.Errorf("failed to parse public key: %w", err)
 		}
 
+		pubKey, ok := pub.(*rsa.PublicKey)
+		if !ok {
+			return nil, fmt.Errorf("server sent an invalid public key type: %T", pub)
+		}
+
 		// Send encrypted password
-		enc, err := encryptPassword(mc.cfg.Passwd, authData, pub.(*rsa.PublicKey))
+		enc, err := encryptPassword(mc.cfg.Passwd, authData, pubKey)
 		if err != nil {
 			return nil, fmt.Errorf("failed to encrypt password with server key: %w", err)
 		}