Swift - Button's action doesn’t get triggered unless long tapped

I've added a Sign in with Apple button in a UIScrollView, the way Apple's documentation suggest;

let signInWithAppleButton = ASAuthorizationAppleIDButton(type: .default, style: .white)
            
signInWithAppleButton.addTarget(self, action: #selector(loginWithApple(_:)), for: .touchUpInside)
signInWithAppleButton.cornerRadius = 12
            
scrollView.addSubview(signInWithAppleButton)

The thing is, the button only responds to very long presses instead of simple taps. I've tried putting it outside the UIScrollView and it worked there!

Here's my UIScrollView's and button's setup;

fileprivate func setupScrollViewConstraints() {
    scrollView.translatesAutoresizingMaskIntoConstraints = false
    
    NSLayoutConstraint.activate([
        scrollView.topAnchor.constraint(equalTo: self.view.topAnchor),
        scrollView.bottomAnchor.constraint(equalTo: self.view.bottomAnchor),
        scrollView.leadingAnchor.constraint(equalTo: self.view.leadingAnchor),
        scrollView.trailingAnchor.constraint(equalTo: self.view.trailingAnchor)
    ])
}

func setupSignInWithAppleButton() -> UIControl? {
    if #available(iOS 13.0, *) {
        let signInWithAppleButton = ASAuthorizationAppleIDButton(type: .default, style: .white)
        
        signInWithAppleButton.addTarget(self, action: #selector(loginWithApple(_:)), for: .touchUpInside)
        signInWithAppleButton.cornerRadius = 12
        
        scrollView.addSubview(signInWithAppleButton)
        
        signInWithAppleButton.translatesAutoresizingMaskIntoConstraints = false
        
        NSLayoutConstraint.activate([
            signInWithAppleButton.topAnchor.constraint(equalTo: accountLabel.bottomAnchor, constant: 8),
            signInWithAppleButton.centerXAnchor.constraint(equalTo: scrollView.centerXAnchor),
            signInWithAppleButton.widthAnchor.constraint(equalToConstant: 210),
            signInWithAppleButton.heightAnchor.constraint(equalToConstant: 45)
        ])
        
        return signInWithAppleButton
    }
    
    return nil
}

Any idea what's breaking it?

from Swift - Button's action doesn’t get triggered unless long tapped