BackTrackZ/email-verifier
BackTrackZ/email-verifier
1
0
Fork 0
mirror of https://github.com/AfterShip/email-verifier.git synced 2025-02-06 09:44:47 +00:00
Code Actions Releases Activity

Chore: support gmail & yahoo smtp check by api (#88)

Browse Source
This commit is contained in:
QiuChengQuan 2023-06-19 17:29:57 +08:00 committed by GitHub
parent e58b22c46c
commit 28ab8910d0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 463 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
smtp.go
View File

@ -7,6 +7,7 @@ import (
"net"
"net/smtp"
"net/url"
"strings"
"sync"
"time"
@ -33,26 +34,35 @@ func (v *Verifier) CheckSMTP(domain, username string) (*SMTP, error) {
}
var ret SMTP
var err error
email := fmt.Sprintf("%s@%s", username, domain)
// Dial any SMTP server that will accept a connection
client, err := newSMTPClient(domain, v.proxyURI)
client, mx, err := newSMTPClient(domain, v.proxyURI)
if err != nil {
return &ret, ParseSMTPError(err)
}
// Sets the HELO/EHLO hostname
if err := client.Hello(v.helloName); err != nil {
return &ret, ParseSMTPError(err)
}
// Sets the from email
if err := client.Mail(v.fromEmail); err != nil {
return &ret, ParseSMTPError(err)
}
// Defer quit the SMTP connection
defer client.Close()
// Check by api when enabled and host recognized.
for _, apiVerifier := range v.apiVerifiers {
if apiVerifier.isSupported(strings.ToLower(mx.Host)) {
return apiVerifier.check(domain, username)
}
}
// Sets the HELO/EHLO hostname
if err = client.Hello(v.helloName); err != nil {
return &ret, ParseSMTPError(err)
}
// Sets the from email
if err = client.Mail(v.fromEmail); err != nil {
return &ret, ParseSMTPError(err)
}
// Host exists if we've successfully formed a connection
ret.HostExists = true
@ -63,7 +73,7 @@ func (v *Verifier) CheckSMTP(domain, username string) (*SMTP, error) {
// Checks the deliver ability of a randomly generated address in
// order to verify the existence of a catch-all and etc.
randomEmail := GenerateRandomEmail(domain)
if err := client.Rcpt(randomEmail); err != nil {
if err = client.Rcpt(randomEmail); err != nil {
if e := ParseSMTPError(err); e != nil {
switch e.Message {
case ErrFullInbox:
@ -94,8 +104,7 @@ func (v *Verifier) CheckSMTP(domain, username string) (*SMTP, error) {
return &ret, nil
}
email := fmt.Sprintf("%s@%s", username, domain)
if err := client.Rcpt(email); err == nil {
if err = client.Rcpt(email); err == nil {
ret.Deliverable = true
}
@ -103,18 +112,19 @@ func (v *Verifier) CheckSMTP(domain, username string) (*SMTP, error) {
}
// newSMTPClient generates a new available SMTP client
func newSMTPClient(domain, proxyURI string) (*smtp.Client, error) {
func newSMTPClient(domain, proxyURI string) (*smtp.Client, *net.MX, error) {
domain = domainToASCII(domain)
mxRecords, err := net.LookupMX(domain)
if err != nil {
return nil, err
return nil, nil, err
}
if len(mxRecords) == 0 {
return nil, errors.New("No MX records found")
return nil, nil, errors.New("No MX records found")
}
// Create a channel for receiving response from
ch := make(chan interface{}, 1)
selectedMXCh := make(chan *net.MX, 1)
// Done indicates if we're still waiting on dial responses
var done bool
@ -123,9 +133,9 @@ func newSMTPClient(domain, proxyURI string) (*smtp.Client, error) {
var mutex sync.Mutex
// Attempt to connect to all SMTP servers concurrently
for _, r := range mxRecords {
for i, r := range mxRecords {
addr := r.Host + smtpPort
index := i
go func() {
c, err := dialSMTP(addr, proxyURI)
if err != nil {
@ -141,6 +151,7 @@ func newSMTPClient(domain, proxyURI string) (*smtp.Client, error) {
case !done:
done = true
ch <- c
selectedMXCh <- mxRecords[index]
default:
c.Close()
}
@ -154,14 +165,14 @@ func newSMTPClient(domain, proxyURI string) (*smtp.Client, error) {
res := <-ch
switch r := res.(type) {
case *smtp.Client:
return r, nil
return r, <-selectedMXCh, nil
case error:
errs = append(errs, r)
if len(errs) == len(mxRecords) {
return nil, errs[0]
return nil, nil, errs[0]
}
default:
return nil, errors.New("Unexpected response dialing SMTP server")
return nil, nil, errors.New("Unexpected response dialing SMTP server")
}
}

13
smtp_by_api.go Normal file
View File

@ -0,0 +1,13 @@
package emailverifier
const (
GMAIL = "gmail"
YAHOO = "yahoo"
)
type smtpAPIVerifier interface {
// isSupported the specific host supports the check by api.
isSupported(host string) bool
// check must be called before isSupported == true
check(domain, username string) (*SMTP, error)
}

53
smtp_by_api_gmail.go Normal file
View File

@ -0,0 +1,53 @@
package emailverifier
import (
"context"
"fmt"
"net/http"
"strings"
"time"
)
const (
glxuPageFormat = "https://mail.google.com/mail/gxlu?email=%s"
)
// See the link below to know why we can use this way to check if a gmail exists.
// https://blog.0day.rocks/abusing-gmail-to-get-previously-unlisted-e-mail-addresses-41544b62b2
func newGmailAPIVerifier(client *http.Client) smtpAPIVerifier {
if client == nil {
client = http.DefaultClient
}
return gmail{
client: client,
}
}
type gmail struct {
client *http.Client
}
func (g gmail) isSupported(host string) bool {
return strings.HasSuffix(host, ".google.com.")
}
func (g gmail) check(domain, username string) (*SMTP, error) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
email := fmt.Sprintf("%s@%s", username, domain)
request, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf(glxuPageFormat, email), nil)
if err != nil {
return nil, err
}
resp, err := g.client.Do(request)
if err != nil {
return &SMTP{}, err
}
defer resp.Body.Close()
emailExists := len(resp.Cookies()) > 0
return &SMTP{
HostExists: true,
Deliverable: emailExists,
}, nil
}

25
smtp_by_api_gmail_test.go Normal file
View File

@ -0,0 +1,25 @@
package emailverifier
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestGmailCheckByAPI(t *testing.T) {
gmailAPIVerifier := newGmailAPIVerifier(nil)
t.Run("email exists", func(tt *testing.T) {
res, err := gmailAPIVerifier.check("gmail.com", "someone")
assert.NoError(t, err)
assert.Equal(t, true, res.HostExists)
assert.Equal(t, true, res.Deliverable)
})
t.Run("invalid email not exists", func(tt *testing.T) {
// username must greater than 6 characters
res, err := gmailAPIVerifier.check("gmail.com", "hello")
assert.NoError(t, err)
assert.Equal(t, true, res.HostExists)
assert.Equal(t, false, res.Deliverable)
})
}

178
smtp_by_api_yahoo.go Normal file
View File

@ -0,0 +1,178 @@
package emailverifier
import (
"bytes"
"context"
"encoding/json"
"errors"
"io/ioutil"
"net/http"
"regexp"
"strings"
"time"
)
const (
SIGNUP_PAGE = "https://login.yahoo.com/account/create?specId=yidregsimplified&lang=en-US&src=&done=https%3A%2F%2Fwww.yahoo.com&display=login"
SIGNUP_API = "https://login.yahoo.com/account/module/create?validateField=userId"
// USER_AGENT Fake one to use in API requests
USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36"
)
// Check yahoo email exists by their login & registration page.
// See https://login.yahoo.com
// See https://login.yahoo.com/account/create
func newYahooAPIVerifier(client *http.Client) smtpAPIVerifier {
if client == nil {
client = http.DefaultClient
}
return yahoo{
client: client,
}
}
type yahoo struct {
client *http.Client
}
type yahooValidateReq struct {
Domain, Username, Acrumb, SessionIndex string
Cookies []*http.Cookie
}
type yahooErrorResp struct {
Errors []errItem `json:"errors"`
}
type errItem struct {
Name string `json:"name"`
Error string `json:"error"`
}
func (y yahoo) isSupported(host string) bool {
// FIXME Is this `contains` too lenient?
return strings.Contains(host, "yahoo")
}
func (y yahoo) check(domain, username string) (*SMTP, error) {
cookies, signUpPageRespBytes, err := y.toSignUpPage()
if err != nil {
return nil, err
}
if len(cookies) == 0 {
return nil, errors.New("yahoo check by api, no cookies")
}
acrumb := getAcrumb(cookies)
if acrumb == "" {
return nil, errors.New("yahoo check by api, no acrumb")
}
sessionIndex := getSessionIndex(signUpPageRespBytes)
if sessionIndex == "" {
return nil, errors.New("yahoo check by api, no sessionIndex")
}
yahooErrResp, err := y.sendValidateRequest(yahooValidateReq{
Domain: domain,
Username: username,
Acrumb: acrumb,
SessionIndex: sessionIndex,
Cookies: cookies,
})
if err != nil {
return nil, err
}
usernameExists := checkUsernameExists(yahooErrResp)
return &SMTP{
HostExists: true,
Deliverable: usernameExists,
}, nil
}
func getSessionIndex(respBytes []byte) string {
re := regexp.MustCompile(`value="([^"]+)" name="sessionIndex"`)
match := re.FindSubmatch(respBytes)
if len(match) > 1 {
return string(match[1])
}
return ""
}
func checkUsernameExists(resp yahooErrorResp) bool {
for _, item := range resp.Errors {
if item.Name == "userId" && item.Error == "IDENTIFIER_EXISTS" {
return true
}
}
return false
}
func (y yahoo) sendValidateRequest(req yahooValidateReq) (yahooErrorResp, error) {
var res yahooErrorResp
data, err := json.Marshal(struct {
Acrumb string `json:"acrumb"`
SpecId string `json:"specId"`
Yid string `json:"userId"`
SessionIndex string `json:"sessionIndex"`
YidDomain string `json:"yidDomain"`
}{
Acrumb: req.Acrumb,
SpecId: "yidregsimplified",
Yid: req.Username,
SessionIndex: req.SessionIndex,
YidDomain: req.Domain,
})
if err != nil {
return res, err
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
request, err := http.NewRequestWithContext(ctx, http.MethodPost, SIGNUP_API, bytes.NewReader(data))
if err != nil {
return res, err
}
for _, c := range req.Cookies {
request.AddCookie(c)
}
request.Header.Add("X-Requested-With", "XMLHttpRequest")
request.Header.Add("Content-Type", "application/json; charset=UTF-8")
resp, err := y.client.Do(request)
if err != nil {
return res, err
}
defer resp.Body.Close()
respBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
return res, err
}
return res, json.Unmarshal(respBytes, &res)
}
func (y yahoo) toSignUpPage() ([]*http.Cookie, []byte, error) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
request, err := http.NewRequestWithContext(ctx, http.MethodGet, SIGNUP_PAGE, nil)
if err != nil {
return nil, nil, err
}
request.Header.Add("User-Agent", USER_AGENT)
resp, err := y.client.Do(request)
if err != nil {
return nil, nil, err
}
defer resp.Body.Close()
respBytes, err := ioutil.ReadAll(resp.Body)
return resp.Cookies(), respBytes, err
}
func getAcrumb(cookies []*http.Cookie) string {
for _, c := range cookies {
re := regexp.MustCompile(`s=(?P<acrumb>[^;^&]*)`)
match := re.FindStringSubmatch(c.Value)
if len(match) > 1 {
return match[1]
}
}
return ""
}

46
smtp_by_api_yahoo_test.go Normal file
View File

@ -0,0 +1,46 @@
package emailverifier
import (
"net/http"
"testing"
"github.com/stretchr/testify/assert"
)
func TestYahooCheckByAPI(t *testing.T) {
yahooAPIVerifier := newYahooAPIVerifier(nil)
t.Run("email exists", func(tt *testing.T) {
res, err := yahooAPIVerifier.check("yahoo.com", "hello")
assert.NoError(t, err)
assert.Equal(t, true, res.HostExists)
assert.Equal(t, true, res.Deliverable)
})
t.Run("invalid email not exists", func(tt *testing.T) {
res, err := yahooAPIVerifier.check("yahoo.com", "123")
assert.NoError(t, err)
assert.Equal(t, true, res.HostExists)
assert.Equal(t, false, res.Deliverable)
})
}
func TestGetAcrumb(t *testing.T) {
cookies0 := []*http.Cookie{
{Value: "123321"},
{Value: "v=1&s=gWKqrs5c&d=A6454c24b|Zt.ZFgb.2T"},
}
acrumb := getAcrumb(cookies0)
assert.Equal(t, acrumb, "gWKqrs5c")
cookies1 := []*http.Cookie{
{Value: "123321"},
{Value: "v=1&s=gWKqrs5c"},
}
acrumb = getAcrumb(cookies1)
assert.Equal(t, acrumb, "gWKqrs5c")
cookies2 := []*http.Cookie{
{Value: "123321"},
}
acrumb = getAcrumb(cookies2)
assert.Equal(t, acrumb, "")
}

87
smtp_test.go
View File

@ -8,6 +8,87 @@ import (
"github.com/stretchr/testify/assert"
)
func TestCheckSMTPUnSupportedVendor(t *testing.T) {
err := verifier.EnableAPIVerifier("unsupported_vendor")
assert.Error(t, err)
}
func TestCheckSMTPOK_ByApi(t *testing.T) {
cases := []struct {
name string
domain string
username string
expected *SMTP
}{
{
name: "gmail exists",
domain: "gmail.com",
username: "someone",
expected: &SMTP{
HostExists: true,
Deliverable: true,
},
},
{
name: "gmail no exists",
domain: "gmail.com",
username: "hello",
expected: &SMTP{
HostExists: true,
Deliverable: false,
},
},
{
name: "yahoo exists",
domain: "yahoo.com",
username: "someone",
expected: &SMTP{
HostExists: true,
Deliverable: true,
},
},
{
name: "myyahoo exists",
domain: "myyahoo.com",
username: "someone",
expected: &SMTP{
HostExists: true,
Deliverable: true,
},
},
{
name: "yahoo no exists",
domain: "yahoo.com",
username: "123",
expected: &SMTP{
HostExists: true,
Deliverable: false,
},
},
{
name: "myyahoo no exists",
domain: "myyahoo.com",
username: "123",
expected: &SMTP{
HostExists: true,
Deliverable: false,
},
},
}
_ = verifier.EnableAPIVerifier(GMAIL)
_ = verifier.EnableAPIVerifier(YAHOO)
defer verifier.DisableAPIVerifier(GMAIL)
defer verifier.DisableAPIVerifier(YAHOO)
for _, c := range cases {
test := c
t.Run(test.name, func(tt *testing.T) {
smtp, err := verifier.CheckSMTP(test.domain, test.username)
assert.NoError(t, err)
assert.Equal(t, test.expected, smtp)
})
}
}
func TestCheckSMTPOK_HostExists(t *testing.T) {
domain := "github.com"
@ -133,7 +214,7 @@ func TestCheckSMTPOK_HostNotExists(t *testing.T) {
func TestNewSMTPClientOK(t *testing.T) {
domain := "gmail.com"
ret, err := newSMTPClient(domain, "")
ret, _, err := newSMTPClient(domain, "")
assert.NotNil(t, ret)
assert.Nil(t, err)
}
@ -141,14 +222,14 @@ func TestNewSMTPClientOK(t *testing.T) {
func TestNewSMTPClientFailed_WithInvalidProxy(t *testing.T) {
domain := "gmail.com"
proxyURI := "socks5://user:password@127.0.0.1:1080?timeout=5s"
ret, err := newSMTPClient(domain, proxyURI)
ret, _, err := newSMTPClient(domain, proxyURI)
assert.Nil(t, ret)
assert.Error(t, err, syscall.ECONNREFUSED)
}
func TestNewSMTPClientFailed(t *testing.T) {
domain := "zzzz171777.com"
ret, err := newSMTPClient(domain, "")
ret, _, err := newSMTPClient(domain, "")
assert.Nil(t, ret)
assert.Error(t, err)
}

40
verifier.go
View File

@ -1,20 +1,22 @@
package emailverifier
import (
"fmt"
"net/http"
"time"
)
// Verifier is an email verifier. Create one by calling NewVerifier
type Verifier struct {
smtpCheckEnabled bool // SMTP check enabled or disabled (disabled by default)
catchAllCheckEnabled bool // SMTP catchAll check enabled or disabled (enabled by default)
domainSuggestEnabled bool // whether suggest a most similar correct domain or not (disabled by default)
gravatarCheckEnabled bool // gravatar check enabled or disabled (disabled by default)
fromEmail string // name to use in the `EHLO:` SMTP command, defaults to "user@example.org"
helloName string // email to use in the `MAIL FROM:` SMTP command. defaults to `localhost`
schedule *schedule // schedule represents a job schedule
proxyURI string // use a SOCKS5 proxy to verify the email,
smtpCheckEnabled bool // SMTP check enabled or disabled (disabled by default)
catchAllCheckEnabled bool // SMTP catchAll check enabled or disabled (enabled by default)
domainSuggestEnabled bool // whether suggest a most similar correct domain or not (disabled by default)
gravatarCheckEnabled bool // gravatar check enabled or disabled (disabled by default)
fromEmail string // name to use in the `EHLO:` SMTP command, defaults to "user@example.org"
helloName string // email to use in the `MAIL FROM:` SMTP command. defaults to `localhost`
schedule *schedule // schedule represents a job schedule
proxyURI string // use a SOCKS5 proxy to verify the email,
apiVerifiers map[string]smtpAPIVerifier // currently support gmail & yahoo, further contributions are welcomed.
}
// Result is the result of Email Verification
@ -47,6 +49,7 @@ func NewVerifier() *Verifier {
fromEmail: defaultFromEmail,
helloName: defaultHelloName,
catchAllCheckEnabled: true,
apiVerifiers: map[string]smtpAPIVerifier{},
}
}
@ -131,6 +134,25 @@ func (v *Verifier) EnableSMTPCheck() *Verifier {
return v
}
// EnableAPIVerifier API verifier is activated when EnableAPIVerifier for the target vendor.
// ** Please know ** that this is a tricky way (but relatively stable) to check if target vendor's email exists.
// If you use this feature in a production environment, please ensure that you have sufficient backup measures in place, as this may encounter rate limiting or other API issues.
func (v *Verifier) EnableAPIVerifier(name string) error {
switch name {
case GMAIL:
v.apiVerifiers[GMAIL] = newGmailAPIVerifier(http.DefaultClient)
case YAHOO:
v.apiVerifiers[YAHOO] = newYahooAPIVerifier(http.DefaultClient)
default:
return fmt.Errorf("unsupported to enable the API verifier for vendor: %s", name)
}
return nil
}
func (v *Verifier) DisableAPIVerifier(name string) {
delete(v.apiVerifiers, name)
}
// DisableSMTPCheck disables check email by smtp
func (v *Verifier) DisableSMTPCheck() *Verifier {
v.smtpCheckEnabled = false