github.com/urfave/cli v1.20.0
github.com/willf/bitset v0.0.0-20180426185212-8ce1146b8621 // indirect
github.com/yohcop/openid-go v0.0.0-20160914080427-2c050d2dae53
- golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad
+ golang.org/x/crypto v0.0.0-20191117063200-497ca9f6d64f
golang.org/x/net v0.0.0-20191101175033-0deb6923b6d9
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad h1:5E5raQxcv+6CZ11RrBYQe5WRbUIWpScjh0kvHZkZIrQ=
golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191117063200-497ca9f6d64f h1:kz4KIr+xcPUsI3VMoqWfPMvtnJ6MGfiVwsWSVzphMO4=
+golang.org/x/crypto v0.0.0-20191117063200-497ca9f6d64f/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4=
// Package acme provides an implementation of the
// Automatic Certificate Management Environment (ACME) spec.
// The intial implementation was based on ACME draft-02 and
-// is now being extended to comply with RFC8555.
+// is now being extended to comply with RFC 8555.
// See https://tools.ietf.org/html/draft-ietf-acme-acme-02
// and https://tools.ietf.org/html/rfc8555 for details.
//
const (
// LetsEncryptURL is the Directory endpoint of Let's Encrypt CA.
- LetsEncryptURL = "https://acme-v01.api.letsencrypt.org/directory"
+ LetsEncryptURL = "https://acme-v02.api.letsencrypt.org/directory"
// ALPNProto is the ALPN protocol name used by a CA server when validating
// tls-alpn-01 challenges.
const (
maxChainLen = 5 // max depth and breadth of a certificate chain
- maxCertSize = 1 << 20 // max size of a certificate, in bytes
+ maxCertSize = 1 << 20 // max size of a certificate, in DER bytes
+ // Used for decoding certs from application/pem-certificate-chain response,
+ // the default when in RFC mode.
+ maxCertChainSize = maxCertSize * maxChainLen
// Max number of collected nonces kept in memory.
// Expect usual peak of 1 or 2.
func (c *Client) accountKID(ctx context.Context) keyID {
c.cacheMu.Lock()
defer c.cacheMu.Unlock()
- if c.dir.OrderURL == "" {
- // Assume legacy CA.
+ if !c.dir.rfcCompliant() {
return noKeyID
}
if c.kid != noKeyID {
}
// CreateCert requests a new certificate using the Certificate Signing Request csr encoded in DER format.
+// It is incompatible with RFC 8555. Callers should use CreateOrderCert when interfacing
+// with an RFC-compliant CA.
+//
// The exp argument indicates the desired certificate validity duration. CA may issue a certificate
// with a different duration.
// If the bundle argument is true, the returned value will also contain the CA (issuer) certificate chain.
// It retries the request until the certificate is successfully retrieved,
// context is cancelled by the caller or an error response is received.
//
-// The returned value will also contain the CA (issuer) certificate if the bundle argument is true.
+// If the bundle argument is true, the returned value also contains the CA (issuer)
+// certificate chain.
//
// FetchCert returns an error if the CA's response or chain was unreasonably large.
// Callers are encouraged to parse the returned value to ensure the certificate is valid
// and has expected features.
func (c *Client) FetchCert(ctx context.Context, url string, bundle bool) ([][]byte, error) {
+ dir, err := c.Discover(ctx)
+ if err != nil {
+ return nil, err
+ }
+ if dir.rfcCompliant() {
+ return c.fetchCertRFC(ctx, url, bundle)
+ }
+
+ // Legacy non-authenticated GET request.
res, err := c.get(ctx, url, wantStatus(http.StatusOK))
if err != nil {
return nil, err
// For instance, the key pair of the certificate may be authorized.
// If the key is nil, c.Key is used instead.
func (c *Client) RevokeCert(ctx context.Context, key crypto.Signer, cert []byte, reason CRLReasonCode) error {
- if _, err := c.Discover(ctx); err != nil {
+ dir, err := c.Discover(ctx)
+ if err != nil {
return err
}
+ if dir.rfcCompliant() {
+ return c.revokeCertRFC(ctx, key, cert, reason)
+ }
+ // Legacy CA.
body := &struct {
Resource string `json:"resource"`
Cert string `json:"certificate"`
Cert: base64.RawURLEncoding.EncodeToString(cert),
Reason: int(reason),
}
- res, err := c.post(ctx, key, c.dir.RevokeURL, body, wantStatus(http.StatusOK))
+ res, err := c.post(ctx, key, dir.RevokeURL, body, wantStatus(http.StatusOK))
if err != nil {
return err
}
// Register calls prompt with a TOS URL provided by the CA. Prompt should report
// whether the caller agrees to the terms. To always accept the terms, the caller can use AcceptTOS.
//
-// When interfacing with RFC compliant CA, non-RFC8555 compliant fields of acct are ignored
+// When interfacing with an RFC-compliant CA, non-RFC 8555 fields of acct are ignored
// and prompt is called if Directory's Terms field is non-zero.
// Also see Error's Instance field for when a CA requires already registered accounts to agree
// to an updated Terms of Service.
if err != nil {
return nil, err
}
-
- // RFC8555 compliant account registration.
- if dir.OrderURL != "" {
+ if dir.rfcCompliant() {
return c.registerRFC(ctx, acct, prompt)
}
// GetReg retrieves an existing account associated with c.Key.
//
-// The url argument is an Account URI used with pre-RFC8555 CAs.
-// It is ignored when interfacing with an RFC compliant CA.
+// The url argument is an Account URI used with pre-RFC 8555 CAs.
+// It is ignored when interfacing with an RFC-compliant CA.
func (c *Client) GetReg(ctx context.Context, url string) (*Account, error) {
dir, err := c.Discover(ctx)
if err != nil {
return nil, err
}
-
- // Assume RFC8555 compliant CA.
- if dir.OrderURL != "" {
+ if dir.rfcCompliant() {
return c.getRegRFC(ctx)
}
// UpdateReg updates an existing registration.
// It returns an updated account copy. The provided account is not modified.
//
-// When interfacing with RFC compliant CAs, a.URI is ignored and the account URL
+// When interfacing with RFC-compliant CAs, a.URI is ignored and the account URL
// associated with c.Key is used instead.
func (c *Client) UpdateReg(ctx context.Context, acct *Account) (*Account, error) {
dir, err := c.Discover(ctx)
if err != nil {
return nil, err
}
-
- // Assume RFC8555 compliant CA.
- if dir.OrderURL != "" {
+ if dir.rfcCompliant() {
return c.updateRegRFC(ctx, acct)
}
return a, nil
}
-// Authorize performs the initial step in an authorization flow.
+// Authorize performs the initial step in the pre-authorization flow,
+// as opposed to order-based flow.
// The caller will then need to choose from and perform a set of returned
// challenges using c.Accept in order to successfully complete authorization.
//
+// Once complete, the caller can use AuthorizeOrder which the CA
+// should provision with the already satisfied authorization.
+// For pre-RFC CAs, the caller can proceed directly to requesting a certificate
+// using CreateCert method.
+//
// If an authorization has been previously granted, the CA may return
-// a valid authorization (Authorization.Status is StatusValid). If so, the caller
-// need not fulfill any challenge and can proceed to requesting a certificate.
+// a valid authorization which has its Status field set to StatusValid.
+//
+// More about pre-authorization can be found at
+// https://tools.ietf.org/html/rfc8555#section-7.4.1.
func (c *Client) Authorize(ctx context.Context, domain string) (*Authorization, error) {
return c.authorize(ctx, "dns", domain)
}
// If a caller needs to poll an authorization until its status is final,
// see the WaitAuthorization method.
func (c *Client) GetAuthorization(ctx context.Context, url string) (*Authorization, error) {
- res, err := c.get(ctx, url, wantStatus(http.StatusOK, http.StatusAccepted))
+ dir, err := c.Discover(ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ var res *http.Response
+ if dir.rfcCompliant() {
+ res, err = c.postAsGet(ctx, url, wantStatus(http.StatusOK))
+ } else {
+ res, err = c.get(ctx, url, wantStatus(http.StatusOK, http.StatusAccepted))
+ }
if err != nil {
return nil, err
}
// The url argument is an Authorization.URI value.
//
// If successful, the caller will be required to obtain a new authorization
-// using the Authorize method before being able to request a new certificate
-// for the domain associated with the authorization.
+// using the Authorize or AuthorizeOrder methods before being able to request
+// a new certificate for the domain associated with the authorization.
//
// It does not revoke existing certificates.
func (c *Client) RevokeAuthorization(ctx context.Context, url string) error {
// In all other cases WaitAuthorization returns an error.
// If the Status is StatusInvalid, the returned error is of type *AuthorizationError.
func (c *Client) WaitAuthorization(ctx context.Context, url string) (*Authorization, error) {
+ // Required for c.accountKID() when in RFC mode.
+ dir, err := c.Discover(ctx)
+ if err != nil {
+ return nil, err
+ }
+ getfn := c.postAsGet
+ if !dir.rfcCompliant() {
+ getfn = c.get
+ }
+
for {
- res, err := c.get(ctx, url, wantStatus(http.StatusOK, http.StatusAccepted))
+ res, err := getfn(ctx, url, wantStatus(http.StatusOK, http.StatusAccepted))
if err != nil {
return nil, err
}
//
// A client typically polls a challenge status using this method.
func (c *Client) GetChallenge(ctx context.Context, url string) (*Challenge, error) {
- res, err := c.get(ctx, url, wantStatus(http.StatusOK, http.StatusAccepted))
+ // Required for c.accountKID() when in RFC mode.
+ dir, err := c.Discover(ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ getfn := c.postAsGet
+ if !dir.rfcCompliant() {
+ getfn = c.get
+ }
+ res, err := getfn(ctx, url, wantStatus(http.StatusOK, http.StatusAccepted))
if err != nil {
return nil, err
}
+
defer res.Body.Close()
v := wireChallenge{URI: url}
if err := json.NewDecoder(res.Body).Decode(&v); err != nil {
// The server will then perform the validation asynchronously.
func (c *Client) Accept(ctx context.Context, chal *Challenge) (*Challenge, error) {
// Required for c.accountKID() when in RFC mode.
- if _, err := c.Discover(ctx); err != nil {
- return nil, err
- }
-
- auth, err := keyAuth(c.Key.Public(), chal.Token)
+ dir, err := c.Discover(ctx)
if err != nil {
return nil, err
}
- req := struct {
- Resource string `json:"resource"`
- Type string `json:"type"`
- Auth string `json:"keyAuthorization"`
- }{
- Resource: "challenge",
- Type: chal.Type,
- Auth: auth,
+ var req interface{} = json.RawMessage("{}") // RFC-compliant CA
+ if !dir.rfcCompliant() {
+ auth, err := keyAuth(c.Key.Public(), chal.Token)
+ if err != nil {
+ return nil, err
+ }
+ req = struct {
+ Resource string `json:"resource"`
+ Type string `json:"type"`
+ Auth string `json:"keyAuthorization"`
+ }{
+ Resource: "challenge",
+ Type: chal.Type,
+ Auth: auth,
+ }
}
res, err := c.post(ctx, nil, chal.URI, req, wantStatus(
http.StatusOK, // according to the spec
}
// TLSSNI01ChallengeCert creates a certificate for TLS-SNI-01 challenge response.
-// Servers can present the certificate to validate the challenge and prove control
-// over a domain name.
-//
-// The implementation is incomplete in that the returned value is a single certificate,
-// computed only for Z0 of the key authorization. ACME CAs are expected to update
-// their implementations to use the newer version, TLS-SNI-02.
-// For more details on TLS-SNI-01 see https://tools.ietf.org/html/draft-ietf-acme-acme-01#section-7.3.
-//
-// The token argument is a Challenge.Token value.
-// If a WithKey option is provided, its private part signs the returned cert,
-// and the public part is used to specify the signee.
-// If no WithKey option is provided, a new ECDSA key is generated using P-256 curve.
//
-// The returned certificate is valid for the next 24 hours and must be presented only when
-// the server name of the TLS ClientHello matches exactly the returned name value.
+// Deprecated: This challenge type is unused in both draft-02 and RFC versions of ACME spec.
func (c *Client) TLSSNI01ChallengeCert(token string, opt ...CertOption) (cert tls.Certificate, name string, err error) {
ka, err := keyAuth(c.Key.Public(), token)
if err != nil {
}
// TLSSNI02ChallengeCert creates a certificate for TLS-SNI-02 challenge response.
-// Servers can present the certificate to validate the challenge and prove control
-// over a domain name. For more details on TLS-SNI-02 see
-// https://tools.ietf.org/html/draft-ietf-acme-acme-03#section-7.3.
-//
-// The token argument is a Challenge.Token value.
-// If a WithKey option is provided, its private part signs the returned cert,
-// and the public part is used to specify the signee.
-// If no WithKey option is provided, a new ECDSA key is generated using P-256 curve.
//
-// The returned certificate is valid for the next 24 hours and must be presented only when
-// the server name in the TLS ClientHello matches exactly the returned name value.
+// Deprecated: This challenge type is unused in both draft-02 and RFC versions of ACME spec.
func (c *Client) TLSSNI02ChallengeCert(token string, opt ...CertOption) (cert tls.Certificate, name string, err error) {
b := sha256.Sum256([]byte(token))
h := hex.EncodeToString(b[:])
return tlsChallengeCert([]string{domain}, newOpt)
}
-// doReg sends all types of registration requests.
+// doReg sends all types of registration requests the old way (pre-RFC world).
// The type of request is identified by typ argument, which is a "resource"
// in the ACME spec terms.
//
"golang.org/x/net/idna"
)
+// DefaultACMEDirectory is the default ACME Directory URL used when the Manager's Client is nil.
+const DefaultACMEDirectory = "https://acme-v02.api.letsencrypt.org/directory"
+
// createCertRetryAfter is how much time to wait before removing a failed state
// entry due to an unsuccessful createCert call.
// This is a variable instead of a const for testing.
// Client is used to perform low-level operations, such as account registration
// and requesting new certificates.
//
- // If Client is nil, a zero-value acme.Client is used with acme.LetsEncryptURL
- // as directory endpoint. If the Client.Key is nil, a new ECDSA P-256 key is
- // generated and, if Cache is not nil, stored in cache.
+ // If Client is nil, a zero-value acme.Client is used with DefaultACMEDirectory
+ // as the directory endpoint.
+ // If the Client.Key is nil, a new ECDSA P-256 key is generated and,
+ // if Cache is not nil, stored in cache.
//
// Mutating the field after the first call of GetCertificate method will have no effect.
Client *acme.Client
renewalMu sync.Mutex
renewal map[certKey]*domainRenewal
- // tokensMu guards the rest of the fields: tryHTTP01, certTokens and httpTokens.
- tokensMu sync.RWMutex
+ // challengeMu guards tryHTTP01, certTokens and httpTokens.
+ challengeMu sync.RWMutex
// tryHTTP01 indicates whether the Manager should try "http-01" challenge type
// during the authorization flow.
tryHTTP01 bool
// and is keyed by the domain name which matches the ClientHello server name.
// The entries are stored for the duration of the authorization flow.
certTokens map[string]*tls.Certificate
+
// nowFunc, if not nil, returns the current time. This may be set for
// testing purposes.
nowFunc func() time.Time
// Check whether this is a token cert requested for TLS-ALPN challenge.
if wantsTokenCert(hello) {
- m.tokensMu.RLock()
- defer m.tokensMu.RUnlock()
+ m.challengeMu.RLock()
+ defer m.challengeMu.RUnlock()
if cert := m.certTokens[name]; cert != nil {
return cert, nil
}
// If HTTPHandler is never called, the Manager will only use the "tls-alpn-01"
// challenge for domain verification.
func (m *Manager) HTTPHandler(fallback http.Handler) http.Handler {
- m.tokensMu.Lock()
- defer m.tokensMu.Unlock()
+ m.challengeMu.Lock()
+ defer m.challengeMu.Unlock()
m.tryHTTP01 = true
if fallback == nil {
// authorizedCert starts the domain ownership verification process and requests a new cert upon success.
// The key argument is the certificate private key.
func (m *Manager) authorizedCert(ctx context.Context, key crypto.Signer, ck certKey) (der [][]byte, leaf *x509.Certificate, err error) {
- client, err := m.acmeClient(ctx)
- if err != nil {
- return nil, nil, err
- }
-
- if err := m.verify(ctx, client, ck.domain); err != nil {
- return nil, nil, err
- }
csr, err := certRequest(key, ck.domain, m.ExtraExtensions)
if err != nil {
return nil, nil, err
}
- der, _, err = client.CreateCert(ctx, csr, 0, true)
+
+ client, err := m.acmeClient(ctx)
if err != nil {
return nil, nil, err
}
- leaf, err = validCert(ck, der, key, m.now())
+ dir, err := client.Discover(ctx)
if err != nil {
return nil, nil, err
}
- return der, leaf, nil
-}
-// revokePendingAuthz revokes all authorizations idenfied by the elements of uri slice.
-// It ignores revocation errors.
-func (m *Manager) revokePendingAuthz(ctx context.Context, uri []string) {
- client, err := m.acmeClient(ctx)
- if err != nil {
- return
+ var chain [][]byte
+ switch {
+ // Pre-RFC legacy CA.
+ case dir.OrderURL == "":
+ if err := m.verify(ctx, client, ck.domain); err != nil {
+ return nil, nil, err
+ }
+ der, _, err := client.CreateCert(ctx, csr, 0, true)
+ if err != nil {
+ return nil, nil, err
+ }
+ chain = der
+ // RFC 8555 compliant CA.
+ default:
+ o, err := m.verifyRFC(ctx, client, ck.domain)
+ if err != nil {
+ return nil, nil, err
+ }
+ der, _, err := client.CreateOrderCert(ctx, o.FinalizeURL, csr, true)
+ if err != nil {
+ return nil, nil, err
+ }
+ chain = der
}
- for _, u := range uri {
- client.RevokeAuthorization(ctx, u)
+ leaf, err = validCert(ck, chain, key, m.now())
+ if err != nil {
+ return nil, nil, err
}
+ return chain, leaf, nil
}
-// verify runs the identifier (domain) authorization flow
+// verify runs the identifier (domain) pre-authorization flow for legacy CAs
// using each applicable ACME challenge type.
func (m *Manager) verify(ctx context.Context, client *acme.Client, domain string) error {
- // The list of challenge types we'll try to fulfill
- // in this specific order.
- challengeTypes := []string{"tls-alpn-01"}
- m.tokensMu.RLock()
- if m.tryHTTP01 {
- challengeTypes = append(challengeTypes, "http-01")
- }
- m.tokensMu.RUnlock()
-
- // Keep track of pending authzs and revoke the ones that did not validate.
- pendingAuthzs := make(map[string]bool)
+ // Remove all hanging authorizations to reduce rate limit quotas
+ // after we're done.
+ var authzURLs []string
defer func() {
- var uri []string
- for k, pending := range pendingAuthzs {
- if pending {
- uri = append(uri, k)
- }
- }
- if len(uri) > 0 {
- // Use "detached" background context.
- // The revocations need not happen in the current verification flow.
- go m.revokePendingAuthz(context.Background(), uri)
- }
+ go m.deactivatePendingAuthz(authzURLs)
}()
// errs accumulates challenge failure errors, printed if all fail
errs := make(map[*acme.Challenge]error)
+ challengeTypes := m.supportedChallengeTypes()
var nextTyp int // challengeType index of the next challenge type to try
for {
// Start domain authorization and get the challenge.
if err != nil {
return err
}
+ authzURLs = append(authzURLs, authz.URI)
// No point in accepting challenges if the authorization status
// is in a final state.
switch authz.Status {
return fmt.Errorf("acme/autocert: invalid authorization %q", authz.URI)
}
- pendingAuthzs[authz.URI] = true
-
// Pick the next preferred challenge.
var chal *acme.Challenge
for chal == nil && nextTyp < len(challengeTypes) {
errs[chal] = err
continue
}
- delete(pendingAuthzs, authz.URI)
return nil
}
}
+// verifyRFC runs the identifier (domain) order-based authorization flow for RFC compliant CAs
+// using each applicable ACME challenge type.
+func (m *Manager) verifyRFC(ctx context.Context, client *acme.Client, domain string) (*acme.Order, error) {
+ // Try each supported challenge type starting with a new order each time.
+ // The nextTyp index of the next challenge type to try is shared across
+ // all order authorizations: if we've tried a challenge type once and it didn't work,
+ // it will most likely not work on another order's authorization either.
+ challengeTypes := m.supportedChallengeTypes()
+ nextTyp := 0 // challengeTypes index
+AuthorizeOrderLoop:
+ for {
+ o, err := client.AuthorizeOrder(ctx, acme.DomainIDs(domain))
+ if err != nil {
+ return nil, err
+ }
+ // Remove all hanging authorizations to reduce rate limit quotas
+ // after we're done.
+ defer func(urls []string) {
+ go m.deactivatePendingAuthz(urls)
+ }(o.AuthzURLs)
+
+ // Check if there's actually anything we need to do.
+ switch o.Status {
+ case acme.StatusReady:
+ // Already authorized.
+ return o, nil
+ case acme.StatusPending:
+ // Continue normal Order-based flow.
+ default:
+ return nil, fmt.Errorf("acme/autocert: invalid new order status %q; order URL: %q", o.Status, o.URI)
+ }
+
+ // Satisfy all pending authorizations.
+ for _, zurl := range o.AuthzURLs {
+ z, err := client.GetAuthorization(ctx, zurl)
+ if err != nil {
+ return nil, err
+ }
+ if z.Status != acme.StatusPending {
+ // We are interested only in pending authorizations.
+ continue
+ }
+ // Pick the next preferred challenge.
+ var chal *acme.Challenge
+ for chal == nil && nextTyp < len(challengeTypes) {
+ chal = pickChallenge(challengeTypes[nextTyp], z.Challenges)
+ nextTyp++
+ }
+ if chal == nil {
+ return nil, fmt.Errorf("acme/autocert: unable to satisfy %q for domain %q: no viable challenge type found", z.URI, domain)
+ }
+ // Respond to the challenge and wait for validation result.
+ cleanup, err := m.fulfill(ctx, client, chal, domain)
+ if err != nil {
+ continue AuthorizeOrderLoop
+ }
+ defer cleanup()
+ if _, err := client.Accept(ctx, chal); err != nil {
+ continue AuthorizeOrderLoop
+ }
+ if _, err := client.WaitAuthorization(ctx, z.URI); err != nil {
+ continue AuthorizeOrderLoop
+ }
+ }
+
+ // All authorizations are satisfied.
+ // Wait for the CA to update the order status.
+ o, err = client.WaitOrder(ctx, o.URI)
+ if err != nil {
+ continue AuthorizeOrderLoop
+ }
+ return o, nil
+ }
+}
+
+func pickChallenge(typ string, chal []*acme.Challenge) *acme.Challenge {
+ for _, c := range chal {
+ if c.Type == typ {
+ return c
+ }
+ }
+ return nil
+}
+
+func (m *Manager) supportedChallengeTypes() []string {
+ m.challengeMu.RLock()
+ defer m.challengeMu.RUnlock()
+ typ := []string{"tls-alpn-01"}
+ if m.tryHTTP01 {
+ typ = append(typ, "http-01")
+ }
+ return typ
+}
+
+// deactivatePendingAuthz relinquishes all authorizations identified by the elements
+// of the provided uri slice which are in "pending" state.
+// It ignores revocation errors.
+//
+// deactivatePendingAuthz takes no context argument and instead runs with its own
+// "detached" context because deactivations are done in a goroutine separate from
+// that of the main issuance or renewal flow.
+func (m *Manager) deactivatePendingAuthz(uri []string) {
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+ defer cancel()
+ client, err := m.acmeClient(ctx)
+ if err != nil {
+ return
+ }
+ for _, u := range uri {
+ z, err := client.GetAuthorization(ctx, u)
+ if err == nil && z.Status == acme.StatusPending {
+ client.RevokeAuthorization(ctx, u)
+ }
+ }
+}
+
// fulfill provisions a response to the challenge chal.
// The cleanup is non-nil only if provisioning succeeded.
func (m *Manager) fulfill(ctx context.Context, client *acme.Client, chal *acme.Challenge, domain string) (cleanup func(), err error) {
return nil, fmt.Errorf("acme/autocert: unknown challenge type %q", chal.Type)
}
-func pickChallenge(typ string, chal []*acme.Challenge) *acme.Challenge {
- for _, c := range chal {
- if c.Type == typ {
- return c
- }
- }
- return nil
-}
-
// putCertToken stores the token certificate with the specified name
// in both m.certTokens map and m.Cache.
func (m *Manager) putCertToken(ctx context.Context, name string, cert *tls.Certificate) {
- m.tokensMu.Lock()
- defer m.tokensMu.Unlock()
+ m.challengeMu.Lock()
+ defer m.challengeMu.Unlock()
if m.certTokens == nil {
m.certTokens = make(map[string]*tls.Certificate)
}
// deleteCertToken removes the token certificate with the specified name
// from both m.certTokens map and m.Cache.
func (m *Manager) deleteCertToken(name string) {
- m.tokensMu.Lock()
- defer m.tokensMu.Unlock()
+ m.challengeMu.Lock()
+ defer m.challengeMu.Unlock()
delete(m.certTokens, name)
if m.Cache != nil {
ck := certKey{domain: name, isToken: true}
// httpToken retrieves an existing http-01 token value from an in-memory map
// or the optional cache.
func (m *Manager) httpToken(ctx context.Context, tokenPath string) ([]byte, error) {
- m.tokensMu.RLock()
- defer m.tokensMu.RUnlock()
+ m.challengeMu.RLock()
+ defer m.challengeMu.RUnlock()
if v, ok := m.httpTokens[tokenPath]; ok {
return v, nil
}
//
// It ignores any error returned from Cache.Put.
func (m *Manager) putHTTPToken(ctx context.Context, tokenPath, val string) {
- m.tokensMu.Lock()
- defer m.tokensMu.Unlock()
+ m.challengeMu.Lock()
+ defer m.challengeMu.Unlock()
if m.httpTokens == nil {
m.httpTokens = make(map[string][]byte)
}
//
// If m.Cache is non-nil, it blocks until Cache.Delete returns without a timeout.
func (m *Manager) deleteHTTPToken(tokenPath string) {
- m.tokensMu.Lock()
- defer m.tokensMu.Unlock()
+ m.challengeMu.Lock()
+ defer m.challengeMu.Unlock()
delete(m.httpTokens, tokenPath)
if m.Cache != nil {
m.Cache.Delete(context.Background(), httpTokenCacheKey(tokenPath))
client := m.Client
if client == nil {
- client = &acme.Client{DirectoryURL: acme.LetsEncryptURL}
+ client = &acme.Client{DirectoryURL: DefaultACMEDirectory}
}
if client.Key == nil {
var err error
}
a := &acme.Account{Contact: contact}
_, err := client.Register(ctx, a, m.Prompt)
- if ae, ok := err.(*acme.Error); err == nil || ok && ae.StatusCode == http.StatusConflict {
- // conflict indicates the key is already registered
+ if err == nil || isAccountAlreadyExist(err) {
m.client = client
err = nil
}
return m.client, err
}
+// isAccountAlreadyExist reports whether the err, as returned from acme.Client.Register,
+// indicates the account has already been registered.
+func isAccountAlreadyExist(err error) bool {
+ if err == acme.ErrAccountAlreadyExists {
+ return true
+ }
+ ae, ok := err.(*acme.Error)
+ return ok && ae.StatusCode == http.StatusConflict
+}
+
func (m *Manager) hostPolicy() HostPolicy {
if m.HostPolicy != nil {
return m.HostPolicy
}
}
+// postAsGet is POST-as-GET, a replacement for GET in RFC8555
+// as described in https://tools.ietf.org/html/rfc8555#section-6.3.
+// It makes a POST request in KID form with zero JWS payload.
+// See nopayload doc comments in jws.go.
+func (c *Client) postAsGet(ctx context.Context, url string, ok resOkay) (*http.Response, error) {
+ return c.post(ctx, nil, url, noPayload, ok)
+}
+
// post issues a signed POST request in JWS format using the provided key
// to the specified URL. If key is nil, c.Key is used instead.
// It returns a non-error value only when ok reports true.
// If key argument is nil and c.accountKID returns a non-zero keyID,
// the request is sent in KID form. Otherwise, JWK form is used.
//
-// In practice, when interfacing with RFC compliant CAs most requests are sent in KID form
+// In practice, when interfacing with RFC-compliant CAs most requests are sent in KID form
// and JWK is used only when KID is unavailable: new account endpoint and certificate
// revocation requests authenticated by a cert key.
// See jwsEncodeJSON for other details.
// See jwsEncodeJSON for details.
const noKeyID = keyID("")
+// noPayload indicates jwsEncodeJSON will encode zero-length octet string
+// in a JWS request. This is called POST-as-GET in RFC 8555 and is used to make
+// authenticated GET requests via POSTing with an empty payload.
+// See https://tools.ietf.org/html/rfc8555#section-6.3 for more details.
+const noPayload = ""
+
// jwsEncodeJSON signs claimset using provided key and a nonce.
// The result is serialized in JSON format containing either kid or jwk
// fields based on the provided keyID value.
phead = fmt.Sprintf(`{"alg":%q,"kid":%q,"nonce":%q,"url":%q}`, alg, kid, nonce, url)
}
phead = base64.RawURLEncoding.EncodeToString([]byte(phead))
- cs, err := json.Marshal(claimset)
- if err != nil {
- return nil, err
+ var payload string
+ if claimset != noPayload {
+ cs, err := json.Marshal(claimset)
+ if err != nil {
+ return nil, err
+ }
+ payload = base64.RawURLEncoding.EncodeToString(cs)
}
- payload := base64.RawURLEncoding.EncodeToString(cs)
hash := sha.New()
hash.Write([]byte(phead + "." + payload))
sig, err := jwsSign(key, sha, hash.Sum(nil))
import (
"context"
+ "crypto"
+ "encoding/base64"
"encoding/json"
+ "encoding/pem"
+ "errors"
"fmt"
+ "io"
+ "io/ioutil"
"net/http"
+ "time"
)
// DeactivateReg permanently disables an existing account associated with c.Key.
// A deactivated account can no longer request certificate issuance or access
// resources related to the account, such as orders or authorizations.
//
-// It works only with RFC8555 compliant CAs.
+// It only works with CAs implementing RFC 8555.
func (c *Client) DeactivateReg(ctx context.Context) error {
url := string(c.accountKID(ctx))
if url == "" {
return nil
}
-// registerRFC is quivalent to c.Register but for RFC-compliant CAs.
+// registerRFC is quivalent to c.Register but for CAs implementing RFC 8555.
// It expects c.Discover to have already been called.
// TODO: Implement externalAccountBinding.
func (c *Client) registerRFC(ctx context.Context, acct *Account, prompt func(tosURL string) bool) (*Account, error) {
return a, nil
}
-// updateGegRFC is equivalent to c.UpdateReg but for RFC-compliant CAs.
+// updateGegRFC is equivalent to c.UpdateReg but for CAs implementing RFC 8555.
// It expects c.Discover to have already been called.
func (c *Client) updateRegRFC(ctx context.Context, a *Account) (*Account, error) {
url := string(c.accountKID(ctx))
return responseAccount(res)
}
-// getGegRFC is equivalent to c.GetReg but for RFC-compliant CAs.
+// getGegRFC is equivalent to c.GetReg but for CAs implementing RFC 8555.
// It expects c.Discover to have already been called.
func (c *Client) getRegRFC(ctx context.Context) (*Account, error) {
req := json.RawMessage(`{"onlyReturnExisting": true}`)
Orders string
}
if err := json.NewDecoder(res.Body).Decode(&v); err != nil {
- return nil, fmt.Errorf("acme: invalid response: %v", err)
+ return nil, fmt.Errorf("acme: invalid account response: %v", err)
}
return &Account{
URI: res.Header.Get("Location"),
OrdersURL: v.Orders,
}, nil
}
+
+// AuthorizeOrder initiates the order-based application for certificate issuance,
+// as opposed to pre-authorization in Authorize.
+// It is only supported by CAs implementing RFC 8555.
+//
+// The caller then needs to fetch each authorization with GetAuthorization,
+// identify those with StatusPending status and fulfill a challenge using Accept.
+// Once all authorizations are satisfied, the caller will typically want to poll
+// order status using WaitOrder until it's in StatusReady state.
+// To finalize the order and obtain a certificate, the caller submits a CSR with CreateOrderCert.
+func (c *Client) AuthorizeOrder(ctx context.Context, id []AuthzID, opt ...OrderOption) (*Order, error) {
+ dir, err := c.Discover(ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ req := struct {
+ Identifiers []wireAuthzID `json:"identifiers"`
+ NotBefore string `json:"notBefore,omitempty"`
+ NotAfter string `json:"notAfter,omitempty"`
+ }{}
+ for _, v := range id {
+ req.Identifiers = append(req.Identifiers, wireAuthzID{
+ Type: v.Type,
+ Value: v.Value,
+ })
+ }
+ for _, o := range opt {
+ switch o := o.(type) {
+ case orderNotBeforeOpt:
+ req.NotBefore = time.Time(o).Format(time.RFC3339)
+ case orderNotAfterOpt:
+ req.NotAfter = time.Time(o).Format(time.RFC3339)
+ default:
+ // Package's fault if we let this happen.
+ panic(fmt.Sprintf("unsupported order option type %T", o))
+ }
+ }
+
+ res, err := c.post(ctx, nil, dir.OrderURL, req, wantStatus(http.StatusCreated))
+ if err != nil {
+ return nil, err
+ }
+ defer res.Body.Close()
+ return responseOrder(res)
+}
+
+// GetOrder retrives an order identified by the given URL.
+// For orders created with AuthorizeOrder, the url value is Order.URI.
+//
+// If a caller needs to poll an order until its status is final,
+// see the WaitOrder method.
+func (c *Client) GetOrder(ctx context.Context, url string) (*Order, error) {
+ if _, err := c.Discover(ctx); err != nil {
+ return nil, err
+ }
+
+ res, err := c.postAsGet(ctx, url, wantStatus(http.StatusOK))
+ if err != nil {
+ return nil, err
+ }
+ defer res.Body.Close()
+ return responseOrder(res)
+}
+
+// WaitOrder polls an order from the given URL until it is in one of the final states,
+// StatusReady, StatusValid or StatusInvalid, the CA responded with a non-retryable error
+// or the context is done.
+//
+// It returns a non-nil Order only if its Status is StatusReady or StatusValid.
+// In all other cases WaitOrder returns an error.
+// If the Status is StatusInvalid, the returned error is of type *OrderError.
+func (c *Client) WaitOrder(ctx context.Context, url string) (*Order, error) {
+ if _, err := c.Discover(ctx); err != nil {
+ return nil, err
+ }
+ for {
+ res, err := c.postAsGet(ctx, url, wantStatus(http.StatusOK))
+ if err != nil {
+ return nil, err
+ }
+ o, err := responseOrder(res)
+ res.Body.Close()
+ switch {
+ case err != nil:
+ // Skip and retry.
+ case o.Status == StatusInvalid:
+ return nil, &OrderError{OrderURL: o.URI, Status: o.Status}
+ case o.Status == StatusReady || o.Status == StatusValid:
+ return o, nil
+ }
+
+ d := retryAfter(res.Header.Get("Retry-After"))
+ if d == 0 {
+ // Default retry-after.
+ // Same reasoning as in WaitAuthorization.
+ d = time.Second
+ }
+ t := time.NewTimer(d)
+ select {
+ case <-ctx.Done():
+ t.Stop()
+ return nil, ctx.Err()
+ case <-t.C:
+ // Retry.
+ }
+ }
+}
+
+func responseOrder(res *http.Response) (*Order, error) {
+ var v struct {
+ Status string
+ Expires time.Time
+ Identifiers []wireAuthzID
+ NotBefore time.Time
+ NotAfter time.Time
+ Error *wireError
+ Authorizations []string
+ Finalize string
+ Certificate string
+ }
+ if err := json.NewDecoder(res.Body).Decode(&v); err != nil {
+ return nil, fmt.Errorf("acme: error reading order: %v", err)
+ }
+ o := &Order{
+ URI: res.Header.Get("Location"),
+ Status: v.Status,
+ Expires: v.Expires,
+ NotBefore: v.NotBefore,
+ NotAfter: v.NotAfter,
+ AuthzURLs: v.Authorizations,
+ FinalizeURL: v.Finalize,
+ CertURL: v.Certificate,
+ }
+ for _, id := range v.Identifiers {
+ o.Identifiers = append(o.Identifiers, AuthzID{Type: id.Type, Value: id.Value})
+ }
+ if v.Error != nil {
+ o.Error = v.Error.error(nil /* headers */)
+ }
+ return o, nil
+}
+
+// CreateOrderCert submits the CSR (Certificate Signing Request) to a CA at the specified URL.
+// The URL is the FinalizeURL field of an Order created with AuthorizeOrder.
+//
+// If the bundle argument is true, the returned value also contain the CA (issuer)
+// certificate chain. Otherwise, only a leaf certificate is returned.
+// The returned URL can be used to re-fetch the certificate using FetchCert.
+//
+// This method is only supported by CAs implementing RFC 8555. See CreateCert for pre-RFC CAs.
+//
+// CreateOrderCert returns an error if the CA's response is unreasonably large.
+// Callers are encouraged to parse the returned value to ensure the certificate is valid and has the expected features.
+func (c *Client) CreateOrderCert(ctx context.Context, url string, csr []byte, bundle bool) (der [][]byte, certURL string, err error) {
+ if _, err := c.Discover(ctx); err != nil { // required by c.accountKID
+ return nil, "", err
+ }
+
+ // RFC describes this as "finalize order" request.
+ req := struct {
+ CSR string `json:"csr"`
+ }{
+ CSR: base64.RawURLEncoding.EncodeToString(csr),
+ }
+ res, err := c.post(ctx, nil, url, req, wantStatus(http.StatusOK))
+ if err != nil {
+ return nil, "", err
+ }
+ defer res.Body.Close()
+ o, err := responseOrder(res)
+ if err != nil {
+ return nil, "", err
+ }
+
+ // Wait for CA to issue the cert if they haven't.
+ if o.Status != StatusValid {
+ o, err = c.WaitOrder(ctx, o.URI)
+ }
+ if err != nil {
+ return nil, "", err
+ }
+ // The only acceptable status post finalize and WaitOrder is "valid".
+ if o.Status != StatusValid {
+ return nil, "", &OrderError{OrderURL: o.URI, Status: o.Status}
+ }
+ crt, err := c.fetchCertRFC(ctx, o.CertURL, bundle)
+ return crt, o.CertURL, err
+}
+
+// fetchCertRFC downloads issued certificate from the given URL.
+// It expects the CA to respond with PEM-encoded certificate chain.
+//
+// The URL argument is the CertURL field of Order.
+func (c *Client) fetchCertRFC(ctx context.Context, url string, bundle bool) ([][]byte, error) {
+ res, err := c.postAsGet(ctx, url, wantStatus(http.StatusOK))
+ if err != nil {
+ return nil, err
+ }
+ defer res.Body.Close()
+
+ // Get all the bytes up to a sane maximum.
+ // Account very roughly for base64 overhead.
+ const max = maxCertChainSize + maxCertChainSize/33
+ b, err := ioutil.ReadAll(io.LimitReader(res.Body, max+1))
+ if err != nil {
+ return nil, fmt.Errorf("acme: fetch cert response stream: %v", err)
+ }
+ if len(b) > max {
+ return nil, errors.New("acme: certificate chain is too big")
+ }
+
+ // Decode PEM chain.
+ var chain [][]byte
+ for {
+ var p *pem.Block
+ p, b = pem.Decode(b)
+ if p == nil {
+ break
+ }
+ if p.Type != "CERTIFICATE" {
+ return nil, fmt.Errorf("acme: invalid PEM cert type %q", p.Type)
+ }
+
+ chain = append(chain, p.Bytes)
+ if !bundle {
+ return chain, nil
+ }
+ if len(chain) > maxChainLen {
+ return nil, errors.New("acme: certificate chain is too long")
+ }
+ }
+ if len(chain) == 0 {
+ return nil, errors.New("acme: certificate chain is empty")
+ }
+ return chain, nil
+}
+
+// sends a cert revocation request in either JWK form when key is non-nil or KID form otherwise.
+func (c *Client) revokeCertRFC(ctx context.Context, key crypto.Signer, cert []byte, reason CRLReasonCode) error {
+ req := &struct {
+ Cert string `json:"certificate"`
+ Reason int `json:"reason"`
+ }{
+ Cert: base64.RawURLEncoding.EncodeToString(cert),
+ Reason: int(reason),
+ }
+ res, err := c.post(ctx, key, c.dir.RevokeURL, req, wantStatus(http.StatusOK))
+ if err != nil {
+ if isAlreadyRevoked(err) {
+ // Assume it is not an error to revoke an already revoked cert.
+ return nil
+ }
+ return err
+ }
+ defer res.Body.Close()
+ return nil
+}
+
+func isAlreadyRevoked(err error) bool {
+ e, ok := err.(*Error)
+ return ok && e.ProblemType == "urn:ietf:params:acme:error:alreadyRevoked"
+}
"time"
)
-// ACME server response statuses used to describe Authorization and Challenge states.
+// ACME status values of Account, Order, Authorization and Challenge objects.
+// See https://tools.ietf.org/html/rfc8555#section-7.1.6 for details.
const (
StatusDeactivated = "deactivated"
+ StatusExpired = "expired"
StatusInvalid = "invalid"
StatusPending = "pending"
StatusProcessing = "processing"
+ StatusReady = "ready"
StatusRevoked = "revoked"
StatusUnknown = "unknown"
StatusValid = "valid"
return fmt.Sprintf("acme: authorization error for %s: %s", a.Identifier, strings.Join(e, "; "))
}
+// OrderError is returned from Client's order related methods.
+// It indicates the order is unusable and the clients should start over with
+// AuthorizeOrder.
+//
+// The clients can still fetch the order object from CA using GetOrder
+// to inspect its state.
+type OrderError struct {
+ OrderURL string
+ Status string
+}
+
+func (oe *OrderError) Error() string {
+ return fmt.Sprintf("acme: order %s status: %s", oe.OrderURL, oe.Status)
+}
+
// RateLimit reports whether err represents a rate limit error and
// any Retry-After duration returned by the server.
//
}
// Account is a user account. It is associated with a private key.
-// Non-RFC8555 fields are empty when interfacing with a compliant CA.
+// Non-RFC 8555 fields are empty when interfacing with a compliant CA.
type Account struct {
// URI is the account unique ID, which is also a URL used to retrieve
// account data from the CA.
- // When interfacing with RFC8555-compliant CAs, URI is the "kid" field
+ // When interfacing with RFC 8555-compliant CAs, URI is the "kid" field
// value in JWS signed requests.
URI string
Contact []string
// Status indicates current account status as returned by the CA.
- // Possible values are "valid", "deactivated", and "revoked".
+ // Possible values are StatusValid, StatusDeactivated, and StatusRevoked.
Status string
// OrdersURL is a URL from which a list of orders submitted by this account
// A value not matching CurrentTerms indicates that the user hasn't agreed
// to the actual Terms of Service of the CA.
//
- // It is non-RFC8555 compliant. Package users can store the ToS they agree to
+ // It is non-RFC 8555 compliant. Package users can store the ToS they agree to
// during Client's Register call in the prompt callback function.
AgreedTerms string
// Actual terms of a CA.
//
- // It is non-RFC8555 compliant. Use Directory's Terms field.
+ // It is non-RFC 8555 compliant. Use Directory's Terms field.
// When a CA updates their terms and requires an account agreement,
// a URL at which instructions to do so is available in Error's Instance field.
CurrentTerms string
// Authz is the authorization URL used to initiate a new authz flow.
//
- // It is non-RFC8555 compliant. Use Directory's AuthzURL or OrderURL.
+ // It is non-RFC 8555 compliant. Use Directory's AuthzURL or OrderURL.
Authz string
// Authorizations is a URI from which a list of authorizations
// granted to this account can be fetched via a GET request.
//
- // It is non-RFC8555 compliant and is obsoleted by OrdersURL.
+ // It is non-RFC 8555 compliant and is obsoleted by OrdersURL.
Authorizations string
// Certificates is a URI from which a list of certificates
// issued for this account can be fetched via a GET request.
//
- // It is non-RFC8555 compliant and is obsoleted by OrdersURL.
+ // It is non-RFC 8555 compliant and is obsoleted by OrdersURL.
Certificates string
}
NonceURL string
// RegURL is an account endpoint URL, allowing for creating new accounts.
- // Pre-RFC8555 CAs also allow modifying existing accounts at this URL.
+ // Pre-RFC 8555 CAs also allow modifying existing accounts at this URL.
RegURL string
// OrderURL is used to initiate the certificate issuance flow
- // as described in RFC8555.
+ // as described in RFC 8555.
OrderURL string
// AuthzURL is used to initiate identifier pre-authorization flow.
AuthzURL string
// CertURL is a new certificate issuance endpoint URL.
- // It is non-RFC8555 compliant and is obsoleted by OrderURL.
+ // It is non-RFC 8555 compliant and is obsoleted by OrderURL.
CertURL string
// RevokeURL is used to initiate a certificate revocation flow.
ExternalAccountRequired bool
}
-// Challenge encodes a returned CA challenge.
-// Its Error field may be non-nil if the challenge is part of an Authorization
-// with StatusInvalid.
-type Challenge struct {
- // Type is the challenge type, e.g. "http-01", "tls-sni-02", "dns-01".
- Type string
+// rfcCompliant reports whether the ACME server implements RFC 8555.
+// Note that some servers may have incomplete RFC implementation
+// even if the returned value is true.
+// If rfcCompliant reports false, the server most likely implements draft-02.
+func (d *Directory) rfcCompliant() bool {
+ return d.OrderURL != ""
+}
- // URI is where a challenge response can be posted to.
+// Order represents a client's request for a certificate.
+// It tracks the request flow progress through to issuance.
+type Order struct {
+ // URI uniquely identifies an order.
URI string
- // Token is a random value that uniquely identifies the challenge.
- Token string
-
- // Status identifies the status of this challenge.
+ // Status represents the current status of the order.
+ // It indicates which action the client should take.
+ //
+ // Possible values are StatusPending, StatusReady, StatusProcessing, StatusValid and StatusInvalid.
+ // Pending means the CA does not believe that the client has fulfilled the requirements.
+ // Ready indicates that the client has fulfilled all the requirements and can submit a CSR
+ // to obtain a certificate. This is done with Client's CreateOrderCert.
+ // Processing means the certificate is being issued.
+ // Valid indicates the CA has issued the certificate. It can be downloaded
+ // from the Order's CertURL. This is done with Client's FetchCert.
+ // Invalid means the certificate will not be issued. Users should consider this order
+ // abandoned.
Status string
- // Error indicates the reason for an authorization failure
- // when this challenge was used.
- // The type of a non-nil value is *Error.
- Error error
+ // Expires is the timestamp after which CA considers this order invalid.
+ Expires time.Time
+
+ // Identifiers contains all identifier objects which the order pertains to.
+ Identifiers []AuthzID
+
+ // NotBefore is the requested value of the notBefore field in the certificate.
+ NotBefore time.Time
+
+ // NotAfter is the requested value of the notAfter field in the certificate.
+ NotAfter time.Time
+
+ // AuthzURLs represents authorizations to complete before a certificate
+ // for identifiers specified in the order can be issued.
+ // It also contains unexpired authorizations that the client has completed
+ // in the past.
+ //
+ // Authorization objects can be fetched using Client's GetAuthorization method.
+ //
+ // The required authorizations are dictated by CA policies.
+ // There may not be a 1:1 relationship between the identifiers and required authorizations.
+ // Required authorizations can be identified by their StatusPending status.
+ //
+ // For orders in the StatusValid or StatusInvalid state these are the authorizations
+ // which were completed.
+ AuthzURLs []string
+
+ // FinalizeURL is the endpoint at which a CSR is submitted to obtain a certificate
+ // once all the authorizations are satisfied.
+ FinalizeURL string
+
+ // CertURL points to the certificate that has been issued in response to this order.
+ CertURL string
+
+ // The error that occurred while processing the order as received from a CA, if any.
+ Error *Error
+}
+
+// OrderOption allows customizing Client.AuthorizeOrder call.
+type OrderOption interface {
+ privateOrderOpt()
+}
+
+// WithOrderNotBefore sets order's NotBefore field.
+func WithOrderNotBefore(t time.Time) OrderOption {
+ return orderNotBeforeOpt(t)
}
+// WithOrderNotAfter sets order's NotAfter field.
+func WithOrderNotAfter(t time.Time) OrderOption {
+ return orderNotAfterOpt(t)
+}
+
+type orderNotBeforeOpt time.Time
+
+func (orderNotBeforeOpt) privateOrderOpt() {}
+
+type orderNotAfterOpt time.Time
+
+func (orderNotAfterOpt) privateOrderOpt() {}
+
// Authorization encodes an authorization response.
type Authorization struct {
// URI uniquely identifies a authorization.
URI string
- // Status identifies the status of an authorization.
+ // Status is the current status of an authorization.
+ // Possible values are StatusPending, StatusValid, StatusInvalid, StatusDeactivated,
+ // StatusExpired and StatusRevoked.
Status string
// Identifier is what the account is authorized to represent.
Identifier AuthzID
+ // The timestamp after which the CA considers the authorization invalid.
+ Expires time.Time
+
+ // Wildcard is true for authorizations of a wildcard domain name.
+ Wildcard bool
+
// Challenges that the client needs to fulfill in order to prove possession
// of the identifier (for pending authorizations).
- // For final authorizations, the challenges that were used.
+ // For valid authorizations, the challenge that was validated.
+ // For invalid authorizations, the challenge that was attempted and failed.
+ //
+ // RFC 8555 compatible CAs require users to fuflfill only one of the challenges.
Challenges []*Challenge
// A collection of sets of challenges, each of which would be sufficient
// Clients must complete a set of challenges that covers at least one set.
// Challenges are identified by their indices in the challenges array.
// If this field is empty, the client needs to complete all challenges.
+ //
+ // This field is unused in RFC 8555.
Combinations [][]int
}
// AuthzID is an identifier that an account is authorized to represent.
type AuthzID struct {
- Type string // The type of identifier, e.g. "dns".
+ Type string // The type of identifier, "dns" or "ip".
Value string // The identifier itself, e.g. "example.org".
}
+// DomainIDs creates a slice of AuthzID with "dns" identifier type.
+func DomainIDs(names ...string) []AuthzID {
+ a := make([]AuthzID, len(names))
+ for i, v := range names {
+ a[i] = AuthzID{Type: "dns", Value: v}
+ }
+ return a
+}
+
+// IPIDs creates a slice of AuthzID with "ip" identifier type.
+// Each element of addr is textual form of an address as defined
+// in RFC1123 Section 2.1 for IPv4 and in RFC5952 Section 4 for IPv6.
+func IPIDs(addr ...string) []AuthzID {
+ a := make([]AuthzID, len(addr))
+ for i, v := range addr {
+ a[i] = AuthzID{Type: "ip", Value: v}
+ }
+ return a
+}
+
+// wireAuthzID is ACME JSON representation of authorization identifier objects.
+type wireAuthzID struct {
+ Type string `json:"type"`
+ Value string `json:"value"`
+}
+
// wireAuthz is ACME JSON representation of Authorization objects.
type wireAuthz struct {
+ Identifier wireAuthzID
Status string
+ Expires time.Time
+ Wildcard bool
Challenges []wireChallenge
Combinations [][]int
- Identifier struct {
- Type string
- Value string
- }
}
func (z *wireAuthz) authorization(uri string) *Authorization {
URI: uri,
Status: z.Status,
Identifier: AuthzID{Type: z.Identifier.Type, Value: z.Identifier.Value},
- Combinations: z.Combinations, // shallow copy
+ Expires: z.Expires,
+ Wildcard: z.Wildcard,
Challenges: make([]*Challenge, len(z.Challenges)),
+ Combinations: z.Combinations, // shallow copy
}
for i, v := range z.Challenges {
a.Challenges[i] = v.challenge()
return err
}
+// Challenge encodes a returned CA challenge.
+// Its Error field may be non-nil if the challenge is part of an Authorization
+// with StatusInvalid.
+type Challenge struct {
+ // Type is the challenge type, e.g. "http-01", "tls-alpn-01", "dns-01".
+ Type string
+
+ // URI is where a challenge response can be posted to.
+ URI string
+
+ // Token is a random value that uniquely identifies the challenge.
+ Token string
+
+ // Status identifies the status of this challenge.
+ // In RFC 8555, possible values are StatusPending, StatusProcessing, StatusValid,
+ // and StatusInvalid.
+ Status string
+
+ // Validated is the time at which the CA validated this challenge.
+ // Always zero value in pre-RFC 8555.
+ Validated time.Time
+
+ // Error indicates the reason for an authorization failure
+ // when this challenge was used.
+ // The type of a non-nil value is *Error.
+ Error error
+}
+
// wireChallenge is ACME JSON challenge representation.
type wireChallenge struct {
- URI string `json:"uri"`
- Type string
- Token string
- Status string
- Error *wireError
+ URL string `json:"url"` // RFC
+ URI string `json:"uri"` // pre-RFC
+ Type string
+ Token string
+ Status string
+ Validated time.Time
+ Error *wireError
}
func (c *wireChallenge) challenge() *Challenge {
v := &Challenge{
- URI: c.URI,
+ URI: c.URL,
Type: c.Type,
Token: c.Token,
Status: c.Status,
}
+ if v.URI == "" {
+ v.URI = c.URI // c.URL was empty; use legacy
+ }
if v.Status == "" {
v.Status = StatusPending
}
--- /dev/null
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build go1.11
+// +build !gccgo,!appengine
+
+package chacha20
+
+const bufSize = 256
+
+//go:noescape
+func xorKeyStreamVX(dst, src []byte, key *[8]uint32, nonce *[3]uint32, counter *uint32)
+
+func (c *Cipher) xorKeyStreamBlocks(dst, src []byte) {
+ xorKeyStreamVX(dst, src, &c.key, &c.nonce, &c.counter)
+}
--- /dev/null
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build go1.11
+// +build !gccgo,!appengine
+
+#include "textflag.h"
+
+#define NUM_ROUNDS 10
+
+// func xorKeyStreamVX(dst, src []byte, key *[8]uint32, nonce *[3]uint32, counter *uint32)
+TEXT ·xorKeyStreamVX(SB), NOSPLIT, $0
+ MOVD dst+0(FP), R1
+ MOVD src+24(FP), R2
+ MOVD src_len+32(FP), R3
+ MOVD key+48(FP), R4
+ MOVD nonce+56(FP), R6
+ MOVD counter+64(FP), R7
+
+ MOVD $·constants(SB), R10
+ MOVD $·incRotMatrix(SB), R11
+
+ MOVW (R7), R20
+
+ AND $~255, R3, R13
+ ADD R2, R13, R12 // R12 for block end
+ AND $255, R3, R13
+loop:
+ MOVD $NUM_ROUNDS, R21
+ VLD1 (R11), [V30.S4, V31.S4]
+
+ // load contants
+ // VLD4R (R10), [V0.S4, V1.S4, V2.S4, V3.S4]
+ WORD $0x4D60E940
+
+ // load keys
+ // VLD4R 16(R4), [V4.S4, V5.S4, V6.S4, V7.S4]
+ WORD $0x4DFFE884
+ // VLD4R 16(R4), [V8.S4, V9.S4, V10.S4, V11.S4]
+ WORD $0x4DFFE888
+ SUB $32, R4
+
+ // load counter + nonce
+ // VLD1R (R7), [V12.S4]
+ WORD $0x4D40C8EC
+
+ // VLD3R (R6), [V13.S4, V14.S4, V15.S4]
+ WORD $0x4D40E8CD
+
+ // update counter
+ VADD V30.S4, V12.S4, V12.S4
+
+chacha:
+ // V0..V3 += V4..V7
+ // V12..V15 <<<= ((V12..V15 XOR V0..V3), 16)
+ VADD V0.S4, V4.S4, V0.S4
+ VADD V1.S4, V5.S4, V1.S4
+ VADD V2.S4, V6.S4, V2.S4
+ VADD V3.S4, V7.S4, V3.S4
+ VEOR V12.B16, V0.B16, V12.B16
+ VEOR V13.B16, V1.B16, V13.B16
+ VEOR V14.B16, V2.B16, V14.B16
+ VEOR V15.B16, V3.B16, V15.B16
+ VREV32 V12.H8, V12.H8
+ VREV32 V13.H8, V13.H8
+ VREV32 V14.H8, V14.H8
+ VREV32 V15.H8, V15.H8
+ // V8..V11 += V12..V15
+ // V4..V7 <<<= ((V4..V7 XOR V8..V11), 12)
+ VADD V8.S4, V12.S4, V8.S4
+ VADD V9.S4, V13.S4, V9.S4
+ VADD V10.S4, V14.S4, V10.S4
+ VADD V11.S4, V15.S4, V11.S4
+ VEOR V8.B16, V4.B16, V16.B16
+ VEOR V9.B16, V5.B16, V17.B16
+ VEOR V10.B16, V6.B16, V18.B16
+ VEOR V11.B16, V7.B16, V19.B16
+ VSHL $12, V16.S4, V4.S4
+ VSHL $12, V17.S4, V5.S4
+ VSHL $12, V18.S4, V6.S4
+ VSHL $12, V19.S4, V7.S4
+ VSRI $20, V16.S4, V4.S4
+ VSRI $20, V17.S4, V5.S4
+ VSRI $20, V18.S4, V6.S4
+ VSRI $20, V19.S4, V7.S4
+
+ // V0..V3 += V4..V7
+ // V12..V15 <<<= ((V12..V15 XOR V0..V3), 8)
+ VADD V0.S4, V4.S4, V0.S4
+ VADD V1.S4, V5.S4, V1.S4
+ VADD V2.S4, V6.S4, V2.S4
+ VADD V3.S4, V7.S4, V3.S4
+ VEOR V12.B16, V0.B16, V12.B16
+ VEOR V13.B16, V1.B16, V13.B16
+ VEOR V14.B16, V2.B16, V14.B16
+ VEOR V15.B16, V3.B16, V15.B16
+ VTBL V31.B16, [V12.B16], V12.B16
+ VTBL V31.B16, [V13.B16], V13.B16
+ VTBL V31.B16, [V14.B16], V14.B16
+ VTBL V31.B16, [V15.B16], V15.B16
+
+ // V8..V11 += V12..V15
+ // V4..V7 <<<= ((V4..V7 XOR V8..V11), 7)
+ VADD V12.S4, V8.S4, V8.S4
+ VADD V13.S4, V9.S4, V9.S4
+ VADD V14.S4, V10.S4, V10.S4
+ VADD V15.S4, V11.S4, V11.S4
+ VEOR V8.B16, V4.B16, V16.B16
+ VEOR V9.B16, V5.B16, V17.B16
+ VEOR V10.B16, V6.B16, V18.B16
+ VEOR V11.B16, V7.B16, V19.B16
+ VSHL $7, V16.S4, V4.S4
+ VSHL $7, V17.S4, V5.S4
+ VSHL $7, V18.S4, V6.S4
+ VSHL $7, V19.S4, V7.S4
+ VSRI $25, V16.S4, V4.S4
+ VSRI $25, V17.S4, V5.S4
+ VSRI $25, V18.S4, V6.S4
+ VSRI $25, V19.S4, V7.S4
+
+ // V0..V3 += V5..V7, V4
+ // V15,V12-V14 <<<= ((V15,V12-V14 XOR V0..V3), 16)
+ VADD V0.S4, V5.S4, V0.S4
+ VADD V1.S4, V6.S4, V1.S4
+ VADD V2.S4, V7.S4, V2.S4
+ VADD V3.S4, V4.S4, V3.S4
+ VEOR V15.B16, V0.B16, V15.B16
+ VEOR V12.B16, V1.B16, V12.B16
+ VEOR V13.B16, V2.B16, V13.B16
+ VEOR V14.B16, V3.B16, V14.B16
+ VREV32 V12.H8, V12.H8
+ VREV32 V13.H8, V13.H8
+ VREV32 V14.H8, V14.H8
+ VREV32 V15.H8, V15.H8
+
+ // V10 += V15; V5 <<<= ((V10 XOR V5), 12)
+ // ...
+ VADD V15.S4, V10.S4, V10.S4
+ VADD V12.S4, V11.S4, V11.S4
+ VADD V13.S4, V8.S4, V8.S4
+ VADD V14.S4, V9.S4, V9.S4
+ VEOR V10.B16, V5.B16, V16.B16
+ VEOR V11.B16, V6.B16, V17.B16
+ VEOR V8.B16, V7.B16, V18.B16
+ VEOR V9.B16, V4.B16, V19.B16
+ VSHL $12, V16.S4, V5.S4
+ VSHL $12, V17.S4, V6.S4
+ VSHL $12, V18.S4, V7.S4
+ VSHL $12, V19.S4, V4.S4
+ VSRI $20, V16.S4, V5.S4
+ VSRI $20, V17.S4, V6.S4
+ VSRI $20, V18.S4, V7.S4
+ VSRI $20, V19.S4, V4.S4
+
+ // V0 += V5; V15 <<<= ((V0 XOR V15), 8)
+ // ...
+ VADD V5.S4, V0.S4, V0.S4
+ VADD V6.S4, V1.S4, V1.S4
+ VADD V7.S4, V2.S4, V2.S4
+ VADD V4.S4, V3.S4, V3.S4
+ VEOR V0.B16, V15.B16, V15.B16
+ VEOR V1.B16, V12.B16, V12.B16
+ VEOR V2.B16, V13.B16, V13.B16
+ VEOR V3.B16, V14.B16, V14.B16
+ VTBL V31.B16, [V12.B16], V12.B16
+ VTBL V31.B16, [V13.B16], V13.B16
+ VTBL V31.B16, [V14.B16], V14.B16
+ VTBL V31.B16, [V15.B16], V15.B16
+
+ // V10 += V15; V5 <<<= ((V10 XOR V5), 7)
+ // ...
+ VADD V15.S4, V10.S4, V10.S4
+ VADD V12.S4, V11.S4, V11.S4
+ VADD V13.S4, V8.S4, V8.S4
+ VADD V14.S4, V9.S4, V9.S4
+ VEOR V10.B16, V5.B16, V16.B16
+ VEOR V11.B16, V6.B16, V17.B16
+ VEOR V8.B16, V7.B16, V18.B16
+ VEOR V9.B16, V4.B16, V19.B16
+ VSHL $7, V16.S4, V5.S4
+ VSHL $7, V17.S4, V6.S4
+ VSHL $7, V18.S4, V7.S4
+ VSHL $7, V19.S4, V4.S4
+ VSRI $25, V16.S4, V5.S4
+ VSRI $25, V17.S4, V6.S4
+ VSRI $25, V18.S4, V7.S4
+ VSRI $25, V19.S4, V4.S4
+
+ SUB $1, R21
+ CBNZ R21, chacha
+
+ // VLD4R (R10), [V16.S4, V17.S4, V18.S4, V19.S4]
+ WORD $0x4D60E950
+
+ // VLD4R 16(R4), [V20.S4, V21.S4, V22.S4, V23.S4]
+ WORD $0x4DFFE894
+ VADD V30.S4, V12.S4, V12.S4
+ VADD V16.S4, V0.S4, V0.S4
+ VADD V17.S4, V1.S4, V1.S4
+ VADD V18.S4, V2.S4, V2.S4
+ VADD V19.S4, V3.S4, V3.S4
+ // VLD4R 16(R4), [V24.S4, V25.S4, V26.S4, V27.S4]
+ WORD $0x4DFFE898
+ // restore R4
+ SUB $32, R4
+
+ // load counter + nonce
+ // VLD1R (R7), [V28.S4]
+ WORD $0x4D40C8FC
+ // VLD3R (R6), [V29.S4, V30.S4, V31.S4]
+ WORD $0x4D40E8DD
+
+ VADD V20.S4, V4.S4, V4.S4
+ VADD V21.S4, V5.S4, V5.S4
+ VADD V22.S4, V6.S4, V6.S4
+ VADD V23.S4, V7.S4, V7.S4
+ VADD V24.S4, V8.S4, V8.S4
+ VADD V25.S4, V9.S4, V9.S4
+ VADD V26.S4, V10.S4, V10.S4
+ VADD V27.S4, V11.S4, V11.S4
+ VADD V28.S4, V12.S4, V12.S4
+ VADD V29.S4, V13.S4, V13.S4
+ VADD V30.S4, V14.S4, V14.S4
+ VADD V31.S4, V15.S4, V15.S4
+
+ VZIP1 V1.S4, V0.S4, V16.S4
+ VZIP2 V1.S4, V0.S4, V17.S4
+ VZIP1 V3.S4, V2.S4, V18.S4
+ VZIP2 V3.S4, V2.S4, V19.S4
+ VZIP1 V5.S4, V4.S4, V20.S4
+ VZIP2 V5.S4, V4.S4, V21.S4
+ VZIP1 V7.S4, V6.S4, V22.S4
+ VZIP2 V7.S4, V6.S4, V23.S4
+ VZIP1 V9.S4, V8.S4, V24.S4
+ VZIP2 V9.S4, V8.S4, V25.S4
+ VZIP1 V11.S4, V10.S4, V26.S4
+ VZIP2 V11.S4, V10.S4, V27.S4
+ VZIP1 V13.S4, V12.S4, V28.S4
+ VZIP2 V13.S4, V12.S4, V29.S4
+ VZIP1 V15.S4, V14.S4, V30.S4
+ VZIP2 V15.S4, V14.S4, V31.S4
+ VZIP1 V18.D2, V16.D2, V0.D2
+ VZIP2 V18.D2, V16.D2, V4.D2
+ VZIP1 V19.D2, V17.D2, V8.D2
+ VZIP2 V19.D2, V17.D2, V12.D2
+ VLD1.P 64(R2), [V16.B16, V17.B16, V18.B16, V19.B16]
+
+ VZIP1 V22.D2, V20.D2, V1.D2
+ VZIP2 V22.D2, V20.D2, V5.D2
+ VZIP1 V23.D2, V21.D2, V9.D2
+ VZIP2 V23.D2, V21.D2, V13.D2
+ VLD1.P 64(R2), [V20.B16, V21.B16, V22.B16, V23.B16]
+ VZIP1 V26.D2, V24.D2, V2.D2
+ VZIP2 V26.D2, V24.D2, V6.D2
+ VZIP1 V27.D2, V25.D2, V10.D2
+ VZIP2 V27.D2, V25.D2, V14.D2
+ VLD1.P 64(R2), [V24.B16, V25.B16, V26.B16, V27.B16]
+ VZIP1 V30.D2, V28.D2, V3.D2
+ VZIP2 V30.D2, V28.D2, V7.D2
+ VZIP1 V31.D2, V29.D2, V11.D2
+ VZIP2 V31.D2, V29.D2, V15.D2
+ VLD1.P 64(R2), [V28.B16, V29.B16, V30.B16, V31.B16]
+ VEOR V0.B16, V16.B16, V16.B16
+ VEOR V1.B16, V17.B16, V17.B16
+ VEOR V2.B16, V18.B16, V18.B16
+ VEOR V3.B16, V19.B16, V19.B16
+ VST1.P [V16.B16, V17.B16, V18.B16, V19.B16], 64(R1)
+ VEOR V4.B16, V20.B16, V20.B16
+ VEOR V5.B16, V21.B16, V21.B16
+ VEOR V6.B16, V22.B16, V22.B16
+ VEOR V7.B16, V23.B16, V23.B16
+ VST1.P [V20.B16, V21.B16, V22.B16, V23.B16], 64(R1)
+ VEOR V8.B16, V24.B16, V24.B16
+ VEOR V9.B16, V25.B16, V25.B16
+ VEOR V10.B16, V26.B16, V26.B16
+ VEOR V11.B16, V27.B16, V27.B16
+ VST1.P [V24.B16, V25.B16, V26.B16, V27.B16], 64(R1)
+ VEOR V12.B16, V28.B16, V28.B16
+ VEOR V13.B16, V29.B16, V29.B16
+ VEOR V14.B16, V30.B16, V30.B16
+ VEOR V15.B16, V31.B16, V31.B16
+ VST1.P [V28.B16, V29.B16, V30.B16, V31.B16], 64(R1)
+
+ ADD $4, R20
+ MOVW R20, (R7) // update counter
+
+ CMP R2, R12
+ BGT loop
+
+ RET
+
+
+DATA ·constants+0x00(SB)/4, $0x61707865
+DATA ·constants+0x04(SB)/4, $0x3320646e
+DATA ·constants+0x08(SB)/4, $0x79622d32
+DATA ·constants+0x0c(SB)/4, $0x6b206574
+GLOBL ·constants(SB), NOPTR|RODATA, $32
+
+DATA ·incRotMatrix+0x00(SB)/4, $0x00000000
+DATA ·incRotMatrix+0x04(SB)/4, $0x00000001
+DATA ·incRotMatrix+0x08(SB)/4, $0x00000002
+DATA ·incRotMatrix+0x0c(SB)/4, $0x00000003
+DATA ·incRotMatrix+0x10(SB)/4, $0x02010003
+DATA ·incRotMatrix+0x14(SB)/4, $0x06050407
+DATA ·incRotMatrix+0x18(SB)/4, $0x0A09080B
+DATA ·incRotMatrix+0x1c(SB)/4, $0x0E0D0C0F
+GLOBL ·incRotMatrix(SB), NOPTR|RODATA, $32
--- /dev/null
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package chacha20 implements the ChaCha20 and XChaCha20 encryption algorithms
+// as specified in RFC 8439 and draft-irtf-cfrg-xchacha-01.
+package chacha20
+
+import (
+ "crypto/cipher"
+ "encoding/binary"
+ "errors"
+ "math/bits"
+
+ "golang.org/x/crypto/internal/subtle"
+)
+
+const (
+ // KeySize is the size of the key used by this cipher, in bytes.
+ KeySize = 32
+
+ // NonceSize is the size of the nonce used with the standard variant of this
+ // cipher, in bytes.
+ //
+ // Note that this is too short to be safely generated at random if the same
+ // key is reused more than 2³² times.
+ NonceSize = 12
+
+ // NonceSizeX is the size of the nonce used with the XChaCha20 variant of
+ // this cipher, in bytes.
+ NonceSizeX = 24
+)
+
+// Cipher is a stateful instance of ChaCha20 or XChaCha20 using a particular key
+// and nonce. A *Cipher implements the cipher.Stream interface.
+type Cipher struct {
+ // The ChaCha20 state is 16 words: 4 constant, 8 of key, 1 of counter
+ // (incremented after each block), and 3 of nonce.
+ key [8]uint32
+ counter uint32
+ nonce [3]uint32
+
+ // The last len bytes of buf are leftover key stream bytes from the previous
+ // XORKeyStream invocation. The size of buf depends on how many blocks are
+ // computed at a time.
+ buf [bufSize]byte
+ len int
+
+ // The counter-independent results of the first round are cached after they
+ // are computed the first time.
+ precompDone bool
+ p1, p5, p9, p13 uint32
+ p2, p6, p10, p14 uint32
+ p3, p7, p11, p15 uint32
+}
+
+var _ cipher.Stream = (*Cipher)(nil)
+
+// NewUnauthenticatedCipher creates a new ChaCha20 stream cipher with the given
+// 32 bytes key and a 12 or 24 bytes nonce. If a nonce of 24 bytes is provided,
+// the XChaCha20 construction will be used. It returns an error if key or nonce
+// have any other length.
+//
+// Note that ChaCha20, like all stream ciphers, is not authenticated and allows
+// attackers to silently tamper with the plaintext. For this reason, it is more
+// appropriate as a building block than as a standalone encryption mechanism.
+// Instead, consider using package golang.org/x/crypto/chacha20poly1305.
+func NewUnauthenticatedCipher(key, nonce []byte) (*Cipher, error) {
+ // This function is split into a wrapper so that the Cipher allocation will
+ // be inlined, and depending on how the caller uses the return value, won't
+ // escape to the heap.
+ c := &Cipher{}
+ return newUnauthenticatedCipher(c, key, nonce)
+}
+
+func newUnauthenticatedCipher(c *Cipher, key, nonce []byte) (*Cipher, error) {
+ if len(key) != KeySize {
+ return nil, errors.New("chacha20: wrong key size")
+ }
+ if len(nonce) == NonceSizeX {
+ // XChaCha20 uses the ChaCha20 core to mix 16 bytes of the nonce into a
+ // derived key, allowing it to operate on a nonce of 24 bytes. See
+ // draft-irtf-cfrg-xchacha-01, Section 2.3.
+ key, _ = HChaCha20(key, nonce[0:16])
+ cNonce := make([]byte, NonceSize)
+ copy(cNonce[4:12], nonce[16:24])
+ nonce = cNonce
+ } else if len(nonce) != NonceSize {
+ return nil, errors.New("chacha20: wrong nonce size")
+ }
+
+ c.key = [8]uint32{
+ binary.LittleEndian.Uint32(key[0:4]),
+ binary.LittleEndian.Uint32(key[4:8]),
+ binary.LittleEndian.Uint32(key[8:12]),
+ binary.LittleEndian.Uint32(key[12:16]),
+ binary.LittleEndian.Uint32(key[16:20]),
+ binary.LittleEndian.Uint32(key[20:24]),
+ binary.LittleEndian.Uint32(key[24:28]),
+ binary.LittleEndian.Uint32(key[28:32]),
+ }
+ c.nonce = [3]uint32{
+ binary.LittleEndian.Uint32(nonce[0:4]),
+ binary.LittleEndian.Uint32(nonce[4:8]),
+ binary.LittleEndian.Uint32(nonce[8:12]),
+ }
+ return c, nil
+}
+
+// The constant first 4 words of the ChaCha20 state.
+const (
+ j0 uint32 = 0x61707865 // expa
+ j1 uint32 = 0x3320646e // nd 3
+ j2 uint32 = 0x79622d32 // 2-by
+ j3 uint32 = 0x6b206574 // te k
+)
+
+const blockSize = 64
+
+// quarterRound is the core of ChaCha20. It shuffles the bits of 4 state words.
+// It's executed 4 times for each of the 20 ChaCha20 rounds, operating on all 16
+// words each round, in columnar or diagonal groups of 4 at a time.
+func quarterRound(a, b, c, d uint32) (uint32, uint32, uint32, uint32) {
+ a += b
+ d ^= a
+ d = bits.RotateLeft32(d, 16)
+ c += d
+ b ^= c
+ b = bits.RotateLeft32(b, 12)
+ a += b
+ d ^= a
+ d = bits.RotateLeft32(d, 8)
+ c += d
+ b ^= c
+ b = bits.RotateLeft32(b, 7)
+ return a, b, c, d
+}
+
+// XORKeyStream XORs each byte in the given slice with a byte from the
+// cipher's key stream. Dst and src must overlap entirely or not at all.
+//
+// If len(dst) < len(src), XORKeyStream will panic. It is acceptable
+// to pass a dst bigger than src, and in that case, XORKeyStream will
+// only update dst[:len(src)] and will not touch the rest of dst.
+//
+// Multiple calls to XORKeyStream behave as if the concatenation of
+// the src buffers was passed in a single run. That is, Cipher
+// maintains state and does not reset at each XORKeyStream call.
+func (s *Cipher) XORKeyStream(dst, src []byte) {
+ if len(src) == 0 {
+ return
+ }
+ if len(dst) < len(src) {
+ panic("chacha20: output smaller than input")
+ }
+ dst = dst[:len(src)]
+ if subtle.InexactOverlap(dst, src) {
+ panic("chacha20: invalid buffer overlap")
+ }
+
+ // First, drain any remaining key stream from a previous XORKeyStream.
+ if s.len != 0 {
+ keyStream := s.buf[bufSize-s.len:]
+ if len(src) < len(keyStream) {
+ keyStream = keyStream[:len(src)]
+ }
+ _ = src[len(keyStream)-1] // bounds check elimination hint
+ for i, b := range keyStream {
+ dst[i] = src[i] ^ b
+ }
+ s.len -= len(keyStream)
+ src = src[len(keyStream):]
+ dst = dst[len(keyStream):]
+ }
+
+ const blocksPerBuf = bufSize / blockSize
+ numBufs := (uint64(len(src)) + bufSize - 1) / bufSize
+ if uint64(s.counter)+numBufs*blocksPerBuf >= 1<<32 {
+ panic("chacha20: counter overflow")
+ }
+
+ // xorKeyStreamBlocks implementations expect input lengths that are a
+ // multiple of bufSize. Platform-specific ones process multiple blocks at a
+ // time, so have bufSizes that are a multiple of blockSize.
+
+ rem := len(src) % bufSize
+ full := len(src) - rem
+
+ if full > 0 {
+ s.xorKeyStreamBlocks(dst[:full], src[:full])
+ }
+
+ // If we have a partial (multi-)block, pad it for xorKeyStreamBlocks, and
+ // keep the leftover keystream for the next XORKeyStream invocation.
+ if rem > 0 {
+ s.buf = [bufSize]byte{}
+ copy(s.buf[:], src[full:])
+ s.xorKeyStreamBlocks(s.buf[:], s.buf[:])
+ s.len = bufSize - copy(dst[full:], s.buf[:])
+ }
+}
+
+func (s *Cipher) xorKeyStreamBlocksGeneric(dst, src []byte) {
+ if len(dst) != len(src) || len(dst)%blockSize != 0 {
+ panic("chacha20: internal error: wrong dst and/or src length")
+ }
+
+ // To generate each block of key stream, the initial cipher state
+ // (represented below) is passed through 20 rounds of shuffling,
+ // alternatively applying quarterRounds by columns (like 1, 5, 9, 13)
+ // or by diagonals (like 1, 6, 11, 12).
+ //
+ // 0:cccccccc 1:cccccccc 2:cccccccc 3:cccccccc
+ // 4:kkkkkkkk 5:kkkkkkkk 6:kkkkkkkk 7:kkkkkkkk
+ // 8:kkkkkkkk 9:kkkkkkkk 10:kkkkkkkk 11:kkkkkkkk
+ // 12:bbbbbbbb 13:nnnnnnnn 14:nnnnnnnn 15:nnnnnnnn
+ //
+ // c=constant k=key b=blockcount n=nonce
+ var (
+ c0, c1, c2, c3 = j0, j1, j2, j3
+ c4, c5, c6, c7 = s.key[0], s.key[1], s.key[2], s.key[3]
+ c8, c9, c10, c11 = s.key[4], s.key[5], s.key[6], s.key[7]
+ _, c13, c14, c15 = s.counter, s.nonce[0], s.nonce[1], s.nonce[2]
+ )
+
+ // Three quarters of the first round don't depend on the counter, so we can
+ // calculate them here, and reuse them for multiple blocks in the loop, and
+ // for future XORKeyStream invocations.
+ if !s.precompDone {
+ s.p1, s.p5, s.p9, s.p13 = quarterRound(c1, c5, c9, c13)
+ s.p2, s.p6, s.p10, s.p14 = quarterRound(c2, c6, c10, c14)
+ s.p3, s.p7, s.p11, s.p15 = quarterRound(c3, c7, c11, c15)
+ s.precompDone = true
+ }
+
+ for i := 0; i < len(src); i += blockSize {
+ // The remainder of the first column round.
+ fcr0, fcr4, fcr8, fcr12 := quarterRound(c0, c4, c8, s.counter)
+
+ // The second diagonal round.
+ x0, x5, x10, x15 := quarterRound(fcr0, s.p5, s.p10, s.p15)
+ x1, x6, x11, x12 := quarterRound(s.p1, s.p6, s.p11, fcr12)
+ x2, x7, x8, x13 := quarterRound(s.p2, s.p7, fcr8, s.p13)
+ x3, x4, x9, x14 := quarterRound(s.p3, fcr4, s.p9, s.p14)
+
+ // The remaining 18 rounds.
+ for i := 0; i < 9; i++ {
+ // Column round.
+ x0, x4, x8, x12 = quarterRound(x0, x4, x8, x12)
+ x1, x5, x9, x13 = quarterRound(x1, x5, x9, x13)
+ x2, x6, x10, x14 = quarterRound(x2, x6, x10, x14)
+ x3, x7, x11, x15 = quarterRound(x3, x7, x11, x15)
+
+ // Diagonal round.
+ x0, x5, x10, x15 = quarterRound(x0, x5, x10, x15)
+ x1, x6, x11, x12 = quarterRound(x1, x6, x11, x12)
+ x2, x7, x8, x13 = quarterRound(x2, x7, x8, x13)
+ x3, x4, x9, x14 = quarterRound(x3, x4, x9, x14)
+ }
+
+ // Finally, add back the initial state to generate the key stream.
+ x0 += c0
+ x1 += c1
+ x2 += c2
+ x3 += c3
+ x4 += c4
+ x5 += c5
+ x6 += c6
+ x7 += c7
+ x8 += c8
+ x9 += c9
+ x10 += c10
+ x11 += c11
+ x12 += s.counter
+ x13 += c13
+ x14 += c14
+ x15 += c15
+
+ s.counter += 1
+ if s.counter == 0 {
+ panic("chacha20: internal error: counter overflow")
+ }
+
+ in, out := src[i:], dst[i:]
+ in, out = in[:blockSize], out[:blockSize] // bounds check elimination hint
+
+ // XOR the key stream with the source and write out the result.
+ xor(out[0:], in[0:], x0)
+ xor(out[4:], in[4:], x1)
+ xor(out[8:], in[8:], x2)
+ xor(out[12:], in[12:], x3)
+ xor(out[16:], in[16:], x4)
+ xor(out[20:], in[20:], x5)
+ xor(out[24:], in[24:], x6)
+ xor(out[28:], in[28:], x7)
+ xor(out[32:], in[32:], x8)
+ xor(out[36:], in[36:], x9)
+ xor(out[40:], in[40:], x10)
+ xor(out[44:], in[44:], x11)
+ xor(out[48:], in[48:], x12)
+ xor(out[52:], in[52:], x13)
+ xor(out[56:], in[56:], x14)
+ xor(out[60:], in[60:], x15)
+ }
+}
+
+// HChaCha20 uses the ChaCha20 core to generate a derived key from a 32 bytes
+// key and a 16 bytes nonce. It returns an error if key or nonce have any other
+// length. It is used as part of the XChaCha20 construction.
+func HChaCha20(key, nonce []byte) ([]byte, error) {
+ // This function is split into a wrapper so that the slice allocation will
+ // be inlined, and depending on how the caller uses the return value, won't
+ // escape to the heap.
+ out := make([]byte, 32)
+ return hChaCha20(out, key, nonce)
+}
+
+func hChaCha20(out, key, nonce []byte) ([]byte, error) {
+ if len(key) != KeySize {
+ return nil, errors.New("chacha20: wrong HChaCha20 key size")
+ }
+ if len(nonce) != 16 {
+ return nil, errors.New("chacha20: wrong HChaCha20 nonce size")
+ }
+
+ x0, x1, x2, x3 := j0, j1, j2, j3
+ x4 := binary.LittleEndian.Uint32(key[0:4])
+ x5 := binary.LittleEndian.Uint32(key[4:8])
+ x6 := binary.LittleEndian.Uint32(key[8:12])
+ x7 := binary.LittleEndian.Uint32(key[12:16])
+ x8 := binary.LittleEndian.Uint32(key[16:20])
+ x9 := binary.LittleEndian.Uint32(key[20:24])
+ x10 := binary.LittleEndian.Uint32(key[24:28])
+ x11 := binary.LittleEndian.Uint32(key[28:32])
+ x12 := binary.LittleEndian.Uint32(nonce[0:4])
+ x13 := binary.LittleEndian.Uint32(nonce[4:8])
+ x14 := binary.LittleEndian.Uint32(nonce[8:12])
+ x15 := binary.LittleEndian.Uint32(nonce[12:16])
+
+ for i := 0; i < 10; i++ {
+ // Diagonal round.
+ x0, x4, x8, x12 = quarterRound(x0, x4, x8, x12)
+ x1, x5, x9, x13 = quarterRound(x1, x5, x9, x13)
+ x2, x6, x10, x14 = quarterRound(x2, x6, x10, x14)
+ x3, x7, x11, x15 = quarterRound(x3, x7, x11, x15)
+
+ // Column round.
+ x0, x5, x10, x15 = quarterRound(x0, x5, x10, x15)
+ x1, x6, x11, x12 = quarterRound(x1, x6, x11, x12)
+ x2, x7, x8, x13 = quarterRound(x2, x7, x8, x13)
+ x3, x4, x9, x14 = quarterRound(x3, x4, x9, x14)
+ }
+
+ _ = out[31] // bounds check elimination hint
+ binary.LittleEndian.PutUint32(out[0:4], x0)
+ binary.LittleEndian.PutUint32(out[4:8], x1)
+ binary.LittleEndian.PutUint32(out[8:12], x2)
+ binary.LittleEndian.PutUint32(out[12:16], x3)
+ binary.LittleEndian.PutUint32(out[16:20], x12)
+ binary.LittleEndian.PutUint32(out[20:24], x13)
+ binary.LittleEndian.PutUint32(out[24:28], x14)
+ binary.LittleEndian.PutUint32(out[28:32], x15)
+ return out, nil
+}
--- /dev/null
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !arm64,!s390x,!ppc64le arm64,!go1.11 gccgo appengine
+
+package chacha20
+
+const bufSize = blockSize
+
+func (s *Cipher) xorKeyStreamBlocks(dst, src []byte) {
+ s.xorKeyStreamBlocksGeneric(dst, src)
+}
--- /dev/null
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !gccgo,!appengine
+
+package chacha20
+
+const bufSize = 256
+
+//go:noescape
+func chaCha20_ctr32_vsx(out, inp *byte, len int, key *[8]uint32, counter *uint32)
+
+func (c *Cipher) xorKeyStreamBlocks(dst, src []byte) {
+ chaCha20_ctr32_vsx(&dst[0], &src[0], len(src), &c.key, &c.counter)
+}
--- /dev/null
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Based on CRYPTOGAMS code with the following comment:
+// # ====================================================================
+// # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
+// # project. The module is, however, dual licensed under OpenSSL and
+// # CRYPTOGAMS licenses depending on where you obtain it. For further
+// # details see http://www.openssl.org/~appro/cryptogams/.
+// # ====================================================================
+
+// Code for the perl script that generates the ppc64 assembler
+// can be found in the cryptogams repository at the link below. It is based on
+// the original from openssl.
+
+// https://github.com/dot-asm/cryptogams/commit/a60f5b50ed908e91
+
+// The differences in this and the original implementation are
+// due to the calling conventions and initialization of constants.
+
+// +build !gccgo,!appengine
+
+#include "textflag.h"
+
+#define OUT R3
+#define INP R4
+#define LEN R5
+#define KEY R6
+#define CNT R7
+#define TMP R15
+
+#define CONSTBASE R16
+#define BLOCKS R17
+
+DATA consts<>+0x00(SB)/8, $0x3320646e61707865
+DATA consts<>+0x08(SB)/8, $0x6b20657479622d32
+DATA consts<>+0x10(SB)/8, $0x0000000000000001
+DATA consts<>+0x18(SB)/8, $0x0000000000000000
+DATA consts<>+0x20(SB)/8, $0x0000000000000004
+DATA consts<>+0x28(SB)/8, $0x0000000000000000
+DATA consts<>+0x30(SB)/8, $0x0a0b08090e0f0c0d
+DATA consts<>+0x38(SB)/8, $0x0203000106070405
+DATA consts<>+0x40(SB)/8, $0x090a0b080d0e0f0c
+DATA consts<>+0x48(SB)/8, $0x0102030005060704
+DATA consts<>+0x50(SB)/8, $0x6170786561707865
+DATA consts<>+0x58(SB)/8, $0x6170786561707865
+DATA consts<>+0x60(SB)/8, $0x3320646e3320646e
+DATA consts<>+0x68(SB)/8, $0x3320646e3320646e
+DATA consts<>+0x70(SB)/8, $0x79622d3279622d32
+DATA consts<>+0x78(SB)/8, $0x79622d3279622d32
+DATA consts<>+0x80(SB)/8, $0x6b2065746b206574
+DATA consts<>+0x88(SB)/8, $0x6b2065746b206574
+DATA consts<>+0x90(SB)/8, $0x0000000100000000
+DATA consts<>+0x98(SB)/8, $0x0000000300000002
+GLOBL consts<>(SB), RODATA, $0xa0
+
+//func chaCha20_ctr32_vsx(out, inp *byte, len int, key *[8]uint32, counter *uint32)
+TEXT ·chaCha20_ctr32_vsx(SB),NOSPLIT,$64-40
+ MOVD out+0(FP), OUT
+ MOVD inp+8(FP), INP
+ MOVD len+16(FP), LEN
+ MOVD key+24(FP), KEY
+ MOVD counter+32(FP), CNT
+
+ // Addressing for constants
+ MOVD $consts<>+0x00(SB), CONSTBASE
+ MOVD $16, R8
+ MOVD $32, R9
+ MOVD $48, R10
+ MOVD $64, R11
+ SRD $6, LEN, BLOCKS
+ // V16
+ LXVW4X (CONSTBASE)(R0), VS48
+ ADD $80,CONSTBASE
+
+ // Load key into V17,V18
+ LXVW4X (KEY)(R0), VS49
+ LXVW4X (KEY)(R8), VS50
+
+ // Load CNT, NONCE into V19
+ LXVW4X (CNT)(R0), VS51
+
+ // Clear V27
+ VXOR V27, V27, V27
+
+ // V28
+ LXVW4X (CONSTBASE)(R11), VS60
+
+ // splat slot from V19 -> V26
+ VSPLTW $0, V19, V26
+
+ VSLDOI $4, V19, V27, V19
+ VSLDOI $12, V27, V19, V19
+
+ VADDUWM V26, V28, V26
+
+ MOVD $10, R14
+ MOVD R14, CTR
+
+loop_outer_vsx:
+ // V0, V1, V2, V3
+ LXVW4X (R0)(CONSTBASE), VS32
+ LXVW4X (R8)(CONSTBASE), VS33
+ LXVW4X (R9)(CONSTBASE), VS34
+ LXVW4X (R10)(CONSTBASE), VS35
+
+ // splat values from V17, V18 into V4-V11
+ VSPLTW $0, V17, V4
+ VSPLTW $1, V17, V5
+ VSPLTW $2, V17, V6
+ VSPLTW $3, V17, V7
+ VSPLTW $0, V18, V8
+ VSPLTW $1, V18, V9
+ VSPLTW $2, V18, V10
+ VSPLTW $3, V18, V11
+
+ // VOR
+ VOR V26, V26, V12
+
+ // splat values from V19 -> V13, V14, V15
+ VSPLTW $1, V19, V13
+ VSPLTW $2, V19, V14
+ VSPLTW $3, V19, V15
+
+ // splat const values
+ VSPLTISW $-16, V27
+ VSPLTISW $12, V28
+ VSPLTISW $8, V29
+ VSPLTISW $7, V30
+
+loop_vsx:
+ VADDUWM V0, V4, V0
+ VADDUWM V1, V5, V1
+ VADDUWM V2, V6, V2
+ VADDUWM V3, V7, V3
+
+ VXOR V12, V0, V12
+ VXOR V13, V1, V13
+ VXOR V14, V2, V14
+ VXOR V15, V3, V15
+
+ VRLW V12, V27, V12
+ VRLW V13, V27, V13
+ VRLW V14, V27, V14
+ VRLW V15, V27, V15
+
+ VADDUWM V8, V12, V8
+ VADDUWM V9, V13, V9
+ VADDUWM V10, V14, V10
+ VADDUWM V11, V15, V11
+
+ VXOR V4, V8, V4
+ VXOR V5, V9, V5
+ VXOR V6, V10, V6
+ VXOR V7, V11, V7
+
+ VRLW V4, V28, V4
+ VRLW V5, V28, V5
+ VRLW V6, V28, V6
+ VRLW V7, V28, V7
+
+ VADDUWM V0, V4, V0
+ VADDUWM V1, V5, V1
+ VADDUWM V2, V6, V2
+ VADDUWM V3, V7, V3
+
+ VXOR V12, V0, V12
+ VXOR V13, V1, V13
+ VXOR V14, V2, V14
+ VXOR V15, V3, V15
+
+ VRLW V12, V29, V12
+ VRLW V13, V29, V13
+ VRLW V14, V29, V14
+ VRLW V15, V29, V15
+
+ VADDUWM V8, V12, V8
+ VADDUWM V9, V13, V9
+ VADDUWM V10, V14, V10
+ VADDUWM V11, V15, V11
+
+ VXOR V4, V8, V4
+ VXOR V5, V9, V5
+ VXOR V6, V10, V6
+ VXOR V7, V11, V7
+
+ VRLW V4, V30, V4
+ VRLW V5, V30, V5
+ VRLW V6, V30, V6
+ VRLW V7, V30, V7
+
+ VADDUWM V0, V5, V0
+ VADDUWM V1, V6, V1
+ VADDUWM V2, V7, V2
+ VADDUWM V3, V4, V3
+
+ VXOR V15, V0, V15
+ VXOR V12, V1, V12
+ VXOR V13, V2, V13
+ VXOR V14, V3, V14
+
+ VRLW V15, V27, V15
+ VRLW V12, V27, V12
+ VRLW V13, V27, V13
+ VRLW V14, V27, V14
+
+ VADDUWM V10, V15, V10
+ VADDUWM V11, V12, V11
+ VADDUWM V8, V13, V8
+ VADDUWM V9, V14, V9
+
+ VXOR V5, V10, V5
+ VXOR V6, V11, V6
+ VXOR V7, V8, V7
+ VXOR V4, V9, V4
+
+ VRLW V5, V28, V5
+ VRLW V6, V28, V6
+ VRLW V7, V28, V7
+ VRLW V4, V28, V4
+
+ VADDUWM V0, V5, V0
+ VADDUWM V1, V6, V1
+ VADDUWM V2, V7, V2
+ VADDUWM V3, V4, V3
+
+ VXOR V15, V0, V15
+ VXOR V12, V1, V12
+ VXOR V13, V2, V13
+ VXOR V14, V3, V14
+
+ VRLW V15, V29, V15
+ VRLW V12, V29, V12
+ VRLW V13, V29, V13
+ VRLW V14, V29, V14
+
+ VADDUWM V10, V15, V10
+ VADDUWM V11, V12, V11
+ VADDUWM V8, V13, V8
+ VADDUWM V9, V14, V9
+
+ VXOR V5, V10, V5
+ VXOR V6, V11, V6
+ VXOR V7, V8, V7
+ VXOR V4, V9, V4
+
+ VRLW V5, V30, V5
+ VRLW V6, V30, V6
+ VRLW V7, V30, V7
+ VRLW V4, V30, V4
+ BC 16, LT, loop_vsx
+
+ VADDUWM V12, V26, V12
+
+ WORD $0x13600F8C // VMRGEW V0, V1, V27
+ WORD $0x13821F8C // VMRGEW V2, V3, V28
+
+ WORD $0x10000E8C // VMRGOW V0, V1, V0
+ WORD $0x10421E8C // VMRGOW V2, V3, V2
+
+ WORD $0x13A42F8C // VMRGEW V4, V5, V29
+ WORD $0x13C63F8C // VMRGEW V6, V7, V30
+
+ XXPERMDI VS32, VS34, $0, VS33
+ XXPERMDI VS32, VS34, $3, VS35
+ XXPERMDI VS59, VS60, $0, VS32
+ XXPERMDI VS59, VS60, $3, VS34
+
+ WORD $0x10842E8C // VMRGOW V4, V5, V4
+ WORD $0x10C63E8C // VMRGOW V6, V7, V6
+
+ WORD $0x13684F8C // VMRGEW V8, V9, V27
+ WORD $0x138A5F8C // VMRGEW V10, V11, V28
+
+ XXPERMDI VS36, VS38, $0, VS37
+ XXPERMDI VS36, VS38, $3, VS39
+ XXPERMDI VS61, VS62, $0, VS36
+ XXPERMDI VS61, VS62, $3, VS38
+
+ WORD $0x11084E8C // VMRGOW V8, V9, V8
+ WORD $0x114A5E8C // VMRGOW V10, V11, V10
+
+ WORD $0x13AC6F8C // VMRGEW V12, V13, V29
+ WORD $0x13CE7F8C // VMRGEW V14, V15, V30
+
+ XXPERMDI VS40, VS42, $0, VS41
+ XXPERMDI VS40, VS42, $3, VS43
+ XXPERMDI VS59, VS60, $0, VS40
+ XXPERMDI VS59, VS60, $3, VS42
+
+ WORD $0x118C6E8C // VMRGOW V12, V13, V12
+ WORD $0x11CE7E8C // VMRGOW V14, V15, V14
+
+ VSPLTISW $4, V27
+ VADDUWM V26, V27, V26
+
+ XXPERMDI VS44, VS46, $0, VS45
+ XXPERMDI VS44, VS46, $3, VS47
+ XXPERMDI VS61, VS62, $0, VS44
+ XXPERMDI VS61, VS62, $3, VS46
+
+ VADDUWM V0, V16, V0
+ VADDUWM V4, V17, V4
+ VADDUWM V8, V18, V8
+ VADDUWM V12, V19, V12
+
+ CMPU LEN, $64
+ BLT tail_vsx
+
+ // Bottom of loop
+ LXVW4X (INP)(R0), VS59
+ LXVW4X (INP)(R8), VS60
+ LXVW4X (INP)(R9), VS61
+ LXVW4X (INP)(R10), VS62
+
+ VXOR V27, V0, V27
+ VXOR V28, V4, V28
+ VXOR V29, V8, V29
+ VXOR V30, V12, V30
+
+ STXVW4X VS59, (OUT)(R0)
+ STXVW4X VS60, (OUT)(R8)
+ ADD $64, INP
+ STXVW4X VS61, (OUT)(R9)
+ ADD $-64, LEN
+ STXVW4X VS62, (OUT)(R10)
+ ADD $64, OUT
+ BEQ done_vsx
+
+ VADDUWM V1, V16, V0
+ VADDUWM V5, V17, V4
+ VADDUWM V9, V18, V8
+ VADDUWM V13, V19, V12
+
+ CMPU LEN, $64
+ BLT tail_vsx
+
+ LXVW4X (INP)(R0), VS59
+ LXVW4X (INP)(R8), VS60
+ LXVW4X (INP)(R9), VS61
+ LXVW4X (INP)(R10), VS62
+ VXOR V27, V0, V27
+
+ VXOR V28, V4, V28
+ VXOR V29, V8, V29
+ VXOR V30, V12, V30
+
+ STXVW4X VS59, (OUT)(R0)
+ STXVW4X VS60, (OUT)(R8)
+ ADD $64, INP
+ STXVW4X VS61, (OUT)(R9)
+ ADD $-64, LEN
+ STXVW4X VS62, (OUT)(V10)
+ ADD $64, OUT
+ BEQ done_vsx
+
+ VADDUWM V2, V16, V0
+ VADDUWM V6, V17, V4
+ VADDUWM V10, V18, V8
+ VADDUWM V14, V19, V12
+
+ CMPU LEN, $64
+ BLT tail_vsx
+
+ LXVW4X (INP)(R0), VS59
+ LXVW4X (INP)(R8), VS60
+ LXVW4X (INP)(R9), VS61
+ LXVW4X (INP)(R10), VS62
+
+ VXOR V27, V0, V27
+ VXOR V28, V4, V28
+ VXOR V29, V8, V29
+ VXOR V30, V12, V30
+
+ STXVW4X VS59, (OUT)(R0)
+ STXVW4X VS60, (OUT)(R8)
+ ADD $64, INP
+ STXVW4X VS61, (OUT)(R9)
+ ADD $-64, LEN
+ STXVW4X VS62, (OUT)(R10)
+ ADD $64, OUT
+ BEQ done_vsx
+
+ VADDUWM V3, V16, V0
+ VADDUWM V7, V17, V4
+ VADDUWM V11, V18, V8
+ VADDUWM V15, V19, V12
+
+ CMPU LEN, $64
+ BLT tail_vsx
+
+ LXVW4X (INP)(R0), VS59
+ LXVW4X (INP)(R8), VS60
+ LXVW4X (INP)(R9), VS61
+ LXVW4X (INP)(R10), VS62
+
+ VXOR V27, V0, V27
+ VXOR V28, V4, V28
+ VXOR V29, V8, V29
+ VXOR V30, V12, V30
+
+ STXVW4X VS59, (OUT)(R0)
+ STXVW4X VS60, (OUT)(R8)
+ ADD $64, INP
+ STXVW4X VS61, (OUT)(R9)
+ ADD $-64, LEN
+ STXVW4X VS62, (OUT)(R10)
+ ADD $64, OUT
+
+ MOVD $10, R14
+ MOVD R14, CTR
+ BNE loop_outer_vsx
+
+done_vsx:
+ // Increment counter by number of 64 byte blocks
+ MOVD (CNT), R14
+ ADD BLOCKS, R14
+ MOVD R14, (CNT)
+ RET
+
+tail_vsx:
+ ADD $32, R1, R11
+ MOVD LEN, CTR
+
+ // Save values on stack to copy from
+ STXVW4X VS32, (R11)(R0)
+ STXVW4X VS36, (R11)(R8)
+ STXVW4X VS40, (R11)(R9)
+ STXVW4X VS44, (R11)(R10)
+ ADD $-1, R11, R12
+ ADD $-1, INP
+ ADD $-1, OUT
+
+looptail_vsx:
+ // Copying the result to OUT
+ // in bytes.
+ MOVBZU 1(R12), KEY
+ MOVBZU 1(INP), TMP
+ XOR KEY, TMP, KEY
+ MOVBU KEY, 1(OUT)
+ BC 16, LT, looptail_vsx
+
+ // Clear the stack values
+ STXVW4X VS48, (R11)(R0)
+ STXVW4X VS48, (R11)(R8)
+ STXVW4X VS48, (R11)(R9)
+ STXVW4X VS48, (R11)(R10)
+ BR done_vsx
--- /dev/null
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !gccgo,!appengine
+
+package chacha20
+
+import "golang.org/x/sys/cpu"
+
+var haveAsm = cpu.S390X.HasVX
+
+const bufSize = 256
+
+// xorKeyStreamVX is an assembly implementation of XORKeyStream. It must only
+// be called when the vector facility is available. Implementation in asm_s390x.s.
+//go:noescape
+func xorKeyStreamVX(dst, src []byte, key *[8]uint32, nonce *[3]uint32, counter *uint32)
+
+func (c *Cipher) xorKeyStreamBlocks(dst, src []byte) {
+ if cpu.S390X.HasVX {
+ xorKeyStreamVX(dst, src, &c.key, &c.nonce, &c.counter)
+ } else {
+ c.xorKeyStreamBlocksGeneric(dst, src)
+ }
+}
--- /dev/null
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !gccgo,!appengine
+
+#include "go_asm.h"
+#include "textflag.h"
+
+// This is an implementation of the ChaCha20 encryption algorithm as
+// specified in RFC 7539. It uses vector instructions to compute
+// 4 keystream blocks in parallel (256 bytes) which are then XORed
+// with the bytes in the input slice.
+
+GLOBL ·constants<>(SB), RODATA|NOPTR, $32
+// BSWAP: swap bytes in each 4-byte element
+DATA ·constants<>+0x00(SB)/4, $0x03020100
+DATA ·constants<>+0x04(SB)/4, $0x07060504
+DATA ·constants<>+0x08(SB)/4, $0x0b0a0908
+DATA ·constants<>+0x0c(SB)/4, $0x0f0e0d0c
+// J0: [j0, j1, j2, j3]
+DATA ·constants<>+0x10(SB)/4, $0x61707865
+DATA ·constants<>+0x14(SB)/4, $0x3320646e
+DATA ·constants<>+0x18(SB)/4, $0x79622d32
+DATA ·constants<>+0x1c(SB)/4, $0x6b206574
+
+#define BSWAP V5
+#define J0 V6
+#define KEY0 V7
+#define KEY1 V8
+#define NONCE V9
+#define CTR V10
+#define M0 V11
+#define M1 V12
+#define M2 V13
+#define M3 V14
+#define INC V15
+#define X0 V16
+#define X1 V17
+#define X2 V18
+#define X3 V19
+#define X4 V20
+#define X5 V21
+#define X6 V22
+#define X7 V23
+#define X8 V24
+#define X9 V25
+#define X10 V26
+#define X11 V27
+#define X12 V28
+#define X13 V29
+#define X14 V30
+#define X15 V31
+
+#define NUM_ROUNDS 20
+
+#define ROUND4(a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3) \
+ VAF a1, a0, a0 \
+ VAF b1, b0, b0 \
+ VAF c1, c0, c0 \
+ VAF d1, d0, d0 \
+ VX a0, a2, a2 \
+ VX b0, b2, b2 \
+ VX c0, c2, c2 \
+ VX d0, d2, d2 \
+ VERLLF $16, a2, a2 \
+ VERLLF $16, b2, b2 \
+ VERLLF $16, c2, c2 \
+ VERLLF $16, d2, d2 \
+ VAF a2, a3, a3 \
+ VAF b2, b3, b3 \
+ VAF c2, c3, c3 \
+ VAF d2, d3, d3 \
+ VX a3, a1, a1 \
+ VX b3, b1, b1 \
+ VX c3, c1, c1 \
+ VX d3, d1, d1 \
+ VERLLF $12, a1, a1 \
+ VERLLF $12, b1, b1 \
+ VERLLF $12, c1, c1 \
+ VERLLF $12, d1, d1 \
+ VAF a1, a0, a0 \
+ VAF b1, b0, b0 \
+ VAF c1, c0, c0 \
+ VAF d1, d0, d0 \
+ VX a0, a2, a2 \
+ VX b0, b2, b2 \
+ VX c0, c2, c2 \
+ VX d0, d2, d2 \
+ VERLLF $8, a2, a2 \
+ VERLLF $8, b2, b2 \
+ VERLLF $8, c2, c2 \
+ VERLLF $8, d2, d2 \
+ VAF a2, a3, a3 \
+ VAF b2, b3, b3 \
+ VAF c2, c3, c3 \
+ VAF d2, d3, d3 \
+ VX a3, a1, a1 \
+ VX b3, b1, b1 \
+ VX c3, c1, c1 \
+ VX d3, d1, d1 \
+ VERLLF $7, a1, a1 \
+ VERLLF $7, b1, b1 \
+ VERLLF $7, c1, c1 \
+ VERLLF $7, d1, d1
+
+#define PERMUTE(mask, v0, v1, v2, v3) \
+ VPERM v0, v0, mask, v0 \
+ VPERM v1, v1, mask, v1 \
+ VPERM v2, v2, mask, v2 \
+ VPERM v3, v3, mask, v3
+
+#define ADDV(x, v0, v1, v2, v3) \
+ VAF x, v0, v0 \
+ VAF x, v1, v1 \
+ VAF x, v2, v2 \
+ VAF x, v3, v3
+
+#define XORV(off, dst, src, v0, v1, v2, v3) \
+ VLM off(src), M0, M3 \
+ PERMUTE(BSWAP, v0, v1, v2, v3) \
+ VX v0, M0, M0 \
+ VX v1, M1, M1 \
+ VX v2, M2, M2 \
+ VX v3, M3, M3 \
+ VSTM M0, M3, off(dst)
+
+#define SHUFFLE(a, b, c, d, t, u, v, w) \
+ VMRHF a, c, t \ // t = {a[0], c[0], a[1], c[1]}
+ VMRHF b, d, u \ // u = {b[0], d[0], b[1], d[1]}
+ VMRLF a, c, v \ // v = {a[2], c[2], a[3], c[3]}
+ VMRLF b, d, w \ // w = {b[2], d[2], b[3], d[3]}
+ VMRHF t, u, a \ // a = {a[0], b[0], c[0], d[0]}
+ VMRLF t, u, b \ // b = {a[1], b[1], c[1], d[1]}
+ VMRHF v, w, c \ // c = {a[2], b[2], c[2], d[2]}
+ VMRLF v, w, d // d = {a[3], b[3], c[3], d[3]}
+
+// func xorKeyStreamVX(dst, src []byte, key *[8]uint32, nonce *[3]uint32, counter *uint32)
+TEXT ·xorKeyStreamVX(SB), NOSPLIT, $0
+ MOVD $·constants<>(SB), R1
+ MOVD dst+0(FP), R2 // R2=&dst[0]
+ LMG src+24(FP), R3, R4 // R3=&src[0] R4=len(src)
+ MOVD key+48(FP), R5 // R5=key
+ MOVD nonce+56(FP), R6 // R6=nonce
+ MOVD counter+64(FP), R7 // R7=counter
+
+ // load BSWAP and J0
+ VLM (R1), BSWAP, J0
+
+ // setup
+ MOVD $95, R0
+ VLM (R5), KEY0, KEY1
+ VLL R0, (R6), NONCE
+ VZERO M0
+ VLEIB $7, $32, M0
+ VSRLB M0, NONCE, NONCE
+
+ // initialize counter values
+ VLREPF (R7), CTR
+ VZERO INC
+ VLEIF $1, $1, INC
+ VLEIF $2, $2, INC
+ VLEIF $3, $3, INC
+ VAF INC, CTR, CTR
+ VREPIF $4, INC
+
+chacha:
+ VREPF $0, J0, X0
+ VREPF $1, J0, X1
+ VREPF $2, J0, X2
+ VREPF $3, J0, X3
+ VREPF $0, KEY0, X4
+ VREPF $1, KEY0, X5
+ VREPF $2, KEY0, X6
+ VREPF $3, KEY0, X7
+ VREPF $0, KEY1, X8
+ VREPF $1, KEY1, X9
+ VREPF $2, KEY1, X10
+ VREPF $3, KEY1, X11
+ VLR CTR, X12
+ VREPF $1, NONCE, X13
+ VREPF $2, NONCE, X14
+ VREPF $3, NONCE, X15
+
+ MOVD $(NUM_ROUNDS/2), R1
+
+loop:
+ ROUND4(X0, X4, X12, X8, X1, X5, X13, X9, X2, X6, X14, X10, X3, X7, X15, X11)
+ ROUND4(X0, X5, X15, X10, X1, X6, X12, X11, X2, X7, X13, X8, X3, X4, X14, X9)
+
+ ADD $-1, R1
+ BNE loop
+
+ // decrement length
+ ADD $-256, R4
+
+ // rearrange vectors
+ SHUFFLE(X0, X1, X2, X3, M0, M1, M2, M3)
+ ADDV(J0, X0, X1, X2, X3)
+ SHUFFLE(X4, X5, X6, X7, M0, M1, M2, M3)
+ ADDV(KEY0, X4, X5, X6, X7)
+ SHUFFLE(X8, X9, X10, X11, M0, M1, M2, M3)
+ ADDV(KEY1, X8, X9, X10, X11)
+ VAF CTR, X12, X12
+ SHUFFLE(X12, X13, X14, X15, M0, M1, M2, M3)
+ ADDV(NONCE, X12, X13, X14, X15)
+
+ // increment counters
+ VAF INC, CTR, CTR
+
+ // xor keystream with plaintext
+ XORV(0*64, R2, R3, X0, X4, X8, X12)
+ XORV(1*64, R2, R3, X1, X5, X9, X13)
+ XORV(2*64, R2, R3, X2, X6, X10, X14)
+ XORV(3*64, R2, R3, X3, X7, X11, X15)
+
+ // increment pointers
+ MOVD $256(R2), R2
+ MOVD $256(R3), R3
+
+ CMPBNE R4, $0, chacha
+
+ VSTEF $0, CTR, (R7)
+ RET
--- /dev/null
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found src the LICENSE file.
+
+package chacha20
+
+import "runtime"
+
+// Platforms that have fast unaligned 32-bit little endian accesses.
+const unaligned = runtime.GOARCH == "386" ||
+ runtime.GOARCH == "amd64" ||
+ runtime.GOARCH == "arm64" ||
+ runtime.GOARCH == "ppc64le" ||
+ runtime.GOARCH == "s390x"
+
+// xor reads a little endian uint32 from src, XORs it with u and
+// places the result in little endian byte order in dst.
+func xor(dst, src []byte, u uint32) {
+ _, _ = src[3], dst[3] // eliminate bounds checks
+ if unaligned {
+ // The compiler should optimize this code into
+ // 32-bit unaligned little endian loads and stores.
+ // TODO: delete once the compiler does a reliably
+ // good job with the generic code below.
+ // See issue #25111 for more details.
+ v := uint32(src[0])
+ v |= uint32(src[1]) << 8
+ v |= uint32(src[2]) << 16
+ v |= uint32(src[3]) << 24
+ v ^= u
+ dst[0] = byte(v)
+ dst[1] = byte(v >> 8)
+ dst[2] = byte(v >> 16)
+ dst[3] = byte(v >> 24)
+ } else {
+ dst[0] = src[0] ^ byte(u)
+ dst[1] = src[1] ^ byte(u>>8)
+ dst[2] = src[2] ^ byte(u>>16)
+ dst[3] = src[3] ^ byte(u>>24)
+ }
+}
+++ /dev/null
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// This code was translated into a form compatible with 6a from the public
-// domain sources in SUPERCOP: https://bench.cr.yp.to/supercop.html
-
-#define REDMASK51 0x0007FFFFFFFFFFFF
+++ /dev/null
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// This code was translated into a form compatible with 6a from the public
-// domain sources in SUPERCOP: https://bench.cr.yp.to/supercop.html
-
-// +build amd64,!gccgo,!appengine
-
-// These constants cannot be encoded in non-MOVQ immediates.
-// We access them directly from memory instead.
-
-DATA ·_121666_213(SB)/8, $996687872
-GLOBL ·_121666_213(SB), 8, $8
-
-DATA ·_2P0(SB)/8, $0xFFFFFFFFFFFDA
-GLOBL ·_2P0(SB), 8, $8
-
-DATA ·_2P1234(SB)/8, $0xFFFFFFFFFFFFE
-GLOBL ·_2P1234(SB), 8, $8
+++ /dev/null
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build amd64,!gccgo,!appengine
-
-// func cswap(inout *[4][5]uint64, v uint64)
-TEXT ·cswap(SB),7,$0
- MOVQ inout+0(FP),DI
- MOVQ v+8(FP),SI
-
- SUBQ $1, SI
- NOTQ SI
- MOVQ SI, X15
- PSHUFD $0x44, X15, X15
-
- MOVOU 0(DI), X0
- MOVOU 16(DI), X2
- MOVOU 32(DI), X4
- MOVOU 48(DI), X6
- MOVOU 64(DI), X8
- MOVOU 80(DI), X1
- MOVOU 96(DI), X3
- MOVOU 112(DI), X5
- MOVOU 128(DI), X7
- MOVOU 144(DI), X9
-
- MOVO X1, X10
- MOVO X3, X11
- MOVO X5, X12
- MOVO X7, X13
- MOVO X9, X14
-
- PXOR X0, X10
- PXOR X2, X11
- PXOR X4, X12
- PXOR X6, X13
- PXOR X8, X14
- PAND X15, X10
- PAND X15, X11
- PAND X15, X12
- PAND X15, X13
- PAND X15, X14
- PXOR X10, X0
- PXOR X10, X1
- PXOR X11, X2
- PXOR X11, X3
- PXOR X12, X4
- PXOR X12, X5
- PXOR X13, X6
- PXOR X13, X7
- PXOR X14, X8
- PXOR X14, X9
-
- MOVOU X0, 0(DI)
- MOVOU X2, 16(DI)
- MOVOU X4, 32(DI)
- MOVOU X6, 48(DI)
- MOVOU X8, 64(DI)
- MOVOU X1, 80(DI)
- MOVOU X3, 96(DI)
- MOVOU X5, 112(DI)
- MOVOU X7, 128(DI)
- MOVOU X9, 144(DI)
- RET
-// Copyright 2013 The Go Authors. All rights reserved.
+// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-// We have an implementation in amd64 assembly so this code is only run on
-// non-amd64 platforms. The amd64 assembly does not support gccgo.
-// +build !amd64 gccgo appengine
-
-package curve25519
+// Package curve25519 provides an implementation of the X25519 function, which
+// performs scalar multiplication on the elliptic curve known as Curve25519.
+// See RFC 7748.
+package curve25519 // import "golang.org/x/crypto/curve25519"
import (
- "encoding/binary"
+ "crypto/subtle"
+ "fmt"
)
-// This code is a port of the public domain, "ref10" implementation of
-// curve25519 from SUPERCOP 20130419 by D. J. Bernstein.
-
-// fieldElement represents an element of the field GF(2^255 - 19). An element
-// t, entries t[0]...t[9], represents the integer t[0]+2^26 t[1]+2^51 t[2]+2^77
-// t[3]+2^102 t[4]+...+2^230 t[9]. Bounds on each t[i] vary depending on
-// context.
-type fieldElement [10]int32
-
-func feZero(fe *fieldElement) {
- for i := range fe {
- fe[i] = 0
- }
-}
-
-func feOne(fe *fieldElement) {
- feZero(fe)
- fe[0] = 1
-}
-
-func feAdd(dst, a, b *fieldElement) {
- for i := range dst {
- dst[i] = a[i] + b[i]
- }
-}
-
-func feSub(dst, a, b *fieldElement) {
- for i := range dst {
- dst[i] = a[i] - b[i]
- }
-}
-
-func feCopy(dst, src *fieldElement) {
- for i := range dst {
- dst[i] = src[i]
- }
-}
-
-// feCSwap replaces (f,g) with (g,f) if b == 1; replaces (f,g) with (f,g) if b == 0.
-//
-// Preconditions: b in {0,1}.
-func feCSwap(f, g *fieldElement, b int32) {
- b = -b
- for i := range f {
- t := b & (f[i] ^ g[i])
- f[i] ^= t
- g[i] ^= t
- }
-}
-
-// load3 reads a 24-bit, little-endian value from in.
-func load3(in []byte) int64 {
- var r int64
- r = int64(in[0])
- r |= int64(in[1]) << 8
- r |= int64(in[2]) << 16
- return r
-}
-
-// load4 reads a 32-bit, little-endian value from in.
-func load4(in []byte) int64 {
- return int64(binary.LittleEndian.Uint32(in))
-}
-
-func feFromBytes(dst *fieldElement, src *[32]byte) {
- h0 := load4(src[:])
- h1 := load3(src[4:]) << 6
- h2 := load3(src[7:]) << 5
- h3 := load3(src[10:]) << 3
- h4 := load3(src[13:]) << 2
- h5 := load4(src[16:])
- h6 := load3(src[20:]) << 7
- h7 := load3(src[23:]) << 5
- h8 := load3(src[26:]) << 4
- h9 := (load3(src[29:]) & 0x7fffff) << 2
-
- var carry [10]int64
- carry[9] = (h9 + 1<<24) >> 25
- h0 += carry[9] * 19
- h9 -= carry[9] << 25
- carry[1] = (h1 + 1<<24) >> 25
- h2 += carry[1]
- h1 -= carry[1] << 25
- carry[3] = (h3 + 1<<24) >> 25
- h4 += carry[3]
- h3 -= carry[3] << 25
- carry[5] = (h5 + 1<<24) >> 25
- h6 += carry[5]
- h5 -= carry[5] << 25
- carry[7] = (h7 + 1<<24) >> 25
- h8 += carry[7]
- h7 -= carry[7] << 25
-
- carry[0] = (h0 + 1<<25) >> 26
- h1 += carry[0]
- h0 -= carry[0] << 26
- carry[2] = (h2 + 1<<25) >> 26
- h3 += carry[2]
- h2 -= carry[2] << 26
- carry[4] = (h4 + 1<<25) >> 26
- h5 += carry[4]
- h4 -= carry[4] << 26
- carry[6] = (h6 + 1<<25) >> 26
- h7 += carry[6]
- h6 -= carry[6] << 26
- carry[8] = (h8 + 1<<25) >> 26
- h9 += carry[8]
- h8 -= carry[8] << 26
-
- dst[0] = int32(h0)
- dst[1] = int32(h1)
- dst[2] = int32(h2)
- dst[3] = int32(h3)
- dst[4] = int32(h4)
- dst[5] = int32(h5)
- dst[6] = int32(h6)
- dst[7] = int32(h7)
- dst[8] = int32(h8)
- dst[9] = int32(h9)
-}
-
-// feToBytes marshals h to s.
-// Preconditions:
-// |h| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc.
-//
-// Write p=2^255-19; q=floor(h/p).
-// Basic claim: q = floor(2^(-255)(h + 19 2^(-25)h9 + 2^(-1))).
-//
-// Proof:
-// Have |h|<=p so |q|<=1 so |19^2 2^(-255) q|<1/4.
-// Also have |h-2^230 h9|<2^230 so |19 2^(-255)(h-2^230 h9)|<1/4.
-//
-// Write y=2^(-1)-19^2 2^(-255)q-19 2^(-255)(h-2^230 h9).
-// Then 0<y<1.
+// ScalarMult sets dst to the product scalar * point.
//
-// Write r=h-pq.
-// Have 0<=r<=p-1=2^255-20.
-// Thus 0<=r+19(2^-255)r<r+19(2^-255)2^255<=2^255-1.
-//
-// Write x=r+19(2^-255)r+y.
-// Then 0<x<2^255 so floor(2^(-255)x) = 0 so floor(q+2^(-255)x) = q.
-//
-// Have q+2^(-255)x = 2^(-255)(h + 19 2^(-25) h9 + 2^(-1))
-// so floor(2^(-255)(h + 19 2^(-25) h9 + 2^(-1))) = q.
-func feToBytes(s *[32]byte, h *fieldElement) {
- var carry [10]int32
-
- q := (19*h[9] + (1 << 24)) >> 25
- q = (h[0] + q) >> 26
- q = (h[1] + q) >> 25
- q = (h[2] + q) >> 26
- q = (h[3] + q) >> 25
- q = (h[4] + q) >> 26
- q = (h[5] + q) >> 25
- q = (h[6] + q) >> 26
- q = (h[7] + q) >> 25
- q = (h[8] + q) >> 26
- q = (h[9] + q) >> 25
-
- // Goal: Output h-(2^255-19)q, which is between 0 and 2^255-20.
- h[0] += 19 * q
- // Goal: Output h-2^255 q, which is between 0 and 2^255-20.
-
- carry[0] = h[0] >> 26
- h[1] += carry[0]
- h[0] -= carry[0] << 26
- carry[1] = h[1] >> 25
- h[2] += carry[1]
- h[1] -= carry[1] << 25
- carry[2] = h[2] >> 26
- h[3] += carry[2]
- h[2] -= carry[2] << 26
- carry[3] = h[3] >> 25
- h[4] += carry[3]
- h[3] -= carry[3] << 25
- carry[4] = h[4] >> 26
- h[5] += carry[4]
- h[4] -= carry[4] << 26
- carry[5] = h[5] >> 25
- h[6] += carry[5]
- h[5] -= carry[5] << 25
- carry[6] = h[6] >> 26
- h[7] += carry[6]
- h[6] -= carry[6] << 26
- carry[7] = h[7] >> 25
- h[8] += carry[7]
- h[7] -= carry[7] << 25
- carry[8] = h[8] >> 26
- h[9] += carry[8]
- h[8] -= carry[8] << 26
- carry[9] = h[9] >> 25
- h[9] -= carry[9] << 25
- // h10 = carry9
-
- // Goal: Output h[0]+...+2^255 h10-2^255 q, which is between 0 and 2^255-20.
- // Have h[0]+...+2^230 h[9] between 0 and 2^255-1;
- // evidently 2^255 h10-2^255 q = 0.
- // Goal: Output h[0]+...+2^230 h[9].
-
- s[0] = byte(h[0] >> 0)
- s[1] = byte(h[0] >> 8)
- s[2] = byte(h[0] >> 16)
- s[3] = byte((h[0] >> 24) | (h[1] << 2))
- s[4] = byte(h[1] >> 6)
- s[5] = byte(h[1] >> 14)
- s[6] = byte((h[1] >> 22) | (h[2] << 3))
- s[7] = byte(h[2] >> 5)
- s[8] = byte(h[2] >> 13)
- s[9] = byte((h[2] >> 21) | (h[3] << 5))
- s[10] = byte(h[3] >> 3)
- s[11] = byte(h[3] >> 11)
- s[12] = byte((h[3] >> 19) | (h[4] << 6))
- s[13] = byte(h[4] >> 2)
- s[14] = byte(h[4] >> 10)
- s[15] = byte(h[4] >> 18)
- s[16] = byte(h[5] >> 0)
- s[17] = byte(h[5] >> 8)
- s[18] = byte(h[5] >> 16)
- s[19] = byte((h[5] >> 24) | (h[6] << 1))
- s[20] = byte(h[6] >> 7)
- s[21] = byte(h[6] >> 15)
- s[22] = byte((h[6] >> 23) | (h[7] << 3))
- s[23] = byte(h[7] >> 5)
- s[24] = byte(h[7] >> 13)
- s[25] = byte((h[7] >> 21) | (h[8] << 4))
- s[26] = byte(h[8] >> 4)
- s[27] = byte(h[8] >> 12)
- s[28] = byte((h[8] >> 20) | (h[9] << 6))
- s[29] = byte(h[9] >> 2)
- s[30] = byte(h[9] >> 10)
- s[31] = byte(h[9] >> 18)
+// Deprecated: when provided a low-order point, ScalarMult will set dst to all
+// zeroes, irrespective of the scalar. Instead, use the X25519 function, which
+// will return an error.
+func ScalarMult(dst, scalar, point *[32]byte) {
+ scalarMult(dst, scalar, point)
}
-// feMul calculates h = f * g
-// Can overlap h with f or g.
-//
-// Preconditions:
-// |f| bounded by 1.1*2^26,1.1*2^25,1.1*2^26,1.1*2^25,etc.
-// |g| bounded by 1.1*2^26,1.1*2^25,1.1*2^26,1.1*2^25,etc.
-//
-// Postconditions:
-// |h| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc.
-//
-// Notes on implementation strategy:
-//
-// Using schoolbook multiplication.
-// Karatsuba would save a little in some cost models.
+// ScalarBaseMult sets dst to the product scalar * base where base is the
+// standard generator.
//
-// Most multiplications by 2 and 19 are 32-bit precomputations;
-// cheaper than 64-bit postcomputations.
-//
-// There is one remaining multiplication by 19 in the carry chain;
-// one *19 precomputation can be merged into this,
-// but the resulting data flow is considerably less clean.
-//
-// There are 12 carries below.
-// 10 of them are 2-way parallelizable and vectorizable.
-// Can get away with 11 carries, but then data flow is much deeper.
-//
-// With tighter constraints on inputs can squeeze carries into int32.
-func feMul(h, f, g *fieldElement) {
- f0 := f[0]
- f1 := f[1]
- f2 := f[2]
- f3 := f[3]
- f4 := f[4]
- f5 := f[5]
- f6 := f[6]
- f7 := f[7]
- f8 := f[8]
- f9 := f[9]
- g0 := g[0]
- g1 := g[1]
- g2 := g[2]
- g3 := g[3]
- g4 := g[4]
- g5 := g[5]
- g6 := g[6]
- g7 := g[7]
- g8 := g[8]
- g9 := g[9]
- g1_19 := 19 * g1 // 1.4*2^29
- g2_19 := 19 * g2 // 1.4*2^30; still ok
- g3_19 := 19 * g3
- g4_19 := 19 * g4
- g5_19 := 19 * g5
- g6_19 := 19 * g6
- g7_19 := 19 * g7
- g8_19 := 19 * g8
- g9_19 := 19 * g9
- f1_2 := 2 * f1
- f3_2 := 2 * f3
- f5_2 := 2 * f5
- f7_2 := 2 * f7
- f9_2 := 2 * f9
- f0g0 := int64(f0) * int64(g0)
- f0g1 := int64(f0) * int64(g1)
- f0g2 := int64(f0) * int64(g2)
- f0g3 := int64(f0) * int64(g3)
- f0g4 := int64(f0) * int64(g4)
- f0g5 := int64(f0) * int64(g5)
- f0g6 := int64(f0) * int64(g6)
- f0g7 := int64(f0) * int64(g7)
- f0g8 := int64(f0) * int64(g8)
- f0g9 := int64(f0) * int64(g9)
- f1g0 := int64(f1) * int64(g0)
- f1g1_2 := int64(f1_2) * int64(g1)
- f1g2 := int64(f1) * int64(g2)
- f1g3_2 := int64(f1_2) * int64(g3)
- f1g4 := int64(f1) * int64(g4)
- f1g5_2 := int64(f1_2) * int64(g5)
- f1g6 := int64(f1) * int64(g6)
- f1g7_2 := int64(f1_2) * int64(g7)
- f1g8 := int64(f1) * int64(g8)
- f1g9_38 := int64(f1_2) * int64(g9_19)
- f2g0 := int64(f2) * int64(g0)
- f2g1 := int64(f2) * int64(g1)
- f2g2 := int64(f2) * int64(g2)
- f2g3 := int64(f2) * int64(g3)
- f2g4 := int64(f2) * int64(g4)
- f2g5 := int64(f2) * int64(g5)
- f2g6 := int64(f2) * int64(g6)
- f2g7 := int64(f2) * int64(g7)
- f2g8_19 := int64(f2) * int64(g8_19)
- f2g9_19 := int64(f2) * int64(g9_19)
- f3g0 := int64(f3) * int64(g0)
- f3g1_2 := int64(f3_2) * int64(g1)
- f3g2 := int64(f3) * int64(g2)
- f3g3_2 := int64(f3_2) * int64(g3)
- f3g4 := int64(f3) * int64(g4)
- f3g5_2 := int64(f3_2) * int64(g5)
- f3g6 := int64(f3) * int64(g6)
- f3g7_38 := int64(f3_2) * int64(g7_19)
- f3g8_19 := int64(f3) * int64(g8_19)
- f3g9_38 := int64(f3_2) * int64(g9_19)
- f4g0 := int64(f4) * int64(g0)
- f4g1 := int64(f4) * int64(g1)
- f4g2 := int64(f4) * int64(g2)
- f4g3 := int64(f4) * int64(g3)
- f4g4 := int64(f4) * int64(g4)
- f4g5 := int64(f4) * int64(g5)
- f4g6_19 := int64(f4) * int64(g6_19)
- f4g7_19 := int64(f4) * int64(g7_19)
- f4g8_19 := int64(f4) * int64(g8_19)
- f4g9_19 := int64(f4) * int64(g9_19)
- f5g0 := int64(f5) * int64(g0)
- f5g1_2 := int64(f5_2) * int64(g1)
- f5g2 := int64(f5) * int64(g2)
- f5g3_2 := int64(f5_2) * int64(g3)
- f5g4 := int64(f5) * int64(g4)
- f5g5_38 := int64(f5_2) * int64(g5_19)
- f5g6_19 := int64(f5) * int64(g6_19)
- f5g7_38 := int64(f5_2) * int64(g7_19)
- f5g8_19 := int64(f5) * int64(g8_19)
- f5g9_38 := int64(f5_2) * int64(g9_19)
- f6g0 := int64(f6) * int64(g0)
- f6g1 := int64(f6) * int64(g1)
- f6g2 := int64(f6) * int64(g2)
- f6g3 := int64(f6) * int64(g3)
- f6g4_19 := int64(f6) * int64(g4_19)
- f6g5_19 := int64(f6) * int64(g5_19)
- f6g6_19 := int64(f6) * int64(g6_19)
- f6g7_19 := int64(f6) * int64(g7_19)
- f6g8_19 := int64(f6) * int64(g8_19)
- f6g9_19 := int64(f6) * int64(g9_19)
- f7g0 := int64(f7) * int64(g0)
- f7g1_2 := int64(f7_2) * int64(g1)
- f7g2 := int64(f7) * int64(g2)
- f7g3_38 := int64(f7_2) * int64(g3_19)
- f7g4_19 := int64(f7) * int64(g4_19)
- f7g5_38 := int64(f7_2) * int64(g5_19)
- f7g6_19 := int64(f7) * int64(g6_19)
- f7g7_38 := int64(f7_2) * int64(g7_19)
- f7g8_19 := int64(f7) * int64(g8_19)
- f7g9_38 := int64(f7_2) * int64(g9_19)
- f8g0 := int64(f8) * int64(g0)
- f8g1 := int64(f8) * int64(g1)
- f8g2_19 := int64(f8) * int64(g2_19)
- f8g3_19 := int64(f8) * int64(g3_19)
- f8g4_19 := int64(f8) * int64(g4_19)
- f8g5_19 := int64(f8) * int64(g5_19)
- f8g6_19 := int64(f8) * int64(g6_19)
- f8g7_19 := int64(f8) * int64(g7_19)
- f8g8_19 := int64(f8) * int64(g8_19)
- f8g9_19 := int64(f8) * int64(g9_19)
- f9g0 := int64(f9) * int64(g0)
- f9g1_38 := int64(f9_2) * int64(g1_19)
- f9g2_19 := int64(f9) * int64(g2_19)
- f9g3_38 := int64(f9_2) * int64(g3_19)
- f9g4_19 := int64(f9) * int64(g4_19)
- f9g5_38 := int64(f9_2) * int64(g5_19)
- f9g6_19 := int64(f9) * int64(g6_19)
- f9g7_38 := int64(f9_2) * int64(g7_19)
- f9g8_19 := int64(f9) * int64(g8_19)
- f9g9_38 := int64(f9_2) * int64(g9_19)
- h0 := f0g0 + f1g9_38 + f2g8_19 + f3g7_38 + f4g6_19 + f5g5_38 + f6g4_19 + f7g3_38 + f8g2_19 + f9g1_38
- h1 := f0g1 + f1g0 + f2g9_19 + f3g8_19 + f4g7_19 + f5g6_19 + f6g5_19 + f7g4_19 + f8g3_19 + f9g2_19
- h2 := f0g2 + f1g1_2 + f2g0 + f3g9_38 + f4g8_19 + f5g7_38 + f6g6_19 + f7g5_38 + f8g4_19 + f9g3_38
- h3 := f0g3 + f1g2 + f2g1 + f3g0 + f4g9_19 + f5g8_19 + f6g7_19 + f7g6_19 + f8g5_19 + f9g4_19
- h4 := f0g4 + f1g3_2 + f2g2 + f3g1_2 + f4g0 + f5g9_38 + f6g8_19 + f7g7_38 + f8g6_19 + f9g5_38
- h5 := f0g5 + f1g4 + f2g3 + f3g2 + f4g1 + f5g0 + f6g9_19 + f7g8_19 + f8g7_19 + f9g6_19
- h6 := f0g6 + f1g5_2 + f2g4 + f3g3_2 + f4g2 + f5g1_2 + f6g0 + f7g9_38 + f8g8_19 + f9g7_38
- h7 := f0g7 + f1g6 + f2g5 + f3g4 + f4g3 + f5g2 + f6g1 + f7g0 + f8g9_19 + f9g8_19
- h8 := f0g8 + f1g7_2 + f2g6 + f3g5_2 + f4g4 + f5g3_2 + f6g2 + f7g1_2 + f8g0 + f9g9_38
- h9 := f0g9 + f1g8 + f2g7 + f3g6 + f4g5 + f5g4 + f6g3 + f7g2 + f8g1 + f9g0
- var carry [10]int64
-
- // |h0| <= (1.1*1.1*2^52*(1+19+19+19+19)+1.1*1.1*2^50*(38+38+38+38+38))
- // i.e. |h0| <= 1.2*2^59; narrower ranges for h2, h4, h6, h8
- // |h1| <= (1.1*1.1*2^51*(1+1+19+19+19+19+19+19+19+19))
- // i.e. |h1| <= 1.5*2^58; narrower ranges for h3, h5, h7, h9
-
- carry[0] = (h0 + (1 << 25)) >> 26
- h1 += carry[0]
- h0 -= carry[0] << 26
- carry[4] = (h4 + (1 << 25)) >> 26
- h5 += carry[4]
- h4 -= carry[4] << 26
- // |h0| <= 2^25
- // |h4| <= 2^25
- // |h1| <= 1.51*2^58
- // |h5| <= 1.51*2^58
-
- carry[1] = (h1 + (1 << 24)) >> 25
- h2 += carry[1]
- h1 -= carry[1] << 25
- carry[5] = (h5 + (1 << 24)) >> 25
- h6 += carry[5]
- h5 -= carry[5] << 25
- // |h1| <= 2^24; from now on fits into int32
- // |h5| <= 2^24; from now on fits into int32
- // |h2| <= 1.21*2^59
- // |h6| <= 1.21*2^59
-
- carry[2] = (h2 + (1 << 25)) >> 26
- h3 += carry[2]
- h2 -= carry[2] << 26
- carry[6] = (h6 + (1 << 25)) >> 26
- h7 += carry[6]
- h6 -= carry[6] << 26
- // |h2| <= 2^25; from now on fits into int32 unchanged
- // |h6| <= 2^25; from now on fits into int32 unchanged
- // |h3| <= 1.51*2^58
- // |h7| <= 1.51*2^58
-
- carry[3] = (h3 + (1 << 24)) >> 25
- h4 += carry[3]
- h3 -= carry[3] << 25
- carry[7] = (h7 + (1 << 24)) >> 25
- h8 += carry[7]
- h7 -= carry[7] << 25
- // |h3| <= 2^24; from now on fits into int32 unchanged
- // |h7| <= 2^24; from now on fits into int32 unchanged
- // |h4| <= 1.52*2^33
- // |h8| <= 1.52*2^33
-
- carry[4] = (h4 + (1 << 25)) >> 26
- h5 += carry[4]
- h4 -= carry[4] << 26
- carry[8] = (h8 + (1 << 25)) >> 26
- h9 += carry[8]
- h8 -= carry[8] << 26
- // |h4| <= 2^25; from now on fits into int32 unchanged
- // |h8| <= 2^25; from now on fits into int32 unchanged
- // |h5| <= 1.01*2^24
- // |h9| <= 1.51*2^58
-
- carry[9] = (h9 + (1 << 24)) >> 25
- h0 += carry[9] * 19
- h9 -= carry[9] << 25
- // |h9| <= 2^24; from now on fits into int32 unchanged
- // |h0| <= 1.8*2^37
-
- carry[0] = (h0 + (1 << 25)) >> 26
- h1 += carry[0]
- h0 -= carry[0] << 26
- // |h0| <= 2^25; from now on fits into int32 unchanged
- // |h1| <= 1.01*2^24
-
- h[0] = int32(h0)
- h[1] = int32(h1)
- h[2] = int32(h2)
- h[3] = int32(h3)
- h[4] = int32(h4)
- h[5] = int32(h5)
- h[6] = int32(h6)
- h[7] = int32(h7)
- h[8] = int32(h8)
- h[9] = int32(h9)
+// It is recommended to use the X25519 function with Basepoint instead, as
+// copying into fixed size arrays can lead to unexpected bugs.
+func ScalarBaseMult(dst, scalar *[32]byte) {
+ ScalarMult(dst, scalar, &basePoint)
}
-// feSquare calculates h = f*f. Can overlap h with f.
-//
-// Preconditions:
-// |f| bounded by 1.1*2^26,1.1*2^25,1.1*2^26,1.1*2^25,etc.
-//
-// Postconditions:
-// |h| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc.
-func feSquare(h, f *fieldElement) {
- f0 := f[0]
- f1 := f[1]
- f2 := f[2]
- f3 := f[3]
- f4 := f[4]
- f5 := f[5]
- f6 := f[6]
- f7 := f[7]
- f8 := f[8]
- f9 := f[9]
- f0_2 := 2 * f0
- f1_2 := 2 * f1
- f2_2 := 2 * f2
- f3_2 := 2 * f3
- f4_2 := 2 * f4
- f5_2 := 2 * f5
- f6_2 := 2 * f6
- f7_2 := 2 * f7
- f5_38 := 38 * f5 // 1.31*2^30
- f6_19 := 19 * f6 // 1.31*2^30
- f7_38 := 38 * f7 // 1.31*2^30
- f8_19 := 19 * f8 // 1.31*2^30
- f9_38 := 38 * f9 // 1.31*2^30
- f0f0 := int64(f0) * int64(f0)
- f0f1_2 := int64(f0_2) * int64(f1)
- f0f2_2 := int64(f0_2) * int64(f2)
- f0f3_2 := int64(f0_2) * int64(f3)
- f0f4_2 := int64(f0_2) * int64(f4)
- f0f5_2 := int64(f0_2) * int64(f5)
- f0f6_2 := int64(f0_2) * int64(f6)
- f0f7_2 := int64(f0_2) * int64(f7)
- f0f8_2 := int64(f0_2) * int64(f8)
- f0f9_2 := int64(f0_2) * int64(f9)
- f1f1_2 := int64(f1_2) * int64(f1)
- f1f2_2 := int64(f1_2) * int64(f2)
- f1f3_4 := int64(f1_2) * int64(f3_2)
- f1f4_2 := int64(f1_2) * int64(f4)
- f1f5_4 := int64(f1_2) * int64(f5_2)
- f1f6_2 := int64(f1_2) * int64(f6)
- f1f7_4 := int64(f1_2) * int64(f7_2)
- f1f8_2 := int64(f1_2) * int64(f8)
- f1f9_76 := int64(f1_2) * int64(f9_38)
- f2f2 := int64(f2) * int64(f2)
- f2f3_2 := int64(f2_2) * int64(f3)
- f2f4_2 := int64(f2_2) * int64(f4)
- f2f5_2 := int64(f2_2) * int64(f5)
- f2f6_2 := int64(f2_2) * int64(f6)
- f2f7_2 := int64(f2_2) * int64(f7)
- f2f8_38 := int64(f2_2) * int64(f8_19)
- f2f9_38 := int64(f2) * int64(f9_38)
- f3f3_2 := int64(f3_2) * int64(f3)
- f3f4_2 := int64(f3_2) * int64(f4)
- f3f5_4 := int64(f3_2) * int64(f5_2)
- f3f6_2 := int64(f3_2) * int64(f6)
- f3f7_76 := int64(f3_2) * int64(f7_38)
- f3f8_38 := int64(f3_2) * int64(f8_19)
- f3f9_76 := int64(f3_2) * int64(f9_38)
- f4f4 := int64(f4) * int64(f4)
- f4f5_2 := int64(f4_2) * int64(f5)
- f4f6_38 := int64(f4_2) * int64(f6_19)
- f4f7_38 := int64(f4) * int64(f7_38)
- f4f8_38 := int64(f4_2) * int64(f8_19)
- f4f9_38 := int64(f4) * int64(f9_38)
- f5f5_38 := int64(f5) * int64(f5_38)
- f5f6_38 := int64(f5_2) * int64(f6_19)
- f5f7_76 := int64(f5_2) * int64(f7_38)
- f5f8_38 := int64(f5_2) * int64(f8_19)
- f5f9_76 := int64(f5_2) * int64(f9_38)
- f6f6_19 := int64(f6) * int64(f6_19)
- f6f7_38 := int64(f6) * int64(f7_38)
- f6f8_38 := int64(f6_2) * int64(f8_19)
- f6f9_38 := int64(f6) * int64(f9_38)
- f7f7_38 := int64(f7) * int64(f7_38)
- f7f8_38 := int64(f7_2) * int64(f8_19)
- f7f9_76 := int64(f7_2) * int64(f9_38)
- f8f8_19 := int64(f8) * int64(f8_19)
- f8f9_38 := int64(f8) * int64(f9_38)
- f9f9_38 := int64(f9) * int64(f9_38)
- h0 := f0f0 + f1f9_76 + f2f8_38 + f3f7_76 + f4f6_38 + f5f5_38
- h1 := f0f1_2 + f2f9_38 + f3f8_38 + f4f7_38 + f5f6_38
- h2 := f0f2_2 + f1f1_2 + f3f9_76 + f4f8_38 + f5f7_76 + f6f6_19
- h3 := f0f3_2 + f1f2_2 + f4f9_38 + f5f8_38 + f6f7_38
- h4 := f0f4_2 + f1f3_4 + f2f2 + f5f9_76 + f6f8_38 + f7f7_38
- h5 := f0f5_2 + f1f4_2 + f2f3_2 + f6f9_38 + f7f8_38
- h6 := f0f6_2 + f1f5_4 + f2f4_2 + f3f3_2 + f7f9_76 + f8f8_19
- h7 := f0f7_2 + f1f6_2 + f2f5_2 + f3f4_2 + f8f9_38
- h8 := f0f8_2 + f1f7_4 + f2f6_2 + f3f5_4 + f4f4 + f9f9_38
- h9 := f0f9_2 + f1f8_2 + f2f7_2 + f3f6_2 + f4f5_2
- var carry [10]int64
-
- carry[0] = (h0 + (1 << 25)) >> 26
- h1 += carry[0]
- h0 -= carry[0] << 26
- carry[4] = (h4 + (1 << 25)) >> 26
- h5 += carry[4]
- h4 -= carry[4] << 26
-
- carry[1] = (h1 + (1 << 24)) >> 25
- h2 += carry[1]
- h1 -= carry[1] << 25
- carry[5] = (h5 + (1 << 24)) >> 25
- h6 += carry[5]
- h5 -= carry[5] << 25
-
- carry[2] = (h2 + (1 << 25)) >> 26
- h3 += carry[2]
- h2 -= carry[2] << 26
- carry[6] = (h6 + (1 << 25)) >> 26
- h7 += carry[6]
- h6 -= carry[6] << 26
-
- carry[3] = (h3 + (1 << 24)) >> 25
- h4 += carry[3]
- h3 -= carry[3] << 25
- carry[7] = (h7 + (1 << 24)) >> 25
- h8 += carry[7]
- h7 -= carry[7] << 25
+const (
+ // ScalarSize is the size of the scalar input to X25519.
+ ScalarSize = 32
+ // PointSize is the size of the point input to X25519.
+ PointSize = 32
+)
- carry[4] = (h4 + (1 << 25)) >> 26
- h5 += carry[4]
- h4 -= carry[4] << 26
- carry[8] = (h8 + (1 << 25)) >> 26
- h9 += carry[8]
- h8 -= carry[8] << 26
+// Basepoint is the canonical Curve25519 generator.
+var Basepoint []byte
- carry[9] = (h9 + (1 << 24)) >> 25
- h0 += carry[9] * 19
- h9 -= carry[9] << 25
+var basePoint = [32]byte{9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
- carry[0] = (h0 + (1 << 25)) >> 26
- h1 += carry[0]
- h0 -= carry[0] << 26
+func init() { Basepoint = basePoint[:] }
- h[0] = int32(h0)
- h[1] = int32(h1)
- h[2] = int32(h2)
- h[3] = int32(h3)
- h[4] = int32(h4)
- h[5] = int32(h5)
- h[6] = int32(h6)
- h[7] = int32(h7)
- h[8] = int32(h8)
- h[9] = int32(h9)
+func checkBasepoint() {
+ if subtle.ConstantTimeCompare(Basepoint, []byte{
+ 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ }) != 1 {
+ panic("curve25519: global Basepoint value was modified")
+ }
}
-// feMul121666 calculates h = f * 121666. Can overlap h with f.
+// X25519 returns the result of the scalar multiplication (scalar * point),
+// according to RFC 7748, Section 5. scalar, point and the return value are
+// slices of 32 bytes.
//
-// Preconditions:
-// |f| bounded by 1.1*2^26,1.1*2^25,1.1*2^26,1.1*2^25,etc.
+// scalar can be generated at random, for example with crypto/rand. point should
+// be either Basepoint or the output of another X25519 call.
//
-// Postconditions:
-// |h| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc.
-func feMul121666(h, f *fieldElement) {
- h0 := int64(f[0]) * 121666
- h1 := int64(f[1]) * 121666
- h2 := int64(f[2]) * 121666
- h3 := int64(f[3]) * 121666
- h4 := int64(f[4]) * 121666
- h5 := int64(f[5]) * 121666
- h6 := int64(f[6]) * 121666
- h7 := int64(f[7]) * 121666
- h8 := int64(f[8]) * 121666
- h9 := int64(f[9]) * 121666
- var carry [10]int64
-
- carry[9] = (h9 + (1 << 24)) >> 25
- h0 += carry[9] * 19
- h9 -= carry[9] << 25
- carry[1] = (h1 + (1 << 24)) >> 25
- h2 += carry[1]
- h1 -= carry[1] << 25
- carry[3] = (h3 + (1 << 24)) >> 25
- h4 += carry[3]
- h3 -= carry[3] << 25
- carry[5] = (h5 + (1 << 24)) >> 25
- h6 += carry[5]
- h5 -= carry[5] << 25
- carry[7] = (h7 + (1 << 24)) >> 25
- h8 += carry[7]
- h7 -= carry[7] << 25
-
- carry[0] = (h0 + (1 << 25)) >> 26
- h1 += carry[0]
- h0 -= carry[0] << 26
- carry[2] = (h2 + (1 << 25)) >> 26
- h3 += carry[2]
- h2 -= carry[2] << 26
- carry[4] = (h4 + (1 << 25)) >> 26
- h5 += carry[4]
- h4 -= carry[4] << 26
- carry[6] = (h6 + (1 << 25)) >> 26
- h7 += carry[6]
- h6 -= carry[6] << 26
- carry[8] = (h8 + (1 << 25)) >> 26
- h9 += carry[8]
- h8 -= carry[8] << 26
-
- h[0] = int32(h0)
- h[1] = int32(h1)
- h[2] = int32(h2)
- h[3] = int32(h3)
- h[4] = int32(h4)
- h[5] = int32(h5)
- h[6] = int32(h6)
- h[7] = int32(h7)
- h[8] = int32(h8)
- h[9] = int32(h9)
-}
-
-// feInvert sets out = z^-1.
-func feInvert(out, z *fieldElement) {
- var t0, t1, t2, t3 fieldElement
- var i int
-
- feSquare(&t0, z)
- for i = 1; i < 1; i++ {
- feSquare(&t0, &t0)
- }
- feSquare(&t1, &t0)
- for i = 1; i < 2; i++ {
- feSquare(&t1, &t1)
- }
- feMul(&t1, z, &t1)
- feMul(&t0, &t0, &t1)
- feSquare(&t2, &t0)
- for i = 1; i < 1; i++ {
- feSquare(&t2, &t2)
- }
- feMul(&t1, &t1, &t2)
- feSquare(&t2, &t1)
- for i = 1; i < 5; i++ {
- feSquare(&t2, &t2)
- }
- feMul(&t1, &t2, &t1)
- feSquare(&t2, &t1)
- for i = 1; i < 10; i++ {
- feSquare(&t2, &t2)
- }
- feMul(&t2, &t2, &t1)
- feSquare(&t3, &t2)
- for i = 1; i < 20; i++ {
- feSquare(&t3, &t3)
- }
- feMul(&t2, &t3, &t2)
- feSquare(&t2, &t2)
- for i = 1; i < 10; i++ {
- feSquare(&t2, &t2)
- }
- feMul(&t1, &t2, &t1)
- feSquare(&t2, &t1)
- for i = 1; i < 50; i++ {
- feSquare(&t2, &t2)
- }
- feMul(&t2, &t2, &t1)
- feSquare(&t3, &t2)
- for i = 1; i < 100; i++ {
- feSquare(&t3, &t3)
- }
- feMul(&t2, &t3, &t2)
- feSquare(&t2, &t2)
- for i = 1; i < 50; i++ {
- feSquare(&t2, &t2)
- }
- feMul(&t1, &t2, &t1)
- feSquare(&t1, &t1)
- for i = 1; i < 5; i++ {
- feSquare(&t1, &t1)
- }
- feMul(out, &t1, &t0)
+// If point is Basepoint (but not if it's a different slice with the same
+// contents) a precomputed implementation might be used for performance.
+func X25519(scalar, point []byte) ([]byte, error) {
+ // Outline the body of function, to let the allocation be inlined in the
+ // caller, and possibly avoid escaping to the heap.
+ var dst [32]byte
+ return x25519(&dst, scalar, point)
}
-func scalarMult(out, in, base *[32]byte) {
- var e [32]byte
-
- copy(e[:], in[:])
- e[0] &= 248
- e[31] &= 127
- e[31] |= 64
-
- var x1, x2, z2, x3, z3, tmp0, tmp1 fieldElement
- feFromBytes(&x1, base)
- feOne(&x2)
- feCopy(&x3, &x1)
- feOne(&z3)
-
- swap := int32(0)
- for pos := 254; pos >= 0; pos-- {
- b := e[pos/8] >> uint(pos&7)
- b &= 1
- swap ^= int32(b)
- feCSwap(&x2, &x3, swap)
- feCSwap(&z2, &z3, swap)
- swap = int32(b)
-
- feSub(&tmp0, &x3, &z3)
- feSub(&tmp1, &x2, &z2)
- feAdd(&x2, &x2, &z2)
- feAdd(&z2, &x3, &z3)
- feMul(&z3, &tmp0, &x2)
- feMul(&z2, &z2, &tmp1)
- feSquare(&tmp0, &tmp1)
- feSquare(&tmp1, &x2)
- feAdd(&x3, &z3, &z2)
- feSub(&z2, &z3, &z2)
- feMul(&x2, &tmp1, &tmp0)
- feSub(&tmp1, &tmp1, &tmp0)
- feSquare(&z2, &z2)
- feMul121666(&z3, &tmp1)
- feSquare(&x3, &x3)
- feAdd(&tmp0, &tmp0, &z3)
- feMul(&z3, &x1, &z2)
- feMul(&z2, &tmp1, &tmp0)
- }
-
- feCSwap(&x2, &x3, swap)
- feCSwap(&z2, &z3, swap)
-
- feInvert(&z2, &z2)
- feMul(&x2, &x2, &z2)
- feToBytes(out, &x2)
+func x25519(dst *[32]byte, scalar, point []byte) ([]byte, error) {
+ var in [32]byte
+ if l := len(scalar); l != 32 {
+ return nil, fmt.Errorf("bad scalar length: %d, expected %d", l, 32)
+ }
+ if l := len(point); l != 32 {
+ return nil, fmt.Errorf("bad point length: %d, expected %d", l, 32)
+ }
+ copy(in[:], scalar)
+ if &point[0] == &Basepoint[0] {
+ checkBasepoint()
+ ScalarBaseMult(dst, &in)
+ } else {
+ var base, zero [32]byte
+ copy(base[:], point)
+ ScalarMult(dst, &in, &base)
+ if subtle.ConstantTimeCompare(dst[:], zero[:]) == 1 {
+ return nil, fmt.Errorf("bad input point: low order point")
+ }
+ }
+ return dst[:], nil
}
--- /dev/null
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build amd64,!gccgo,!appengine,!purego
+
+package curve25519
+
+// These functions are implemented in the .s files. The names of the functions
+// in the rest of the file are also taken from the SUPERCOP sources to help
+// people following along.
+
+//go:noescape
+
+func cswap(inout *[5]uint64, v uint64)
+
+//go:noescape
+
+func ladderstep(inout *[5][5]uint64)
+
+//go:noescape
+
+func freeze(inout *[5]uint64)
+
+//go:noescape
+
+func mul(dest, a, b *[5]uint64)
+
+//go:noescape
+
+func square(out, in *[5]uint64)
+
+// mladder uses a Montgomery ladder to calculate (xr/zr) *= s.
+func mladder(xr, zr *[5]uint64, s *[32]byte) {
+ var work [5][5]uint64
+
+ work[0] = *xr
+ setint(&work[1], 1)
+ setint(&work[2], 0)
+ work[3] = *xr
+ setint(&work[4], 1)
+
+ j := uint(6)
+ var prevbit byte
+
+ for i := 31; i >= 0; i-- {
+ for j < 8 {
+ bit := ((*s)[i] >> j) & 1
+ swap := bit ^ prevbit
+ prevbit = bit
+ cswap(&work[1], uint64(swap))
+ ladderstep(&work)
+ j--
+ }
+ j = 7
+ }
+
+ *xr = work[1]
+ *zr = work[2]
+}
+
+func scalarMult(out, in, base *[32]byte) {
+ var e [32]byte
+ copy(e[:], (*in)[:])
+ e[0] &= 248
+ e[31] &= 127
+ e[31] |= 64
+
+ var t, z [5]uint64
+ unpack(&t, base)
+ mladder(&t, &z, &e)
+ invert(&z, &z)
+ mul(&t, &t, &z)
+ pack(out, &t)
+}
+
+func setint(r *[5]uint64, v uint64) {
+ r[0] = v
+ r[1] = 0
+ r[2] = 0
+ r[3] = 0
+ r[4] = 0
+}
+
+// unpack sets r = x where r consists of 5, 51-bit limbs in little-endian
+// order.
+func unpack(r *[5]uint64, x *[32]byte) {
+ r[0] = uint64(x[0]) |
+ uint64(x[1])<<8 |
+ uint64(x[2])<<16 |
+ uint64(x[3])<<24 |
+ uint64(x[4])<<32 |
+ uint64(x[5])<<40 |
+ uint64(x[6]&7)<<48
+
+ r[1] = uint64(x[6])>>3 |
+ uint64(x[7])<<5 |
+ uint64(x[8])<<13 |
+ uint64(x[9])<<21 |
+ uint64(x[10])<<29 |
+ uint64(x[11])<<37 |
+ uint64(x[12]&63)<<45
+
+ r[2] = uint64(x[12])>>6 |
+ uint64(x[13])<<2 |
+ uint64(x[14])<<10 |
+ uint64(x[15])<<18 |
+ uint64(x[16])<<26 |
+ uint64(x[17])<<34 |
+ uint64(x[18])<<42 |
+ uint64(x[19]&1)<<50
+
+ r[3] = uint64(x[19])>>1 |
+ uint64(x[20])<<7 |
+ uint64(x[21])<<15 |
+ uint64(x[22])<<23 |
+ uint64(x[23])<<31 |
+ uint64(x[24])<<39 |
+ uint64(x[25]&15)<<47
+
+ r[4] = uint64(x[25])>>4 |
+ uint64(x[26])<<4 |
+ uint64(x[27])<<12 |
+ uint64(x[28])<<20 |
+ uint64(x[29])<<28 |
+ uint64(x[30])<<36 |
+ uint64(x[31]&127)<<44
+}
+
+// pack sets out = x where out is the usual, little-endian form of the 5,
+// 51-bit limbs in x.
+func pack(out *[32]byte, x *[5]uint64) {
+ t := *x
+ freeze(&t)
+
+ out[0] = byte(t[0])
+ out[1] = byte(t[0] >> 8)
+ out[2] = byte(t[0] >> 16)
+ out[3] = byte(t[0] >> 24)
+ out[4] = byte(t[0] >> 32)
+ out[5] = byte(t[0] >> 40)
+ out[6] = byte(t[0] >> 48)
+
+ out[6] ^= byte(t[1]<<3) & 0xf8
+ out[7] = byte(t[1] >> 5)
+ out[8] = byte(t[1] >> 13)
+ out[9] = byte(t[1] >> 21)
+ out[10] = byte(t[1] >> 29)
+ out[11] = byte(t[1] >> 37)
+ out[12] = byte(t[1] >> 45)
+
+ out[12] ^= byte(t[2]<<6) & 0xc0
+ out[13] = byte(t[2] >> 2)
+ out[14] = byte(t[2] >> 10)
+ out[15] = byte(t[2] >> 18)
+ out[16] = byte(t[2] >> 26)
+ out[17] = byte(t[2] >> 34)
+ out[18] = byte(t[2] >> 42)
+ out[19] = byte(t[2] >> 50)
+
+ out[19] ^= byte(t[3]<<1) & 0xfe
+ out[20] = byte(t[3] >> 7)
+ out[21] = byte(t[3] >> 15)
+ out[22] = byte(t[3] >> 23)
+ out[23] = byte(t[3] >> 31)
+ out[24] = byte(t[3] >> 39)
+ out[25] = byte(t[3] >> 47)
+
+ out[25] ^= byte(t[4]<<4) & 0xf0
+ out[26] = byte(t[4] >> 4)
+ out[27] = byte(t[4] >> 12)
+ out[28] = byte(t[4] >> 20)
+ out[29] = byte(t[4] >> 28)
+ out[30] = byte(t[4] >> 36)
+ out[31] = byte(t[4] >> 44)
+}
+
+// invert calculates r = x^-1 mod p using Fermat's little theorem.
+func invert(r *[5]uint64, x *[5]uint64) {
+ var z2, z9, z11, z2_5_0, z2_10_0, z2_20_0, z2_50_0, z2_100_0, t [5]uint64
+
+ square(&z2, x) /* 2 */
+ square(&t, &z2) /* 4 */
+ square(&t, &t) /* 8 */
+ mul(&z9, &t, x) /* 9 */
+ mul(&z11, &z9, &z2) /* 11 */
+ square(&t, &z11) /* 22 */
+ mul(&z2_5_0, &t, &z9) /* 2^5 - 2^0 = 31 */
+
+ square(&t, &z2_5_0) /* 2^6 - 2^1 */
+ for i := 1; i < 5; i++ { /* 2^20 - 2^10 */
+ square(&t, &t)
+ }
+ mul(&z2_10_0, &t, &z2_5_0) /* 2^10 - 2^0 */
+
+ square(&t, &z2_10_0) /* 2^11 - 2^1 */
+ for i := 1; i < 10; i++ { /* 2^20 - 2^10 */
+ square(&t, &t)
+ }
+ mul(&z2_20_0, &t, &z2_10_0) /* 2^20 - 2^0 */
+
+ square(&t, &z2_20_0) /* 2^21 - 2^1 */
+ for i := 1; i < 20; i++ { /* 2^40 - 2^20 */
+ square(&t, &t)
+ }
+ mul(&t, &t, &z2_20_0) /* 2^40 - 2^0 */
+
+ square(&t, &t) /* 2^41 - 2^1 */
+ for i := 1; i < 10; i++ { /* 2^50 - 2^10 */
+ square(&t, &t)
+ }
+ mul(&z2_50_0, &t, &z2_10_0) /* 2^50 - 2^0 */
+
+ square(&t, &z2_50_0) /* 2^51 - 2^1 */
+ for i := 1; i < 50; i++ { /* 2^100 - 2^50 */
+ square(&t, &t)
+ }
+ mul(&z2_100_0, &t, &z2_50_0) /* 2^100 - 2^0 */
+
+ square(&t, &z2_100_0) /* 2^101 - 2^1 */
+ for i := 1; i < 100; i++ { /* 2^200 - 2^100 */
+ square(&t, &t)
+ }
+ mul(&t, &t, &z2_100_0) /* 2^200 - 2^0 */
+
+ square(&t, &t) /* 2^201 - 2^1 */
+ for i := 1; i < 50; i++ { /* 2^250 - 2^50 */
+ square(&t, &t)
+ }
+ mul(&t, &t, &z2_50_0) /* 2^250 - 2^0 */
+
+ square(&t, &t) /* 2^251 - 2^1 */
+ square(&t, &t) /* 2^252 - 2^2 */
+ square(&t, &t) /* 2^253 - 2^3 */
+
+ square(&t, &t) /* 2^254 - 2^4 */
+
+ square(&t, &t) /* 2^255 - 2^5 */
+ mul(r, &t, &z11) /* 2^255 - 21 */
+}
--- /dev/null
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This code was translated into a form compatible with 6a from the public
+// domain sources in SUPERCOP: https://bench.cr.yp.to/supercop.html
+
+// +build amd64,!gccgo,!appengine,!purego
+
+#define REDMASK51 0x0007FFFFFFFFFFFF
+
+// These constants cannot be encoded in non-MOVQ immediates.
+// We access them directly from memory instead.
+
+DATA ·_121666_213(SB)/8, $996687872
+GLOBL ·_121666_213(SB), 8, $8
+
+DATA ·_2P0(SB)/8, $0xFFFFFFFFFFFDA
+GLOBL ·_2P0(SB), 8, $8
+
+DATA ·_2P1234(SB)/8, $0xFFFFFFFFFFFFE
+GLOBL ·_2P1234(SB), 8, $8
+
+// func freeze(inout *[5]uint64)
+TEXT ·freeze(SB),7,$0-8
+ MOVQ inout+0(FP), DI
+
+ MOVQ 0(DI),SI
+ MOVQ 8(DI),DX
+ MOVQ 16(DI),CX
+ MOVQ 24(DI),R8
+ MOVQ 32(DI),R9
+ MOVQ $REDMASK51,AX
+ MOVQ AX,R10
+ SUBQ $18,R10
+ MOVQ $3,R11
+REDUCELOOP:
+ MOVQ SI,R12
+ SHRQ $51,R12
+ ANDQ AX,SI
+ ADDQ R12,DX
+ MOVQ DX,R12
+ SHRQ $51,R12
+ ANDQ AX,DX
+ ADDQ R12,CX
+ MOVQ CX,R12
+ SHRQ $51,R12
+ ANDQ AX,CX
+ ADDQ R12,R8
+ MOVQ R8,R12
+ SHRQ $51,R12
+ ANDQ AX,R8
+ ADDQ R12,R9
+ MOVQ R9,R12
+ SHRQ $51,R12
+ ANDQ AX,R9
+ IMUL3Q $19,R12,R12
+ ADDQ R12,SI
+ SUBQ $1,R11
+ JA REDUCELOOP
+ MOVQ $1,R12
+ CMPQ R10,SI
+ CMOVQLT R11,R12
+ CMPQ AX,DX
+ CMOVQNE R11,R12
+ CMPQ AX,CX
+ CMOVQNE R11,R12
+ CMPQ AX,R8
+ CMOVQNE R11,R12
+ CMPQ AX,R9
+ CMOVQNE R11,R12
+ NEGQ R12
+ ANDQ R12,AX
+ ANDQ R12,R10
+ SUBQ R10,SI
+ SUBQ AX,DX
+ SUBQ AX,CX
+ SUBQ AX,R8
+ SUBQ AX,R9
+ MOVQ SI,0(DI)
+ MOVQ DX,8(DI)
+ MOVQ CX,16(DI)
+ MOVQ R8,24(DI)
+ MOVQ R9,32(DI)
+ RET
+
+// func ladderstep(inout *[5][5]uint64)
+TEXT ·ladderstep(SB),0,$296-8
+ MOVQ inout+0(FP),DI
+
+ MOVQ 40(DI),SI
+ MOVQ 48(DI),DX
+ MOVQ 56(DI),CX
+ MOVQ 64(DI),R8
+ MOVQ 72(DI),R9
+ MOVQ SI,AX
+ MOVQ DX,R10
+ MOVQ CX,R11
+ MOVQ R8,R12
+ MOVQ R9,R13
+ ADDQ ·_2P0(SB),AX
+ ADDQ ·_2P1234(SB),R10
+ ADDQ ·_2P1234(SB),R11
+ ADDQ ·_2P1234(SB),R12
+ ADDQ ·_2P1234(SB),R13
+ ADDQ 80(DI),SI
+ ADDQ 88(DI),DX
+ ADDQ 96(DI),CX
+ ADDQ 104(DI),R8
+ ADDQ 112(DI),R9
+ SUBQ 80(DI),AX
+ SUBQ 88(DI),R10
+ SUBQ 96(DI),R11
+ SUBQ 104(DI),R12
+ SUBQ 112(DI),R13
+ MOVQ SI,0(SP)
+ MOVQ DX,8(SP)
+ MOVQ CX,16(SP)
+ MOVQ R8,24(SP)
+ MOVQ R9,32(SP)
+ MOVQ AX,40(SP)
+ MOVQ R10,48(SP)
+ MOVQ R11,56(SP)
+ MOVQ R12,64(SP)
+ MOVQ R13,72(SP)
+ MOVQ 40(SP),AX
+ MULQ 40(SP)
+ MOVQ AX,SI
+ MOVQ DX,CX
+ MOVQ 40(SP),AX
+ SHLQ $1,AX
+ MULQ 48(SP)
+ MOVQ AX,R8
+ MOVQ DX,R9
+ MOVQ 40(SP),AX
+ SHLQ $1,AX
+ MULQ 56(SP)
+ MOVQ AX,R10
+ MOVQ DX,R11
+ MOVQ 40(SP),AX
+ SHLQ $1,AX
+ MULQ 64(SP)
+ MOVQ AX,R12
+ MOVQ DX,R13
+ MOVQ 40(SP),AX
+ SHLQ $1,AX
+ MULQ 72(SP)
+ MOVQ AX,R14
+ MOVQ DX,R15
+ MOVQ 48(SP),AX
+ MULQ 48(SP)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 48(SP),AX
+ SHLQ $1,AX
+ MULQ 56(SP)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ 48(SP),AX
+ SHLQ $1,AX
+ MULQ 64(SP)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 48(SP),DX
+ IMUL3Q $38,DX,AX
+ MULQ 72(SP)
+ ADDQ AX,SI
+ ADCQ DX,CX
+ MOVQ 56(SP),AX
+ MULQ 56(SP)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 56(SP),DX
+ IMUL3Q $38,DX,AX
+ MULQ 64(SP)
+ ADDQ AX,SI
+ ADCQ DX,CX
+ MOVQ 56(SP),DX
+ IMUL3Q $38,DX,AX
+ MULQ 72(SP)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 64(SP),DX
+ IMUL3Q $19,DX,AX
+ MULQ 64(SP)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 64(SP),DX
+ IMUL3Q $38,DX,AX
+ MULQ 72(SP)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 72(SP),DX
+ IMUL3Q $19,DX,AX
+ MULQ 72(SP)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ $REDMASK51,DX
+ SHLQ $13,SI,CX
+ ANDQ DX,SI
+ SHLQ $13,R8,R9
+ ANDQ DX,R8
+ ADDQ CX,R8
+ SHLQ $13,R10,R11
+ ANDQ DX,R10
+ ADDQ R9,R10
+ SHLQ $13,R12,R13
+ ANDQ DX,R12
+ ADDQ R11,R12
+ SHLQ $13,R14,R15
+ ANDQ DX,R14
+ ADDQ R13,R14
+ IMUL3Q $19,R15,CX
+ ADDQ CX,SI
+ MOVQ SI,CX
+ SHRQ $51,CX
+ ADDQ R8,CX
+ ANDQ DX,SI
+ MOVQ CX,R8
+ SHRQ $51,CX
+ ADDQ R10,CX
+ ANDQ DX,R8
+ MOVQ CX,R9
+ SHRQ $51,CX
+ ADDQ R12,CX
+ ANDQ DX,R9
+ MOVQ CX,AX
+ SHRQ $51,CX
+ ADDQ R14,CX
+ ANDQ DX,AX
+ MOVQ CX,R10
+ SHRQ $51,CX
+ IMUL3Q $19,CX,CX
+ ADDQ CX,SI
+ ANDQ DX,R10
+ MOVQ SI,80(SP)
+ MOVQ R8,88(SP)
+ MOVQ R9,96(SP)
+ MOVQ AX,104(SP)
+ MOVQ R10,112(SP)
+ MOVQ 0(SP),AX
+ MULQ 0(SP)
+ MOVQ AX,SI
+ MOVQ DX,CX
+ MOVQ 0(SP),AX
+ SHLQ $1,AX
+ MULQ 8(SP)
+ MOVQ AX,R8
+ MOVQ DX,R9
+ MOVQ 0(SP),AX
+ SHLQ $1,AX
+ MULQ 16(SP)
+ MOVQ AX,R10
+ MOVQ DX,R11
+ MOVQ 0(SP),AX
+ SHLQ $1,AX
+ MULQ 24(SP)
+ MOVQ AX,R12
+ MOVQ DX,R13
+ MOVQ 0(SP),AX
+ SHLQ $1,AX
+ MULQ 32(SP)
+ MOVQ AX,R14
+ MOVQ DX,R15
+ MOVQ 8(SP),AX
+ MULQ 8(SP)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 8(SP),AX
+ SHLQ $1,AX
+ MULQ 16(SP)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ 8(SP),AX
+ SHLQ $1,AX
+ MULQ 24(SP)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 8(SP),DX
+ IMUL3Q $38,DX,AX
+ MULQ 32(SP)
+ ADDQ AX,SI
+ ADCQ DX,CX
+ MOVQ 16(SP),AX
+ MULQ 16(SP)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 16(SP),DX
+ IMUL3Q $38,DX,AX
+ MULQ 24(SP)
+ ADDQ AX,SI
+ ADCQ DX,CX
+ MOVQ 16(SP),DX
+ IMUL3Q $38,DX,AX
+ MULQ 32(SP)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 24(SP),DX
+ IMUL3Q $19,DX,AX
+ MULQ 24(SP)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 24(SP),DX
+ IMUL3Q $38,DX,AX
+ MULQ 32(SP)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 32(SP),DX
+ IMUL3Q $19,DX,AX
+ MULQ 32(SP)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ $REDMASK51,DX
+ SHLQ $13,SI,CX
+ ANDQ DX,SI
+ SHLQ $13,R8,R9
+ ANDQ DX,R8
+ ADDQ CX,R8
+ SHLQ $13,R10,R11
+ ANDQ DX,R10
+ ADDQ R9,R10
+ SHLQ $13,R12,R13
+ ANDQ DX,R12
+ ADDQ R11,R12
+ SHLQ $13,R14,R15
+ ANDQ DX,R14
+ ADDQ R13,R14
+ IMUL3Q $19,R15,CX
+ ADDQ CX,SI
+ MOVQ SI,CX
+ SHRQ $51,CX
+ ADDQ R8,CX
+ ANDQ DX,SI
+ MOVQ CX,R8
+ SHRQ $51,CX
+ ADDQ R10,CX
+ ANDQ DX,R8
+ MOVQ CX,R9
+ SHRQ $51,CX
+ ADDQ R12,CX
+ ANDQ DX,R9
+ MOVQ CX,AX
+ SHRQ $51,CX
+ ADDQ R14,CX
+ ANDQ DX,AX
+ MOVQ CX,R10
+ SHRQ $51,CX
+ IMUL3Q $19,CX,CX
+ ADDQ CX,SI
+ ANDQ DX,R10
+ MOVQ SI,120(SP)
+ MOVQ R8,128(SP)
+ MOVQ R9,136(SP)
+ MOVQ AX,144(SP)
+ MOVQ R10,152(SP)
+ MOVQ SI,SI
+ MOVQ R8,DX
+ MOVQ R9,CX
+ MOVQ AX,R8
+ MOVQ R10,R9
+ ADDQ ·_2P0(SB),SI
+ ADDQ ·_2P1234(SB),DX
+ ADDQ ·_2P1234(SB),CX
+ ADDQ ·_2P1234(SB),R8
+ ADDQ ·_2P1234(SB),R9
+ SUBQ 80(SP),SI
+ SUBQ 88(SP),DX
+ SUBQ 96(SP),CX
+ SUBQ 104(SP),R8
+ SUBQ 112(SP),R9
+ MOVQ SI,160(SP)
+ MOVQ DX,168(SP)
+ MOVQ CX,176(SP)
+ MOVQ R8,184(SP)
+ MOVQ R9,192(SP)
+ MOVQ 120(DI),SI
+ MOVQ 128(DI),DX
+ MOVQ 136(DI),CX
+ MOVQ 144(DI),R8
+ MOVQ 152(DI),R9
+ MOVQ SI,AX
+ MOVQ DX,R10
+ MOVQ CX,R11
+ MOVQ R8,R12
+ MOVQ R9,R13
+ ADDQ ·_2P0(SB),AX
+ ADDQ ·_2P1234(SB),R10
+ ADDQ ·_2P1234(SB),R11
+ ADDQ ·_2P1234(SB),R12
+ ADDQ ·_2P1234(SB),R13
+ ADDQ 160(DI),SI
+ ADDQ 168(DI),DX
+ ADDQ 176(DI),CX
+ ADDQ 184(DI),R8
+ ADDQ 192(DI),R9
+ SUBQ 160(DI),AX
+ SUBQ 168(DI),R10
+ SUBQ 176(DI),R11
+ SUBQ 184(DI),R12
+ SUBQ 192(DI),R13
+ MOVQ SI,200(SP)
+ MOVQ DX,208(SP)
+ MOVQ CX,216(SP)
+ MOVQ R8,224(SP)
+ MOVQ R9,232(SP)
+ MOVQ AX,240(SP)
+ MOVQ R10,248(SP)
+ MOVQ R11,256(SP)
+ MOVQ R12,264(SP)
+ MOVQ R13,272(SP)
+ MOVQ 224(SP),SI
+ IMUL3Q $19,SI,AX
+ MOVQ AX,280(SP)
+ MULQ 56(SP)
+ MOVQ AX,SI
+ MOVQ DX,CX
+ MOVQ 232(SP),DX
+ IMUL3Q $19,DX,AX
+ MOVQ AX,288(SP)
+ MULQ 48(SP)
+ ADDQ AX,SI
+ ADCQ DX,CX
+ MOVQ 200(SP),AX
+ MULQ 40(SP)
+ ADDQ AX,SI
+ ADCQ DX,CX
+ MOVQ 200(SP),AX
+ MULQ 48(SP)
+ MOVQ AX,R8
+ MOVQ DX,R9
+ MOVQ 200(SP),AX
+ MULQ 56(SP)
+ MOVQ AX,R10
+ MOVQ DX,R11
+ MOVQ 200(SP),AX
+ MULQ 64(SP)
+ MOVQ AX,R12
+ MOVQ DX,R13
+ MOVQ 200(SP),AX
+ MULQ 72(SP)
+ MOVQ AX,R14
+ MOVQ DX,R15
+ MOVQ 208(SP),AX
+ MULQ 40(SP)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 208(SP),AX
+ MULQ 48(SP)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 208(SP),AX
+ MULQ 56(SP)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ 208(SP),AX
+ MULQ 64(SP)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 208(SP),DX
+ IMUL3Q $19,DX,AX
+ MULQ 72(SP)
+ ADDQ AX,SI
+ ADCQ DX,CX
+ MOVQ 216(SP),AX
+ MULQ 40(SP)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 216(SP),AX
+ MULQ 48(SP)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ 216(SP),AX
+ MULQ 56(SP)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 216(SP),DX
+ IMUL3Q $19,DX,AX
+ MULQ 64(SP)
+ ADDQ AX,SI
+ ADCQ DX,CX
+ MOVQ 216(SP),DX
+ IMUL3Q $19,DX,AX
+ MULQ 72(SP)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 224(SP),AX
+ MULQ 40(SP)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ 224(SP),AX
+ MULQ 48(SP)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 280(SP),AX
+ MULQ 64(SP)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 280(SP),AX
+ MULQ 72(SP)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 232(SP),AX
+ MULQ 40(SP)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 288(SP),AX
+ MULQ 56(SP)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 288(SP),AX
+ MULQ 64(SP)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 288(SP),AX
+ MULQ 72(SP)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ $REDMASK51,DX
+ SHLQ $13,SI,CX
+ ANDQ DX,SI
+ SHLQ $13,R8,R9
+ ANDQ DX,R8
+ ADDQ CX,R8
+ SHLQ $13,R10,R11
+ ANDQ DX,R10
+ ADDQ R9,R10
+ SHLQ $13,R12,R13
+ ANDQ DX,R12
+ ADDQ R11,R12
+ SHLQ $13,R14,R15
+ ANDQ DX,R14
+ ADDQ R13,R14
+ IMUL3Q $19,R15,CX
+ ADDQ CX,SI
+ MOVQ SI,CX
+ SHRQ $51,CX
+ ADDQ R8,CX
+ MOVQ CX,R8
+ SHRQ $51,CX
+ ANDQ DX,SI
+ ADDQ R10,CX
+ MOVQ CX,R9
+ SHRQ $51,CX
+ ANDQ DX,R8
+ ADDQ R12,CX
+ MOVQ CX,AX
+ SHRQ $51,CX
+ ANDQ DX,R9
+ ADDQ R14,CX
+ MOVQ CX,R10
+ SHRQ $51,CX
+ ANDQ DX,AX
+ IMUL3Q $19,CX,CX
+ ADDQ CX,SI
+ ANDQ DX,R10
+ MOVQ SI,40(SP)
+ MOVQ R8,48(SP)
+ MOVQ R9,56(SP)
+ MOVQ AX,64(SP)
+ MOVQ R10,72(SP)
+ MOVQ 264(SP),SI
+ IMUL3Q $19,SI,AX
+ MOVQ AX,200(SP)
+ MULQ 16(SP)
+ MOVQ AX,SI
+ MOVQ DX,CX
+ MOVQ 272(SP),DX
+ IMUL3Q $19,DX,AX
+ MOVQ AX,208(SP)
+ MULQ 8(SP)
+ ADDQ AX,SI
+ ADCQ DX,CX
+ MOVQ 240(SP),AX
+ MULQ 0(SP)
+ ADDQ AX,SI
+ ADCQ DX,CX
+ MOVQ 240(SP),AX
+ MULQ 8(SP)
+ MOVQ AX,R8
+ MOVQ DX,R9
+ MOVQ 240(SP),AX
+ MULQ 16(SP)
+ MOVQ AX,R10
+ MOVQ DX,R11
+ MOVQ 240(SP),AX
+ MULQ 24(SP)
+ MOVQ AX,R12
+ MOVQ DX,R13
+ MOVQ 240(SP),AX
+ MULQ 32(SP)
+ MOVQ AX,R14
+ MOVQ DX,R15
+ MOVQ 248(SP),AX
+ MULQ 0(SP)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 248(SP),AX
+ MULQ 8(SP)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 248(SP),AX
+ MULQ 16(SP)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ 248(SP),AX
+ MULQ 24(SP)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 248(SP),DX
+ IMUL3Q $19,DX,AX
+ MULQ 32(SP)
+ ADDQ AX,SI
+ ADCQ DX,CX
+ MOVQ 256(SP),AX
+ MULQ 0(SP)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 256(SP),AX
+ MULQ 8(SP)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ 256(SP),AX
+ MULQ 16(SP)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 256(SP),DX
+ IMUL3Q $19,DX,AX
+ MULQ 24(SP)
+ ADDQ AX,SI
+ ADCQ DX,CX
+ MOVQ 256(SP),DX
+ IMUL3Q $19,DX,AX
+ MULQ 32(SP)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 264(SP),AX
+ MULQ 0(SP)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ 264(SP),AX
+ MULQ 8(SP)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 200(SP),AX
+ MULQ 24(SP)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 200(SP),AX
+ MULQ 32(SP)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 272(SP),AX
+ MULQ 0(SP)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 208(SP),AX
+ MULQ 16(SP)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 208(SP),AX
+ MULQ 24(SP)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 208(SP),AX
+ MULQ 32(SP)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ $REDMASK51,DX
+ SHLQ $13,SI,CX
+ ANDQ DX,SI
+ SHLQ $13,R8,R9
+ ANDQ DX,R8
+ ADDQ CX,R8
+ SHLQ $13,R10,R11
+ ANDQ DX,R10
+ ADDQ R9,R10
+ SHLQ $13,R12,R13
+ ANDQ DX,R12
+ ADDQ R11,R12
+ SHLQ $13,R14,R15
+ ANDQ DX,R14
+ ADDQ R13,R14
+ IMUL3Q $19,R15,CX
+ ADDQ CX,SI
+ MOVQ SI,CX
+ SHRQ $51,CX
+ ADDQ R8,CX
+ MOVQ CX,R8
+ SHRQ $51,CX
+ ANDQ DX,SI
+ ADDQ R10,CX
+ MOVQ CX,R9
+ SHRQ $51,CX
+ ANDQ DX,R8
+ ADDQ R12,CX
+ MOVQ CX,AX
+ SHRQ $51,CX
+ ANDQ DX,R9
+ ADDQ R14,CX
+ MOVQ CX,R10
+ SHRQ $51,CX
+ ANDQ DX,AX
+ IMUL3Q $19,CX,CX
+ ADDQ CX,SI
+ ANDQ DX,R10
+ MOVQ SI,DX
+ MOVQ R8,CX
+ MOVQ R9,R11
+ MOVQ AX,R12
+ MOVQ R10,R13
+ ADDQ ·_2P0(SB),DX
+ ADDQ ·_2P1234(SB),CX
+ ADDQ ·_2P1234(SB),R11
+ ADDQ ·_2P1234(SB),R12
+ ADDQ ·_2P1234(SB),R13
+ ADDQ 40(SP),SI
+ ADDQ 48(SP),R8
+ ADDQ 56(SP),R9
+ ADDQ 64(SP),AX
+ ADDQ 72(SP),R10
+ SUBQ 40(SP),DX
+ SUBQ 48(SP),CX
+ SUBQ 56(SP),R11
+ SUBQ 64(SP),R12
+ SUBQ 72(SP),R13
+ MOVQ SI,120(DI)
+ MOVQ R8,128(DI)
+ MOVQ R9,136(DI)
+ MOVQ AX,144(DI)
+ MOVQ R10,152(DI)
+ MOVQ DX,160(DI)
+ MOVQ CX,168(DI)
+ MOVQ R11,176(DI)
+ MOVQ R12,184(DI)
+ MOVQ R13,192(DI)
+ MOVQ 120(DI),AX
+ MULQ 120(DI)
+ MOVQ AX,SI
+ MOVQ DX,CX
+ MOVQ 120(DI),AX
+ SHLQ $1,AX
+ MULQ 128(DI)
+ MOVQ AX,R8
+ MOVQ DX,R9
+ MOVQ 120(DI),AX
+ SHLQ $1,AX
+ MULQ 136(DI)
+ MOVQ AX,R10
+ MOVQ DX,R11
+ MOVQ 120(DI),AX
+ SHLQ $1,AX
+ MULQ 144(DI)
+ MOVQ AX,R12
+ MOVQ DX,R13
+ MOVQ 120(DI),AX
+ SHLQ $1,AX
+ MULQ 152(DI)
+ MOVQ AX,R14
+ MOVQ DX,R15
+ MOVQ 128(DI),AX
+ MULQ 128(DI)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 128(DI),AX
+ SHLQ $1,AX
+ MULQ 136(DI)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ 128(DI),AX
+ SHLQ $1,AX
+ MULQ 144(DI)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 128(DI),DX
+ IMUL3Q $38,DX,AX
+ MULQ 152(DI)
+ ADDQ AX,SI
+ ADCQ DX,CX
+ MOVQ 136(DI),AX
+ MULQ 136(DI)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 136(DI),DX
+ IMUL3Q $38,DX,AX
+ MULQ 144(DI)
+ ADDQ AX,SI
+ ADCQ DX,CX
+ MOVQ 136(DI),DX
+ IMUL3Q $38,DX,AX
+ MULQ 152(DI)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 144(DI),DX
+ IMUL3Q $19,DX,AX
+ MULQ 144(DI)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 144(DI),DX
+ IMUL3Q $38,DX,AX
+ MULQ 152(DI)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 152(DI),DX
+ IMUL3Q $19,DX,AX
+ MULQ 152(DI)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ $REDMASK51,DX
+ SHLQ $13,SI,CX
+ ANDQ DX,SI
+ SHLQ $13,R8,R9
+ ANDQ DX,R8
+ ADDQ CX,R8
+ SHLQ $13,R10,R11
+ ANDQ DX,R10
+ ADDQ R9,R10
+ SHLQ $13,R12,R13
+ ANDQ DX,R12
+ ADDQ R11,R12
+ SHLQ $13,R14,R15
+ ANDQ DX,R14
+ ADDQ R13,R14
+ IMUL3Q $19,R15,CX
+ ADDQ CX,SI
+ MOVQ SI,CX
+ SHRQ $51,CX
+ ADDQ R8,CX
+ ANDQ DX,SI
+ MOVQ CX,R8
+ SHRQ $51,CX
+ ADDQ R10,CX
+ ANDQ DX,R8
+ MOVQ CX,R9
+ SHRQ $51,CX
+ ADDQ R12,CX
+ ANDQ DX,R9
+ MOVQ CX,AX
+ SHRQ $51,CX
+ ADDQ R14,CX
+ ANDQ DX,AX
+ MOVQ CX,R10
+ SHRQ $51,CX
+ IMUL3Q $19,CX,CX
+ ADDQ CX,SI
+ ANDQ DX,R10
+ MOVQ SI,120(DI)
+ MOVQ R8,128(DI)
+ MOVQ R9,136(DI)
+ MOVQ AX,144(DI)
+ MOVQ R10,152(DI)
+ MOVQ 160(DI),AX
+ MULQ 160(DI)
+ MOVQ AX,SI
+ MOVQ DX,CX
+ MOVQ 160(DI),AX
+ SHLQ $1,AX
+ MULQ 168(DI)
+ MOVQ AX,R8
+ MOVQ DX,R9
+ MOVQ 160(DI),AX
+ SHLQ $1,AX
+ MULQ 176(DI)
+ MOVQ AX,R10
+ MOVQ DX,R11
+ MOVQ 160(DI),AX
+ SHLQ $1,AX
+ MULQ 184(DI)
+ MOVQ AX,R12
+ MOVQ DX,R13
+ MOVQ 160(DI),AX
+ SHLQ $1,AX
+ MULQ 192(DI)
+ MOVQ AX,R14
+ MOVQ DX,R15
+ MOVQ 168(DI),AX
+ MULQ 168(DI)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 168(DI),AX
+ SHLQ $1,AX
+ MULQ 176(DI)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ 168(DI),AX
+ SHLQ $1,AX
+ MULQ 184(DI)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 168(DI),DX
+ IMUL3Q $38,DX,AX
+ MULQ 192(DI)
+ ADDQ AX,SI
+ ADCQ DX,CX
+ MOVQ 176(DI),AX
+ MULQ 176(DI)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 176(DI),DX
+ IMUL3Q $38,DX,AX
+ MULQ 184(DI)
+ ADDQ AX,SI
+ ADCQ DX,CX
+ MOVQ 176(DI),DX
+ IMUL3Q $38,DX,AX
+ MULQ 192(DI)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 184(DI),DX
+ IMUL3Q $19,DX,AX
+ MULQ 184(DI)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 184(DI),DX
+ IMUL3Q $38,DX,AX
+ MULQ 192(DI)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 192(DI),DX
+ IMUL3Q $19,DX,AX
+ MULQ 192(DI)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ $REDMASK51,DX
+ SHLQ $13,SI,CX
+ ANDQ DX,SI
+ SHLQ $13,R8,R9
+ ANDQ DX,R8
+ ADDQ CX,R8
+ SHLQ $13,R10,R11
+ ANDQ DX,R10
+ ADDQ R9,R10
+ SHLQ $13,R12,R13
+ ANDQ DX,R12
+ ADDQ R11,R12
+ SHLQ $13,R14,R15
+ ANDQ DX,R14
+ ADDQ R13,R14
+ IMUL3Q $19,R15,CX
+ ADDQ CX,SI
+ MOVQ SI,CX
+ SHRQ $51,CX
+ ADDQ R8,CX
+ ANDQ DX,SI
+ MOVQ CX,R8
+ SHRQ $51,CX
+ ADDQ R10,CX
+ ANDQ DX,R8
+ MOVQ CX,R9
+ SHRQ $51,CX
+ ADDQ R12,CX
+ ANDQ DX,R9
+ MOVQ CX,AX
+ SHRQ $51,CX
+ ADDQ R14,CX
+ ANDQ DX,AX
+ MOVQ CX,R10
+ SHRQ $51,CX
+ IMUL3Q $19,CX,CX
+ ADDQ CX,SI
+ ANDQ DX,R10
+ MOVQ SI,160(DI)
+ MOVQ R8,168(DI)
+ MOVQ R9,176(DI)
+ MOVQ AX,184(DI)
+ MOVQ R10,192(DI)
+ MOVQ 184(DI),SI
+ IMUL3Q $19,SI,AX
+ MOVQ AX,0(SP)
+ MULQ 16(DI)
+ MOVQ AX,SI
+ MOVQ DX,CX
+ MOVQ 192(DI),DX
+ IMUL3Q $19,DX,AX
+ MOVQ AX,8(SP)
+ MULQ 8(DI)
+ ADDQ AX,SI
+ ADCQ DX,CX
+ MOVQ 160(DI),AX
+ MULQ 0(DI)
+ ADDQ AX,SI
+ ADCQ DX,CX
+ MOVQ 160(DI),AX
+ MULQ 8(DI)
+ MOVQ AX,R8
+ MOVQ DX,R9
+ MOVQ 160(DI),AX
+ MULQ 16(DI)
+ MOVQ AX,R10
+ MOVQ DX,R11
+ MOVQ 160(DI),AX
+ MULQ 24(DI)
+ MOVQ AX,R12
+ MOVQ DX,R13
+ MOVQ 160(DI),AX
+ MULQ 32(DI)
+ MOVQ AX,R14
+ MOVQ DX,R15
+ MOVQ 168(DI),AX
+ MULQ 0(DI)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 168(DI),AX
+ MULQ 8(DI)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 168(DI),AX
+ MULQ 16(DI)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ 168(DI),AX
+ MULQ 24(DI)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 168(DI),DX
+ IMUL3Q $19,DX,AX
+ MULQ 32(DI)
+ ADDQ AX,SI
+ ADCQ DX,CX
+ MOVQ 176(DI),AX
+ MULQ 0(DI)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 176(DI),AX
+ MULQ 8(DI)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ 176(DI),AX
+ MULQ 16(DI)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 176(DI),DX
+ IMUL3Q $19,DX,AX
+ MULQ 24(DI)
+ ADDQ AX,SI
+ ADCQ DX,CX
+ MOVQ 176(DI),DX
+ IMUL3Q $19,DX,AX
+ MULQ 32(DI)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 184(DI),AX
+ MULQ 0(DI)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ 184(DI),AX
+ MULQ 8(DI)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 0(SP),AX
+ MULQ 24(DI)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 0(SP),AX
+ MULQ 32(DI)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 192(DI),AX
+ MULQ 0(DI)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 8(SP),AX
+ MULQ 16(DI)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 8(SP),AX
+ MULQ 24(DI)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 8(SP),AX
+ MULQ 32(DI)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ $REDMASK51,DX
+ SHLQ $13,SI,CX
+ ANDQ DX,SI
+ SHLQ $13,R8,R9
+ ANDQ DX,R8
+ ADDQ CX,R8
+ SHLQ $13,R10,R11
+ ANDQ DX,R10
+ ADDQ R9,R10
+ SHLQ $13,R12,R13
+ ANDQ DX,R12
+ ADDQ R11,R12
+ SHLQ $13,R14,R15
+ ANDQ DX,R14
+ ADDQ R13,R14
+ IMUL3Q $19,R15,CX
+ ADDQ CX,SI
+ MOVQ SI,CX
+ SHRQ $51,CX
+ ADDQ R8,CX
+ MOVQ CX,R8
+ SHRQ $51,CX
+ ANDQ DX,SI
+ ADDQ R10,CX
+ MOVQ CX,R9
+ SHRQ $51,CX
+ ANDQ DX,R8
+ ADDQ R12,CX
+ MOVQ CX,AX
+ SHRQ $51,CX
+ ANDQ DX,R9
+ ADDQ R14,CX
+ MOVQ CX,R10
+ SHRQ $51,CX
+ ANDQ DX,AX
+ IMUL3Q $19,CX,CX
+ ADDQ CX,SI
+ ANDQ DX,R10
+ MOVQ SI,160(DI)
+ MOVQ R8,168(DI)
+ MOVQ R9,176(DI)
+ MOVQ AX,184(DI)
+ MOVQ R10,192(DI)
+ MOVQ 144(SP),SI
+ IMUL3Q $19,SI,AX
+ MOVQ AX,0(SP)
+ MULQ 96(SP)
+ MOVQ AX,SI
+ MOVQ DX,CX
+ MOVQ 152(SP),DX
+ IMUL3Q $19,DX,AX
+ MOVQ AX,8(SP)
+ MULQ 88(SP)
+ ADDQ AX,SI
+ ADCQ DX,CX
+ MOVQ 120(SP),AX
+ MULQ 80(SP)
+ ADDQ AX,SI
+ ADCQ DX,CX
+ MOVQ 120(SP),AX
+ MULQ 88(SP)
+ MOVQ AX,R8
+ MOVQ DX,R9
+ MOVQ 120(SP),AX
+ MULQ 96(SP)
+ MOVQ AX,R10
+ MOVQ DX,R11
+ MOVQ 120(SP),AX
+ MULQ 104(SP)
+ MOVQ AX,R12
+ MOVQ DX,R13
+ MOVQ 120(SP),AX
+ MULQ 112(SP)
+ MOVQ AX,R14
+ MOVQ DX,R15
+ MOVQ 128(SP),AX
+ MULQ 80(SP)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 128(SP),AX
+ MULQ 88(SP)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 128(SP),AX
+ MULQ 96(SP)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ 128(SP),AX
+ MULQ 104(SP)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 128(SP),DX
+ IMUL3Q $19,DX,AX
+ MULQ 112(SP)
+ ADDQ AX,SI
+ ADCQ DX,CX
+ MOVQ 136(SP),AX
+ MULQ 80(SP)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 136(SP),AX
+ MULQ 88(SP)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ 136(SP),AX
+ MULQ 96(SP)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 136(SP),DX
+ IMUL3Q $19,DX,AX
+ MULQ 104(SP)
+ ADDQ AX,SI
+ ADCQ DX,CX
+ MOVQ 136(SP),DX
+ IMUL3Q $19,DX,AX
+ MULQ 112(SP)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 144(SP),AX
+ MULQ 80(SP)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ 144(SP),AX
+ MULQ 88(SP)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 0(SP),AX
+ MULQ 104(SP)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 0(SP),AX
+ MULQ 112(SP)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 152(SP),AX
+ MULQ 80(SP)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 8(SP),AX
+ MULQ 96(SP)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 8(SP),AX
+ MULQ 104(SP)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 8(SP),AX
+ MULQ 112(SP)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ $REDMASK51,DX
+ SHLQ $13,SI,CX
+ ANDQ DX,SI
+ SHLQ $13,R8,R9
+ ANDQ DX,R8
+ ADDQ CX,R8
+ SHLQ $13,R10,R11
+ ANDQ DX,R10
+ ADDQ R9,R10
+ SHLQ $13,R12,R13
+ ANDQ DX,R12
+ ADDQ R11,R12
+ SHLQ $13,R14,R15
+ ANDQ DX,R14
+ ADDQ R13,R14
+ IMUL3Q $19,R15,CX
+ ADDQ CX,SI
+ MOVQ SI,CX
+ SHRQ $51,CX
+ ADDQ R8,CX
+ MOVQ CX,R8
+ SHRQ $51,CX
+ ANDQ DX,SI
+ ADDQ R10,CX
+ MOVQ CX,R9
+ SHRQ $51,CX
+ ANDQ DX,R8
+ ADDQ R12,CX
+ MOVQ CX,AX
+ SHRQ $51,CX
+ ANDQ DX,R9
+ ADDQ R14,CX
+ MOVQ CX,R10
+ SHRQ $51,CX
+ ANDQ DX,AX
+ IMUL3Q $19,CX,CX
+ ADDQ CX,SI
+ ANDQ DX,R10
+ MOVQ SI,40(DI)
+ MOVQ R8,48(DI)
+ MOVQ R9,56(DI)
+ MOVQ AX,64(DI)
+ MOVQ R10,72(DI)
+ MOVQ 160(SP),AX
+ MULQ ·_121666_213(SB)
+ SHRQ $13,AX
+ MOVQ AX,SI
+ MOVQ DX,CX
+ MOVQ 168(SP),AX
+ MULQ ·_121666_213(SB)
+ SHRQ $13,AX
+ ADDQ AX,CX
+ MOVQ DX,R8
+ MOVQ 176(SP),AX
+ MULQ ·_121666_213(SB)
+ SHRQ $13,AX
+ ADDQ AX,R8
+ MOVQ DX,R9
+ MOVQ 184(SP),AX
+ MULQ ·_121666_213(SB)
+ SHRQ $13,AX
+ ADDQ AX,R9
+ MOVQ DX,R10
+ MOVQ 192(SP),AX
+ MULQ ·_121666_213(SB)
+ SHRQ $13,AX
+ ADDQ AX,R10
+ IMUL3Q $19,DX,DX
+ ADDQ DX,SI
+ ADDQ 80(SP),SI
+ ADDQ 88(SP),CX
+ ADDQ 96(SP),R8
+ ADDQ 104(SP),R9
+ ADDQ 112(SP),R10
+ MOVQ SI,80(DI)
+ MOVQ CX,88(DI)
+ MOVQ R8,96(DI)
+ MOVQ R9,104(DI)
+ MOVQ R10,112(DI)
+ MOVQ 104(DI),SI
+ IMUL3Q $19,SI,AX
+ MOVQ AX,0(SP)
+ MULQ 176(SP)
+ MOVQ AX,SI
+ MOVQ DX,CX
+ MOVQ 112(DI),DX
+ IMUL3Q $19,DX,AX
+ MOVQ AX,8(SP)
+ MULQ 168(SP)
+ ADDQ AX,SI
+ ADCQ DX,CX
+ MOVQ 80(DI),AX
+ MULQ 160(SP)
+ ADDQ AX,SI
+ ADCQ DX,CX
+ MOVQ 80(DI),AX
+ MULQ 168(SP)
+ MOVQ AX,R8
+ MOVQ DX,R9
+ MOVQ 80(DI),AX
+ MULQ 176(SP)
+ MOVQ AX,R10
+ MOVQ DX,R11
+ MOVQ 80(DI),AX
+ MULQ 184(SP)
+ MOVQ AX,R12
+ MOVQ DX,R13
+ MOVQ 80(DI),AX
+ MULQ 192(SP)
+ MOVQ AX,R14
+ MOVQ DX,R15
+ MOVQ 88(DI),AX
+ MULQ 160(SP)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 88(DI),AX
+ MULQ 168(SP)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 88(DI),AX
+ MULQ 176(SP)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ 88(DI),AX
+ MULQ 184(SP)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 88(DI),DX
+ IMUL3Q $19,DX,AX
+ MULQ 192(SP)
+ ADDQ AX,SI
+ ADCQ DX,CX
+ MOVQ 96(DI),AX
+ MULQ 160(SP)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 96(DI),AX
+ MULQ 168(SP)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ 96(DI),AX
+ MULQ 176(SP)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 96(DI),DX
+ IMUL3Q $19,DX,AX
+ MULQ 184(SP)
+ ADDQ AX,SI
+ ADCQ DX,CX
+ MOVQ 96(DI),DX
+ IMUL3Q $19,DX,AX
+ MULQ 192(SP)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 104(DI),AX
+ MULQ 160(SP)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ 104(DI),AX
+ MULQ 168(SP)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 0(SP),AX
+ MULQ 184(SP)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 0(SP),AX
+ MULQ 192(SP)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 112(DI),AX
+ MULQ 160(SP)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 8(SP),AX
+ MULQ 176(SP)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 8(SP),AX
+ MULQ 184(SP)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 8(SP),AX
+ MULQ 192(SP)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ $REDMASK51,DX
+ SHLQ $13,SI,CX
+ ANDQ DX,SI
+ SHLQ $13,R8,R9
+ ANDQ DX,R8
+ ADDQ CX,R8
+ SHLQ $13,R10,R11
+ ANDQ DX,R10
+ ADDQ R9,R10
+ SHLQ $13,R12,R13
+ ANDQ DX,R12
+ ADDQ R11,R12
+ SHLQ $13,R14,R15
+ ANDQ DX,R14
+ ADDQ R13,R14
+ IMUL3Q $19,R15,CX
+ ADDQ CX,SI
+ MOVQ SI,CX
+ SHRQ $51,CX
+ ADDQ R8,CX
+ MOVQ CX,R8
+ SHRQ $51,CX
+ ANDQ DX,SI
+ ADDQ R10,CX
+ MOVQ CX,R9
+ SHRQ $51,CX
+ ANDQ DX,R8
+ ADDQ R12,CX
+ MOVQ CX,AX
+ SHRQ $51,CX
+ ANDQ DX,R9
+ ADDQ R14,CX
+ MOVQ CX,R10
+ SHRQ $51,CX
+ ANDQ DX,AX
+ IMUL3Q $19,CX,CX
+ ADDQ CX,SI
+ ANDQ DX,R10
+ MOVQ SI,80(DI)
+ MOVQ R8,88(DI)
+ MOVQ R9,96(DI)
+ MOVQ AX,104(DI)
+ MOVQ R10,112(DI)
+ RET
+
+// func cswap(inout *[4][5]uint64, v uint64)
+TEXT ·cswap(SB),7,$0
+ MOVQ inout+0(FP),DI
+ MOVQ v+8(FP),SI
+
+ SUBQ $1, SI
+ NOTQ SI
+ MOVQ SI, X15
+ PSHUFD $0x44, X15, X15
+
+ MOVOU 0(DI), X0
+ MOVOU 16(DI), X2
+ MOVOU 32(DI), X4
+ MOVOU 48(DI), X6
+ MOVOU 64(DI), X8
+ MOVOU 80(DI), X1
+ MOVOU 96(DI), X3
+ MOVOU 112(DI), X5
+ MOVOU 128(DI), X7
+ MOVOU 144(DI), X9
+
+ MOVO X1, X10
+ MOVO X3, X11
+ MOVO X5, X12
+ MOVO X7, X13
+ MOVO X9, X14
+
+ PXOR X0, X10
+ PXOR X2, X11
+ PXOR X4, X12
+ PXOR X6, X13
+ PXOR X8, X14
+ PAND X15, X10
+ PAND X15, X11
+ PAND X15, X12
+ PAND X15, X13
+ PAND X15, X14
+ PXOR X10, X0
+ PXOR X10, X1
+ PXOR X11, X2
+ PXOR X11, X3
+ PXOR X12, X4
+ PXOR X12, X5
+ PXOR X13, X6
+ PXOR X13, X7
+ PXOR X14, X8
+ PXOR X14, X9
+
+ MOVOU X0, 0(DI)
+ MOVOU X2, 16(DI)
+ MOVOU X4, 32(DI)
+ MOVOU X6, 48(DI)
+ MOVOU X8, 64(DI)
+ MOVOU X1, 80(DI)
+ MOVOU X3, 96(DI)
+ MOVOU X5, 112(DI)
+ MOVOU X7, 128(DI)
+ MOVOU X9, 144(DI)
+ RET
+
+// func mul(dest, a, b *[5]uint64)
+TEXT ·mul(SB),0,$16-24
+ MOVQ dest+0(FP), DI
+ MOVQ a+8(FP), SI
+ MOVQ b+16(FP), DX
+
+ MOVQ DX,CX
+ MOVQ 24(SI),DX
+ IMUL3Q $19,DX,AX
+ MOVQ AX,0(SP)
+ MULQ 16(CX)
+ MOVQ AX,R8
+ MOVQ DX,R9
+ MOVQ 32(SI),DX
+ IMUL3Q $19,DX,AX
+ MOVQ AX,8(SP)
+ MULQ 8(CX)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 0(SI),AX
+ MULQ 0(CX)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 0(SI),AX
+ MULQ 8(CX)
+ MOVQ AX,R10
+ MOVQ DX,R11
+ MOVQ 0(SI),AX
+ MULQ 16(CX)
+ MOVQ AX,R12
+ MOVQ DX,R13
+ MOVQ 0(SI),AX
+ MULQ 24(CX)
+ MOVQ AX,R14
+ MOVQ DX,R15
+ MOVQ 0(SI),AX
+ MULQ 32(CX)
+ MOVQ AX,BX
+ MOVQ DX,BP
+ MOVQ 8(SI),AX
+ MULQ 0(CX)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 8(SI),AX
+ MULQ 8(CX)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ 8(SI),AX
+ MULQ 16(CX)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 8(SI),AX
+ MULQ 24(CX)
+ ADDQ AX,BX
+ ADCQ DX,BP
+ MOVQ 8(SI),DX
+ IMUL3Q $19,DX,AX
+ MULQ 32(CX)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 16(SI),AX
+ MULQ 0(CX)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ 16(SI),AX
+ MULQ 8(CX)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 16(SI),AX
+ MULQ 16(CX)
+ ADDQ AX,BX
+ ADCQ DX,BP
+ MOVQ 16(SI),DX
+ IMUL3Q $19,DX,AX
+ MULQ 24(CX)
+ ADDQ AX,R8
+ ADCQ DX,R9
+ MOVQ 16(SI),DX
+ IMUL3Q $19,DX,AX
+ MULQ 32(CX)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 24(SI),AX
+ MULQ 0(CX)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ 24(SI),AX
+ MULQ 8(CX)
+ ADDQ AX,BX
+ ADCQ DX,BP
+ MOVQ 0(SP),AX
+ MULQ 24(CX)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 0(SP),AX
+ MULQ 32(CX)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ 32(SI),AX
+ MULQ 0(CX)
+ ADDQ AX,BX
+ ADCQ DX,BP
+ MOVQ 8(SP),AX
+ MULQ 16(CX)
+ ADDQ AX,R10
+ ADCQ DX,R11
+ MOVQ 8(SP),AX
+ MULQ 24(CX)
+ ADDQ AX,R12
+ ADCQ DX,R13
+ MOVQ 8(SP),AX
+ MULQ 32(CX)
+ ADDQ AX,R14
+ ADCQ DX,R15
+ MOVQ $REDMASK51,SI
+ SHLQ $13,R8,R9
+ ANDQ SI,R8
+ SHLQ $13,R10,R11
+ ANDQ SI,R10
+ ADDQ R9,R10
+ SHLQ $13,R12,R13
+ ANDQ SI,R12
+ ADDQ R11,R12
+ SHLQ $13,R14,R15
+ ANDQ SI,R14
+ ADDQ R13,R14
+ SHLQ $13,BX,BP
+ ANDQ SI,BX
+ ADDQ R15,BX
+ IMUL3Q $19,BP,DX
+ ADDQ DX,R8
+ MOVQ R8,DX
+ SHRQ $51,DX
+ ADDQ R10,DX
+ MOVQ DX,CX
+ SHRQ $51,DX
+ ANDQ SI,R8
+ ADDQ R12,DX
+ MOVQ DX,R9
+ SHRQ $51,DX
+ ANDQ SI,CX
+ ADDQ R14,DX
+ MOVQ DX,AX
+ SHRQ $51,DX
+ ANDQ SI,R9
+ ADDQ BX,DX
+ MOVQ DX,R10
+ SHRQ $51,DX
+ ANDQ SI,AX
+ IMUL3Q $19,DX,DX
+ ADDQ DX,R8
+ ANDQ SI,R10
+ MOVQ R8,0(DI)
+ MOVQ CX,8(DI)
+ MOVQ R9,16(DI)
+ MOVQ AX,24(DI)
+ MOVQ R10,32(DI)
+ RET
+
+// func square(out, in *[5]uint64)
+TEXT ·square(SB),7,$0-16
+ MOVQ out+0(FP), DI
+ MOVQ in+8(FP), SI
+
+ MOVQ 0(SI),AX
+ MULQ 0(SI)
+ MOVQ AX,CX
+ MOVQ DX,R8
+ MOVQ 0(SI),AX
+ SHLQ $1,AX
+ MULQ 8(SI)
+ MOVQ AX,R9
+ MOVQ DX,R10
+ MOVQ 0(SI),AX
+ SHLQ $1,AX
+ MULQ 16(SI)
+ MOVQ AX,R11
+ MOVQ DX,R12
+ MOVQ 0(SI),AX
+ SHLQ $1,AX
+ MULQ 24(SI)
+ MOVQ AX,R13
+ MOVQ DX,R14
+ MOVQ 0(SI),AX
+ SHLQ $1,AX
+ MULQ 32(SI)
+ MOVQ AX,R15
+ MOVQ DX,BX
+ MOVQ 8(SI),AX
+ MULQ 8(SI)
+ ADDQ AX,R11
+ ADCQ DX,R12
+ MOVQ 8(SI),AX
+ SHLQ $1,AX
+ MULQ 16(SI)
+ ADDQ AX,R13
+ ADCQ DX,R14
+ MOVQ 8(SI),AX
+ SHLQ $1,AX
+ MULQ 24(SI)
+ ADDQ AX,R15
+ ADCQ DX,BX
+ MOVQ 8(SI),DX
+ IMUL3Q $38,DX,AX
+ MULQ 32(SI)
+ ADDQ AX,CX
+ ADCQ DX,R8
+ MOVQ 16(SI),AX
+ MULQ 16(SI)
+ ADDQ AX,R15
+ ADCQ DX,BX
+ MOVQ 16(SI),DX
+ IMUL3Q $38,DX,AX
+ MULQ 24(SI)
+ ADDQ AX,CX
+ ADCQ DX,R8
+ MOVQ 16(SI),DX
+ IMUL3Q $38,DX,AX
+ MULQ 32(SI)
+ ADDQ AX,R9
+ ADCQ DX,R10
+ MOVQ 24(SI),DX
+ IMUL3Q $19,DX,AX
+ MULQ 24(SI)
+ ADDQ AX,R9
+ ADCQ DX,R10
+ MOVQ 24(SI),DX
+ IMUL3Q $38,DX,AX
+ MULQ 32(SI)
+ ADDQ AX,R11
+ ADCQ DX,R12
+ MOVQ 32(SI),DX
+ IMUL3Q $19,DX,AX
+ MULQ 32(SI)
+ ADDQ AX,R13
+ ADCQ DX,R14
+ MOVQ $REDMASK51,SI
+ SHLQ $13,CX,R8
+ ANDQ SI,CX
+ SHLQ $13,R9,R10
+ ANDQ SI,R9
+ ADDQ R8,R9
+ SHLQ $13,R11,R12
+ ANDQ SI,R11
+ ADDQ R10,R11
+ SHLQ $13,R13,R14
+ ANDQ SI,R13
+ ADDQ R12,R13
+ SHLQ $13,R15,BX
+ ANDQ SI,R15
+ ADDQ R14,R15
+ IMUL3Q $19,BX,DX
+ ADDQ DX,CX
+ MOVQ CX,DX
+ SHRQ $51,DX
+ ADDQ R9,DX
+ ANDQ SI,CX
+ MOVQ DX,R8
+ SHRQ $51,DX
+ ADDQ R11,DX
+ ANDQ SI,R8
+ MOVQ DX,R9
+ SHRQ $51,DX
+ ADDQ R13,DX
+ ANDQ SI,R9
+ MOVQ DX,AX
+ SHRQ $51,DX
+ ADDQ R15,DX
+ ANDQ SI,AX
+ MOVQ DX,R10
+ SHRQ $51,DX
+ IMUL3Q $19,DX,DX
+ ADDQ DX,CX
+ ANDQ SI,R10
+ MOVQ CX,0(DI)
+ MOVQ R8,8(DI)
+ MOVQ R9,16(DI)
+ MOVQ AX,24(DI)
+ MOVQ R10,32(DI)
+ RET
--- /dev/null
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package curve25519
+
+import "encoding/binary"
+
+// This code is a port of the public domain, "ref10" implementation of
+// curve25519 from SUPERCOP 20130419 by D. J. Bernstein.
+
+// fieldElement represents an element of the field GF(2^255 - 19). An element
+// t, entries t[0]...t[9], represents the integer t[0]+2^26 t[1]+2^51 t[2]+2^77
+// t[3]+2^102 t[4]+...+2^230 t[9]. Bounds on each t[i] vary depending on
+// context.
+type fieldElement [10]int32
+
+func feZero(fe *fieldElement) {
+ for i := range fe {
+ fe[i] = 0
+ }
+}
+
+func feOne(fe *fieldElement) {
+ feZero(fe)
+ fe[0] = 1
+}
+
+func feAdd(dst, a, b *fieldElement) {
+ for i := range dst {
+ dst[i] = a[i] + b[i]
+ }
+}
+
+func feSub(dst, a, b *fieldElement) {
+ for i := range dst {
+ dst[i] = a[i] - b[i]
+ }
+}
+
+func feCopy(dst, src *fieldElement) {
+ for i := range dst {
+ dst[i] = src[i]
+ }
+}
+
+// feCSwap replaces (f,g) with (g,f) if b == 1; replaces (f,g) with (f,g) if b == 0.
+//
+// Preconditions: b in {0,1}.
+func feCSwap(f, g *fieldElement, b int32) {
+ b = -b
+ for i := range f {
+ t := b & (f[i] ^ g[i])
+ f[i] ^= t
+ g[i] ^= t
+ }
+}
+
+// load3 reads a 24-bit, little-endian value from in.
+func load3(in []byte) int64 {
+ var r int64
+ r = int64(in[0])
+ r |= int64(in[1]) << 8
+ r |= int64(in[2]) << 16
+ return r
+}
+
+// load4 reads a 32-bit, little-endian value from in.
+func load4(in []byte) int64 {
+ return int64(binary.LittleEndian.Uint32(in))
+}
+
+func feFromBytes(dst *fieldElement, src *[32]byte) {
+ h0 := load4(src[:])
+ h1 := load3(src[4:]) << 6
+ h2 := load3(src[7:]) << 5
+ h3 := load3(src[10:]) << 3
+ h4 := load3(src[13:]) << 2
+ h5 := load4(src[16:])
+ h6 := load3(src[20:]) << 7
+ h7 := load3(src[23:]) << 5
+ h8 := load3(src[26:]) << 4
+ h9 := (load3(src[29:]) & 0x7fffff) << 2
+
+ var carry [10]int64
+ carry[9] = (h9 + 1<<24) >> 25
+ h0 += carry[9] * 19
+ h9 -= carry[9] << 25
+ carry[1] = (h1 + 1<<24) >> 25
+ h2 += carry[1]
+ h1 -= carry[1] << 25
+ carry[3] = (h3 + 1<<24) >> 25
+ h4 += carry[3]
+ h3 -= carry[3] << 25
+ carry[5] = (h5 + 1<<24) >> 25
+ h6 += carry[5]
+ h5 -= carry[5] << 25
+ carry[7] = (h7 + 1<<24) >> 25
+ h8 += carry[7]
+ h7 -= carry[7] << 25
+
+ carry[0] = (h0 + 1<<25) >> 26
+ h1 += carry[0]
+ h0 -= carry[0] << 26
+ carry[2] = (h2 + 1<<25) >> 26
+ h3 += carry[2]
+ h2 -= carry[2] << 26
+ carry[4] = (h4 + 1<<25) >> 26
+ h5 += carry[4]
+ h4 -= carry[4] << 26
+ carry[6] = (h6 + 1<<25) >> 26
+ h7 += carry[6]
+ h6 -= carry[6] << 26
+ carry[8] = (h8 + 1<<25) >> 26
+ h9 += carry[8]
+ h8 -= carry[8] << 26
+
+ dst[0] = int32(h0)
+ dst[1] = int32(h1)
+ dst[2] = int32(h2)
+ dst[3] = int32(h3)
+ dst[4] = int32(h4)
+ dst[5] = int32(h5)
+ dst[6] = int32(h6)
+ dst[7] = int32(h7)
+ dst[8] = int32(h8)
+ dst[9] = int32(h9)
+}
+
+// feToBytes marshals h to s.
+// Preconditions:
+// |h| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc.
+//
+// Write p=2^255-19; q=floor(h/p).
+// Basic claim: q = floor(2^(-255)(h + 19 2^(-25)h9 + 2^(-1))).
+//
+// Proof:
+// Have |h|<=p so |q|<=1 so |19^2 2^(-255) q|<1/4.
+// Also have |h-2^230 h9|<2^230 so |19 2^(-255)(h-2^230 h9)|<1/4.
+//
+// Write y=2^(-1)-19^2 2^(-255)q-19 2^(-255)(h-2^230 h9).
+// Then 0<y<1.
+//
+// Write r=h-pq.
+// Have 0<=r<=p-1=2^255-20.
+// Thus 0<=r+19(2^-255)r<r+19(2^-255)2^255<=2^255-1.
+//
+// Write x=r+19(2^-255)r+y.
+// Then 0<x<2^255 so floor(2^(-255)x) = 0 so floor(q+2^(-255)x) = q.
+//
+// Have q+2^(-255)x = 2^(-255)(h + 19 2^(-25) h9 + 2^(-1))
+// so floor(2^(-255)(h + 19 2^(-25) h9 + 2^(-1))) = q.
+func feToBytes(s *[32]byte, h *fieldElement) {
+ var carry [10]int32
+
+ q := (19*h[9] + (1 << 24)) >> 25
+ q = (h[0] + q) >> 26
+ q = (h[1] + q) >> 25
+ q = (h[2] + q) >> 26
+ q = (h[3] + q) >> 25
+ q = (h[4] + q) >> 26
+ q = (h[5] + q) >> 25
+ q = (h[6] + q) >> 26
+ q = (h[7] + q) >> 25
+ q = (h[8] + q) >> 26
+ q = (h[9] + q) >> 25
+
+ // Goal: Output h-(2^255-19)q, which is between 0 and 2^255-20.
+ h[0] += 19 * q
+ // Goal: Output h-2^255 q, which is between 0 and 2^255-20.
+
+ carry[0] = h[0] >> 26
+ h[1] += carry[0]
+ h[0] -= carry[0] << 26
+ carry[1] = h[1] >> 25
+ h[2] += carry[1]
+ h[1] -= carry[1] << 25
+ carry[2] = h[2] >> 26
+ h[3] += carry[2]
+ h[2] -= carry[2] << 26
+ carry[3] = h[3] >> 25
+ h[4] += carry[3]
+ h[3] -= carry[3] << 25
+ carry[4] = h[4] >> 26
+ h[5] += carry[4]
+ h[4] -= carry[4] << 26
+ carry[5] = h[5] >> 25
+ h[6] += carry[5]
+ h[5] -= carry[5] << 25
+ carry[6] = h[6] >> 26
+ h[7] += carry[6]
+ h[6] -= carry[6] << 26
+ carry[7] = h[7] >> 25
+ h[8] += carry[7]
+ h[7] -= carry[7] << 25
+ carry[8] = h[8] >> 26
+ h[9] += carry[8]
+ h[8] -= carry[8] << 26
+ carry[9] = h[9] >> 25
+ h[9] -= carry[9] << 25
+ // h10 = carry9
+
+ // Goal: Output h[0]+...+2^255 h10-2^255 q, which is between 0 and 2^255-20.
+ // Have h[0]+...+2^230 h[9] between 0 and 2^255-1;
+ // evidently 2^255 h10-2^255 q = 0.
+ // Goal: Output h[0]+...+2^230 h[9].
+
+ s[0] = byte(h[0] >> 0)
+ s[1] = byte(h[0] >> 8)
+ s[2] = byte(h[0] >> 16)
+ s[3] = byte((h[0] >> 24) | (h[1] << 2))
+ s[4] = byte(h[1] >> 6)
+ s[5] = byte(h[1] >> 14)
+ s[6] = byte((h[1] >> 22) | (h[2] << 3))
+ s[7] = byte(h[2] >> 5)
+ s[8] = byte(h[2] >> 13)
+ s[9] = byte((h[2] >> 21) | (h[3] << 5))
+ s[10] = byte(h[3] >> 3)
+ s[11] = byte(h[3] >> 11)
+ s[12] = byte((h[3] >> 19) | (h[4] << 6))
+ s[13] = byte(h[4] >> 2)
+ s[14] = byte(h[4] >> 10)
+ s[15] = byte(h[4] >> 18)
+ s[16] = byte(h[5] >> 0)
+ s[17] = byte(h[5] >> 8)
+ s[18] = byte(h[5] >> 16)
+ s[19] = byte((h[5] >> 24) | (h[6] << 1))
+ s[20] = byte(h[6] >> 7)
+ s[21] = byte(h[6] >> 15)
+ s[22] = byte((h[6] >> 23) | (h[7] << 3))
+ s[23] = byte(h[7] >> 5)
+ s[24] = byte(h[7] >> 13)
+ s[25] = byte((h[7] >> 21) | (h[8] << 4))
+ s[26] = byte(h[8] >> 4)
+ s[27] = byte(h[8] >> 12)
+ s[28] = byte((h[8] >> 20) | (h[9] << 6))
+ s[29] = byte(h[9] >> 2)
+ s[30] = byte(h[9] >> 10)
+ s[31] = byte(h[9] >> 18)
+}
+
+// feMul calculates h = f * g
+// Can overlap h with f or g.
+//
+// Preconditions:
+// |f| bounded by 1.1*2^26,1.1*2^25,1.1*2^26,1.1*2^25,etc.
+// |g| bounded by 1.1*2^26,1.1*2^25,1.1*2^26,1.1*2^25,etc.
+//
+// Postconditions:
+// |h| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc.
+//
+// Notes on implementation strategy:
+//
+// Using schoolbook multiplication.
+// Karatsuba would save a little in some cost models.
+//
+// Most multiplications by 2 and 19 are 32-bit precomputations;
+// cheaper than 64-bit postcomputations.
+//
+// There is one remaining multiplication by 19 in the carry chain;
+// one *19 precomputation can be merged into this,
+// but the resulting data flow is considerably less clean.
+//
+// There are 12 carries below.
+// 10 of them are 2-way parallelizable and vectorizable.
+// Can get away with 11 carries, but then data flow is much deeper.
+//
+// With tighter constraints on inputs can squeeze carries into int32.
+func feMul(h, f, g *fieldElement) {
+ f0 := f[0]
+ f1 := f[1]
+ f2 := f[2]
+ f3 := f[3]
+ f4 := f[4]
+ f5 := f[5]
+ f6 := f[6]
+ f7 := f[7]
+ f8 := f[8]
+ f9 := f[9]
+ g0 := g[0]
+ g1 := g[1]
+ g2 := g[2]
+ g3 := g[3]
+ g4 := g[4]
+ g5 := g[5]
+ g6 := g[6]
+ g7 := g[7]
+ g8 := g[8]
+ g9 := g[9]
+ g1_19 := 19 * g1 // 1.4*2^29
+ g2_19 := 19 * g2 // 1.4*2^30; still ok
+ g3_19 := 19 * g3
+ g4_19 := 19 * g4
+ g5_19 := 19 * g5
+ g6_19 := 19 * g6
+ g7_19 := 19 * g7
+ g8_19 := 19 * g8
+ g9_19 := 19 * g9
+ f1_2 := 2 * f1
+ f3_2 := 2 * f3
+ f5_2 := 2 * f5
+ f7_2 := 2 * f7
+ f9_2 := 2 * f9
+ f0g0 := int64(f0) * int64(g0)
+ f0g1 := int64(f0) * int64(g1)
+ f0g2 := int64(f0) * int64(g2)
+ f0g3 := int64(f0) * int64(g3)
+ f0g4 := int64(f0) * int64(g4)
+ f0g5 := int64(f0) * int64(g5)
+ f0g6 := int64(f0) * int64(g6)
+ f0g7 := int64(f0) * int64(g7)
+ f0g8 := int64(f0) * int64(g8)
+ f0g9 := int64(f0) * int64(g9)
+ f1g0 := int64(f1) * int64(g0)
+ f1g1_2 := int64(f1_2) * int64(g1)
+ f1g2 := int64(f1) * int64(g2)
+ f1g3_2 := int64(f1_2) * int64(g3)
+ f1g4 := int64(f1) * int64(g4)
+ f1g5_2 := int64(f1_2) * int64(g5)
+ f1g6 := int64(f1) * int64(g6)
+ f1g7_2 := int64(f1_2) * int64(g7)
+ f1g8 := int64(f1) * int64(g8)
+ f1g9_38 := int64(f1_2) * int64(g9_19)
+ f2g0 := int64(f2) * int64(g0)
+ f2g1 := int64(f2) * int64(g1)
+ f2g2 := int64(f2) * int64(g2)
+ f2g3 := int64(f2) * int64(g3)
+ f2g4 := int64(f2) * int64(g4)
+ f2g5 := int64(f2) * int64(g5)
+ f2g6 := int64(f2) * int64(g6)
+ f2g7 := int64(f2) * int64(g7)
+ f2g8_19 := int64(f2) * int64(g8_19)
+ f2g9_19 := int64(f2) * int64(g9_19)
+ f3g0 := int64(f3) * int64(g0)
+ f3g1_2 := int64(f3_2) * int64(g1)
+ f3g2 := int64(f3) * int64(g2)
+ f3g3_2 := int64(f3_2) * int64(g3)
+ f3g4 := int64(f3) * int64(g4)
+ f3g5_2 := int64(f3_2) * int64(g5)
+ f3g6 := int64(f3) * int64(g6)
+ f3g7_38 := int64(f3_2) * int64(g7_19)
+ f3g8_19 := int64(f3) * int64(g8_19)
+ f3g9_38 := int64(f3_2) * int64(g9_19)
+ f4g0 := int64(f4) * int64(g0)
+ f4g1 := int64(f4) * int64(g1)
+ f4g2 := int64(f4) * int64(g2)
+ f4g3 := int64(f4) * int64(g3)
+ f4g4 := int64(f4) * int64(g4)
+ f4g5 := int64(f4) * int64(g5)
+ f4g6_19 := int64(f4) * int64(g6_19)
+ f4g7_19 := int64(f4) * int64(g7_19)
+ f4g8_19 := int64(f4) * int64(g8_19)
+ f4g9_19 := int64(f4) * int64(g9_19)
+ f5g0 := int64(f5) * int64(g0)
+ f5g1_2 := int64(f5_2) * int64(g1)
+ f5g2 := int64(f5) * int64(g2)
+ f5g3_2 := int64(f5_2) * int64(g3)
+ f5g4 := int64(f5) * int64(g4)
+ f5g5_38 := int64(f5_2) * int64(g5_19)
+ f5g6_19 := int64(f5) * int64(g6_19)
+ f5g7_38 := int64(f5_2) * int64(g7_19)
+ f5g8_19 := int64(f5) * int64(g8_19)
+ f5g9_38 := int64(f5_2) * int64(g9_19)
+ f6g0 := int64(f6) * int64(g0)
+ f6g1 := int64(f6) * int64(g1)
+ f6g2 := int64(f6) * int64(g2)
+ f6g3 := int64(f6) * int64(g3)
+ f6g4_19 := int64(f6) * int64(g4_19)
+ f6g5_19 := int64(f6) * int64(g5_19)
+ f6g6_19 := int64(f6) * int64(g6_19)
+ f6g7_19 := int64(f6) * int64(g7_19)
+ f6g8_19 := int64(f6) * int64(g8_19)
+ f6g9_19 := int64(f6) * int64(g9_19)
+ f7g0 := int64(f7) * int64(g0)
+ f7g1_2 := int64(f7_2) * int64(g1)
+ f7g2 := int64(f7) * int64(g2)
+ f7g3_38 := int64(f7_2) * int64(g3_19)
+ f7g4_19 := int64(f7) * int64(g4_19)
+ f7g5_38 := int64(f7_2) * int64(g5_19)
+ f7g6_19 := int64(f7) * int64(g6_19)
+ f7g7_38 := int64(f7_2) * int64(g7_19)
+ f7g8_19 := int64(f7) * int64(g8_19)
+ f7g9_38 := int64(f7_2) * int64(g9_19)
+ f8g0 := int64(f8) * int64(g0)
+ f8g1 := int64(f8) * int64(g1)
+ f8g2_19 := int64(f8) * int64(g2_19)
+ f8g3_19 := int64(f8) * int64(g3_19)
+ f8g4_19 := int64(f8) * int64(g4_19)
+ f8g5_19 := int64(f8) * int64(g5_19)
+ f8g6_19 := int64(f8) * int64(g6_19)
+ f8g7_19 := int64(f8) * int64(g7_19)
+ f8g8_19 := int64(f8) * int64(g8_19)
+ f8g9_19 := int64(f8) * int64(g9_19)
+ f9g0 := int64(f9) * int64(g0)
+ f9g1_38 := int64(f9_2) * int64(g1_19)
+ f9g2_19 := int64(f9) * int64(g2_19)
+ f9g3_38 := int64(f9_2) * int64(g3_19)
+ f9g4_19 := int64(f9) * int64(g4_19)
+ f9g5_38 := int64(f9_2) * int64(g5_19)
+ f9g6_19 := int64(f9) * int64(g6_19)
+ f9g7_38 := int64(f9_2) * int64(g7_19)
+ f9g8_19 := int64(f9) * int64(g8_19)
+ f9g9_38 := int64(f9_2) * int64(g9_19)
+ h0 := f0g0 + f1g9_38 + f2g8_19 + f3g7_38 + f4g6_19 + f5g5_38 + f6g4_19 + f7g3_38 + f8g2_19 + f9g1_38
+ h1 := f0g1 + f1g0 + f2g9_19 + f3g8_19 + f4g7_19 + f5g6_19 + f6g5_19 + f7g4_19 + f8g3_19 + f9g2_19
+ h2 := f0g2 + f1g1_2 + f2g0 + f3g9_38 + f4g8_19 + f5g7_38 + f6g6_19 + f7g5_38 + f8g4_19 + f9g3_38
+ h3 := f0g3 + f1g2 + f2g1 + f3g0 + f4g9_19 + f5g8_19 + f6g7_19 + f7g6_19 + f8g5_19 + f9g4_19
+ h4 := f0g4 + f1g3_2 + f2g2 + f3g1_2 + f4g0 + f5g9_38 + f6g8_19 + f7g7_38 + f8g6_19 + f9g5_38
+ h5 := f0g5 + f1g4 + f2g3 + f3g2 + f4g1 + f5g0 + f6g9_19 + f7g8_19 + f8g7_19 + f9g6_19
+ h6 := f0g6 + f1g5_2 + f2g4 + f3g3_2 + f4g2 + f5g1_2 + f6g0 + f7g9_38 + f8g8_19 + f9g7_38
+ h7 := f0g7 + f1g6 + f2g5 + f3g4 + f4g3 + f5g2 + f6g1 + f7g0 + f8g9_19 + f9g8_19
+ h8 := f0g8 + f1g7_2 + f2g6 + f3g5_2 + f4g4 + f5g3_2 + f6g2 + f7g1_2 + f8g0 + f9g9_38
+ h9 := f0g9 + f1g8 + f2g7 + f3g6 + f4g5 + f5g4 + f6g3 + f7g2 + f8g1 + f9g0
+ var carry [10]int64
+
+ // |h0| <= (1.1*1.1*2^52*(1+19+19+19+19)+1.1*1.1*2^50*(38+38+38+38+38))
+ // i.e. |h0| <= 1.2*2^59; narrower ranges for h2, h4, h6, h8
+ // |h1| <= (1.1*1.1*2^51*(1+1+19+19+19+19+19+19+19+19))
+ // i.e. |h1| <= 1.5*2^58; narrower ranges for h3, h5, h7, h9
+
+ carry[0] = (h0 + (1 << 25)) >> 26
+ h1 += carry[0]
+ h0 -= carry[0] << 26
+ carry[4] = (h4 + (1 << 25)) >> 26
+ h5 += carry[4]
+ h4 -= carry[4] << 26
+ // |h0| <= 2^25
+ // |h4| <= 2^25
+ // |h1| <= 1.51*2^58
+ // |h5| <= 1.51*2^58
+
+ carry[1] = (h1 + (1 << 24)) >> 25
+ h2 += carry[1]
+ h1 -= carry[1] << 25
+ carry[5] = (h5 + (1 << 24)) >> 25
+ h6 += carry[5]
+ h5 -= carry[5] << 25
+ // |h1| <= 2^24; from now on fits into int32
+ // |h5| <= 2^24; from now on fits into int32
+ // |h2| <= 1.21*2^59
+ // |h6| <= 1.21*2^59
+
+ carry[2] = (h2 + (1 << 25)) >> 26
+ h3 += carry[2]
+ h2 -= carry[2] << 26
+ carry[6] = (h6 + (1 << 25)) >> 26
+ h7 += carry[6]
+ h6 -= carry[6] << 26
+ // |h2| <= 2^25; from now on fits into int32 unchanged
+ // |h6| <= 2^25; from now on fits into int32 unchanged
+ // |h3| <= 1.51*2^58
+ // |h7| <= 1.51*2^58
+
+ carry[3] = (h3 + (1 << 24)) >> 25
+ h4 += carry[3]
+ h3 -= carry[3] << 25
+ carry[7] = (h7 + (1 << 24)) >> 25
+ h8 += carry[7]
+ h7 -= carry[7] << 25
+ // |h3| <= 2^24; from now on fits into int32 unchanged
+ // |h7| <= 2^24; from now on fits into int32 unchanged
+ // |h4| <= 1.52*2^33
+ // |h8| <= 1.52*2^33
+
+ carry[4] = (h4 + (1 << 25)) >> 26
+ h5 += carry[4]
+ h4 -= carry[4] << 26
+ carry[8] = (h8 + (1 << 25)) >> 26
+ h9 += carry[8]
+ h8 -= carry[8] << 26
+ // |h4| <= 2^25; from now on fits into int32 unchanged
+ // |h8| <= 2^25; from now on fits into int32 unchanged
+ // |h5| <= 1.01*2^24
+ // |h9| <= 1.51*2^58
+
+ carry[9] = (h9 + (1 << 24)) >> 25
+ h0 += carry[9] * 19
+ h9 -= carry[9] << 25
+ // |h9| <= 2^24; from now on fits into int32 unchanged
+ // |h0| <= 1.8*2^37
+
+ carry[0] = (h0 + (1 << 25)) >> 26
+ h1 += carry[0]
+ h0 -= carry[0] << 26
+ // |h0| <= 2^25; from now on fits into int32 unchanged
+ // |h1| <= 1.01*2^24
+
+ h[0] = int32(h0)
+ h[1] = int32(h1)
+ h[2] = int32(h2)
+ h[3] = int32(h3)
+ h[4] = int32(h4)
+ h[5] = int32(h5)
+ h[6] = int32(h6)
+ h[7] = int32(h7)
+ h[8] = int32(h8)
+ h[9] = int32(h9)
+}
+
+// feSquare calculates h = f*f. Can overlap h with f.
+//
+// Preconditions:
+// |f| bounded by 1.1*2^26,1.1*2^25,1.1*2^26,1.1*2^25,etc.
+//
+// Postconditions:
+// |h| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc.
+func feSquare(h, f *fieldElement) {
+ f0 := f[0]
+ f1 := f[1]
+ f2 := f[2]
+ f3 := f[3]
+ f4 := f[4]
+ f5 := f[5]
+ f6 := f[6]
+ f7 := f[7]
+ f8 := f[8]
+ f9 := f[9]
+ f0_2 := 2 * f0
+ f1_2 := 2 * f1
+ f2_2 := 2 * f2
+ f3_2 := 2 * f3
+ f4_2 := 2 * f4
+ f5_2 := 2 * f5
+ f6_2 := 2 * f6
+ f7_2 := 2 * f7
+ f5_38 := 38 * f5 // 1.31*2^30
+ f6_19 := 19 * f6 // 1.31*2^30
+ f7_38 := 38 * f7 // 1.31*2^30
+ f8_19 := 19 * f8 // 1.31*2^30
+ f9_38 := 38 * f9 // 1.31*2^30
+ f0f0 := int64(f0) * int64(f0)
+ f0f1_2 := int64(f0_2) * int64(f1)
+ f0f2_2 := int64(f0_2) * int64(f2)
+ f0f3_2 := int64(f0_2) * int64(f3)
+ f0f4_2 := int64(f0_2) * int64(f4)
+ f0f5_2 := int64(f0_2) * int64(f5)
+ f0f6_2 := int64(f0_2) * int64(f6)
+ f0f7_2 := int64(f0_2) * int64(f7)
+ f0f8_2 := int64(f0_2) * int64(f8)
+ f0f9_2 := int64(f0_2) * int64(f9)
+ f1f1_2 := int64(f1_2) * int64(f1)
+ f1f2_2 := int64(f1_2) * int64(f2)
+ f1f3_4 := int64(f1_2) * int64(f3_2)
+ f1f4_2 := int64(f1_2) * int64(f4)
+ f1f5_4 := int64(f1_2) * int64(f5_2)
+ f1f6_2 := int64(f1_2) * int64(f6)
+ f1f7_4 := int64(f1_2) * int64(f7_2)
+ f1f8_2 := int64(f1_2) * int64(f8)
+ f1f9_76 := int64(f1_2) * int64(f9_38)
+ f2f2 := int64(f2) * int64(f2)
+ f2f3_2 := int64(f2_2) * int64(f3)
+ f2f4_2 := int64(f2_2) * int64(f4)
+ f2f5_2 := int64(f2_2) * int64(f5)
+ f2f6_2 := int64(f2_2) * int64(f6)
+ f2f7_2 := int64(f2_2) * int64(f7)
+ f2f8_38 := int64(f2_2) * int64(f8_19)
+ f2f9_38 := int64(f2) * int64(f9_38)
+ f3f3_2 := int64(f3_2) * int64(f3)
+ f3f4_2 := int64(f3_2) * int64(f4)
+ f3f5_4 := int64(f3_2) * int64(f5_2)
+ f3f6_2 := int64(f3_2) * int64(f6)
+ f3f7_76 := int64(f3_2) * int64(f7_38)
+ f3f8_38 := int64(f3_2) * int64(f8_19)
+ f3f9_76 := int64(f3_2) * int64(f9_38)
+ f4f4 := int64(f4) * int64(f4)
+ f4f5_2 := int64(f4_2) * int64(f5)
+ f4f6_38 := int64(f4_2) * int64(f6_19)
+ f4f7_38 := int64(f4) * int64(f7_38)
+ f4f8_38 := int64(f4_2) * int64(f8_19)
+ f4f9_38 := int64(f4) * int64(f9_38)
+ f5f5_38 := int64(f5) * int64(f5_38)
+ f5f6_38 := int64(f5_2) * int64(f6_19)
+ f5f7_76 := int64(f5_2) * int64(f7_38)
+ f5f8_38 := int64(f5_2) * int64(f8_19)
+ f5f9_76 := int64(f5_2) * int64(f9_38)
+ f6f6_19 := int64(f6) * int64(f6_19)
+ f6f7_38 := int64(f6) * int64(f7_38)
+ f6f8_38 := int64(f6_2) * int64(f8_19)
+ f6f9_38 := int64(f6) * int64(f9_38)
+ f7f7_38 := int64(f7) * int64(f7_38)
+ f7f8_38 := int64(f7_2) * int64(f8_19)
+ f7f9_76 := int64(f7_2) * int64(f9_38)
+ f8f8_19 := int64(f8) * int64(f8_19)
+ f8f9_38 := int64(f8) * int64(f9_38)
+ f9f9_38 := int64(f9) * int64(f9_38)
+ h0 := f0f0 + f1f9_76 + f2f8_38 + f3f7_76 + f4f6_38 + f5f5_38
+ h1 := f0f1_2 + f2f9_38 + f3f8_38 + f4f7_38 + f5f6_38
+ h2 := f0f2_2 + f1f1_2 + f3f9_76 + f4f8_38 + f5f7_76 + f6f6_19
+ h3 := f0f3_2 + f1f2_2 + f4f9_38 + f5f8_38 + f6f7_38
+ h4 := f0f4_2 + f1f3_4 + f2f2 + f5f9_76 + f6f8_38 + f7f7_38
+ h5 := f0f5_2 + f1f4_2 + f2f3_2 + f6f9_38 + f7f8_38
+ h6 := f0f6_2 + f1f5_4 + f2f4_2 + f3f3_2 + f7f9_76 + f8f8_19
+ h7 := f0f7_2 + f1f6_2 + f2f5_2 + f3f4_2 + f8f9_38
+ h8 := f0f8_2 + f1f7_4 + f2f6_2 + f3f5_4 + f4f4 + f9f9_38
+ h9 := f0f9_2 + f1f8_2 + f2f7_2 + f3f6_2 + f4f5_2
+ var carry [10]int64
+
+ carry[0] = (h0 + (1 << 25)) >> 26
+ h1 += carry[0]
+ h0 -= carry[0] << 26
+ carry[4] = (h4 + (1 << 25)) >> 26
+ h5 += carry[4]
+ h4 -= carry[4] << 26
+
+ carry[1] = (h1 + (1 << 24)) >> 25
+ h2 += carry[1]
+ h1 -= carry[1] << 25
+ carry[5] = (h5 + (1 << 24)) >> 25
+ h6 += carry[5]
+ h5 -= carry[5] << 25
+
+ carry[2] = (h2 + (1 << 25)) >> 26
+ h3 += carry[2]
+ h2 -= carry[2] << 26
+ carry[6] = (h6 + (1 << 25)) >> 26
+ h7 += carry[6]
+ h6 -= carry[6] << 26
+
+ carry[3] = (h3 + (1 << 24)) >> 25
+ h4 += carry[3]
+ h3 -= carry[3] << 25
+ carry[7] = (h7 + (1 << 24)) >> 25
+ h8 += carry[7]
+ h7 -= carry[7] << 25
+
+ carry[4] = (h4 + (1 << 25)) >> 26
+ h5 += carry[4]
+ h4 -= carry[4] << 26
+ carry[8] = (h8 + (1 << 25)) >> 26
+ h9 += carry[8]
+ h8 -= carry[8] << 26
+
+ carry[9] = (h9 + (1 << 24)) >> 25
+ h0 += carry[9] * 19
+ h9 -= carry[9] << 25
+
+ carry[0] = (h0 + (1 << 25)) >> 26
+ h1 += carry[0]
+ h0 -= carry[0] << 26
+
+ h[0] = int32(h0)
+ h[1] = int32(h1)
+ h[2] = int32(h2)
+ h[3] = int32(h3)
+ h[4] = int32(h4)
+ h[5] = int32(h5)
+ h[6] = int32(h6)
+ h[7] = int32(h7)
+ h[8] = int32(h8)
+ h[9] = int32(h9)
+}
+
+// feMul121666 calculates h = f * 121666. Can overlap h with f.
+//
+// Preconditions:
+// |f| bounded by 1.1*2^26,1.1*2^25,1.1*2^26,1.1*2^25,etc.
+//
+// Postconditions:
+// |h| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc.
+func feMul121666(h, f *fieldElement) {
+ h0 := int64(f[0]) * 121666
+ h1 := int64(f[1]) * 121666
+ h2 := int64(f[2]) * 121666
+ h3 := int64(f[3]) * 121666
+ h4 := int64(f[4]) * 121666
+ h5 := int64(f[5]) * 121666
+ h6 := int64(f[6]) * 121666
+ h7 := int64(f[7]) * 121666
+ h8 := int64(f[8]) * 121666
+ h9 := int64(f[9]) * 121666
+ var carry [10]int64
+
+ carry[9] = (h9 + (1 << 24)) >> 25
+ h0 += carry[9] * 19
+ h9 -= carry[9] << 25
+ carry[1] = (h1 + (1 << 24)) >> 25
+ h2 += carry[1]
+ h1 -= carry[1] << 25
+ carry[3] = (h3 + (1 << 24)) >> 25
+ h4 += carry[3]
+ h3 -= carry[3] << 25
+ carry[5] = (h5 + (1 << 24)) >> 25
+ h6 += carry[5]
+ h5 -= carry[5] << 25
+ carry[7] = (h7 + (1 << 24)) >> 25
+ h8 += carry[7]
+ h7 -= carry[7] << 25
+
+ carry[0] = (h0 + (1 << 25)) >> 26
+ h1 += carry[0]
+ h0 -= carry[0] << 26
+ carry[2] = (h2 + (1 << 25)) >> 26
+ h3 += carry[2]
+ h2 -= carry[2] << 26
+ carry[4] = (h4 + (1 << 25)) >> 26
+ h5 += carry[4]
+ h4 -= carry[4] << 26
+ carry[6] = (h6 + (1 << 25)) >> 26
+ h7 += carry[6]
+ h6 -= carry[6] << 26
+ carry[8] = (h8 + (1 << 25)) >> 26
+ h9 += carry[8]
+ h8 -= carry[8] << 26
+
+ h[0] = int32(h0)
+ h[1] = int32(h1)
+ h[2] = int32(h2)
+ h[3] = int32(h3)
+ h[4] = int32(h4)
+ h[5] = int32(h5)
+ h[6] = int32(h6)
+ h[7] = int32(h7)
+ h[8] = int32(h8)
+ h[9] = int32(h9)
+}
+
+// feInvert sets out = z^-1.
+func feInvert(out, z *fieldElement) {
+ var t0, t1, t2, t3 fieldElement
+ var i int
+
+ feSquare(&t0, z)
+ for i = 1; i < 1; i++ {
+ feSquare(&t0, &t0)
+ }
+ feSquare(&t1, &t0)
+ for i = 1; i < 2; i++ {
+ feSquare(&t1, &t1)
+ }
+ feMul(&t1, z, &t1)
+ feMul(&t0, &t0, &t1)
+ feSquare(&t2, &t0)
+ for i = 1; i < 1; i++ {
+ feSquare(&t2, &t2)
+ }
+ feMul(&t1, &t1, &t2)
+ feSquare(&t2, &t1)
+ for i = 1; i < 5; i++ {
+ feSquare(&t2, &t2)
+ }
+ feMul(&t1, &t2, &t1)
+ feSquare(&t2, &t1)
+ for i = 1; i < 10; i++ {
+ feSquare(&t2, &t2)
+ }
+ feMul(&t2, &t2, &t1)
+ feSquare(&t3, &t2)
+ for i = 1; i < 20; i++ {
+ feSquare(&t3, &t3)
+ }
+ feMul(&t2, &t3, &t2)
+ feSquare(&t2, &t2)
+ for i = 1; i < 10; i++ {
+ feSquare(&t2, &t2)
+ }
+ feMul(&t1, &t2, &t1)
+ feSquare(&t2, &t1)
+ for i = 1; i < 50; i++ {
+ feSquare(&t2, &t2)
+ }
+ feMul(&t2, &t2, &t1)
+ feSquare(&t3, &t2)
+ for i = 1; i < 100; i++ {
+ feSquare(&t3, &t3)
+ }
+ feMul(&t2, &t3, &t2)
+ feSquare(&t2, &t2)
+ for i = 1; i < 50; i++ {
+ feSquare(&t2, &t2)
+ }
+ feMul(&t1, &t2, &t1)
+ feSquare(&t1, &t1)
+ for i = 1; i < 5; i++ {
+ feSquare(&t1, &t1)
+ }
+ feMul(out, &t1, &t0)
+}
+
+func scalarMultGeneric(out, in, base *[32]byte) {
+ var e [32]byte
+
+ copy(e[:], in[:])
+ e[0] &= 248
+ e[31] &= 127
+ e[31] |= 64
+
+ var x1, x2, z2, x3, z3, tmp0, tmp1 fieldElement
+ feFromBytes(&x1, base)
+ feOne(&x2)
+ feCopy(&x3, &x1)
+ feOne(&z3)
+
+ swap := int32(0)
+ for pos := 254; pos >= 0; pos-- {
+ b := e[pos/8] >> uint(pos&7)
+ b &= 1
+ swap ^= int32(b)
+ feCSwap(&x2, &x3, swap)
+ feCSwap(&z2, &z3, swap)
+ swap = int32(b)
+
+ feSub(&tmp0, &x3, &z3)
+ feSub(&tmp1, &x2, &z2)
+ feAdd(&x2, &x2, &z2)
+ feAdd(&z2, &x3, &z3)
+ feMul(&z3, &tmp0, &x2)
+ feMul(&z2, &z2, &tmp1)
+ feSquare(&tmp0, &tmp1)
+ feSquare(&tmp1, &x2)
+ feAdd(&x3, &z3, &z2)
+ feSub(&z2, &z3, &z2)
+ feMul(&x2, &tmp1, &tmp0)
+ feSub(&tmp1, &tmp1, &tmp0)
+ feSquare(&z2, &z2)
+ feMul121666(&z3, &tmp1)
+ feSquare(&x3, &x3)
+ feAdd(&tmp0, &tmp0, &z3)
+ feMul(&z3, &x1, &z2)
+ feMul(&z2, &tmp1, &tmp0)
+ }
+
+ feCSwap(&x2, &x3, swap)
+ feCSwap(&z2, &z3, swap)
+
+ feInvert(&z2, &z2)
+ feMul(&x2, &x2, &z2)
+ feToBytes(out, &x2)
+}
--- /dev/null
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !amd64 gccgo appengine purego
+
+package curve25519
+
+func scalarMult(out, in, base *[32]byte) {
+ scalarMultGeneric(out, in, base)
+}
+++ /dev/null
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package curve25519 provides an implementation of scalar multiplication on
-// the elliptic curve known as curve25519. See https://cr.yp.to/ecdh.html
-package curve25519 // import "golang.org/x/crypto/curve25519"
-
-// basePoint is the x coordinate of the generator of the curve.
-var basePoint = [32]byte{9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
-
-// ScalarMult sets dst to the product in*base where dst and base are the x
-// coordinates of group points and all values are in little-endian form.
-func ScalarMult(dst, in, base *[32]byte) {
- scalarMult(dst, in, base)
-}
-
-// ScalarBaseMult sets dst to the product in*base where dst and base are the x
-// coordinates of group points, base is the standard generator and all values
-// are in little-endian form.
-func ScalarBaseMult(dst, in *[32]byte) {
- ScalarMult(dst, in, &basePoint)
-}
+++ /dev/null
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// This code was translated into a form compatible with 6a from the public
-// domain sources in SUPERCOP: https://bench.cr.yp.to/supercop.html
-
-// +build amd64,!gccgo,!appengine
-
-#include "const_amd64.h"
-
-// func freeze(inout *[5]uint64)
-TEXT ·freeze(SB),7,$0-8
- MOVQ inout+0(FP), DI
-
- MOVQ 0(DI),SI
- MOVQ 8(DI),DX
- MOVQ 16(DI),CX
- MOVQ 24(DI),R8
- MOVQ 32(DI),R9
- MOVQ $REDMASK51,AX
- MOVQ AX,R10
- SUBQ $18,R10
- MOVQ $3,R11
-REDUCELOOP:
- MOVQ SI,R12
- SHRQ $51,R12
- ANDQ AX,SI
- ADDQ R12,DX
- MOVQ DX,R12
- SHRQ $51,R12
- ANDQ AX,DX
- ADDQ R12,CX
- MOVQ CX,R12
- SHRQ $51,R12
- ANDQ AX,CX
- ADDQ R12,R8
- MOVQ R8,R12
- SHRQ $51,R12
- ANDQ AX,R8
- ADDQ R12,R9
- MOVQ R9,R12
- SHRQ $51,R12
- ANDQ AX,R9
- IMUL3Q $19,R12,R12
- ADDQ R12,SI
- SUBQ $1,R11
- JA REDUCELOOP
- MOVQ $1,R12
- CMPQ R10,SI
- CMOVQLT R11,R12
- CMPQ AX,DX
- CMOVQNE R11,R12
- CMPQ AX,CX
- CMOVQNE R11,R12
- CMPQ AX,R8
- CMOVQNE R11,R12
- CMPQ AX,R9
- CMOVQNE R11,R12
- NEGQ R12
- ANDQ R12,AX
- ANDQ R12,R10
- SUBQ R10,SI
- SUBQ AX,DX
- SUBQ AX,CX
- SUBQ AX,R8
- SUBQ AX,R9
- MOVQ SI,0(DI)
- MOVQ DX,8(DI)
- MOVQ CX,16(DI)
- MOVQ R8,24(DI)
- MOVQ R9,32(DI)
- RET
+++ /dev/null
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// This code was translated into a form compatible with 6a from the public
-// domain sources in SUPERCOP: https://bench.cr.yp.to/supercop.html
-
-// +build amd64,!gccgo,!appengine
-
-#include "const_amd64.h"
-
-// func ladderstep(inout *[5][5]uint64)
-TEXT ·ladderstep(SB),0,$296-8
- MOVQ inout+0(FP),DI
-
- MOVQ 40(DI),SI
- MOVQ 48(DI),DX
- MOVQ 56(DI),CX
- MOVQ 64(DI),R8
- MOVQ 72(DI),R9
- MOVQ SI,AX
- MOVQ DX,R10
- MOVQ CX,R11
- MOVQ R8,R12
- MOVQ R9,R13
- ADDQ ·_2P0(SB),AX
- ADDQ ·_2P1234(SB),R10
- ADDQ ·_2P1234(SB),R11
- ADDQ ·_2P1234(SB),R12
- ADDQ ·_2P1234(SB),R13
- ADDQ 80(DI),SI
- ADDQ 88(DI),DX
- ADDQ 96(DI),CX
- ADDQ 104(DI),R8
- ADDQ 112(DI),R9
- SUBQ 80(DI),AX
- SUBQ 88(DI),R10
- SUBQ 96(DI),R11
- SUBQ 104(DI),R12
- SUBQ 112(DI),R13
- MOVQ SI,0(SP)
- MOVQ DX,8(SP)
- MOVQ CX,16(SP)
- MOVQ R8,24(SP)
- MOVQ R9,32(SP)
- MOVQ AX,40(SP)
- MOVQ R10,48(SP)
- MOVQ R11,56(SP)
- MOVQ R12,64(SP)
- MOVQ R13,72(SP)
- MOVQ 40(SP),AX
- MULQ 40(SP)
- MOVQ AX,SI
- MOVQ DX,CX
- MOVQ 40(SP),AX
- SHLQ $1,AX
- MULQ 48(SP)
- MOVQ AX,R8
- MOVQ DX,R9
- MOVQ 40(SP),AX
- SHLQ $1,AX
- MULQ 56(SP)
- MOVQ AX,R10
- MOVQ DX,R11
- MOVQ 40(SP),AX
- SHLQ $1,AX
- MULQ 64(SP)
- MOVQ AX,R12
- MOVQ DX,R13
- MOVQ 40(SP),AX
- SHLQ $1,AX
- MULQ 72(SP)
- MOVQ AX,R14
- MOVQ DX,R15
- MOVQ 48(SP),AX
- MULQ 48(SP)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 48(SP),AX
- SHLQ $1,AX
- MULQ 56(SP)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ 48(SP),AX
- SHLQ $1,AX
- MULQ 64(SP)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 48(SP),DX
- IMUL3Q $38,DX,AX
- MULQ 72(SP)
- ADDQ AX,SI
- ADCQ DX,CX
- MOVQ 56(SP),AX
- MULQ 56(SP)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 56(SP),DX
- IMUL3Q $38,DX,AX
- MULQ 64(SP)
- ADDQ AX,SI
- ADCQ DX,CX
- MOVQ 56(SP),DX
- IMUL3Q $38,DX,AX
- MULQ 72(SP)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 64(SP),DX
- IMUL3Q $19,DX,AX
- MULQ 64(SP)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 64(SP),DX
- IMUL3Q $38,DX,AX
- MULQ 72(SP)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 72(SP),DX
- IMUL3Q $19,DX,AX
- MULQ 72(SP)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ $REDMASK51,DX
- SHLQ $13,SI,CX
- ANDQ DX,SI
- SHLQ $13,R8,R9
- ANDQ DX,R8
- ADDQ CX,R8
- SHLQ $13,R10,R11
- ANDQ DX,R10
- ADDQ R9,R10
- SHLQ $13,R12,R13
- ANDQ DX,R12
- ADDQ R11,R12
- SHLQ $13,R14,R15
- ANDQ DX,R14
- ADDQ R13,R14
- IMUL3Q $19,R15,CX
- ADDQ CX,SI
- MOVQ SI,CX
- SHRQ $51,CX
- ADDQ R8,CX
- ANDQ DX,SI
- MOVQ CX,R8
- SHRQ $51,CX
- ADDQ R10,CX
- ANDQ DX,R8
- MOVQ CX,R9
- SHRQ $51,CX
- ADDQ R12,CX
- ANDQ DX,R9
- MOVQ CX,AX
- SHRQ $51,CX
- ADDQ R14,CX
- ANDQ DX,AX
- MOVQ CX,R10
- SHRQ $51,CX
- IMUL3Q $19,CX,CX
- ADDQ CX,SI
- ANDQ DX,R10
- MOVQ SI,80(SP)
- MOVQ R8,88(SP)
- MOVQ R9,96(SP)
- MOVQ AX,104(SP)
- MOVQ R10,112(SP)
- MOVQ 0(SP),AX
- MULQ 0(SP)
- MOVQ AX,SI
- MOVQ DX,CX
- MOVQ 0(SP),AX
- SHLQ $1,AX
- MULQ 8(SP)
- MOVQ AX,R8
- MOVQ DX,R9
- MOVQ 0(SP),AX
- SHLQ $1,AX
- MULQ 16(SP)
- MOVQ AX,R10
- MOVQ DX,R11
- MOVQ 0(SP),AX
- SHLQ $1,AX
- MULQ 24(SP)
- MOVQ AX,R12
- MOVQ DX,R13
- MOVQ 0(SP),AX
- SHLQ $1,AX
- MULQ 32(SP)
- MOVQ AX,R14
- MOVQ DX,R15
- MOVQ 8(SP),AX
- MULQ 8(SP)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 8(SP),AX
- SHLQ $1,AX
- MULQ 16(SP)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ 8(SP),AX
- SHLQ $1,AX
- MULQ 24(SP)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 8(SP),DX
- IMUL3Q $38,DX,AX
- MULQ 32(SP)
- ADDQ AX,SI
- ADCQ DX,CX
- MOVQ 16(SP),AX
- MULQ 16(SP)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 16(SP),DX
- IMUL3Q $38,DX,AX
- MULQ 24(SP)
- ADDQ AX,SI
- ADCQ DX,CX
- MOVQ 16(SP),DX
- IMUL3Q $38,DX,AX
- MULQ 32(SP)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 24(SP),DX
- IMUL3Q $19,DX,AX
- MULQ 24(SP)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 24(SP),DX
- IMUL3Q $38,DX,AX
- MULQ 32(SP)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 32(SP),DX
- IMUL3Q $19,DX,AX
- MULQ 32(SP)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ $REDMASK51,DX
- SHLQ $13,SI,CX
- ANDQ DX,SI
- SHLQ $13,R8,R9
- ANDQ DX,R8
- ADDQ CX,R8
- SHLQ $13,R10,R11
- ANDQ DX,R10
- ADDQ R9,R10
- SHLQ $13,R12,R13
- ANDQ DX,R12
- ADDQ R11,R12
- SHLQ $13,R14,R15
- ANDQ DX,R14
- ADDQ R13,R14
- IMUL3Q $19,R15,CX
- ADDQ CX,SI
- MOVQ SI,CX
- SHRQ $51,CX
- ADDQ R8,CX
- ANDQ DX,SI
- MOVQ CX,R8
- SHRQ $51,CX
- ADDQ R10,CX
- ANDQ DX,R8
- MOVQ CX,R9
- SHRQ $51,CX
- ADDQ R12,CX
- ANDQ DX,R9
- MOVQ CX,AX
- SHRQ $51,CX
- ADDQ R14,CX
- ANDQ DX,AX
- MOVQ CX,R10
- SHRQ $51,CX
- IMUL3Q $19,CX,CX
- ADDQ CX,SI
- ANDQ DX,R10
- MOVQ SI,120(SP)
- MOVQ R8,128(SP)
- MOVQ R9,136(SP)
- MOVQ AX,144(SP)
- MOVQ R10,152(SP)
- MOVQ SI,SI
- MOVQ R8,DX
- MOVQ R9,CX
- MOVQ AX,R8
- MOVQ R10,R9
- ADDQ ·_2P0(SB),SI
- ADDQ ·_2P1234(SB),DX
- ADDQ ·_2P1234(SB),CX
- ADDQ ·_2P1234(SB),R8
- ADDQ ·_2P1234(SB),R9
- SUBQ 80(SP),SI
- SUBQ 88(SP),DX
- SUBQ 96(SP),CX
- SUBQ 104(SP),R8
- SUBQ 112(SP),R9
- MOVQ SI,160(SP)
- MOVQ DX,168(SP)
- MOVQ CX,176(SP)
- MOVQ R8,184(SP)
- MOVQ R9,192(SP)
- MOVQ 120(DI),SI
- MOVQ 128(DI),DX
- MOVQ 136(DI),CX
- MOVQ 144(DI),R8
- MOVQ 152(DI),R9
- MOVQ SI,AX
- MOVQ DX,R10
- MOVQ CX,R11
- MOVQ R8,R12
- MOVQ R9,R13
- ADDQ ·_2P0(SB),AX
- ADDQ ·_2P1234(SB),R10
- ADDQ ·_2P1234(SB),R11
- ADDQ ·_2P1234(SB),R12
- ADDQ ·_2P1234(SB),R13
- ADDQ 160(DI),SI
- ADDQ 168(DI),DX
- ADDQ 176(DI),CX
- ADDQ 184(DI),R8
- ADDQ 192(DI),R9
- SUBQ 160(DI),AX
- SUBQ 168(DI),R10
- SUBQ 176(DI),R11
- SUBQ 184(DI),R12
- SUBQ 192(DI),R13
- MOVQ SI,200(SP)
- MOVQ DX,208(SP)
- MOVQ CX,216(SP)
- MOVQ R8,224(SP)
- MOVQ R9,232(SP)
- MOVQ AX,240(SP)
- MOVQ R10,248(SP)
- MOVQ R11,256(SP)
- MOVQ R12,264(SP)
- MOVQ R13,272(SP)
- MOVQ 224(SP),SI
- IMUL3Q $19,SI,AX
- MOVQ AX,280(SP)
- MULQ 56(SP)
- MOVQ AX,SI
- MOVQ DX,CX
- MOVQ 232(SP),DX
- IMUL3Q $19,DX,AX
- MOVQ AX,288(SP)
- MULQ 48(SP)
- ADDQ AX,SI
- ADCQ DX,CX
- MOVQ 200(SP),AX
- MULQ 40(SP)
- ADDQ AX,SI
- ADCQ DX,CX
- MOVQ 200(SP),AX
- MULQ 48(SP)
- MOVQ AX,R8
- MOVQ DX,R9
- MOVQ 200(SP),AX
- MULQ 56(SP)
- MOVQ AX,R10
- MOVQ DX,R11
- MOVQ 200(SP),AX
- MULQ 64(SP)
- MOVQ AX,R12
- MOVQ DX,R13
- MOVQ 200(SP),AX
- MULQ 72(SP)
- MOVQ AX,R14
- MOVQ DX,R15
- MOVQ 208(SP),AX
- MULQ 40(SP)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 208(SP),AX
- MULQ 48(SP)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 208(SP),AX
- MULQ 56(SP)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ 208(SP),AX
- MULQ 64(SP)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 208(SP),DX
- IMUL3Q $19,DX,AX
- MULQ 72(SP)
- ADDQ AX,SI
- ADCQ DX,CX
- MOVQ 216(SP),AX
- MULQ 40(SP)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 216(SP),AX
- MULQ 48(SP)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ 216(SP),AX
- MULQ 56(SP)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 216(SP),DX
- IMUL3Q $19,DX,AX
- MULQ 64(SP)
- ADDQ AX,SI
- ADCQ DX,CX
- MOVQ 216(SP),DX
- IMUL3Q $19,DX,AX
- MULQ 72(SP)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 224(SP),AX
- MULQ 40(SP)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ 224(SP),AX
- MULQ 48(SP)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 280(SP),AX
- MULQ 64(SP)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 280(SP),AX
- MULQ 72(SP)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 232(SP),AX
- MULQ 40(SP)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 288(SP),AX
- MULQ 56(SP)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 288(SP),AX
- MULQ 64(SP)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 288(SP),AX
- MULQ 72(SP)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ $REDMASK51,DX
- SHLQ $13,SI,CX
- ANDQ DX,SI
- SHLQ $13,R8,R9
- ANDQ DX,R8
- ADDQ CX,R8
- SHLQ $13,R10,R11
- ANDQ DX,R10
- ADDQ R9,R10
- SHLQ $13,R12,R13
- ANDQ DX,R12
- ADDQ R11,R12
- SHLQ $13,R14,R15
- ANDQ DX,R14
- ADDQ R13,R14
- IMUL3Q $19,R15,CX
- ADDQ CX,SI
- MOVQ SI,CX
- SHRQ $51,CX
- ADDQ R8,CX
- MOVQ CX,R8
- SHRQ $51,CX
- ANDQ DX,SI
- ADDQ R10,CX
- MOVQ CX,R9
- SHRQ $51,CX
- ANDQ DX,R8
- ADDQ R12,CX
- MOVQ CX,AX
- SHRQ $51,CX
- ANDQ DX,R9
- ADDQ R14,CX
- MOVQ CX,R10
- SHRQ $51,CX
- ANDQ DX,AX
- IMUL3Q $19,CX,CX
- ADDQ CX,SI
- ANDQ DX,R10
- MOVQ SI,40(SP)
- MOVQ R8,48(SP)
- MOVQ R9,56(SP)
- MOVQ AX,64(SP)
- MOVQ R10,72(SP)
- MOVQ 264(SP),SI
- IMUL3Q $19,SI,AX
- MOVQ AX,200(SP)
- MULQ 16(SP)
- MOVQ AX,SI
- MOVQ DX,CX
- MOVQ 272(SP),DX
- IMUL3Q $19,DX,AX
- MOVQ AX,208(SP)
- MULQ 8(SP)
- ADDQ AX,SI
- ADCQ DX,CX
- MOVQ 240(SP),AX
- MULQ 0(SP)
- ADDQ AX,SI
- ADCQ DX,CX
- MOVQ 240(SP),AX
- MULQ 8(SP)
- MOVQ AX,R8
- MOVQ DX,R9
- MOVQ 240(SP),AX
- MULQ 16(SP)
- MOVQ AX,R10
- MOVQ DX,R11
- MOVQ 240(SP),AX
- MULQ 24(SP)
- MOVQ AX,R12
- MOVQ DX,R13
- MOVQ 240(SP),AX
- MULQ 32(SP)
- MOVQ AX,R14
- MOVQ DX,R15
- MOVQ 248(SP),AX
- MULQ 0(SP)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 248(SP),AX
- MULQ 8(SP)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 248(SP),AX
- MULQ 16(SP)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ 248(SP),AX
- MULQ 24(SP)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 248(SP),DX
- IMUL3Q $19,DX,AX
- MULQ 32(SP)
- ADDQ AX,SI
- ADCQ DX,CX
- MOVQ 256(SP),AX
- MULQ 0(SP)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 256(SP),AX
- MULQ 8(SP)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ 256(SP),AX
- MULQ 16(SP)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 256(SP),DX
- IMUL3Q $19,DX,AX
- MULQ 24(SP)
- ADDQ AX,SI
- ADCQ DX,CX
- MOVQ 256(SP),DX
- IMUL3Q $19,DX,AX
- MULQ 32(SP)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 264(SP),AX
- MULQ 0(SP)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ 264(SP),AX
- MULQ 8(SP)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 200(SP),AX
- MULQ 24(SP)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 200(SP),AX
- MULQ 32(SP)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 272(SP),AX
- MULQ 0(SP)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 208(SP),AX
- MULQ 16(SP)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 208(SP),AX
- MULQ 24(SP)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 208(SP),AX
- MULQ 32(SP)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ $REDMASK51,DX
- SHLQ $13,SI,CX
- ANDQ DX,SI
- SHLQ $13,R8,R9
- ANDQ DX,R8
- ADDQ CX,R8
- SHLQ $13,R10,R11
- ANDQ DX,R10
- ADDQ R9,R10
- SHLQ $13,R12,R13
- ANDQ DX,R12
- ADDQ R11,R12
- SHLQ $13,R14,R15
- ANDQ DX,R14
- ADDQ R13,R14
- IMUL3Q $19,R15,CX
- ADDQ CX,SI
- MOVQ SI,CX
- SHRQ $51,CX
- ADDQ R8,CX
- MOVQ CX,R8
- SHRQ $51,CX
- ANDQ DX,SI
- ADDQ R10,CX
- MOVQ CX,R9
- SHRQ $51,CX
- ANDQ DX,R8
- ADDQ R12,CX
- MOVQ CX,AX
- SHRQ $51,CX
- ANDQ DX,R9
- ADDQ R14,CX
- MOVQ CX,R10
- SHRQ $51,CX
- ANDQ DX,AX
- IMUL3Q $19,CX,CX
- ADDQ CX,SI
- ANDQ DX,R10
- MOVQ SI,DX
- MOVQ R8,CX
- MOVQ R9,R11
- MOVQ AX,R12
- MOVQ R10,R13
- ADDQ ·_2P0(SB),DX
- ADDQ ·_2P1234(SB),CX
- ADDQ ·_2P1234(SB),R11
- ADDQ ·_2P1234(SB),R12
- ADDQ ·_2P1234(SB),R13
- ADDQ 40(SP),SI
- ADDQ 48(SP),R8
- ADDQ 56(SP),R9
- ADDQ 64(SP),AX
- ADDQ 72(SP),R10
- SUBQ 40(SP),DX
- SUBQ 48(SP),CX
- SUBQ 56(SP),R11
- SUBQ 64(SP),R12
- SUBQ 72(SP),R13
- MOVQ SI,120(DI)
- MOVQ R8,128(DI)
- MOVQ R9,136(DI)
- MOVQ AX,144(DI)
- MOVQ R10,152(DI)
- MOVQ DX,160(DI)
- MOVQ CX,168(DI)
- MOVQ R11,176(DI)
- MOVQ R12,184(DI)
- MOVQ R13,192(DI)
- MOVQ 120(DI),AX
- MULQ 120(DI)
- MOVQ AX,SI
- MOVQ DX,CX
- MOVQ 120(DI),AX
- SHLQ $1,AX
- MULQ 128(DI)
- MOVQ AX,R8
- MOVQ DX,R9
- MOVQ 120(DI),AX
- SHLQ $1,AX
- MULQ 136(DI)
- MOVQ AX,R10
- MOVQ DX,R11
- MOVQ 120(DI),AX
- SHLQ $1,AX
- MULQ 144(DI)
- MOVQ AX,R12
- MOVQ DX,R13
- MOVQ 120(DI),AX
- SHLQ $1,AX
- MULQ 152(DI)
- MOVQ AX,R14
- MOVQ DX,R15
- MOVQ 128(DI),AX
- MULQ 128(DI)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 128(DI),AX
- SHLQ $1,AX
- MULQ 136(DI)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ 128(DI),AX
- SHLQ $1,AX
- MULQ 144(DI)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 128(DI),DX
- IMUL3Q $38,DX,AX
- MULQ 152(DI)
- ADDQ AX,SI
- ADCQ DX,CX
- MOVQ 136(DI),AX
- MULQ 136(DI)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 136(DI),DX
- IMUL3Q $38,DX,AX
- MULQ 144(DI)
- ADDQ AX,SI
- ADCQ DX,CX
- MOVQ 136(DI),DX
- IMUL3Q $38,DX,AX
- MULQ 152(DI)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 144(DI),DX
- IMUL3Q $19,DX,AX
- MULQ 144(DI)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 144(DI),DX
- IMUL3Q $38,DX,AX
- MULQ 152(DI)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 152(DI),DX
- IMUL3Q $19,DX,AX
- MULQ 152(DI)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ $REDMASK51,DX
- SHLQ $13,SI,CX
- ANDQ DX,SI
- SHLQ $13,R8,R9
- ANDQ DX,R8
- ADDQ CX,R8
- SHLQ $13,R10,R11
- ANDQ DX,R10
- ADDQ R9,R10
- SHLQ $13,R12,R13
- ANDQ DX,R12
- ADDQ R11,R12
- SHLQ $13,R14,R15
- ANDQ DX,R14
- ADDQ R13,R14
- IMUL3Q $19,R15,CX
- ADDQ CX,SI
- MOVQ SI,CX
- SHRQ $51,CX
- ADDQ R8,CX
- ANDQ DX,SI
- MOVQ CX,R8
- SHRQ $51,CX
- ADDQ R10,CX
- ANDQ DX,R8
- MOVQ CX,R9
- SHRQ $51,CX
- ADDQ R12,CX
- ANDQ DX,R9
- MOVQ CX,AX
- SHRQ $51,CX
- ADDQ R14,CX
- ANDQ DX,AX
- MOVQ CX,R10
- SHRQ $51,CX
- IMUL3Q $19,CX,CX
- ADDQ CX,SI
- ANDQ DX,R10
- MOVQ SI,120(DI)
- MOVQ R8,128(DI)
- MOVQ R9,136(DI)
- MOVQ AX,144(DI)
- MOVQ R10,152(DI)
- MOVQ 160(DI),AX
- MULQ 160(DI)
- MOVQ AX,SI
- MOVQ DX,CX
- MOVQ 160(DI),AX
- SHLQ $1,AX
- MULQ 168(DI)
- MOVQ AX,R8
- MOVQ DX,R9
- MOVQ 160(DI),AX
- SHLQ $1,AX
- MULQ 176(DI)
- MOVQ AX,R10
- MOVQ DX,R11
- MOVQ 160(DI),AX
- SHLQ $1,AX
- MULQ 184(DI)
- MOVQ AX,R12
- MOVQ DX,R13
- MOVQ 160(DI),AX
- SHLQ $1,AX
- MULQ 192(DI)
- MOVQ AX,R14
- MOVQ DX,R15
- MOVQ 168(DI),AX
- MULQ 168(DI)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 168(DI),AX
- SHLQ $1,AX
- MULQ 176(DI)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ 168(DI),AX
- SHLQ $1,AX
- MULQ 184(DI)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 168(DI),DX
- IMUL3Q $38,DX,AX
- MULQ 192(DI)
- ADDQ AX,SI
- ADCQ DX,CX
- MOVQ 176(DI),AX
- MULQ 176(DI)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 176(DI),DX
- IMUL3Q $38,DX,AX
- MULQ 184(DI)
- ADDQ AX,SI
- ADCQ DX,CX
- MOVQ 176(DI),DX
- IMUL3Q $38,DX,AX
- MULQ 192(DI)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 184(DI),DX
- IMUL3Q $19,DX,AX
- MULQ 184(DI)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 184(DI),DX
- IMUL3Q $38,DX,AX
- MULQ 192(DI)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 192(DI),DX
- IMUL3Q $19,DX,AX
- MULQ 192(DI)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ $REDMASK51,DX
- SHLQ $13,SI,CX
- ANDQ DX,SI
- SHLQ $13,R8,R9
- ANDQ DX,R8
- ADDQ CX,R8
- SHLQ $13,R10,R11
- ANDQ DX,R10
- ADDQ R9,R10
- SHLQ $13,R12,R13
- ANDQ DX,R12
- ADDQ R11,R12
- SHLQ $13,R14,R15
- ANDQ DX,R14
- ADDQ R13,R14
- IMUL3Q $19,R15,CX
- ADDQ CX,SI
- MOVQ SI,CX
- SHRQ $51,CX
- ADDQ R8,CX
- ANDQ DX,SI
- MOVQ CX,R8
- SHRQ $51,CX
- ADDQ R10,CX
- ANDQ DX,R8
- MOVQ CX,R9
- SHRQ $51,CX
- ADDQ R12,CX
- ANDQ DX,R9
- MOVQ CX,AX
- SHRQ $51,CX
- ADDQ R14,CX
- ANDQ DX,AX
- MOVQ CX,R10
- SHRQ $51,CX
- IMUL3Q $19,CX,CX
- ADDQ CX,SI
- ANDQ DX,R10
- MOVQ SI,160(DI)
- MOVQ R8,168(DI)
- MOVQ R9,176(DI)
- MOVQ AX,184(DI)
- MOVQ R10,192(DI)
- MOVQ 184(DI),SI
- IMUL3Q $19,SI,AX
- MOVQ AX,0(SP)
- MULQ 16(DI)
- MOVQ AX,SI
- MOVQ DX,CX
- MOVQ 192(DI),DX
- IMUL3Q $19,DX,AX
- MOVQ AX,8(SP)
- MULQ 8(DI)
- ADDQ AX,SI
- ADCQ DX,CX
- MOVQ 160(DI),AX
- MULQ 0(DI)
- ADDQ AX,SI
- ADCQ DX,CX
- MOVQ 160(DI),AX
- MULQ 8(DI)
- MOVQ AX,R8
- MOVQ DX,R9
- MOVQ 160(DI),AX
- MULQ 16(DI)
- MOVQ AX,R10
- MOVQ DX,R11
- MOVQ 160(DI),AX
- MULQ 24(DI)
- MOVQ AX,R12
- MOVQ DX,R13
- MOVQ 160(DI),AX
- MULQ 32(DI)
- MOVQ AX,R14
- MOVQ DX,R15
- MOVQ 168(DI),AX
- MULQ 0(DI)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 168(DI),AX
- MULQ 8(DI)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 168(DI),AX
- MULQ 16(DI)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ 168(DI),AX
- MULQ 24(DI)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 168(DI),DX
- IMUL3Q $19,DX,AX
- MULQ 32(DI)
- ADDQ AX,SI
- ADCQ DX,CX
- MOVQ 176(DI),AX
- MULQ 0(DI)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 176(DI),AX
- MULQ 8(DI)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ 176(DI),AX
- MULQ 16(DI)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 176(DI),DX
- IMUL3Q $19,DX,AX
- MULQ 24(DI)
- ADDQ AX,SI
- ADCQ DX,CX
- MOVQ 176(DI),DX
- IMUL3Q $19,DX,AX
- MULQ 32(DI)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 184(DI),AX
- MULQ 0(DI)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ 184(DI),AX
- MULQ 8(DI)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 0(SP),AX
- MULQ 24(DI)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 0(SP),AX
- MULQ 32(DI)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 192(DI),AX
- MULQ 0(DI)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 8(SP),AX
- MULQ 16(DI)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 8(SP),AX
- MULQ 24(DI)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 8(SP),AX
- MULQ 32(DI)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ $REDMASK51,DX
- SHLQ $13,SI,CX
- ANDQ DX,SI
- SHLQ $13,R8,R9
- ANDQ DX,R8
- ADDQ CX,R8
- SHLQ $13,R10,R11
- ANDQ DX,R10
- ADDQ R9,R10
- SHLQ $13,R12,R13
- ANDQ DX,R12
- ADDQ R11,R12
- SHLQ $13,R14,R15
- ANDQ DX,R14
- ADDQ R13,R14
- IMUL3Q $19,R15,CX
- ADDQ CX,SI
- MOVQ SI,CX
- SHRQ $51,CX
- ADDQ R8,CX
- MOVQ CX,R8
- SHRQ $51,CX
- ANDQ DX,SI
- ADDQ R10,CX
- MOVQ CX,R9
- SHRQ $51,CX
- ANDQ DX,R8
- ADDQ R12,CX
- MOVQ CX,AX
- SHRQ $51,CX
- ANDQ DX,R9
- ADDQ R14,CX
- MOVQ CX,R10
- SHRQ $51,CX
- ANDQ DX,AX
- IMUL3Q $19,CX,CX
- ADDQ CX,SI
- ANDQ DX,R10
- MOVQ SI,160(DI)
- MOVQ R8,168(DI)
- MOVQ R9,176(DI)
- MOVQ AX,184(DI)
- MOVQ R10,192(DI)
- MOVQ 144(SP),SI
- IMUL3Q $19,SI,AX
- MOVQ AX,0(SP)
- MULQ 96(SP)
- MOVQ AX,SI
- MOVQ DX,CX
- MOVQ 152(SP),DX
- IMUL3Q $19,DX,AX
- MOVQ AX,8(SP)
- MULQ 88(SP)
- ADDQ AX,SI
- ADCQ DX,CX
- MOVQ 120(SP),AX
- MULQ 80(SP)
- ADDQ AX,SI
- ADCQ DX,CX
- MOVQ 120(SP),AX
- MULQ 88(SP)
- MOVQ AX,R8
- MOVQ DX,R9
- MOVQ 120(SP),AX
- MULQ 96(SP)
- MOVQ AX,R10
- MOVQ DX,R11
- MOVQ 120(SP),AX
- MULQ 104(SP)
- MOVQ AX,R12
- MOVQ DX,R13
- MOVQ 120(SP),AX
- MULQ 112(SP)
- MOVQ AX,R14
- MOVQ DX,R15
- MOVQ 128(SP),AX
- MULQ 80(SP)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 128(SP),AX
- MULQ 88(SP)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 128(SP),AX
- MULQ 96(SP)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ 128(SP),AX
- MULQ 104(SP)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 128(SP),DX
- IMUL3Q $19,DX,AX
- MULQ 112(SP)
- ADDQ AX,SI
- ADCQ DX,CX
- MOVQ 136(SP),AX
- MULQ 80(SP)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 136(SP),AX
- MULQ 88(SP)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ 136(SP),AX
- MULQ 96(SP)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 136(SP),DX
- IMUL3Q $19,DX,AX
- MULQ 104(SP)
- ADDQ AX,SI
- ADCQ DX,CX
- MOVQ 136(SP),DX
- IMUL3Q $19,DX,AX
- MULQ 112(SP)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 144(SP),AX
- MULQ 80(SP)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ 144(SP),AX
- MULQ 88(SP)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 0(SP),AX
- MULQ 104(SP)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 0(SP),AX
- MULQ 112(SP)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 152(SP),AX
- MULQ 80(SP)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 8(SP),AX
- MULQ 96(SP)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 8(SP),AX
- MULQ 104(SP)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 8(SP),AX
- MULQ 112(SP)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ $REDMASK51,DX
- SHLQ $13,SI,CX
- ANDQ DX,SI
- SHLQ $13,R8,R9
- ANDQ DX,R8
- ADDQ CX,R8
- SHLQ $13,R10,R11
- ANDQ DX,R10
- ADDQ R9,R10
- SHLQ $13,R12,R13
- ANDQ DX,R12
- ADDQ R11,R12
- SHLQ $13,R14,R15
- ANDQ DX,R14
- ADDQ R13,R14
- IMUL3Q $19,R15,CX
- ADDQ CX,SI
- MOVQ SI,CX
- SHRQ $51,CX
- ADDQ R8,CX
- MOVQ CX,R8
- SHRQ $51,CX
- ANDQ DX,SI
- ADDQ R10,CX
- MOVQ CX,R9
- SHRQ $51,CX
- ANDQ DX,R8
- ADDQ R12,CX
- MOVQ CX,AX
- SHRQ $51,CX
- ANDQ DX,R9
- ADDQ R14,CX
- MOVQ CX,R10
- SHRQ $51,CX
- ANDQ DX,AX
- IMUL3Q $19,CX,CX
- ADDQ CX,SI
- ANDQ DX,R10
- MOVQ SI,40(DI)
- MOVQ R8,48(DI)
- MOVQ R9,56(DI)
- MOVQ AX,64(DI)
- MOVQ R10,72(DI)
- MOVQ 160(SP),AX
- MULQ ·_121666_213(SB)
- SHRQ $13,AX
- MOVQ AX,SI
- MOVQ DX,CX
- MOVQ 168(SP),AX
- MULQ ·_121666_213(SB)
- SHRQ $13,AX
- ADDQ AX,CX
- MOVQ DX,R8
- MOVQ 176(SP),AX
- MULQ ·_121666_213(SB)
- SHRQ $13,AX
- ADDQ AX,R8
- MOVQ DX,R9
- MOVQ 184(SP),AX
- MULQ ·_121666_213(SB)
- SHRQ $13,AX
- ADDQ AX,R9
- MOVQ DX,R10
- MOVQ 192(SP),AX
- MULQ ·_121666_213(SB)
- SHRQ $13,AX
- ADDQ AX,R10
- IMUL3Q $19,DX,DX
- ADDQ DX,SI
- ADDQ 80(SP),SI
- ADDQ 88(SP),CX
- ADDQ 96(SP),R8
- ADDQ 104(SP),R9
- ADDQ 112(SP),R10
- MOVQ SI,80(DI)
- MOVQ CX,88(DI)
- MOVQ R8,96(DI)
- MOVQ R9,104(DI)
- MOVQ R10,112(DI)
- MOVQ 104(DI),SI
- IMUL3Q $19,SI,AX
- MOVQ AX,0(SP)
- MULQ 176(SP)
- MOVQ AX,SI
- MOVQ DX,CX
- MOVQ 112(DI),DX
- IMUL3Q $19,DX,AX
- MOVQ AX,8(SP)
- MULQ 168(SP)
- ADDQ AX,SI
- ADCQ DX,CX
- MOVQ 80(DI),AX
- MULQ 160(SP)
- ADDQ AX,SI
- ADCQ DX,CX
- MOVQ 80(DI),AX
- MULQ 168(SP)
- MOVQ AX,R8
- MOVQ DX,R9
- MOVQ 80(DI),AX
- MULQ 176(SP)
- MOVQ AX,R10
- MOVQ DX,R11
- MOVQ 80(DI),AX
- MULQ 184(SP)
- MOVQ AX,R12
- MOVQ DX,R13
- MOVQ 80(DI),AX
- MULQ 192(SP)
- MOVQ AX,R14
- MOVQ DX,R15
- MOVQ 88(DI),AX
- MULQ 160(SP)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 88(DI),AX
- MULQ 168(SP)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 88(DI),AX
- MULQ 176(SP)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ 88(DI),AX
- MULQ 184(SP)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 88(DI),DX
- IMUL3Q $19,DX,AX
- MULQ 192(SP)
- ADDQ AX,SI
- ADCQ DX,CX
- MOVQ 96(DI),AX
- MULQ 160(SP)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 96(DI),AX
- MULQ 168(SP)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ 96(DI),AX
- MULQ 176(SP)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 96(DI),DX
- IMUL3Q $19,DX,AX
- MULQ 184(SP)
- ADDQ AX,SI
- ADCQ DX,CX
- MOVQ 96(DI),DX
- IMUL3Q $19,DX,AX
- MULQ 192(SP)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 104(DI),AX
- MULQ 160(SP)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ 104(DI),AX
- MULQ 168(SP)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 0(SP),AX
- MULQ 184(SP)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 0(SP),AX
- MULQ 192(SP)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 112(DI),AX
- MULQ 160(SP)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 8(SP),AX
- MULQ 176(SP)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 8(SP),AX
- MULQ 184(SP)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 8(SP),AX
- MULQ 192(SP)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ $REDMASK51,DX
- SHLQ $13,SI,CX
- ANDQ DX,SI
- SHLQ $13,R8,R9
- ANDQ DX,R8
- ADDQ CX,R8
- SHLQ $13,R10,R11
- ANDQ DX,R10
- ADDQ R9,R10
- SHLQ $13,R12,R13
- ANDQ DX,R12
- ADDQ R11,R12
- SHLQ $13,R14,R15
- ANDQ DX,R14
- ADDQ R13,R14
- IMUL3Q $19,R15,CX
- ADDQ CX,SI
- MOVQ SI,CX
- SHRQ $51,CX
- ADDQ R8,CX
- MOVQ CX,R8
- SHRQ $51,CX
- ANDQ DX,SI
- ADDQ R10,CX
- MOVQ CX,R9
- SHRQ $51,CX
- ANDQ DX,R8
- ADDQ R12,CX
- MOVQ CX,AX
- SHRQ $51,CX
- ANDQ DX,R9
- ADDQ R14,CX
- MOVQ CX,R10
- SHRQ $51,CX
- ANDQ DX,AX
- IMUL3Q $19,CX,CX
- ADDQ CX,SI
- ANDQ DX,R10
- MOVQ SI,80(DI)
- MOVQ R8,88(DI)
- MOVQ R9,96(DI)
- MOVQ AX,104(DI)
- MOVQ R10,112(DI)
- RET
+++ /dev/null
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build amd64,!gccgo,!appengine
-
-package curve25519
-
-// These functions are implemented in the .s files. The names of the functions
-// in the rest of the file are also taken from the SUPERCOP sources to help
-// people following along.
-
-//go:noescape
-
-func cswap(inout *[5]uint64, v uint64)
-
-//go:noescape
-
-func ladderstep(inout *[5][5]uint64)
-
-//go:noescape
-
-func freeze(inout *[5]uint64)
-
-//go:noescape
-
-func mul(dest, a, b *[5]uint64)
-
-//go:noescape
-
-func square(out, in *[5]uint64)
-
-// mladder uses a Montgomery ladder to calculate (xr/zr) *= s.
-func mladder(xr, zr *[5]uint64, s *[32]byte) {
- var work [5][5]uint64
-
- work[0] = *xr
- setint(&work[1], 1)
- setint(&work[2], 0)
- work[3] = *xr
- setint(&work[4], 1)
-
- j := uint(6)
- var prevbit byte
-
- for i := 31; i >= 0; i-- {
- for j < 8 {
- bit := ((*s)[i] >> j) & 1
- swap := bit ^ prevbit
- prevbit = bit
- cswap(&work[1], uint64(swap))
- ladderstep(&work)
- j--
- }
- j = 7
- }
-
- *xr = work[1]
- *zr = work[2]
-}
-
-func scalarMult(out, in, base *[32]byte) {
- var e [32]byte
- copy(e[:], (*in)[:])
- e[0] &= 248
- e[31] &= 127
- e[31] |= 64
-
- var t, z [5]uint64
- unpack(&t, base)
- mladder(&t, &z, &e)
- invert(&z, &z)
- mul(&t, &t, &z)
- pack(out, &t)
-}
-
-func setint(r *[5]uint64, v uint64) {
- r[0] = v
- r[1] = 0
- r[2] = 0
- r[3] = 0
- r[4] = 0
-}
-
-// unpack sets r = x where r consists of 5, 51-bit limbs in little-endian
-// order.
-func unpack(r *[5]uint64, x *[32]byte) {
- r[0] = uint64(x[0]) |
- uint64(x[1])<<8 |
- uint64(x[2])<<16 |
- uint64(x[3])<<24 |
- uint64(x[4])<<32 |
- uint64(x[5])<<40 |
- uint64(x[6]&7)<<48
-
- r[1] = uint64(x[6])>>3 |
- uint64(x[7])<<5 |
- uint64(x[8])<<13 |
- uint64(x[9])<<21 |
- uint64(x[10])<<29 |
- uint64(x[11])<<37 |
- uint64(x[12]&63)<<45
-
- r[2] = uint64(x[12])>>6 |
- uint64(x[13])<<2 |
- uint64(x[14])<<10 |
- uint64(x[15])<<18 |
- uint64(x[16])<<26 |
- uint64(x[17])<<34 |
- uint64(x[18])<<42 |
- uint64(x[19]&1)<<50
-
- r[3] = uint64(x[19])>>1 |
- uint64(x[20])<<7 |
- uint64(x[21])<<15 |
- uint64(x[22])<<23 |
- uint64(x[23])<<31 |
- uint64(x[24])<<39 |
- uint64(x[25]&15)<<47
-
- r[4] = uint64(x[25])>>4 |
- uint64(x[26])<<4 |
- uint64(x[27])<<12 |
- uint64(x[28])<<20 |
- uint64(x[29])<<28 |
- uint64(x[30])<<36 |
- uint64(x[31]&127)<<44
-}
-
-// pack sets out = x where out is the usual, little-endian form of the 5,
-// 51-bit limbs in x.
-func pack(out *[32]byte, x *[5]uint64) {
- t := *x
- freeze(&t)
-
- out[0] = byte(t[0])
- out[1] = byte(t[0] >> 8)
- out[2] = byte(t[0] >> 16)
- out[3] = byte(t[0] >> 24)
- out[4] = byte(t[0] >> 32)
- out[5] = byte(t[0] >> 40)
- out[6] = byte(t[0] >> 48)
-
- out[6] ^= byte(t[1]<<3) & 0xf8
- out[7] = byte(t[1] >> 5)
- out[8] = byte(t[1] >> 13)
- out[9] = byte(t[1] >> 21)
- out[10] = byte(t[1] >> 29)
- out[11] = byte(t[1] >> 37)
- out[12] = byte(t[1] >> 45)
-
- out[12] ^= byte(t[2]<<6) & 0xc0
- out[13] = byte(t[2] >> 2)
- out[14] = byte(t[2] >> 10)
- out[15] = byte(t[2] >> 18)
- out[16] = byte(t[2] >> 26)
- out[17] = byte(t[2] >> 34)
- out[18] = byte(t[2] >> 42)
- out[19] = byte(t[2] >> 50)
-
- out[19] ^= byte(t[3]<<1) & 0xfe
- out[20] = byte(t[3] >> 7)
- out[21] = byte(t[3] >> 15)
- out[22] = byte(t[3] >> 23)
- out[23] = byte(t[3] >> 31)
- out[24] = byte(t[3] >> 39)
- out[25] = byte(t[3] >> 47)
-
- out[25] ^= byte(t[4]<<4) & 0xf0
- out[26] = byte(t[4] >> 4)
- out[27] = byte(t[4] >> 12)
- out[28] = byte(t[4] >> 20)
- out[29] = byte(t[4] >> 28)
- out[30] = byte(t[4] >> 36)
- out[31] = byte(t[4] >> 44)
-}
-
-// invert calculates r = x^-1 mod p using Fermat's little theorem.
-func invert(r *[5]uint64, x *[5]uint64) {
- var z2, z9, z11, z2_5_0, z2_10_0, z2_20_0, z2_50_0, z2_100_0, t [5]uint64
-
- square(&z2, x) /* 2 */
- square(&t, &z2) /* 4 */
- square(&t, &t) /* 8 */
- mul(&z9, &t, x) /* 9 */
- mul(&z11, &z9, &z2) /* 11 */
- square(&t, &z11) /* 22 */
- mul(&z2_5_0, &t, &z9) /* 2^5 - 2^0 = 31 */
-
- square(&t, &z2_5_0) /* 2^6 - 2^1 */
- for i := 1; i < 5; i++ { /* 2^20 - 2^10 */
- square(&t, &t)
- }
- mul(&z2_10_0, &t, &z2_5_0) /* 2^10 - 2^0 */
-
- square(&t, &z2_10_0) /* 2^11 - 2^1 */
- for i := 1; i < 10; i++ { /* 2^20 - 2^10 */
- square(&t, &t)
- }
- mul(&z2_20_0, &t, &z2_10_0) /* 2^20 - 2^0 */
-
- square(&t, &z2_20_0) /* 2^21 - 2^1 */
- for i := 1; i < 20; i++ { /* 2^40 - 2^20 */
- square(&t, &t)
- }
- mul(&t, &t, &z2_20_0) /* 2^40 - 2^0 */
-
- square(&t, &t) /* 2^41 - 2^1 */
- for i := 1; i < 10; i++ { /* 2^50 - 2^10 */
- square(&t, &t)
- }
- mul(&z2_50_0, &t, &z2_10_0) /* 2^50 - 2^0 */
-
- square(&t, &z2_50_0) /* 2^51 - 2^1 */
- for i := 1; i < 50; i++ { /* 2^100 - 2^50 */
- square(&t, &t)
- }
- mul(&z2_100_0, &t, &z2_50_0) /* 2^100 - 2^0 */
-
- square(&t, &z2_100_0) /* 2^101 - 2^1 */
- for i := 1; i < 100; i++ { /* 2^200 - 2^100 */
- square(&t, &t)
- }
- mul(&t, &t, &z2_100_0) /* 2^200 - 2^0 */
-
- square(&t, &t) /* 2^201 - 2^1 */
- for i := 1; i < 50; i++ { /* 2^250 - 2^50 */
- square(&t, &t)
- }
- mul(&t, &t, &z2_50_0) /* 2^250 - 2^0 */
-
- square(&t, &t) /* 2^251 - 2^1 */
- square(&t, &t) /* 2^252 - 2^2 */
- square(&t, &t) /* 2^253 - 2^3 */
-
- square(&t, &t) /* 2^254 - 2^4 */
-
- square(&t, &t) /* 2^255 - 2^5 */
- mul(r, &t, &z11) /* 2^255 - 21 */
-}
+++ /dev/null
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// This code was translated into a form compatible with 6a from the public
-// domain sources in SUPERCOP: https://bench.cr.yp.to/supercop.html
-
-// +build amd64,!gccgo,!appengine
-
-#include "const_amd64.h"
-
-// func mul(dest, a, b *[5]uint64)
-TEXT ·mul(SB),0,$16-24
- MOVQ dest+0(FP), DI
- MOVQ a+8(FP), SI
- MOVQ b+16(FP), DX
-
- MOVQ DX,CX
- MOVQ 24(SI),DX
- IMUL3Q $19,DX,AX
- MOVQ AX,0(SP)
- MULQ 16(CX)
- MOVQ AX,R8
- MOVQ DX,R9
- MOVQ 32(SI),DX
- IMUL3Q $19,DX,AX
- MOVQ AX,8(SP)
- MULQ 8(CX)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 0(SI),AX
- MULQ 0(CX)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 0(SI),AX
- MULQ 8(CX)
- MOVQ AX,R10
- MOVQ DX,R11
- MOVQ 0(SI),AX
- MULQ 16(CX)
- MOVQ AX,R12
- MOVQ DX,R13
- MOVQ 0(SI),AX
- MULQ 24(CX)
- MOVQ AX,R14
- MOVQ DX,R15
- MOVQ 0(SI),AX
- MULQ 32(CX)
- MOVQ AX,BX
- MOVQ DX,BP
- MOVQ 8(SI),AX
- MULQ 0(CX)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 8(SI),AX
- MULQ 8(CX)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ 8(SI),AX
- MULQ 16(CX)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 8(SI),AX
- MULQ 24(CX)
- ADDQ AX,BX
- ADCQ DX,BP
- MOVQ 8(SI),DX
- IMUL3Q $19,DX,AX
- MULQ 32(CX)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 16(SI),AX
- MULQ 0(CX)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ 16(SI),AX
- MULQ 8(CX)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 16(SI),AX
- MULQ 16(CX)
- ADDQ AX,BX
- ADCQ DX,BP
- MOVQ 16(SI),DX
- IMUL3Q $19,DX,AX
- MULQ 24(CX)
- ADDQ AX,R8
- ADCQ DX,R9
- MOVQ 16(SI),DX
- IMUL3Q $19,DX,AX
- MULQ 32(CX)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 24(SI),AX
- MULQ 0(CX)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ 24(SI),AX
- MULQ 8(CX)
- ADDQ AX,BX
- ADCQ DX,BP
- MOVQ 0(SP),AX
- MULQ 24(CX)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 0(SP),AX
- MULQ 32(CX)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ 32(SI),AX
- MULQ 0(CX)
- ADDQ AX,BX
- ADCQ DX,BP
- MOVQ 8(SP),AX
- MULQ 16(CX)
- ADDQ AX,R10
- ADCQ DX,R11
- MOVQ 8(SP),AX
- MULQ 24(CX)
- ADDQ AX,R12
- ADCQ DX,R13
- MOVQ 8(SP),AX
- MULQ 32(CX)
- ADDQ AX,R14
- ADCQ DX,R15
- MOVQ $REDMASK51,SI
- SHLQ $13,R8,R9
- ANDQ SI,R8
- SHLQ $13,R10,R11
- ANDQ SI,R10
- ADDQ R9,R10
- SHLQ $13,R12,R13
- ANDQ SI,R12
- ADDQ R11,R12
- SHLQ $13,R14,R15
- ANDQ SI,R14
- ADDQ R13,R14
- SHLQ $13,BX,BP
- ANDQ SI,BX
- ADDQ R15,BX
- IMUL3Q $19,BP,DX
- ADDQ DX,R8
- MOVQ R8,DX
- SHRQ $51,DX
- ADDQ R10,DX
- MOVQ DX,CX
- SHRQ $51,DX
- ANDQ SI,R8
- ADDQ R12,DX
- MOVQ DX,R9
- SHRQ $51,DX
- ANDQ SI,CX
- ADDQ R14,DX
- MOVQ DX,AX
- SHRQ $51,DX
- ANDQ SI,R9
- ADDQ BX,DX
- MOVQ DX,R10
- SHRQ $51,DX
- ANDQ SI,AX
- IMUL3Q $19,DX,DX
- ADDQ DX,R8
- ANDQ SI,R10
- MOVQ R8,0(DI)
- MOVQ CX,8(DI)
- MOVQ R9,16(DI)
- MOVQ AX,24(DI)
- MOVQ R10,32(DI)
- RET
+++ /dev/null
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// This code was translated into a form compatible with 6a from the public
-// domain sources in SUPERCOP: https://bench.cr.yp.to/supercop.html
-
-// +build amd64,!gccgo,!appengine
-
-#include "const_amd64.h"
-
-// func square(out, in *[5]uint64)
-TEXT ·square(SB),7,$0-16
- MOVQ out+0(FP), DI
- MOVQ in+8(FP), SI
-
- MOVQ 0(SI),AX
- MULQ 0(SI)
- MOVQ AX,CX
- MOVQ DX,R8
- MOVQ 0(SI),AX
- SHLQ $1,AX
- MULQ 8(SI)
- MOVQ AX,R9
- MOVQ DX,R10
- MOVQ 0(SI),AX
- SHLQ $1,AX
- MULQ 16(SI)
- MOVQ AX,R11
- MOVQ DX,R12
- MOVQ 0(SI),AX
- SHLQ $1,AX
- MULQ 24(SI)
- MOVQ AX,R13
- MOVQ DX,R14
- MOVQ 0(SI),AX
- SHLQ $1,AX
- MULQ 32(SI)
- MOVQ AX,R15
- MOVQ DX,BX
- MOVQ 8(SI),AX
- MULQ 8(SI)
- ADDQ AX,R11
- ADCQ DX,R12
- MOVQ 8(SI),AX
- SHLQ $1,AX
- MULQ 16(SI)
- ADDQ AX,R13
- ADCQ DX,R14
- MOVQ 8(SI),AX
- SHLQ $1,AX
- MULQ 24(SI)
- ADDQ AX,R15
- ADCQ DX,BX
- MOVQ 8(SI),DX
- IMUL3Q $38,DX,AX
- MULQ 32(SI)
- ADDQ AX,CX
- ADCQ DX,R8
- MOVQ 16(SI),AX
- MULQ 16(SI)
- ADDQ AX,R15
- ADCQ DX,BX
- MOVQ 16(SI),DX
- IMUL3Q $38,DX,AX
- MULQ 24(SI)
- ADDQ AX,CX
- ADCQ DX,R8
- MOVQ 16(SI),DX
- IMUL3Q $38,DX,AX
- MULQ 32(SI)
- ADDQ AX,R9
- ADCQ DX,R10
- MOVQ 24(SI),DX
- IMUL3Q $19,DX,AX
- MULQ 24(SI)
- ADDQ AX,R9
- ADCQ DX,R10
- MOVQ 24(SI),DX
- IMUL3Q $38,DX,AX
- MULQ 32(SI)
- ADDQ AX,R11
- ADCQ DX,R12
- MOVQ 32(SI),DX
- IMUL3Q $19,DX,AX
- MULQ 32(SI)
- ADDQ AX,R13
- ADCQ DX,R14
- MOVQ $REDMASK51,SI
- SHLQ $13,CX,R8
- ANDQ SI,CX
- SHLQ $13,R9,R10
- ANDQ SI,R9
- ADDQ R8,R9
- SHLQ $13,R11,R12
- ANDQ SI,R11
- ADDQ R10,R11
- SHLQ $13,R13,R14
- ANDQ SI,R13
- ADDQ R12,R13
- SHLQ $13,R15,BX
- ANDQ SI,R15
- ADDQ R14,R15
- IMUL3Q $19,BX,DX
- ADDQ DX,CX
- MOVQ CX,DX
- SHRQ $51,DX
- ADDQ R9,DX
- ANDQ SI,CX
- MOVQ DX,R8
- SHRQ $51,DX
- ADDQ R11,DX
- ANDQ SI,R8
- MOVQ DX,R9
- SHRQ $51,DX
- ADDQ R13,DX
- ANDQ SI,R9
- MOVQ DX,AX
- SHRQ $51,DX
- ADDQ R15,DX
- ANDQ SI,AX
- MOVQ DX,R10
- SHRQ $51,DX
- IMUL3Q $19,DX,DX
- ADDQ DX,CX
- ANDQ SI,R10
- MOVQ CX,0(DI)
- MOVQ R8,8(DI)
- MOVQ R9,16(DI)
- MOVQ AX,24(DI)
- MOVQ R10,32(DI)
- RET
+++ /dev/null
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build go1.11
-// +build !gccgo,!appengine
-
-#include "textflag.h"
-
-#define NUM_ROUNDS 10
-
-// func xorKeyStreamVX(dst, src []byte, key *[8]uint32, nonce *[3]uint32, counter *uint32)
-TEXT ·xorKeyStreamVX(SB), NOSPLIT, $0
- MOVD dst+0(FP), R1
- MOVD src+24(FP), R2
- MOVD src_len+32(FP), R3
- MOVD key+48(FP), R4
- MOVD nonce+56(FP), R6
- MOVD counter+64(FP), R7
-
- MOVD $·constants(SB), R10
- MOVD $·incRotMatrix(SB), R11
-
- MOVW (R7), R20
-
- AND $~255, R3, R13
- ADD R2, R13, R12 // R12 for block end
- AND $255, R3, R13
-loop:
- MOVD $NUM_ROUNDS, R21
- VLD1 (R11), [V30.S4, V31.S4]
-
- // load contants
- // VLD4R (R10), [V0.S4, V1.S4, V2.S4, V3.S4]
- WORD $0x4D60E940
-
- // load keys
- // VLD4R 16(R4), [V4.S4, V5.S4, V6.S4, V7.S4]
- WORD $0x4DFFE884
- // VLD4R 16(R4), [V8.S4, V9.S4, V10.S4, V11.S4]
- WORD $0x4DFFE888
- SUB $32, R4
-
- // load counter + nonce
- // VLD1R (R7), [V12.S4]
- WORD $0x4D40C8EC
-
- // VLD3R (R6), [V13.S4, V14.S4, V15.S4]
- WORD $0x4D40E8CD
-
- // update counter
- VADD V30.S4, V12.S4, V12.S4
-
-chacha:
- // V0..V3 += V4..V7
- // V12..V15 <<<= ((V12..V15 XOR V0..V3), 16)
- VADD V0.S4, V4.S4, V0.S4
- VADD V1.S4, V5.S4, V1.S4
- VADD V2.S4, V6.S4, V2.S4
- VADD V3.S4, V7.S4, V3.S4
- VEOR V12.B16, V0.B16, V12.B16
- VEOR V13.B16, V1.B16, V13.B16
- VEOR V14.B16, V2.B16, V14.B16
- VEOR V15.B16, V3.B16, V15.B16
- VREV32 V12.H8, V12.H8
- VREV32 V13.H8, V13.H8
- VREV32 V14.H8, V14.H8
- VREV32 V15.H8, V15.H8
- // V8..V11 += V12..V15
- // V4..V7 <<<= ((V4..V7 XOR V8..V11), 12)
- VADD V8.S4, V12.S4, V8.S4
- VADD V9.S4, V13.S4, V9.S4
- VADD V10.S4, V14.S4, V10.S4
- VADD V11.S4, V15.S4, V11.S4
- VEOR V8.B16, V4.B16, V16.B16
- VEOR V9.B16, V5.B16, V17.B16
- VEOR V10.B16, V6.B16, V18.B16
- VEOR V11.B16, V7.B16, V19.B16
- VSHL $12, V16.S4, V4.S4
- VSHL $12, V17.S4, V5.S4
- VSHL $12, V18.S4, V6.S4
- VSHL $12, V19.S4, V7.S4
- VSRI $20, V16.S4, V4.S4
- VSRI $20, V17.S4, V5.S4
- VSRI $20, V18.S4, V6.S4
- VSRI $20, V19.S4, V7.S4
-
- // V0..V3 += V4..V7
- // V12..V15 <<<= ((V12..V15 XOR V0..V3), 8)
- VADD V0.S4, V4.S4, V0.S4
- VADD V1.S4, V5.S4, V1.S4
- VADD V2.S4, V6.S4, V2.S4
- VADD V3.S4, V7.S4, V3.S4
- VEOR V12.B16, V0.B16, V12.B16
- VEOR V13.B16, V1.B16, V13.B16
- VEOR V14.B16, V2.B16, V14.B16
- VEOR V15.B16, V3.B16, V15.B16
- VTBL V31.B16, [V12.B16], V12.B16
- VTBL V31.B16, [V13.B16], V13.B16
- VTBL V31.B16, [V14.B16], V14.B16
- VTBL V31.B16, [V15.B16], V15.B16
-
- // V8..V11 += V12..V15
- // V4..V7 <<<= ((V4..V7 XOR V8..V11), 7)
- VADD V12.S4, V8.S4, V8.S4
- VADD V13.S4, V9.S4, V9.S4
- VADD V14.S4, V10.S4, V10.S4
- VADD V15.S4, V11.S4, V11.S4
- VEOR V8.B16, V4.B16, V16.B16
- VEOR V9.B16, V5.B16, V17.B16
- VEOR V10.B16, V6.B16, V18.B16
- VEOR V11.B16, V7.B16, V19.B16
- VSHL $7, V16.S4, V4.S4
- VSHL $7, V17.S4, V5.S4
- VSHL $7, V18.S4, V6.S4
- VSHL $7, V19.S4, V7.S4
- VSRI $25, V16.S4, V4.S4
- VSRI $25, V17.S4, V5.S4
- VSRI $25, V18.S4, V6.S4
- VSRI $25, V19.S4, V7.S4
-
- // V0..V3 += V5..V7, V4
- // V15,V12-V14 <<<= ((V15,V12-V14 XOR V0..V3), 16)
- VADD V0.S4, V5.S4, V0.S4
- VADD V1.S4, V6.S4, V1.S4
- VADD V2.S4, V7.S4, V2.S4
- VADD V3.S4, V4.S4, V3.S4
- VEOR V15.B16, V0.B16, V15.B16
- VEOR V12.B16, V1.B16, V12.B16
- VEOR V13.B16, V2.B16, V13.B16
- VEOR V14.B16, V3.B16, V14.B16
- VREV32 V12.H8, V12.H8
- VREV32 V13.H8, V13.H8
- VREV32 V14.H8, V14.H8
- VREV32 V15.H8, V15.H8
-
- // V10 += V15; V5 <<<= ((V10 XOR V5), 12)
- // ...
- VADD V15.S4, V10.S4, V10.S4
- VADD V12.S4, V11.S4, V11.S4
- VADD V13.S4, V8.S4, V8.S4
- VADD V14.S4, V9.S4, V9.S4
- VEOR V10.B16, V5.B16, V16.B16
- VEOR V11.B16, V6.B16, V17.B16
- VEOR V8.B16, V7.B16, V18.B16
- VEOR V9.B16, V4.B16, V19.B16
- VSHL $12, V16.S4, V5.S4
- VSHL $12, V17.S4, V6.S4
- VSHL $12, V18.S4, V7.S4
- VSHL $12, V19.S4, V4.S4
- VSRI $20, V16.S4, V5.S4
- VSRI $20, V17.S4, V6.S4
- VSRI $20, V18.S4, V7.S4
- VSRI $20, V19.S4, V4.S4
-
- // V0 += V5; V15 <<<= ((V0 XOR V15), 8)
- // ...
- VADD V5.S4, V0.S4, V0.S4
- VADD V6.S4, V1.S4, V1.S4
- VADD V7.S4, V2.S4, V2.S4
- VADD V4.S4, V3.S4, V3.S4
- VEOR V0.B16, V15.B16, V15.B16
- VEOR V1.B16, V12.B16, V12.B16
- VEOR V2.B16, V13.B16, V13.B16
- VEOR V3.B16, V14.B16, V14.B16
- VTBL V31.B16, [V12.B16], V12.B16
- VTBL V31.B16, [V13.B16], V13.B16
- VTBL V31.B16, [V14.B16], V14.B16
- VTBL V31.B16, [V15.B16], V15.B16
-
- // V10 += V15; V5 <<<= ((V10 XOR V5), 7)
- // ...
- VADD V15.S4, V10.S4, V10.S4
- VADD V12.S4, V11.S4, V11.S4
- VADD V13.S4, V8.S4, V8.S4
- VADD V14.S4, V9.S4, V9.S4
- VEOR V10.B16, V5.B16, V16.B16
- VEOR V11.B16, V6.B16, V17.B16
- VEOR V8.B16, V7.B16, V18.B16
- VEOR V9.B16, V4.B16, V19.B16
- VSHL $7, V16.S4, V5.S4
- VSHL $7, V17.S4, V6.S4
- VSHL $7, V18.S4, V7.S4
- VSHL $7, V19.S4, V4.S4
- VSRI $25, V16.S4, V5.S4
- VSRI $25, V17.S4, V6.S4
- VSRI $25, V18.S4, V7.S4
- VSRI $25, V19.S4, V4.S4
-
- SUB $1, R21
- CBNZ R21, chacha
-
- // VLD4R (R10), [V16.S4, V17.S4, V18.S4, V19.S4]
- WORD $0x4D60E950
-
- // VLD4R 16(R4), [V20.S4, V21.S4, V22.S4, V23.S4]
- WORD $0x4DFFE894
- VADD V30.S4, V12.S4, V12.S4
- VADD V16.S4, V0.S4, V0.S4
- VADD V17.S4, V1.S4, V1.S4
- VADD V18.S4, V2.S4, V2.S4
- VADD V19.S4, V3.S4, V3.S4
- // VLD4R 16(R4), [V24.S4, V25.S4, V26.S4, V27.S4]
- WORD $0x4DFFE898
- // restore R4
- SUB $32, R4
-
- // load counter + nonce
- // VLD1R (R7), [V28.S4]
- WORD $0x4D40C8FC
- // VLD3R (R6), [V29.S4, V30.S4, V31.S4]
- WORD $0x4D40E8DD
-
- VADD V20.S4, V4.S4, V4.S4
- VADD V21.S4, V5.S4, V5.S4
- VADD V22.S4, V6.S4, V6.S4
- VADD V23.S4, V7.S4, V7.S4
- VADD V24.S4, V8.S4, V8.S4
- VADD V25.S4, V9.S4, V9.S4
- VADD V26.S4, V10.S4, V10.S4
- VADD V27.S4, V11.S4, V11.S4
- VADD V28.S4, V12.S4, V12.S4
- VADD V29.S4, V13.S4, V13.S4
- VADD V30.S4, V14.S4, V14.S4
- VADD V31.S4, V15.S4, V15.S4
-
- VZIP1 V1.S4, V0.S4, V16.S4
- VZIP2 V1.S4, V0.S4, V17.S4
- VZIP1 V3.S4, V2.S4, V18.S4
- VZIP2 V3.S4, V2.S4, V19.S4
- VZIP1 V5.S4, V4.S4, V20.S4
- VZIP2 V5.S4, V4.S4, V21.S4
- VZIP1 V7.S4, V6.S4, V22.S4
- VZIP2 V7.S4, V6.S4, V23.S4
- VZIP1 V9.S4, V8.S4, V24.S4
- VZIP2 V9.S4, V8.S4, V25.S4
- VZIP1 V11.S4, V10.S4, V26.S4
- VZIP2 V11.S4, V10.S4, V27.S4
- VZIP1 V13.S4, V12.S4, V28.S4
- VZIP2 V13.S4, V12.S4, V29.S4
- VZIP1 V15.S4, V14.S4, V30.S4
- VZIP2 V15.S4, V14.S4, V31.S4
- VZIP1 V18.D2, V16.D2, V0.D2
- VZIP2 V18.D2, V16.D2, V4.D2
- VZIP1 V19.D2, V17.D2, V8.D2
- VZIP2 V19.D2, V17.D2, V12.D2
- VLD1.P 64(R2), [V16.B16, V17.B16, V18.B16, V19.B16]
-
- VZIP1 V22.D2, V20.D2, V1.D2
- VZIP2 V22.D2, V20.D2, V5.D2
- VZIP1 V23.D2, V21.D2, V9.D2
- VZIP2 V23.D2, V21.D2, V13.D2
- VLD1.P 64(R2), [V20.B16, V21.B16, V22.B16, V23.B16]
- VZIP1 V26.D2, V24.D2, V2.D2
- VZIP2 V26.D2, V24.D2, V6.D2
- VZIP1 V27.D2, V25.D2, V10.D2
- VZIP2 V27.D2, V25.D2, V14.D2
- VLD1.P 64(R2), [V24.B16, V25.B16, V26.B16, V27.B16]
- VZIP1 V30.D2, V28.D2, V3.D2
- VZIP2 V30.D2, V28.D2, V7.D2
- VZIP1 V31.D2, V29.D2, V11.D2
- VZIP2 V31.D2, V29.D2, V15.D2
- VLD1.P 64(R2), [V28.B16, V29.B16, V30.B16, V31.B16]
- VEOR V0.B16, V16.B16, V16.B16
- VEOR V1.B16, V17.B16, V17.B16
- VEOR V2.B16, V18.B16, V18.B16
- VEOR V3.B16, V19.B16, V19.B16
- VST1.P [V16.B16, V17.B16, V18.B16, V19.B16], 64(R1)
- VEOR V4.B16, V20.B16, V20.B16
- VEOR V5.B16, V21.B16, V21.B16
- VEOR V6.B16, V22.B16, V22.B16
- VEOR V7.B16, V23.B16, V23.B16
- VST1.P [V20.B16, V21.B16, V22.B16, V23.B16], 64(R1)
- VEOR V8.B16, V24.B16, V24.B16
- VEOR V9.B16, V25.B16, V25.B16
- VEOR V10.B16, V26.B16, V26.B16
- VEOR V11.B16, V27.B16, V27.B16
- VST1.P [V24.B16, V25.B16, V26.B16, V27.B16], 64(R1)
- VEOR V12.B16, V28.B16, V28.B16
- VEOR V13.B16, V29.B16, V29.B16
- VEOR V14.B16, V30.B16, V30.B16
- VEOR V15.B16, V31.B16, V31.B16
- VST1.P [V28.B16, V29.B16, V30.B16, V31.B16], 64(R1)
-
- ADD $4, R20
- MOVW R20, (R7) // update counter
-
- CMP R2, R12
- BGT loop
-
- RET
-
-
-DATA ·constants+0x00(SB)/4, $0x61707865
-DATA ·constants+0x04(SB)/4, $0x3320646e
-DATA ·constants+0x08(SB)/4, $0x79622d32
-DATA ·constants+0x0c(SB)/4, $0x6b206574
-GLOBL ·constants(SB), NOPTR|RODATA, $32
-
-DATA ·incRotMatrix+0x00(SB)/4, $0x00000000
-DATA ·incRotMatrix+0x04(SB)/4, $0x00000001
-DATA ·incRotMatrix+0x08(SB)/4, $0x00000002
-DATA ·incRotMatrix+0x0c(SB)/4, $0x00000003
-DATA ·incRotMatrix+0x10(SB)/4, $0x02010003
-DATA ·incRotMatrix+0x14(SB)/4, $0x06050407
-DATA ·incRotMatrix+0x18(SB)/4, $0x0A09080B
-DATA ·incRotMatrix+0x1c(SB)/4, $0x0E0D0C0F
-GLOBL ·incRotMatrix(SB), NOPTR|RODATA, $32
+++ /dev/null
-// Copyright 2019 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Based on CRYPTOGAMS code with the following comment:
-// # ====================================================================
-// # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
-// # project. The module is, however, dual licensed under OpenSSL and
-// # CRYPTOGAMS licenses depending on where you obtain it. For further
-// # details see http://www.openssl.org/~appro/cryptogams/.
-// # ====================================================================
-
-// Original code can be found at the link below:
-// https://github.com/dot-asm/cryptogams/commit/a60f5b50ed908e91e5c39ca79126a4a876d5d8ff
-
-// There are some differences between CRYPTOGAMS code and this one. The round
-// loop for "_int" isn't the same as the original. Some adjustments were
-// necessary because there are less vector registers available. For example, some
-// X variables (r12, r13, r14, and r15) share the same register used by the
-// counter. The original code uses ctr to name the counter. Here we use CNT
-// because golang uses CTR as the counter register name.
-
-// +build ppc64le,!gccgo,!appengine
-
-#include "textflag.h"
-
-#define OUT R3
-#define INP R4
-#define LEN R5
-#define KEY R6
-#define CNT R7
-
-#define TEMP R8
-
-#define X0 R11
-#define X1 R12
-#define X2 R14
-#define X3 R15
-#define X4 R16
-#define X5 R17
-#define X6 R18
-#define X7 R19
-#define X8 R20
-#define X9 R21
-#define X10 R22
-#define X11 R23
-#define X12 R24
-#define X13 R25
-#define X14 R26
-#define X15 R27
-
-#define CON0 X0
-#define CON1 X1
-#define CON2 X2
-#define CON3 X3
-
-#define KEY0 X4
-#define KEY1 X5
-#define KEY2 X6
-#define KEY3 X7
-#define KEY4 X8
-#define KEY5 X9
-#define KEY6 X10
-#define KEY7 X11
-
-#define CNT0 X12
-#define CNT1 X13
-#define CNT2 X14
-#define CNT3 X15
-
-#define TMP0 R9
-#define TMP1 R10
-#define TMP2 R28
-#define TMP3 R29
-
-#define CONSTS R8
-
-#define A0 V0
-#define B0 V1
-#define C0 V2
-#define D0 V3
-#define A1 V4
-#define B1 V5
-#define C1 V6
-#define D1 V7
-#define A2 V8
-#define B2 V9
-#define C2 V10
-#define D2 V11
-#define T0 V12
-#define T1 V13
-#define T2 V14
-
-#define K0 V15
-#define K1 V16
-#define K2 V17
-#define K3 V18
-#define K4 V19
-#define K5 V20
-
-#define FOUR V21
-#define SIXTEEN V22
-#define TWENTY4 V23
-#define TWENTY V24
-#define TWELVE V25
-#define TWENTY5 V26
-#define SEVEN V27
-
-#define INPPERM V28
-#define OUTPERM V29
-#define OUTMASK V30
-
-#define DD0 V31
-#define DD1 SEVEN
-#define DD2 T0
-#define DD3 T1
-#define DD4 T2
-
-DATA ·consts+0x00(SB)/8, $0x3320646e61707865
-DATA ·consts+0x08(SB)/8, $0x6b20657479622d32
-DATA ·consts+0x10(SB)/8, $0x0000000000000001
-DATA ·consts+0x18(SB)/8, $0x0000000000000000
-DATA ·consts+0x20(SB)/8, $0x0000000000000004
-DATA ·consts+0x28(SB)/8, $0x0000000000000000
-DATA ·consts+0x30(SB)/8, $0x0a0b08090e0f0c0d
-DATA ·consts+0x38(SB)/8, $0x0203000106070405
-DATA ·consts+0x40(SB)/8, $0x090a0b080d0e0f0c
-DATA ·consts+0x48(SB)/8, $0x0102030005060704
-GLOBL ·consts(SB), RODATA, $80
-
-//func chaCha20_ctr32_vmx(out, inp *byte, len int, key *[32]byte, counter *[16]byte)
-TEXT ·chaCha20_ctr32_vmx(SB),NOSPLIT|NOFRAME,$0
- // Load the arguments inside the registers
- MOVD out+0(FP), OUT
- MOVD inp+8(FP), INP
- MOVD len+16(FP), LEN
- MOVD key+24(FP), KEY
- MOVD counter+32(FP), CNT
-
- MOVD $·consts(SB), CONSTS // point to consts addr
-
- MOVD $16, X0
- MOVD $32, X1
- MOVD $48, X2
- MOVD $64, X3
- MOVD $31, X4
- MOVD $15, X5
-
- // Load key
- LVX (KEY)(R0), K1
- LVSR (KEY)(R0), T0
- LVX (KEY)(X0), K2
- LVX (KEY)(X4), DD0
-
- // Load counter
- LVX (CNT)(R0), K3
- LVSR (CNT)(R0), T1
- LVX (CNT)(X5), DD1
-
- // Load constants
- LVX (CONSTS)(R0), K0
- LVX (CONSTS)(X0), K5
- LVX (CONSTS)(X1), FOUR
- LVX (CONSTS)(X2), SIXTEEN
- LVX (CONSTS)(X3), TWENTY4
-
- // Align key and counter
- VPERM K2, K1, T0, K1
- VPERM DD0, K2, T0, K2
- VPERM DD1, K3, T1, K3
-
- // Load counter to GPR
- MOVWZ 0(CNT), CNT0
- MOVWZ 4(CNT), CNT1
- MOVWZ 8(CNT), CNT2
- MOVWZ 12(CNT), CNT3
-
- // Adjust vectors for the initial state
- VADDUWM K3, K5, K3
- VADDUWM K3, K5, K4
- VADDUWM K4, K5, K5
-
- // Synthesized constants
- VSPLTISW $-12, TWENTY
- VSPLTISW $12, TWELVE
- VSPLTISW $-7, TWENTY5
-
- VXOR T0, T0, T0
- VSPLTISW $-1, OUTMASK
- LVSR (INP)(R0), INPPERM
- LVSL (OUT)(R0), OUTPERM
- VPERM OUTMASK, T0, OUTPERM, OUTMASK
-
-loop_outer_vmx:
- // Load constant
- MOVD $0x61707865, CON0
- MOVD $0x3320646e, CON1
- MOVD $0x79622d32, CON2
- MOVD $0x6b206574, CON3
-
- VOR K0, K0, A0
- VOR K0, K0, A1
- VOR K0, K0, A2
- VOR K1, K1, B0
-
- MOVD $10, TEMP
-
- // Load key to GPR
- MOVWZ 0(KEY), X4
- MOVWZ 4(KEY), X5
- MOVWZ 8(KEY), X6
- MOVWZ 12(KEY), X7
- VOR K1, K1, B1
- VOR K1, K1, B2
- MOVWZ 16(KEY), X8
- MOVWZ 0(CNT), X12
- MOVWZ 20(KEY), X9
- MOVWZ 4(CNT), X13
- VOR K2, K2, C0
- VOR K2, K2, C1
- MOVWZ 24(KEY), X10
- MOVWZ 8(CNT), X14
- VOR K2, K2, C2
- VOR K3, K3, D0
- MOVWZ 28(KEY), X11
- MOVWZ 12(CNT), X15
- VOR K4, K4, D1
- VOR K5, K5, D2
-
- MOVD X4, TMP0
- MOVD X5, TMP1
- MOVD X6, TMP2
- MOVD X7, TMP3
- VSPLTISW $7, SEVEN
-
- MOVD TEMP, CTR
-
-loop_vmx:
- // CRYPTOGAMS uses a macro to create a loop using perl. This isn't possible
- // using assembly macros. Therefore, the macro expansion result was used
- // in order to maintain the algorithm efficiency.
- // This loop generates three keystream blocks using VMX instructions and,
- // in parallel, one keystream block using scalar instructions.
- ADD X4, X0, X0
- ADD X5, X1, X1
- VADDUWM A0, B0, A0
- VADDUWM A1, B1, A1
- ADD X6, X2, X2
- ADD X7, X3, X3
- VADDUWM A2, B2, A2
- VXOR D0, A0, D0
- XOR X0, X12, X12
- XOR X1, X13, X13
- VXOR D1, A1, D1
- VXOR D2, A2, D2
- XOR X2, X14, X14
- XOR X3, X15, X15
- VPERM D0, D0, SIXTEEN, D0
- VPERM D1, D1, SIXTEEN, D1
- ROTLW $16, X12, X12
- ROTLW $16, X13, X13
- VPERM D2, D2, SIXTEEN, D2
- VADDUWM C0, D0, C0
- ROTLW $16, X14, X14
- ROTLW $16, X15, X15
- VADDUWM C1, D1, C1
- VADDUWM C2, D2, C2
- ADD X12, X8, X8
- ADD X13, X9, X9
- VXOR B0, C0, T0
- VXOR B1, C1, T1
- ADD X14, X10, X10
- ADD X15, X11, X11
- VXOR B2, C2, T2
- VRLW T0, TWELVE, B0
- XOR X8, X4, X4
- XOR X9, X5, X5
- VRLW T1, TWELVE, B1
- VRLW T2, TWELVE, B2
- XOR X10, X6, X6
- XOR X11, X7, X7
- VADDUWM A0, B0, A0
- VADDUWM A1, B1, A1
- ROTLW $12, X4, X4
- ROTLW $12, X5, X5
- VADDUWM A2, B2, A2
- VXOR D0, A0, D0
- ROTLW $12, X6, X6
- ROTLW $12, X7, X7
- VXOR D1, A1, D1
- VXOR D2, A2, D2
- ADD X4, X0, X0
- ADD X5, X1, X1
- VPERM D0, D0, TWENTY4, D0
- VPERM D1, D1, TWENTY4, D1
- ADD X6, X2, X2
- ADD X7, X3, X3
- VPERM D2, D2, TWENTY4, D2
- VADDUWM C0, D0, C0
- XOR X0, X12, X12
- XOR X1, X13, X13
- VADDUWM C1, D1, C1
- VADDUWM C2, D2, C2
- XOR X2, X14, X14
- XOR X3, X15, X15
- VXOR B0, C0, T0
- VXOR B1, C1, T1
- ROTLW $8, X12, X12
- ROTLW $8, X13, X13
- VXOR B2, C2, T2
- VRLW T0, SEVEN, B0
- ROTLW $8, X14, X14
- ROTLW $8, X15, X15
- VRLW T1, SEVEN, B1
- VRLW T2, SEVEN, B2
- ADD X12, X8, X8
- ADD X13, X9, X9
- VSLDOI $8, C0, C0, C0
- VSLDOI $8, C1, C1, C1
- ADD X14, X10, X10
- ADD X15, X11, X11
- VSLDOI $8, C2, C2, C2
- VSLDOI $12, B0, B0, B0
- XOR X8, X4, X4
- XOR X9, X5, X5
- VSLDOI $12, B1, B1, B1
- VSLDOI $12, B2, B2, B2
- XOR X10, X6, X6
- XOR X11, X7, X7
- VSLDOI $4, D0, D0, D0
- VSLDOI $4, D1, D1, D1
- ROTLW $7, X4, X4
- ROTLW $7, X5, X5
- VSLDOI $4, D2, D2, D2
- VADDUWM A0, B0, A0
- ROTLW $7, X6, X6
- ROTLW $7, X7, X7
- VADDUWM A1, B1, A1
- VADDUWM A2, B2, A2
- ADD X5, X0, X0
- ADD X6, X1, X1
- VXOR D0, A0, D0
- VXOR D1, A1, D1
- ADD X7, X2, X2
- ADD X4, X3, X3
- VXOR D2, A2, D2
- VPERM D0, D0, SIXTEEN, D0
- XOR X0, X15, X15
- XOR X1, X12, X12
- VPERM D1, D1, SIXTEEN, D1
- VPERM D2, D2, SIXTEEN, D2
- XOR X2, X13, X13
- XOR X3, X14, X14
- VADDUWM C0, D0, C0
- VADDUWM C1, D1, C1
- ROTLW $16, X15, X15
- ROTLW $16, X12, X12
- VADDUWM C2, D2, C2
- VXOR B0, C0, T0
- ROTLW $16, X13, X13
- ROTLW $16, X14, X14
- VXOR B1, C1, T1
- VXOR B2, C2, T2
- ADD X15, X10, X10
- ADD X12, X11, X11
- VRLW T0, TWELVE, B0
- VRLW T1, TWELVE, B1
- ADD X13, X8, X8
- ADD X14, X9, X9
- VRLW T2, TWELVE, B2
- VADDUWM A0, B0, A0
- XOR X10, X5, X5
- XOR X11, X6, X6
- VADDUWM A1, B1, A1
- VADDUWM A2, B2, A2
- XOR X8, X7, X7
- XOR X9, X4, X4
- VXOR D0, A0, D0
- VXOR D1, A1, D1
- ROTLW $12, X5, X5
- ROTLW $12, X6, X6
- VXOR D2, A2, D2
- VPERM D0, D0, TWENTY4, D0
- ROTLW $12, X7, X7
- ROTLW $12, X4, X4
- VPERM D1, D1, TWENTY4, D1
- VPERM D2, D2, TWENTY4, D2
- ADD X5, X0, X0
- ADD X6, X1, X1
- VADDUWM C0, D0, C0
- VADDUWM C1, D1, C1
- ADD X7, X2, X2
- ADD X4, X3, X3
- VADDUWM C2, D2, C2
- VXOR B0, C0, T0
- XOR X0, X15, X15
- XOR X1, X12, X12
- VXOR B1, C1, T1
- VXOR B2, C2, T2
- XOR X2, X13, X13
- XOR X3, X14, X14
- VRLW T0, SEVEN, B0
- VRLW T1, SEVEN, B1
- ROTLW $8, X15, X15
- ROTLW $8, X12, X12
- VRLW T2, SEVEN, B2
- VSLDOI $8, C0, C0, C0
- ROTLW $8, X13, X13
- ROTLW $8, X14, X14
- VSLDOI $8, C1, C1, C1
- VSLDOI $8, C2, C2, C2
- ADD X15, X10, X10
- ADD X12, X11, X11
- VSLDOI $4, B0, B0, B0
- VSLDOI $4, B1, B1, B1
- ADD X13, X8, X8
- ADD X14, X9, X9
- VSLDOI $4, B2, B2, B2
- VSLDOI $12, D0, D0, D0
- XOR X10, X5, X5
- XOR X11, X6, X6
- VSLDOI $12, D1, D1, D1
- VSLDOI $12, D2, D2, D2
- XOR X8, X7, X7
- XOR X9, X4, X4
- ROTLW $7, X5, X5
- ROTLW $7, X6, X6
- ROTLW $7, X7, X7
- ROTLW $7, X4, X4
- BC 0x10, 0, loop_vmx
-
- SUB $256, LEN, LEN
-
- // Accumulate key block
- ADD $0x61707865, X0, X0
- ADD $0x3320646e, X1, X1
- ADD $0x79622d32, X2, X2
- ADD $0x6b206574, X3, X3
- ADD TMP0, X4, X4
- ADD TMP1, X5, X5
- ADD TMP2, X6, X6
- ADD TMP3, X7, X7
- MOVWZ 16(KEY), TMP0
- MOVWZ 20(KEY), TMP1
- MOVWZ 24(KEY), TMP2
- MOVWZ 28(KEY), TMP3
- ADD TMP0, X8, X8
- ADD TMP1, X9, X9
- ADD TMP2, X10, X10
- ADD TMP3, X11, X11
-
- MOVWZ 12(CNT), TMP0
- MOVWZ 8(CNT), TMP1
- MOVWZ 4(CNT), TMP2
- MOVWZ 0(CNT), TEMP
- ADD TMP0, X15, X15
- ADD TMP1, X14, X14
- ADD TMP2, X13, X13
- ADD TEMP, X12, X12
-
- // Accumulate key block
- VADDUWM A0, K0, A0
- VADDUWM A1, K0, A1
- VADDUWM A2, K0, A2
- VADDUWM B0, K1, B0
- VADDUWM B1, K1, B1
- VADDUWM B2, K1, B2
- VADDUWM C0, K2, C0
- VADDUWM C1, K2, C1
- VADDUWM C2, K2, C2
- VADDUWM D0, K3, D0
- VADDUWM D1, K4, D1
- VADDUWM D2, K5, D2
-
- // Increment counter
- ADD $4, TEMP, TEMP
- MOVW TEMP, 0(CNT)
-
- VADDUWM K3, FOUR, K3
- VADDUWM K4, FOUR, K4
- VADDUWM K5, FOUR, K5
-
- // XOR the input slice (INP) with the keystream, which is stored in GPRs (X0-X3).
-
- // Load input (aligned or not)
- MOVWZ 0(INP), TMP0
- MOVWZ 4(INP), TMP1
- MOVWZ 8(INP), TMP2
- MOVWZ 12(INP), TMP3
-
- // XOR with input
- XOR TMP0, X0, X0
- XOR TMP1, X1, X1
- XOR TMP2, X2, X2
- XOR TMP3, X3, X3
- MOVWZ 16(INP), TMP0
- MOVWZ 20(INP), TMP1
- MOVWZ 24(INP), TMP2
- MOVWZ 28(INP), TMP3
- XOR TMP0, X4, X4
- XOR TMP1, X5, X5
- XOR TMP2, X6, X6
- XOR TMP3, X7, X7
- MOVWZ 32(INP), TMP0
- MOVWZ 36(INP), TMP1
- MOVWZ 40(INP), TMP2
- MOVWZ 44(INP), TMP3
- XOR TMP0, X8, X8
- XOR TMP1, X9, X9
- XOR TMP2, X10, X10
- XOR TMP3, X11, X11
- MOVWZ 48(INP), TMP0
- MOVWZ 52(INP), TMP1
- MOVWZ 56(INP), TMP2
- MOVWZ 60(INP), TMP3
- XOR TMP0, X12, X12
- XOR TMP1, X13, X13
- XOR TMP2, X14, X14
- XOR TMP3, X15, X15
-
- // Store output (aligned or not)
- MOVW X0, 0(OUT)
- MOVW X1, 4(OUT)
- MOVW X2, 8(OUT)
- MOVW X3, 12(OUT)
-
- ADD $64, INP, INP // INP points to the end of the slice for the alignment code below
-
- MOVW X4, 16(OUT)
- MOVD $16, TMP0
- MOVW X5, 20(OUT)
- MOVD $32, TMP1
- MOVW X6, 24(OUT)
- MOVD $48, TMP2
- MOVW X7, 28(OUT)
- MOVD $64, TMP3
- MOVW X8, 32(OUT)
- MOVW X9, 36(OUT)
- MOVW X10, 40(OUT)
- MOVW X11, 44(OUT)
- MOVW X12, 48(OUT)
- MOVW X13, 52(OUT)
- MOVW X14, 56(OUT)
- MOVW X15, 60(OUT)
- ADD $64, OUT, OUT
-
- // Load input
- LVX (INP)(R0), DD0
- LVX (INP)(TMP0), DD1
- LVX (INP)(TMP1), DD2
- LVX (INP)(TMP2), DD3
- LVX (INP)(TMP3), DD4
- ADD $64, INP, INP
-
- VPERM DD1, DD0, INPPERM, DD0 // Align input
- VPERM DD2, DD1, INPPERM, DD1
- VPERM DD3, DD2, INPPERM, DD2
- VPERM DD4, DD3, INPPERM, DD3
- VXOR A0, DD0, A0 // XOR with input
- VXOR B0, DD1, B0
- LVX (INP)(TMP0), DD1 // Keep loading input
- VXOR C0, DD2, C0
- LVX (INP)(TMP1), DD2
- VXOR D0, DD3, D0
- LVX (INP)(TMP2), DD3
- LVX (INP)(TMP3), DD0
- ADD $64, INP, INP
- MOVD $63, TMP3 // 63 is not a typo
- VPERM A0, A0, OUTPERM, A0
- VPERM B0, B0, OUTPERM, B0
- VPERM C0, C0, OUTPERM, C0
- VPERM D0, D0, OUTPERM, D0
-
- VPERM DD1, DD4, INPPERM, DD4 // Align input
- VPERM DD2, DD1, INPPERM, DD1
- VPERM DD3, DD2, INPPERM, DD2
- VPERM DD0, DD3, INPPERM, DD3
- VXOR A1, DD4, A1
- VXOR B1, DD1, B1
- LVX (INP)(TMP0), DD1 // Keep loading
- VXOR C1, DD2, C1
- LVX (INP)(TMP1), DD2
- VXOR D1, DD3, D1
- LVX (INP)(TMP2), DD3
-
- // Note that the LVX address is always rounded down to the nearest 16-byte
- // boundary, and that it always points to at most 15 bytes beyond the end of
- // the slice, so we cannot cross a page boundary.
- LVX (INP)(TMP3), DD4 // Redundant in aligned case.
- ADD $64, INP, INP
- VPERM A1, A1, OUTPERM, A1 // Pre-misalign output
- VPERM B1, B1, OUTPERM, B1
- VPERM C1, C1, OUTPERM, C1
- VPERM D1, D1, OUTPERM, D1
-
- VPERM DD1, DD0, INPPERM, DD0 // Align Input
- VPERM DD2, DD1, INPPERM, DD1
- VPERM DD3, DD2, INPPERM, DD2
- VPERM DD4, DD3, INPPERM, DD3
- VXOR A2, DD0, A2
- VXOR B2, DD1, B2
- VXOR C2, DD2, C2
- VXOR D2, DD3, D2
- VPERM A2, A2, OUTPERM, A2
- VPERM B2, B2, OUTPERM, B2
- VPERM C2, C2, OUTPERM, C2
- VPERM D2, D2, OUTPERM, D2
-
- ANDCC $15, OUT, X1 // Is out aligned?
- MOVD OUT, X0
-
- VSEL A0, B0, OUTMASK, DD0 // Collect pre-misaligned output
- VSEL B0, C0, OUTMASK, DD1
- VSEL C0, D0, OUTMASK, DD2
- VSEL D0, A1, OUTMASK, DD3
- VSEL A1, B1, OUTMASK, B0
- VSEL B1, C1, OUTMASK, C0
- VSEL C1, D1, OUTMASK, D0
- VSEL D1, A2, OUTMASK, A1
- VSEL A2, B2, OUTMASK, B1
- VSEL B2, C2, OUTMASK, C1
- VSEL C2, D2, OUTMASK, D1
-
- STVX DD0, (OUT+TMP0)
- STVX DD1, (OUT+TMP1)
- STVX DD2, (OUT+TMP2)
- ADD $64, OUT, OUT
- STVX DD3, (OUT+R0)
- STVX B0, (OUT+TMP0)
- STVX C0, (OUT+TMP1)
- STVX D0, (OUT+TMP2)
- ADD $64, OUT, OUT
- STVX A1, (OUT+R0)
- STVX B1, (OUT+TMP0)
- STVX C1, (OUT+TMP1)
- STVX D1, (OUT+TMP2)
- ADD $64, OUT, OUT
-
- BEQ aligned_vmx
-
- SUB X1, OUT, X2 // in misaligned case edges
- MOVD $0, X3 // are written byte-by-byte
-
-unaligned_tail_vmx:
- STVEBX D2, (X2+X3)
- ADD $1, X3, X3
- CMPW X3, X1
- BNE unaligned_tail_vmx
- SUB X1, X0, X2
-
-unaligned_head_vmx:
- STVEBX A0, (X2+X1)
- CMPW X1, $15
- ADD $1, X1, X1
- BNE unaligned_head_vmx
-
- CMPU LEN, $255 // done with 256-byte block yet?
- BGT loop_outer_vmx
-
- JMP done_vmx
-
-aligned_vmx:
- STVX A0, (X0+R0)
- CMPU LEN, $255 // done with 256-byte block yet?
- BGT loop_outer_vmx
-
-done_vmx:
- RET
+++ /dev/null
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build go1.11
-// +build !gccgo
-
-package chacha20
-
-const (
- haveAsm = true
- bufSize = 256
-)
-
-//go:noescape
-func xorKeyStreamVX(dst, src []byte, key *[8]uint32, nonce *[3]uint32, counter *uint32)
-
-func (c *Cipher) xorKeyStreamAsm(dst, src []byte) {
-
- if len(src) >= bufSize {
- xorKeyStreamVX(dst, src, &c.key, &c.nonce, &c.counter)
- }
-
- if len(src)%bufSize != 0 {
- i := len(src) - len(src)%bufSize
- c.buf = [bufSize]byte{}
- copy(c.buf[:], src[i:])
- xorKeyStreamVX(c.buf[:], c.buf[:], &c.key, &c.nonce, &c.counter)
- c.len = bufSize - copy(dst[i:], c.buf[:len(src)%bufSize])
- }
-}
+++ /dev/null
-// Copyright 2016 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package ChaCha20 implements the core ChaCha20 function as specified
-// in https://tools.ietf.org/html/rfc7539#section-2.3.
-package chacha20
-
-import (
- "crypto/cipher"
- "encoding/binary"
-
- "golang.org/x/crypto/internal/subtle"
-)
-
-// assert that *Cipher implements cipher.Stream
-var _ cipher.Stream = (*Cipher)(nil)
-
-// Cipher is a stateful instance of ChaCha20 using a particular key
-// and nonce. A *Cipher implements the cipher.Stream interface.
-type Cipher struct {
- key [8]uint32
- counter uint32 // incremented after each block
- nonce [3]uint32
- buf [bufSize]byte // buffer for unused keystream bytes
- len int // number of unused keystream bytes at end of buf
-}
-
-// New creates a new ChaCha20 stream cipher with the given key and nonce.
-// The initial counter value is set to 0.
-func New(key [8]uint32, nonce [3]uint32) *Cipher {
- return &Cipher{key: key, nonce: nonce}
-}
-
-// ChaCha20 constants spelling "expand 32-byte k"
-const (
- j0 uint32 = 0x61707865
- j1 uint32 = 0x3320646e
- j2 uint32 = 0x79622d32
- j3 uint32 = 0x6b206574
-)
-
-func quarterRound(a, b, c, d uint32) (uint32, uint32, uint32, uint32) {
- a += b
- d ^= a
- d = (d << 16) | (d >> 16)
- c += d
- b ^= c
- b = (b << 12) | (b >> 20)
- a += b
- d ^= a
- d = (d << 8) | (d >> 24)
- c += d
- b ^= c
- b = (b << 7) | (b >> 25)
- return a, b, c, d
-}
-
-// XORKeyStream XORs each byte in the given slice with a byte from the
-// cipher's key stream. Dst and src must overlap entirely or not at all.
-//
-// If len(dst) < len(src), XORKeyStream will panic. It is acceptable
-// to pass a dst bigger than src, and in that case, XORKeyStream will
-// only update dst[:len(src)] and will not touch the rest of dst.
-//
-// Multiple calls to XORKeyStream behave as if the concatenation of
-// the src buffers was passed in a single run. That is, Cipher
-// maintains state and does not reset at each XORKeyStream call.
-func (s *Cipher) XORKeyStream(dst, src []byte) {
- if len(dst) < len(src) {
- panic("chacha20: output smaller than input")
- }
- if subtle.InexactOverlap(dst[:len(src)], src) {
- panic("chacha20: invalid buffer overlap")
- }
-
- // xor src with buffered keystream first
- if s.len != 0 {
- buf := s.buf[len(s.buf)-s.len:]
- if len(src) < len(buf) {
- buf = buf[:len(src)]
- }
- td, ts := dst[:len(buf)], src[:len(buf)] // BCE hint
- for i, b := range buf {
- td[i] = ts[i] ^ b
- }
- s.len -= len(buf)
- if s.len != 0 {
- return
- }
- s.buf = [len(s.buf)]byte{} // zero the empty buffer
- src = src[len(buf):]
- dst = dst[len(buf):]
- }
-
- if len(src) == 0 {
- return
- }
- if haveAsm {
- if uint64(len(src))+uint64(s.counter)*64 > (1<<38)-64 {
- panic("chacha20: counter overflow")
- }
- s.xorKeyStreamAsm(dst, src)
- return
- }
-
- // set up a 64-byte buffer to pad out the final block if needed
- // (hoisted out of the main loop to avoid spills)
- rem := len(src) % 64 // length of final block
- fin := len(src) - rem // index of final block
- if rem > 0 {
- copy(s.buf[len(s.buf)-64:], src[fin:])
- }
-
- // pre-calculate most of the first round
- s1, s5, s9, s13 := quarterRound(j1, s.key[1], s.key[5], s.nonce[0])
- s2, s6, s10, s14 := quarterRound(j2, s.key[2], s.key[6], s.nonce[1])
- s3, s7, s11, s15 := quarterRound(j3, s.key[3], s.key[7], s.nonce[2])
-
- n := len(src)
- src, dst = src[:n:n], dst[:n:n] // BCE hint
- for i := 0; i < n; i += 64 {
- // calculate the remainder of the first round
- s0, s4, s8, s12 := quarterRound(j0, s.key[0], s.key[4], s.counter)
-
- // execute the second round
- x0, x5, x10, x15 := quarterRound(s0, s5, s10, s15)
- x1, x6, x11, x12 := quarterRound(s1, s6, s11, s12)
- x2, x7, x8, x13 := quarterRound(s2, s7, s8, s13)
- x3, x4, x9, x14 := quarterRound(s3, s4, s9, s14)
-
- // execute the remaining 18 rounds
- for i := 0; i < 9; i++ {
- x0, x4, x8, x12 = quarterRound(x0, x4, x8, x12)
- x1, x5, x9, x13 = quarterRound(x1, x5, x9, x13)
- x2, x6, x10, x14 = quarterRound(x2, x6, x10, x14)
- x3, x7, x11, x15 = quarterRound(x3, x7, x11, x15)
-
- x0, x5, x10, x15 = quarterRound(x0, x5, x10, x15)
- x1, x6, x11, x12 = quarterRound(x1, x6, x11, x12)
- x2, x7, x8, x13 = quarterRound(x2, x7, x8, x13)
- x3, x4, x9, x14 = quarterRound(x3, x4, x9, x14)
- }
-
- x0 += j0
- x1 += j1
- x2 += j2
- x3 += j3
-
- x4 += s.key[0]
- x5 += s.key[1]
- x6 += s.key[2]
- x7 += s.key[3]
- x8 += s.key[4]
- x9 += s.key[5]
- x10 += s.key[6]
- x11 += s.key[7]
-
- x12 += s.counter
- x13 += s.nonce[0]
- x14 += s.nonce[1]
- x15 += s.nonce[2]
-
- // increment the counter
- s.counter += 1
- if s.counter == 0 {
- panic("chacha20: counter overflow")
- }
-
- // pad to 64 bytes if needed
- in, out := src[i:], dst[i:]
- if i == fin {
- // src[fin:] has already been copied into s.buf before
- // the main loop
- in, out = s.buf[len(s.buf)-64:], s.buf[len(s.buf)-64:]
- }
- in, out = in[:64], out[:64] // BCE hint
-
- // XOR the key stream with the source and write out the result
- xor(out[0:], in[0:], x0)
- xor(out[4:], in[4:], x1)
- xor(out[8:], in[8:], x2)
- xor(out[12:], in[12:], x3)
- xor(out[16:], in[16:], x4)
- xor(out[20:], in[20:], x5)
- xor(out[24:], in[24:], x6)
- xor(out[28:], in[28:], x7)
- xor(out[32:], in[32:], x8)
- xor(out[36:], in[36:], x9)
- xor(out[40:], in[40:], x10)
- xor(out[44:], in[44:], x11)
- xor(out[48:], in[48:], x12)
- xor(out[52:], in[52:], x13)
- xor(out[56:], in[56:], x14)
- xor(out[60:], in[60:], x15)
- }
- // copy any trailing bytes out of the buffer and into dst
- if rem != 0 {
- s.len = 64 - rem
- copy(dst[fin:], s.buf[len(s.buf)-64:])
- }
-}
-
-// Advance discards bytes in the key stream until the next 64 byte block
-// boundary is reached and updates the counter accordingly. If the key
-// stream is already at a block boundary no bytes will be discarded and
-// the counter will be unchanged.
-func (s *Cipher) Advance() {
- s.len -= s.len % 64
- if s.len == 0 {
- s.buf = [len(s.buf)]byte{}
- }
-}
-
-// XORKeyStream crypts bytes from in to out using the given key and counters.
-// In and out must overlap entirely or not at all. Counter contains the raw
-// ChaCha20 counter bytes (i.e. block counter followed by nonce).
-func XORKeyStream(out, in []byte, counter *[16]byte, key *[32]byte) {
- s := Cipher{
- key: [8]uint32{
- binary.LittleEndian.Uint32(key[0:4]),
- binary.LittleEndian.Uint32(key[4:8]),
- binary.LittleEndian.Uint32(key[8:12]),
- binary.LittleEndian.Uint32(key[12:16]),
- binary.LittleEndian.Uint32(key[16:20]),
- binary.LittleEndian.Uint32(key[20:24]),
- binary.LittleEndian.Uint32(key[24:28]),
- binary.LittleEndian.Uint32(key[28:32]),
- },
- nonce: [3]uint32{
- binary.LittleEndian.Uint32(counter[4:8]),
- binary.LittleEndian.Uint32(counter[8:12]),
- binary.LittleEndian.Uint32(counter[12:16]),
- },
- counter: binary.LittleEndian.Uint32(counter[0:4]),
- }
- s.XORKeyStream(out, in)
-}
-
-// HChaCha20 uses the ChaCha20 core to generate a derived key from a key and a
-// nonce. It should only be used as part of the XChaCha20 construction.
-func HChaCha20(key *[8]uint32, nonce *[4]uint32) [8]uint32 {
- x0, x1, x2, x3 := j0, j1, j2, j3
- x4, x5, x6, x7 := key[0], key[1], key[2], key[3]
- x8, x9, x10, x11 := key[4], key[5], key[6], key[7]
- x12, x13, x14, x15 := nonce[0], nonce[1], nonce[2], nonce[3]
-
- for i := 0; i < 10; i++ {
- x0, x4, x8, x12 = quarterRound(x0, x4, x8, x12)
- x1, x5, x9, x13 = quarterRound(x1, x5, x9, x13)
- x2, x6, x10, x14 = quarterRound(x2, x6, x10, x14)
- x3, x7, x11, x15 = quarterRound(x3, x7, x11, x15)
-
- x0, x5, x10, x15 = quarterRound(x0, x5, x10, x15)
- x1, x6, x11, x12 = quarterRound(x1, x6, x11, x12)
- x2, x7, x8, x13 = quarterRound(x2, x7, x8, x13)
- x3, x4, x9, x14 = quarterRound(x3, x4, x9, x14)
- }
-
- var out [8]uint32
- out[0], out[1], out[2], out[3] = x0, x1, x2, x3
- out[4], out[5], out[6], out[7] = x12, x13, x14, x15
- return out
-}
+++ /dev/null
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build !ppc64le,!arm64,!s390x arm64,!go1.11 gccgo appengine
-
-package chacha20
-
-const (
- bufSize = 64
- haveAsm = false
-)
-
-func (*Cipher) xorKeyStreamAsm(dst, src []byte) {
- panic("not implemented")
-}
+++ /dev/null
-// Copyright 2019 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build ppc64le,!gccgo,!appengine
-
-package chacha20
-
-import "encoding/binary"
-
-const (
- bufSize = 256
- haveAsm = true
-)
-
-//go:noescape
-func chaCha20_ctr32_vmx(out, inp *byte, len int, key *[8]uint32, counter *uint32)
-
-func (c *Cipher) xorKeyStreamAsm(dst, src []byte) {
- if len(src) >= bufSize {
- chaCha20_ctr32_vmx(&dst[0], &src[0], len(src)-len(src)%bufSize, &c.key, &c.counter)
- }
- if len(src)%bufSize != 0 {
- chaCha20_ctr32_vmx(&c.buf[0], &c.buf[0], bufSize, &c.key, &c.counter)
- start := len(src) - len(src)%bufSize
- ts, td, tb := src[start:], dst[start:], c.buf[:]
- // Unroll loop to XOR 32 bytes per iteration.
- for i := 0; i < len(ts)-32; i += 32 {
- td, tb = td[:len(ts)], tb[:len(ts)] // bounds check elimination
- s0 := binary.LittleEndian.Uint64(ts[0:8])
- s1 := binary.LittleEndian.Uint64(ts[8:16])
- s2 := binary.LittleEndian.Uint64(ts[16:24])
- s3 := binary.LittleEndian.Uint64(ts[24:32])
- b0 := binary.LittleEndian.Uint64(tb[0:8])
- b1 := binary.LittleEndian.Uint64(tb[8:16])
- b2 := binary.LittleEndian.Uint64(tb[16:24])
- b3 := binary.LittleEndian.Uint64(tb[24:32])
- binary.LittleEndian.PutUint64(td[0:8], s0^b0)
- binary.LittleEndian.PutUint64(td[8:16], s1^b1)
- binary.LittleEndian.PutUint64(td[16:24], s2^b2)
- binary.LittleEndian.PutUint64(td[24:32], s3^b3)
- ts, td, tb = ts[32:], td[32:], tb[32:]
- }
- td, tb = td[:len(ts)], tb[:len(ts)] // bounds check elimination
- for i, v := range ts {
- td[i] = tb[i] ^ v
- }
- c.len = bufSize - (len(src) % bufSize)
-
- }
-
-}
+++ /dev/null
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build s390x,!gccgo,!appengine
-
-package chacha20
-
-import (
- "golang.org/x/sys/cpu"
-)
-
-var haveAsm = cpu.S390X.HasVX
-
-const bufSize = 256
-
-// xorKeyStreamVX is an assembly implementation of XORKeyStream. It must only
-// be called when the vector facility is available.
-// Implementation in asm_s390x.s.
-//go:noescape
-func xorKeyStreamVX(dst, src []byte, key *[8]uint32, nonce *[3]uint32, counter *uint32, buf *[256]byte, len *int)
-
-func (c *Cipher) xorKeyStreamAsm(dst, src []byte) {
- xorKeyStreamVX(dst, src, &c.key, &c.nonce, &c.counter, &c.buf, &c.len)
-}
-
-// EXRL targets, DO NOT CALL!
-func mvcSrcToBuf()
-func mvcBufToDst()
+++ /dev/null
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build s390x,!gccgo,!appengine
-
-#include "go_asm.h"
-#include "textflag.h"
-
-// This is an implementation of the ChaCha20 encryption algorithm as
-// specified in RFC 7539. It uses vector instructions to compute
-// 4 keystream blocks in parallel (256 bytes) which are then XORed
-// with the bytes in the input slice.
-
-GLOBL ·constants<>(SB), RODATA|NOPTR, $32
-// BSWAP: swap bytes in each 4-byte element
-DATA ·constants<>+0x00(SB)/4, $0x03020100
-DATA ·constants<>+0x04(SB)/4, $0x07060504
-DATA ·constants<>+0x08(SB)/4, $0x0b0a0908
-DATA ·constants<>+0x0c(SB)/4, $0x0f0e0d0c
-// J0: [j0, j1, j2, j3]
-DATA ·constants<>+0x10(SB)/4, $0x61707865
-DATA ·constants<>+0x14(SB)/4, $0x3320646e
-DATA ·constants<>+0x18(SB)/4, $0x79622d32
-DATA ·constants<>+0x1c(SB)/4, $0x6b206574
-
-// EXRL targets:
-TEXT ·mvcSrcToBuf(SB), NOFRAME|NOSPLIT, $0
- MVC $1, (R1), (R8)
- RET
-
-TEXT ·mvcBufToDst(SB), NOFRAME|NOSPLIT, $0
- MVC $1, (R8), (R9)
- RET
-
-#define BSWAP V5
-#define J0 V6
-#define KEY0 V7
-#define KEY1 V8
-#define NONCE V9
-#define CTR V10
-#define M0 V11
-#define M1 V12
-#define M2 V13
-#define M3 V14
-#define INC V15
-#define X0 V16
-#define X1 V17
-#define X2 V18
-#define X3 V19
-#define X4 V20
-#define X5 V21
-#define X6 V22
-#define X7 V23
-#define X8 V24
-#define X9 V25
-#define X10 V26
-#define X11 V27
-#define X12 V28
-#define X13 V29
-#define X14 V30
-#define X15 V31
-
-#define NUM_ROUNDS 20
-
-#define ROUND4(a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3) \
- VAF a1, a0, a0 \
- VAF b1, b0, b0 \
- VAF c1, c0, c0 \
- VAF d1, d0, d0 \
- VX a0, a2, a2 \
- VX b0, b2, b2 \
- VX c0, c2, c2 \
- VX d0, d2, d2 \
- VERLLF $16, a2, a2 \
- VERLLF $16, b2, b2 \
- VERLLF $16, c2, c2 \
- VERLLF $16, d2, d2 \
- VAF a2, a3, a3 \
- VAF b2, b3, b3 \
- VAF c2, c3, c3 \
- VAF d2, d3, d3 \
- VX a3, a1, a1 \
- VX b3, b1, b1 \
- VX c3, c1, c1 \
- VX d3, d1, d1 \
- VERLLF $12, a1, a1 \
- VERLLF $12, b1, b1 \
- VERLLF $12, c1, c1 \
- VERLLF $12, d1, d1 \
- VAF a1, a0, a0 \
- VAF b1, b0, b0 \
- VAF c1, c0, c0 \
- VAF d1, d0, d0 \
- VX a0, a2, a2 \
- VX b0, b2, b2 \
- VX c0, c2, c2 \
- VX d0, d2, d2 \
- VERLLF $8, a2, a2 \
- VERLLF $8, b2, b2 \
- VERLLF $8, c2, c2 \
- VERLLF $8, d2, d2 \
- VAF a2, a3, a3 \
- VAF b2, b3, b3 \
- VAF c2, c3, c3 \
- VAF d2, d3, d3 \
- VX a3, a1, a1 \
- VX b3, b1, b1 \
- VX c3, c1, c1 \
- VX d3, d1, d1 \
- VERLLF $7, a1, a1 \
- VERLLF $7, b1, b1 \
- VERLLF $7, c1, c1 \
- VERLLF $7, d1, d1
-
-#define PERMUTE(mask, v0, v1, v2, v3) \
- VPERM v0, v0, mask, v0 \
- VPERM v1, v1, mask, v1 \
- VPERM v2, v2, mask, v2 \
- VPERM v3, v3, mask, v3
-
-#define ADDV(x, v0, v1, v2, v3) \
- VAF x, v0, v0 \
- VAF x, v1, v1 \
- VAF x, v2, v2 \
- VAF x, v3, v3
-
-#define XORV(off, dst, src, v0, v1, v2, v3) \
- VLM off(src), M0, M3 \
- PERMUTE(BSWAP, v0, v1, v2, v3) \
- VX v0, M0, M0 \
- VX v1, M1, M1 \
- VX v2, M2, M2 \
- VX v3, M3, M3 \
- VSTM M0, M3, off(dst)
-
-#define SHUFFLE(a, b, c, d, t, u, v, w) \
- VMRHF a, c, t \ // t = {a[0], c[0], a[1], c[1]}
- VMRHF b, d, u \ // u = {b[0], d[0], b[1], d[1]}
- VMRLF a, c, v \ // v = {a[2], c[2], a[3], c[3]}
- VMRLF b, d, w \ // w = {b[2], d[2], b[3], d[3]}
- VMRHF t, u, a \ // a = {a[0], b[0], c[0], d[0]}
- VMRLF t, u, b \ // b = {a[1], b[1], c[1], d[1]}
- VMRHF v, w, c \ // c = {a[2], b[2], c[2], d[2]}
- VMRLF v, w, d // d = {a[3], b[3], c[3], d[3]}
-
-// func xorKeyStreamVX(dst, src []byte, key *[8]uint32, nonce *[3]uint32, counter *uint32, buf *[256]byte, len *int)
-TEXT ·xorKeyStreamVX(SB), NOSPLIT, $0
- MOVD $·constants<>(SB), R1
- MOVD dst+0(FP), R2 // R2=&dst[0]
- LMG src+24(FP), R3, R4 // R3=&src[0] R4=len(src)
- MOVD key+48(FP), R5 // R5=key
- MOVD nonce+56(FP), R6 // R6=nonce
- MOVD counter+64(FP), R7 // R7=counter
- MOVD buf+72(FP), R8 // R8=buf
- MOVD len+80(FP), R9 // R9=len
-
- // load BSWAP and J0
- VLM (R1), BSWAP, J0
-
- // set up tail buffer
- ADD $-1, R4, R12
- MOVBZ R12, R12
- CMPUBEQ R12, $255, aligned
- MOVD R4, R1
- AND $~255, R1
- MOVD $(R3)(R1*1), R1
- EXRL $·mvcSrcToBuf(SB), R12
- MOVD $255, R0
- SUB R12, R0
- MOVD R0, (R9) // update len
-
-aligned:
- // setup
- MOVD $95, R0
- VLM (R5), KEY0, KEY1
- VLL R0, (R6), NONCE
- VZERO M0
- VLEIB $7, $32, M0
- VSRLB M0, NONCE, NONCE
-
- // initialize counter values
- VLREPF (R7), CTR
- VZERO INC
- VLEIF $1, $1, INC
- VLEIF $2, $2, INC
- VLEIF $3, $3, INC
- VAF INC, CTR, CTR
- VREPIF $4, INC
-
-chacha:
- VREPF $0, J0, X0
- VREPF $1, J0, X1
- VREPF $2, J0, X2
- VREPF $3, J0, X3
- VREPF $0, KEY0, X4
- VREPF $1, KEY0, X5
- VREPF $2, KEY0, X6
- VREPF $3, KEY0, X7
- VREPF $0, KEY1, X8
- VREPF $1, KEY1, X9
- VREPF $2, KEY1, X10
- VREPF $3, KEY1, X11
- VLR CTR, X12
- VREPF $1, NONCE, X13
- VREPF $2, NONCE, X14
- VREPF $3, NONCE, X15
-
- MOVD $(NUM_ROUNDS/2), R1
-
-loop:
- ROUND4(X0, X4, X12, X8, X1, X5, X13, X9, X2, X6, X14, X10, X3, X7, X15, X11)
- ROUND4(X0, X5, X15, X10, X1, X6, X12, X11, X2, X7, X13, X8, X3, X4, X14, X9)
-
- ADD $-1, R1
- BNE loop
-
- // decrement length
- ADD $-256, R4
- BLT tail
-
-continue:
- // rearrange vectors
- SHUFFLE(X0, X1, X2, X3, M0, M1, M2, M3)
- ADDV(J0, X0, X1, X2, X3)
- SHUFFLE(X4, X5, X6, X7, M0, M1, M2, M3)
- ADDV(KEY0, X4, X5, X6, X7)
- SHUFFLE(X8, X9, X10, X11, M0, M1, M2, M3)
- ADDV(KEY1, X8, X9, X10, X11)
- VAF CTR, X12, X12
- SHUFFLE(X12, X13, X14, X15, M0, M1, M2, M3)
- ADDV(NONCE, X12, X13, X14, X15)
-
- // increment counters
- VAF INC, CTR, CTR
-
- // xor keystream with plaintext
- XORV(0*64, R2, R3, X0, X4, X8, X12)
- XORV(1*64, R2, R3, X1, X5, X9, X13)
- XORV(2*64, R2, R3, X2, X6, X10, X14)
- XORV(3*64, R2, R3, X3, X7, X11, X15)
-
- // increment pointers
- MOVD $256(R2), R2
- MOVD $256(R3), R3
-
- CMPBNE R4, $0, chacha
- CMPUBEQ R12, $255, return
- EXRL $·mvcBufToDst(SB), R12 // len was updated during setup
-
-return:
- VSTEF $0, CTR, (R7)
- RET
-
-tail:
- MOVD R2, R9
- MOVD R8, R2
- MOVD R8, R3
- MOVD $0, R4
- JMP continue
+++ /dev/null
-// Copyright 2018 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found src the LICENSE file.
-
-package chacha20
-
-import (
- "runtime"
-)
-
-// Platforms that have fast unaligned 32-bit little endian accesses.
-const unaligned = runtime.GOARCH == "386" ||
- runtime.GOARCH == "amd64" ||
- runtime.GOARCH == "arm64" ||
- runtime.GOARCH == "ppc64le" ||
- runtime.GOARCH == "s390x"
-
-// xor reads a little endian uint32 from src, XORs it with u and
-// places the result in little endian byte order in dst.
-func xor(dst, src []byte, u uint32) {
- _, _ = src[3], dst[3] // eliminate bounds checks
- if unaligned {
- // The compiler should optimize this code into
- // 32-bit unaligned little endian loads and stores.
- // TODO: delete once the compiler does a reliably
- // good job with the generic code below.
- // See issue #25111 for more details.
- v := uint32(src[0])
- v |= uint32(src[1]) << 8
- v |= uint32(src[2]) << 16
- v |= uint32(src[3]) << 24
- v ^= u
- dst[0] = byte(v)
- dst[1] = byte(v >> 8)
- dst[2] = byte(v >> 16)
- dst[3] = byte(v >> 24)
- } else {
- dst[0] = src[0] ^ byte(u)
- dst[1] = src[1] ^ byte(u>>8)
- dst[2] = src[2] ^ byte(u>>16)
- dst[3] = src[3] ^ byte(u>>24)
- }
-}
// Bleichenbacher, Advances in Cryptology (Crypto '98),
func Decrypt(priv *PrivateKey, c1, c2 *big.Int) (msg []byte, err error) {
s := new(big.Int).Exp(c1, priv.X, priv.P)
- s.ModInverse(s, priv.P)
+ if s.ModInverse(s, priv.P) == nil {
+ return nil, errors.New("elgamal: invalid private key")
+ }
s.Mul(s, c2)
s.Mod(s, priv.P)
em := s.Bytes()
package packet
import (
+ "crypto"
"crypto/rsa"
"encoding/binary"
"io"
// padding oracle attacks.
switch priv.PubKeyAlgo {
case PubKeyAlgoRSA, PubKeyAlgoRSAEncryptOnly:
- k := priv.PrivateKey.(*rsa.PrivateKey)
- b, err = rsa.DecryptPKCS1v15(config.Random(), k, padToKeySize(&k.PublicKey, e.encryptedMPI1.bytes))
+ // Supports both *rsa.PrivateKey and crypto.Decrypter
+ k := priv.PrivateKey.(crypto.Decrypter)
+ b, err = k.Decrypt(config.Random(), padToKeySize(k.Public().(*rsa.PublicKey), e.encryptedMPI1.bytes), nil)
case PubKeyAlgoElGamal:
c1 := new(big.Int).SetBytes(e.encryptedMPI1.bytes)
c2 := new(big.Int).SetBytes(e.encryptedMPI2.bytes)
encryptedData []byte
cipher CipherFunction
s2k func(out, in []byte)
- PrivateKey interface{} // An *{rsa|dsa|ecdsa}.PrivateKey or a crypto.Signer.
+ PrivateKey interface{} // An *{rsa|dsa|ecdsa}.PrivateKey or crypto.Signer/crypto.Decrypter (Decryptor RSA only).
sha1Checksum bool
iv []byte
}
--- /dev/null
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !go1.13
+
+package poly1305
+
+// Generic fallbacks for the math/bits intrinsics, copied from
+// src/math/bits/bits.go. They were added in Go 1.12, but Add64 and Sum64 had
+// variable time fallbacks until Go 1.13.
+
+func bitsAdd64(x, y, carry uint64) (sum, carryOut uint64) {
+ sum = x + y + carry
+ carryOut = ((x & y) | ((x | y) &^ sum)) >> 63
+ return
+}
+
+func bitsSub64(x, y, borrow uint64) (diff, borrowOut uint64) {
+ diff = x - y - borrow
+ borrowOut = ((^x & y) | (^(x ^ y) & diff)) >> 63
+ return
+}
+
+func bitsMul64(x, y uint64) (hi, lo uint64) {
+ const mask32 = 1<<32 - 1
+ x0 := x & mask32
+ x1 := x >> 32
+ y0 := y & mask32
+ y1 := y >> 32
+ w0 := x0 * y0
+ t := x1*y0 + w0>>32
+ w1 := t & mask32
+ w2 := t >> 32
+ w1 += x0 * y1
+ hi = x1*y1 + w2 + w1>>32
+ lo = x * y
+ return
+}
--- /dev/null
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build go1.13
+
+package poly1305
+
+import "math/bits"
+
+func bitsAdd64(x, y, carry uint64) (sum, carryOut uint64) {
+ return bits.Add64(x, y, carry)
+}
+
+func bitsSub64(x, y, borrow uint64) (diff, borrowOut uint64) {
+ return bits.Sub64(x, y, borrow)
+}
+
+func bitsMul64(x, y uint64) (hi, lo uint64) {
+ return bits.Mul64(x, y)
+}
// TagSize is the size, in bytes, of a poly1305 authenticator.
const TagSize = 16
-// Verify returns true if mac is a valid authenticator for m with the given
-// key.
+// Sum generates an authenticator for msg using a one-time key and puts the
+// 16-byte result into out. Authenticating two different messages with the same
+// key allows an attacker to forge messages at will.
+func Sum(out *[16]byte, m []byte, key *[32]byte) {
+ sum(out, m, key)
+}
+
+// Verify returns true if mac is a valid authenticator for m with the given key.
func Verify(mac *[16]byte, m []byte, key *[32]byte) bool {
var tmp [16]byte
Sum(&tmp, m, key)
package poly1305
//go:noescape
-func initialize(state *[7]uint64, key *[32]byte)
+func update(state *macState, msg []byte)
-//go:noescape
-func update(state *[7]uint64, msg []byte)
-
-//go:noescape
-func finalize(tag *[TagSize]byte, state *[7]uint64)
-
-// Sum generates an authenticator for m using a one-time key and puts the
-// 16-byte result into out. Authenticating two different messages with the same
-// key allows an attacker to forge messages at will.
-func Sum(out *[16]byte, m []byte, key *[32]byte) {
+func sum(out *[16]byte, m []byte, key *[32]byte) {
h := newMAC(key)
h.Write(m)
h.Sum(out)
}
func newMAC(key *[32]byte) (h mac) {
- initialize(&h.state, key)
+ initialize(key, &h.r, &h.s)
return
}
-type mac struct {
- state [7]uint64 // := uint64{ h0, h1, h2, r0, r1, pad0, pad1 }
-
- buffer [TagSize]byte
- offset int
-}
+// mac is a wrapper for macGeneric that redirects calls that would have gone to
+// updateGeneric to update.
+//
+// Its Write and Sum methods are otherwise identical to the macGeneric ones, but
+// using function pointers would carry a major performance cost.
+type mac struct{ macGeneric }
-func (h *mac) Write(p []byte) (n int, err error) {
- n = len(p)
+func (h *mac) Write(p []byte) (int, error) {
+ nn := len(p)
if h.offset > 0 {
- remaining := TagSize - h.offset
- if n < remaining {
- h.offset += copy(h.buffer[h.offset:], p)
- return n, nil
+ n := copy(h.buffer[h.offset:], p)
+ if h.offset+n < TagSize {
+ h.offset += n
+ return nn, nil
}
- copy(h.buffer[h.offset:], p[:remaining])
- p = p[remaining:]
+ p = p[n:]
h.offset = 0
- update(&h.state, h.buffer[:])
+ update(&h.macState, h.buffer[:])
}
- if nn := len(p) - (len(p) % TagSize); nn > 0 {
- update(&h.state, p[:nn])
- p = p[nn:]
+ if n := len(p) - (len(p) % TagSize); n > 0 {
+ update(&h.macState, p[:n])
+ p = p[n:]
}
if len(p) > 0 {
h.offset += copy(h.buffer[h.offset:], p)
}
- return n, nil
+ return nn, nil
}
func (h *mac) Sum(out *[16]byte) {
- state := h.state
+ state := h.macState
if h.offset > 0 {
update(&state, h.buffer[:h.offset])
}
- finalize(out, &state)
+ finalize(out, &state.h, &state.s)
}
ADCQ t3, h1; \
ADCQ $0, h2
-DATA ·poly1305Mask<>+0x00(SB)/8, $0x0FFFFFFC0FFFFFFF
-DATA ·poly1305Mask<>+0x08(SB)/8, $0x0FFFFFFC0FFFFFFC
-GLOBL ·poly1305Mask<>(SB), RODATA, $16
-
// func update(state *[7]uint64, msg []byte)
TEXT ·update(SB), $0-32
MOVQ state+0(FP), DI
MOVQ R9, 8(DI)
MOVQ R10, 16(DI)
RET
-
-// func initialize(state *[7]uint64, key *[32]byte)
-TEXT ·initialize(SB), $0-16
- MOVQ state+0(FP), DI
- MOVQ key+8(FP), SI
-
- // state[0...7] is initialized with zero
- MOVOU 0(SI), X0
- MOVOU 16(SI), X1
- MOVOU ·poly1305Mask<>(SB), X2
- PAND X2, X0
- MOVOU X0, 24(DI)
- MOVOU X1, 40(DI)
- RET
-
-// func finalize(tag *[TagSize]byte, state *[7]uint64)
-TEXT ·finalize(SB), $0-16
- MOVQ tag+0(FP), DI
- MOVQ state+8(FP), SI
-
- MOVQ 0(SI), AX
- MOVQ 8(SI), BX
- MOVQ 16(SI), CX
- MOVQ AX, R8
- MOVQ BX, R9
- SUBQ $0xFFFFFFFFFFFFFFFB, AX
- SBBQ $0xFFFFFFFFFFFFFFFF, BX
- SBBQ $3, CX
- CMOVQCS R8, AX
- CMOVQCS R9, BX
- ADDQ 40(SI), AX
- ADCQ 48(SI), BX
-
- MOVQ AX, 0(DI)
- MOVQ BX, 8(DI)
- RET
package poly1305
-// This function is implemented in sum_arm.s
+// poly1305_auth_armv6 is implemented in sum_arm.s
//go:noescape
func poly1305_auth_armv6(out *[16]byte, m *byte, mlen uint32, key *[32]byte)
-// Sum generates an authenticator for m using a one-time key and puts the
-// 16-byte result into out. Authenticating two different messages with the same
-// key allows an attacker to forge messages at will.
-func Sum(out *[16]byte, m []byte, key *[32]byte) {
+func sum(out *[16]byte, m []byte, key *[32]byte) {
var mPtr *byte
if len(m) > 0 {
mPtr = &m[0]
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
+// This file provides the generic implementation of Sum and MAC. Other files
+// might provide optimized assembly implementations of some of this code.
+
package poly1305
import "encoding/binary"
-const (
- msgBlock = uint32(1 << 24)
- finalBlock = uint32(0)
-)
+// Poly1305 [RFC 7539] is a relatively simple algorithm: the authentication tag
+// for a 64 bytes message is approximately
+//
+// s + m[0:16] * r⁴ + m[16:32] * r³ + m[32:48] * r² + m[48:64] * r mod 2¹³⁰ - 5
+//
+// for some secret r and s. It can be computed sequentially like
+//
+// for len(msg) > 0:
+// h += read(msg, 16)
+// h *= r
+// h %= 2¹³⁰ - 5
+// return h + s
+//
+// All the complexity is about doing performant constant-time math on numbers
+// larger than any available numeric type.
-// sumGeneric generates an authenticator for msg using a one-time key and
-// puts the 16-byte result into out. This is the generic implementation of
-// Sum and should be called if no assembly implementation is available.
func sumGeneric(out *[TagSize]byte, msg []byte, key *[32]byte) {
h := newMACGeneric(key)
h.Write(msg)
}
func newMACGeneric(key *[32]byte) (h macGeneric) {
- h.r[0] = binary.LittleEndian.Uint32(key[0:]) & 0x3ffffff
- h.r[1] = (binary.LittleEndian.Uint32(key[3:]) >> 2) & 0x3ffff03
- h.r[2] = (binary.LittleEndian.Uint32(key[6:]) >> 4) & 0x3ffc0ff
- h.r[3] = (binary.LittleEndian.Uint32(key[9:]) >> 6) & 0x3f03fff
- h.r[4] = (binary.LittleEndian.Uint32(key[12:]) >> 8) & 0x00fffff
-
- h.s[0] = binary.LittleEndian.Uint32(key[16:])
- h.s[1] = binary.LittleEndian.Uint32(key[20:])
- h.s[2] = binary.LittleEndian.Uint32(key[24:])
- h.s[3] = binary.LittleEndian.Uint32(key[28:])
+ initialize(key, &h.r, &h.s)
return
}
+// macState holds numbers in saturated 64-bit little-endian limbs. That is,
+// the value of [x0, x1, x2] is x[0] + x[1] * 2⁶⁴ + x[2] * 2¹²⁸.
+type macState struct {
+ // h is the main accumulator. It is to be interpreted modulo 2¹³⁰ - 5, but
+ // can grow larger during and after rounds.
+ h [3]uint64
+ // r and s are the private key components.
+ r [2]uint64
+ s [2]uint64
+}
+
type macGeneric struct {
- h, r [5]uint32
- s [4]uint32
+ macState
buffer [TagSize]byte
offset int
}
-func (h *macGeneric) Write(p []byte) (n int, err error) {
- n = len(p)
+// Write splits the incoming message into TagSize chunks, and passes them to
+// update. It buffers incomplete chunks.
+func (h *macGeneric) Write(p []byte) (int, error) {
+ nn := len(p)
if h.offset > 0 {
- remaining := TagSize - h.offset
- if n < remaining {
- h.offset += copy(h.buffer[h.offset:], p)
- return n, nil
+ n := copy(h.buffer[h.offset:], p)
+ if h.offset+n < TagSize {
+ h.offset += n
+ return nn, nil
}
- copy(h.buffer[h.offset:], p[:remaining])
- p = p[remaining:]
+ p = p[n:]
h.offset = 0
- updateGeneric(h.buffer[:], msgBlock, &(h.h), &(h.r))
+ updateGeneric(&h.macState, h.buffer[:])
}
- if nn := len(p) - (len(p) % TagSize); nn > 0 {
- updateGeneric(p, msgBlock, &(h.h), &(h.r))
- p = p[nn:]
+ if n := len(p) - (len(p) % TagSize); n > 0 {
+ updateGeneric(&h.macState, p[:n])
+ p = p[n:]
}
if len(p) > 0 {
h.offset += copy(h.buffer[h.offset:], p)
}
- return n, nil
+ return nn, nil
}
-func (h *macGeneric) Sum(out *[16]byte) {
- H, R := h.h, h.r
+// Sum flushes the last incomplete chunk from the buffer, if any, and generates
+// the MAC output. It does not modify its state, in order to allow for multiple
+// calls to Sum, even if no Write is allowed after Sum.
+func (h *macGeneric) Sum(out *[TagSize]byte) {
+ state := h.macState
if h.offset > 0 {
- var buffer [TagSize]byte
- copy(buffer[:], h.buffer[:h.offset])
- buffer[h.offset] = 1 // invariant: h.offset < TagSize
- updateGeneric(buffer[:], finalBlock, &H, &R)
+ updateGeneric(&state, h.buffer[:h.offset])
}
- finalizeGeneric(out, &H, &(h.s))
+ finalize(out, &state.h, &state.s)
+}
+
+// [rMask0, rMask1] is the specified Poly1305 clamping mask in little-endian. It
+// clears some bits of the secret coefficient to make it possible to implement
+// multiplication more efficiently.
+const (
+ rMask0 = 0x0FFFFFFC0FFFFFFF
+ rMask1 = 0x0FFFFFFC0FFFFFFC
+)
+
+func initialize(key *[32]byte, r, s *[2]uint64) {
+ r[0] = binary.LittleEndian.Uint64(key[0:8]) & rMask0
+ r[1] = binary.LittleEndian.Uint64(key[8:16]) & rMask1
+ s[0] = binary.LittleEndian.Uint64(key[16:24])
+ s[1] = binary.LittleEndian.Uint64(key[24:32])
+}
+
+// uint128 holds a 128-bit number as two 64-bit limbs, for use with the
+// bits.Mul64 and bits.Add64 intrinsics.
+type uint128 struct {
+ lo, hi uint64
+}
+
+func mul64(a, b uint64) uint128 {
+ hi, lo := bitsMul64(a, b)
+ return uint128{lo, hi}
}
-func updateGeneric(msg []byte, flag uint32, h, r *[5]uint32) {
- h0, h1, h2, h3, h4 := h[0], h[1], h[2], h[3], h[4]
- r0, r1, r2, r3, r4 := uint64(r[0]), uint64(r[1]), uint64(r[2]), uint64(r[3]), uint64(r[4])
- R1, R2, R3, R4 := r1*5, r2*5, r3*5, r4*5
-
- for len(msg) >= TagSize {
- // h += msg
- h0 += binary.LittleEndian.Uint32(msg[0:]) & 0x3ffffff
- h1 += (binary.LittleEndian.Uint32(msg[3:]) >> 2) & 0x3ffffff
- h2 += (binary.LittleEndian.Uint32(msg[6:]) >> 4) & 0x3ffffff
- h3 += (binary.LittleEndian.Uint32(msg[9:]) >> 6) & 0x3ffffff
- h4 += (binary.LittleEndian.Uint32(msg[12:]) >> 8) | flag
-
- // h *= r
- d0 := (uint64(h0) * r0) + (uint64(h1) * R4) + (uint64(h2) * R3) + (uint64(h3) * R2) + (uint64(h4) * R1)
- d1 := (d0 >> 26) + (uint64(h0) * r1) + (uint64(h1) * r0) + (uint64(h2) * R4) + (uint64(h3) * R3) + (uint64(h4) * R2)
- d2 := (d1 >> 26) + (uint64(h0) * r2) + (uint64(h1) * r1) + (uint64(h2) * r0) + (uint64(h3) * R4) + (uint64(h4) * R3)
- d3 := (d2 >> 26) + (uint64(h0) * r3) + (uint64(h1) * r2) + (uint64(h2) * r1) + (uint64(h3) * r0) + (uint64(h4) * R4)
- d4 := (d3 >> 26) + (uint64(h0) * r4) + (uint64(h1) * r3) + (uint64(h2) * r2) + (uint64(h3) * r1) + (uint64(h4) * r0)
-
- // h %= p
- h0 = uint32(d0) & 0x3ffffff
- h1 = uint32(d1) & 0x3ffffff
- h2 = uint32(d2) & 0x3ffffff
- h3 = uint32(d3) & 0x3ffffff
- h4 = uint32(d4) & 0x3ffffff
-
- h0 += uint32(d4>>26) * 5
- h1 += h0 >> 26
- h0 = h0 & 0x3ffffff
-
- msg = msg[TagSize:]
+func add128(a, b uint128) uint128 {
+ lo, c := bitsAdd64(a.lo, b.lo, 0)
+ hi, c := bitsAdd64(a.hi, b.hi, c)
+ if c != 0 {
+ panic("poly1305: unexpected overflow")
}
+ return uint128{lo, hi}
+}
- h[0], h[1], h[2], h[3], h[4] = h0, h1, h2, h3, h4
+func shiftRightBy2(a uint128) uint128 {
+ a.lo = a.lo>>2 | (a.hi&3)<<62
+ a.hi = a.hi >> 2
+ return a
}
-func finalizeGeneric(out *[TagSize]byte, h *[5]uint32, s *[4]uint32) {
- h0, h1, h2, h3, h4 := h[0], h[1], h[2], h[3], h[4]
-
- // h %= p reduction
- h2 += h1 >> 26
- h1 &= 0x3ffffff
- h3 += h2 >> 26
- h2 &= 0x3ffffff
- h4 += h3 >> 26
- h3 &= 0x3ffffff
- h0 += 5 * (h4 >> 26)
- h4 &= 0x3ffffff
- h1 += h0 >> 26
- h0 &= 0x3ffffff
-
- // h - p
- t0 := h0 + 5
- t1 := h1 + (t0 >> 26)
- t2 := h2 + (t1 >> 26)
- t3 := h3 + (t2 >> 26)
- t4 := h4 + (t3 >> 26) - (1 << 26)
- t0 &= 0x3ffffff
- t1 &= 0x3ffffff
- t2 &= 0x3ffffff
- t3 &= 0x3ffffff
-
- // select h if h < p else h - p
- t_mask := (t4 >> 31) - 1
- h_mask := ^t_mask
- h0 = (h0 & h_mask) | (t0 & t_mask)
- h1 = (h1 & h_mask) | (t1 & t_mask)
- h2 = (h2 & h_mask) | (t2 & t_mask)
- h3 = (h3 & h_mask) | (t3 & t_mask)
- h4 = (h4 & h_mask) | (t4 & t_mask)
-
- // h %= 2^128
- h0 |= h1 << 26
- h1 = ((h1 >> 6) | (h2 << 20))
- h2 = ((h2 >> 12) | (h3 << 14))
- h3 = ((h3 >> 18) | (h4 << 8))
-
- // s: the s part of the key
- // tag = (h + s) % (2^128)
- t := uint64(h0) + uint64(s[0])
- h0 = uint32(t)
- t = uint64(h1) + uint64(s[1]) + (t >> 32)
- h1 = uint32(t)
- t = uint64(h2) + uint64(s[2]) + (t >> 32)
- h2 = uint32(t)
- t = uint64(h3) + uint64(s[3]) + (t >> 32)
- h3 = uint32(t)
-
- binary.LittleEndian.PutUint32(out[0:], h0)
- binary.LittleEndian.PutUint32(out[4:], h1)
- binary.LittleEndian.PutUint32(out[8:], h2)
- binary.LittleEndian.PutUint32(out[12:], h3)
+// updateGeneric absorbs msg into the state.h accumulator. For each chunk m of
+// 128 bits of message, it computes
+//
+// h₊ = (h + m) * r mod 2¹³⁰ - 5
+//
+// If the msg length is not a multiple of TagSize, it assumes the last
+// incomplete chunk is the final one.
+func updateGeneric(state *macState, msg []byte) {
+ h0, h1, h2 := state.h[0], state.h[1], state.h[2]
+ r0, r1 := state.r[0], state.r[1]
+
+ for len(msg) > 0 {
+ var c uint64
+
+ // For the first step, h + m, we use a chain of bits.Add64 intrinsics.
+ // The resulting value of h might exceed 2¹³⁰ - 5, but will be partially
+ // reduced at the end of the multiplication below.
+ //
+ // The spec requires us to set a bit just above the message size, not to
+ // hide leading zeroes. For full chunks, that's 1 << 128, so we can just
+ // add 1 to the most significant (2¹²⁸) limb, h2.
+ if len(msg) >= TagSize {
+ h0, c = bitsAdd64(h0, binary.LittleEndian.Uint64(msg[0:8]), 0)
+ h1, c = bitsAdd64(h1, binary.LittleEndian.Uint64(msg[8:16]), c)
+ h2 += c + 1
+
+ msg = msg[TagSize:]
+ } else {
+ var buf [TagSize]byte
+ copy(buf[:], msg)
+ buf[len(msg)] = 1
+
+ h0, c = bitsAdd64(h0, binary.LittleEndian.Uint64(buf[0:8]), 0)
+ h1, c = bitsAdd64(h1, binary.LittleEndian.Uint64(buf[8:16]), c)
+ h2 += c
+
+ msg = nil
+ }
+
+ // Multiplication of big number limbs is similar to elementary school
+ // columnar multiplication. Instead of digits, there are 64-bit limbs.
+ //
+ // We are multiplying a 3 limbs number, h, by a 2 limbs number, r.
+ //
+ // h2 h1 h0 x
+ // r1 r0 =
+ // ----------------
+ // h2r0 h1r0 h0r0 <-- individual 128-bit products
+ // + h2r1 h1r1 h0r1
+ // ------------------------
+ // m3 m2 m1 m0 <-- result in 128-bit overlapping limbs
+ // ------------------------
+ // m3.hi m2.hi m1.hi m0.hi <-- carry propagation
+ // + m3.lo m2.lo m1.lo m0.lo
+ // -------------------------------
+ // t4 t3 t2 t1 t0 <-- final result in 64-bit limbs
+ //
+ // The main difference from pen-and-paper multiplication is that we do
+ // carry propagation in a separate step, as if we wrote two digit sums
+ // at first (the 128-bit limbs), and then carried the tens all at once.
+
+ h0r0 := mul64(h0, r0)
+ h1r0 := mul64(h1, r0)
+ h2r0 := mul64(h2, r0)
+ h0r1 := mul64(h0, r1)
+ h1r1 := mul64(h1, r1)
+ h2r1 := mul64(h2, r1)
+
+ // Since h2 is known to be at most 7 (5 + 1 + 1), and r0 and r1 have their
+ // top 4 bits cleared by rMask{0,1}, we know that their product is not going
+ // to overflow 64 bits, so we can ignore the high part of the products.
+ //
+ // This also means that the product doesn't have a fifth limb (t4).
+ if h2r0.hi != 0 {
+ panic("poly1305: unexpected overflow")
+ }
+ if h2r1.hi != 0 {
+ panic("poly1305: unexpected overflow")
+ }
+
+ m0 := h0r0
+ m1 := add128(h1r0, h0r1) // These two additions don't overflow thanks again
+ m2 := add128(h2r0, h1r1) // to the 4 masked bits at the top of r0 and r1.
+ m3 := h2r1
+
+ t0 := m0.lo
+ t1, c := bitsAdd64(m1.lo, m0.hi, 0)
+ t2, c := bitsAdd64(m2.lo, m1.hi, c)
+ t3, _ := bitsAdd64(m3.lo, m2.hi, c)
+
+ // Now we have the result as 4 64-bit limbs, and we need to reduce it
+ // modulo 2¹³⁰ - 5. The special shape of this Crandall prime lets us do
+ // a cheap partial reduction according to the reduction identity
+ //
+ // c * 2¹³⁰ + n = c * 5 + n mod 2¹³⁰ - 5
+ //
+ // because 2¹³⁰ = 5 mod 2¹³⁰ - 5. Partial reduction since the result is
+ // likely to be larger than 2¹³⁰ - 5, but still small enough to fit the
+ // assumptions we make about h in the rest of the code.
+ //
+ // See also https://speakerdeck.com/gtank/engineering-prime-numbers?slide=23
+
+ // We split the final result at the 2¹³⁰ mark into h and cc, the carry.
+ // Note that the carry bits are effectively shifted left by 2, in other
+ // words, cc = c * 4 for the c in the reduction identity.
+ h0, h1, h2 = t0, t1, t2&maskLow2Bits
+ cc := uint128{t2 & maskNotLow2Bits, t3}
+
+ // To add c * 5 to h, we first add cc = c * 4, and then add (cc >> 2) = c.
+
+ h0, c = bitsAdd64(h0, cc.lo, 0)
+ h1, c = bitsAdd64(h1, cc.hi, c)
+ h2 += c
+
+ cc = shiftRightBy2(cc)
+
+ h0, c = bitsAdd64(h0, cc.lo, 0)
+ h1, c = bitsAdd64(h1, cc.hi, c)
+ h2 += c
+
+ // h2 is at most 3 + 1 + 1 = 5, making the whole of h at most
+ //
+ // 5 * 2¹²⁸ + (2¹²⁸ - 1) = 6 * 2¹²⁸ - 1
+ }
+
+ state.h[0], state.h[1], state.h[2] = h0, h1, h2
+}
+
+const (
+ maskLow2Bits uint64 = 0x0000000000000003
+ maskNotLow2Bits uint64 = ^maskLow2Bits
+)
+
+// select64 returns x if v == 1 and y if v == 0, in constant time.
+func select64(v, x, y uint64) uint64 { return ^(v-1)&x | (v-1)&y }
+
+// [p0, p1, p2] is 2¹³⁰ - 5 in little endian order.
+const (
+ p0 = 0xFFFFFFFFFFFFFFFB
+ p1 = 0xFFFFFFFFFFFFFFFF
+ p2 = 0x0000000000000003
+)
+
+// finalize completes the modular reduction of h and computes
+//
+// out = h + s mod 2¹²⁸
+//
+func finalize(out *[TagSize]byte, h *[3]uint64, s *[2]uint64) {
+ h0, h1, h2 := h[0], h[1], h[2]
+
+ // After the partial reduction in updateGeneric, h might be more than
+ // 2¹³⁰ - 5, but will be less than 2 * (2¹³⁰ - 5). To complete the reduction
+ // in constant time, we compute t = h - (2¹³⁰ - 5), and select h as the
+ // result if the subtraction underflows, and t otherwise.
+
+ hMinusP0, b := bitsSub64(h0, p0, 0)
+ hMinusP1, b := bitsSub64(h1, p1, b)
+ _, b = bitsSub64(h2, p2, b)
+
+ // h = h if h < p else h - p
+ h0 = select64(b, h0, hMinusP0)
+ h1 = select64(b, h1, hMinusP1)
+
+ // Finally, we compute the last Poly1305 step
+ //
+ // tag = h + s mod 2¹²⁸
+ //
+ // by just doing a wide addition with the 128 low bits of h and discarding
+ // the overflow.
+ h0, c := bitsAdd64(h0, s[0], 0)
+ h1, _ = bitsAdd64(h1, s[1], c)
+
+ binary.LittleEndian.PutUint64(out[0:8], h0)
+ binary.LittleEndian.PutUint64(out[8:16], h1)
}
package poly1305
-// Sum generates an authenticator for msg using a one-time key and puts the
-// 16-byte result into out. Authenticating two different messages with the same
-// key allows an attacker to forge messages at will.
-func Sum(out *[TagSize]byte, msg []byte, key *[32]byte) {
+func sum(out *[TagSize]byte, msg []byte, key *[32]byte) {
h := newMAC(key)
h.Write(msg)
h.Sum(out)
package poly1305
//go:noescape
-func initialize(state *[7]uint64, key *[32]byte)
+func update(state *macState, msg []byte)
-//go:noescape
-func update(state *[7]uint64, msg []byte)
-
-//go:noescape
-func finalize(tag *[TagSize]byte, state *[7]uint64)
-
-// Sum generates an authenticator for m using a one-time key and puts the
-// 16-byte result into out. Authenticating two different messages with the same
-// key allows an attacker to forge messages at will.
-func Sum(out *[16]byte, m []byte, key *[32]byte) {
+func sum(out *[16]byte, m []byte, key *[32]byte) {
h := newMAC(key)
h.Write(m)
h.Sum(out)
}
func newMAC(key *[32]byte) (h mac) {
- initialize(&h.state, key)
+ initialize(key, &h.r, &h.s)
return
}
-type mac struct {
- state [7]uint64 // := uint64{ h0, h1, h2, r0, r1, pad0, pad1 }
-
- buffer [TagSize]byte
- offset int
-}
+// mac is a wrapper for macGeneric that redirects calls that would have gone to
+// updateGeneric to update.
+//
+// Its Write and Sum methods are otherwise identical to the macGeneric ones, but
+// using function pointers would carry a major performance cost.
+type mac struct{ macGeneric }
-func (h *mac) Write(p []byte) (n int, err error) {
- n = len(p)
+func (h *mac) Write(p []byte) (int, error) {
+ nn := len(p)
if h.offset > 0 {
- remaining := TagSize - h.offset
- if n < remaining {
- h.offset += copy(h.buffer[h.offset:], p)
- return n, nil
+ n := copy(h.buffer[h.offset:], p)
+ if h.offset+n < TagSize {
+ h.offset += n
+ return nn, nil
}
- copy(h.buffer[h.offset:], p[:remaining])
- p = p[remaining:]
+ p = p[n:]
h.offset = 0
- update(&h.state, h.buffer[:])
+ update(&h.macState, h.buffer[:])
}
- if nn := len(p) - (len(p) % TagSize); nn > 0 {
- update(&h.state, p[:nn])
- p = p[nn:]
+ if n := len(p) - (len(p) % TagSize); n > 0 {
+ update(&h.macState, p[:n])
+ p = p[n:]
}
if len(p) > 0 {
h.offset += copy(h.buffer[h.offset:], p)
}
- return n, nil
+ return nn, nil
}
func (h *mac) Sum(out *[16]byte) {
- state := h.state
+ state := h.macState
if h.offset > 0 {
update(&state, h.buffer[:h.offset])
}
- finalize(out, &state)
+ finalize(out, &state.h, &state.s)
}
GLOBL ·poly1305Mask<>(SB), RODATA, $16
// func update(state *[7]uint64, msg []byte)
-
TEXT ·update(SB), $0-32
MOVD state+0(FP), R3
MOVD msg_base+8(FP), R4
MOVD R9, 8(R3)
MOVD R10, 16(R3)
RET
-
-// func initialize(state *[7]uint64, key *[32]byte)
-TEXT ·initialize(SB), $0-16
- MOVD state+0(FP), R3
- MOVD key+8(FP), R4
-
- // state[0...7] is initialized with zero
- // Load key
- MOVD 0(R4), R5
- MOVD 8(R4), R6
- MOVD 16(R4), R7
- MOVD 24(R4), R8
-
- // Address of key mask
- MOVD $·poly1305Mask<>(SB), R9
-
- // Save original key in state
- MOVD R7, 40(R3)
- MOVD R8, 48(R3)
-
- // Get mask
- MOVD (R9), R7
- MOVD 8(R9), R8
-
- // And with key
- AND R5, R7, R5
- AND R6, R8, R6
-
- // Save masked key in state
- MOVD R5, 24(R3)
- MOVD R6, 32(R3)
- RET
-
-// func finalize(tag *[TagSize]byte, state *[7]uint64)
-TEXT ·finalize(SB), $0-16
- MOVD tag+0(FP), R3
- MOVD state+8(FP), R4
-
- // Get h0, h1, h2 from state
- MOVD 0(R4), R5
- MOVD 8(R4), R6
- MOVD 16(R4), R7
-
- // Save h0, h1
- MOVD R5, R8
- MOVD R6, R9
- MOVD $3, R20
- MOVD $-1, R21
- SUBC $-5, R5
- SUBE R21, R6
- SUBE R20, R7
- MOVD $0, R21
- SUBZE R21
-
- // Check for carry
- CMP $0, R21
- ISEL $2, R5, R8, R5
- ISEL $2, R6, R9, R6
- MOVD 40(R4), R8
- MOVD 48(R4), R9
- ADDC R8, R5
- ADDE R9, R6
- MOVD R5, 0(R3)
- MOVD R6, 8(R3)
- RET
//go:noescape
func poly1305vmsl(out *[16]byte, m *byte, mlen uint64, key *[32]byte)
-// Sum generates an authenticator for m using a one-time key and puts the
-// 16-byte result into out. Authenticating two different messages with the same
-// key allows an attacker to forge messages at will.
-func Sum(out *[16]byte, m []byte, key *[32]byte) {
+func sum(out *[16]byte, m []byte, key *[32]byte) {
if cpu.S390X.HasVX {
var mPtr *byte
if len(m) > 0 {
"hash"
"io"
"io/ioutil"
- "math/bits"
- "golang.org/x/crypto/internal/chacha20"
+ "golang.org/x/crypto/chacha20"
"golang.org/x/crypto/poly1305"
)
// the methods here also implement padding, which RFC4253 Section 6
// also requires of stream ciphers.
type chacha20Poly1305Cipher struct {
- lengthKey [8]uint32
- contentKey [8]uint32
+ lengthKey [32]byte
+ contentKey [32]byte
buf []byte
}
buf: make([]byte, 256),
}
- for i := range c.contentKey {
- c.contentKey[i] = binary.LittleEndian.Uint32(key[i*4 : (i+1)*4])
- }
- for i := range c.lengthKey {
- c.lengthKey[i] = binary.LittleEndian.Uint32(key[(i+8)*4 : (i+9)*4])
- }
+ copy(c.contentKey[:], key[:32])
+ copy(c.lengthKey[:], key[32:])
return c, nil
}
func (c *chacha20Poly1305Cipher) readCipherPacket(seqNum uint32, r io.Reader) ([]byte, error) {
- nonce := [3]uint32{0, 0, bits.ReverseBytes32(seqNum)}
- s := chacha20.New(c.contentKey, nonce)
- var polyKey [32]byte
+ nonce := make([]byte, 12)
+ binary.BigEndian.PutUint32(nonce[8:], seqNum)
+ s, err := chacha20.NewUnauthenticatedCipher(c.contentKey[:], nonce)
+ if err != nil {
+ return nil, err
+ }
+ var polyKey, discardBuf [32]byte
s.XORKeyStream(polyKey[:], polyKey[:])
- s.Advance() // skip next 32 bytes
+ s.XORKeyStream(discardBuf[:], discardBuf[:]) // skip the next 32 bytes
encryptedLength := c.buf[:4]
if _, err := io.ReadFull(r, encryptedLength); err != nil {
}
var lenBytes [4]byte
- chacha20.New(c.lengthKey, nonce).XORKeyStream(lenBytes[:], encryptedLength)
+ ls, err := chacha20.NewUnauthenticatedCipher(c.lengthKey[:], nonce)
+ if err != nil {
+ return nil, err
+ }
+ ls.XORKeyStream(lenBytes[:], encryptedLength)
length := binary.BigEndian.Uint32(lenBytes[:])
if length > maxPacket {
}
func (c *chacha20Poly1305Cipher) writeCipherPacket(seqNum uint32, w io.Writer, rand io.Reader, payload []byte) error {
- nonce := [3]uint32{0, 0, bits.ReverseBytes32(seqNum)}
- s := chacha20.New(c.contentKey, nonce)
- var polyKey [32]byte
+ nonce := make([]byte, 12)
+ binary.BigEndian.PutUint32(nonce[8:], seqNum)
+ s, err := chacha20.NewUnauthenticatedCipher(c.contentKey[:], nonce)
+ if err != nil {
+ return err
+ }
+ var polyKey, discardBuf [32]byte
s.XORKeyStream(polyKey[:], polyKey[:])
- s.Advance() // skip next 32 bytes
+ s.XORKeyStream(discardBuf[:], discardBuf[:]) // skip the next 32 bytes
// There is no blocksize, so fall back to multiple of 8 byte
// padding, as described in RFC 4253, Sec 6.
}
binary.BigEndian.PutUint32(c.buf, uint32(1+len(payload)+padding))
- chacha20.New(c.lengthKey, nonce).XORKeyStream(c.buf, c.buf[:4])
+ ls, err := chacha20.NewUnauthenticatedCipher(c.lengthKey[:], nonce)
+ if err != nil {
+ return err
+ }
+ ls.XORKeyStream(c.buf, c.buf[:4])
c.buf[4] = byte(padding)
copy(c.buf[5:], payload)
packetEnd := 5 + len(payload) + padding
HostKey: hostKeyBytes,
Signature: sig,
Hash: crypto.SHA1,
- }, nil
+ }, err
}
// ecdh performs Elliptic Curve Diffie-Hellman key exchange as
go.mongodb.org/mongo-driver/bson/bsontype
go.mongodb.org/mongo-driver/bson/primitive
go.mongodb.org/mongo-driver/x/bsonx/bsoncore
-# golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad
+# golang.org/x/crypto v0.0.0-20191117063200-497ca9f6d64f
golang.org/x/crypto/acme
golang.org/x/crypto/acme/autocert
golang.org/x/crypto/argon2
golang.org/x/crypto/blake2b
golang.org/x/crypto/blowfish
golang.org/x/crypto/cast5
+golang.org/x/crypto/chacha20
golang.org/x/crypto/curve25519
golang.org/x/crypto/ed25519
golang.org/x/crypto/ed25519/internal/edwards25519
-golang.org/x/crypto/internal/chacha20
golang.org/x/crypto/internal/subtle
golang.org/x/crypto/md4
golang.org/x/crypto/openpgp
projects / gitea.git / commitdiff