From 43845cfccd9bde89553e766758a3076e14dc8e5b Mon Sep 17 00:00:00 2001 From: Jackson Owens Date: Thu, 25 Jun 2020 11:29:47 -0400 Subject: [PATCH 01/49] bump Pebble to 560ed0b8a18c 560ed0b compaction: Don't do a move compaction within L0. b822482 compaction: Exclude largest key when extending L0 compactions if it's sentinel 585d322 internal/metamorphic: add checkpointOp, compactOp and flushOp 5bdf31a cmd/pebble: enable L0-sublevel compactions and flush splits d478834 db: switch compaction concurrency signal from level score to L0 read-amp c577a10 db: force flush memtables with range deletions after delay fd3f2e3 db: add memtable count, duration and byte rate to flush event 730159f db: tweak the compaction scoring heuristics b34dbe9 db: tighten merging iterator's seek conditions c08345d tool: Allow vertical scrolling if visualization overflows 747113e db: fix race in TestTableStats 90dbbfc *: {white,black}list -> {allow,deny}list --- github.com/cockroachdb/pebble/batch.go | 6 + github.com/cockroachdb/pebble/compaction.go | 59 ++- .../cockroachdb/pebble/compaction_picker.go | 233 ++++++---- github.com/cockroachdb/pebble/db.go | 20 +- github.com/cockroachdb/pebble/event.go | 24 +- github.com/cockroachdb/pebble/flushable.go | 3 + .../pebble/internal/manifest/l0_sublevels.go | 20 +- github.com/cockroachdb/pebble/merging_iter.go | 54 ++- github.com/cockroachdb/pebble/open.go | 1 + github.com/cockroachdb/pebble/options.go | 40 +- .../cockroachdb/pebble/tool/lsm_data.go | 2 + .../pebble/vfs/syncing_file_linux.go | 6 +- github.com/cockroachdb/pebble/vfs/vfs.go | 2 +- golang.org/x/sys/cpu/byteorder.go | 11 +- golang.org/x/sys/cpu/cpu_arm64.go | 8 +- golang.org/x/sys/unix/syscall_linux.go | 28 +- golang.org/x/sys/unix/zerrors_linux.go | 66 +-- golang.org/x/sys/unix/zerrors_linux_386.go | 1 + golang.org/x/sys/unix/zerrors_linux_amd64.go | 1 + golang.org/x/sys/unix/zerrors_linux_arm.go | 1 + golang.org/x/sys/unix/zerrors_linux_arm64.go | 1 + golang.org/x/sys/unix/zerrors_linux_mips.go | 1 + golang.org/x/sys/unix/zerrors_linux_mips64.go | 1 + .../x/sys/unix/zerrors_linux_mips64le.go | 1 + golang.org/x/sys/unix/zerrors_linux_mipsle.go | 1 + golang.org/x/sys/unix/zerrors_linux_ppc64.go | 1 + .../x/sys/unix/zerrors_linux_ppc64le.go | 1 + .../x/sys/unix/zerrors_linux_riscv64.go | 1 + golang.org/x/sys/unix/zerrors_linux_s390x.go | 1 + .../x/sys/unix/zerrors_linux_sparc64.go | 1 + golang.org/x/sys/unix/ztypes_freebsd_arm.go | 12 +- golang.org/x/sys/unix/ztypes_linux.go | 416 +++++++++++------- golang.org/x/sys/windows/memory_windows.go | 5 + golang.org/x/sys/windows/syscall_windows.go | 2 + golang.org/x/sys/windows/zsyscall_windows.go | 19 + modules.txt | 4 +- 36 files changed, 695 insertions(+), 359 deletions(-) diff --git a/github.com/cockroachdb/pebble/batch.go b/github.com/cockroachdb/pebble/batch.go index bf7f9f6cdd..3c400bd774 100644 --- a/github.com/cockroachdb/pebble/batch.go +++ b/github.com/cockroachdb/pebble/batch.go @@ -196,6 +196,10 @@ type Batch struct { // data whenever Repr() is called. count uint64 + // The count of range deletions in the batch. Updated every time a range + // deletion is added. + countRangeDels uint64 + // A deferredOp struct, stored in the Batch so that a pointer can be returned // from the *Deferred() methods rather than a value. deferredOp DeferredBatchOp @@ -594,6 +598,7 @@ func (b *Batch) DeleteRange(start, end []byte, _ *WriteOptions) error { // with the end key. func (b *Batch) DeleteRangeDeferred(startLen, endLen int) *DeferredBatchOp { b.prepareDeferredKeyValueRecord(startLen, endLen, InternalKeyKindRangeDelete) + b.countRangeDels++ if b.index != nil { b.tombstones = nil // Range deletions are rare, so we lazily allocate the index for them. @@ -749,6 +754,7 @@ func (b *Batch) init(cap int) { // Commits and Closes take care of releasing resources when appropriate. func (b *Batch) Reset() { b.count = 0 + b.countRangeDels = 0 if b.data != nil { if cap(b.data) > batchMaxRetainedSize { // If the capacity of the buffer is larger than our maximum diff --git a/github.com/cockroachdb/pebble/compaction.go b/github.com/cockroachdb/pebble/compaction.go index 6be91ff938..008fbabebf 100644 --- a/github.com/cockroachdb/pebble/compaction.go +++ b/github.com/cockroachdb/pebble/compaction.go @@ -284,7 +284,7 @@ func (c *compaction) setupInputs() { // Call the L0-specific compaction extension method. Similar logic as // c.grow. Additional L0 files are optionally added to the compaction at // this step. - if c.version.L0Sublevels.ExtendL0ForBaseCompactionTo(c.smallest.UserKey, c.largest.UserKey, c.lcf) { + if c.version.L0Sublevels.ExtendL0ForBaseCompactionTo(c.smallest, c.largest, c.lcf) { c.inputs[0] = c.inputs[0][:0] for j := range c.lcf.FilesIncluded { if c.lcf.FilesIncluded[j] { @@ -958,6 +958,53 @@ func (d *DB) maybeScheduleFlush() { go d.flush() } +func (d *DB) maybeScheduleDelayedFlush(tbl *memTable) { + var mem *flushableEntry + for _, m := range d.mu.mem.queue { + if m.flushable == tbl { + mem = m + break + } + } + if mem == nil || mem.flushForced || mem.delayedFlushForced { + return + } + mem.delayedFlushForced = true + go func() { + timer := time.NewTimer(d.opts.Experimental.DeleteRangeFlushDelay) + defer timer.Stop() + + select { + case <-d.closedCh: + return + case <-mem.flushed: + return + case <-timer.C: + d.commit.mu.Lock() + defer d.commit.mu.Unlock() + d.mu.Lock() + defer d.mu.Unlock() + + // NB: The timer may fire concurrently with a call to Close. If a + // Close call beat us to acquiring d.mu, d.closed is 1, and it's + // too late to flush anything. Otherwise, the Close call will + // block on locking d.mu until we've finished scheduling the flush + // and set `d.mu.compact.flushing` to true. Close will wait for + // the current flush to complete. + if atomic.LoadInt32(&d.closed) != 0 { + return + } + + if d.mu.mem.mutable == tbl { + d.makeRoomForWrite(nil) + } else { + mem.flushForced = true + d.maybeScheduleFlush() + } + } + }() +} + func (d *DB) flush() { pprof.Do(context.Background(), flushLabels, func(context.Context) { d.mu.Lock() @@ -1014,7 +1061,9 @@ func (d *DB) flush1() error { d.mu.nextJobID++ d.opts.EventListener.FlushBegin(FlushInfo{ JobID: jobID, + Input: n, }) + startTime := d.timeNow() flushPacer := (pacer)(nilPacer) if d.opts.private.enablePacing { @@ -1029,9 +1078,11 @@ func (d *DB) flush1() error { ve, pendingOutputs, err := d.runCompaction(jobID, c, flushPacer) info := FlushInfo{ - JobID: jobID, - Done: true, - Err: err, + JobID: jobID, + Input: n, + Duration: d.timeNow().Sub(startTime), + Done: true, + Err: err, } if err == nil { for i := range ve.NewFiles { diff --git a/github.com/cockroachdb/pebble/compaction_picker.go b/github.com/cockroachdb/pebble/compaction_picker.go index 186e16ca50..8ffc38d3cc 100644 --- a/github.com/cockroachdb/pebble/compaction_picker.go +++ b/github.com/cockroachdb/pebble/compaction_picker.go @@ -5,10 +5,13 @@ package pebble import ( + "bytes" + "fmt" "math" "sort" "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/humanize" "github.com/cockroachdb/pebble/internal/manifest" ) @@ -71,8 +74,9 @@ func newCompactionPicker( // compaction picker. type pickedCompactionInfo struct { // The score of the level to be compacted. - score float64 - level int + score float64 + origScore float64 + level int // The level to compact to. outputLevel int // The file in level that will be compacted. Additional files may be picked by the @@ -113,9 +117,6 @@ type compactionPickerByScore struct { // added to L0. estimatedMaxWAmp float64 - // smoothedLevelMultiplier is the size ratio between one level and the next. - smoothedLevelMultiplier float64 - // levelMaxBytes holds the dynamically adjusted max bytes setting for each // level. levelMaxBytes [numLevels]int64 @@ -184,11 +185,14 @@ func (p *compactionPickerByScore) estimatedCompactionDebt(l0ExtraSize uint64) ui func (p *compactionPickerByScore) initLevelMaxBytes(inProgressCompactions []compactionInfo) { // The levelMaxBytes calculations here differ from RocksDB in two ways: // - // 1. The use of bottomLevelSize vs maxLevelSize. RocksDB uses the size of - // the maximum level in L1-L6, rather than the size of the bottommost - // non-empty level. In practice this seems to have little impact. + // 1. The use of dbSize vs maxLevelSize. RocksDB uses the size of the maximum + // level in L1-L6, rather than determining the size of the bottom level + // based on the total amount of data in the dB. The RocksDB calculation is + // problematic if L0 contains a significant fraction of data, or if the + // level sizes are roughly equal and thus there is a significant fraction + // of data outside of the largest level. // - // 2. Not adjusting the size of base level based on L0. RocksDB computes + // 2. Not adjusting the size of Lbase based on L0. RocksDB computes // baseBytesMax as the maximum of the configured LBaseMaxBytes and the // size of L0. This is problematic because baseBytesMax is used to compute // the max size of lower levels. A very large baseBytesMax will result in @@ -196,26 +200,17 @@ func (p *compactionPickerByScore) initLevelMaxBytes(inProgressCompactions []comp // those levels not to be compacted even when they should be // compacted. This often results in "inverted" LSM shapes where Ln is // larger than Ln+1. - // - // TODO(peter): An alternative to the current calculation of bottomLevelSize - // is to compute the total number of bytes in the LSM and then compute - // bottomLevelSize as 90% of that value (presuming a level multiplier of - // 10). This computation has the advantage of being stable: it changes at the - // rate that data is inserted into the DB, independently of - // compactions. Unfortunately, it performed worse experimentally and often - // resulted in "inverted" LSM shapes where L5 was significantly larger than - // L6. The reason for this inversion was not clear. - - // Determine the first non-empty level and the bottom level size. + + // Determine the first non-empty level and the total DB size. firstNonEmptyLevel := -1 - var bottomLevelSize int64 + var dbSize int64 for level := 1; level < numLevels; level++ { levelSize := int64(totalSize(p.vers.Levels[level])) if levelSize > 0 { if firstNonEmptyLevel == -1 { firstNonEmptyLevel = level } - bottomLevelSize = levelSize + dbSize += levelSize } } for _, c := range inProgressCompactions { @@ -234,7 +229,7 @@ func (p *compactionPickerByScore) initLevelMaxBytes(inProgressCompactions []comp p.levelMaxBytes[level] = math.MaxInt64 } - if bottomLevelSize == 0 { + if dbSize == 0 { // No levels for L1 and up contain any data. Target L0 compactions for the // last level or to the level to which there is an ongoing L0 compaction. p.baseLevel = numLevels - 1 @@ -244,44 +239,36 @@ func (p *compactionPickerByScore) initLevelMaxBytes(inProgressCompactions []comp return } - levelMultiplier := 10.0 - - baseBytesMax := p.opts.LBaseMaxBytes - baseBytesMin := int64(float64(baseBytesMax) / levelMultiplier) + const levelMultiplier = 10 + dbSize += int64(totalSize(p.vers.Levels[0])) + bottomLevelSize := dbSize - dbSize/levelMultiplier curLevelSize := bottomLevelSize for level := numLevels - 2; level >= firstNonEmptyLevel; level-- { curLevelSize = int64(float64(curLevelSize) / levelMultiplier) } - if curLevelSize <= baseBytesMin { - // If we make target size of last level to be bottomLevelSize, target size - // of the first non-empty level would be smaller than baseBytesMin. We set - // it be baseBytesMin. - p.baseLevel = firstNonEmptyLevel - } else { - // Compute base level (where L0 data is compacted to). - p.baseLevel = firstNonEmptyLevel - for p.baseLevel > 1 && curLevelSize > baseBytesMax { - p.baseLevel-- - curLevelSize = int64(float64(curLevelSize) / levelMultiplier) - } + // Compute base level (where L0 data is compacted to). + baseBytesMax := p.opts.LBaseMaxBytes + p.baseLevel = firstNonEmptyLevel + for p.baseLevel > 1 && curLevelSize > baseBytesMax { + p.baseLevel-- + curLevelSize = int64(float64(curLevelSize) / levelMultiplier) } + smoothedLevelMultiplier := 1.0 if p.baseLevel < numLevels-1 { - p.smoothedLevelMultiplier = math.Pow( + smoothedLevelMultiplier = math.Pow( float64(bottomLevelSize)/float64(baseBytesMax), 1.0/float64(numLevels-p.baseLevel-1)) - } else { - p.smoothedLevelMultiplier = 1.0 } - p.estimatedMaxWAmp = float64(numLevels-p.baseLevel) * (p.smoothedLevelMultiplier + 1) + p.estimatedMaxWAmp = float64(numLevels-p.baseLevel) * (smoothedLevelMultiplier + 1) levelSize := float64(baseBytesMax) for level := p.baseLevel; level < numLevels; level++ { if level > p.baseLevel && levelSize > 0 { - levelSize *= p.smoothedLevelMultiplier + levelSize *= smoothedLevelMultiplier } // Round the result since test cases use small target level sizes, which // can be impacted by floating-point imprecision + integer truncation. @@ -323,13 +310,48 @@ func (p *compactionPickerByScore) calculateScores( scores[0] = p.calculateL0Score(inProgressCompactions) sizeAdjust := calculateSizeAdjust(inProgressCompactions) - for level := 1; level < numLevels-1; level++ { - // Use the "compensated" file size when scoring. The file size is - // compensated by artifically inflating it to account for other - // priorities like reclaiming disk space beneath range tombstones. + for level := 1; level < numLevels; level++ { levelSize := int64(totalCompensatedSize(p.vers.Levels[level])) + sizeAdjust[level] scores[level].score = float64(levelSize) / float64(p.levelMaxBytes[level]) + scores[level].origScore = scores[level].score + } + + // Adjust each level's score by the score of the next level. If the next + // level has a high score, and is thus a priority for compaction, this + // reduces the priority for compacting the current level. If the next level + // has a low score (i.e. it is below its target size), this increases the + // priority for compacting the current level. + // + // The effect of this adjustment is to help prioritize compactions in lower + // levels. The following shows the adjusted score and original score. In this + // scenario, L0 has 68 sublevels. L3 (a.k.a. Lbase) is significantly above + // its target size. The original score prioritizes compactions from those two + // levels, but doing so ends up causing a future problem: data piles up in + // the higher levels, starving L5->L6 compactions, and to a lesser degree + // starving L4->L5 compactions. + // + // adjusted original + // score score size max-size + // L0 3.2 68.0 2.2 G - + // L3 3.2 21.1 1.3 G 64 M + // L4 3.4 6.7 3.1 G 467 M + // L5 3.4 2.0 6.6 G 3.3 G + // L6 0.6 0.6 14 G 24 G + var prevLevel int + for level := p.baseLevel; level < numLevels; level++ { + if scores[prevLevel].score >= 1 { + // Avoid absurdly large scores by placing a floor on the score that we'll + // adjust a level by. The value of 0.01 was chosen somewhat arbitrarily + const minScore = 0.01 + if scores[level].score >= minScore { + scores[prevLevel].score /= scores[level].score + } else { + scores[prevLevel].score /= minScore + } + } + prevLevel = level } + sort.Sort(sortCompactionLevelsDecreasingScore(scores[:])) return scores } @@ -344,10 +366,10 @@ func (p *compactionPickerByScore) calculateL0Score( // If L0Sublevels are present, we use the sublevel count as opposed to // the L0 file count to score this level. The base vs intra-L0 // compaction determination happens in pickAuto, not here. - info.score = float64(p.vers.L0Sublevels.MaxDepthAfterOngoingCompactions()) / - float64(p.opts.L0CompactionThreshold) + info.score = float64(2 * p.vers.L0Sublevels.MaxDepthAfterOngoingCompactions()) return info } + // TODO(peter): The current scoring logic precludes concurrent L0->Lbase // compactions in most cases because if there is an in-progress L0->Lbase // compaction we'll instead preferentially score an intra-L0 compaction. One @@ -497,51 +519,85 @@ func (p *compactionPickerByScore) pickFile(level, outputLevel int) int { // If a score-based compaction cannot be found, pickAuto falls back to looking // for a forced compaction (identified by FileMetadata.MarkedForCompaction). func (p *compactionPickerByScore) pickAuto(env compactionEnv) (c *compaction) { - // highPriorityThreshold controls compaction concurrency. If there is already - // a compaction in progress, highPriorityThreshold is set to the minimum - // score needed for a concurrent compaction to be initiated. Since all level - // scores are >= 0, a positive value will cause compactions to be - // disabled. We set highPriorityThreshold to a value only when a there is at - // least one in-progress compaction. Concurrent compactions are useful for - // ensuring that compaction doesn't fall far behind, but concurrent - // compactions can have an adverse affect on write throughput. - // - // There are a variety of possibilities for choosing highPriorityThreshold: - // - A fixed value: 1.5. - // - A value linear in the number of in-progress compactions: 2, 3, 4. - // - A value exponential in the number of in-progress compactions: 2, 4, 8. - // - // A fixed value tends to allow too much compaction concurrency. There was - // only a minor difference between the linear and exponential values, making - // the choice of exponential below somewhat arbitrary. - // - // For comparison, RocksDB alternates between only allowing a single - // compaction at a time to allowing the configured maximum number of - // concurrent compactions depending on whether compaction-debt has gotten too - // large or the number of L0 sstables has reached 2x the L0 compaction - // threshold. In testing, it is usually the latter condition that triggers - // concurrent compactions in RocksDB. - var highPriorityThreshold float64 - if len(env.inProgressCompactions) > 0 { - // Exponential high priority threshold: 2, 4, 8, ... - highPriorityThreshold = float64(int(1) << len(env.inProgressCompactions)) + // Compaction concurrency is controlled by L0 read-amp. We allow one + // additional compaction per L0CompactionConcurrency sublevels. Compaction + // concurrency is tied to L0 sublevels as that signal is independent of the + // database size. + if n := len(env.inProgressCompactions); n > 0 { + var l0ReadAmp int + if p.opts.Experimental.L0SublevelCompactions { + l0ReadAmp = p.vers.L0Sublevels.MaxDepthAfterOngoingCompactions() + } else { + l0ReadAmp = len(p.vers.Levels[0]) + } + if l0ReadAmp < n*p.opts.Experimental.L0CompactionConcurrency { + return nil + } } scores := p.calculateScores(env.inProgressCompactions) + // TODO(peter): Either remove, or change this into an event sent to the + // EventListener. + logCompaction := func(c *compaction) { + var buf bytes.Buffer + for i := 0; i < numLevels; i++ { + if i != 0 && i < p.baseLevel { + continue + } + + var info *pickedCompactionInfo + for j := range scores { + if scores[j].level == i { + info = &scores[j] + break + } + } + + marker := " " + if c.startLevel == info.level { + marker = "*" + } + fmt.Fprintf(&buf, " %sL%d: %5.1f %5.1f %8s %8s", + marker, info.level, info.score, info.origScore, + humanize.Int64(int64(totalCompensatedSize(p.vers.Levels[info.level]))), + humanize.Int64(p.levelMaxBytes[info.level]), + ) + + count := 0 + for i := range env.inProgressCompactions { + c := &env.inProgressCompactions[i] + if c.startLevel != info.level { + continue + } + count++ + if count == 1 { + fmt.Fprintf(&buf, " [") + } else { + fmt.Fprintf(&buf, " ") + } + fmt.Fprintf(&buf, "L%d->L%d", c.startLevel, c.outputLevel) + } + if count > 0 { + fmt.Fprintf(&buf, "]") + } + fmt.Fprintf(&buf, "\n") + } + p.opts.Logger.Infof("pickAuto: L%d->L%d\n%s", + c.startLevel, c.outputLevel, buf.String()) + } + // Check for a score-based compaction. "scores" has been sorted in order of // decreasing score. For each level with a score >= 1, we attempt to find a // compaction anchored at at that level. for i := range scores { info := &scores[i] - if info.score < highPriorityThreshold { - // Don't start a low priority compaction if there is already a compaction - // running. - return nil - } if info.score < 1 { break } + if info.level == numLevels-1 { + continue + } if info.level == 0 && p.opts.Experimental.L0SublevelCompactions { c = pickL0(env, p.opts, p.vers, p.baseLevel) @@ -549,6 +605,10 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (c *compaction) { // concurrently. if c != nil && !inputAlreadyCompacting(c) { c.score = info.score + // TODO(peter): remove + if false { + logCompaction(c) + } return c } continue @@ -563,6 +623,10 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (c *compaction) { // Fail-safe to protect against compacting the same sstable concurrently. if c != nil && !inputAlreadyCompacting(c) { c.score = info.score + // TODO(peter): remove + if false { + logCompaction(c) + } return c } } @@ -684,6 +748,9 @@ func pickL0(env compactionEnv, opts *Options, vers *version, baseLevel int) (c * } if len(c.inputs[0]) == 0 { opts.Logger.Fatalf("empty compaction chosen") + } else if len(c.inputs[0]) == 1 { + // A single-file intra-L0 compaction is unproductive. + return nil } c.smallest, c.largest = manifest.KeyRange(c.cmp, c.inputs[0], nil) c.setupInuseKeyRanges() diff --git a/github.com/cockroachdb/pebble/db.go b/github.com/cockroachdb/pebble/db.go index fecb58f1d0..108609d072 100644 --- a/github.com/cockroachdb/pebble/db.go +++ b/github.com/cockroachdb/pebble/db.go @@ -197,7 +197,8 @@ type DB struct { // updates. logRecycler logRecycler - closed int32 // updated atomically + closed int32 // updated atomically + closedCh chan struct{} // The count and size of referenced memtables. This includes memtables // present in DB.mu.mem.queue, as well as memtables that have been flushed @@ -564,6 +565,16 @@ func (d *DB) commitApply(b *Batch, mem *memTable) error { if err != nil { return err } + + // If the batch contains range tombstones and the database is configured + // to flush range deletions, schedule a delayed flush so that disk space + // may be reclaimed without additional writes or an explicit flush. + if b.countRangeDels > 0 && d.opts.Experimental.DeleteRangeFlushDelay > 0 { + d.mu.Lock() + d.maybeScheduleDelayedFlush(mem) + d.mu.Unlock() + } + if mem.writerUnref() { d.mu.Lock() d.maybeScheduleFlush() @@ -829,6 +840,7 @@ func (d *DB) Close() error { panic(ErrClosed) } atomic.StoreInt32(&d.closed, 1) + close(d.closedCh) defer d.opts.Cache.Unref() @@ -1267,11 +1279,11 @@ func (d *DB) makeRoomForWrite(b *Batch) error { continue } } - l0FileCount := len(d.mu.versions.currentVersion().Levels[0]) + l0ReadAmp := len(d.mu.versions.currentVersion().Levels[0]) if d.opts.Experimental.L0SublevelCompactions { - l0FileCount = d.mu.versions.currentVersion().L0Sublevels.ReadAmplification() + l0ReadAmp = d.mu.versions.currentVersion().L0Sublevels.ReadAmplification() } - if l0FileCount >= d.opts.L0StopWritesThreshold { + if l0ReadAmp >= d.opts.L0StopWritesThreshold { // There are too many level-0 files, so we wait. if !stalled { stalled = true diff --git a/github.com/cockroachdb/pebble/event.go b/github.com/cockroachdb/pebble/event.go index 7489422536..45e9243045 100644 --- a/github.com/cockroachdb/pebble/event.go +++ b/github.com/cockroachdb/pebble/event.go @@ -100,11 +100,14 @@ type FlushInfo struct { JobID int // Reason is the reason for the flush. Reason string + // Input contains the count of input memtables that were flushed. + Input int // Output contains the ouptut table generated by the flush. The output info // is empty for the flush begin event. - Output []TableInfo - Done bool - Err error + Output []TableInfo + Duration time.Duration + Done bool + Err error } func (i FlushInfo) String() string { @@ -112,14 +115,21 @@ func (i FlushInfo) String() string { return fmt.Sprintf("[JOB %d] flush error: %s", i.JobID, i.Err) } + plural := "s" + if i.Input == 1 { + plural = "" + } if !i.Done { - return fmt.Sprintf("[JOB %d] flushing to L0", i.JobID) + return fmt.Sprintf("[JOB %d] flushing %d memtable%s to L0", i.JobID, i.Input, plural) } - return fmt.Sprintf("[JOB %d] flushed to L0 [%s] (%s)", - i.JobID, + outputSize := tablesTotalSize(i.Output) + return fmt.Sprintf("[JOB %d] flushed %d memtable%s to L0 [%s] (%s), in %.1fs, output rate %s/s", + i.JobID, i.Input, plural, formatFileNums(i.Output), - humanize.Uint64(tablesTotalSize(i.Output))) + humanize.Uint64(outputSize), + i.Duration.Seconds(), + humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))) } // ManifestCreateInfo contains info about a manifest creation event. diff --git a/github.com/cockroachdb/pebble/flushable.go b/github.com/cockroachdb/pebble/flushable.go index 6e56fad18a..e7eba129d9 100644 --- a/github.com/cockroachdb/pebble/flushable.go +++ b/github.com/cockroachdb/pebble/flushable.go @@ -33,6 +33,9 @@ type flushableEntry struct { // flushForced indicates whether a flush was forced on this memtable (either // manual, or due to ingestion). Protected by DB.mu. flushForced bool + // delayedFlushForced indicates whether a timer has been set to force a flush + // on this memtable at some point in the future. Protected by DB.mu + delayedFlushForced bool // logNum corresponds to the WAL that contains the records present in the // receiver. logNum FileNum diff --git a/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go b/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go index 47cfd55682..0c260c6c3c 100644 --- a/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go +++ b/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go @@ -1237,21 +1237,29 @@ func (s *L0Sublevels) intraL0CompactionUsingSeed( // L0CompactionFiles to cover all L0 files in the specified key interval, // by calling extendCandidateToRectangle. func (s *L0Sublevels) ExtendL0ForBaseCompactionTo( - smallest []byte, largest []byte, candidate *L0CompactionFiles, + smallest, largest InternalKey, candidate *L0CompactionFiles, ) bool { firstIntervalIndex := sort.Search(len(s.orderedIntervals), func(i int) bool { // Need to start at >= smallest since if we widen too much we may miss // an Lbase file that overlaps with an L0 file that will get picked in // this widening, which would be bad. This interval will not start with // an immediate successor key. - return s.cmp(smallest, s.orderedIntervals[i].startKey.key) <= 0 + return s.cmp(smallest.UserKey, s.orderedIntervals[i].startKey.key) <= 0 }) // First interval that starts at or beyond the largest. This interval will not // start with an immediate successor key. - lastIntervalIndex := sort.Search(len(s.orderedIntervals), func(i int) bool { - return s.cmp(largest, s.orderedIntervals[i].startKey.key) < 0 - }) - // Right now, lastIntervalIndex has a start that's higher than largest. + var lastIntervalIndex int + if largest.Trailer == base.InternalKeyRangeDeleteSentinel { + // largest.UserKey is excluded. Do not expand L0 to include it. + lastIntervalIndex = sort.Search(len(s.orderedIntervals), func(i int) bool { + return s.cmp(largest.UserKey, s.orderedIntervals[i].startKey.key) <= 0 + }) + } else { + lastIntervalIndex = sort.Search(len(s.orderedIntervals), func(i int) bool { + return s.cmp(largest.UserKey, s.orderedIntervals[i].startKey.key) < 0 + }) + } + // Right now, lastIntervalIndex has a startKey that extends beyond largest. // The previous interval, by definition, has an end key higher than largest. // Iterate back twice to get the last interval that's completely within // [smallest, largest]. Except in the case where we went past the end of the diff --git a/github.com/cockroachdb/pebble/merging_iter.go b/github.com/cockroachdb/pebble/merging_iter.go index 8f7d3e19d9..779e8f8298 100644 --- a/github.com/cockroachdb/pebble/merging_iter.go +++ b/github.com/cockroachdb/pebble/merging_iter.go @@ -366,7 +366,32 @@ func (m *mergingIter) switchToMinHeap() { if l == cur { continue } - if l.iterKey == nil || l.iterKey.Kind() == base.InternalKeyKindRangeDelete { + + // If the iterator is exhausted, it may be out of bounds if range + // deletions modified our search key as we descended. we need to + // reposition it within the search bounds. If the current key is a + // range tombstone, the iterator might still be exhausted but at a + // sstable boundary sentinel. It would be okay to reposition an + // interator like this only through successive Next calls, except that + // it would violate the levelIter's invariants by causing it to return + // a key before the lower bound. + // + // bounds = [ f, _ ) + // L0: [ b ] [ f* z ] + // L1: [ a |----| k y ] + // L2: [ c (d) ] [ e g m ] + // L3: [ x ] + // + // * - current key [] - table bounds () - heap item + // + // In the above diagram, the L2 iterator is positioned at a sstable + // boundary (d) outside the lower bound (f). It arrived here from a + // seek whose seek-key was modified by a range tombstone. If we called + // Next on the L2 iterator, it would return e, violating its lower + // bound. Instead, we seek it to >= f and Next from there. + + if l.iterKey == nil || (l.iterKey.Kind() == base.InternalKeyKindRangeDelete && + m.heap.cmp(l.iterKey.UserKey, m.lower) < 0) { if m.lower != nil { l.iterKey, l.iterValue = l.iter.SeekGE(m.lower) } else { @@ -415,7 +440,32 @@ func (m *mergingIter) switchToMaxHeap() { if l == cur { continue } - if l.iterKey == nil || l.iterKey.Kind() == base.InternalKeyKindRangeDelete { + + // If the iterator is exhausted, it may be out of bounds if range + // deletions modified our search key as we descended. we need to + // reposition it within the search bounds. If the current key is a + // range tombstone, the iterator might still be exhausted but at a + // sstable boundary sentinel. It would be okay to reposition an + // interator like this only through successive Prev calls, except that + // it would violate the levelIter's invariants by causing it to return + // a key beyond the upper bound. + // + // bounds = [ _, g ) + // L0: [ b ] [ f* z ] + // L1: [ a |-------| k y ] + // L2: [ c d ] h [(i) m ] + // L3: [ e x ] + // + // * - current key [] - table bounds () - heap item + // + // In the above diagram, the L2 iterator is positioned at a sstable + // boundary (i) outside the upper bound (g). It arrived here from a + // seek whose seek-key was modified by a range tombstone. If we called + // Prev on the L2 iterator, it would return h, violating its upper + // bound. Instead, we seek it to < g, and Prev from there. + + if l.iterKey == nil || (l.iterKey.Kind() == base.InternalKeyKindRangeDelete && + m.heap.cmp(l.iterKey.UserKey, m.upper) >= 0) { if m.upper != nil { l.iterKey, l.iterValue = l.iter.SeekLT(m.upper) } else { diff --git a/github.com/cockroachdb/pebble/open.go b/github.com/cockroachdb/pebble/open.go index 2d4fa707cb..372bbd4c92 100644 --- a/github.com/cockroachdb/pebble/open.go +++ b/github.com/cockroachdb/pebble/open.go @@ -65,6 +65,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { abbreviatedKey: opts.Comparer.AbbreviatedKey, largeBatchThreshold: (opts.MemTableSize - int(memTableEmptySize)) / 2, logRecycler: logRecycler{limit: opts.MemTableStopWritesThreshold + 1}, + closedCh: make(chan struct{}), } defer func() { diff --git a/github.com/cockroachdb/pebble/options.go b/github.com/cockroachdb/pebble/options.go index 9f9b272e0c..57da2feb96 100644 --- a/github.com/cockroachdb/pebble/options.go +++ b/github.com/cockroachdb/pebble/options.go @@ -303,6 +303,11 @@ type Options struct { // TODO(bilal): Experiment with this option to pick a good value. FlushSplitBytes int64 + // The threshold of L0 read-amplification at which compaction concurrency + // is enabled. Every multiple of this value enables another concurrent + // compaction up to MaxConcurrentCompactions. + L0CompactionConcurrency int + // L0SublevelCompactions enables the use of L0 sublevel-based compaction // picking logic. Defaults to false for now. This logic will become // a non-configurable default once it's better tuned. @@ -311,6 +316,12 @@ type Options struct { // L0CompactionThreshold and L0StopWritesThreshold to refer to L0 // read amplification as opposed to the count of L0 files. L0SublevelCompactions bool + + // DeleteRangeFlushDelay configures how long the database should wait + // before forcing a flush of a memtable that contains a range + // deletion. Disk space cannot be reclaimed until the range deletion + // is flushed. No automatic flush occurs if zero. + DeleteRangeFlushDelay time.Duration } // Filters is a map from filter policy name to filter policy. It is used for @@ -324,11 +335,13 @@ type Options struct { // The default value uses the underlying operating system's file system. FS vfs.FS - // The number of files necessary to trigger an L0 compaction. + // The amount of L0 read-amplification necessary to trigger an L0 compaction. L0CompactionThreshold int - // Hard limit on the number of L0 files. Writes are stopped when this - // threshold is reached. + // Hard limit on L0 read-amplification. Writes are stopped when this + // threshold is reached. If Experimental.L0SublevelCompactions is enabled + // this threshold is measured against the number of L0 sublevels. Otherwise + // it is measured against the number of files in L0. L0StopWritesThreshold int // The maximum number of bytes for LBase. The base level is the level which @@ -388,8 +401,9 @@ type Options struct { // default is 1 MB/s. MinFlushRate int - // MaxConcurrentCompactions specifies the maximum number of concurrent compactions. The - // default is 1. + // MaxConcurrentCompactions specifies the maximum number of concurrent + // compactions. The default is 1. Concurrent compactions are only performed + // when L0 read-amplification passes the L0CompactionConcurrency threshold. MaxConcurrentCompactions int // ReadOnly indicates that the DB should be opened in read-only mode. Writes @@ -466,6 +480,9 @@ func (o *Options) EnsureDefaults() *Options { if o.FS == nil { o.FS = vfs.Default } + if o.Experimental.L0CompactionConcurrency <= 0 { + o.Experimental.L0CompactionConcurrency = 10 + } if o.L0CompactionThreshold <= 0 { o.L0CompactionThreshold = 4 } @@ -585,10 +602,13 @@ func (o *Options) String() string { fmt.Fprintf(&buf, " cache_size=%d\n", cacheSize) fmt.Fprintf(&buf, " cleaner=%s\n", o.Cleaner) fmt.Fprintf(&buf, " comparer=%s\n", o.Comparer.Name) + fmt.Fprintf(&buf, " delete_range_flush_delay=%s\n", o.Experimental.DeleteRangeFlushDelay) fmt.Fprintf(&buf, " disable_wal=%t\n", o.DisableWAL) fmt.Fprintf(&buf, " flush_split_bytes=%d\n", o.Experimental.FlushSplitBytes) + fmt.Fprintf(&buf, " l0_compaction_concurrency=%d\n", o.Experimental.L0CompactionConcurrency) fmt.Fprintf(&buf, " l0_compaction_threshold=%d\n", o.L0CompactionThreshold) fmt.Fprintf(&buf, " l0_stop_writes_threshold=%d\n", o.L0StopWritesThreshold) + fmt.Fprintf(&buf, " l0_sublevel_compactions=%t\n", o.Experimental.L0SublevelCompactions) fmt.Fprintf(&buf, " lbase_max_bytes=%d\n", o.LBaseMaxBytes) fmt.Fprintf(&buf, " max_concurrent_compactions=%d\n", o.MaxConcurrentCompactions) fmt.Fprintf(&buf, " max_manifest_file_size=%d\n", o.MaxManifestFileSize) @@ -736,14 +756,20 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error { o.Comparer, err = hooks.NewComparer(value) } } + case "delete_range_flush_delay": + o.Experimental.DeleteRangeFlushDelay, err = time.ParseDuration(value) case "disable_wal": o.DisableWAL, err = strconv.ParseBool(value) case "flush_split_bytes": o.Experimental.FlushSplitBytes, err = strconv.ParseInt(value, 10, 64) + case "l0_compaction_concurrency": + o.Experimental.L0CompactionConcurrency, err = strconv.Atoi(value) case "l0_compaction_threshold": o.L0CompactionThreshold, err = strconv.Atoi(value) case "l0_stop_writes_threshold": o.L0StopWritesThreshold, err = strconv.Atoi(value) + case "l0_sublevel_compactions": + o.Experimental.L0SublevelCompactions, err = strconv.ParseBool(value) case "lbase_max_bytes": o.LBaseMaxBytes, err = strconv.ParseInt(value, 10, 64) case "max_concurrent_compactions": @@ -889,6 +915,10 @@ func (o *Options) Validate() error { // is no need to check for zero values. var buf strings.Builder + if o.Experimental.L0CompactionConcurrency < 1 { + fmt.Fprintf(&buf, "L0CompactionConcurrency (%d) must be >= 1\n", + o.Experimental.L0CompactionConcurrency) + } if o.L0StopWritesThreshold < o.L0CompactionThreshold { fmt.Fprintf(&buf, "L0StopWritesThreshold (%d) must be >= L0CompactionThreshold (%d)\n", o.L0StopWritesThreshold, o.L0CompactionThreshold) diff --git a/github.com/cockroachdb/pebble/tool/lsm_data.go b/github.com/cockroachdb/pebble/tool/lsm_data.go index babe898a6d..5b7753f4e8 100644 --- a/github.com/cockroachdb/pebble/tool/lsm_data.go +++ b/github.com/cockroachdb/pebble/tool/lsm_data.go @@ -65,6 +65,7 @@ body { #container { height: 100vh; width: 100%; + overflow-y: scroll; } #header { @@ -248,6 +249,7 @@ let version = { levelHeights[0] = sublevelHeight; } levelOffsets = generateLevelOffsets(); + vis.style("height", levelOffsets[6] + 100); }, onCheckboxChange: function(value) { diff --git a/github.com/cockroachdb/pebble/vfs/syncing_file_linux.go b/github.com/cockroachdb/pebble/vfs/syncing_file_linux.go index b8e4bfbe57..df204d891e 100644 --- a/github.com/cockroachdb/pebble/vfs/syncing_file_linux.go +++ b/github.com/cockroachdb/pebble/vfs/syncing_file_linux.go @@ -25,9 +25,9 @@ func isSyncRangeSupported(fd uintptr) bool { return false } - // Whitelist which filesystems we allow using sync_file_range with as some - // filesystems treat that syscall as a noop (notably ZFS). A whitelist is - // used instead of a blacklist in order to have a more graceful failure mode + // Allowlist which filesystems we allow using sync_file_range with as some + // filesystems treat that syscall as a noop (notably ZFS). A allowlist is + // used instead of a denylist in order to have a more graceful failure mode // in case a filesystem we haven't tested is encountered. Currently only // ext2/3/4 are known to work properly. const extMagic = 0xef53 diff --git a/github.com/cockroachdb/pebble/vfs/vfs.go b/github.com/cockroachdb/pebble/vfs/vfs.go index a1c0b5697d..1f0a416c0d 100644 --- a/github.com/cockroachdb/pebble/vfs/vfs.go +++ b/github.com/cockroachdb/pebble/vfs/vfs.go @@ -251,7 +251,7 @@ func LinkOrCopy(fs FS, oldname, newname string) error { if err == nil { return nil } - // Whitelist a handful of errors which we know won't be fixed by copying the + // Permit a handful of errors which we know won't be fixed by copying the // file. Note that we don't check for the specifics of the error code as it // isn't easy to do so in a portable manner. On Unix we'd have to check for // LinkError.Err == syscall.EXDEV. On Windows we'd have to check for diff --git a/golang.org/x/sys/cpu/byteorder.go b/golang.org/x/sys/cpu/byteorder.go index ed8da8deac..dcbb14ef35 100644 --- a/golang.org/x/sys/cpu/byteorder.go +++ b/golang.org/x/sys/cpu/byteorder.go @@ -39,20 +39,25 @@ func (bigEndian) Uint64(b []byte) uint64 { uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 } -// hostByteOrder returns binary.LittleEndian on little-endian machines and -// binary.BigEndian on big-endian machines. +// hostByteOrder returns littleEndian on little-endian machines and +// bigEndian on big-endian machines. func hostByteOrder() byteOrder { switch runtime.GOARCH { case "386", "amd64", "amd64p32", + "alpha", "arm", "arm64", "mipsle", "mips64le", "mips64p32le", + "nios2", "ppc64le", - "riscv", "riscv64": + "riscv", "riscv64", + "sh": return littleEndian{} case "armbe", "arm64be", + "m68k", "mips", "mips64", "mips64p32", "ppc", "ppc64", "s390", "s390x", + "shbe", "sparc", "sparc64": return bigEndian{} } diff --git a/golang.org/x/sys/cpu/cpu_arm64.go b/golang.org/x/sys/cpu/cpu_arm64.go index 9c87677aef..7bcb36c7bb 100644 --- a/golang.org/x/sys/cpu/cpu_arm64.go +++ b/golang.org/x/sys/cpu/cpu_arm64.go @@ -10,8 +10,14 @@ const cacheLineSize = 64 func init() { switch runtime.GOOS { - case "android", "darwin": + case "android", "darwin", "netbsd": // Android and iOS don't seem to allow reading these registers. + // + // NetBSD: + // ID_AA64ISAR0_EL1 is a privileged register and cannot be read from EL0. + // It can be read via sysctl(3). Example for future implementers: + // https://nxr.netbsd.org/xref/src/usr.sbin/cpuctl/arch/aarch64.c + // // Fake the minimal features expected by // TestARM64minimalFeatures. ARM64.HasASIMD = true diff --git a/golang.org/x/sys/unix/syscall_linux.go b/golang.org/x/sys/unix/syscall_linux.go index 942a4bbf74..e50e4cb276 100644 --- a/golang.org/x/sys/unix/syscall_linux.go +++ b/golang.org/x/sys/unix/syscall_linux.go @@ -97,6 +97,12 @@ func IoctlSetRTCTime(fd int, value *RTCTime) error { return err } +func IoctlSetRTCWkAlrm(fd int, value *RTCWkAlrm) error { + err := ioctl(fd, RTC_WKALM_SET, uintptr(unsafe.Pointer(value))) + runtime.KeepAlive(value) + return err +} + func IoctlGetUint32(fd int, req uint) (uint32, error) { var value uint32 err := ioctl(fd, req, uintptr(unsafe.Pointer(&value))) @@ -109,6 +115,12 @@ func IoctlGetRTCTime(fd int) (*RTCTime, error) { return &value, err } +func IoctlGetRTCWkAlrm(fd int) (*RTCWkAlrm, error) { + var value RTCWkAlrm + err := ioctl(fd, RTC_WKALM_RD, uintptr(unsafe.Pointer(&value))) + return &value, err +} + //sys Linkat(olddirfd int, oldpath string, newdirfd int, newpath string, flags int) (err error) func Link(oldpath string, newpath string) (err error) { @@ -1938,6 +1950,20 @@ func Vmsplice(fd int, iovs []Iovec, flags int) (int, error) { return int(n), nil } +func isGroupMember(gid int) bool { + groups, err := Getgroups() + if err != nil { + return false + } + + for _, g := range groups { + if g == gid { + return true + } + } + return false +} + //sys faccessat(dirfd int, path string, mode uint32) (err error) func Faccessat(dirfd int, path string, mode uint32, flags int) (err error) { @@ -1995,7 +2021,7 @@ func Faccessat(dirfd int, path string, mode uint32, flags int) (err error) { gid = Getgid() } - if uint32(gid) == st.Gid { + if uint32(gid) == st.Gid || isGroupMember(gid) { fmode = (st.Mode >> 3) & 7 } else { fmode = st.Mode & 7 diff --git a/golang.org/x/sys/unix/zerrors_linux.go b/golang.org/x/sys/unix/zerrors_linux.go index 6e3cfec46c..f8bd50c11b 100644 --- a/golang.org/x/sys/unix/zerrors_linux.go +++ b/golang.org/x/sys/unix/zerrors_linux.go @@ -160,78 +160,28 @@ const ( BPF_A = 0x10 BPF_ABS = 0x20 BPF_ADD = 0x0 - BPF_ADJ_ROOM_ENCAP_L2_MASK = 0xff - BPF_ADJ_ROOM_ENCAP_L2_SHIFT = 0x38 BPF_ALU = 0x4 BPF_ALU64 = 0x7 BPF_AND = 0x50 - BPF_ANY = 0x0 BPF_ARSH = 0xc0 BPF_B = 0x10 BPF_BUILD_ID_SIZE = 0x14 BPF_CALL = 0x80 - BPF_DEVCG_ACC_MKNOD = 0x1 - BPF_DEVCG_ACC_READ = 0x2 - BPF_DEVCG_ACC_WRITE = 0x4 - BPF_DEVCG_DEV_BLOCK = 0x1 - BPF_DEVCG_DEV_CHAR = 0x2 BPF_DIV = 0x30 BPF_DW = 0x18 BPF_END = 0xd0 - BPF_EXIST = 0x2 BPF_EXIT = 0x90 - BPF_FLOW_DISSECTOR_F_PARSE_1ST_FRAG = 0x1 - BPF_FLOW_DISSECTOR_F_STOP_AT_ENCAP = 0x4 - BPF_FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL = 0x2 BPF_FROM_BE = 0x8 BPF_FROM_LE = 0x0 BPF_FS_MAGIC = 0xcafe4a11 - BPF_F_ADJ_ROOM_ENCAP_L3_IPV4 = 0x2 - BPF_F_ADJ_ROOM_ENCAP_L3_IPV6 = 0x4 - BPF_F_ADJ_ROOM_ENCAP_L4_GRE = 0x8 - BPF_F_ADJ_ROOM_ENCAP_L4_UDP = 0x10 - BPF_F_ADJ_ROOM_FIXED_GSO = 0x1 BPF_F_ALLOW_MULTI = 0x2 BPF_F_ALLOW_OVERRIDE = 0x1 BPF_F_ANY_ALIGNMENT = 0x2 - BPF_F_CLONE = 0x200 - BPF_F_CTXLEN_MASK = 0xfffff00000000 - BPF_F_CURRENT_CPU = 0xffffffff - BPF_F_CURRENT_NETNS = -0x1 - BPF_F_DONT_FRAGMENT = 0x4 - BPF_F_FAST_STACK_CMP = 0x200 - BPF_F_HDR_FIELD_MASK = 0xf - BPF_F_INDEX_MASK = 0xffffffff - BPF_F_INGRESS = 0x1 - BPF_F_INVALIDATE_HASH = 0x2 - BPF_F_LOCK = 0x4 - BPF_F_MARK_ENFORCE = 0x40 - BPF_F_MARK_MANGLED_0 = 0x20 - BPF_F_MMAPABLE = 0x400 - BPF_F_NO_COMMON_LRU = 0x2 - BPF_F_NO_PREALLOC = 0x1 - BPF_F_NUMA_NODE = 0x4 - BPF_F_PSEUDO_HDR = 0x10 BPF_F_QUERY_EFFECTIVE = 0x1 - BPF_F_RDONLY = 0x8 - BPF_F_RDONLY_PROG = 0x80 - BPF_F_RECOMPUTE_CSUM = 0x1 BPF_F_REPLACE = 0x4 - BPF_F_REUSE_STACKID = 0x400 - BPF_F_SEQ_NUMBER = 0x8 - BPF_F_SKIP_FIELD_MASK = 0xff - BPF_F_STACK_BUILD_ID = 0x20 BPF_F_STRICT_ALIGNMENT = 0x1 - BPF_F_SYSCTL_BASE_NAME = 0x1 BPF_F_TEST_RND_HI32 = 0x4 BPF_F_TEST_STATE_FREQ = 0x8 - BPF_F_TUNINFO_IPV6 = 0x1 - BPF_F_USER_BUILD_ID = 0x800 - BPF_F_USER_STACK = 0x100 - BPF_F_WRONLY = 0x10 - BPF_F_WRONLY_PROG = 0x100 - BPF_F_ZERO_CSUM_TX = 0x2 - BPF_F_ZERO_SEED = 0x40 BPF_H = 0x8 BPF_IMM = 0x0 BPF_IND = 0x40 @@ -267,7 +217,6 @@ const ( BPF_MUL = 0x20 BPF_NEG = 0x80 BPF_NET_OFF = -0x100000 - BPF_NOEXIST = 0x1 BPF_OBJ_NAME_LEN = 0x10 BPF_OR = 0x40 BPF_PSEUDO_CALL = 0x1 @@ -275,12 +224,6 @@ const ( BPF_PSEUDO_MAP_VALUE = 0x2 BPF_RET = 0x6 BPF_RSH = 0x70 - BPF_SK_STORAGE_GET_F_CREATE = 0x1 - BPF_SOCK_OPS_ALL_CB_FLAGS = 0xf - BPF_SOCK_OPS_RETRANS_CB_FLAG = 0x2 - BPF_SOCK_OPS_RTO_CB_FLAG = 0x1 - BPF_SOCK_OPS_RTT_CB_FLAG = 0x8 - BPF_SOCK_OPS_STATE_CB_FLAG = 0x4 BPF_ST = 0x2 BPF_STX = 0x3 BPF_SUB = 0x10 @@ -378,12 +321,14 @@ const ( CLOCK_TXINT = 0x3 CLONE_ARGS_SIZE_VER0 = 0x40 CLONE_ARGS_SIZE_VER1 = 0x50 + CLONE_ARGS_SIZE_VER2 = 0x58 CLONE_CHILD_CLEARTID = 0x200000 CLONE_CHILD_SETTID = 0x1000000 CLONE_CLEAR_SIGHAND = 0x100000000 CLONE_DETACHED = 0x400000 CLONE_FILES = 0x400 CLONE_FS = 0x200 + CLONE_INTO_CGROUP = 0x200000000 CLONE_IO = 0x80000000 CLONE_NEWCGROUP = 0x2000000 CLONE_NEWIPC = 0x8000000 @@ -598,7 +543,9 @@ const ( FAN_DELETE = 0x200 FAN_DELETE_SELF = 0x400 FAN_DENY = 0x2 + FAN_DIR_MODIFY = 0x80000 FAN_ENABLE_AUDIT = 0x40 + FAN_EVENT_INFO_TYPE_DFID_NAME = 0x2 FAN_EVENT_INFO_TYPE_FID = 0x1 FAN_EVENT_METADATA_LEN = 0x18 FAN_EVENT_ON_CHILD = 0x8000000 @@ -2108,8 +2055,6 @@ const ( TCOFLUSH = 0x1 TCOOFF = 0x0 TCOON = 0x1 - TCP_BPF_IW = 0x3e9 - TCP_BPF_SNDCWND_CLAMP = 0x3ea TCP_CC_INFO = 0x1a TCP_CM_INQ = 0x24 TCP_CONGESTION = 0xd @@ -2384,8 +2329,9 @@ const ( XDP_COPY = 0x2 XDP_FLAGS_DRV_MODE = 0x4 XDP_FLAGS_HW_MODE = 0x8 - XDP_FLAGS_MASK = 0xf + XDP_FLAGS_MASK = 0x1f XDP_FLAGS_MODES = 0xe + XDP_FLAGS_REPLACE = 0x10 XDP_FLAGS_SKB_MODE = 0x2 XDP_FLAGS_UPDATE_IF_NOEXIST = 0x1 XDP_MMAP_OFFSETS = 0x1 diff --git a/golang.org/x/sys/unix/zerrors_linux_386.go b/golang.org/x/sys/unix/zerrors_linux_386.go index 5e974110d9..8d207b041e 100644 --- a/golang.org/x/sys/unix/zerrors_linux_386.go +++ b/golang.org/x/sys/unix/zerrors_linux_386.go @@ -75,6 +75,7 @@ const ( FP_XSTATE_MAGIC2 = 0x46505845 FS_IOC_ENABLE_VERITY = 0x40806685 FS_IOC_GETFLAGS = 0x80046601 + FS_IOC_GET_ENCRYPTION_NONCE = 0x8010661b FS_IOC_GET_ENCRYPTION_POLICY = 0x400c6615 FS_IOC_GET_ENCRYPTION_PWSALT = 0x40106614 FS_IOC_SET_ENCRYPTION_POLICY = 0x800c6613 diff --git a/golang.org/x/sys/unix/zerrors_linux_amd64.go b/golang.org/x/sys/unix/zerrors_linux_amd64.go index 47a57fe468..c4bf9cb80f 100644 --- a/golang.org/x/sys/unix/zerrors_linux_amd64.go +++ b/golang.org/x/sys/unix/zerrors_linux_amd64.go @@ -75,6 +75,7 @@ const ( FP_XSTATE_MAGIC2 = 0x46505845 FS_IOC_ENABLE_VERITY = 0x40806685 FS_IOC_GETFLAGS = 0x80086601 + FS_IOC_GET_ENCRYPTION_NONCE = 0x8010661b FS_IOC_GET_ENCRYPTION_POLICY = 0x400c6615 FS_IOC_GET_ENCRYPTION_PWSALT = 0x40106614 FS_IOC_SET_ENCRYPTION_POLICY = 0x800c6613 diff --git a/golang.org/x/sys/unix/zerrors_linux_arm.go b/golang.org/x/sys/unix/zerrors_linux_arm.go index df2eea4bb7..0cab0522e6 100644 --- a/golang.org/x/sys/unix/zerrors_linux_arm.go +++ b/golang.org/x/sys/unix/zerrors_linux_arm.go @@ -74,6 +74,7 @@ const ( FLUSHO = 0x1000 FS_IOC_ENABLE_VERITY = 0x40806685 FS_IOC_GETFLAGS = 0x80046601 + FS_IOC_GET_ENCRYPTION_NONCE = 0x8010661b FS_IOC_GET_ENCRYPTION_POLICY = 0x400c6615 FS_IOC_GET_ENCRYPTION_PWSALT = 0x40106614 FS_IOC_SET_ENCRYPTION_POLICY = 0x800c6613 diff --git a/golang.org/x/sys/unix/zerrors_linux_arm64.go b/golang.org/x/sys/unix/zerrors_linux_arm64.go index 4e1214217f..370d0a7f59 100644 --- a/golang.org/x/sys/unix/zerrors_linux_arm64.go +++ b/golang.org/x/sys/unix/zerrors_linux_arm64.go @@ -77,6 +77,7 @@ const ( FPSIMD_MAGIC = 0x46508001 FS_IOC_ENABLE_VERITY = 0x40806685 FS_IOC_GETFLAGS = 0x80086601 + FS_IOC_GET_ENCRYPTION_NONCE = 0x8010661b FS_IOC_GET_ENCRYPTION_POLICY = 0x400c6615 FS_IOC_GET_ENCRYPTION_PWSALT = 0x40106614 FS_IOC_SET_ENCRYPTION_POLICY = 0x800c6613 diff --git a/golang.org/x/sys/unix/zerrors_linux_mips.go b/golang.org/x/sys/unix/zerrors_linux_mips.go index a23b08029a..fbf2f3174e 100644 --- a/golang.org/x/sys/unix/zerrors_linux_mips.go +++ b/golang.org/x/sys/unix/zerrors_linux_mips.go @@ -74,6 +74,7 @@ const ( FLUSHO = 0x2000 FS_IOC_ENABLE_VERITY = 0x80806685 FS_IOC_GETFLAGS = 0x40046601 + FS_IOC_GET_ENCRYPTION_NONCE = 0x4010661b FS_IOC_GET_ENCRYPTION_POLICY = 0x800c6615 FS_IOC_GET_ENCRYPTION_PWSALT = 0x80106614 FS_IOC_SET_ENCRYPTION_POLICY = 0x400c6613 diff --git a/golang.org/x/sys/unix/zerrors_linux_mips64.go b/golang.org/x/sys/unix/zerrors_linux_mips64.go index a5a921e43b..25e74b30a9 100644 --- a/golang.org/x/sys/unix/zerrors_linux_mips64.go +++ b/golang.org/x/sys/unix/zerrors_linux_mips64.go @@ -74,6 +74,7 @@ const ( FLUSHO = 0x2000 FS_IOC_ENABLE_VERITY = 0x80806685 FS_IOC_GETFLAGS = 0x40086601 + FS_IOC_GET_ENCRYPTION_NONCE = 0x4010661b FS_IOC_GET_ENCRYPTION_POLICY = 0x800c6615 FS_IOC_GET_ENCRYPTION_PWSALT = 0x80106614 FS_IOC_SET_ENCRYPTION_POLICY = 0x400c6613 diff --git a/golang.org/x/sys/unix/zerrors_linux_mips64le.go b/golang.org/x/sys/unix/zerrors_linux_mips64le.go index d088e197bd..4ecc0bca34 100644 --- a/golang.org/x/sys/unix/zerrors_linux_mips64le.go +++ b/golang.org/x/sys/unix/zerrors_linux_mips64le.go @@ -74,6 +74,7 @@ const ( FLUSHO = 0x2000 FS_IOC_ENABLE_VERITY = 0x80806685 FS_IOC_GETFLAGS = 0x40086601 + FS_IOC_GET_ENCRYPTION_NONCE = 0x4010661b FS_IOC_GET_ENCRYPTION_POLICY = 0x800c6615 FS_IOC_GET_ENCRYPTION_PWSALT = 0x80106614 FS_IOC_SET_ENCRYPTION_POLICY = 0x400c6613 diff --git a/golang.org/x/sys/unix/zerrors_linux_mipsle.go b/golang.org/x/sys/unix/zerrors_linux_mipsle.go index 0ddf9d5fe8..dfb8f88a7e 100644 --- a/golang.org/x/sys/unix/zerrors_linux_mipsle.go +++ b/golang.org/x/sys/unix/zerrors_linux_mipsle.go @@ -74,6 +74,7 @@ const ( FLUSHO = 0x2000 FS_IOC_ENABLE_VERITY = 0x80806685 FS_IOC_GETFLAGS = 0x40046601 + FS_IOC_GET_ENCRYPTION_NONCE = 0x4010661b FS_IOC_GET_ENCRYPTION_POLICY = 0x800c6615 FS_IOC_GET_ENCRYPTION_PWSALT = 0x80106614 FS_IOC_SET_ENCRYPTION_POLICY = 0x400c6613 diff --git a/golang.org/x/sys/unix/zerrors_linux_ppc64.go b/golang.org/x/sys/unix/zerrors_linux_ppc64.go index a93ffc1807..72d8dad5b8 100644 --- a/golang.org/x/sys/unix/zerrors_linux_ppc64.go +++ b/golang.org/x/sys/unix/zerrors_linux_ppc64.go @@ -74,6 +74,7 @@ const ( FLUSHO = 0x800000 FS_IOC_ENABLE_VERITY = 0x80806685 FS_IOC_GETFLAGS = 0x40086601 + FS_IOC_GET_ENCRYPTION_NONCE = 0x4010661b FS_IOC_GET_ENCRYPTION_POLICY = 0x800c6615 FS_IOC_GET_ENCRYPTION_PWSALT = 0x80106614 FS_IOC_SET_ENCRYPTION_POLICY = 0x400c6613 diff --git a/golang.org/x/sys/unix/zerrors_linux_ppc64le.go b/golang.org/x/sys/unix/zerrors_linux_ppc64le.go index c1ea48b95f..ca0e7b5262 100644 --- a/golang.org/x/sys/unix/zerrors_linux_ppc64le.go +++ b/golang.org/x/sys/unix/zerrors_linux_ppc64le.go @@ -74,6 +74,7 @@ const ( FLUSHO = 0x800000 FS_IOC_ENABLE_VERITY = 0x80806685 FS_IOC_GETFLAGS = 0x40086601 + FS_IOC_GET_ENCRYPTION_NONCE = 0x4010661b FS_IOC_GET_ENCRYPTION_POLICY = 0x800c6615 FS_IOC_GET_ENCRYPTION_PWSALT = 0x80106614 FS_IOC_SET_ENCRYPTION_POLICY = 0x400c6613 diff --git a/golang.org/x/sys/unix/zerrors_linux_riscv64.go b/golang.org/x/sys/unix/zerrors_linux_riscv64.go index 7def950ba5..147511a974 100644 --- a/golang.org/x/sys/unix/zerrors_linux_riscv64.go +++ b/golang.org/x/sys/unix/zerrors_linux_riscv64.go @@ -74,6 +74,7 @@ const ( FLUSHO = 0x1000 FS_IOC_ENABLE_VERITY = 0x40806685 FS_IOC_GETFLAGS = 0x80086601 + FS_IOC_GET_ENCRYPTION_NONCE = 0x8010661b FS_IOC_GET_ENCRYPTION_POLICY = 0x400c6615 FS_IOC_GET_ENCRYPTION_PWSALT = 0x40106614 FS_IOC_SET_ENCRYPTION_POLICY = 0x800c6613 diff --git a/golang.org/x/sys/unix/zerrors_linux_s390x.go b/golang.org/x/sys/unix/zerrors_linux_s390x.go index d39293c871..517349dafa 100644 --- a/golang.org/x/sys/unix/zerrors_linux_s390x.go +++ b/golang.org/x/sys/unix/zerrors_linux_s390x.go @@ -74,6 +74,7 @@ const ( FLUSHO = 0x1000 FS_IOC_ENABLE_VERITY = 0x40806685 FS_IOC_GETFLAGS = 0x80086601 + FS_IOC_GET_ENCRYPTION_NONCE = 0x8010661b FS_IOC_GET_ENCRYPTION_POLICY = 0x400c6615 FS_IOC_GET_ENCRYPTION_PWSALT = 0x40106614 FS_IOC_SET_ENCRYPTION_POLICY = 0x800c6613 diff --git a/golang.org/x/sys/unix/zerrors_linux_sparc64.go b/golang.org/x/sys/unix/zerrors_linux_sparc64.go index 3ff3ec681b..094822465b 100644 --- a/golang.org/x/sys/unix/zerrors_linux_sparc64.go +++ b/golang.org/x/sys/unix/zerrors_linux_sparc64.go @@ -78,6 +78,7 @@ const ( FLUSHO = 0x1000 FS_IOC_ENABLE_VERITY = 0x80806685 FS_IOC_GETFLAGS = 0x40086601 + FS_IOC_GET_ENCRYPTION_NONCE = 0x4010661b FS_IOC_GET_ENCRYPTION_POLICY = 0x800c6615 FS_IOC_GET_ENCRYPTION_PWSALT = 0x80106614 FS_IOC_SET_ENCRYPTION_POLICY = 0x400c6613 diff --git a/golang.org/x/sys/unix/ztypes_freebsd_arm.go b/golang.org/x/sys/unix/ztypes_freebsd_arm.go index 6f79227d74..b91c2ae0f0 100644 --- a/golang.org/x/sys/unix/ztypes_freebsd_arm.go +++ b/golang.org/x/sys/unix/ztypes_freebsd_arm.go @@ -125,9 +125,9 @@ type Statfs_t struct { Owner uint32 Fsid Fsid Charspare [80]int8 - Fstypename [16]int8 - Mntfromname [1024]int8 - Mntonname [1024]int8 + Fstypename [16]byte + Mntfromname [1024]byte + Mntonname [1024]byte } type statfs_freebsd11_t struct { @@ -150,9 +150,9 @@ type statfs_freebsd11_t struct { Owner uint32 Fsid Fsid Charspare [80]int8 - Fstypename [16]int8 - Mntfromname [88]int8 - Mntonname [88]int8 + Fstypename [16]byte + Mntfromname [88]byte + Mntonname [88]byte } type Flock_t struct { diff --git a/golang.org/x/sys/unix/ztypes_linux.go b/golang.org/x/sys/unix/ztypes_linux.go index 416f7767e7..27d67ac8f5 100644 --- a/golang.org/x/sys/unix/ztypes_linux.go +++ b/golang.org/x/sys/unix/ztypes_linux.go @@ -1871,175 +1871,249 @@ const ( ) const ( - BPF_REG_0 = 0x0 - BPF_REG_1 = 0x1 - BPF_REG_2 = 0x2 - BPF_REG_3 = 0x3 - BPF_REG_4 = 0x4 - BPF_REG_5 = 0x5 - BPF_REG_6 = 0x6 - BPF_REG_7 = 0x7 - BPF_REG_8 = 0x8 - BPF_REG_9 = 0x9 - BPF_REG_10 = 0xa - BPF_MAP_CREATE = 0x0 - BPF_MAP_LOOKUP_ELEM = 0x1 - BPF_MAP_UPDATE_ELEM = 0x2 - BPF_MAP_DELETE_ELEM = 0x3 - BPF_MAP_GET_NEXT_KEY = 0x4 - BPF_PROG_LOAD = 0x5 - BPF_OBJ_PIN = 0x6 - BPF_OBJ_GET = 0x7 - BPF_PROG_ATTACH = 0x8 - BPF_PROG_DETACH = 0x9 - BPF_PROG_TEST_RUN = 0xa - BPF_PROG_GET_NEXT_ID = 0xb - BPF_MAP_GET_NEXT_ID = 0xc - BPF_PROG_GET_FD_BY_ID = 0xd - BPF_MAP_GET_FD_BY_ID = 0xe - BPF_OBJ_GET_INFO_BY_FD = 0xf - BPF_PROG_QUERY = 0x10 - BPF_RAW_TRACEPOINT_OPEN = 0x11 - BPF_BTF_LOAD = 0x12 - BPF_BTF_GET_FD_BY_ID = 0x13 - BPF_TASK_FD_QUERY = 0x14 - BPF_MAP_LOOKUP_AND_DELETE_ELEM = 0x15 - BPF_MAP_FREEZE = 0x16 - BPF_BTF_GET_NEXT_ID = 0x17 - BPF_MAP_TYPE_UNSPEC = 0x0 - BPF_MAP_TYPE_HASH = 0x1 - BPF_MAP_TYPE_ARRAY = 0x2 - BPF_MAP_TYPE_PROG_ARRAY = 0x3 - BPF_MAP_TYPE_PERF_EVENT_ARRAY = 0x4 - BPF_MAP_TYPE_PERCPU_HASH = 0x5 - BPF_MAP_TYPE_PERCPU_ARRAY = 0x6 - BPF_MAP_TYPE_STACK_TRACE = 0x7 - BPF_MAP_TYPE_CGROUP_ARRAY = 0x8 - BPF_MAP_TYPE_LRU_HASH = 0x9 - BPF_MAP_TYPE_LRU_PERCPU_HASH = 0xa - BPF_MAP_TYPE_LPM_TRIE = 0xb - BPF_MAP_TYPE_ARRAY_OF_MAPS = 0xc - BPF_MAP_TYPE_HASH_OF_MAPS = 0xd - BPF_MAP_TYPE_DEVMAP = 0xe - BPF_MAP_TYPE_SOCKMAP = 0xf - BPF_MAP_TYPE_CPUMAP = 0x10 - BPF_MAP_TYPE_XSKMAP = 0x11 - BPF_MAP_TYPE_SOCKHASH = 0x12 - BPF_MAP_TYPE_CGROUP_STORAGE = 0x13 - BPF_MAP_TYPE_REUSEPORT_SOCKARRAY = 0x14 - BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE = 0x15 - BPF_MAP_TYPE_QUEUE = 0x16 - BPF_MAP_TYPE_STACK = 0x17 - BPF_MAP_TYPE_SK_STORAGE = 0x18 - BPF_MAP_TYPE_DEVMAP_HASH = 0x19 - BPF_PROG_TYPE_UNSPEC = 0x0 - BPF_PROG_TYPE_SOCKET_FILTER = 0x1 - BPF_PROG_TYPE_KPROBE = 0x2 - BPF_PROG_TYPE_SCHED_CLS = 0x3 - BPF_PROG_TYPE_SCHED_ACT = 0x4 - BPF_PROG_TYPE_TRACEPOINT = 0x5 - BPF_PROG_TYPE_XDP = 0x6 - BPF_PROG_TYPE_PERF_EVENT = 0x7 - BPF_PROG_TYPE_CGROUP_SKB = 0x8 - BPF_PROG_TYPE_CGROUP_SOCK = 0x9 - BPF_PROG_TYPE_LWT_IN = 0xa - BPF_PROG_TYPE_LWT_OUT = 0xb - BPF_PROG_TYPE_LWT_XMIT = 0xc - BPF_PROG_TYPE_SOCK_OPS = 0xd - BPF_PROG_TYPE_SK_SKB = 0xe - BPF_PROG_TYPE_CGROUP_DEVICE = 0xf - BPF_PROG_TYPE_SK_MSG = 0x10 - BPF_PROG_TYPE_RAW_TRACEPOINT = 0x11 - BPF_PROG_TYPE_CGROUP_SOCK_ADDR = 0x12 - BPF_PROG_TYPE_LWT_SEG6LOCAL = 0x13 - BPF_PROG_TYPE_LIRC_MODE2 = 0x14 - BPF_PROG_TYPE_SK_REUSEPORT = 0x15 - BPF_PROG_TYPE_FLOW_DISSECTOR = 0x16 - BPF_PROG_TYPE_CGROUP_SYSCTL = 0x17 - BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE = 0x18 - BPF_PROG_TYPE_CGROUP_SOCKOPT = 0x19 - BPF_PROG_TYPE_TRACING = 0x1a - BPF_CGROUP_INET_INGRESS = 0x0 - BPF_CGROUP_INET_EGRESS = 0x1 - BPF_CGROUP_INET_SOCK_CREATE = 0x2 - BPF_CGROUP_SOCK_OPS = 0x3 - BPF_SK_SKB_STREAM_PARSER = 0x4 - BPF_SK_SKB_STREAM_VERDICT = 0x5 - BPF_CGROUP_DEVICE = 0x6 - BPF_SK_MSG_VERDICT = 0x7 - BPF_CGROUP_INET4_BIND = 0x8 - BPF_CGROUP_INET6_BIND = 0x9 - BPF_CGROUP_INET4_CONNECT = 0xa - BPF_CGROUP_INET6_CONNECT = 0xb - BPF_CGROUP_INET4_POST_BIND = 0xc - BPF_CGROUP_INET6_POST_BIND = 0xd - BPF_CGROUP_UDP4_SENDMSG = 0xe - BPF_CGROUP_UDP6_SENDMSG = 0xf - BPF_LIRC_MODE2 = 0x10 - BPF_FLOW_DISSECTOR = 0x11 - BPF_CGROUP_SYSCTL = 0x12 - BPF_CGROUP_UDP4_RECVMSG = 0x13 - BPF_CGROUP_UDP6_RECVMSG = 0x14 - BPF_CGROUP_GETSOCKOPT = 0x15 - BPF_CGROUP_SETSOCKOPT = 0x16 - BPF_TRACE_RAW_TP = 0x17 - BPF_TRACE_FENTRY = 0x18 - BPF_TRACE_FEXIT = 0x19 - BPF_STACK_BUILD_ID_EMPTY = 0x0 - BPF_STACK_BUILD_ID_VALID = 0x1 - BPF_STACK_BUILD_ID_IP = 0x2 - BPF_ADJ_ROOM_NET = 0x0 - BPF_ADJ_ROOM_MAC = 0x1 - BPF_HDR_START_MAC = 0x0 - BPF_HDR_START_NET = 0x1 - BPF_LWT_ENCAP_SEG6 = 0x0 - BPF_LWT_ENCAP_SEG6_INLINE = 0x1 - BPF_LWT_ENCAP_IP = 0x2 - BPF_OK = 0x0 - BPF_DROP = 0x2 - BPF_REDIRECT = 0x7 - BPF_LWT_REROUTE = 0x80 - BPF_SOCK_OPS_VOID = 0x0 - BPF_SOCK_OPS_TIMEOUT_INIT = 0x1 - BPF_SOCK_OPS_RWND_INIT = 0x2 - BPF_SOCK_OPS_TCP_CONNECT_CB = 0x3 - BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB = 0x4 - BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB = 0x5 - BPF_SOCK_OPS_NEEDS_ECN = 0x6 - BPF_SOCK_OPS_BASE_RTT = 0x7 - BPF_SOCK_OPS_RTO_CB = 0x8 - BPF_SOCK_OPS_RETRANS_CB = 0x9 - BPF_SOCK_OPS_STATE_CB = 0xa - BPF_SOCK_OPS_TCP_LISTEN_CB = 0xb - BPF_SOCK_OPS_RTT_CB = 0xc - BPF_TCP_ESTABLISHED = 0x1 - BPF_TCP_SYN_SENT = 0x2 - BPF_TCP_SYN_RECV = 0x3 - BPF_TCP_FIN_WAIT1 = 0x4 - BPF_TCP_FIN_WAIT2 = 0x5 - BPF_TCP_TIME_WAIT = 0x6 - BPF_TCP_CLOSE = 0x7 - BPF_TCP_CLOSE_WAIT = 0x8 - BPF_TCP_LAST_ACK = 0x9 - BPF_TCP_LISTEN = 0xa - BPF_TCP_CLOSING = 0xb - BPF_TCP_NEW_SYN_RECV = 0xc - BPF_TCP_MAX_STATES = 0xd - BPF_FIB_LKUP_RET_SUCCESS = 0x0 - BPF_FIB_LKUP_RET_BLACKHOLE = 0x1 - BPF_FIB_LKUP_RET_UNREACHABLE = 0x2 - BPF_FIB_LKUP_RET_PROHIBIT = 0x3 - BPF_FIB_LKUP_RET_NOT_FWDED = 0x4 - BPF_FIB_LKUP_RET_FWD_DISABLED = 0x5 - BPF_FIB_LKUP_RET_UNSUPP_LWT = 0x6 - BPF_FIB_LKUP_RET_NO_NEIGH = 0x7 - BPF_FIB_LKUP_RET_FRAG_NEEDED = 0x8 - BPF_FD_TYPE_RAW_TRACEPOINT = 0x0 - BPF_FD_TYPE_TRACEPOINT = 0x1 - BPF_FD_TYPE_KPROBE = 0x2 - BPF_FD_TYPE_KRETPROBE = 0x3 - BPF_FD_TYPE_UPROBE = 0x4 - BPF_FD_TYPE_URETPROBE = 0x5 + BPF_REG_0 = 0x0 + BPF_REG_1 = 0x1 + BPF_REG_2 = 0x2 + BPF_REG_3 = 0x3 + BPF_REG_4 = 0x4 + BPF_REG_5 = 0x5 + BPF_REG_6 = 0x6 + BPF_REG_7 = 0x7 + BPF_REG_8 = 0x8 + BPF_REG_9 = 0x9 + BPF_REG_10 = 0xa + BPF_MAP_CREATE = 0x0 + BPF_MAP_LOOKUP_ELEM = 0x1 + BPF_MAP_UPDATE_ELEM = 0x2 + BPF_MAP_DELETE_ELEM = 0x3 + BPF_MAP_GET_NEXT_KEY = 0x4 + BPF_PROG_LOAD = 0x5 + BPF_OBJ_PIN = 0x6 + BPF_OBJ_GET = 0x7 + BPF_PROG_ATTACH = 0x8 + BPF_PROG_DETACH = 0x9 + BPF_PROG_TEST_RUN = 0xa + BPF_PROG_GET_NEXT_ID = 0xb + BPF_MAP_GET_NEXT_ID = 0xc + BPF_PROG_GET_FD_BY_ID = 0xd + BPF_MAP_GET_FD_BY_ID = 0xe + BPF_OBJ_GET_INFO_BY_FD = 0xf + BPF_PROG_QUERY = 0x10 + BPF_RAW_TRACEPOINT_OPEN = 0x11 + BPF_BTF_LOAD = 0x12 + BPF_BTF_GET_FD_BY_ID = 0x13 + BPF_TASK_FD_QUERY = 0x14 + BPF_MAP_LOOKUP_AND_DELETE_ELEM = 0x15 + BPF_MAP_FREEZE = 0x16 + BPF_BTF_GET_NEXT_ID = 0x17 + BPF_MAP_LOOKUP_BATCH = 0x18 + BPF_MAP_LOOKUP_AND_DELETE_BATCH = 0x19 + BPF_MAP_UPDATE_BATCH = 0x1a + BPF_MAP_DELETE_BATCH = 0x1b + BPF_LINK_CREATE = 0x1c + BPF_LINK_UPDATE = 0x1d + BPF_MAP_TYPE_UNSPEC = 0x0 + BPF_MAP_TYPE_HASH = 0x1 + BPF_MAP_TYPE_ARRAY = 0x2 + BPF_MAP_TYPE_PROG_ARRAY = 0x3 + BPF_MAP_TYPE_PERF_EVENT_ARRAY = 0x4 + BPF_MAP_TYPE_PERCPU_HASH = 0x5 + BPF_MAP_TYPE_PERCPU_ARRAY = 0x6 + BPF_MAP_TYPE_STACK_TRACE = 0x7 + BPF_MAP_TYPE_CGROUP_ARRAY = 0x8 + BPF_MAP_TYPE_LRU_HASH = 0x9 + BPF_MAP_TYPE_LRU_PERCPU_HASH = 0xa + BPF_MAP_TYPE_LPM_TRIE = 0xb + BPF_MAP_TYPE_ARRAY_OF_MAPS = 0xc + BPF_MAP_TYPE_HASH_OF_MAPS = 0xd + BPF_MAP_TYPE_DEVMAP = 0xe + BPF_MAP_TYPE_SOCKMAP = 0xf + BPF_MAP_TYPE_CPUMAP = 0x10 + BPF_MAP_TYPE_XSKMAP = 0x11 + BPF_MAP_TYPE_SOCKHASH = 0x12 + BPF_MAP_TYPE_CGROUP_STORAGE = 0x13 + BPF_MAP_TYPE_REUSEPORT_SOCKARRAY = 0x14 + BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE = 0x15 + BPF_MAP_TYPE_QUEUE = 0x16 + BPF_MAP_TYPE_STACK = 0x17 + BPF_MAP_TYPE_SK_STORAGE = 0x18 + BPF_MAP_TYPE_DEVMAP_HASH = 0x19 + BPF_MAP_TYPE_STRUCT_OPS = 0x1a + BPF_PROG_TYPE_UNSPEC = 0x0 + BPF_PROG_TYPE_SOCKET_FILTER = 0x1 + BPF_PROG_TYPE_KPROBE = 0x2 + BPF_PROG_TYPE_SCHED_CLS = 0x3 + BPF_PROG_TYPE_SCHED_ACT = 0x4 + BPF_PROG_TYPE_TRACEPOINT = 0x5 + BPF_PROG_TYPE_XDP = 0x6 + BPF_PROG_TYPE_PERF_EVENT = 0x7 + BPF_PROG_TYPE_CGROUP_SKB = 0x8 + BPF_PROG_TYPE_CGROUP_SOCK = 0x9 + BPF_PROG_TYPE_LWT_IN = 0xa + BPF_PROG_TYPE_LWT_OUT = 0xb + BPF_PROG_TYPE_LWT_XMIT = 0xc + BPF_PROG_TYPE_SOCK_OPS = 0xd + BPF_PROG_TYPE_SK_SKB = 0xe + BPF_PROG_TYPE_CGROUP_DEVICE = 0xf + BPF_PROG_TYPE_SK_MSG = 0x10 + BPF_PROG_TYPE_RAW_TRACEPOINT = 0x11 + BPF_PROG_TYPE_CGROUP_SOCK_ADDR = 0x12 + BPF_PROG_TYPE_LWT_SEG6LOCAL = 0x13 + BPF_PROG_TYPE_LIRC_MODE2 = 0x14 + BPF_PROG_TYPE_SK_REUSEPORT = 0x15 + BPF_PROG_TYPE_FLOW_DISSECTOR = 0x16 + BPF_PROG_TYPE_CGROUP_SYSCTL = 0x17 + BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE = 0x18 + BPF_PROG_TYPE_CGROUP_SOCKOPT = 0x19 + BPF_PROG_TYPE_TRACING = 0x1a + BPF_PROG_TYPE_STRUCT_OPS = 0x1b + BPF_PROG_TYPE_EXT = 0x1c + BPF_PROG_TYPE_LSM = 0x1d + BPF_CGROUP_INET_INGRESS = 0x0 + BPF_CGROUP_INET_EGRESS = 0x1 + BPF_CGROUP_INET_SOCK_CREATE = 0x2 + BPF_CGROUP_SOCK_OPS = 0x3 + BPF_SK_SKB_STREAM_PARSER = 0x4 + BPF_SK_SKB_STREAM_VERDICT = 0x5 + BPF_CGROUP_DEVICE = 0x6 + BPF_SK_MSG_VERDICT = 0x7 + BPF_CGROUP_INET4_BIND = 0x8 + BPF_CGROUP_INET6_BIND = 0x9 + BPF_CGROUP_INET4_CONNECT = 0xa + BPF_CGROUP_INET6_CONNECT = 0xb + BPF_CGROUP_INET4_POST_BIND = 0xc + BPF_CGROUP_INET6_POST_BIND = 0xd + BPF_CGROUP_UDP4_SENDMSG = 0xe + BPF_CGROUP_UDP6_SENDMSG = 0xf + BPF_LIRC_MODE2 = 0x10 + BPF_FLOW_DISSECTOR = 0x11 + BPF_CGROUP_SYSCTL = 0x12 + BPF_CGROUP_UDP4_RECVMSG = 0x13 + BPF_CGROUP_UDP6_RECVMSG = 0x14 + BPF_CGROUP_GETSOCKOPT = 0x15 + BPF_CGROUP_SETSOCKOPT = 0x16 + BPF_TRACE_RAW_TP = 0x17 + BPF_TRACE_FENTRY = 0x18 + BPF_TRACE_FEXIT = 0x19 + BPF_MODIFY_RETURN = 0x1a + BPF_LSM_MAC = 0x1b + BPF_ANY = 0x0 + BPF_NOEXIST = 0x1 + BPF_EXIST = 0x2 + BPF_F_LOCK = 0x4 + BPF_F_NO_PREALLOC = 0x1 + BPF_F_NO_COMMON_LRU = 0x2 + BPF_F_NUMA_NODE = 0x4 + BPF_F_RDONLY = 0x8 + BPF_F_WRONLY = 0x10 + BPF_F_STACK_BUILD_ID = 0x20 + BPF_F_ZERO_SEED = 0x40 + BPF_F_RDONLY_PROG = 0x80 + BPF_F_WRONLY_PROG = 0x100 + BPF_F_CLONE = 0x200 + BPF_F_MMAPABLE = 0x400 + BPF_STACK_BUILD_ID_EMPTY = 0x0 + BPF_STACK_BUILD_ID_VALID = 0x1 + BPF_STACK_BUILD_ID_IP = 0x2 + BPF_F_RECOMPUTE_CSUM = 0x1 + BPF_F_INVALIDATE_HASH = 0x2 + BPF_F_HDR_FIELD_MASK = 0xf + BPF_F_PSEUDO_HDR = 0x10 + BPF_F_MARK_MANGLED_0 = 0x20 + BPF_F_MARK_ENFORCE = 0x40 + BPF_F_INGRESS = 0x1 + BPF_F_TUNINFO_IPV6 = 0x1 + BPF_F_SKIP_FIELD_MASK = 0xff + BPF_F_USER_STACK = 0x100 + BPF_F_FAST_STACK_CMP = 0x200 + BPF_F_REUSE_STACKID = 0x400 + BPF_F_USER_BUILD_ID = 0x800 + BPF_F_ZERO_CSUM_TX = 0x2 + BPF_F_DONT_FRAGMENT = 0x4 + BPF_F_SEQ_NUMBER = 0x8 + BPF_F_INDEX_MASK = 0xffffffff + BPF_F_CURRENT_CPU = 0xffffffff + BPF_F_CTXLEN_MASK = 0xfffff00000000 + BPF_F_CURRENT_NETNS = -0x1 + BPF_F_ADJ_ROOM_FIXED_GSO = 0x1 + BPF_F_ADJ_ROOM_ENCAP_L3_IPV4 = 0x2 + BPF_F_ADJ_ROOM_ENCAP_L3_IPV6 = 0x4 + BPF_F_ADJ_ROOM_ENCAP_L4_GRE = 0x8 + BPF_F_ADJ_ROOM_ENCAP_L4_UDP = 0x10 + BPF_ADJ_ROOM_ENCAP_L2_MASK = 0xff + BPF_ADJ_ROOM_ENCAP_L2_SHIFT = 0x38 + BPF_F_SYSCTL_BASE_NAME = 0x1 + BPF_SK_STORAGE_GET_F_CREATE = 0x1 + BPF_F_GET_BRANCH_RECORDS_SIZE = 0x1 + BPF_ADJ_ROOM_NET = 0x0 + BPF_ADJ_ROOM_MAC = 0x1 + BPF_HDR_START_MAC = 0x0 + BPF_HDR_START_NET = 0x1 + BPF_LWT_ENCAP_SEG6 = 0x0 + BPF_LWT_ENCAP_SEG6_INLINE = 0x1 + BPF_LWT_ENCAP_IP = 0x2 + BPF_OK = 0x0 + BPF_DROP = 0x2 + BPF_REDIRECT = 0x7 + BPF_LWT_REROUTE = 0x80 + BPF_SOCK_OPS_RTO_CB_FLAG = 0x1 + BPF_SOCK_OPS_RETRANS_CB_FLAG = 0x2 + BPF_SOCK_OPS_STATE_CB_FLAG = 0x4 + BPF_SOCK_OPS_RTT_CB_FLAG = 0x8 + BPF_SOCK_OPS_ALL_CB_FLAGS = 0xf + BPF_SOCK_OPS_VOID = 0x0 + BPF_SOCK_OPS_TIMEOUT_INIT = 0x1 + BPF_SOCK_OPS_RWND_INIT = 0x2 + BPF_SOCK_OPS_TCP_CONNECT_CB = 0x3 + BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB = 0x4 + BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB = 0x5 + BPF_SOCK_OPS_NEEDS_ECN = 0x6 + BPF_SOCK_OPS_BASE_RTT = 0x7 + BPF_SOCK_OPS_RTO_CB = 0x8 + BPF_SOCK_OPS_RETRANS_CB = 0x9 + BPF_SOCK_OPS_STATE_CB = 0xa + BPF_SOCK_OPS_TCP_LISTEN_CB = 0xb + BPF_SOCK_OPS_RTT_CB = 0xc + BPF_TCP_ESTABLISHED = 0x1 + BPF_TCP_SYN_SENT = 0x2 + BPF_TCP_SYN_RECV = 0x3 + BPF_TCP_FIN_WAIT1 = 0x4 + BPF_TCP_FIN_WAIT2 = 0x5 + BPF_TCP_TIME_WAIT = 0x6 + BPF_TCP_CLOSE = 0x7 + BPF_TCP_CLOSE_WAIT = 0x8 + BPF_TCP_LAST_ACK = 0x9 + BPF_TCP_LISTEN = 0xa + BPF_TCP_CLOSING = 0xb + BPF_TCP_NEW_SYN_RECV = 0xc + BPF_TCP_MAX_STATES = 0xd + TCP_BPF_IW = 0x3e9 + TCP_BPF_SNDCWND_CLAMP = 0x3ea + BPF_DEVCG_ACC_MKNOD = 0x1 + BPF_DEVCG_ACC_READ = 0x2 + BPF_DEVCG_ACC_WRITE = 0x4 + BPF_DEVCG_DEV_BLOCK = 0x1 + BPF_DEVCG_DEV_CHAR = 0x2 + BPF_FIB_LOOKUP_DIRECT = 0x1 + BPF_FIB_LOOKUP_OUTPUT = 0x2 + BPF_FIB_LKUP_RET_SUCCESS = 0x0 + BPF_FIB_LKUP_RET_BLACKHOLE = 0x1 + BPF_FIB_LKUP_RET_UNREACHABLE = 0x2 + BPF_FIB_LKUP_RET_PROHIBIT = 0x3 + BPF_FIB_LKUP_RET_NOT_FWDED = 0x4 + BPF_FIB_LKUP_RET_FWD_DISABLED = 0x5 + BPF_FIB_LKUP_RET_UNSUPP_LWT = 0x6 + BPF_FIB_LKUP_RET_NO_NEIGH = 0x7 + BPF_FIB_LKUP_RET_FRAG_NEEDED = 0x8 + BPF_FD_TYPE_RAW_TRACEPOINT = 0x0 + BPF_FD_TYPE_TRACEPOINT = 0x1 + BPF_FD_TYPE_KPROBE = 0x2 + BPF_FD_TYPE_KRETPROBE = 0x3 + BPF_FD_TYPE_UPROBE = 0x4 + BPF_FD_TYPE_URETPROBE = 0x5 + BPF_FLOW_DISSECTOR_F_PARSE_1ST_FRAG = 0x1 + BPF_FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL = 0x2 + BPF_FLOW_DISSECTOR_F_STOP_AT_ENCAP = 0x4 ) const ( @@ -2205,7 +2279,7 @@ const ( DEVLINK_CMD_DPIPE_ENTRIES_GET = 0x20 DEVLINK_CMD_DPIPE_HEADERS_GET = 0x21 DEVLINK_CMD_DPIPE_TABLE_COUNTERS_SET = 0x22 - DEVLINK_CMD_MAX = 0x44 + DEVLINK_CMD_MAX = 0x48 DEVLINK_PORT_TYPE_NOTSET = 0x0 DEVLINK_PORT_TYPE_AUTO = 0x1 DEVLINK_PORT_TYPE_ETH = 0x2 @@ -2285,7 +2359,7 @@ const ( DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE = 0x3c DEVLINK_ATTR_PAD = 0x3d DEVLINK_ATTR_ESWITCH_ENCAP_MODE = 0x3e - DEVLINK_ATTR_MAX = 0x8c + DEVLINK_ATTR_MAX = 0x90 DEVLINK_DPIPE_FIELD_MAPPING_TYPE_NONE = 0x0 DEVLINK_DPIPE_FIELD_MAPPING_TYPE_IFINDEX = 0x1 DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT = 0x0 diff --git a/golang.org/x/sys/windows/memory_windows.go b/golang.org/x/sys/windows/memory_windows.go index f80a4204f0..e409d76f0f 100644 --- a/golang.org/x/sys/windows/memory_windows.go +++ b/golang.org/x/sys/windows/memory_windows.go @@ -23,4 +23,9 @@ const ( PAGE_EXECUTE_READ = 0x20 PAGE_EXECUTE_READWRITE = 0x40 PAGE_EXECUTE_WRITECOPY = 0x80 + + QUOTA_LIMITS_HARDWS_MIN_DISABLE = 0x00000002 + QUOTA_LIMITS_HARDWS_MIN_ENABLE = 0x00000001 + QUOTA_LIMITS_HARDWS_MAX_DISABLE = 0x00000008 + QUOTA_LIMITS_HARDWS_MAX_ENABLE = 0x00000004 ) diff --git a/golang.org/x/sys/windows/syscall_windows.go b/golang.org/x/sys/windows/syscall_windows.go index 12c0544cb5..62cf70e9f6 100644 --- a/golang.org/x/sys/windows/syscall_windows.go +++ b/golang.org/x/sys/windows/syscall_windows.go @@ -308,6 +308,8 @@ func NewCallbackCDecl(fn interface{}) uintptr { //sys GetProcessId(process Handle) (id uint32, err error) //sys OpenThread(desiredAccess uint32, inheritHandle bool, threadId uint32) (handle Handle, err error) //sys SetProcessPriorityBoost(process Handle, disable bool) (err error) = kernel32.SetProcessPriorityBoost +//sys GetProcessWorkingSetSizeEx(hProcess Handle, lpMinimumWorkingSetSize *uintptr, lpMaximumWorkingSetSize *uintptr, flags *uint32) +//sys SetProcessWorkingSetSizeEx(hProcess Handle, dwMinimumWorkingSetSize uintptr, dwMaximumWorkingSetSize uintptr, flags uint32) (err error) // Volume Management Functions //sys DefineDosDevice(flags uint32, deviceName *uint16, targetPath *uint16) (err error) = DefineDosDeviceW diff --git a/golang.org/x/sys/windows/zsyscall_windows.go b/golang.org/x/sys/windows/zsyscall_windows.go index 2aa4fa642a..8a562feed0 100644 --- a/golang.org/x/sys/windows/zsyscall_windows.go +++ b/golang.org/x/sys/windows/zsyscall_windows.go @@ -217,6 +217,8 @@ var ( procGetProcessId = modkernel32.NewProc("GetProcessId") procOpenThread = modkernel32.NewProc("OpenThread") procSetProcessPriorityBoost = modkernel32.NewProc("SetProcessPriorityBoost") + procGetProcessWorkingSetSizeEx = modkernel32.NewProc("GetProcessWorkingSetSizeEx") + procSetProcessWorkingSetSizeEx = modkernel32.NewProc("SetProcessWorkingSetSizeEx") procDefineDosDeviceW = modkernel32.NewProc("DefineDosDeviceW") procDeleteVolumeMountPointW = modkernel32.NewProc("DeleteVolumeMountPointW") procFindFirstVolumeW = modkernel32.NewProc("FindFirstVolumeW") @@ -2414,6 +2416,23 @@ func SetProcessPriorityBoost(process Handle, disable bool) (err error) { return } +func GetProcessWorkingSetSizeEx(hProcess Handle, lpMinimumWorkingSetSize *uintptr, lpMaximumWorkingSetSize *uintptr, flags *uint32) { + syscall.Syscall6(procGetProcessWorkingSetSizeEx.Addr(), 4, uintptr(hProcess), uintptr(unsafe.Pointer(lpMinimumWorkingSetSize)), uintptr(unsafe.Pointer(lpMaximumWorkingSetSize)), uintptr(unsafe.Pointer(flags)), 0, 0) + return +} + +func SetProcessWorkingSetSizeEx(hProcess Handle, dwMinimumWorkingSetSize uintptr, dwMaximumWorkingSetSize uintptr, flags uint32) (err error) { + r1, _, e1 := syscall.Syscall6(procSetProcessWorkingSetSizeEx.Addr(), 4, uintptr(hProcess), uintptr(dwMinimumWorkingSetSize), uintptr(dwMaximumWorkingSetSize), uintptr(flags), 0, 0) + if r1 == 0 { + if e1 != 0 { + err = errnoErr(e1) + } else { + err = syscall.EINVAL + } + } + return +} + func DefineDosDevice(flags uint32, deviceName *uint16, targetPath *uint16) (err error) { r1, _, e1 := syscall.Syscall(procDefineDosDeviceW.Addr(), 3, uintptr(flags), uintptr(unsafe.Pointer(deviceName)), uintptr(unsafe.Pointer(targetPath))) if r1 == 0 { diff --git a/modules.txt b/modules.txt index 938959b57a..8541726460 100644 --- a/modules.txt +++ b/modules.txt @@ -185,7 +185,7 @@ github.com/cockroachdb/gostdlib/x/tools/internal/semver github.com/cockroachdb/gostdlib/x/tools/internal/span # github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f github.com/cockroachdb/logtags -# github.com/cockroachdb/pebble v0.0.0-20200618174744-bc7bb07c0a28 +# github.com/cockroachdb/pebble v0.0.0-20200624214307-560ed0b8a18c github.com/cockroachdb/pebble github.com/cockroachdb/pebble/bloom github.com/cockroachdb/pebble/internal/arenaskl @@ -672,7 +672,7 @@ golang.org/x/perf/storage/benchfmt # golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e golang.org/x/sync/errgroup golang.org/x/sync/syncmap -# golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 +# golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4 golang.org/x/sys/cpu golang.org/x/sys/internal/unsafeheader golang.org/x/sys/unix From d970f4b1057ee051e809cd4c13e710c8760ef93f Mon Sep 17 00:00:00 2001 From: Peter Mattis Date: Fri, 26 Jun 2020 11:05:39 -0400 Subject: [PATCH 02/49] bump pebble to 3b86a4009c3b --- github.com/cockroachdb/pebble/batch.go | 12 +- github.com/cockroachdb/pebble/compaction.go | 194 +++++++++--------- .../cockroachdb/pebble/compaction_picker.go | 87 ++++---- github.com/cockroachdb/pebble/db.go | 3 +- github.com/cockroachdb/pebble/event.go | 56 +++-- github.com/cockroachdb/pebble/ingest.go | 2 +- .../cockroachdb/pebble/sstable/block.go | 71 +++---- .../cockroachdb/pebble/sstable/reader.go | 70 ++++--- .../cockroachdb/pebble/sstable/unsafe.go | 11 +- golang.org/x/sys/unix/mkerrors.sh | 2 +- golang.org/x/sys/unix/zerrors_linux_386.go | 1 + golang.org/x/sys/unix/zerrors_linux_amd64.go | 1 + golang.org/x/sys/unix/zerrors_linux_arm.go | 1 + golang.org/x/sys/unix/zerrors_linux_arm64.go | 1 + golang.org/x/sys/unix/zerrors_linux_mips.go | 1 + golang.org/x/sys/unix/zerrors_linux_mips64.go | 1 + .../x/sys/unix/zerrors_linux_mips64le.go | 1 + golang.org/x/sys/unix/zerrors_linux_mipsle.go | 1 + golang.org/x/sys/unix/zerrors_linux_ppc64.go | 1 + .../x/sys/unix/zerrors_linux_ppc64le.go | 1 + .../x/sys/unix/zerrors_linux_riscv64.go | 1 + golang.org/x/sys/unix/zerrors_linux_s390x.go | 1 + .../x/sys/unix/zerrors_linux_sparc64.go | 1 + modules.txt | 4 +- 24 files changed, 283 insertions(+), 242 deletions(-) diff --git a/github.com/cockroachdb/pebble/batch.go b/github.com/cockroachdb/pebble/batch.go index 3c400bd774..8b415e79a8 100644 --- a/github.com/cockroachdb/pebble/batch.go +++ b/github.com/cockroachdb/pebble/batch.go @@ -832,21 +832,21 @@ func (b *Batch) Reader() BatchReader { func batchDecodeStr(data []byte) (odata []byte, s []byte, ok bool) { var v uint32 var n int - src := (*[5]uint8)(unsafe.Pointer(&data[0])) - if a := (*src)[0]; a < 128 { + ptr := unsafe.Pointer(&data[0]) + if a := *((*uint8)(ptr)); a < 128 { v = uint32(a) n = 1 - } else if a, b := a&0x7f, (*src)[1]; b < 128 { + } else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 { v = uint32(b)<<7 | uint32(a) n = 2 - } else if b, c := b&0x7f, (*src)[2]; c < 128 { + } else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 { v = uint32(c)<<14 | uint32(b)<<7 | uint32(a) n = 3 - } else if c, d := c&0x7f, (*src)[3]; d < 128 { + } else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 { v = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a) n = 4 } else { - d, e := d&0x7f, (*src)[4] + d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4))) v = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a) n = 5 } diff --git a/github.com/cockroachdb/pebble/compaction.go b/github.com/cockroachdb/pebble/compaction.go index 008fbabebf..7dadd47233 100644 --- a/github.com/cockroachdb/pebble/compaction.go +++ b/github.com/cockroachdb/pebble/compaction.go @@ -70,6 +70,11 @@ type userKeyRange struct { start, end []byte } +type compactionLevel struct { + level int + files []*fileMetadata +} + // compaction is a table compaction from one level to the next, starting from a // given version. type compaction struct { @@ -82,11 +87,13 @@ type compaction struct { // startLevel is the level that is being compacted. Inputs from startLevel // and outputLevel will be merged to produce a set of outputLevel files. - startLevel int + startLevel *compactionLevel // outputLevel is the level that files are being produced in. outputLevel is // equal to startLevel+1 except when startLevel is 0 in which case it is // equal to compactionPicker.baseLevel(). - outputLevel int + outputLevel *compactionLevel + + inputs []compactionLevel // maxOutputFileSize is the maximum size of an individual table created // during compaction. @@ -113,8 +120,7 @@ type compaction struct { // compaction routine to know how many bytes have been flushed before the flush // is applied. atomicBytesIterated *uint64 - // inputs are the tables to be compacted. - inputs [2][]*fileMetadata + // The boundaries of the input data. smallest InternalKey largest InternalKey @@ -173,18 +179,20 @@ func newCompaction( // bytes, we want to adjust the range to [1,numLevels]. adjustedOutputLevel := 1 + outputLevel - baseLevel - return &compaction{ + c := &compaction{ cmp: opts.Comparer.Compare, formatKey: opts.Comparer.FormatKey, logger: opts.Logger, version: cur, - startLevel: startLevel, - outputLevel: outputLevel, + inputs: []compactionLevel{{level: startLevel}, {level: outputLevel}}, maxOutputFileSize: uint64(opts.Level(adjustedOutputLevel).TargetFileSize), maxOverlapBytes: maxGrandparentOverlapBytes(opts, adjustedOutputLevel), maxExpandedBytes: expandedCompactionByteSizeLimit(opts, adjustedOutputLevel), atomicBytesIterated: bytesCompacted, } + c.startLevel = &c.inputs[0] + c.outputLevel = &c.inputs[1] + return c } func newFlush( @@ -195,14 +203,15 @@ func newFlush( formatKey: opts.Comparer.FormatKey, logger: opts.Logger, version: cur, - startLevel: -1, - outputLevel: 0, + inputs: []compactionLevel{{level: -1}, {level: 0}}, maxOutputFileSize: math.MaxUint64, maxOverlapBytes: math.MaxUint64, maxExpandedBytes: math.MaxUint64, flushing: flushing, atomicBytesIterated: bytesFlushed, } + c.startLevel = &c.inputs[0] + c.outputLevel = &c.inputs[1] if cur.L0Sublevels != nil { c.l0Limits = cur.L0Sublevels.FlushSplitKeys() } @@ -269,38 +278,38 @@ func newFlush( // whether the compaction was automatically scheduled or user initiated. func (c *compaction) setupInputs() { // Expand the initial inputs to a clean cut. - c.inputs[0] = c.expandInputs(c.startLevel, c.inputs[0]) - c.smallest, c.largest = manifest.KeyRange(c.cmp, c.inputs[0], nil) + c.startLevel.files = c.expandInputs(c.startLevel.level, c.startLevel.files) + c.smallest, c.largest = manifest.KeyRange(c.cmp, c.startLevel.files, nil) // Determine the sstables in the output level which overlap with the input // sstables, and then expand those tables to a clean cut. - c.inputs[1] = c.version.Overlaps(c.outputLevel, c.cmp, c.smallest.UserKey, c.largest.UserKey) - c.inputs[1] = c.expandInputs(c.outputLevel, c.inputs[1]) - c.smallest, c.largest = manifest.KeyRange(c.cmp, c.inputs[0], c.inputs[1]) + c.outputLevel.files = c.version.Overlaps(c.outputLevel.level, c.cmp, c.smallest.UserKey, c.largest.UserKey) + c.outputLevel.files = c.expandInputs(c.outputLevel.level, c.outputLevel.files) + c.smallest, c.largest = manifest.KeyRange(c.cmp, c.startLevel.files, c.outputLevel.files) - // Grow the sstables in c.startLevel as long as it doesn't affect the number - // of sstables included from c.outputLevel. - if c.lcf != nil && c.startLevel == 0 && c.outputLevel != 0 { + // Grow the sstables in c.startLevel.level as long as it doesn't affect the number + // of sstables included from c.outputLevel.level. + if c.lcf != nil && c.startLevel.level == 0 && c.outputLevel.level != 0 { // Call the L0-specific compaction extension method. Similar logic as // c.grow. Additional L0 files are optionally added to the compaction at // this step. if c.version.L0Sublevels.ExtendL0ForBaseCompactionTo(c.smallest, c.largest, c.lcf) { - c.inputs[0] = c.inputs[0][:0] + c.startLevel.files = c.startLevel.files[:0] for j := range c.lcf.FilesIncluded { if c.lcf.FilesIncluded[j] { - c.inputs[0] = append(c.inputs[0], c.version.Levels[0][j]) + c.startLevel.files = append(c.startLevel.files, c.version.Levels[0][j]) } } - c.smallest, c.largest = manifest.KeyRange(c.cmp, c.inputs[0], c.inputs[1]) + c.smallest, c.largest = manifest.KeyRange(c.cmp, c.startLevel.files, c.outputLevel.files) } } else if c.grow(c.smallest, c.largest) { - c.smallest, c.largest = manifest.KeyRange(c.cmp, c.inputs[0], c.inputs[1]) + c.smallest, c.largest = manifest.KeyRange(c.cmp, c.startLevel.files, c.outputLevel.files) } // Compute the set of outputLevel+1 files that overlap this compaction (these // are the grandparent sstables). - if c.outputLevel+1 < numLevels { - c.grandparents = c.version.Overlaps(c.outputLevel+1, c.cmp, c.smallest.UserKey, c.largest.UserKey) + if c.outputLevel.level+1 < numLevels { + c.grandparents = c.version.Overlaps(c.outputLevel.level+1, c.cmp, c.smallest.UserKey, c.largest.UserKey) } c.setupInuseKeyRanges() @@ -403,31 +412,31 @@ func (c *compaction) expandInputs(level int, inputs []*fileMetadata) []*fileMeta // c.level+1 files in the compaction, and returns whether the inputs grew. sm // and la are the smallest and largest InternalKeys in all of the inputs. func (c *compaction) grow(sm, la InternalKey) bool { - if len(c.inputs[1]) == 0 { + if len(c.outputLevel.files) == 0 { return false } - grow0 := c.version.Overlaps(c.startLevel, c.cmp, sm.UserKey, la.UserKey) - grow0 = c.expandInputs(c.startLevel, grow0) - if len(grow0) <= len(c.inputs[0]) { + grow0 := c.version.Overlaps(c.startLevel.level, c.cmp, sm.UserKey, la.UserKey) + grow0 = c.expandInputs(c.startLevel.level, grow0) + if len(grow0) <= len(c.startLevel.files) { return false } - if totalSize(grow0)+totalSize(c.inputs[1]) >= c.maxExpandedBytes { + if totalSize(grow0)+totalSize(c.outputLevel.files) >= c.maxExpandedBytes { return false } sm1, la1 := manifest.KeyRange(c.cmp, grow0, nil) - grow1 := c.version.Overlaps(c.outputLevel, c.cmp, sm1.UserKey, la1.UserKey) - grow1 = c.expandInputs(c.outputLevel, grow1) - if len(grow1) != len(c.inputs[1]) { + grow1 := c.version.Overlaps(c.outputLevel.level, c.cmp, sm1.UserKey, la1.UserKey) + grow1 = c.expandInputs(c.outputLevel.level, grow1) + if len(grow1) != len(c.outputLevel.files) { return false } - c.inputs[0] = grow0 - c.inputs[1] = grow1 + c.startLevel.files = grow0 + c.outputLevel.files = grow1 return true } func (c *compaction) setupInuseKeyRanges() { - level := c.outputLevel + 1 - if c.outputLevel == 0 { + level := c.outputLevel.level + 1 + if c.outputLevel.level == 0 { // Level 0 can contain overlapping sstables so we need to check it for // overlaps. level = 0 @@ -478,7 +487,7 @@ func (c *compaction) trivialMove() bool { // such a move if there is lots of overlapping grandparent data. Otherwise, // the move could create a parent file that will require a very expensive // merge later on. - if len(c.inputs[0]) == 1 && len(c.inputs[1]) == 0 && + if len(c.startLevel.files) == 1 && len(c.outputLevel.files) == 0 && totalSize(c.grandparents) <= c.maxOverlapBytes { return true } @@ -525,7 +534,7 @@ func (c *compaction) findGrandparentLimit(start []byte) []byte { // this function passes through to findGrandparentLimit. func (c *compaction) findL0Limit(start []byte) []byte { grandparentLimit := c.findGrandparentLimit(start) - if c.startLevel > -1 || c.outputLevel != 0 || len(c.l0Limits) == 0 { + if c.startLevel.level > -1 || c.outputLevel.level != 0 || len(c.l0Limits) == 0 { return grandparentLimit } index := sort.Search(len(c.l0Limits), func(i int) bool { @@ -569,7 +578,7 @@ func (c *compaction) elideTombstone(key []byte) bool { // elideRangeTombstone returns true if it is ok to elide the specified range // tombstone. A return value of true guarantees that there are no key/value -// pairs at c.outputLevel+1 or higher that possibly overlap the specified +// pairs at c.outputLevel.level+1 or higher that possibly overlap the specified // tombstone. func (c *compaction) elideRangeTombstone(start, end []byte) bool { // Disable range tombstone elision if the testing knob for that is enabled, @@ -606,7 +615,7 @@ func (c *compaction) elideRangeTombstone(start, end []byte) bool { // the specified sstable (identified by a pointer to its fileMetadata). func (c *compaction) atomicUnitBounds(f *fileMetadata) (lower, upper []byte) { for i := range c.inputs { - files := c.inputs[i] + files := c.inputs[i].files for j := range files { if f == files[j] { // Note that if this file is in a multi-file atomic compaction unit, this file @@ -688,16 +697,16 @@ func (c *compaction) newInputIter(newIters tableNewIters) (_ internalIterator, r // Check that the LSM ordering invariants are ok in order to prevent // generating corrupted sstables due to a violation of those invariants. - if c.startLevel >= 0 { - if err := manifest.CheckOrdering(c.cmp, c.formatKey, manifest.Level(c.startLevel), c.inputs[0]); err != nil { + if c.startLevel.level >= 0 { + if err := manifest.CheckOrdering(c.cmp, c.formatKey, manifest.Level(c.startLevel.level), c.startLevel.files); err != nil { c.logger.Fatalf("%s", err) } } - if err := manifest.CheckOrdering(c.cmp, c.formatKey, manifest.Level(c.outputLevel), c.inputs[1]); err != nil { + if err := manifest.CheckOrdering(c.cmp, c.formatKey, manifest.Level(c.outputLevel.level), c.outputLevel.files); err != nil { c.logger.Fatalf("%s", err) } - iters := make([]internalIterator, 0, 2*len(c.inputs[0])+1) + iters := make([]internalIterator, 0, 2*len(c.startLevel.files)+1) defer func() { if retErr != nil { for _, iter := range iters { @@ -757,14 +766,14 @@ func (c *compaction) newInputIter(newIters tableNewIters) (_ internalIterator, r } iterOpts := IterOptions{logger: c.logger} - if c.startLevel != 0 { - iters = append(iters, newLevelIter(iterOpts, c.cmp, newIters, c.inputs[0], - manifest.Level(c.startLevel), &c.bytesIterated)) - iters = append(iters, newLevelIter(iterOpts, c.cmp, newRangeDelIter, c.inputs[0], - manifest.Level(c.startLevel), &c.bytesIterated)) + if c.startLevel.level != 0 { + iters = append(iters, newLevelIter(iterOpts, c.cmp, newIters, c.startLevel.files, + manifest.Level(c.startLevel.level), &c.bytesIterated)) + iters = append(iters, newLevelIter(iterOpts, c.cmp, newRangeDelIter, c.startLevel.files, + manifest.Level(c.startLevel.level), &c.bytesIterated)) } else { - for i := range c.inputs[0] { - f := c.inputs[0][i] + for i := range c.startLevel.files { + f := c.startLevel.files[i] iter, rangeDelIter, err := newIters(f, nil /* iter options */, &c.bytesIterated) if err != nil { return nil, errors.Wrapf(err, "pebble: could not open table %s", errors.Safe(f.FileNum)) @@ -776,10 +785,10 @@ func (c *compaction) newInputIter(newIters tableNewIters) (_ internalIterator, r } } - iters = append(iters, newLevelIter(iterOpts, c.cmp, newIters, c.inputs[1], - manifest.Level(c.outputLevel), &c.bytesIterated)) - iters = append(iters, newLevelIter(iterOpts, c.cmp, newRangeDelIter, c.inputs[1], - manifest.Level(c.outputLevel), &c.bytesIterated)) + iters = append(iters, newLevelIter(iterOpts, c.cmp, newIters, c.outputLevel.files, + manifest.Level(c.outputLevel.level), &c.bytesIterated)) + iters = append(iters, newLevelIter(iterOpts, c.cmp, newRangeDelIter, c.outputLevel.files, + manifest.Level(c.outputLevel.level), &c.bytesIterated)) return newMergingIter(c.logger, c.cmp, iters...), nil } @@ -790,12 +799,12 @@ func (c *compaction) String() string { var buf bytes.Buffer for i := range c.inputs { - level := c.startLevel + level := c.startLevel.level if i == 1 { - level = c.outputLevel + level = c.outputLevel.level } fmt.Fprintf(&buf, "%d:", level) - for _, f := range c.inputs[i] { + for _, f := range c.inputs[i].files { fmt.Fprintf(&buf, " %s:%s-%s", f.FileNum, f.Smallest, f.Largest) } fmt.Fprintf(&buf, "\n") @@ -817,14 +826,14 @@ type manualCompaction struct { func (d *DB) addInProgressCompaction(c *compaction) { d.mu.compact.inProgress[c] = struct{}{} var isBase, isIntraL0 bool - for _, files := range c.inputs { - for _, f := range files { + for _, cl := range c.inputs { + for _, f := range cl.files { if f.Compacting { - d.opts.Logger.Fatalf("L%d->L%d: %s already being compacted", c.startLevel, c.outputLevel, f.FileNum) + d.opts.Logger.Fatalf("L%d->L%d: %s already being compacted", c.startLevel.level, c.outputLevel.level, f.FileNum) } f.Compacting = true - if c.startLevel == 0 { - if c.outputLevel == 0 { + if c.startLevel.level == 0 { + if c.outputLevel.level == 0 { f.IsIntraL0Compacting = true isIntraL0 = true } else { @@ -835,9 +844,9 @@ func (d *DB) addInProgressCompaction(c *compaction) { } if (isIntraL0 || isBase) && c.version.L0Sublevels != nil { - l0Inputs := [][]*fileMetadata{c.inputs[0]} + l0Inputs := [][]*fileMetadata{c.startLevel.files} if isIntraL0 { - l0Inputs = c.inputs[:] + l0Inputs = append(l0Inputs, c.outputLevel.files) } if err := c.version.L0Sublevels.UpdateStateForStartedCompaction(l0Inputs, isBase); err != nil { d.opts.Logger.Fatalf("could not update state for compaction: %s", err) @@ -852,10 +861,10 @@ func (d *DB) addInProgressCompaction(c *compaction) { strs := make([]string, 0, len(d.mu.compact.inProgress)) for c := range d.mu.compact.inProgress { var s string - if c.startLevel == -1 { - s = fmt.Sprintf("mem->L%d", c.outputLevel) + if c.startLevel.level == -1 { + s = fmt.Sprintf("mem->L%d", c.outputLevel.level) } else { - s = fmt.Sprintf("L%d->L%d:%.1f", c.startLevel, c.outputLevel, c.score) + s = fmt.Sprintf("L%d->L%d:%.1f", c.startLevel.level, c.outputLevel.level, c.score) } strs = append(strs, s) } @@ -875,10 +884,10 @@ func (d *DB) addInProgressCompaction(c *compaction) { // DB.mu must be held when calling this method. All writes to the manifest // for this compaction should have completed by this point. func (d *DB) removeInProgressCompaction(c *compaction) { - for _, files := range c.inputs { - for _, f := range files { + for _, cl := range c.inputs { + for _, f := range cl.files { if !f.Compacting { - d.opts.Logger.Fatalf("L%d->L%d: %s already being compacted", c.startLevel, c.outputLevel, f.FileNum) + d.opts.Logger.Fatalf("L%d->L%d: %s already being compacted", c.startLevel.level, c.outputLevel.level, f.FileNum) } f.Compacting = false f.IsIntraL0Compacting = false @@ -1240,15 +1249,18 @@ func (d *DB) compact1(c *compaction, errChannel chan error) (err error) { d.mu.nextJobID++ info := CompactionInfo{ JobID: jobID, + Input: []LevelInfo{ + {Level: c.startLevel.level}, + {Level: c.outputLevel.level}, + }, + Output: LevelInfo{Level: c.outputLevel.level}, } - info.Input.Level = c.startLevel - info.Output.Level = c.outputLevel - for i := range c.inputs { - for j := range c.inputs[i] { - m := c.inputs[i][j] - info.Input.Tables[i] = append(info.Input.Tables[i], m.TableInfo()) + for i, cl := range c.inputs { + for _, m := range cl.files { + info.Input[i].Tables = append(info.Input[i].Tables, m.TableInfo()) } } + d.opts.EventListener.CompactionBegin(info) startTime := d.timeNow() @@ -1315,19 +1327,19 @@ func (d *DB) runCompaction( // the move could create a parent file that will require a very expensive // merge later on. if c.trivialMove() { - meta := c.inputs[0][0] + meta := c.startLevel.files[0] c.metrics = map[int]*LevelMetrics{ - c.outputLevel: &LevelMetrics{ + c.outputLevel.level: &LevelMetrics{ BytesMoved: meta.Size, TablesMoved: 1, }, } ve := &versionEdit{ DeletedFiles: map[deletedFileEntry]bool{ - deletedFileEntry{Level: c.startLevel, FileNum: meta.FileNum}: true, + deletedFileEntry{Level: c.startLevel.level, FileNum: meta.FileNum}: true, }, NewFiles: []newFileEntry{ - {Level: c.outputLevel, Meta: meta}, + {Level: c.outputLevel.level, Meta: meta}, }, } return ve, nil, nil @@ -1380,15 +1392,15 @@ func (d *DB) runCompaction( } metrics := &LevelMetrics{ - BytesIn: totalSize(c.inputs[0]), - BytesRead: totalSize(c.inputs[1]), + BytesIn: totalSize(c.startLevel.files), + BytesRead: totalSize(c.outputLevel.files), } metrics.BytesRead += metrics.BytesIn c.metrics = map[int]*LevelMetrics{ - c.outputLevel: metrics, + c.outputLevel.level: metrics, } - writerOpts := d.opts.MakeWriterOptions(c.outputLevel) + writerOpts := d.opts.MakeWriterOptions(c.outputLevel.level) newOutput := func() error { d.mu.Lock() @@ -1420,7 +1432,7 @@ func (d *DB) runCompaction( tw = sstable.NewWriter(file, writerOpts, cacheOpts, internalTableOpt) ve.NewFiles = append(ve.NewFiles, newFileEntry{ - Level: c.outputLevel, + Level: c.outputLevel.level, Meta: &fileMetadata{ FileNum: fileNum, CreationTime: time.Now().Unix(), @@ -1429,7 +1441,7 @@ func (d *DB) runCompaction( return nil } - splittingFlush := c.startLevel < 0 && c.outputLevel == 0 && d.opts.Experimental.FlushSplitBytes > 0 + splittingFlush := c.startLevel.level < 0 && c.outputLevel.level == 0 && d.opts.Experimental.FlushSplitBytes > 0 // finishOutput is called for an sstable with the first key of the next sstable, and for the // last sstable with an empty key. @@ -1793,14 +1805,10 @@ func (d *DB) runCompaction( } } - for i := range c.inputs { - level := c.startLevel - if i == 1 { - level = c.outputLevel - } - for _, f := range c.inputs[i] { + for _, cl := range c.inputs { + for _, f := range cl.files { ve.DeletedFiles[deletedFileEntry{ - Level: level, + Level: cl.level, FileNum: f.FileNum, }] = true } diff --git a/github.com/cockroachdb/pebble/compaction_picker.go b/github.com/cockroachdb/pebble/compaction_picker.go index 8ffc38d3cc..ebfb0a5482 100644 --- a/github.com/cockroachdb/pebble/compaction_picker.go +++ b/github.com/cockroachdb/pebble/compaction_picker.go @@ -39,9 +39,8 @@ type compactionPicker interface { // Information about in-progress compactions provided to the compaction picker. These are used to // constrain the new compactions that will be picked. type compactionInfo struct { - startLevel int + inputs []compactionLevel outputLevel int - inputs [2][]*fileMetadata } type sortCompactionLevelsDecreasingScore []pickedCompactionInfo @@ -217,7 +216,7 @@ func (p *compactionPickerByScore) initLevelMaxBytes(inProgressCompactions []comp if c.outputLevel == 0 { continue } - if c.startLevel == 0 && (firstNonEmptyLevel == -1 || c.outputLevel < firstNonEmptyLevel) { + if c.inputs[0].level == 0 && (firstNonEmptyLevel == -1 || c.outputLevel < firstNonEmptyLevel) { firstNonEmptyLevel = c.outputLevel } } @@ -291,10 +290,16 @@ func calculateSizeAdjust(inProgressCompactions []compactionInfo) [numLevels]int6 var sizeAdjust [numLevels]int64 for i := range inProgressCompactions { c := &inProgressCompactions[i] - compensated := int64(totalCompensatedSize(c.inputs[0])) - real := int64(totalSize(c.inputs[0])) - sizeAdjust[c.startLevel] -= compensated - sizeAdjust[c.outputLevel] += real + + for _, input := range c.inputs { + real := int64(totalSize(input.files)) + compensated := int64(totalCompensatedSize(input.files)) + + if input.level != c.outputLevel { + sizeAdjust[input.level] -= compensated + sizeAdjust[c.outputLevel] += real + } + } } return sizeAdjust } @@ -402,7 +407,7 @@ func (p *compactionPickerByScore) calculateL0Score( // compaction. var l0Compaction bool for i := range inProgressCompactions { - if inProgressCompactions[i].startLevel == 0 && + if inProgressCompactions[i].inputs[0].level == 0 && inProgressCompactions[i].outputLevel != 0 { l0Compaction = true break @@ -555,7 +560,7 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (c *compaction) { } marker := " " - if c.startLevel == info.level { + if c.startLevel.level == info.level { marker = "*" } fmt.Fprintf(&buf, " %sL%d: %5.1f %5.1f %8s %8s", @@ -567,7 +572,7 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (c *compaction) { count := 0 for i := range env.inProgressCompactions { c := &env.inProgressCompactions[i] - if c.startLevel != info.level { + if c.inputs[0].level != info.level { continue } count++ @@ -576,7 +581,7 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (c *compaction) { } else { fmt.Fprintf(&buf, " ") } - fmt.Fprintf(&buf, "L%d->L%d", c.startLevel, c.outputLevel) + fmt.Fprintf(&buf, "L%d->L%d", c.inputs[0].level, c.outputLevel) } if count > 0 { fmt.Fprintf(&buf, "]") @@ -584,7 +589,7 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (c *compaction) { fmt.Fprintf(&buf, "\n") } p.opts.Logger.Infof("pickAuto: L%d->L%d\n%s", - c.startLevel, c.outputLevel, buf.String()) + c.startLevel.level, c.outputLevel.level, buf.String()) } // Check for a score-based compaction. "scores" has been sorted in order of @@ -675,16 +680,16 @@ func pickAutoHelper( } c = newCompaction(opts, vers, cInfo.level, baseLevel, env.bytesCompacted) - if c.outputLevel != cInfo.outputLevel { + if c.outputLevel.level != cInfo.outputLevel { panic("pebble: compaction picked unexpected output level") } - c.inputs[0] = vers.Levels[c.startLevel][cInfo.file : cInfo.file+1] + c.startLevel.files = vers.Levels[c.startLevel.level][cInfo.file : cInfo.file+1] // Files in level 0 may overlap each other, so pick up all overlapping ones. - if c.startLevel == 0 { + if c.startLevel.level == 0 { cmp := opts.Comparer.Compare - smallest, largest := manifest.KeyRange(cmp, c.inputs[0], nil) - c.inputs[0] = vers.Overlaps(0, cmp, smallest.UserKey, largest.UserKey) - if len(c.inputs[0]) == 0 { + smallest, largest := manifest.KeyRange(cmp, c.startLevel.files, nil) + c.startLevel.files = vers.Overlaps(0, cmp, smallest.UserKey, largest.UserKey) + if len(c.startLevel.files) == 0 { panic("pebble: empty compaction") } } @@ -714,17 +719,17 @@ func pickL0(env compactionEnv, opts *Options, vers *version, baseLevel int) (c * // pick contiguous sequences of files in p.vers.Levels[0]. c = newCompaction(opts, vers, 0, baseLevel, env.bytesCompacted) c.lcf = lcf - if c.outputLevel != baseLevel { - opts.Logger.Fatalf("compaction picked unexpected output level: %d != %d", c.outputLevel, baseLevel) + if c.outputLevel.level != baseLevel { + opts.Logger.Fatalf("compaction picked unexpected output level: %d != %d", c.outputLevel.level, baseLevel) } - c.inputs[0] = make([]*manifest.FileMetadata, 0, len(lcf.Files)) + c.startLevel.files = make([]*manifest.FileMetadata, 0, len(lcf.Files)) for j := range lcf.FilesIncluded { if lcf.FilesIncluded[j] { - c.inputs[0] = append(c.inputs[0], vers.Levels[0][j]) + c.startLevel.files = append(c.startLevel.files, vers.Levels[0][j]) } } c.setupInputs() - if len(c.inputs[0]) == 0 { + if len(c.startLevel.files) == 0 { opts.Logger.Fatalf("empty compaction chosen") } return c @@ -740,19 +745,19 @@ func pickL0(env compactionEnv, opts *Options, vers *version, baseLevel int) (c * if lcf != nil { c = newCompaction(opts, vers, 0, 0, env.bytesCompacted) c.lcf = lcf - c.inputs[0] = make([]*manifest.FileMetadata, 0, len(lcf.Files)) + c.startLevel.files = make([]*manifest.FileMetadata, 0, len(lcf.Files)) for j := range lcf.FilesIncluded { if lcf.FilesIncluded[j] { - c.inputs[0] = append(c.inputs[0], vers.Levels[0][j]) + c.startLevel.files = append(c.startLevel.files, vers.Levels[0][j]) } } - if len(c.inputs[0]) == 0 { + if len(c.startLevel.files) == 0 { opts.Logger.Fatalf("empty compaction chosen") - } else if len(c.inputs[0]) == 1 { + } else if len(c.startLevel.files) == 1 { // A single-file intra-L0 compaction is unproductive. return nil } - c.smallest, c.largest = manifest.KeyRange(c.cmp, c.inputs[0], nil) + c.smallest, c.largest = manifest.KeyRange(c.cmp, c.startLevel.files, nil) c.setupInuseKeyRanges() // Output only a single sstable for intra-L0 compactions. // Now that we have the ability to split flushes, we could conceivably @@ -844,8 +849,8 @@ func pickIntraL0(env compactionEnv, opts *Options, vers *version) (c *compaction } c = newCompaction(opts, vers, 0, 0, env.bytesCompacted) - c.inputs[0] = l0Files[begin:end] - c.smallest, c.largest = manifest.KeyRange(c.cmp, c.inputs[0], nil) + c.startLevel.files = l0Files[begin:end] + c.smallest, c.largest = manifest.KeyRange(c.cmp, c.startLevel.files, nil) c.setupInuseKeyRanges() // Output only a single sstable for intra-L0 compactions. There is no current // benefit to outputting multiple tables, because other parts of the code @@ -885,7 +890,7 @@ func (p *compactionPickerByScore) pickManual( if c == nil { return nil, false } - if c.outputLevel != outputLevel { + if c.outputLevel.level != outputLevel { panic("pebble: compaction picked unexpected output level") } // Fail-safe to protect against compacting the same sstable concurrently. @@ -899,10 +904,10 @@ func pickManualHelper( env compactionEnv, opts *Options, manual *manualCompaction, vers *version, baseLevel int, ) (c *compaction) { c = newCompaction(opts, vers, manual.level, baseLevel, env.bytesCompacted) - manual.outputLevel = c.outputLevel + manual.outputLevel = c.outputLevel.level cmp := opts.Comparer.Compare - c.inputs[0] = vers.Overlaps(manual.level, cmp, manual.start.UserKey, manual.end.UserKey) - if len(c.inputs[0]) == 0 { + c.startLevel.files = vers.Overlaps(manual.level, cmp, manual.start.UserKey, manual.end.UserKey) + if len(c.startLevel.files) == 0 { // Nothing to do return nil } @@ -915,8 +920,8 @@ func (p *compactionPickerByScore) forceBaseLevel1() { } func inputAlreadyCompacting(c *compaction) bool { - for _, inputs := range c.inputs { - for _, f := range inputs { + for _, cl := range c.inputs { + for _, f := range cl.files { if f.Compacting { return true } @@ -929,10 +934,12 @@ func conflictsWithInProgress( level int, outputLevel int, inProgressCompactions []compactionInfo, ) bool { for _, c := range inProgressCompactions { - if level == c.startLevel || - level == c.outputLevel || - outputLevel == c.startLevel || - outputLevel == c.outputLevel { + for _, in := range c.inputs { + if in.level == level || in.level == outputLevel { + return true + } + } + if c.outputLevel == level || c.outputLevel == outputLevel { return true } } diff --git a/github.com/cockroachdb/pebble/db.go b/github.com/cockroachdb/pebble/db.go index 108609d072..0675af6bcc 100644 --- a/github.com/cockroachdb/pebble/db.go +++ b/github.com/cockroachdb/pebble/db.go @@ -1458,9 +1458,8 @@ func (d *DB) getInProgressCompactionInfoLocked(finishing *compaction) (rv []comp for c := range d.mu.compact.inProgress { if len(c.flushing) == 0 && (finishing == nil || c != finishing) { rv = append(rv, compactionInfo{ - startLevel: c.startLevel, - outputLevel: c.outputLevel, inputs: c.inputs, + outputLevel: c.outputLevel.level, }) } } diff --git a/github.com/cockroachdb/pebble/event.go b/github.com/cockroachdb/pebble/event.go index 45e9243045..2032a018f3 100644 --- a/github.com/cockroachdb/pebble/event.go +++ b/github.com/cockroachdb/pebble/event.go @@ -7,6 +7,7 @@ package pebble import ( "bytes" "fmt" + "io" "strings" "time" @@ -36,26 +37,30 @@ func formatFileNums(tables []TableInfo) string { return buf.String() } +// LevelInfo contains info pertaining to a partificular level. +type LevelInfo struct { + Level int + Tables []TableInfo +} + +func (i LevelInfo) String() string { + return fmt.Sprintf("L%d [%s] (%s)", + i.Level, + formatFileNums(i.Tables), + humanize.Uint64(tablesTotalSize(i.Tables))) +} + // CompactionInfo contains the info for a compaction event. type CompactionInfo struct { // JobID is the ID of the compaction job. JobID int // Reason is the reason for the compaction. Reason string - // Input contains the input tables for the compaction. A compaction is - // performed from Input.Level to Output.Level. Input.Tables[0] contains the - // inputs from Input.Level and Input.Tables[1] contains the inputs from - // Output.Level. - Input struct { - Level int - Tables [2][]TableInfo - } + // Input contains the input tables for the compaction organized by level. + Input []LevelInfo // Output contains the output tables generated by the compaction. The output // tables are empty for the compaction begin event. - Output struct { - Level int - Tables []TableInfo - } + Output LevelInfo Duration time.Duration Done bool Err error @@ -66,27 +71,20 @@ func (i CompactionInfo) String() string { return fmt.Sprintf("[JOB %d] compaction to L%d error: %s", i.JobID, i.Output.Level, i.Err) } - + var inputLevelsBuf bytes.Buffer + for j, levelInfo := range i.Input { + if j > 0 { + io.WriteString(&inputLevelsBuf, " + ") + } + io.WriteString(&inputLevelsBuf, levelInfo.String()) + } if !i.Done { - return fmt.Sprintf("[JOB %d] compacting L%d [%s] (%s) + L%d [%s] (%s)", - i.JobID, - i.Input.Level, - formatFileNums(i.Input.Tables[0]), - humanize.Uint64(tablesTotalSize(i.Input.Tables[0])), - i.Output.Level, - formatFileNums(i.Input.Tables[1]), - humanize.Uint64(tablesTotalSize(i.Input.Tables[1]))) + return fmt.Sprintf("[JOB %d] compacting %s", i.JobID, inputLevelsBuf.String()) } - outputSize := tablesTotalSize(i.Output.Tables) - return fmt.Sprintf("[JOB %d] compacted L%d [%s] (%s) + L%d [%s] (%s) -> L%d [%s] (%s), in %.1fs, output rate %s/s", + return fmt.Sprintf("[JOB %d] compacted %s -> L%d [%s] (%s), in %.1fs, output rate %s/s", i.JobID, - i.Input.Level, - formatFileNums(i.Input.Tables[0]), - humanize.Uint64(tablesTotalSize(i.Input.Tables[0])), - i.Output.Level, - formatFileNums(i.Input.Tables[1]), - humanize.Uint64(tablesTotalSize(i.Input.Tables[1])), + inputLevelsBuf.String(), i.Output.Level, formatFileNums(i.Output.Tables), humanize.Uint64(outputSize), diff --git a/github.com/cockroachdb/pebble/ingest.go b/github.com/cockroachdb/pebble/ingest.go index 20f2d70569..654e554189 100644 --- a/github.com/cockroachdb/pebble/ingest.go +++ b/github.com/cockroachdb/pebble/ingest.go @@ -445,7 +445,7 @@ func ingestTargetLevel( // negative (else we'd have returned earlier). overlaps := false for c := range compactions { - if level != c.outputLevel { + if level != c.outputLevel.level { continue } if cmp(meta.Smallest.UserKey, c.largest.UserKey) <= 0 && diff --git a/github.com/cockroachdb/pebble/sstable/block.go b/github.com/cockroachdb/pebble/sstable/block.go index 42dfb355a5..d0a5661d61 100644 --- a/github.com/cockroachdb/pebble/sstable/block.go +++ b/github.com/cockroachdb/pebble/sstable/block.go @@ -305,61 +305,58 @@ func (i *blockIter) readEntry() { // TODO(peter): remove this hack if go:inline is ever supported. var shared uint32 - src := (*[5]uint8)(ptr) - if a := (*src)[0]; a < 128 { + if a := *((*uint8)(ptr)); a < 128 { shared = uint32(a) ptr = unsafe.Pointer(uintptr(ptr) + 1) - } else if a, b := a&0x7f, (*src)[1]; b < 128 { + } else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 { shared = uint32(b)<<7 | uint32(a) ptr = unsafe.Pointer(uintptr(ptr) + 2) - } else if b, c := b&0x7f, (*src)[2]; c < 128 { + } else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 { shared = uint32(c)<<14 | uint32(b)<<7 | uint32(a) ptr = unsafe.Pointer(uintptr(ptr) + 3) - } else if c, d := c&0x7f, (*src)[3]; d < 128 { + } else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 { shared = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a) ptr = unsafe.Pointer(uintptr(ptr) + 4) } else { - d, e := d&0x7f, (*src)[4] + d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4))) shared = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a) ptr = unsafe.Pointer(uintptr(ptr) + 5) } var unshared uint32 - src = (*[5]uint8)(ptr) - if a := (*src)[0]; a < 128 { + if a := *((*uint8)(ptr)); a < 128 { unshared = uint32(a) ptr = unsafe.Pointer(uintptr(ptr) + 1) - } else if a, b := a&0x7f, (*src)[1]; b < 128 { + } else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 { unshared = uint32(b)<<7 | uint32(a) ptr = unsafe.Pointer(uintptr(ptr) + 2) - } else if b, c := b&0x7f, (*src)[2]; c < 128 { + } else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 { unshared = uint32(c)<<14 | uint32(b)<<7 | uint32(a) ptr = unsafe.Pointer(uintptr(ptr) + 3) - } else if c, d := c&0x7f, (*src)[3]; d < 128 { + } else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 { unshared = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a) ptr = unsafe.Pointer(uintptr(ptr) + 4) } else { - d, e := d&0x7f, (*src)[4] + d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4))) unshared = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a) ptr = unsafe.Pointer(uintptr(ptr) + 5) } var value uint32 - src = (*[5]uint8)(ptr) - if a := (*src)[0]; a < 128 { + if a := *((*uint8)(ptr)); a < 128 { value = uint32(a) ptr = unsafe.Pointer(uintptr(ptr) + 1) - } else if a, b := a&0x7f, (*src)[1]; b < 128 { + } else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 { value = uint32(b)<<7 | uint32(a) ptr = unsafe.Pointer(uintptr(ptr) + 2) - } else if b, c := b&0x7f, (*src)[2]; c < 128 { + } else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 { value = uint32(c)<<14 | uint32(b)<<7 | uint32(a) ptr = unsafe.Pointer(uintptr(ptr) + 3) - } else if c, d := c&0x7f, (*src)[3]; d < 128 { + } else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 { value = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a) ptr = unsafe.Pointer(uintptr(ptr) + 4) } else { - d, e := d&0x7f, (*src)[4] + d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4))) value = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a) ptr = unsafe.Pointer(uintptr(ptr) + 5) } @@ -447,32 +444,31 @@ func (i *blockIter) SeekGE(key []byte) (*InternalKey, []byte) { // sought. See the comment in readEntry for why we manually inline the // varint decoding. var v1 uint32 - src := (*[5]uint8)(ptr) - if a := (*src)[0]; a < 128 { + if a := *((*uint8)(ptr)); a < 128 { v1 = uint32(a) ptr = unsafe.Pointer(uintptr(ptr) + 1) - } else if a, b := a&0x7f, (*src)[1]; b < 128 { + } else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 { v1 = uint32(b)<<7 | uint32(a) ptr = unsafe.Pointer(uintptr(ptr) + 2) - } else if b, c := b&0x7f, (*src)[2]; c < 128 { + } else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 { v1 = uint32(c)<<14 | uint32(b)<<7 | uint32(a) ptr = unsafe.Pointer(uintptr(ptr) + 3) - } else if c, d := c&0x7f, (*src)[3]; d < 128 { + } else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 { v1 = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a) ptr = unsafe.Pointer(uintptr(ptr) + 4) } else { - d, e := d&0x7f, (*src)[4] + d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4))) v1 = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a) ptr = unsafe.Pointer(uintptr(ptr) + 5) } - if src := (*[5]uint8)(ptr); (*src)[0] < 128 { + if *((*uint8)(ptr)) < 128 { ptr = unsafe.Pointer(uintptr(ptr) + 1) - } else if (*src)[1] < 128 { + } else if *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))) < 128 { ptr = unsafe.Pointer(uintptr(ptr) + 2) - } else if (*src)[2] < 128 { + } else if *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))) < 128 { ptr = unsafe.Pointer(uintptr(ptr) + 3) - } else if (*src)[3] < 128 { + } else if *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))) < 128 { ptr = unsafe.Pointer(uintptr(ptr) + 4) } else { ptr = unsafe.Pointer(uintptr(ptr) + 5) @@ -559,32 +555,31 @@ func (i *blockIter) SeekLT(key []byte) (*InternalKey, []byte) { // sought. See the comment in readEntry for why we manually inline the // varint decoding. var v1 uint32 - src := (*[5]uint8)(ptr) - if a := (*src)[0]; a < 128 { + if a := *((*uint8)(ptr)); a < 128 { v1 = uint32(a) ptr = unsafe.Pointer(uintptr(ptr) + 1) - } else if a, b := a&0x7f, (*src)[1]; b < 128 { + } else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 { v1 = uint32(b)<<7 | uint32(a) ptr = unsafe.Pointer(uintptr(ptr) + 2) - } else if b, c := b&0x7f, (*src)[2]; c < 128 { + } else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 { v1 = uint32(c)<<14 | uint32(b)<<7 | uint32(a) ptr = unsafe.Pointer(uintptr(ptr) + 3) - } else if c, d := c&0x7f, (*src)[3]; d < 128 { + } else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 { v1 = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a) ptr = unsafe.Pointer(uintptr(ptr) + 4) } else { - d, e := d&0x7f, (*src)[4] + d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4))) v1 = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a) ptr = unsafe.Pointer(uintptr(ptr) + 5) } - if src := (*[5]uint8)(ptr); (*src)[0] < 128 { + if *((*uint8)(ptr)) < 128 { ptr = unsafe.Pointer(uintptr(ptr) + 1) - } else if (*src)[1] < 128 { + } else if *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))) < 128 { ptr = unsafe.Pointer(uintptr(ptr) + 2) - } else if (*src)[2] < 128 { + } else if *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))) < 128 { ptr = unsafe.Pointer(uintptr(ptr) + 3) - } else if (*src)[3] < 128 { + } else if *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))) < 128 { ptr = unsafe.Pointer(uintptr(ptr) + 4) } else { ptr = unsafe.Pointer(uintptr(ptr) + 5) diff --git a/github.com/cockroachdb/pebble/sstable/reader.go b/github.com/cockroachdb/pebble/sstable/reader.go index 0ae5973267..49529feb15 100644 --- a/github.com/cockroachdb/pebble/sstable/reader.go +++ b/github.com/cockroachdb/pebble/sstable/reader.go @@ -35,8 +35,8 @@ const ( minFileReadsForReadahead = 2 // TODO(bilal): Have the initial size value be a factor of the block size, // as opposed to a hardcoded value. - initialReadaheadSize = 64 << 10 /* 64KB */ - maxReadaheadSize = 512 << 10 /* 512KB */ + initialReadaheadSize = 64 << 10 /* 64KB */ + maxReadaheadSize = 512 << 10 /* 512KB */ ) // decodeBlockHandle returns the block handle encoded at the start of src, as @@ -419,13 +419,17 @@ func (i *singleLevelIterator) skipForward() (*InternalKey, []byte) { i.data.invalidate() break } - if i.loadBlock() { - if key, val := i.data.First(); key != nil { - if i.blockUpper != nil && i.cmp(key.UserKey, i.blockUpper) >= 0 { - return nil, nil - } - return key, val + if !i.loadBlock() { + if i.err != nil { + break } + continue + } + if key, val := i.data.First(); key != nil { + if i.blockUpper != nil && i.cmp(key.UserKey, i.blockUpper) >= 0 { + return nil, nil + } + return key, val } } return nil, nil @@ -437,16 +441,20 @@ func (i *singleLevelIterator) skipBackward() (*InternalKey, []byte) { i.data.invalidate() break } - if i.loadBlock() { - key, val := i.data.Last() - if key == nil { - return nil, nil - } - if i.blockLower != nil && i.cmp(key.UserKey, i.blockLower) < 0 { - return nil, nil + if !i.loadBlock() { + if i.err != nil { + break } - return key, val + continue + } + key, val := i.data.Last() + if key == nil { + return nil, nil } + if i.blockLower != nil && i.cmp(key.UserKey, i.blockLower) < 0 { + return nil, nil + } + return key, val } return nil, nil } @@ -564,10 +572,14 @@ func (i *compactionIterator) skipForward(key *InternalKey, val []byte) (*Interna if key, _ := i.index.Next(); key == nil { break } - if i.loadBlock() { - if key, val = i.data.First(); key != nil { + if !i.loadBlock() { + if i.err != nil { break } + continue + } + if key, val = i.data.First(); key != nil { + break } } } @@ -783,6 +795,9 @@ func (i *twoLevelIterator) Prev() (*InternalKey, []byte) { func (i *twoLevelIterator) skipForward() (*InternalKey, []byte) { for { + if i.err != nil { + return nil, nil + } if i.singleLevelIterator.valid() { // The iterator is positioned at valid record in the current data block // which implies the previous positioning call reached the upper bound. @@ -804,6 +819,9 @@ func (i *twoLevelIterator) skipForward() (*InternalKey, []byte) { func (i *twoLevelIterator) skipBackward() (*InternalKey, []byte) { for { + if i.err != nil { + return nil, nil + } if i.singleLevelIterator.valid() { // The iterator is positioned at valid record in the current data block // which implies the previous positioning call reached the lower bound. @@ -963,7 +981,7 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 { // readahead may not be beneficial with a small readahead size, but over // time the readahead size would increase exponentially to make it // beneficial. - if currentReadEnd >= rs.limit && offset <= rs.limit + maxReadaheadSize { + if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize { // We are doing a read in the interval ahead of // the last readahead range. In the diagrams below, ++++ is the last // readahead range, ==== is the range represented by @@ -997,7 +1015,7 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 { } return rs.prevSize } - if currentReadEnd < rs.limit - rs.prevSize || offset > rs.limit + maxReadaheadSize { + if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+maxReadaheadSize { // The above conditional has rs.limit > rs.prevSize to confirm that // rs.limit - rs.prevSize would not underflow. // We read too far away from rs.limit to benefit from readahead in @@ -1039,7 +1057,7 @@ func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 { rs.numReads++ return 0 } - if currentReadEnd >= rs.limit && offset <= rs.limit + maxReadaheadSize { + if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize { // Blocks are being read sequentially and would benefit from readahead // down the line. // @@ -1328,13 +1346,15 @@ func (r *Reader) readRangeDel() (cache.Handle, error) { } // readBlock reads and decompresses a block from disk into memory. -func (r *Reader) readBlock(bh BlockHandle, transform blockTransform, raState *readaheadState) (cache.Handle, error) { +func (r *Reader) readBlock( + bh BlockHandle, transform blockTransform, raState *readaheadState, +) (cache.Handle, error) { if h := r.opts.Cache.Get(r.cacheID, r.fileNum, bh.Offset); h.Get() != nil { return h, nil } if raState != nil { - if readaheadSize := raState.maybeReadahead(int64(bh.Offset), int64(bh.Length + blockTrailerLen)); readaheadSize > 0 { + if readaheadSize := raState.maybeReadahead(int64(bh.Offset), int64(bh.Length+blockTrailerLen)); readaheadSize > 0 { _ = vfs.Prefetch(r.file, bh.Offset, uint64(readaheadSize)) } } @@ -1350,7 +1370,9 @@ func (r *Reader) readBlock(bh BlockHandle, transform blockTransform, raState *re checksum1 := crc.New(b[:bh.Length+1]).Value() if checksum0 != checksum1 { r.opts.Cache.Free(v) - return cache.Handle{}, errors.New("pebble/table: invalid table (checksum mismatch)") + return cache.Handle{}, errors.Newf( + "pebble/table: invalid table %s (checksum mismatch at %d/%d)", + errors.Safe(r.fileNum), errors.Safe(bh.Offset), errors.Safe(bh.Length)) } typ := b[bh.Length] diff --git a/github.com/cockroachdb/pebble/sstable/unsafe.go b/github.com/cockroachdb/pebble/sstable/unsafe.go index da9ef351a8..0030be0456 100644 --- a/github.com/cockroachdb/pebble/sstable/unsafe.go +++ b/github.com/cockroachdb/pebble/sstable/unsafe.go @@ -11,21 +11,20 @@ func getBytes(ptr unsafe.Pointer, length int) []byte { } func decodeVarint(ptr unsafe.Pointer) (uint32, unsafe.Pointer) { - src := (*[5]uint8)(ptr) - if a := (*src)[0]; a < 128 { + if a := *((*uint8)(ptr)); a < 128 { return uint32(a), unsafe.Pointer(uintptr(ptr) + 1) - } else if a, b := a&0x7f, (*src)[1]; b < 128 { + } else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 { return uint32(b)<<7 | uint32(a), unsafe.Pointer(uintptr(ptr) + 2) - } else if b, c := b&0x7f, (*src)[2]; c < 128 { + } else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 { return uint32(c)<<14 | uint32(b)<<7 | uint32(a), unsafe.Pointer(uintptr(ptr) + 3) - } else if c, d := c&0x7f, (*src)[3]; d < 128 { + } else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 { return uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a), unsafe.Pointer(uintptr(ptr) + 4) } else { - d, e := d&0x7f, (*src)[4] + d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4))) return uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a), unsafe.Pointer(uintptr(ptr) + 5) } diff --git a/golang.org/x/sys/unix/mkerrors.sh b/golang.org/x/sys/unix/mkerrors.sh index 780e387e3f..08f8230d6d 100644 --- a/golang.org/x/sys/unix/mkerrors.sh +++ b/golang.org/x/sys/unix/mkerrors.sh @@ -509,7 +509,7 @@ ccflags="$@" $2 ~ /^CAP_/ || $2 ~ /^ALG_/ || $2 ~ /^FS_(POLICY_FLAGS|KEY_DESC|ENCRYPTION_MODE|[A-Z0-9_]+_KEY_SIZE)/ || - $2 ~ /^FS_IOC_.*(ENCRYPTION|VERITY|GETFLAGS)/ || + $2 ~ /^FS_IOC_.*(ENCRYPTION|VERITY|[GS]ETFLAGS)/ || $2 ~ /^FS_VERITY_/ || $2 ~ /^FSCRYPT_/ || $2 ~ /^GRND_/ || diff --git a/golang.org/x/sys/unix/zerrors_linux_386.go b/golang.org/x/sys/unix/zerrors_linux_386.go index 8d207b041e..11b25f68c2 100644 --- a/golang.org/x/sys/unix/zerrors_linux_386.go +++ b/golang.org/x/sys/unix/zerrors_linux_386.go @@ -78,6 +78,7 @@ const ( FS_IOC_GET_ENCRYPTION_NONCE = 0x8010661b FS_IOC_GET_ENCRYPTION_POLICY = 0x400c6615 FS_IOC_GET_ENCRYPTION_PWSALT = 0x40106614 + FS_IOC_SETFLAGS = 0x40046602 FS_IOC_SET_ENCRYPTION_POLICY = 0x800c6613 F_GETLK = 0xc F_GETLK64 = 0xc diff --git a/golang.org/x/sys/unix/zerrors_linux_amd64.go b/golang.org/x/sys/unix/zerrors_linux_amd64.go index c4bf9cb80f..f92cff6ea0 100644 --- a/golang.org/x/sys/unix/zerrors_linux_amd64.go +++ b/golang.org/x/sys/unix/zerrors_linux_amd64.go @@ -78,6 +78,7 @@ const ( FS_IOC_GET_ENCRYPTION_NONCE = 0x8010661b FS_IOC_GET_ENCRYPTION_POLICY = 0x400c6615 FS_IOC_GET_ENCRYPTION_PWSALT = 0x40106614 + FS_IOC_SETFLAGS = 0x40086602 FS_IOC_SET_ENCRYPTION_POLICY = 0x800c6613 F_GETLK = 0x5 F_GETLK64 = 0x5 diff --git a/golang.org/x/sys/unix/zerrors_linux_arm.go b/golang.org/x/sys/unix/zerrors_linux_arm.go index 0cab0522e6..12bcbf88d6 100644 --- a/golang.org/x/sys/unix/zerrors_linux_arm.go +++ b/golang.org/x/sys/unix/zerrors_linux_arm.go @@ -77,6 +77,7 @@ const ( FS_IOC_GET_ENCRYPTION_NONCE = 0x8010661b FS_IOC_GET_ENCRYPTION_POLICY = 0x400c6615 FS_IOC_GET_ENCRYPTION_PWSALT = 0x40106614 + FS_IOC_SETFLAGS = 0x40046602 FS_IOC_SET_ENCRYPTION_POLICY = 0x800c6613 F_GETLK = 0xc F_GETLK64 = 0xc diff --git a/golang.org/x/sys/unix/zerrors_linux_arm64.go b/golang.org/x/sys/unix/zerrors_linux_arm64.go index 370d0a7f59..8b0e024b94 100644 --- a/golang.org/x/sys/unix/zerrors_linux_arm64.go +++ b/golang.org/x/sys/unix/zerrors_linux_arm64.go @@ -80,6 +80,7 @@ const ( FS_IOC_GET_ENCRYPTION_NONCE = 0x8010661b FS_IOC_GET_ENCRYPTION_POLICY = 0x400c6615 FS_IOC_GET_ENCRYPTION_PWSALT = 0x40106614 + FS_IOC_SETFLAGS = 0x40086602 FS_IOC_SET_ENCRYPTION_POLICY = 0x800c6613 F_GETLK = 0x5 F_GETLK64 = 0x5 diff --git a/golang.org/x/sys/unix/zerrors_linux_mips.go b/golang.org/x/sys/unix/zerrors_linux_mips.go index fbf2f3174e..eeadea943f 100644 --- a/golang.org/x/sys/unix/zerrors_linux_mips.go +++ b/golang.org/x/sys/unix/zerrors_linux_mips.go @@ -77,6 +77,7 @@ const ( FS_IOC_GET_ENCRYPTION_NONCE = 0x4010661b FS_IOC_GET_ENCRYPTION_POLICY = 0x800c6615 FS_IOC_GET_ENCRYPTION_PWSALT = 0x80106614 + FS_IOC_SETFLAGS = 0x80046602 FS_IOC_SET_ENCRYPTION_POLICY = 0x400c6613 F_GETLK = 0x21 F_GETLK64 = 0x21 diff --git a/golang.org/x/sys/unix/zerrors_linux_mips64.go b/golang.org/x/sys/unix/zerrors_linux_mips64.go index 25e74b30a9..0be6c4ccc0 100644 --- a/golang.org/x/sys/unix/zerrors_linux_mips64.go +++ b/golang.org/x/sys/unix/zerrors_linux_mips64.go @@ -77,6 +77,7 @@ const ( FS_IOC_GET_ENCRYPTION_NONCE = 0x4010661b FS_IOC_GET_ENCRYPTION_POLICY = 0x800c6615 FS_IOC_GET_ENCRYPTION_PWSALT = 0x80106614 + FS_IOC_SETFLAGS = 0x80086602 FS_IOC_SET_ENCRYPTION_POLICY = 0x400c6613 F_GETLK = 0xe F_GETLK64 = 0xe diff --git a/golang.org/x/sys/unix/zerrors_linux_mips64le.go b/golang.org/x/sys/unix/zerrors_linux_mips64le.go index 4ecc0bca34..0880b745c1 100644 --- a/golang.org/x/sys/unix/zerrors_linux_mips64le.go +++ b/golang.org/x/sys/unix/zerrors_linux_mips64le.go @@ -77,6 +77,7 @@ const ( FS_IOC_GET_ENCRYPTION_NONCE = 0x4010661b FS_IOC_GET_ENCRYPTION_POLICY = 0x800c6615 FS_IOC_GET_ENCRYPTION_PWSALT = 0x80106614 + FS_IOC_SETFLAGS = 0x80086602 FS_IOC_SET_ENCRYPTION_POLICY = 0x400c6613 F_GETLK = 0xe F_GETLK64 = 0xe diff --git a/golang.org/x/sys/unix/zerrors_linux_mipsle.go b/golang.org/x/sys/unix/zerrors_linux_mipsle.go index dfb8f88a7e..c8a66627aa 100644 --- a/golang.org/x/sys/unix/zerrors_linux_mipsle.go +++ b/golang.org/x/sys/unix/zerrors_linux_mipsle.go @@ -77,6 +77,7 @@ const ( FS_IOC_GET_ENCRYPTION_NONCE = 0x4010661b FS_IOC_GET_ENCRYPTION_POLICY = 0x800c6615 FS_IOC_GET_ENCRYPTION_PWSALT = 0x80106614 + FS_IOC_SETFLAGS = 0x80046602 FS_IOC_SET_ENCRYPTION_POLICY = 0x400c6613 F_GETLK = 0x21 F_GETLK64 = 0x21 diff --git a/golang.org/x/sys/unix/zerrors_linux_ppc64.go b/golang.org/x/sys/unix/zerrors_linux_ppc64.go index 72d8dad5b8..97aae63f16 100644 --- a/golang.org/x/sys/unix/zerrors_linux_ppc64.go +++ b/golang.org/x/sys/unix/zerrors_linux_ppc64.go @@ -77,6 +77,7 @@ const ( FS_IOC_GET_ENCRYPTION_NONCE = 0x4010661b FS_IOC_GET_ENCRYPTION_POLICY = 0x800c6615 FS_IOC_GET_ENCRYPTION_PWSALT = 0x80106614 + FS_IOC_SETFLAGS = 0x80086602 FS_IOC_SET_ENCRYPTION_POLICY = 0x400c6613 F_GETLK = 0x5 F_GETLK64 = 0xc diff --git a/golang.org/x/sys/unix/zerrors_linux_ppc64le.go b/golang.org/x/sys/unix/zerrors_linux_ppc64le.go index ca0e7b5262..b0c3b0664f 100644 --- a/golang.org/x/sys/unix/zerrors_linux_ppc64le.go +++ b/golang.org/x/sys/unix/zerrors_linux_ppc64le.go @@ -77,6 +77,7 @@ const ( FS_IOC_GET_ENCRYPTION_NONCE = 0x4010661b FS_IOC_GET_ENCRYPTION_POLICY = 0x800c6615 FS_IOC_GET_ENCRYPTION_PWSALT = 0x80106614 + FS_IOC_SETFLAGS = 0x80086602 FS_IOC_SET_ENCRYPTION_POLICY = 0x400c6613 F_GETLK = 0x5 F_GETLK64 = 0xc diff --git a/golang.org/x/sys/unix/zerrors_linux_riscv64.go b/golang.org/x/sys/unix/zerrors_linux_riscv64.go index 147511a974..0c05181935 100644 --- a/golang.org/x/sys/unix/zerrors_linux_riscv64.go +++ b/golang.org/x/sys/unix/zerrors_linux_riscv64.go @@ -77,6 +77,7 @@ const ( FS_IOC_GET_ENCRYPTION_NONCE = 0x8010661b FS_IOC_GET_ENCRYPTION_POLICY = 0x400c6615 FS_IOC_GET_ENCRYPTION_PWSALT = 0x40106614 + FS_IOC_SETFLAGS = 0x40086602 FS_IOC_SET_ENCRYPTION_POLICY = 0x800c6613 F_GETLK = 0x5 F_GETLK64 = 0x5 diff --git a/golang.org/x/sys/unix/zerrors_linux_s390x.go b/golang.org/x/sys/unix/zerrors_linux_s390x.go index 517349dafa..0b96bd462e 100644 --- a/golang.org/x/sys/unix/zerrors_linux_s390x.go +++ b/golang.org/x/sys/unix/zerrors_linux_s390x.go @@ -77,6 +77,7 @@ const ( FS_IOC_GET_ENCRYPTION_NONCE = 0x8010661b FS_IOC_GET_ENCRYPTION_POLICY = 0x400c6615 FS_IOC_GET_ENCRYPTION_PWSALT = 0x40106614 + FS_IOC_SETFLAGS = 0x40086602 FS_IOC_SET_ENCRYPTION_POLICY = 0x800c6613 F_GETLK = 0x5 F_GETLK64 = 0x5 diff --git a/golang.org/x/sys/unix/zerrors_linux_sparc64.go b/golang.org/x/sys/unix/zerrors_linux_sparc64.go index 094822465b..bd5c305779 100644 --- a/golang.org/x/sys/unix/zerrors_linux_sparc64.go +++ b/golang.org/x/sys/unix/zerrors_linux_sparc64.go @@ -81,6 +81,7 @@ const ( FS_IOC_GET_ENCRYPTION_NONCE = 0x4010661b FS_IOC_GET_ENCRYPTION_POLICY = 0x800c6615 FS_IOC_GET_ENCRYPTION_PWSALT = 0x80106614 + FS_IOC_SETFLAGS = 0x80086602 FS_IOC_SET_ENCRYPTION_POLICY = 0x400c6613 F_GETLK = 0x7 F_GETLK64 = 0x7 diff --git a/modules.txt b/modules.txt index 8541726460..b94f5edcee 100644 --- a/modules.txt +++ b/modules.txt @@ -185,7 +185,7 @@ github.com/cockroachdb/gostdlib/x/tools/internal/semver github.com/cockroachdb/gostdlib/x/tools/internal/span # github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f github.com/cockroachdb/logtags -# github.com/cockroachdb/pebble v0.0.0-20200624214307-560ed0b8a18c +# github.com/cockroachdb/pebble v0.0.0-20200626145914-3b86a4009c3b github.com/cockroachdb/pebble github.com/cockroachdb/pebble/bloom github.com/cockroachdb/pebble/internal/arenaskl @@ -672,7 +672,7 @@ golang.org/x/perf/storage/benchfmt # golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e golang.org/x/sync/errgroup golang.org/x/sync/syncmap -# golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4 +# golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae golang.org/x/sys/cpu golang.org/x/sys/internal/unsafeheader golang.org/x/sys/unix From 9c42e4cee71e0a680e9b05087bce111c96c4aa53 Mon Sep 17 00:00:00 2001 From: Oliver Tan Date: Fri, 26 Jun 2020 14:45:19 -0700 Subject: [PATCH 03/49] update vendor to go1.14 --- modules.txt | 178 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) diff --git a/modules.txt b/modules.txt index b94f5edcee..7071ed3265 100644 --- a/modules.txt +++ b/modules.txt @@ -1,4 +1,5 @@ # cloud.google.com/go v0.34.0 +## explicit cloud.google.com/go/compute/metadata cloud.google.com/go/iam cloud.google.com/go/internal @@ -9,6 +10,7 @@ cloud.google.com/go/storage # github.com/Azure/azure-pipeline-go v0.1.8 github.com/Azure/azure-pipeline-go/pipeline # github.com/Azure/azure-sdk-for-go v33.4.0+incompatible +## explicit github.com/Azure/azure-sdk-for-go/profiles/latest/compute/mgmt/compute github.com/Azure/azure-sdk-for-go/profiles/latest/network/mgmt/network github.com/Azure/azure-sdk-for-go/profiles/latest/resources/mgmt/resources @@ -19,34 +21,42 @@ github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2018-06-01/subscriptio github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-05-01/resources github.com/Azure/azure-sdk-for-go/version # github.com/Azure/azure-storage-blob-go v0.0.0-20190104215108-45d0c5e3638e +## explicit github.com/Azure/azure-storage-blob-go/azblob # github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 github.com/Azure/go-ansiterm github.com/Azure/go-ansiterm/winterm # github.com/Azure/go-autorest/autorest v0.10.2 +## explicit github.com/Azure/go-autorest/autorest github.com/Azure/go-autorest/autorest/azure # github.com/Azure/go-autorest/autorest/adal v0.8.2 github.com/Azure/go-autorest/autorest/adal # github.com/Azure/go-autorest/autorest/azure/auth v0.4.2 +## explicit github.com/Azure/go-autorest/autorest/azure/auth # github.com/Azure/go-autorest/autorest/azure/cli v0.3.1 github.com/Azure/go-autorest/autorest/azure/cli # github.com/Azure/go-autorest/autorest/date v0.2.0 github.com/Azure/go-autorest/autorest/date # github.com/Azure/go-autorest/autorest/to v0.3.0 +## explicit github.com/Azure/go-autorest/autorest/to # github.com/Azure/go-autorest/autorest/validation v0.2.0 +## explicit github.com/Azure/go-autorest/autorest/validation # github.com/Azure/go-autorest/logger v0.1.0 github.com/Azure/go-autorest/logger # github.com/Azure/go-autorest/tracing v0.5.0 github.com/Azure/go-autorest/tracing # github.com/BurntSushi/toml v0.3.1 +## explicit github.com/BurntSushi/toml # github.com/DataDog/zstd v1.4.4 +## explicit github.com/DataDog/zstd # github.com/MichaelTJones/walk v0.0.0-20161122175330-4748e29d5718 +## explicit github.com/MichaelTJones/walk # github.com/Microsoft/go-winio v0.4.14 github.com/Microsoft/go-winio @@ -54,23 +64,32 @@ github.com/Microsoft/go-winio/pkg/guid # github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 github.com/Nvveen/Gotty # github.com/PuerkitoBio/goquery v1.5.0 +## explicit github.com/PuerkitoBio/goquery # github.com/Shopify/sarama v1.22.2-0.20190604114437-cd910a683f9f +## explicit github.com/Shopify/sarama # github.com/Shopify/toxiproxy v2.1.4+incompatible +## explicit github.com/Shopify/toxiproxy/client # github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 +## explicit github.com/StackExchange/wmi # github.com/VividCortex/ewma v1.1.1 +## explicit github.com/VividCortex/ewma # github.com/abourget/teamcity v0.0.0-00010101000000-000000000000 => github.com/cockroachdb/teamcity v0.0.0-20180905144921-8ca25c33eb11 +## explicit github.com/abourget/teamcity # github.com/andy-kimball/arenaskl v0.0.0-20200617143215-f701008588b9 +## explicit github.com/andy-kimball/arenaskl github.com/andy-kimball/arenaskl/internal/fastrand # github.com/andybalholm/cascadia v1.2.0 +## explicit github.com/andybalholm/cascadia # github.com/apache/arrow/go/arrow v0.0.0-20200610220642-670890229854 +## explicit github.com/apache/arrow/go/arrow github.com/apache/arrow/go/arrow/array github.com/apache/arrow/go/arrow/bitutil @@ -80,10 +99,13 @@ github.com/apache/arrow/go/arrow/internal/cpu github.com/apache/arrow/go/arrow/internal/debug github.com/apache/arrow/go/arrow/memory # github.com/apache/thrift v0.0.0-20181211084444-2b7365c54f82 +## explicit github.com/apache/thrift/lib/go/thrift # github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e +## explicit github.com/armon/circbuf # github.com/aws/aws-sdk-go v1.16.19 +## explicit github.com/aws/aws-sdk-go/aws github.com/aws/aws-sdk-go/aws/awserr github.com/aws/aws-sdk-go/aws/awsutil @@ -121,33 +143,47 @@ github.com/aws/aws-sdk-go/service/s3/s3iface github.com/aws/aws-sdk-go/service/s3/s3manager github.com/aws/aws-sdk-go/service/sts # github.com/axiomhq/hyperloglog v0.0.0-20181223111420-4b99d0c2c99e +## explicit github.com/axiomhq/hyperloglog # github.com/benesch/cgosymbolizer v0.0.0-20180702220239-70e1ee2b39d3 +## explicit github.com/benesch/cgosymbolizer # github.com/beorn7/perks v1.0.1 github.com/beorn7/perks/quantile # github.com/biogo/store v0.0.0-20160505134755-913427a1d5e8 +## explicit github.com/biogo/store/llrb # github.com/cenkalti/backoff v2.1.1+incompatible +## explicit github.com/cenkalti/backoff # github.com/client9/misspell v0.3.4 +## explicit github.com/client9/misspell github.com/client9/misspell/cmd/misspell +# github.com/cockroachdb/apd v1.1.0 +## explicit # github.com/cockroachdb/apd/v2 v2.0.2 +## explicit github.com/cockroachdb/apd/v2 # github.com/cockroachdb/circuitbreaker v2.2.2-0.20190114160014-a614b14ccf63+incompatible +## explicit github.com/cockroachdb/circuitbreaker # github.com/cockroachdb/cmux v0.0.0-20170110192607-30d10be49292 +## explicit github.com/cockroachdb/cmux # github.com/cockroachdb/cockroach-go v0.0.0-20200504194139-73ffeee90b62 +## explicit github.com/cockroachdb/cockroach-go/crdb # github.com/cockroachdb/crlfmt v0.0.0-20200116191136-a78e1c207bc0 +## explicit github.com/cockroachdb/crlfmt github.com/cockroachdb/crlfmt/internal/parser github.com/cockroachdb/crlfmt/internal/render # github.com/cockroachdb/datadriven v0.0.0-20200212141702-aca09668cb24 +## explicit github.com/cockroachdb/datadriven # github.com/cockroachdb/errors v1.5.0 +## explicit github.com/cockroachdb/errors github.com/cockroachdb/errors/assert github.com/cockroachdb/errors/barriers @@ -167,8 +203,10 @@ github.com/cockroachdb/errors/telemetrykeys github.com/cockroachdb/errors/testutils github.com/cockroachdb/errors/withstack # github.com/cockroachdb/go-test-teamcity v0.0.0-20191211140407-cff980ad0a55 +## explicit github.com/cockroachdb/go-test-teamcity # github.com/cockroachdb/gostdlib v1.13.0 +## explicit github.com/cockroachdb/gostdlib/cmd/gofmt github.com/cockroachdb/gostdlib/go/format github.com/cockroachdb/gostdlib/go/printer @@ -184,8 +222,10 @@ github.com/cockroachdb/gostdlib/x/tools/internal/module github.com/cockroachdb/gostdlib/x/tools/internal/semver github.com/cockroachdb/gostdlib/x/tools/internal/span # github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f +## explicit github.com/cockroachdb/logtags # github.com/cockroachdb/pebble v0.0.0-20200626145914-3b86a4009c3b +## explicit github.com/cockroachdb/pebble github.com/cockroachdb/pebble/bloom github.com/cockroachdb/pebble/internal/arenaskl @@ -209,20 +249,27 @@ github.com/cockroachdb/pebble/sstable github.com/cockroachdb/pebble/tool github.com/cockroachdb/pebble/vfs # github.com/cockroachdb/redact v0.0.0-20200622112456-cd282804bbd3 +## explicit github.com/cockroachdb/redact # github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd +## explicit github.com/cockroachdb/returncheck # github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 +## explicit github.com/cockroachdb/sentry-go # github.com/cockroachdb/stress v0.0.0-20170808184505-29b5d31b4c3a +## explicit github.com/cockroachdb/stress # github.com/cockroachdb/ttycolor v0.0.0-20180709150743-a1d5aaeb377d +## explicit github.com/cockroachdb/ttycolor # github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd +## explicit github.com/codahale/hdrhistogram # github.com/cpuguy83/go-md2man v1.0.10 github.com/cpuguy83/go-md2man/md2man # github.com/dave/dst v0.24.0 +## explicit github.com/dave/dst github.com/dave/dst/decorator github.com/dave/dst/decorator/resolver @@ -234,13 +281,16 @@ github.com/davecgh/go-spew/spew # github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/dgrijalva/jwt-go # github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc +## explicit github.com/dgryski/go-metro # github.com/dimchansky/utfbom v1.1.0 github.com/dimchansky/utfbom # github.com/docker/distribution v2.7.0+incompatible +## explicit github.com/docker/distribution/digestset github.com/docker/distribution/reference # github.com/docker/docker v17.12.0-ce-rc1.0.20190115172544-0dc531243dd3+incompatible +## explicit github.com/docker/docker/api github.com/docker/docker/api/types github.com/docker/docker/api/types/blkiodev @@ -264,40 +314,56 @@ github.com/docker/docker/pkg/stdcopy github.com/docker/docker/pkg/term github.com/docker/docker/pkg/term/windows # github.com/docker/go-connections v0.4.0 +## explicit github.com/docker/go-connections/nat github.com/docker/go-connections/sockets github.com/docker/go-connections/tlsconfig # github.com/docker/go-units v0.4.0 +## explicit github.com/docker/go-units # github.com/dustin/go-humanize v1.0.0 +## explicit github.com/dustin/go-humanize # github.com/eapache/go-resiliency v1.2.0 +## explicit github.com/eapache/go-resiliency/breaker # github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 github.com/eapache/go-xerial-snappy # github.com/eapache/queue v1.1.0 github.com/eapache/queue # github.com/edsrzf/mmap-go v1.0.0 +## explicit github.com/edsrzf/mmap-go # github.com/elastic/gosigar v0.10.0 +## explicit github.com/elastic/gosigar github.com/elastic/gosigar/sys/windows # github.com/elazarl/go-bindata-assetfs v1.0.0 +## explicit github.com/elazarl/go-bindata-assetfs # github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a +## explicit github.com/facebookgo/clock +# github.com/frankban/quicktest v1.7.3 +## explicit # github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9 +## explicit github.com/ghemawat/stream # github.com/ghodss/yaml v1.0.0 github.com/ghodss/yaml # github.com/go-logfmt/logfmt v0.4.0 github.com/go-logfmt/logfmt # github.com/go-ole/go-ole v1.2.2 +## explicit github.com/go-ole/go-ole github.com/go-ole/go-ole/oleutil # github.com/go-sql-driver/mysql v1.4.1-0.20181218123637-c45f530f8e7f +## explicit github.com/go-sql-driver/mysql +# github.com/gofrs/uuid v3.3.0+incompatible +## explicit # github.com/gogo/protobuf v1.3.1 => github.com/cockroachdb/gogoproto v1.2.1-0.20190102194534-ca10b809dba0 +## explicit github.com/gogo/protobuf/gogoproto github.com/gogo/protobuf/jsonpb github.com/gogo/protobuf/plugin/compare @@ -327,16 +393,22 @@ github.com/gogo/protobuf/types github.com/gogo/protobuf/vanity github.com/gogo/protobuf/vanity/command # github.com/golang-commonmark/html v0.0.0-20180910111043-7d7c804e1d46 +## explicit github.com/golang-commonmark/html # github.com/golang-commonmark/linkify v0.0.0-20180910111149-f05efb453a0e +## explicit github.com/golang-commonmark/linkify # github.com/golang-commonmark/markdown v0.0.0-20180910011815-a8f139058164 +## explicit github.com/golang-commonmark/markdown # github.com/golang-commonmark/mdurl v0.0.0-20180910110917-8d018c6567d6 +## explicit github.com/golang-commonmark/mdurl # github.com/golang-commonmark/puny v0.0.0-20180910110745-050be392d8b8 +## explicit github.com/golang-commonmark/puny # github.com/golang/geo v0.0.0-20200319012246-673a6f80352d +## explicit github.com/golang/geo/r1 github.com/golang/geo/r2 github.com/golang/geo/r3 @@ -345,6 +417,7 @@ github.com/golang/geo/s2 # github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/glog # github.com/golang/protobuf v1.4.2 +## explicit github.com/golang/protobuf/descriptor github.com/golang/protobuf/jsonpb github.com/golang/protobuf/proto @@ -356,12 +429,16 @@ github.com/golang/protobuf/ptypes/duration github.com/golang/protobuf/ptypes/timestamp github.com/golang/protobuf/ptypes/wrappers # github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf +## explicit github.com/golang/snappy # github.com/google/btree v1.0.0 +## explicit github.com/google/btree # github.com/google/flatbuffers v1.11.0 +## explicit github.com/google/flatbuffers/go # github.com/google/go-cmp v0.4.0 +## explicit github.com/google/go-cmp/cmp github.com/google/go-cmp/cmp/cmpopts github.com/google/go-cmp/cmp/internal/diff @@ -369,10 +446,14 @@ github.com/google/go-cmp/cmp/internal/flags github.com/google/go-cmp/cmp/internal/function github.com/google/go-cmp/cmp/internal/value # github.com/google/go-github v17.0.0+incompatible +## explicit github.com/google/go-github/github # github.com/google/go-querystring v1.0.0 github.com/google/go-querystring/query +# github.com/google/martian v2.1.0+incompatible +## explicit # github.com/google/pprof v0.0.0-20190109223431-e84dfd68c163 +## explicit github.com/google/pprof/driver github.com/google/pprof/internal/binutils github.com/google/pprof/internal/driver @@ -389,14 +470,20 @@ github.com/google/pprof/third_party/d3 github.com/google/pprof/third_party/d3flamegraph github.com/google/pprof/third_party/svgpan # github.com/googleapis/gax-go v2.0.2+incompatible +## explicit github.com/googleapis/gax-go # github.com/gorhill/cronexpr v0.0.0-20140423231348-a557574d6c02 +## explicit github.com/gorhill/cronexpr +# github.com/gorilla/mux v1.7.4 +## explicit # github.com/gorilla/websocket v1.4.0 github.com/gorilla/websocket # github.com/goware/modvendor v0.3.0 +## explicit github.com/goware/modvendor # github.com/grpc-ecosystem/grpc-gateway v1.13.0 => github.com/cockroachdb/grpc-gateway v1.14.6-0.20200519165156-52697fc4a249 +## explicit github.com/grpc-ecosystem/grpc-gateway/codegenerator github.com/grpc-ecosystem/grpc-gateway/internal github.com/grpc-ecosystem/grpc-gateway/internal/casing @@ -408,16 +495,22 @@ github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway/internal/gengatew github.com/grpc-ecosystem/grpc-gateway/runtime github.com/grpc-ecosystem/grpc-gateway/utilities # github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 +## explicit github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc # github.com/hashicorp/go-uuid v1.0.2 +## explicit github.com/hashicorp/go-uuid # github.com/ianlancetaylor/cgosymbolizer v0.0.0-20170921033129-f5072df9c550 +## explicit github.com/ianlancetaylor/cgosymbolizer # github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 github.com/ianlancetaylor/demangle # github.com/inconshreveable/mousetrap v1.0.0 github.com/inconshreveable/mousetrap +# github.com/jackc/fake v0.0.0-20150926172116-812a484cc733 +## explicit # github.com/jackc/pgx v3.6.2+incompatible +## explicit github.com/jackc/pgx github.com/jackc/pgx/chunkreader github.com/jackc/pgx/internal/sanitize @@ -425,44 +518,57 @@ github.com/jackc/pgx/pgio github.com/jackc/pgx/pgproto3 github.com/jackc/pgx/pgtype # github.com/jaegertracing/jaeger v1.17.0 +## explicit github.com/jaegertracing/jaeger/model/json # github.com/jcmturner/gofork v1.0.0 +## explicit github.com/jcmturner/gofork/encoding/asn1 github.com/jcmturner/gofork/x/crypto/pbkdf2 # github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af github.com/jmespath/go-jmespath # github.com/kevinburke/go-bindata v3.13.0+incompatible +## explicit github.com/kevinburke/go-bindata github.com/kevinburke/go-bindata/go-bindata # github.com/kisielk/errcheck v1.2.0 +## explicit github.com/kisielk/errcheck github.com/kisielk/errcheck/internal/errcheck # github.com/kisielk/gotool v1.0.0 +## explicit github.com/kisielk/gotool github.com/kisielk/gotool/internal/load # github.com/knz/go-libedit v1.10.1 +## explicit github.com/knz/go-libedit github.com/knz/go-libedit/common github.com/knz/go-libedit/other github.com/knz/go-libedit/unix # github.com/knz/strtime v0.0.0-20200318182718-be999391ffa9 +## explicit github.com/knz/strtime # github.com/konsorten/go-windows-terminal-sequences v1.0.2 +## explicit github.com/konsorten/go-windows-terminal-sequences # github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 github.com/kr/logfmt # github.com/kr/pretty v0.2.0 +## explicit github.com/kr/pretty # github.com/kr/text v0.2.0 +## explicit github.com/kr/text # github.com/leanovate/gopter v0.2.5-0.20190402064358-634a59d12406 +## explicit github.com/leanovate/gopter github.com/leanovate/gopter/prop # github.com/lib/pq v1.5.2 +## explicit github.com/lib/pq github.com/lib/pq/oid github.com/lib/pq/scram # github.com/lightstep/lightstep-tracer-go v0.15.6 +## explicit github.com/lightstep/lightstep-tracer-go github.com/lightstep/lightstep-tracer-go/collectorpb github.com/lightstep/lightstep-tracer-go/lightstep/rand @@ -470,14 +576,23 @@ github.com/lightstep/lightstep-tracer-go/lightstep_thrift github.com/lightstep/lightstep-tracer-go/lightsteppb github.com/lightstep/lightstep-tracer-go/thrift_0_9_2/lib/go/thrift # github.com/linkedin/goavro/v2 v2.8.1 +## explicit github.com/linkedin/goavro/v2 # github.com/lufia/iostat v1.0.0 +## explicit github.com/lufia/iostat +# github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5 +## explicit +# github.com/lusis/slack-test v0.0.0-20190426140909-c40012f20018 +## explicit # github.com/maruel/panicparse v1.1.2-0.20180806203336-f20d4c4d746f +## explicit github.com/maruel/panicparse/stack # github.com/marusama/semaphore v0.0.0-20190110074507-6952cef993b2 +## explicit github.com/marusama/semaphore # github.com/mattn/go-isatty v0.0.9 +## explicit github.com/mattn/go-isatty # github.com/mattn/go-runewidth v0.0.7 github.com/mattn/go-runewidth @@ -485,10 +600,12 @@ github.com/mattn/go-runewidth github.com/mattn/go-zglob github.com/mattn/go-zglob/fastwalk # github.com/mattn/goveralls v0.0.2 +## explicit github.com/mattn/goveralls # github.com/matttproud/golang_protobuf_extensions v1.0.1 github.com/matttproud/golang_protobuf_extensions/pbutil # github.com/mibk/dupl v1.0.0 +## explicit github.com/mibk/dupl github.com/mibk/dupl/job github.com/mibk/dupl/output @@ -498,68 +615,93 @@ github.com/mibk/dupl/syntax/golang # github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/go-homedir # github.com/mitchellh/reflectwalk v1.0.0 +## explicit github.com/mitchellh/reflectwalk # github.com/mmatczuk/go_generics v0.0.0-20181212143635-0aaa050f9bab +## explicit github.com/mmatczuk/go_generics/cmd/go_generics github.com/mmatczuk/go_generics/globals # github.com/montanaflynn/stats v0.4.1-0.20190115100425-713f2944833c +## explicit github.com/montanaflynn/stats # github.com/nlopes/slack v0.4.0 +## explicit github.com/nlopes/slack github.com/nlopes/slack/slackutilsx # github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5 => github.com/cockroachdb/tablewriter v0.0.5-0.20200105123400-bd15540e8847 +## explicit github.com/olekukonko/tablewriter # github.com/opencontainers/go-digest v1.0.0-rc1 github.com/opencontainers/go-digest # github.com/opencontainers/image-spec v1.0.1 github.com/opencontainers/image-spec/specs-go github.com/opencontainers/image-spec/specs-go/v1 +# github.com/opennota/wd v0.0.0-20180911144301-b446539ab1e7 +## explicit # github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492 +## explicit github.com/opentracing-contrib/go-observer # github.com/opentracing/opentracing-go v1.1.0 +## explicit github.com/opentracing/opentracing-go github.com/opentracing/opentracing-go/ext github.com/opentracing/opentracing-go/log # github.com/openzipkin-contrib/zipkin-go-opentracing v0.3.5 +## explicit github.com/openzipkin-contrib/zipkin-go-opentracing github.com/openzipkin-contrib/zipkin-go-opentracing/flag github.com/openzipkin-contrib/zipkin-go-opentracing/thrift/gen-go/scribe github.com/openzipkin-contrib/zipkin-go-opentracing/thrift/gen-go/zipkincore github.com/openzipkin-contrib/zipkin-go-opentracing/types github.com/openzipkin-contrib/zipkin-go-opentracing/wire +# github.com/pborman/uuid v1.2.0 +## explicit +# github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea +## explicit # github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 +## explicit github.com/petermattis/goid # github.com/pierrec/lz4 v2.4.1+incompatible +## explicit github.com/pierrec/lz4 github.com/pierrec/lz4/internal/xxh32 # github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 +## explicit github.com/pkg/browser # github.com/pkg/errors v0.9.1 github.com/pkg/errors # github.com/pmezard/go-difflib v1.0.0 +## explicit github.com/pmezard/go-difflib/difflib # github.com/prometheus/client_golang v1.1.0 +## explicit github.com/prometheus/client_golang/prometheus github.com/prometheus/client_golang/prometheus/graphite github.com/prometheus/client_golang/prometheus/internal # github.com/prometheus/client_model v0.2.0 +## explicit github.com/prometheus/client_model/go # github.com/prometheus/common v0.9.1 +## explicit github.com/prometheus/common/expfmt github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg github.com/prometheus/common/model # github.com/prometheus/procfs v0.0.10 +## explicit github.com/prometheus/procfs github.com/prometheus/procfs/internal/fs github.com/prometheus/procfs/internal/util # github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 +## explicit github.com/rcrowley/go-metrics github.com/rcrowley/go-metrics/exp # github.com/russross/blackfriday v1.5.2 github.com/russross/blackfriday # github.com/sasha-s/go-deadlock v0.2.0 +## explicit github.com/sasha-s/go-deadlock # github.com/shirou/gopsutil v2.18.12+incompatible +## explicit github.com/shirou/gopsutil/cpu github.com/shirou/gopsutil/disk github.com/shirou/gopsutil/host @@ -569,18 +711,25 @@ github.com/shirou/gopsutil/mem github.com/shirou/gopsutil/net github.com/shirou/gopsutil/process # github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 +## explicit github.com/shirou/w32 +# github.com/shopspring/decimal v1.2.0 +## explicit # github.com/sirupsen/logrus v1.4.2 github.com/sirupsen/logrus # github.com/spf13/cobra v0.0.5 +## explicit github.com/spf13/cobra github.com/spf13/cobra/doc # github.com/spf13/pflag v1.0.5 +## explicit github.com/spf13/pflag # github.com/stretchr/testify v1.5.1 +## explicit github.com/stretchr/testify/assert github.com/stretchr/testify/require # github.com/twpayne/go-geom v1.3.0 +## explicit github.com/twpayne/go-geom github.com/twpayne/go-geom/bigxy github.com/twpayne/go-geom/encoding/ewkb @@ -600,8 +749,10 @@ github.com/twpayne/go-geom/xy/orientation # github.com/twpayne/go-kml v1.5.0 github.com/twpayne/go-kml # github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad +## explicit github.com/wadey/gocovmerge # go.etcd.io/etcd v0.0.0-00010101000000-000000000000 => github.com/cockroachdb/etcd v0.4.7-0.20200615211340-a17df30d5955 +## explicit go.etcd.io/etcd/raft go.etcd.io/etcd/raft/confchange go.etcd.io/etcd/raft/quorum @@ -623,6 +774,7 @@ go.opencensus.io/trace/internal go.opencensus.io/trace/propagation go.opencensus.io/trace/tracestate # golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 +## explicit golang.org/x/crypto/bcrypt golang.org/x/crypto/blowfish golang.org/x/crypto/chacha20 @@ -641,11 +793,14 @@ golang.org/x/crypto/ssh/internal/bcrypt_pbkdf golang.org/x/crypto/ssh/knownhosts golang.org/x/crypto/ssh/terminal # golang.org/x/exp v0.0.0-20200513190911-00229845015e +## explicit golang.org/x/exp/rand # golang.org/x/lint v0.0.0-20200130185559-910be7a94367 +## explicit golang.org/x/lint golang.org/x/lint/golint # golang.org/x/net v0.0.0-20200602114024-627f9648deb9 +## explicit golang.org/x/net/context golang.org/x/net/context/ctxhttp golang.org/x/net/html @@ -659,25 +814,30 @@ golang.org/x/net/internal/timeseries golang.org/x/net/proxy golang.org/x/net/trace # golang.org/x/oauth2 v0.0.0-20190115181402-5dab4167f31c +## explicit golang.org/x/oauth2 golang.org/x/oauth2/google golang.org/x/oauth2/internal golang.org/x/oauth2/jws golang.org/x/oauth2/jwt # golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852 +## explicit golang.org/x/perf/benchstat golang.org/x/perf/cmd/benchstat golang.org/x/perf/internal/stats golang.org/x/perf/storage/benchfmt # golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e +## explicit golang.org/x/sync/errgroup golang.org/x/sync/syncmap # golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae +## explicit golang.org/x/sys/cpu golang.org/x/sys/internal/unsafeheader golang.org/x/sys/unix golang.org/x/sys/windows # golang.org/x/text v0.3.2 +## explicit golang.org/x/text/cases golang.org/x/text/collate golang.org/x/text/internal @@ -694,8 +854,10 @@ golang.org/x/text/unicode/bidi golang.org/x/text/unicode/norm golang.org/x/text/width # golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 +## explicit golang.org/x/time/rate # golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5 +## explicit golang.org/x/tools/cmd/goyacc golang.org/x/tools/cmd/stringer golang.org/x/tools/container/intsets @@ -759,6 +921,7 @@ golang.org/x/tools/txtar golang.org/x/xerrors golang.org/x/xerrors/internal # google.golang.org/api v0.1.0 +## explicit google.golang.org/api/gensupport google.golang.org/api/googleapi google.golang.org/api/googleapi/internal/uritemplates @@ -782,6 +945,7 @@ google.golang.org/appengine/internal/remote_api google.golang.org/appengine/internal/urlfetch google.golang.org/appengine/urlfetch # google.golang.org/genproto v0.0.0-20200218151345-dad8c97a84f5 +## explicit google.golang.org/genproto/googleapis/api/annotations google.golang.org/genproto/googleapis/api/httpbody google.golang.org/genproto/googleapis/iam/v1 @@ -790,6 +954,7 @@ google.golang.org/genproto/googleapis/rpc/status google.golang.org/genproto/googleapis/type/expr google.golang.org/genproto/protobuf/field_mask # google.golang.org/grpc v1.29.1 +## explicit google.golang.org/grpc google.golang.org/grpc/attributes google.golang.org/grpc/backoff @@ -872,7 +1037,10 @@ google.golang.org/protobuf/types/pluginpb gopkg.in/jcmturner/aescts.v1 # gopkg.in/jcmturner/dnsutils.v1 v1.0.1 gopkg.in/jcmturner/dnsutils.v1 +# gopkg.in/jcmturner/goidentity.v3 v3.0.0 +## explicit # gopkg.in/jcmturner/gokrb5.v7 v7.5.0 +## explicit gopkg.in/jcmturner/gokrb5.v7/asn1tools gopkg.in/jcmturner/gokrb5.v7/client gopkg.in/jcmturner/gokrb5.v7/config @@ -907,8 +1075,10 @@ gopkg.in/jcmturner/gokrb5.v7/types gopkg.in/jcmturner/rpc.v1/mstypes gopkg.in/jcmturner/rpc.v1/ndr # gopkg.in/yaml.v2 v2.3.0 => github.com/cockroachdb/yaml v0.0.0-20180705215940-0e2822948641 +## explicit gopkg.in/yaml.v2 # honnef.co/go/tools v0.0.0-20190530104931-1f0868a609b7 +## explicit honnef.co/go/tools/arg honnef.co/go/tools/cmd/staticcheck honnef.co/go/tools/config @@ -935,6 +1105,7 @@ honnef.co/go/tools/stylecheck honnef.co/go/tools/unused honnef.co/go/tools/version # vitess.io/vitess v0.0.0-00010101000000-000000000000 => github.com/cockroachdb/vitess v2.2.0-rc.1.0.20180830030426-1740ce8b3188+incompatible +## explicit vitess.io/vitess/go/bytes2 vitess.io/vitess/go/hack vitess.io/vitess/go/sqltypes @@ -944,3 +1115,10 @@ vitess.io/vitess/go/vt/proto/vtgate vitess.io/vitess/go/vt/proto/vtrpc vitess.io/vitess/go/vt/sqlparser vitess.io/vitess/go/vt/vterrors +# github.com/gogo/protobuf => github.com/cockroachdb/gogoproto v1.2.1-0.20190102194534-ca10b809dba0 +# github.com/grpc-ecosystem/grpc-gateway => github.com/cockroachdb/grpc-gateway v1.14.6-0.20200519165156-52697fc4a249 +# github.com/olekukonko/tablewriter => github.com/cockroachdb/tablewriter v0.0.5-0.20200105123400-bd15540e8847 +# github.com/abourget/teamcity => github.com/cockroachdb/teamcity v0.0.0-20180905144921-8ca25c33eb11 +# vitess.io/vitess => github.com/cockroachdb/vitess v2.2.0-rc.1.0.20180830030426-1740ce8b3188+incompatible +# gopkg.in/yaml.v2 => github.com/cockroachdb/yaml v0.0.0-20180705215940-0e2822948641 +# go.etcd.io/etcd => github.com/cockroachdb/etcd v0.4.7-0.20200615211340-a17df30d5955 From 51544aebe7ad27d8fe0aebdbf92f860ed3d76669 Mon Sep 17 00:00:00 2001 From: Jordan Lewis Date: Tue, 30 Jun 2020 14:56:18 -0400 Subject: [PATCH 04/49] add jordanlewis/gcassert This upgrades testify and go/analysis. --- github.com/jordanlewis/gcassert/.gitignore | 2 + github.com/jordanlewis/gcassert/README.md | 83 + github.com/jordanlewis/gcassert/gcassert.go | 246 ++ github.com/jordanlewis/gcassert/go.mod | 8 + github.com/jordanlewis/gcassert/go.sum | 31 + github.com/stretchr/testify/LICENSE | 2 +- ...ssertion_order.go => assertion_compare.go} | 173 +- .../testify/assert/assertion_format.go | 40 +- .../testify/assert/assertion_forward.go | 62 +- .../stretchr/testify/assert/assertions.go | 191 +- .../testify/assert/http_assertions.go | 25 +- .../stretchr/testify/require/require.go | 74 +- .../testify/require/require_forward.go | 62 +- golang.org/x/mod/LICENSE | 27 + golang.org/x/mod/PATENTS | 22 + golang.org/x/mod/semver/semver.go | 388 +++ golang.org/x/tools/go/analysis/doc.go | 9 + .../go/analysis/internal/checker/checker.go | 9 +- .../tools/go/analysis/passes/printf/printf.go | 1 + .../x/tools/go/gcexportdata/gcexportdata.go | 12 +- .../x/tools/go/internal/gcimporter/bexport.go | 852 +++++ .../x/tools/go/internal/gcimporter/bimport.go | 1039 ++++++ .../go/internal/gcimporter/gcimporter.go | 11 +- .../x/tools/go/internal/gcimporter/iexport.go | 20 - .../x/tools/go/internal/gcimporter/iimport.go | 170 - golang.org/x/tools/go/packages/golist.go | 24 +- .../x/tools/go/packages/golist_overlay.go | 59 +- golang.org/x/tools/go/packages/packages.go | 69 +- .../internal/analysisinternal/analysis.go | 11 +- .../x/tools/internal/gocommand/invoke.go | 129 +- .../x/tools/internal/gocommand/vendor.go | 102 + .../internal/packagesinternal/packages.go | 25 +- golang.org/x/tools/internal/span/uri.go | 6 +- .../x/tools/internal/testenv/testenv.go | 90 + .../x/tools/internal/typesinternal/types.go | 28 + gopkg.in/yaml.v3/.travis.yml | 16 + gopkg.in/yaml.v3/LICENSE | 50 + gopkg.in/yaml.v3/NOTICE | 13 + gopkg.in/yaml.v3/README.md | 150 + gopkg.in/yaml.v3/apic.go | 746 ++++ gopkg.in/yaml.v3/decode.go | 931 +++++ gopkg.in/yaml.v3/emitterc.go | 1992 +++++++++++ gopkg.in/yaml.v3/encode.go | 561 +++ gopkg.in/yaml.v3/go.mod | 5 + gopkg.in/yaml.v3/parserc.go | 1229 +++++++ gopkg.in/yaml.v3/readerc.go | 434 +++ gopkg.in/yaml.v3/resolve.go | 326 ++ gopkg.in/yaml.v3/scannerc.go | 3025 +++++++++++++++++ gopkg.in/yaml.v3/sorter.go | 134 + gopkg.in/yaml.v3/writerc.go | 48 + gopkg.in/yaml.v3/yaml.go | 662 ++++ gopkg.in/yaml.v3/yamlh.go | 805 +++++ gopkg.in/yaml.v3/yamlprivateh.go | 198 ++ modules.txt | 13 +- 54 files changed, 14936 insertions(+), 504 deletions(-) create mode 100644 github.com/jordanlewis/gcassert/.gitignore create mode 100644 github.com/jordanlewis/gcassert/README.md create mode 100644 github.com/jordanlewis/gcassert/gcassert.go create mode 100644 github.com/jordanlewis/gcassert/go.mod create mode 100644 github.com/jordanlewis/gcassert/go.sum rename github.com/stretchr/testify/assert/{assertion_order.go => assertion_compare.go} (62%) create mode 100644 golang.org/x/mod/LICENSE create mode 100644 golang.org/x/mod/PATENTS create mode 100644 golang.org/x/mod/semver/semver.go create mode 100644 golang.org/x/tools/go/internal/gcimporter/bexport.go create mode 100644 golang.org/x/tools/go/internal/gcimporter/bimport.go create mode 100644 golang.org/x/tools/internal/gocommand/vendor.go create mode 100644 golang.org/x/tools/internal/typesinternal/types.go create mode 100644 gopkg.in/yaml.v3/.travis.yml create mode 100644 gopkg.in/yaml.v3/LICENSE create mode 100644 gopkg.in/yaml.v3/NOTICE create mode 100644 gopkg.in/yaml.v3/README.md create mode 100644 gopkg.in/yaml.v3/apic.go create mode 100644 gopkg.in/yaml.v3/decode.go create mode 100644 gopkg.in/yaml.v3/emitterc.go create mode 100644 gopkg.in/yaml.v3/encode.go create mode 100644 gopkg.in/yaml.v3/go.mod create mode 100644 gopkg.in/yaml.v3/parserc.go create mode 100644 gopkg.in/yaml.v3/readerc.go create mode 100644 gopkg.in/yaml.v3/resolve.go create mode 100644 gopkg.in/yaml.v3/scannerc.go create mode 100644 gopkg.in/yaml.v3/sorter.go create mode 100644 gopkg.in/yaml.v3/writerc.go create mode 100644 gopkg.in/yaml.v3/yaml.go create mode 100644 gopkg.in/yaml.v3/yamlh.go create mode 100644 gopkg.in/yaml.v3/yamlprivateh.go diff --git a/github.com/jordanlewis/gcassert/.gitignore b/github.com/jordanlewis/gcassert/.gitignore new file mode 100644 index 0000000000..0f6ec8e845 --- /dev/null +++ b/github.com/jordanlewis/gcassert/.gitignore @@ -0,0 +1,2 @@ +.idea +gcassert diff --git a/github.com/jordanlewis/gcassert/README.md b/github.com/jordanlewis/gcassert/README.md new file mode 100644 index 0000000000..16e1af0191 --- /dev/null +++ b/github.com/jordanlewis/gcassert/README.md @@ -0,0 +1,83 @@ +# gcassert + +gcassert is a program for making assertions about compiler decisions in +Golang programs, via inline comment directives like `//gcassert:inline`. + +## Example + +Given a file `foo.go`: + +``` +package foo + +func addOne(i int) int { + return i+1 +} + +func a(ints []int) int { + var sum int + for i := range ints { + //gcassert:bce,inline + sum += addOne(ints[i]) + + sum += ints[i] //gcassert:bce + } + return sum +} +``` + +The inline `//gcassert` directive will cause `gcassert` to fail if the line +`sum += addOne(ints[i])` is either not inlined or contains bounds checks. + +`//gcassert` comments expect a comma-separated list of directives after +`//gcassert:`. They can be included above the line in question or after, as an +inline comment. + +## Installation + +``` +go get github.com/jordanlewis/gcassert/cmd/gcassert +``` + +## Usage + +Run gcassert on packages containing gcassert directives, like this: + +``` +gcassert ./package/path +``` + +The program will output all lines that had a gcassert directive that wasn't +respected by the compiler. + +For example, running on the testdata directory in this library will produce the +following output: + +``` +$ gcassert ./testdata +testdata/bce.go:8: fmt.Println(ints[5]): Found IsInBounds +testdata/bce.go:16: sum += notInlinable(ints[i]): call was not inlined +testdata/inline.go:22: sum += notInlinable(i): call was not inlined +``` + +Inspecting each of the listed lines will show a `//gcassert` directive +that wasn't upheld when running the compiler on the package. + +## Directives + + +``` +//gcassert:inline +``` + +The inline directive asserts that the following statement contains a function +that is inlined by the compiler. If the function does not get inlined, gcassert +will fail. + +``` +//gcassert:bce +``` + +The bce directive asserts that the following statement contains a slice index +that has no necessary bounds checks. If the compiler adds bounds checks, +gcassert will fail. diff --git a/github.com/jordanlewis/gcassert/gcassert.go b/github.com/jordanlewis/gcassert/gcassert.go new file mode 100644 index 0000000000..ceb93d58fd --- /dev/null +++ b/github.com/jordanlewis/gcassert/gcassert.go @@ -0,0 +1,246 @@ +// Copyright 2020 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package gcassert + +import ( + "bufio" + "errors" + "fmt" + "go/ast" + "go/printer" + "go/token" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + + "golang.org/x/tools/go/packages" +) + +type assertDirective int + +const ( + noDirective assertDirective = iota + inline + bce +) + +func stringToDirective(s string) (assertDirective, error) { + switch s { + case "inline": + return inline, nil + case "bce": + return bce, nil + } + return noDirective, errors.New(fmt.Sprintf("no such directive %s", s)) +} + +type lineInfo struct { + n ast.Node + directives []assertDirective + // passedDirective is a map from index into the directives slice to a + // boolean that says whether or not the directive succeeded, in the case + // of directives like inlining that have compiler output if they passed. + // For directives like bce that have compiler output if they failed, there's + // no entry in this map. + passedDirective map[int]bool +} + +var gcAssertRegex = regexp.MustCompile(`//gcassert:([\w,]+)`) + +type assertVisitor struct { + commentMap ast.CommentMap + + directiveMap map[int]lineInfo + fileSet *token.FileSet +} + +func newAssertVisitor(commentMap ast.CommentMap, fileSet *token.FileSet) assertVisitor { + return assertVisitor{ + commentMap: commentMap, + fileSet: fileSet, + directiveMap: make(map[int]lineInfo), + } +} + +func (v assertVisitor) Visit(node ast.Node) (w ast.Visitor) { + if node == nil { + return w + } + m := v.commentMap[node] +COMMENTLOOP: + for _, g := range m { + for _, c := range g.List { + matches := gcAssertRegex.FindStringSubmatch(c.Text) + if len(matches) == 0 { + continue COMMENTLOOP + } + // The 0th match is the whole string, and the 1st match is the + // gcassert directive(s). + directiveStrings := strings.Split(matches[1], ",") + + pos := node.Pos() + lineNumber := v.fileSet.Position(pos).Line + lineInfo := v.directiveMap[lineNumber] + lineInfo.n = node + for _, s := range directiveStrings { + directive, err := stringToDirective(s) + if err != nil { + continue + } + lineInfo.directives = append(lineInfo.directives, directive) + } + v.directiveMap[lineNumber] = lineInfo + } + } + return v +} + +// GCAssert searches through the packages at the input path and writes failures +// to comply with //gcassert directives to the given io.Writer. +func GCAssert(path string, w io.Writer) error { + fileSet := token.NewFileSet() + pkgs, err := packages.Load(&packages.Config{ + Mode: packages.NeedName | packages.NeedFiles | packages.NeedSyntax | packages.NeedCompiledGoFiles, + Fset: fileSet, + }, path) + directiveMap, err := parseDirectives(pkgs, fileSet) + if err != nil { + return err + } + + // Next: invoke Go compiler with -m flags to get the compiler to print + // its optimization decisions. + + args := append([]string{"build", "-gcflags=-m -m -d=ssa/check_bce/debug=1"}, "./"+path) + cmd := exec.Command("go", args...) + cwd, err := os.Getwd() + if err != nil { + return err + } + cmd.Dir = cwd + pr, pw := io.Pipe() + cmd.Stdout = pw + cmd.Stderr = pw + cmdErr := make(chan error, 1) + go func() { + cmdErr <- cmd.Run() + pw.Close() + }() + + scanner := bufio.NewScanner(pr) + optInfo := regexp.MustCompile(`([\.\/\w]+):(\d+):\d+: (.*)`) + boundsCheck := "Found IsInBounds" + sliceBoundsCheck := "Found SliceIsInBounds" + + for scanner.Scan() { + line := scanner.Text() + matches := optInfo.FindStringSubmatch(line) + if len(matches) != 0 { + path := matches[1] + lineNo, err := strconv.Atoi(matches[2]) + if err != nil { + return err + } + message := matches[3] + + absPath, err := filepath.Abs(path) + if err != nil { + return err + } + if lineToDirectives := directiveMap[absPath]; lineToDirectives != nil { + info := lineToDirectives[lineNo] + if info.passedDirective == nil { + info.passedDirective = make(map[int]bool) + lineToDirectives[lineNo] = info + } + for i, d := range info.directives { + switch d { + case bce: + if message == boundsCheck || message == sliceBoundsCheck { + // Error! We found a bounds check where the user expected + // there to be none. + // Print out the user's code lineNo that failed the assertion, + // the assertion itself, and the compiler output that + // proved that the assertion failed. + if err := printAssertionFailure(cwd, fileSet, info, w, message); err != nil { + return err + } + } + case inline: + if strings.HasPrefix(message, "inlining call to") { + info.passedDirective[i] = true + } + } + } + } + } + } + + for _, lineToDirectives := range directiveMap { + for _, info := range lineToDirectives { + for i, d := range info.directives { + // An inlining directive passes if it has compiler output. For + // each inlining directive, check if there was matching compiler + // output and fail if not. + if d == inline { + if !info.passedDirective[i] { + if err := printAssertionFailure( + cwd, fileSet, info, w, "call was not inlined"); err != nil { + return err + } + } + } + } + } + } + return nil +} + +func printAssertionFailure(cwd string, fileSet *token.FileSet, info lineInfo, w io.Writer, message string) error { + var buf strings.Builder + _ = printer.Fprint(&buf, fileSet, info.n) + pos := fileSet.Position(info.n.Pos()) + relPath, err := filepath.Rel(cwd, pos.Filename) + if err != nil { + return err + } + fmt.Fprintf(w, "%s:%d:\t%s: %s\n", relPath, pos.Line, buf.String(), message) + return nil +} + +type directiveMap map[string]map[int]lineInfo + +func parseDirectives(pkgs []*packages.Package, fileSet *token.FileSet) (directiveMap, error) { + fileDirectiveMap := make(directiveMap) + for _, pkg := range pkgs { + for i, file := range pkg.Syntax { + commentMap := ast.NewCommentMap(fileSet, file, file.Comments) + + v := newAssertVisitor(commentMap, fileSet) + // First: find all lines of code annotated with our gcassert directives. + ast.Walk(v, file) + + file := pkg.CompiledGoFiles[i] + idx := strings.LastIndex(file, pkg.PkgPath) + if idx == -1 { + return nil, fmt.Errorf("couldn't find file %s in package %s", file, pkg.PkgPath) + } + if len(v.directiveMap) > 0 { + fileDirectiveMap[file] = v.directiveMap + } + } + } + return fileDirectiveMap, nil +} diff --git a/github.com/jordanlewis/gcassert/go.mod b/github.com/jordanlewis/gcassert/go.mod new file mode 100644 index 0000000000..98c57315c3 --- /dev/null +++ b/github.com/jordanlewis/gcassert/go.mod @@ -0,0 +1,8 @@ +module github.com/jordanlewis/gcassert + +go 1.13 + +require ( + github.com/stretchr/testify v1.6.1 + golang.org/x/tools v0.0.0-20200626171337-aa94e735be7f +) diff --git a/github.com/jordanlewis/gcassert/go.sum b/github.com/jordanlewis/gcassert/go.sum new file mode 100644 index 0000000000..46cfcb564f --- /dev/null +++ b/github.com/jordanlewis/gcassert/go.sum @@ -0,0 +1,31 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200626171337-aa94e735be7f h1:JcoF/bowzCDI+MXu1yLqQGNO3ibqWsWq+Sk7pOT218w= +golang.org/x/tools v0.0.0-20200626171337-aa94e735be7f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/github.com/stretchr/testify/LICENSE b/github.com/stretchr/testify/LICENSE index f38ec5956b..4b0421cf9e 100644 --- a/github.com/stretchr/testify/LICENSE +++ b/github.com/stretchr/testify/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2012-2018 Mat Ryer and Tyler Bunnell +Copyright (c) 2012-2020 Mat Ryer, Tyler Bunnell and contributors. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/github.com/stretchr/testify/assert/assertion_order.go b/github.com/stretchr/testify/assert/assertion_compare.go similarity index 62% rename from github.com/stretchr/testify/assert/assertion_order.go rename to github.com/stretchr/testify/assert/assertion_compare.go index 15a486ca6e..dc200395ce 100644 --- a/github.com/stretchr/testify/assert/assertion_order.go +++ b/github.com/stretchr/testify/assert/assertion_compare.go @@ -5,20 +5,28 @@ import ( "reflect" ) -func compare(obj1, obj2 interface{}, kind reflect.Kind) (int, bool) { +type CompareType int + +const ( + compareLess CompareType = iota - 1 + compareEqual + compareGreater +) + +func compare(obj1, obj2 interface{}, kind reflect.Kind) (CompareType, bool) { switch kind { case reflect.Int: { intobj1 := obj1.(int) intobj2 := obj2.(int) if intobj1 > intobj2 { - return -1, true + return compareGreater, true } if intobj1 == intobj2 { - return 0, true + return compareEqual, true } if intobj1 < intobj2 { - return 1, true + return compareLess, true } } case reflect.Int8: @@ -26,13 +34,13 @@ func compare(obj1, obj2 interface{}, kind reflect.Kind) (int, bool) { int8obj1 := obj1.(int8) int8obj2 := obj2.(int8) if int8obj1 > int8obj2 { - return -1, true + return compareGreater, true } if int8obj1 == int8obj2 { - return 0, true + return compareEqual, true } if int8obj1 < int8obj2 { - return 1, true + return compareLess, true } } case reflect.Int16: @@ -40,13 +48,13 @@ func compare(obj1, obj2 interface{}, kind reflect.Kind) (int, bool) { int16obj1 := obj1.(int16) int16obj2 := obj2.(int16) if int16obj1 > int16obj2 { - return -1, true + return compareGreater, true } if int16obj1 == int16obj2 { - return 0, true + return compareEqual, true } if int16obj1 < int16obj2 { - return 1, true + return compareLess, true } } case reflect.Int32: @@ -54,13 +62,13 @@ func compare(obj1, obj2 interface{}, kind reflect.Kind) (int, bool) { int32obj1 := obj1.(int32) int32obj2 := obj2.(int32) if int32obj1 > int32obj2 { - return -1, true + return compareGreater, true } if int32obj1 == int32obj2 { - return 0, true + return compareEqual, true } if int32obj1 < int32obj2 { - return 1, true + return compareLess, true } } case reflect.Int64: @@ -68,13 +76,13 @@ func compare(obj1, obj2 interface{}, kind reflect.Kind) (int, bool) { int64obj1 := obj1.(int64) int64obj2 := obj2.(int64) if int64obj1 > int64obj2 { - return -1, true + return compareGreater, true } if int64obj1 == int64obj2 { - return 0, true + return compareEqual, true } if int64obj1 < int64obj2 { - return 1, true + return compareLess, true } } case reflect.Uint: @@ -82,13 +90,13 @@ func compare(obj1, obj2 interface{}, kind reflect.Kind) (int, bool) { uintobj1 := obj1.(uint) uintobj2 := obj2.(uint) if uintobj1 > uintobj2 { - return -1, true + return compareGreater, true } if uintobj1 == uintobj2 { - return 0, true + return compareEqual, true } if uintobj1 < uintobj2 { - return 1, true + return compareLess, true } } case reflect.Uint8: @@ -96,13 +104,13 @@ func compare(obj1, obj2 interface{}, kind reflect.Kind) (int, bool) { uint8obj1 := obj1.(uint8) uint8obj2 := obj2.(uint8) if uint8obj1 > uint8obj2 { - return -1, true + return compareGreater, true } if uint8obj1 == uint8obj2 { - return 0, true + return compareEqual, true } if uint8obj1 < uint8obj2 { - return 1, true + return compareLess, true } } case reflect.Uint16: @@ -110,13 +118,13 @@ func compare(obj1, obj2 interface{}, kind reflect.Kind) (int, bool) { uint16obj1 := obj1.(uint16) uint16obj2 := obj2.(uint16) if uint16obj1 > uint16obj2 { - return -1, true + return compareGreater, true } if uint16obj1 == uint16obj2 { - return 0, true + return compareEqual, true } if uint16obj1 < uint16obj2 { - return 1, true + return compareLess, true } } case reflect.Uint32: @@ -124,13 +132,13 @@ func compare(obj1, obj2 interface{}, kind reflect.Kind) (int, bool) { uint32obj1 := obj1.(uint32) uint32obj2 := obj2.(uint32) if uint32obj1 > uint32obj2 { - return -1, true + return compareGreater, true } if uint32obj1 == uint32obj2 { - return 0, true + return compareEqual, true } if uint32obj1 < uint32obj2 { - return 1, true + return compareLess, true } } case reflect.Uint64: @@ -138,13 +146,13 @@ func compare(obj1, obj2 interface{}, kind reflect.Kind) (int, bool) { uint64obj1 := obj1.(uint64) uint64obj2 := obj2.(uint64) if uint64obj1 > uint64obj2 { - return -1, true + return compareGreater, true } if uint64obj1 == uint64obj2 { - return 0, true + return compareEqual, true } if uint64obj1 < uint64obj2 { - return 1, true + return compareLess, true } } case reflect.Float32: @@ -152,13 +160,13 @@ func compare(obj1, obj2 interface{}, kind reflect.Kind) (int, bool) { float32obj1 := obj1.(float32) float32obj2 := obj2.(float32) if float32obj1 > float32obj2 { - return -1, true + return compareGreater, true } if float32obj1 == float32obj2 { - return 0, true + return compareEqual, true } if float32obj1 < float32obj2 { - return 1, true + return compareLess, true } } case reflect.Float64: @@ -166,13 +174,13 @@ func compare(obj1, obj2 interface{}, kind reflect.Kind) (int, bool) { float64obj1 := obj1.(float64) float64obj2 := obj2.(float64) if float64obj1 > float64obj2 { - return -1, true + return compareGreater, true } if float64obj1 == float64obj2 { - return 0, true + return compareEqual, true } if float64obj1 < float64obj2 { - return 1, true + return compareLess, true } } case reflect.String: @@ -180,18 +188,18 @@ func compare(obj1, obj2 interface{}, kind reflect.Kind) (int, bool) { stringobj1 := obj1.(string) stringobj2 := obj2.(string) if stringobj1 > stringobj2 { - return -1, true + return compareGreater, true } if stringobj1 == stringobj2 { - return 0, true + return compareEqual, true } if stringobj1 < stringobj2 { - return 1, true + return compareLess, true } } } - return 0, false + return compareEqual, false } // Greater asserts that the first element is greater than the second @@ -200,26 +208,7 @@ func compare(obj1, obj2 interface{}, kind reflect.Kind) (int, bool) { // assert.Greater(t, float64(2), float64(1)) // assert.Greater(t, "b", "a") func Greater(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - - e1Kind := reflect.ValueOf(e1).Kind() - e2Kind := reflect.ValueOf(e2).Kind() - if e1Kind != e2Kind { - return Fail(t, "Elements should be the same type", msgAndArgs...) - } - - res, isComparable := compare(e1, e2, e1Kind) - if !isComparable { - return Fail(t, fmt.Sprintf("Can not compare type \"%s\"", reflect.TypeOf(e1)), msgAndArgs...) - } - - if res != -1 { - return Fail(t, fmt.Sprintf("\"%v\" is not greater than \"%v\"", e1, e2), msgAndArgs...) - } - - return true + return compareTwoValues(t, e1, e2, []CompareType{compareGreater}, "\"%v\" is not greater than \"%v\"", msgAndArgs) } // GreaterOrEqual asserts that the first element is greater than or equal to the second @@ -229,26 +218,7 @@ func Greater(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface // assert.GreaterOrEqual(t, "b", "a") // assert.GreaterOrEqual(t, "b", "b") func GreaterOrEqual(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - - e1Kind := reflect.ValueOf(e1).Kind() - e2Kind := reflect.ValueOf(e2).Kind() - if e1Kind != e2Kind { - return Fail(t, "Elements should be the same type", msgAndArgs...) - } - - res, isComparable := compare(e1, e2, e1Kind) - if !isComparable { - return Fail(t, fmt.Sprintf("Can not compare type \"%s\"", reflect.TypeOf(e1)), msgAndArgs...) - } - - if res != -1 && res != 0 { - return Fail(t, fmt.Sprintf("\"%v\" is not greater than or equal to \"%v\"", e1, e2), msgAndArgs...) - } - - return true + return compareTwoValues(t, e1, e2, []CompareType{compareGreater, compareEqual}, "\"%v\" is not greater than or equal to \"%v\"", msgAndArgs) } // Less asserts that the first element is less than the second @@ -257,26 +227,7 @@ func GreaterOrEqual(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...in // assert.Less(t, float64(1), float64(2)) // assert.Less(t, "a", "b") func Less(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - - e1Kind := reflect.ValueOf(e1).Kind() - e2Kind := reflect.ValueOf(e2).Kind() - if e1Kind != e2Kind { - return Fail(t, "Elements should be the same type", msgAndArgs...) - } - - res, isComparable := compare(e1, e2, e1Kind) - if !isComparable { - return Fail(t, fmt.Sprintf("Can not compare type \"%s\"", reflect.TypeOf(e1)), msgAndArgs...) - } - - if res != 1 { - return Fail(t, fmt.Sprintf("\"%v\" is not less than \"%v\"", e1, e2), msgAndArgs...) - } - - return true + return compareTwoValues(t, e1, e2, []CompareType{compareLess}, "\"%v\" is not less than \"%v\"", msgAndArgs) } // LessOrEqual asserts that the first element is less than or equal to the second @@ -286,6 +237,10 @@ func Less(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) // assert.LessOrEqual(t, "a", "b") // assert.LessOrEqual(t, "b", "b") func LessOrEqual(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) bool { + return compareTwoValues(t, e1, e2, []CompareType{compareLess, compareEqual}, "\"%v\" is not less than or equal to \"%v\"", msgAndArgs) +} + +func compareTwoValues(t TestingT, e1 interface{}, e2 interface{}, allowedComparesResults []CompareType, failMessage string, msgAndArgs ...interface{}) bool { if h, ok := t.(tHelper); ok { h.Helper() } @@ -296,14 +251,24 @@ func LessOrEqual(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...inter return Fail(t, "Elements should be the same type", msgAndArgs...) } - res, isComparable := compare(e1, e2, e1Kind) + compareResult, isComparable := compare(e1, e2, e1Kind) if !isComparable { return Fail(t, fmt.Sprintf("Can not compare type \"%s\"", reflect.TypeOf(e1)), msgAndArgs...) } - if res != 1 && res != 0 { - return Fail(t, fmt.Sprintf("\"%v\" is not less than or equal to \"%v\"", e1, e2), msgAndArgs...) + if !containsValue(allowedComparesResults, compareResult) { + return Fail(t, fmt.Sprintf(failMessage, e1, e2), msgAndArgs...) } return true } + +func containsValue(values []CompareType, value CompareType) bool { + for _, v := range values { + if v == value { + return true + } + } + + return false +} diff --git a/github.com/stretchr/testify/assert/assertion_format.go b/github.com/stretchr/testify/assert/assertion_format.go index bf89ecd21f..49370eb167 100644 --- a/github.com/stretchr/testify/assert/assertion_format.go +++ b/github.com/stretchr/testify/assert/assertion_format.go @@ -93,7 +93,7 @@ func EqualErrorf(t TestingT, theError error, errString string, msg string, args // EqualValuesf asserts that two objects are equal or convertable to the same types // and equal. // -// assert.EqualValuesf(t, uint32(123, "error message %s", "formatted"), int32(123)) +// assert.EqualValuesf(t, uint32(123), int32(123), "error message %s", "formatted") func EqualValuesf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) bool { if h, ok := t.(tHelper); ok { h.Helper() @@ -127,7 +127,7 @@ func Eventuallyf(t TestingT, condition func() bool, waitFor time.Duration, tick // Exactlyf asserts that two objects are equal in value and type. // -// assert.Exactlyf(t, int32(123, "error message %s", "formatted"), int64(123)) +// assert.Exactlyf(t, int32(123), int64(123), "error message %s", "formatted") func Exactlyf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) bool { if h, ok := t.(tHelper); ok { h.Helper() @@ -173,7 +173,7 @@ func FileExistsf(t TestingT, path string, msg string, args ...interface{}) bool // Greaterf asserts that the first element is greater than the second // // assert.Greaterf(t, 2, 1, "error message %s", "formatted") -// assert.Greaterf(t, float64(2, "error message %s", "formatted"), float64(1)) +// assert.Greaterf(t, float64(2), float64(1), "error message %s", "formatted") // assert.Greaterf(t, "b", "a", "error message %s", "formatted") func Greaterf(t TestingT, e1 interface{}, e2 interface{}, msg string, args ...interface{}) bool { if h, ok := t.(tHelper); ok { @@ -225,7 +225,7 @@ func HTTPBodyNotContainsf(t TestingT, handler http.HandlerFunc, method string, u // // assert.HTTPErrorf(t, myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} // -// Returns whether the assertion was successful (true, "error message %s", "formatted") or not (false). +// Returns whether the assertion was successful (true) or not (false). func HTTPErrorf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) bool { if h, ok := t.(tHelper); ok { h.Helper() @@ -237,7 +237,7 @@ func HTTPErrorf(t TestingT, handler http.HandlerFunc, method string, url string, // // assert.HTTPRedirectf(t, myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} // -// Returns whether the assertion was successful (true, "error message %s", "formatted") or not (false). +// Returns whether the assertion was successful (true) or not (false). func HTTPRedirectf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) bool { if h, ok := t.(tHelper); ok { h.Helper() @@ -245,6 +245,18 @@ func HTTPRedirectf(t TestingT, handler http.HandlerFunc, method string, url stri return HTTPRedirect(t, handler, method, url, values, append([]interface{}{msg}, args...)...) } +// HTTPStatusCodef asserts that a specified handler returns a specified status code. +// +// assert.HTTPStatusCodef(t, myHandler, "GET", "/notImplemented", nil, 501, "error message %s", "formatted") +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPStatusCodef(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, statuscode int, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return HTTPStatusCode(t, handler, method, url, values, statuscode, append([]interface{}{msg}, args...)...) +} + // HTTPSuccessf asserts that a specified handler returns a success status code. // // assert.HTTPSuccessf(t, myHandler, "POST", "http://www.google.com", nil, "error message %s", "formatted") @@ -259,7 +271,7 @@ func HTTPSuccessf(t TestingT, handler http.HandlerFunc, method string, url strin // Implementsf asserts that an object is implemented by the specified interface. // -// assert.Implementsf(t, (*MyInterface, "error message %s", "formatted")(nil), new(MyObject)) +// assert.Implementsf(t, (*MyInterface)(nil), new(MyObject), "error message %s", "formatted") func Implementsf(t TestingT, interfaceObject interface{}, object interface{}, msg string, args ...interface{}) bool { if h, ok := t.(tHelper); ok { h.Helper() @@ -341,7 +353,7 @@ func Lenf(t TestingT, object interface{}, length int, msg string, args ...interf // Lessf asserts that the first element is less than the second // // assert.Lessf(t, 1, 2, "error message %s", "formatted") -// assert.Lessf(t, float64(1, "error message %s", "formatted"), float64(2)) +// assert.Lessf(t, float64(1), float64(2), "error message %s", "formatted") // assert.Lessf(t, "a", "b", "error message %s", "formatted") func Lessf(t TestingT, e1 interface{}, e2 interface{}, msg string, args ...interface{}) bool { if h, ok := t.(tHelper); ok { @@ -454,6 +466,16 @@ func NotEqualf(t TestingT, expected interface{}, actual interface{}, msg string, return NotEqual(t, expected, actual, append([]interface{}{msg}, args...)...) } +// NotEqualValuesf asserts that two objects are not equal even when converted to the same type +// +// assert.NotEqualValuesf(t, obj1, obj2, "error message %s", "formatted") +func NotEqualValuesf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return NotEqualValues(t, expected, actual, append([]interface{}{msg}, args...)...) +} + // NotNilf asserts that the specified object is not nil. // // assert.NotNilf(t, err, "error message %s", "formatted") @@ -476,7 +498,7 @@ func NotPanicsf(t TestingT, f PanicTestFunc, msg string, args ...interface{}) bo // NotRegexpf asserts that a specified regexp does not match a string. // -// assert.NotRegexpf(t, regexp.MustCompile("starts", "error message %s", "formatted"), "it's starting") +// assert.NotRegexpf(t, regexp.MustCompile("starts"), "it's starting", "error message %s", "formatted") // assert.NotRegexpf(t, "^start", "it's not starting", "error message %s", "formatted") func NotRegexpf(t TestingT, rx interface{}, str interface{}, msg string, args ...interface{}) bool { if h, ok := t.(tHelper); ok { @@ -552,7 +574,7 @@ func PanicsWithValuef(t TestingT, expected interface{}, f PanicTestFunc, msg str // Regexpf asserts that a specified regexp matches a string. // -// assert.Regexpf(t, regexp.MustCompile("start", "error message %s", "formatted"), "it's starting") +// assert.Regexpf(t, regexp.MustCompile("start"), "it's starting", "error message %s", "formatted") // assert.Regexpf(t, "start...$", "it's not starting", "error message %s", "formatted") func Regexpf(t TestingT, rx interface{}, str interface{}, msg string, args ...interface{}) bool { if h, ok := t.(tHelper); ok { diff --git a/github.com/stretchr/testify/assert/assertion_forward.go b/github.com/stretchr/testify/assert/assertion_forward.go index 75ecdcaa2f..9db889427a 100644 --- a/github.com/stretchr/testify/assert/assertion_forward.go +++ b/github.com/stretchr/testify/assert/assertion_forward.go @@ -169,7 +169,7 @@ func (a *Assertions) EqualValues(expected interface{}, actual interface{}, msgAn // EqualValuesf asserts that two objects are equal or convertable to the same types // and equal. // -// a.EqualValuesf(uint32(123, "error message %s", "formatted"), int32(123)) +// a.EqualValuesf(uint32(123), int32(123), "error message %s", "formatted") func (a *Assertions) EqualValuesf(expected interface{}, actual interface{}, msg string, args ...interface{}) bool { if h, ok := a.t.(tHelper); ok { h.Helper() @@ -251,7 +251,7 @@ func (a *Assertions) Exactly(expected interface{}, actual interface{}, msgAndArg // Exactlyf asserts that two objects are equal in value and type. // -// a.Exactlyf(int32(123, "error message %s", "formatted"), int64(123)) +// a.Exactlyf(int32(123), int64(123), "error message %s", "formatted") func (a *Assertions) Exactlyf(expected interface{}, actual interface{}, msg string, args ...interface{}) bool { if h, ok := a.t.(tHelper); ok { h.Helper() @@ -370,7 +370,7 @@ func (a *Assertions) GreaterOrEqualf(e1 interface{}, e2 interface{}, msg string, // Greaterf asserts that the first element is greater than the second // // a.Greaterf(2, 1, "error message %s", "formatted") -// a.Greaterf(float64(2, "error message %s", "formatted"), float64(1)) +// a.Greaterf(float64(2), float64(1), "error message %s", "formatted") // a.Greaterf("b", "a", "error message %s", "formatted") func (a *Assertions) Greaterf(e1 interface{}, e2 interface{}, msg string, args ...interface{}) bool { if h, ok := a.t.(tHelper); ok { @@ -447,7 +447,7 @@ func (a *Assertions) HTTPError(handler http.HandlerFunc, method string, url stri // // a.HTTPErrorf(myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} // -// Returns whether the assertion was successful (true, "error message %s", "formatted") or not (false). +// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) HTTPErrorf(handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) bool { if h, ok := a.t.(tHelper); ok { h.Helper() @@ -471,7 +471,7 @@ func (a *Assertions) HTTPRedirect(handler http.HandlerFunc, method string, url s // // a.HTTPRedirectf(myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} // -// Returns whether the assertion was successful (true, "error message %s", "formatted") or not (false). +// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) HTTPRedirectf(handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) bool { if h, ok := a.t.(tHelper); ok { h.Helper() @@ -479,6 +479,30 @@ func (a *Assertions) HTTPRedirectf(handler http.HandlerFunc, method string, url return HTTPRedirectf(a.t, handler, method, url, values, msg, args...) } +// HTTPStatusCode asserts that a specified handler returns a specified status code. +// +// a.HTTPStatusCode(myHandler, "GET", "/notImplemented", nil, 501) +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPStatusCode(handler http.HandlerFunc, method string, url string, values url.Values, statuscode int, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return HTTPStatusCode(a.t, handler, method, url, values, statuscode, msgAndArgs...) +} + +// HTTPStatusCodef asserts that a specified handler returns a specified status code. +// +// a.HTTPStatusCodef(myHandler, "GET", "/notImplemented", nil, 501, "error message %s", "formatted") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPStatusCodef(handler http.HandlerFunc, method string, url string, values url.Values, statuscode int, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return HTTPStatusCodef(a.t, handler, method, url, values, statuscode, msg, args...) +} + // HTTPSuccess asserts that a specified handler returns a success status code. // // a.HTTPSuccess(myHandler, "POST", "http://www.google.com", nil) @@ -515,7 +539,7 @@ func (a *Assertions) Implements(interfaceObject interface{}, object interface{}, // Implementsf asserts that an object is implemented by the specified interface. // -// a.Implementsf((*MyInterface, "error message %s", "formatted")(nil), new(MyObject)) +// a.Implementsf((*MyInterface)(nil), new(MyObject), "error message %s", "formatted") func (a *Assertions) Implementsf(interfaceObject interface{}, object interface{}, msg string, args ...interface{}) bool { if h, ok := a.t.(tHelper); ok { h.Helper() @@ -706,7 +730,7 @@ func (a *Assertions) LessOrEqualf(e1 interface{}, e2 interface{}, msg string, ar // Lessf asserts that the first element is less than the second // // a.Lessf(1, 2, "error message %s", "formatted") -// a.Lessf(float64(1, "error message %s", "formatted"), float64(2)) +// a.Lessf(float64(1), float64(2), "error message %s", "formatted") // a.Lessf("a", "b", "error message %s", "formatted") func (a *Assertions) Lessf(e1 interface{}, e2 interface{}, msg string, args ...interface{}) bool { if h, ok := a.t.(tHelper); ok { @@ -884,6 +908,26 @@ func (a *Assertions) NotEqual(expected interface{}, actual interface{}, msgAndAr return NotEqual(a.t, expected, actual, msgAndArgs...) } +// NotEqualValues asserts that two objects are not equal even when converted to the same type +// +// a.NotEqualValues(obj1, obj2) +func (a *Assertions) NotEqualValues(expected interface{}, actual interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotEqualValues(a.t, expected, actual, msgAndArgs...) +} + +// NotEqualValuesf asserts that two objects are not equal even when converted to the same type +// +// a.NotEqualValuesf(obj1, obj2, "error message %s", "formatted") +func (a *Assertions) NotEqualValuesf(expected interface{}, actual interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return NotEqualValuesf(a.t, expected, actual, msg, args...) +} + // NotEqualf asserts that the specified values are NOT equal. // // a.NotEqualf(obj1, obj2, "error message %s", "formatted") @@ -950,7 +994,7 @@ func (a *Assertions) NotRegexp(rx interface{}, str interface{}, msgAndArgs ...in // NotRegexpf asserts that a specified regexp does not match a string. // -// a.NotRegexpf(regexp.MustCompile("starts", "error message %s", "formatted"), "it's starting") +// a.NotRegexpf(regexp.MustCompile("starts"), "it's starting", "error message %s", "formatted") // a.NotRegexpf("^start", "it's not starting", "error message %s", "formatted") func (a *Assertions) NotRegexpf(rx interface{}, str interface{}, msg string, args ...interface{}) bool { if h, ok := a.t.(tHelper); ok { @@ -1102,7 +1146,7 @@ func (a *Assertions) Regexp(rx interface{}, str interface{}, msgAndArgs ...inter // Regexpf asserts that a specified regexp matches a string. // -// a.Regexpf(regexp.MustCompile("start", "error message %s", "formatted"), "it's starting") +// a.Regexpf(regexp.MustCompile("start"), "it's starting", "error message %s", "formatted") // a.Regexpf("start...$", "it's not starting", "error message %s", "formatted") func (a *Assertions) Regexpf(rx interface{}, str interface{}, msg string, args ...interface{}) bool { if h, ok := a.t.(tHelper); ok { diff --git a/github.com/stretchr/testify/assert/assertions.go b/github.com/stretchr/testify/assert/assertions.go index bdd81389a9..914a10d83a 100644 --- a/github.com/stretchr/testify/assert/assertions.go +++ b/github.com/stretchr/testify/assert/assertions.go @@ -19,7 +19,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/pmezard/go-difflib/difflib" - yaml "gopkg.in/yaml.v2" + yaml "gopkg.in/yaml.v3" ) //go:generate sh -c "cd ../_codegen && go build && cd - && ../_codegen/_codegen -output-package=assert -template=assertion_format.go.tmpl" @@ -45,7 +45,7 @@ type BoolAssertionFunc func(TestingT, bool, ...interface{}) bool // for table driven tests. type ErrorAssertionFunc func(TestingT, error, ...interface{}) bool -// Comparison a custom function that returns true on success and false on failure +// Comparison is a custom function that returns true on success and false on failure type Comparison func() (success bool) /* @@ -104,11 +104,11 @@ the problem actually occurred in calling code.*/ // failed. func CallerInfo() []string { - pc := uintptr(0) - file := "" - line := 0 - ok := false - name := "" + var pc uintptr + var ok bool + var file string + var line int + var name string callers := []string{} for i := 0; ; i++ { @@ -429,14 +429,27 @@ func samePointers(first, second interface{}) bool { // to a type conversion in the Go grammar. func formatUnequalValues(expected, actual interface{}) (e string, a string) { if reflect.TypeOf(expected) != reflect.TypeOf(actual) { - return fmt.Sprintf("%T(%#v)", expected, expected), - fmt.Sprintf("%T(%#v)", actual, actual) + return fmt.Sprintf("%T(%s)", expected, truncatingFormat(expected)), + fmt.Sprintf("%T(%s)", actual, truncatingFormat(actual)) } switch expected.(type) { case time.Duration: return fmt.Sprintf("%v", expected), fmt.Sprintf("%v", actual) } - return fmt.Sprintf("%#v", expected), fmt.Sprintf("%#v", actual) + return truncatingFormat(expected), truncatingFormat(actual) +} + +// truncatingFormat formats the data and truncates it if it's too long. +// +// This helps keep formatted error messages lines from exceeding the +// bufio.MaxScanTokenSize max line length that the go testing framework imposes. +func truncatingFormat(data interface{}) string { + value := fmt.Sprintf("%#v", data) + max := bufio.MaxScanTokenSize - 100 // Give us some space the type info too if needed. + if len(value) > max { + value = value[0:max] + "<... truncated>" + } + return value } // EqualValues asserts that two objects are equal or convertable to the same types @@ -483,12 +496,12 @@ func Exactly(t TestingT, expected, actual interface{}, msgAndArgs ...interface{} // // assert.NotNil(t, err) func NotNil(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } if !isNil(object) { return true } + if h, ok := t.(tHelper); ok { + h.Helper() + } return Fail(t, "Expected value not to be nil.", msgAndArgs...) } @@ -529,12 +542,12 @@ func isNil(object interface{}) bool { // // assert.Nil(t, err) func Nil(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } if isNil(object) { return true } + if h, ok := t.(tHelper); ok { + h.Helper() + } return Fail(t, fmt.Sprintf("Expected nil, but got: %#v", object), msgAndArgs...) } @@ -571,12 +584,11 @@ func isEmpty(object interface{}) bool { // // assert.Empty(t, obj) func Empty(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - pass := isEmpty(object) if !pass { + if h, ok := t.(tHelper); ok { + h.Helper() + } Fail(t, fmt.Sprintf("Should be empty, but was %v", object), msgAndArgs...) } @@ -591,12 +603,11 @@ func Empty(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { // assert.Equal(t, "two", obj[1]) // } func NotEmpty(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - pass := !isEmpty(object) if !pass { + if h, ok := t.(tHelper); ok { + h.Helper() + } Fail(t, fmt.Sprintf("Should NOT be empty, but was %v", object), msgAndArgs...) } @@ -639,16 +650,10 @@ func Len(t TestingT, object interface{}, length int, msgAndArgs ...interface{}) // // assert.True(t, myBool) func True(t TestingT, value bool, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - if h, ok := t.(interface { - Helper() - }); ok { - h.Helper() - } - - if value != true { + if !value { + if h, ok := t.(tHelper); ok { + h.Helper() + } return Fail(t, "Should be true", msgAndArgs...) } @@ -660,11 +665,10 @@ func True(t TestingT, value bool, msgAndArgs ...interface{}) bool { // // assert.False(t, myBool) func False(t TestingT, value bool, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - - if value != false { + if value { + if h, ok := t.(tHelper); ok { + h.Helper() + } return Fail(t, "Should be false", msgAndArgs...) } @@ -695,6 +699,21 @@ func NotEqual(t TestingT, expected, actual interface{}, msgAndArgs ...interface{ } +// NotEqualValues asserts that two objects are not equal even when converted to the same type +// +// assert.NotEqualValues(t, obj1, obj2) +func NotEqualValues(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + + if ObjectsAreEqualValues(expected, actual) { + return Fail(t, fmt.Sprintf("Should not be: %#v\n", actual), msgAndArgs...) + } + + return true +} + // containsElement try loop over the list check if the list includes the element. // return (false, false) if impossible. // return (true, false) if element was not found. @@ -747,10 +766,10 @@ func Contains(t TestingT, s, contains interface{}, msgAndArgs ...interface{}) bo ok, found := includeElement(s, contains) if !ok { - return Fail(t, fmt.Sprintf("\"%s\" could not be applied builtin len()", s), msgAndArgs...) + return Fail(t, fmt.Sprintf("%#v could not be applied builtin len()", s), msgAndArgs...) } if !found { - return Fail(t, fmt.Sprintf("\"%s\" does not contain \"%s\"", s, contains), msgAndArgs...) + return Fail(t, fmt.Sprintf("%#v does not contain %#v", s, contains), msgAndArgs...) } return true @@ -881,27 +900,39 @@ func ElementsMatch(t TestingT, listA, listB interface{}, msgAndArgs ...interface return true } - aKind := reflect.TypeOf(listA).Kind() - bKind := reflect.TypeOf(listB).Kind() + if !isList(t, listA, msgAndArgs...) || !isList(t, listB, msgAndArgs...) { + return false + } - if aKind != reflect.Array && aKind != reflect.Slice { - return Fail(t, fmt.Sprintf("%q has an unsupported type %s", listA, aKind), msgAndArgs...) + extraA, extraB := diffLists(listA, listB) + + if len(extraA) == 0 && len(extraB) == 0 { + return true } - if bKind != reflect.Array && bKind != reflect.Slice { - return Fail(t, fmt.Sprintf("%q has an unsupported type %s", listB, bKind), msgAndArgs...) + return Fail(t, formatListDiff(listA, listB, extraA, extraB), msgAndArgs...) +} + +// isList checks that the provided value is array or slice. +func isList(t TestingT, list interface{}, msgAndArgs ...interface{}) (ok bool) { + kind := reflect.TypeOf(list).Kind() + if kind != reflect.Array && kind != reflect.Slice { + return Fail(t, fmt.Sprintf("%q has an unsupported type %s, expecting array or slice", list, kind), + msgAndArgs...) } + return true +} +// diffLists diffs two arrays/slices and returns slices of elements that are only in A and only in B. +// If some element is present multiple times, each instance is counted separately (e.g. if something is 2x in A and +// 5x in B, it will be 0x in extraA and 3x in extraB). The order of items in both lists is ignored. +func diffLists(listA, listB interface{}) (extraA, extraB []interface{}) { aValue := reflect.ValueOf(listA) bValue := reflect.ValueOf(listB) aLen := aValue.Len() bLen := bValue.Len() - if aLen != bLen { - return Fail(t, fmt.Sprintf("lengths don't match: %d != %d", aLen, bLen), msgAndArgs...) - } - // Mark indexes in bValue that we already used visited := make([]bool, bLen) for i := 0; i < aLen; i++ { @@ -918,11 +949,38 @@ func ElementsMatch(t TestingT, listA, listB interface{}, msgAndArgs ...interface } } if !found { - return Fail(t, fmt.Sprintf("element %s appears more times in %s than in %s", element, aValue, bValue), msgAndArgs...) + extraA = append(extraA, element) } } - return true + for j := 0; j < bLen; j++ { + if visited[j] { + continue + } + extraB = append(extraB, bValue.Index(j).Interface()) + } + + return +} + +func formatListDiff(listA, listB interface{}, extraA, extraB []interface{}) string { + var msg bytes.Buffer + + msg.WriteString("elements differ") + if len(extraA) > 0 { + msg.WriteString("\n\nextra elements in list A:\n") + msg.WriteString(spewConfig.Sdump(extraA)) + } + if len(extraB) > 0 { + msg.WriteString("\n\nextra elements in list B:\n") + msg.WriteString(spewConfig.Sdump(extraB)) + } + msg.WriteString("\n\nlistA:\n") + msg.WriteString(spewConfig.Sdump(listA)) + msg.WriteString("\n\nlistB:\n") + msg.WriteString(spewConfig.Sdump(listB)) + + return msg.String() } // Condition uses a Comparison to assert a complex condition. @@ -1058,6 +1116,8 @@ func toFloat(x interface{}) (float64, bool) { xok := true switch xn := x.(type) { + case uint: + xf = float64(xn) case uint8: xf = float64(xn) case uint16: @@ -1079,7 +1139,7 @@ func toFloat(x interface{}) (float64, bool) { case float32: xf = float64(xn) case float64: - xf = float64(xn) + xf = xn case time.Duration: xf = float64(xn) default: @@ -1193,6 +1253,9 @@ func calcRelativeError(expected, actual interface{}) (float64, error) { if !aok { return 0, fmt.Errorf("expected value %q cannot be converted to float", expected) } + if math.IsNaN(af) { + return 0, errors.New("expected value must not be NaN") + } if af == 0 { return 0, fmt.Errorf("expected value must have a value other than zero to calculate the relative error") } @@ -1200,6 +1263,9 @@ func calcRelativeError(expected, actual interface{}) (float64, error) { if !bok { return 0, fmt.Errorf("actual value %q cannot be converted to float", actual) } + if math.IsNaN(bf) { + return 0, errors.New("actual value must not be NaN") + } return math.Abs(af-bf) / math.Abs(af), nil } @@ -1209,6 +1275,9 @@ func InEpsilon(t TestingT, expected, actual interface{}, epsilon float64, msgAnd if h, ok := t.(tHelper); ok { h.Helper() } + if math.IsNaN(epsilon) { + return Fail(t, "epsilon must not be NaN") + } actualEpsilon, err := calcRelativeError(expected, actual) if err != nil { return Fail(t, err.Error(), msgAndArgs...) @@ -1256,10 +1325,10 @@ func InEpsilonSlice(t TestingT, expected, actual interface{}, epsilon float64, m // assert.Equal(t, expectedObj, actualObj) // } func NoError(t TestingT, err error, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } if err != nil { + if h, ok := t.(tHelper); ok { + h.Helper() + } return Fail(t, fmt.Sprintf("Received unexpected error:\n%+v", err), msgAndArgs...) } @@ -1273,11 +1342,10 @@ func NoError(t TestingT, err error, msgAndArgs ...interface{}) bool { // assert.Equal(t, expectedError, err) // } func Error(t TestingT, err error, msgAndArgs ...interface{}) bool { - if h, ok := t.(tHelper); ok { - h.Helper() - } - if err == nil { + if h, ok := t.(tHelper); ok { + h.Helper() + } return Fail(t, "An error is expected but got nil.", msgAndArgs...) } @@ -1553,6 +1621,7 @@ var spewConfig = spew.ConfigState{ DisablePointerAddresses: true, DisableCapacities: true, SortKeys: true, + DisableMethods: true, } type tHelper interface { diff --git a/github.com/stretchr/testify/assert/http_assertions.go b/github.com/stretchr/testify/assert/http_assertions.go index df46fa777a..4ed341dd28 100644 --- a/github.com/stretchr/testify/assert/http_assertions.go +++ b/github.com/stretchr/testify/assert/http_assertions.go @@ -33,7 +33,6 @@ func HTTPSuccess(t TestingT, handler http.HandlerFunc, method, url string, value code, err := httpCode(handler, method, url, values) if err != nil { Fail(t, fmt.Sprintf("Failed to build test request, got error: %s", err)) - return false } isSuccessCode := code >= http.StatusOK && code <= http.StatusPartialContent @@ -56,7 +55,6 @@ func HTTPRedirect(t TestingT, handler http.HandlerFunc, method, url string, valu code, err := httpCode(handler, method, url, values) if err != nil { Fail(t, fmt.Sprintf("Failed to build test request, got error: %s", err)) - return false } isRedirectCode := code >= http.StatusMultipleChoices && code <= http.StatusTemporaryRedirect @@ -79,7 +77,6 @@ func HTTPError(t TestingT, handler http.HandlerFunc, method, url string, values code, err := httpCode(handler, method, url, values) if err != nil { Fail(t, fmt.Sprintf("Failed to build test request, got error: %s", err)) - return false } isErrorCode := code >= http.StatusBadRequest @@ -90,6 +87,28 @@ func HTTPError(t TestingT, handler http.HandlerFunc, method, url string, values return isErrorCode } +// HTTPStatusCode asserts that a specified handler returns a specified status code. +// +// assert.HTTPStatusCode(t, myHandler, "GET", "/notImplemented", nil, 501) +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPStatusCode(t TestingT, handler http.HandlerFunc, method, url string, values url.Values, statuscode int, msgAndArgs ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + code, err := httpCode(handler, method, url, values) + if err != nil { + Fail(t, fmt.Sprintf("Failed to build test request, got error: %s", err)) + } + + successful := code == statuscode + if !successful { + Fail(t, fmt.Sprintf("Expected HTTP status code %d for %q but received %d", statuscode, url+"?"+values.Encode(), code)) + } + + return successful +} + // HTTPBody is a helper that returns HTTP body of the response. It returns // empty string if building a new request fails. func HTTPBody(handler http.HandlerFunc, method, url string, values url.Values) string { diff --git a/github.com/stretchr/testify/require/require.go b/github.com/stretchr/testify/require/require.go index cf6c7b5664..ec4624b282 100644 --- a/github.com/stretchr/testify/require/require.go +++ b/github.com/stretchr/testify/require/require.go @@ -212,7 +212,7 @@ func EqualValues(t TestingT, expected interface{}, actual interface{}, msgAndArg // EqualValuesf asserts that two objects are equal or convertable to the same types // and equal. // -// assert.EqualValuesf(t, uint32(123, "error message %s", "formatted"), int32(123)) +// assert.EqualValuesf(t, uint32(123), int32(123), "error message %s", "formatted") func EqualValuesf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) { if h, ok := t.(tHelper); ok { h.Helper() @@ -315,7 +315,7 @@ func Exactly(t TestingT, expected interface{}, actual interface{}, msgAndArgs .. // Exactlyf asserts that two objects are equal in value and type. // -// assert.Exactlyf(t, int32(123, "error message %s", "formatted"), int64(123)) +// assert.Exactlyf(t, int32(123), int64(123), "error message %s", "formatted") func Exactlyf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) { if h, ok := t.(tHelper); ok { h.Helper() @@ -470,7 +470,7 @@ func GreaterOrEqualf(t TestingT, e1 interface{}, e2 interface{}, msg string, arg // Greaterf asserts that the first element is greater than the second // // assert.Greaterf(t, 2, 1, "error message %s", "formatted") -// assert.Greaterf(t, float64(2, "error message %s", "formatted"), float64(1)) +// assert.Greaterf(t, float64(2), float64(1), "error message %s", "formatted") // assert.Greaterf(t, "b", "a", "error message %s", "formatted") func Greaterf(t TestingT, e1 interface{}, e2 interface{}, msg string, args ...interface{}) { if h, ok := t.(tHelper); ok { @@ -565,7 +565,7 @@ func HTTPError(t TestingT, handler http.HandlerFunc, method string, url string, // // assert.HTTPErrorf(t, myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} // -// Returns whether the assertion was successful (true, "error message %s", "formatted") or not (false). +// Returns whether the assertion was successful (true) or not (false). func HTTPErrorf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) { if h, ok := t.(tHelper); ok { h.Helper() @@ -595,7 +595,7 @@ func HTTPRedirect(t TestingT, handler http.HandlerFunc, method string, url strin // // assert.HTTPRedirectf(t, myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} // -// Returns whether the assertion was successful (true, "error message %s", "formatted") or not (false). +// Returns whether the assertion was successful (true) or not (false). func HTTPRedirectf(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) { if h, ok := t.(tHelper); ok { h.Helper() @@ -606,6 +606,36 @@ func HTTPRedirectf(t TestingT, handler http.HandlerFunc, method string, url stri t.FailNow() } +// HTTPStatusCode asserts that a specified handler returns a specified status code. +// +// assert.HTTPStatusCode(t, myHandler, "GET", "/notImplemented", nil, 501) +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPStatusCode(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, statuscode int, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.HTTPStatusCode(t, handler, method, url, values, statuscode, msgAndArgs...) { + return + } + t.FailNow() +} + +// HTTPStatusCodef asserts that a specified handler returns a specified status code. +// +// assert.HTTPStatusCodef(t, myHandler, "GET", "/notImplemented", nil, 501, "error message %s", "formatted") +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPStatusCodef(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, statuscode int, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.HTTPStatusCodef(t, handler, method, url, values, statuscode, msg, args...) { + return + } + t.FailNow() +} + // HTTPSuccess asserts that a specified handler returns a success status code. // // assert.HTTPSuccess(t, myHandler, "POST", "http://www.google.com", nil) @@ -651,7 +681,7 @@ func Implements(t TestingT, interfaceObject interface{}, object interface{}, msg // Implementsf asserts that an object is implemented by the specified interface. // -// assert.Implementsf(t, (*MyInterface, "error message %s", "formatted")(nil), new(MyObject)) +// assert.Implementsf(t, (*MyInterface)(nil), new(MyObject), "error message %s", "formatted") func Implementsf(t TestingT, interfaceObject interface{}, object interface{}, msg string, args ...interface{}) { if h, ok := t.(tHelper); ok { h.Helper() @@ -902,7 +932,7 @@ func LessOrEqualf(t TestingT, e1 interface{}, e2 interface{}, msg string, args . // Lessf asserts that the first element is less than the second // // assert.Lessf(t, 1, 2, "error message %s", "formatted") -// assert.Lessf(t, float64(1, "error message %s", "formatted"), float64(2)) +// assert.Lessf(t, float64(1), float64(2), "error message %s", "formatted") // assert.Lessf(t, "a", "b", "error message %s", "formatted") func Lessf(t TestingT, e1 interface{}, e2 interface{}, msg string, args ...interface{}) { if h, ok := t.(tHelper); ok { @@ -1128,6 +1158,32 @@ func NotEqual(t TestingT, expected interface{}, actual interface{}, msgAndArgs . t.FailNow() } +// NotEqualValues asserts that two objects are not equal even when converted to the same type +// +// assert.NotEqualValues(t, obj1, obj2) +func NotEqualValues(t TestingT, expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotEqualValues(t, expected, actual, msgAndArgs...) { + return + } + t.FailNow() +} + +// NotEqualValuesf asserts that two objects are not equal even when converted to the same type +// +// assert.NotEqualValuesf(t, obj1, obj2, "error message %s", "formatted") +func NotEqualValuesf(t TestingT, expected interface{}, actual interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.NotEqualValuesf(t, expected, actual, msg, args...) { + return + } + t.FailNow() +} + // NotEqualf asserts that the specified values are NOT equal. // // assert.NotEqualf(t, obj1, obj2, "error message %s", "formatted") @@ -1212,7 +1268,7 @@ func NotRegexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interf // NotRegexpf asserts that a specified regexp does not match a string. // -// assert.NotRegexpf(t, regexp.MustCompile("starts", "error message %s", "formatted"), "it's starting") +// assert.NotRegexpf(t, regexp.MustCompile("starts"), "it's starting", "error message %s", "formatted") // assert.NotRegexpf(t, "^start", "it's not starting", "error message %s", "formatted") func NotRegexpf(t TestingT, rx interface{}, str interface{}, msg string, args ...interface{}) { if h, ok := t.(tHelper); ok { @@ -1406,7 +1462,7 @@ func Regexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interface // Regexpf asserts that a specified regexp matches a string. // -// assert.Regexpf(t, regexp.MustCompile("start", "error message %s", "formatted"), "it's starting") +// assert.Regexpf(t, regexp.MustCompile("start"), "it's starting", "error message %s", "formatted") // assert.Regexpf(t, "start...$", "it's not starting", "error message %s", "formatted") func Regexpf(t TestingT, rx interface{}, str interface{}, msg string, args ...interface{}) { if h, ok := t.(tHelper); ok { diff --git a/github.com/stretchr/testify/require/require_forward.go b/github.com/stretchr/testify/require/require_forward.go index 5aac226df8..103d7dcb6a 100644 --- a/github.com/stretchr/testify/require/require_forward.go +++ b/github.com/stretchr/testify/require/require_forward.go @@ -170,7 +170,7 @@ func (a *Assertions) EqualValues(expected interface{}, actual interface{}, msgAn // EqualValuesf asserts that two objects are equal or convertable to the same types // and equal. // -// a.EqualValuesf(uint32(123, "error message %s", "formatted"), int32(123)) +// a.EqualValuesf(uint32(123), int32(123), "error message %s", "formatted") func (a *Assertions) EqualValuesf(expected interface{}, actual interface{}, msg string, args ...interface{}) { if h, ok := a.t.(tHelper); ok { h.Helper() @@ -252,7 +252,7 @@ func (a *Assertions) Exactly(expected interface{}, actual interface{}, msgAndArg // Exactlyf asserts that two objects are equal in value and type. // -// a.Exactlyf(int32(123, "error message %s", "formatted"), int64(123)) +// a.Exactlyf(int32(123), int64(123), "error message %s", "formatted") func (a *Assertions) Exactlyf(expected interface{}, actual interface{}, msg string, args ...interface{}) { if h, ok := a.t.(tHelper); ok { h.Helper() @@ -371,7 +371,7 @@ func (a *Assertions) GreaterOrEqualf(e1 interface{}, e2 interface{}, msg string, // Greaterf asserts that the first element is greater than the second // // a.Greaterf(2, 1, "error message %s", "formatted") -// a.Greaterf(float64(2, "error message %s", "formatted"), float64(1)) +// a.Greaterf(float64(2), float64(1), "error message %s", "formatted") // a.Greaterf("b", "a", "error message %s", "formatted") func (a *Assertions) Greaterf(e1 interface{}, e2 interface{}, msg string, args ...interface{}) { if h, ok := a.t.(tHelper); ok { @@ -448,7 +448,7 @@ func (a *Assertions) HTTPError(handler http.HandlerFunc, method string, url stri // // a.HTTPErrorf(myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} // -// Returns whether the assertion was successful (true, "error message %s", "formatted") or not (false). +// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) HTTPErrorf(handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) { if h, ok := a.t.(tHelper); ok { h.Helper() @@ -472,7 +472,7 @@ func (a *Assertions) HTTPRedirect(handler http.HandlerFunc, method string, url s // // a.HTTPRedirectf(myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} // -// Returns whether the assertion was successful (true, "error message %s", "formatted") or not (false). +// Returns whether the assertion was successful (true) or not (false). func (a *Assertions) HTTPRedirectf(handler http.HandlerFunc, method string, url string, values url.Values, msg string, args ...interface{}) { if h, ok := a.t.(tHelper); ok { h.Helper() @@ -480,6 +480,30 @@ func (a *Assertions) HTTPRedirectf(handler http.HandlerFunc, method string, url HTTPRedirectf(a.t, handler, method, url, values, msg, args...) } +// HTTPStatusCode asserts that a specified handler returns a specified status code. +// +// a.HTTPStatusCode(myHandler, "GET", "/notImplemented", nil, 501) +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPStatusCode(handler http.HandlerFunc, method string, url string, values url.Values, statuscode int, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + HTTPStatusCode(a.t, handler, method, url, values, statuscode, msgAndArgs...) +} + +// HTTPStatusCodef asserts that a specified handler returns a specified status code. +// +// a.HTTPStatusCodef(myHandler, "GET", "/notImplemented", nil, 501, "error message %s", "formatted") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPStatusCodef(handler http.HandlerFunc, method string, url string, values url.Values, statuscode int, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + HTTPStatusCodef(a.t, handler, method, url, values, statuscode, msg, args...) +} + // HTTPSuccess asserts that a specified handler returns a success status code. // // a.HTTPSuccess(myHandler, "POST", "http://www.google.com", nil) @@ -516,7 +540,7 @@ func (a *Assertions) Implements(interfaceObject interface{}, object interface{}, // Implementsf asserts that an object is implemented by the specified interface. // -// a.Implementsf((*MyInterface, "error message %s", "formatted")(nil), new(MyObject)) +// a.Implementsf((*MyInterface)(nil), new(MyObject), "error message %s", "formatted") func (a *Assertions) Implementsf(interfaceObject interface{}, object interface{}, msg string, args ...interface{}) { if h, ok := a.t.(tHelper); ok { h.Helper() @@ -707,7 +731,7 @@ func (a *Assertions) LessOrEqualf(e1 interface{}, e2 interface{}, msg string, ar // Lessf asserts that the first element is less than the second // // a.Lessf(1, 2, "error message %s", "formatted") -// a.Lessf(float64(1, "error message %s", "formatted"), float64(2)) +// a.Lessf(float64(1), float64(2), "error message %s", "formatted") // a.Lessf("a", "b", "error message %s", "formatted") func (a *Assertions) Lessf(e1 interface{}, e2 interface{}, msg string, args ...interface{}) { if h, ok := a.t.(tHelper); ok { @@ -885,6 +909,26 @@ func (a *Assertions) NotEqual(expected interface{}, actual interface{}, msgAndAr NotEqual(a.t, expected, actual, msgAndArgs...) } +// NotEqualValues asserts that two objects are not equal even when converted to the same type +// +// a.NotEqualValues(obj1, obj2) +func (a *Assertions) NotEqualValues(expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotEqualValues(a.t, expected, actual, msgAndArgs...) +} + +// NotEqualValuesf asserts that two objects are not equal even when converted to the same type +// +// a.NotEqualValuesf(obj1, obj2, "error message %s", "formatted") +func (a *Assertions) NotEqualValuesf(expected interface{}, actual interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + NotEqualValuesf(a.t, expected, actual, msg, args...) +} + // NotEqualf asserts that the specified values are NOT equal. // // a.NotEqualf(obj1, obj2, "error message %s", "formatted") @@ -951,7 +995,7 @@ func (a *Assertions) NotRegexp(rx interface{}, str interface{}, msgAndArgs ...in // NotRegexpf asserts that a specified regexp does not match a string. // -// a.NotRegexpf(regexp.MustCompile("starts", "error message %s", "formatted"), "it's starting") +// a.NotRegexpf(regexp.MustCompile("starts"), "it's starting", "error message %s", "formatted") // a.NotRegexpf("^start", "it's not starting", "error message %s", "formatted") func (a *Assertions) NotRegexpf(rx interface{}, str interface{}, msg string, args ...interface{}) { if h, ok := a.t.(tHelper); ok { @@ -1103,7 +1147,7 @@ func (a *Assertions) Regexp(rx interface{}, str interface{}, msgAndArgs ...inter // Regexpf asserts that a specified regexp matches a string. // -// a.Regexpf(regexp.MustCompile("start", "error message %s", "formatted"), "it's starting") +// a.Regexpf(regexp.MustCompile("start"), "it's starting", "error message %s", "formatted") // a.Regexpf("start...$", "it's not starting", "error message %s", "formatted") func (a *Assertions) Regexpf(rx interface{}, str interface{}, msg string, args ...interface{}) { if h, ok := a.t.(tHelper); ok { diff --git a/golang.org/x/mod/LICENSE b/golang.org/x/mod/LICENSE new file mode 100644 index 0000000000..6a66aea5ea --- /dev/null +++ b/golang.org/x/mod/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/golang.org/x/mod/PATENTS b/golang.org/x/mod/PATENTS new file mode 100644 index 0000000000..733099041f --- /dev/null +++ b/golang.org/x/mod/PATENTS @@ -0,0 +1,22 @@ +Additional IP Rights Grant (Patents) + +"This implementation" means the copyrightable works distributed by +Google as part of the Go project. + +Google hereby grants to You a perpetual, worldwide, non-exclusive, +no-charge, royalty-free, irrevocable (except as stated in this section) +patent license to make, have made, use, offer to sell, sell, import, +transfer and otherwise run, modify and propagate the contents of this +implementation of Go, where such license applies only to those patent +claims, both currently owned or controlled by Google and acquired in +the future, licensable by Google that are necessarily infringed by this +implementation of Go. This grant does not include claims that would be +infringed only as a consequence of further modification of this +implementation. If you or your agent or exclusive licensee institute or +order or agree to the institution of patent litigation against any +entity (including a cross-claim or counterclaim in a lawsuit) alleging +that this implementation of Go or any code incorporated within this +implementation of Go constitutes direct or contributory patent +infringement, or inducement of patent infringement, then any patent +rights granted to you under this License for this implementation of Go +shall terminate as of the date such litigation is filed. diff --git a/golang.org/x/mod/semver/semver.go b/golang.org/x/mod/semver/semver.go new file mode 100644 index 0000000000..2988e3cf9c --- /dev/null +++ b/golang.org/x/mod/semver/semver.go @@ -0,0 +1,388 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package semver implements comparison of semantic version strings. +// In this package, semantic version strings must begin with a leading "v", +// as in "v1.0.0". +// +// The general form of a semantic version string accepted by this package is +// +// vMAJOR[.MINOR[.PATCH[-PRERELEASE][+BUILD]]] +// +// where square brackets indicate optional parts of the syntax; +// MAJOR, MINOR, and PATCH are decimal integers without extra leading zeros; +// PRERELEASE and BUILD are each a series of non-empty dot-separated identifiers +// using only alphanumeric characters and hyphens; and +// all-numeric PRERELEASE identifiers must not have leading zeros. +// +// This package follows Semantic Versioning 2.0.0 (see semver.org) +// with two exceptions. First, it requires the "v" prefix. Second, it recognizes +// vMAJOR and vMAJOR.MINOR (with no prerelease or build suffixes) +// as shorthands for vMAJOR.0.0 and vMAJOR.MINOR.0. +package semver + +// parsed returns the parsed form of a semantic version string. +type parsed struct { + major string + minor string + patch string + short string + prerelease string + build string + err string +} + +// IsValid reports whether v is a valid semantic version string. +func IsValid(v string) bool { + _, ok := parse(v) + return ok +} + +// Canonical returns the canonical formatting of the semantic version v. +// It fills in any missing .MINOR or .PATCH and discards build metadata. +// Two semantic versions compare equal only if their canonical formattings +// are identical strings. +// The canonical invalid semantic version is the empty string. +func Canonical(v string) string { + p, ok := parse(v) + if !ok { + return "" + } + if p.build != "" { + return v[:len(v)-len(p.build)] + } + if p.short != "" { + return v + p.short + } + return v +} + +// Major returns the major version prefix of the semantic version v. +// For example, Major("v2.1.0") == "v2". +// If v is an invalid semantic version string, Major returns the empty string. +func Major(v string) string { + pv, ok := parse(v) + if !ok { + return "" + } + return v[:1+len(pv.major)] +} + +// MajorMinor returns the major.minor version prefix of the semantic version v. +// For example, MajorMinor("v2.1.0") == "v2.1". +// If v is an invalid semantic version string, MajorMinor returns the empty string. +func MajorMinor(v string) string { + pv, ok := parse(v) + if !ok { + return "" + } + i := 1 + len(pv.major) + if j := i + 1 + len(pv.minor); j <= len(v) && v[i] == '.' && v[i+1:j] == pv.minor { + return v[:j] + } + return v[:i] + "." + pv.minor +} + +// Prerelease returns the prerelease suffix of the semantic version v. +// For example, Prerelease("v2.1.0-pre+meta") == "-pre". +// If v is an invalid semantic version string, Prerelease returns the empty string. +func Prerelease(v string) string { + pv, ok := parse(v) + if !ok { + return "" + } + return pv.prerelease +} + +// Build returns the build suffix of the semantic version v. +// For example, Build("v2.1.0+meta") == "+meta". +// If v is an invalid semantic version string, Build returns the empty string. +func Build(v string) string { + pv, ok := parse(v) + if !ok { + return "" + } + return pv.build +} + +// Compare returns an integer comparing two versions according to +// semantic version precedence. +// The result will be 0 if v == w, -1 if v < w, or +1 if v > w. +// +// An invalid semantic version string is considered less than a valid one. +// All invalid semantic version strings compare equal to each other. +func Compare(v, w string) int { + pv, ok1 := parse(v) + pw, ok2 := parse(w) + if !ok1 && !ok2 { + return 0 + } + if !ok1 { + return -1 + } + if !ok2 { + return +1 + } + if c := compareInt(pv.major, pw.major); c != 0 { + return c + } + if c := compareInt(pv.minor, pw.minor); c != 0 { + return c + } + if c := compareInt(pv.patch, pw.patch); c != 0 { + return c + } + return comparePrerelease(pv.prerelease, pw.prerelease) +} + +// Max canonicalizes its arguments and then returns the version string +// that compares greater. +func Max(v, w string) string { + v = Canonical(v) + w = Canonical(w) + if Compare(v, w) > 0 { + return v + } + return w +} + +func parse(v string) (p parsed, ok bool) { + if v == "" || v[0] != 'v' { + p.err = "missing v prefix" + return + } + p.major, v, ok = parseInt(v[1:]) + if !ok { + p.err = "bad major version" + return + } + if v == "" { + p.minor = "0" + p.patch = "0" + p.short = ".0.0" + return + } + if v[0] != '.' { + p.err = "bad minor prefix" + ok = false + return + } + p.minor, v, ok = parseInt(v[1:]) + if !ok { + p.err = "bad minor version" + return + } + if v == "" { + p.patch = "0" + p.short = ".0" + return + } + if v[0] != '.' { + p.err = "bad patch prefix" + ok = false + return + } + p.patch, v, ok = parseInt(v[1:]) + if !ok { + p.err = "bad patch version" + return + } + if len(v) > 0 && v[0] == '-' { + p.prerelease, v, ok = parsePrerelease(v) + if !ok { + p.err = "bad prerelease" + return + } + } + if len(v) > 0 && v[0] == '+' { + p.build, v, ok = parseBuild(v) + if !ok { + p.err = "bad build" + return + } + } + if v != "" { + p.err = "junk on end" + ok = false + return + } + ok = true + return +} + +func parseInt(v string) (t, rest string, ok bool) { + if v == "" { + return + } + if v[0] < '0' || '9' < v[0] { + return + } + i := 1 + for i < len(v) && '0' <= v[i] && v[i] <= '9' { + i++ + } + if v[0] == '0' && i != 1 { + return + } + return v[:i], v[i:], true +} + +func parsePrerelease(v string) (t, rest string, ok bool) { + // "A pre-release version MAY be denoted by appending a hyphen and + // a series of dot separated identifiers immediately following the patch version. + // Identifiers MUST comprise only ASCII alphanumerics and hyphen [0-9A-Za-z-]. + // Identifiers MUST NOT be empty. Numeric identifiers MUST NOT include leading zeroes." + if v == "" || v[0] != '-' { + return + } + i := 1 + start := 1 + for i < len(v) && v[i] != '+' { + if !isIdentChar(v[i]) && v[i] != '.' { + return + } + if v[i] == '.' { + if start == i || isBadNum(v[start:i]) { + return + } + start = i + 1 + } + i++ + } + if start == i || isBadNum(v[start:i]) { + return + } + return v[:i], v[i:], true +} + +func parseBuild(v string) (t, rest string, ok bool) { + if v == "" || v[0] != '+' { + return + } + i := 1 + start := 1 + for i < len(v) { + if !isIdentChar(v[i]) && v[i] != '.' { + return + } + if v[i] == '.' { + if start == i { + return + } + start = i + 1 + } + i++ + } + if start == i { + return + } + return v[:i], v[i:], true +} + +func isIdentChar(c byte) bool { + return 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' || c == '-' +} + +func isBadNum(v string) bool { + i := 0 + for i < len(v) && '0' <= v[i] && v[i] <= '9' { + i++ + } + return i == len(v) && i > 1 && v[0] == '0' +} + +func isNum(v string) bool { + i := 0 + for i < len(v) && '0' <= v[i] && v[i] <= '9' { + i++ + } + return i == len(v) +} + +func compareInt(x, y string) int { + if x == y { + return 0 + } + if len(x) < len(y) { + return -1 + } + if len(x) > len(y) { + return +1 + } + if x < y { + return -1 + } else { + return +1 + } +} + +func comparePrerelease(x, y string) int { + // "When major, minor, and patch are equal, a pre-release version has + // lower precedence than a normal version. + // Example: 1.0.0-alpha < 1.0.0. + // Precedence for two pre-release versions with the same major, minor, + // and patch version MUST be determined by comparing each dot separated + // identifier from left to right until a difference is found as follows: + // identifiers consisting of only digits are compared numerically and + // identifiers with letters or hyphens are compared lexically in ASCII + // sort order. Numeric identifiers always have lower precedence than + // non-numeric identifiers. A larger set of pre-release fields has a + // higher precedence than a smaller set, if all of the preceding + // identifiers are equal. + // Example: 1.0.0-alpha < 1.0.0-alpha.1 < 1.0.0-alpha.beta < + // 1.0.0-beta < 1.0.0-beta.2 < 1.0.0-beta.11 < 1.0.0-rc.1 < 1.0.0." + if x == y { + return 0 + } + if x == "" { + return +1 + } + if y == "" { + return -1 + } + for x != "" && y != "" { + x = x[1:] // skip - or . + y = y[1:] // skip - or . + var dx, dy string + dx, x = nextIdent(x) + dy, y = nextIdent(y) + if dx != dy { + ix := isNum(dx) + iy := isNum(dy) + if ix != iy { + if ix { + return -1 + } else { + return +1 + } + } + if ix { + if len(dx) < len(dy) { + return -1 + } + if len(dx) > len(dy) { + return +1 + } + } + if dx < dy { + return -1 + } else { + return +1 + } + } + } + if x == "" { + return -1 + } else { + return +1 + } +} + +func nextIdent(x string) (dx, rest string) { + i := 0 + for i < len(x) && x[i] != '.' { + i++ + } + return x[:i], x[i:] +} diff --git a/golang.org/x/tools/go/analysis/doc.go b/golang.org/x/tools/go/analysis/doc.go index ea56b724e8..fb17a0e415 100644 --- a/golang.org/x/tools/go/analysis/doc.go +++ b/golang.org/x/tools/go/analysis/doc.go @@ -170,6 +170,15 @@ Diagnostic is defined as: The optional Category field is a short identifier that classifies the kind of message when an analysis produces several kinds of diagnostic. +Many analyses want to associate diagnostics with a severity level. +Because Diagnostic does not have a severity level field, an Analyzer's +diagnostics effectively all have the same severity level. To separate which +diagnostics are high severity and which are low severity, expose multiple +Analyzers instead. Analyzers should also be separated when their +diagnostics belong in different groups, or could be tagged differently +before being shown to the end user. Analyzers should document their severity +level to help downstream tools surface diagnostics properly. + Most Analyzers inspect typed Go syntax trees, but a few, such as asmdecl and buildtag, inspect the raw text of Go source files or even non-Go files such as assembly. To report a diagnostic against a line of a diff --git a/golang.org/x/tools/go/analysis/internal/checker/checker.go b/golang.org/x/tools/go/analysis/internal/checker/checker.go index a26f801b58..5ccfb16374 100644 --- a/golang.org/x/tools/go/analysis/internal/checker/checker.go +++ b/golang.org/x/tools/go/analysis/internal/checker/checker.go @@ -800,8 +800,13 @@ func exportedFrom(obj types.Object, pkg *types.Package) bool { return obj.Exported() && obj.Pkg() == pkg || obj.Type().(*types.Signature).Recv() != nil case *types.Var: - return obj.Exported() && obj.Pkg() == pkg || - obj.IsField() + if obj.IsField() { + return true + } + // we can't filter more aggressively than this because we need + // to consider function parameters exported, but have no way + // of telling apart function parameters from local variables. + return obj.Pkg() == pkg case *types.TypeName, *types.Const: return true } diff --git a/golang.org/x/tools/go/analysis/passes/printf/printf.go b/golang.org/x/tools/go/analysis/passes/printf/printf.go index 14f3a47610..ddad4c796c 100644 --- a/golang.org/x/tools/go/analysis/passes/printf/printf.go +++ b/golang.org/x/tools/go/analysis/passes/printf/printf.go @@ -805,6 +805,7 @@ var printVerbs = []printVerb{ {'g', sharpNumFlag, argFloat | argComplex}, {'G', sharpNumFlag, argFloat | argComplex}, {'o', sharpNumFlag, argInt | argPointer}, + {'O', sharpNumFlag, argInt | argPointer}, {'p', "-#", argPointer}, {'q', " -+.0#", argRune | argInt | argString}, {'s', " -+.0", argString}, diff --git a/golang.org/x/tools/go/gcexportdata/gcexportdata.go b/golang.org/x/tools/go/gcexportdata/gcexportdata.go index 0d51acad99..f8363d8faa 100644 --- a/golang.org/x/tools/go/gcexportdata/gcexportdata.go +++ b/golang.org/x/tools/go/gcexportdata/gcexportdata.go @@ -85,11 +85,15 @@ func Read(in io.Reader, fset *token.FileSet, imports map[string]*types.Package, return gcimporter.ImportData(imports, path, path, bytes.NewReader(data)) } - // The indexed export format starts with an 'i'. - if len(data) == 0 || data[0] != 'i' { - return nil, fmt.Errorf("unknown export data format") + // The indexed export format starts with an 'i'; the older + // binary export format starts with a 'c', 'd', or 'v' + // (from "version"). Select appropriate importer. + if len(data) > 0 && data[0] == 'i' { + _, pkg, err := gcimporter.IImportData(fset, imports, data[1:], path) + return pkg, err } - _, pkg, err := gcimporter.IImportData(fset, imports, data[1:], path) + + _, pkg, err := gcimporter.BImportData(fset, imports, data, path) return pkg, err } diff --git a/golang.org/x/tools/go/internal/gcimporter/bexport.go b/golang.org/x/tools/go/internal/gcimporter/bexport.go new file mode 100644 index 0000000000..a807d0aaa2 --- /dev/null +++ b/golang.org/x/tools/go/internal/gcimporter/bexport.go @@ -0,0 +1,852 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Binary package export. +// This file was derived from $GOROOT/src/cmd/compile/internal/gc/bexport.go; +// see that file for specification of the format. + +package gcimporter + +import ( + "bytes" + "encoding/binary" + "fmt" + "go/ast" + "go/constant" + "go/token" + "go/types" + "math" + "math/big" + "sort" + "strings" +) + +// If debugFormat is set, each integer and string value is preceded by a marker +// and position information in the encoding. This mechanism permits an importer +// to recognize immediately when it is out of sync. The importer recognizes this +// mode automatically (i.e., it can import export data produced with debugging +// support even if debugFormat is not set at the time of import). This mode will +// lead to massively larger export data (by a factor of 2 to 3) and should only +// be enabled during development and debugging. +// +// NOTE: This flag is the first flag to enable if importing dies because of +// (suspected) format errors, and whenever a change is made to the format. +const debugFormat = false // default: false + +// If trace is set, debugging output is printed to std out. +const trace = false // default: false + +// Current export format version. Increase with each format change. +// Note: The latest binary (non-indexed) export format is at version 6. +// This exporter is still at level 4, but it doesn't matter since +// the binary importer can handle older versions just fine. +// 6: package height (CL 105038) -- NOT IMPLEMENTED HERE +// 5: improved position encoding efficiency (issue 20080, CL 41619) -- NOT IMPLEMEMTED HERE +// 4: type name objects support type aliases, uses aliasTag +// 3: Go1.8 encoding (same as version 2, aliasTag defined but never used) +// 2: removed unused bool in ODCL export (compiler only) +// 1: header format change (more regular), export package for _ struct fields +// 0: Go1.7 encoding +const exportVersion = 4 + +// trackAllTypes enables cycle tracking for all types, not just named +// types. The existing compiler invariants assume that unnamed types +// that are not completely set up are not used, or else there are spurious +// errors. +// If disabled, only named types are tracked, possibly leading to slightly +// less efficient encoding in rare cases. It also prevents the export of +// some corner-case type declarations (but those are not handled correctly +// with with the textual export format either). +// TODO(gri) enable and remove once issues caused by it are fixed +const trackAllTypes = false + +type exporter struct { + fset *token.FileSet + out bytes.Buffer + + // object -> index maps, indexed in order of serialization + strIndex map[string]int + pkgIndex map[*types.Package]int + typIndex map[types.Type]int + + // position encoding + posInfoFormat bool + prevFile string + prevLine int + + // debugging support + written int // bytes written + indent int // for trace +} + +// internalError represents an error generated inside this package. +type internalError string + +func (e internalError) Error() string { return "gcimporter: " + string(e) } + +func internalErrorf(format string, args ...interface{}) error { + return internalError(fmt.Sprintf(format, args...)) +} + +// BExportData returns binary export data for pkg. +// If no file set is provided, position info will be missing. +func BExportData(fset *token.FileSet, pkg *types.Package) (b []byte, err error) { + defer func() { + if e := recover(); e != nil { + if ierr, ok := e.(internalError); ok { + err = ierr + return + } + // Not an internal error; panic again. + panic(e) + } + }() + + p := exporter{ + fset: fset, + strIndex: map[string]int{"": 0}, // empty string is mapped to 0 + pkgIndex: make(map[*types.Package]int), + typIndex: make(map[types.Type]int), + posInfoFormat: true, // TODO(gri) might become a flag, eventually + } + + // write version info + // The version string must start with "version %d" where %d is the version + // number. Additional debugging information may follow after a blank; that + // text is ignored by the importer. + p.rawStringln(fmt.Sprintf("version %d", exportVersion)) + var debug string + if debugFormat { + debug = "debug" + } + p.rawStringln(debug) // cannot use p.bool since it's affected by debugFormat; also want to see this clearly + p.bool(trackAllTypes) + p.bool(p.posInfoFormat) + + // --- generic export data --- + + // populate type map with predeclared "known" types + for index, typ := range predeclared() { + p.typIndex[typ] = index + } + if len(p.typIndex) != len(predeclared()) { + return nil, internalError("duplicate entries in type map?") + } + + // write package data + p.pkg(pkg, true) + if trace { + p.tracef("\n") + } + + // write objects + objcount := 0 + scope := pkg.Scope() + for _, name := range scope.Names() { + if !ast.IsExported(name) { + continue + } + if trace { + p.tracef("\n") + } + p.obj(scope.Lookup(name)) + objcount++ + } + + // indicate end of list + if trace { + p.tracef("\n") + } + p.tag(endTag) + + // for self-verification only (redundant) + p.int(objcount) + + if trace { + p.tracef("\n") + } + + // --- end of export data --- + + return p.out.Bytes(), nil +} + +func (p *exporter) pkg(pkg *types.Package, emptypath bool) { + if pkg == nil { + panic(internalError("unexpected nil pkg")) + } + + // if we saw the package before, write its index (>= 0) + if i, ok := p.pkgIndex[pkg]; ok { + p.index('P', i) + return + } + + // otherwise, remember the package, write the package tag (< 0) and package data + if trace { + p.tracef("P%d = { ", len(p.pkgIndex)) + defer p.tracef("} ") + } + p.pkgIndex[pkg] = len(p.pkgIndex) + + p.tag(packageTag) + p.string(pkg.Name()) + if emptypath { + p.string("") + } else { + p.string(pkg.Path()) + } +} + +func (p *exporter) obj(obj types.Object) { + switch obj := obj.(type) { + case *types.Const: + p.tag(constTag) + p.pos(obj) + p.qualifiedName(obj) + p.typ(obj.Type()) + p.value(obj.Val()) + + case *types.TypeName: + if obj.IsAlias() { + p.tag(aliasTag) + p.pos(obj) + p.qualifiedName(obj) + } else { + p.tag(typeTag) + } + p.typ(obj.Type()) + + case *types.Var: + p.tag(varTag) + p.pos(obj) + p.qualifiedName(obj) + p.typ(obj.Type()) + + case *types.Func: + p.tag(funcTag) + p.pos(obj) + p.qualifiedName(obj) + sig := obj.Type().(*types.Signature) + p.paramList(sig.Params(), sig.Variadic()) + p.paramList(sig.Results(), false) + + default: + panic(internalErrorf("unexpected object %v (%T)", obj, obj)) + } +} + +func (p *exporter) pos(obj types.Object) { + if !p.posInfoFormat { + return + } + + file, line := p.fileLine(obj) + if file == p.prevFile { + // common case: write line delta + // delta == 0 means different file or no line change + delta := line - p.prevLine + p.int(delta) + if delta == 0 { + p.int(-1) // -1 means no file change + } + } else { + // different file + p.int(0) + // Encode filename as length of common prefix with previous + // filename, followed by (possibly empty) suffix. Filenames + // frequently share path prefixes, so this can save a lot + // of space and make export data size less dependent on file + // path length. The suffix is unlikely to be empty because + // file names tend to end in ".go". + n := commonPrefixLen(p.prevFile, file) + p.int(n) // n >= 0 + p.string(file[n:]) // write suffix only + p.prevFile = file + p.int(line) + } + p.prevLine = line +} + +func (p *exporter) fileLine(obj types.Object) (file string, line int) { + if p.fset != nil { + pos := p.fset.Position(obj.Pos()) + file = pos.Filename + line = pos.Line + } + return +} + +func commonPrefixLen(a, b string) int { + if len(a) > len(b) { + a, b = b, a + } + // len(a) <= len(b) + i := 0 + for i < len(a) && a[i] == b[i] { + i++ + } + return i +} + +func (p *exporter) qualifiedName(obj types.Object) { + p.string(obj.Name()) + p.pkg(obj.Pkg(), false) +} + +func (p *exporter) typ(t types.Type) { + if t == nil { + panic(internalError("nil type")) + } + + // Possible optimization: Anonymous pointer types *T where + // T is a named type are common. We could canonicalize all + // such types *T to a single type PT = *T. This would lead + // to at most one *T entry in typIndex, and all future *T's + // would be encoded as the respective index directly. Would + // save 1 byte (pointerTag) per *T and reduce the typIndex + // size (at the cost of a canonicalization map). We can do + // this later, without encoding format change. + + // if we saw the type before, write its index (>= 0) + if i, ok := p.typIndex[t]; ok { + p.index('T', i) + return + } + + // otherwise, remember the type, write the type tag (< 0) and type data + if trackAllTypes { + if trace { + p.tracef("T%d = {>\n", len(p.typIndex)) + defer p.tracef("<\n} ") + } + p.typIndex[t] = len(p.typIndex) + } + + switch t := t.(type) { + case *types.Named: + if !trackAllTypes { + // if we don't track all types, track named types now + p.typIndex[t] = len(p.typIndex) + } + + p.tag(namedTag) + p.pos(t.Obj()) + p.qualifiedName(t.Obj()) + p.typ(t.Underlying()) + if !types.IsInterface(t) { + p.assocMethods(t) + } + + case *types.Array: + p.tag(arrayTag) + p.int64(t.Len()) + p.typ(t.Elem()) + + case *types.Slice: + p.tag(sliceTag) + p.typ(t.Elem()) + + case *dddSlice: + p.tag(dddTag) + p.typ(t.elem) + + case *types.Struct: + p.tag(structTag) + p.fieldList(t) + + case *types.Pointer: + p.tag(pointerTag) + p.typ(t.Elem()) + + case *types.Signature: + p.tag(signatureTag) + p.paramList(t.Params(), t.Variadic()) + p.paramList(t.Results(), false) + + case *types.Interface: + p.tag(interfaceTag) + p.iface(t) + + case *types.Map: + p.tag(mapTag) + p.typ(t.Key()) + p.typ(t.Elem()) + + case *types.Chan: + p.tag(chanTag) + p.int(int(3 - t.Dir())) // hack + p.typ(t.Elem()) + + default: + panic(internalErrorf("unexpected type %T: %s", t, t)) + } +} + +func (p *exporter) assocMethods(named *types.Named) { + // Sort methods (for determinism). + var methods []*types.Func + for i := 0; i < named.NumMethods(); i++ { + methods = append(methods, named.Method(i)) + } + sort.Sort(methodsByName(methods)) + + p.int(len(methods)) + + if trace && methods != nil { + p.tracef("associated methods {>\n") + } + + for i, m := range methods { + if trace && i > 0 { + p.tracef("\n") + } + + p.pos(m) + name := m.Name() + p.string(name) + if !exported(name) { + p.pkg(m.Pkg(), false) + } + + sig := m.Type().(*types.Signature) + p.paramList(types.NewTuple(sig.Recv()), false) + p.paramList(sig.Params(), sig.Variadic()) + p.paramList(sig.Results(), false) + p.int(0) // dummy value for go:nointerface pragma - ignored by importer + } + + if trace && methods != nil { + p.tracef("<\n} ") + } +} + +type methodsByName []*types.Func + +func (x methodsByName) Len() int { return len(x) } +func (x methodsByName) Swap(i, j int) { x[i], x[j] = x[j], x[i] } +func (x methodsByName) Less(i, j int) bool { return x[i].Name() < x[j].Name() } + +func (p *exporter) fieldList(t *types.Struct) { + if trace && t.NumFields() > 0 { + p.tracef("fields {>\n") + defer p.tracef("<\n} ") + } + + p.int(t.NumFields()) + for i := 0; i < t.NumFields(); i++ { + if trace && i > 0 { + p.tracef("\n") + } + p.field(t.Field(i)) + p.string(t.Tag(i)) + } +} + +func (p *exporter) field(f *types.Var) { + if !f.IsField() { + panic(internalError("field expected")) + } + + p.pos(f) + p.fieldName(f) + p.typ(f.Type()) +} + +func (p *exporter) iface(t *types.Interface) { + // TODO(gri): enable importer to load embedded interfaces, + // then emit Embeddeds and ExplicitMethods separately here. + p.int(0) + + n := t.NumMethods() + if trace && n > 0 { + p.tracef("methods {>\n") + defer p.tracef("<\n} ") + } + p.int(n) + for i := 0; i < n; i++ { + if trace && i > 0 { + p.tracef("\n") + } + p.method(t.Method(i)) + } +} + +func (p *exporter) method(m *types.Func) { + sig := m.Type().(*types.Signature) + if sig.Recv() == nil { + panic(internalError("method expected")) + } + + p.pos(m) + p.string(m.Name()) + if m.Name() != "_" && !ast.IsExported(m.Name()) { + p.pkg(m.Pkg(), false) + } + + // interface method; no need to encode receiver. + p.paramList(sig.Params(), sig.Variadic()) + p.paramList(sig.Results(), false) +} + +func (p *exporter) fieldName(f *types.Var) { + name := f.Name() + + if f.Anonymous() { + // anonymous field - we distinguish between 3 cases: + // 1) field name matches base type name and is exported + // 2) field name matches base type name and is not exported + // 3) field name doesn't match base type name (alias name) + bname := basetypeName(f.Type()) + if name == bname { + if ast.IsExported(name) { + name = "" // 1) we don't need to know the field name or package + } else { + name = "?" // 2) use unexported name "?" to force package export + } + } else { + // 3) indicate alias and export name as is + // (this requires an extra "@" but this is a rare case) + p.string("@") + } + } + + p.string(name) + if name != "" && !ast.IsExported(name) { + p.pkg(f.Pkg(), false) + } +} + +func basetypeName(typ types.Type) string { + switch typ := deref(typ).(type) { + case *types.Basic: + return typ.Name() + case *types.Named: + return typ.Obj().Name() + default: + return "" // unnamed type + } +} + +func (p *exporter) paramList(params *types.Tuple, variadic bool) { + // use negative length to indicate unnamed parameters + // (look at the first parameter only since either all + // names are present or all are absent) + n := params.Len() + if n > 0 && params.At(0).Name() == "" { + n = -n + } + p.int(n) + for i := 0; i < params.Len(); i++ { + q := params.At(i) + t := q.Type() + if variadic && i == params.Len()-1 { + t = &dddSlice{t.(*types.Slice).Elem()} + } + p.typ(t) + if n > 0 { + name := q.Name() + p.string(name) + if name != "_" { + p.pkg(q.Pkg(), false) + } + } + p.string("") // no compiler-specific info + } +} + +func (p *exporter) value(x constant.Value) { + if trace { + p.tracef("= ") + } + + switch x.Kind() { + case constant.Bool: + tag := falseTag + if constant.BoolVal(x) { + tag = trueTag + } + p.tag(tag) + + case constant.Int: + if v, exact := constant.Int64Val(x); exact { + // common case: x fits into an int64 - use compact encoding + p.tag(int64Tag) + p.int64(v) + return + } + // uncommon case: large x - use float encoding + // (powers of 2 will be encoded efficiently with exponent) + p.tag(floatTag) + p.float(constant.ToFloat(x)) + + case constant.Float: + p.tag(floatTag) + p.float(x) + + case constant.Complex: + p.tag(complexTag) + p.float(constant.Real(x)) + p.float(constant.Imag(x)) + + case constant.String: + p.tag(stringTag) + p.string(constant.StringVal(x)) + + case constant.Unknown: + // package contains type errors + p.tag(unknownTag) + + default: + panic(internalErrorf("unexpected value %v (%T)", x, x)) + } +} + +func (p *exporter) float(x constant.Value) { + if x.Kind() != constant.Float { + panic(internalErrorf("unexpected constant %v, want float", x)) + } + // extract sign (there is no -0) + sign := constant.Sign(x) + if sign == 0 { + // x == 0 + p.int(0) + return + } + // x != 0 + + var f big.Float + if v, exact := constant.Float64Val(x); exact { + // float64 + f.SetFloat64(v) + } else if num, denom := constant.Num(x), constant.Denom(x); num.Kind() == constant.Int { + // TODO(gri): add big.Rat accessor to constant.Value. + r := valueToRat(num) + f.SetRat(r.Quo(r, valueToRat(denom))) + } else { + // Value too large to represent as a fraction => inaccessible. + // TODO(gri): add big.Float accessor to constant.Value. + f.SetFloat64(math.MaxFloat64) // FIXME + } + + // extract exponent such that 0.5 <= m < 1.0 + var m big.Float + exp := f.MantExp(&m) + + // extract mantissa as *big.Int + // - set exponent large enough so mant satisfies mant.IsInt() + // - get *big.Int from mant + m.SetMantExp(&m, int(m.MinPrec())) + mant, acc := m.Int(nil) + if acc != big.Exact { + panic(internalError("internal error")) + } + + p.int(sign) + p.int(exp) + p.string(string(mant.Bytes())) +} + +func valueToRat(x constant.Value) *big.Rat { + // Convert little-endian to big-endian. + // I can't believe this is necessary. + bytes := constant.Bytes(x) + for i := 0; i < len(bytes)/2; i++ { + bytes[i], bytes[len(bytes)-1-i] = bytes[len(bytes)-1-i], bytes[i] + } + return new(big.Rat).SetInt(new(big.Int).SetBytes(bytes)) +} + +func (p *exporter) bool(b bool) bool { + if trace { + p.tracef("[") + defer p.tracef("= %v] ", b) + } + + x := 0 + if b { + x = 1 + } + p.int(x) + return b +} + +// ---------------------------------------------------------------------------- +// Low-level encoders + +func (p *exporter) index(marker byte, index int) { + if index < 0 { + panic(internalError("invalid index < 0")) + } + if debugFormat { + p.marker('t') + } + if trace { + p.tracef("%c%d ", marker, index) + } + p.rawInt64(int64(index)) +} + +func (p *exporter) tag(tag int) { + if tag >= 0 { + panic(internalError("invalid tag >= 0")) + } + if debugFormat { + p.marker('t') + } + if trace { + p.tracef("%s ", tagString[-tag]) + } + p.rawInt64(int64(tag)) +} + +func (p *exporter) int(x int) { + p.int64(int64(x)) +} + +func (p *exporter) int64(x int64) { + if debugFormat { + p.marker('i') + } + if trace { + p.tracef("%d ", x) + } + p.rawInt64(x) +} + +func (p *exporter) string(s string) { + if debugFormat { + p.marker('s') + } + if trace { + p.tracef("%q ", s) + } + // if we saw the string before, write its index (>= 0) + // (the empty string is mapped to 0) + if i, ok := p.strIndex[s]; ok { + p.rawInt64(int64(i)) + return + } + // otherwise, remember string and write its negative length and bytes + p.strIndex[s] = len(p.strIndex) + p.rawInt64(-int64(len(s))) + for i := 0; i < len(s); i++ { + p.rawByte(s[i]) + } +} + +// marker emits a marker byte and position information which makes +// it easy for a reader to detect if it is "out of sync". Used for +// debugFormat format only. +func (p *exporter) marker(m byte) { + p.rawByte(m) + // Enable this for help tracking down the location + // of an incorrect marker when running in debugFormat. + if false && trace { + p.tracef("#%d ", p.written) + } + p.rawInt64(int64(p.written)) +} + +// rawInt64 should only be used by low-level encoders. +func (p *exporter) rawInt64(x int64) { + var tmp [binary.MaxVarintLen64]byte + n := binary.PutVarint(tmp[:], x) + for i := 0; i < n; i++ { + p.rawByte(tmp[i]) + } +} + +// rawStringln should only be used to emit the initial version string. +func (p *exporter) rawStringln(s string) { + for i := 0; i < len(s); i++ { + p.rawByte(s[i]) + } + p.rawByte('\n') +} + +// rawByte is the bottleneck interface to write to p.out. +// rawByte escapes b as follows (any encoding does that +// hides '$'): +// +// '$' => '|' 'S' +// '|' => '|' '|' +// +// Necessary so other tools can find the end of the +// export data by searching for "$$". +// rawByte should only be used by low-level encoders. +func (p *exporter) rawByte(b byte) { + switch b { + case '$': + // write '$' as '|' 'S' + b = 'S' + fallthrough + case '|': + // write '|' as '|' '|' + p.out.WriteByte('|') + p.written++ + } + p.out.WriteByte(b) + p.written++ +} + +// tracef is like fmt.Printf but it rewrites the format string +// to take care of indentation. +func (p *exporter) tracef(format string, args ...interface{}) { + if strings.ContainsAny(format, "<>\n") { + var buf bytes.Buffer + for i := 0; i < len(format); i++ { + // no need to deal with runes + ch := format[i] + switch ch { + case '>': + p.indent++ + continue + case '<': + p.indent-- + continue + } + buf.WriteByte(ch) + if ch == '\n' { + for j := p.indent; j > 0; j-- { + buf.WriteString(". ") + } + } + } + format = buf.String() + } + fmt.Printf(format, args...) +} + +// Debugging support. +// (tagString is only used when tracing is enabled) +var tagString = [...]string{ + // Packages + -packageTag: "package", + + // Types + -namedTag: "named type", + -arrayTag: "array", + -sliceTag: "slice", + -dddTag: "ddd", + -structTag: "struct", + -pointerTag: "pointer", + -signatureTag: "signature", + -interfaceTag: "interface", + -mapTag: "map", + -chanTag: "chan", + + // Values + -falseTag: "false", + -trueTag: "true", + -int64Tag: "int64", + -floatTag: "float", + -fractionTag: "fraction", + -complexTag: "complex", + -stringTag: "string", + -unknownTag: "unknown", + + // Type aliases + -aliasTag: "alias", +} diff --git a/golang.org/x/tools/go/internal/gcimporter/bimport.go b/golang.org/x/tools/go/internal/gcimporter/bimport.go new file mode 100644 index 0000000000..e9f73d14a1 --- /dev/null +++ b/golang.org/x/tools/go/internal/gcimporter/bimport.go @@ -0,0 +1,1039 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file is a copy of $GOROOT/src/go/internal/gcimporter/bimport.go. + +package gcimporter + +import ( + "encoding/binary" + "fmt" + "go/constant" + "go/token" + "go/types" + "sort" + "strconv" + "strings" + "sync" + "unicode" + "unicode/utf8" +) + +type importer struct { + imports map[string]*types.Package + data []byte + importpath string + buf []byte // for reading strings + version int // export format version + + // object lists + strList []string // in order of appearance + pathList []string // in order of appearance + pkgList []*types.Package // in order of appearance + typList []types.Type // in order of appearance + interfaceList []*types.Interface // for delayed completion only + trackAllTypes bool + + // position encoding + posInfoFormat bool + prevFile string + prevLine int + fake fakeFileSet + + // debugging support + debugFormat bool + read int // bytes read +} + +// BImportData imports a package from the serialized package data +// and returns the number of bytes consumed and a reference to the package. +// If the export data version is not recognized or the format is otherwise +// compromised, an error is returned. +func BImportData(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string) (_ int, pkg *types.Package, err error) { + // catch panics and return them as errors + const currentVersion = 6 + version := -1 // unknown version + defer func() { + if e := recover(); e != nil { + // Return a (possibly nil or incomplete) package unchanged (see #16088). + if version > currentVersion { + err = fmt.Errorf("cannot import %q (%v), export data is newer version - update tool", path, e) + } else { + err = fmt.Errorf("cannot import %q (%v), possibly version skew - reinstall package", path, e) + } + } + }() + + p := importer{ + imports: imports, + data: data, + importpath: path, + version: version, + strList: []string{""}, // empty string is mapped to 0 + pathList: []string{""}, // empty string is mapped to 0 + fake: fakeFileSet{ + fset: fset, + files: make(map[string]*token.File), + }, + } + + // read version info + var versionstr string + if b := p.rawByte(); b == 'c' || b == 'd' { + // Go1.7 encoding; first byte encodes low-level + // encoding format (compact vs debug). + // For backward-compatibility only (avoid problems with + // old installed packages). Newly compiled packages use + // the extensible format string. + // TODO(gri) Remove this support eventually; after Go1.8. + if b == 'd' { + p.debugFormat = true + } + p.trackAllTypes = p.rawByte() == 'a' + p.posInfoFormat = p.int() != 0 + versionstr = p.string() + if versionstr == "v1" { + version = 0 + } + } else { + // Go1.8 extensible encoding + // read version string and extract version number (ignore anything after the version number) + versionstr = p.rawStringln(b) + if s := strings.SplitN(versionstr, " ", 3); len(s) >= 2 && s[0] == "version" { + if v, err := strconv.Atoi(s[1]); err == nil && v > 0 { + version = v + } + } + } + p.version = version + + // read version specific flags - extend as necessary + switch p.version { + // case currentVersion: + // ... + // fallthrough + case currentVersion, 5, 4, 3, 2, 1: + p.debugFormat = p.rawStringln(p.rawByte()) == "debug" + p.trackAllTypes = p.int() != 0 + p.posInfoFormat = p.int() != 0 + case 0: + // Go1.7 encoding format - nothing to do here + default: + errorf("unknown bexport format version %d (%q)", p.version, versionstr) + } + + // --- generic export data --- + + // populate typList with predeclared "known" types + p.typList = append(p.typList, predeclared()...) + + // read package data + pkg = p.pkg() + + // read objects of phase 1 only (see cmd/compile/internal/gc/bexport.go) + objcount := 0 + for { + tag := p.tagOrIndex() + if tag == endTag { + break + } + p.obj(tag) + objcount++ + } + + // self-verification + if count := p.int(); count != objcount { + errorf("got %d objects; want %d", objcount, count) + } + + // ignore compiler-specific import data + + // complete interfaces + // TODO(gri) re-investigate if we still need to do this in a delayed fashion + for _, typ := range p.interfaceList { + typ.Complete() + } + + // record all referenced packages as imports + list := append(([]*types.Package)(nil), p.pkgList[1:]...) + sort.Sort(byPath(list)) + pkg.SetImports(list) + + // package was imported completely and without errors + pkg.MarkComplete() + + return p.read, pkg, nil +} + +func errorf(format string, args ...interface{}) { + panic(fmt.Sprintf(format, args...)) +} + +func (p *importer) pkg() *types.Package { + // if the package was seen before, i is its index (>= 0) + i := p.tagOrIndex() + if i >= 0 { + return p.pkgList[i] + } + + // otherwise, i is the package tag (< 0) + if i != packageTag { + errorf("unexpected package tag %d version %d", i, p.version) + } + + // read package data + name := p.string() + var path string + if p.version >= 5 { + path = p.path() + } else { + path = p.string() + } + if p.version >= 6 { + p.int() // package height; unused by go/types + } + + // we should never see an empty package name + if name == "" { + errorf("empty package name in import") + } + + // an empty path denotes the package we are currently importing; + // it must be the first package we see + if (path == "") != (len(p.pkgList) == 0) { + errorf("package path %q for pkg index %d", path, len(p.pkgList)) + } + + // if the package was imported before, use that one; otherwise create a new one + if path == "" { + path = p.importpath + } + pkg := p.imports[path] + if pkg == nil { + pkg = types.NewPackage(path, name) + p.imports[path] = pkg + } else if pkg.Name() != name { + errorf("conflicting names %s and %s for package %q", pkg.Name(), name, path) + } + p.pkgList = append(p.pkgList, pkg) + + return pkg +} + +// objTag returns the tag value for each object kind. +func objTag(obj types.Object) int { + switch obj.(type) { + case *types.Const: + return constTag + case *types.TypeName: + return typeTag + case *types.Var: + return varTag + case *types.Func: + return funcTag + default: + errorf("unexpected object: %v (%T)", obj, obj) // panics + panic("unreachable") + } +} + +func sameObj(a, b types.Object) bool { + // Because unnamed types are not canonicalized, we cannot simply compare types for + // (pointer) identity. + // Ideally we'd check equality of constant values as well, but this is good enough. + return objTag(a) == objTag(b) && types.Identical(a.Type(), b.Type()) +} + +func (p *importer) declare(obj types.Object) { + pkg := obj.Pkg() + if alt := pkg.Scope().Insert(obj); alt != nil { + // This can only trigger if we import a (non-type) object a second time. + // Excluding type aliases, this cannot happen because 1) we only import a package + // once; and b) we ignore compiler-specific export data which may contain + // functions whose inlined function bodies refer to other functions that + // were already imported. + // However, type aliases require reexporting the original type, so we need + // to allow it (see also the comment in cmd/compile/internal/gc/bimport.go, + // method importer.obj, switch case importing functions). + // TODO(gri) review/update this comment once the gc compiler handles type aliases. + if !sameObj(obj, alt) { + errorf("inconsistent import:\n\t%v\npreviously imported as:\n\t%v\n", obj, alt) + } + } +} + +func (p *importer) obj(tag int) { + switch tag { + case constTag: + pos := p.pos() + pkg, name := p.qualifiedName() + typ := p.typ(nil, nil) + val := p.value() + p.declare(types.NewConst(pos, pkg, name, typ, val)) + + case aliasTag: + // TODO(gri) verify type alias hookup is correct + pos := p.pos() + pkg, name := p.qualifiedName() + typ := p.typ(nil, nil) + p.declare(types.NewTypeName(pos, pkg, name, typ)) + + case typeTag: + p.typ(nil, nil) + + case varTag: + pos := p.pos() + pkg, name := p.qualifiedName() + typ := p.typ(nil, nil) + p.declare(types.NewVar(pos, pkg, name, typ)) + + case funcTag: + pos := p.pos() + pkg, name := p.qualifiedName() + params, isddd := p.paramList() + result, _ := p.paramList() + sig := types.NewSignature(nil, params, result, isddd) + p.declare(types.NewFunc(pos, pkg, name, sig)) + + default: + errorf("unexpected object tag %d", tag) + } +} + +const deltaNewFile = -64 // see cmd/compile/internal/gc/bexport.go + +func (p *importer) pos() token.Pos { + if !p.posInfoFormat { + return token.NoPos + } + + file := p.prevFile + line := p.prevLine + delta := p.int() + line += delta + if p.version >= 5 { + if delta == deltaNewFile { + if n := p.int(); n >= 0 { + // file changed + file = p.path() + line = n + } + } + } else { + if delta == 0 { + if n := p.int(); n >= 0 { + // file changed + file = p.prevFile[:n] + p.string() + line = p.int() + } + } + } + p.prevFile = file + p.prevLine = line + + return p.fake.pos(file, line, 0) +} + +// Synthesize a token.Pos +type fakeFileSet struct { + fset *token.FileSet + files map[string]*token.File +} + +func (s *fakeFileSet) pos(file string, line, column int) token.Pos { + // TODO(mdempsky): Make use of column. + + // Since we don't know the set of needed file positions, we + // reserve maxlines positions per file. + const maxlines = 64 * 1024 + f := s.files[file] + if f == nil { + f = s.fset.AddFile(file, -1, maxlines) + s.files[file] = f + // Allocate the fake linebreak indices on first use. + // TODO(adonovan): opt: save ~512KB using a more complex scheme? + fakeLinesOnce.Do(func() { + fakeLines = make([]int, maxlines) + for i := range fakeLines { + fakeLines[i] = i + } + }) + f.SetLines(fakeLines) + } + + if line > maxlines { + line = 1 + } + + // Treat the file as if it contained only newlines + // and column=1: use the line number as the offset. + return f.Pos(line - 1) +} + +var ( + fakeLines []int + fakeLinesOnce sync.Once +) + +func (p *importer) qualifiedName() (pkg *types.Package, name string) { + name = p.string() + pkg = p.pkg() + return +} + +func (p *importer) record(t types.Type) { + p.typList = append(p.typList, t) +} + +// A dddSlice is a types.Type representing ...T parameters. +// It only appears for parameter types and does not escape +// the importer. +type dddSlice struct { + elem types.Type +} + +func (t *dddSlice) Underlying() types.Type { return t } +func (t *dddSlice) String() string { return "..." + t.elem.String() } + +// parent is the package which declared the type; parent == nil means +// the package currently imported. The parent package is needed for +// exported struct fields and interface methods which don't contain +// explicit package information in the export data. +// +// A non-nil tname is used as the "owner" of the result type; i.e., +// the result type is the underlying type of tname. tname is used +// to give interface methods a named receiver type where possible. +func (p *importer) typ(parent *types.Package, tname *types.Named) types.Type { + // if the type was seen before, i is its index (>= 0) + i := p.tagOrIndex() + if i >= 0 { + return p.typList[i] + } + + // otherwise, i is the type tag (< 0) + switch i { + case namedTag: + // read type object + pos := p.pos() + parent, name := p.qualifiedName() + scope := parent.Scope() + obj := scope.Lookup(name) + + // if the object doesn't exist yet, create and insert it + if obj == nil { + obj = types.NewTypeName(pos, parent, name, nil) + scope.Insert(obj) + } + + if _, ok := obj.(*types.TypeName); !ok { + errorf("pkg = %s, name = %s => %s", parent, name, obj) + } + + // associate new named type with obj if it doesn't exist yet + t0 := types.NewNamed(obj.(*types.TypeName), nil, nil) + + // but record the existing type, if any + tname := obj.Type().(*types.Named) // tname is either t0 or the existing type + p.record(tname) + + // read underlying type + t0.SetUnderlying(p.typ(parent, t0)) + + // interfaces don't have associated methods + if types.IsInterface(t0) { + return tname + } + + // read associated methods + for i := p.int(); i > 0; i-- { + // TODO(gri) replace this with something closer to fieldName + pos := p.pos() + name := p.string() + if !exported(name) { + p.pkg() + } + + recv, _ := p.paramList() // TODO(gri) do we need a full param list for the receiver? + params, isddd := p.paramList() + result, _ := p.paramList() + p.int() // go:nointerface pragma - discarded + + sig := types.NewSignature(recv.At(0), params, result, isddd) + t0.AddMethod(types.NewFunc(pos, parent, name, sig)) + } + + return tname + + case arrayTag: + t := new(types.Array) + if p.trackAllTypes { + p.record(t) + } + + n := p.int64() + *t = *types.NewArray(p.typ(parent, nil), n) + return t + + case sliceTag: + t := new(types.Slice) + if p.trackAllTypes { + p.record(t) + } + + *t = *types.NewSlice(p.typ(parent, nil)) + return t + + case dddTag: + t := new(dddSlice) + if p.trackAllTypes { + p.record(t) + } + + t.elem = p.typ(parent, nil) + return t + + case structTag: + t := new(types.Struct) + if p.trackAllTypes { + p.record(t) + } + + *t = *types.NewStruct(p.fieldList(parent)) + return t + + case pointerTag: + t := new(types.Pointer) + if p.trackAllTypes { + p.record(t) + } + + *t = *types.NewPointer(p.typ(parent, nil)) + return t + + case signatureTag: + t := new(types.Signature) + if p.trackAllTypes { + p.record(t) + } + + params, isddd := p.paramList() + result, _ := p.paramList() + *t = *types.NewSignature(nil, params, result, isddd) + return t + + case interfaceTag: + // Create a dummy entry in the type list. This is safe because we + // cannot expect the interface type to appear in a cycle, as any + // such cycle must contain a named type which would have been + // first defined earlier. + // TODO(gri) Is this still true now that we have type aliases? + // See issue #23225. + n := len(p.typList) + if p.trackAllTypes { + p.record(nil) + } + + var embeddeds []types.Type + for n := p.int(); n > 0; n-- { + p.pos() + embeddeds = append(embeddeds, p.typ(parent, nil)) + } + + t := newInterface(p.methodList(parent, tname), embeddeds) + p.interfaceList = append(p.interfaceList, t) + if p.trackAllTypes { + p.typList[n] = t + } + return t + + case mapTag: + t := new(types.Map) + if p.trackAllTypes { + p.record(t) + } + + key := p.typ(parent, nil) + val := p.typ(parent, nil) + *t = *types.NewMap(key, val) + return t + + case chanTag: + t := new(types.Chan) + if p.trackAllTypes { + p.record(t) + } + + dir := chanDir(p.int()) + val := p.typ(parent, nil) + *t = *types.NewChan(dir, val) + return t + + default: + errorf("unexpected type tag %d", i) // panics + panic("unreachable") + } +} + +func chanDir(d int) types.ChanDir { + // tag values must match the constants in cmd/compile/internal/gc/go.go + switch d { + case 1 /* Crecv */ : + return types.RecvOnly + case 2 /* Csend */ : + return types.SendOnly + case 3 /* Cboth */ : + return types.SendRecv + default: + errorf("unexpected channel dir %d", d) + return 0 + } +} + +func (p *importer) fieldList(parent *types.Package) (fields []*types.Var, tags []string) { + if n := p.int(); n > 0 { + fields = make([]*types.Var, n) + tags = make([]string, n) + for i := range fields { + fields[i], tags[i] = p.field(parent) + } + } + return +} + +func (p *importer) field(parent *types.Package) (*types.Var, string) { + pos := p.pos() + pkg, name, alias := p.fieldName(parent) + typ := p.typ(parent, nil) + tag := p.string() + + anonymous := false + if name == "" { + // anonymous field - typ must be T or *T and T must be a type name + switch typ := deref(typ).(type) { + case *types.Basic: // basic types are named types + pkg = nil // // objects defined in Universe scope have no package + name = typ.Name() + case *types.Named: + name = typ.Obj().Name() + default: + errorf("named base type expected") + } + anonymous = true + } else if alias { + // anonymous field: we have an explicit name because it's an alias + anonymous = true + } + + return types.NewField(pos, pkg, name, typ, anonymous), tag +} + +func (p *importer) methodList(parent *types.Package, baseType *types.Named) (methods []*types.Func) { + if n := p.int(); n > 0 { + methods = make([]*types.Func, n) + for i := range methods { + methods[i] = p.method(parent, baseType) + } + } + return +} + +func (p *importer) method(parent *types.Package, baseType *types.Named) *types.Func { + pos := p.pos() + pkg, name, _ := p.fieldName(parent) + // If we don't have a baseType, use a nil receiver. + // A receiver using the actual interface type (which + // we don't know yet) will be filled in when we call + // types.Interface.Complete. + var recv *types.Var + if baseType != nil { + recv = types.NewVar(token.NoPos, parent, "", baseType) + } + params, isddd := p.paramList() + result, _ := p.paramList() + sig := types.NewSignature(recv, params, result, isddd) + return types.NewFunc(pos, pkg, name, sig) +} + +func (p *importer) fieldName(parent *types.Package) (pkg *types.Package, name string, alias bool) { + name = p.string() + pkg = parent + if pkg == nil { + // use the imported package instead + pkg = p.pkgList[0] + } + if p.version == 0 && name == "_" { + // version 0 didn't export a package for _ fields + return + } + switch name { + case "": + // 1) field name matches base type name and is exported: nothing to do + case "?": + // 2) field name matches base type name and is not exported: need package + name = "" + pkg = p.pkg() + case "@": + // 3) field name doesn't match type name (alias) + name = p.string() + alias = true + fallthrough + default: + if !exported(name) { + pkg = p.pkg() + } + } + return +} + +func (p *importer) paramList() (*types.Tuple, bool) { + n := p.int() + if n == 0 { + return nil, false + } + // negative length indicates unnamed parameters + named := true + if n < 0 { + n = -n + named = false + } + // n > 0 + params := make([]*types.Var, n) + isddd := false + for i := range params { + params[i], isddd = p.param(named) + } + return types.NewTuple(params...), isddd +} + +func (p *importer) param(named bool) (*types.Var, bool) { + t := p.typ(nil, nil) + td, isddd := t.(*dddSlice) + if isddd { + t = types.NewSlice(td.elem) + } + + var pkg *types.Package + var name string + if named { + name = p.string() + if name == "" { + errorf("expected named parameter") + } + if name != "_" { + pkg = p.pkg() + } + if i := strings.Index(name, "·"); i > 0 { + name = name[:i] // cut off gc-specific parameter numbering + } + } + + // read and discard compiler-specific info + p.string() + + return types.NewVar(token.NoPos, pkg, name, t), isddd +} + +func exported(name string) bool { + ch, _ := utf8.DecodeRuneInString(name) + return unicode.IsUpper(ch) +} + +func (p *importer) value() constant.Value { + switch tag := p.tagOrIndex(); tag { + case falseTag: + return constant.MakeBool(false) + case trueTag: + return constant.MakeBool(true) + case int64Tag: + return constant.MakeInt64(p.int64()) + case floatTag: + return p.float() + case complexTag: + re := p.float() + im := p.float() + return constant.BinaryOp(re, token.ADD, constant.MakeImag(im)) + case stringTag: + return constant.MakeString(p.string()) + case unknownTag: + return constant.MakeUnknown() + default: + errorf("unexpected value tag %d", tag) // panics + panic("unreachable") + } +} + +func (p *importer) float() constant.Value { + sign := p.int() + if sign == 0 { + return constant.MakeInt64(0) + } + + exp := p.int() + mant := []byte(p.string()) // big endian + + // remove leading 0's if any + for len(mant) > 0 && mant[0] == 0 { + mant = mant[1:] + } + + // convert to little endian + // TODO(gri) go/constant should have a more direct conversion function + // (e.g., once it supports a big.Float based implementation) + for i, j := 0, len(mant)-1; i < j; i, j = i+1, j-1 { + mant[i], mant[j] = mant[j], mant[i] + } + + // adjust exponent (constant.MakeFromBytes creates an integer value, + // but mant represents the mantissa bits such that 0.5 <= mant < 1.0) + exp -= len(mant) << 3 + if len(mant) > 0 { + for msd := mant[len(mant)-1]; msd&0x80 == 0; msd <<= 1 { + exp++ + } + } + + x := constant.MakeFromBytes(mant) + switch { + case exp < 0: + d := constant.Shift(constant.MakeInt64(1), token.SHL, uint(-exp)) + x = constant.BinaryOp(x, token.QUO, d) + case exp > 0: + x = constant.Shift(x, token.SHL, uint(exp)) + } + + if sign < 0 { + x = constant.UnaryOp(token.SUB, x, 0) + } + return x +} + +// ---------------------------------------------------------------------------- +// Low-level decoders + +func (p *importer) tagOrIndex() int { + if p.debugFormat { + p.marker('t') + } + + return int(p.rawInt64()) +} + +func (p *importer) int() int { + x := p.int64() + if int64(int(x)) != x { + errorf("exported integer too large") + } + return int(x) +} + +func (p *importer) int64() int64 { + if p.debugFormat { + p.marker('i') + } + + return p.rawInt64() +} + +func (p *importer) path() string { + if p.debugFormat { + p.marker('p') + } + // if the path was seen before, i is its index (>= 0) + // (the empty string is at index 0) + i := p.rawInt64() + if i >= 0 { + return p.pathList[i] + } + // otherwise, i is the negative path length (< 0) + a := make([]string, -i) + for n := range a { + a[n] = p.string() + } + s := strings.Join(a, "/") + p.pathList = append(p.pathList, s) + return s +} + +func (p *importer) string() string { + if p.debugFormat { + p.marker('s') + } + // if the string was seen before, i is its index (>= 0) + // (the empty string is at index 0) + i := p.rawInt64() + if i >= 0 { + return p.strList[i] + } + // otherwise, i is the negative string length (< 0) + if n := int(-i); n <= cap(p.buf) { + p.buf = p.buf[:n] + } else { + p.buf = make([]byte, n) + } + for i := range p.buf { + p.buf[i] = p.rawByte() + } + s := string(p.buf) + p.strList = append(p.strList, s) + return s +} + +func (p *importer) marker(want byte) { + if got := p.rawByte(); got != want { + errorf("incorrect marker: got %c; want %c (pos = %d)", got, want, p.read) + } + + pos := p.read + if n := int(p.rawInt64()); n != pos { + errorf("incorrect position: got %d; want %d", n, pos) + } +} + +// rawInt64 should only be used by low-level decoders. +func (p *importer) rawInt64() int64 { + i, err := binary.ReadVarint(p) + if err != nil { + errorf("read error: %v", err) + } + return i +} + +// rawStringln should only be used to read the initial version string. +func (p *importer) rawStringln(b byte) string { + p.buf = p.buf[:0] + for b != '\n' { + p.buf = append(p.buf, b) + b = p.rawByte() + } + return string(p.buf) +} + +// needed for binary.ReadVarint in rawInt64 +func (p *importer) ReadByte() (byte, error) { + return p.rawByte(), nil +} + +// byte is the bottleneck interface for reading p.data. +// It unescapes '|' 'S' to '$' and '|' '|' to '|'. +// rawByte should only be used by low-level decoders. +func (p *importer) rawByte() byte { + b := p.data[0] + r := 1 + if b == '|' { + b = p.data[1] + r = 2 + switch b { + case 'S': + b = '$' + case '|': + // nothing to do + default: + errorf("unexpected escape sequence in export data") + } + } + p.data = p.data[r:] + p.read += r + return b + +} + +// ---------------------------------------------------------------------------- +// Export format + +// Tags. Must be < 0. +const ( + // Objects + packageTag = -(iota + 1) + constTag + typeTag + varTag + funcTag + endTag + + // Types + namedTag + arrayTag + sliceTag + dddTag + structTag + pointerTag + signatureTag + interfaceTag + mapTag + chanTag + + // Values + falseTag + trueTag + int64Tag + floatTag + fractionTag // not used by gc + complexTag + stringTag + nilTag // only used by gc (appears in exported inlined function bodies) + unknownTag // not used by gc (only appears in packages with errors) + + // Type aliases + aliasTag +) + +var predeclOnce sync.Once +var predecl []types.Type // initialized lazily + +func predeclared() []types.Type { + predeclOnce.Do(func() { + // initialize lazily to be sure that all + // elements have been initialized before + predecl = []types.Type{ // basic types + types.Typ[types.Bool], + types.Typ[types.Int], + types.Typ[types.Int8], + types.Typ[types.Int16], + types.Typ[types.Int32], + types.Typ[types.Int64], + types.Typ[types.Uint], + types.Typ[types.Uint8], + types.Typ[types.Uint16], + types.Typ[types.Uint32], + types.Typ[types.Uint64], + types.Typ[types.Uintptr], + types.Typ[types.Float32], + types.Typ[types.Float64], + types.Typ[types.Complex64], + types.Typ[types.Complex128], + types.Typ[types.String], + + // basic type aliases + types.Universe.Lookup("byte").Type(), + types.Universe.Lookup("rune").Type(), + + // error + types.Universe.Lookup("error").Type(), + + // untyped types + types.Typ[types.UntypedBool], + types.Typ[types.UntypedInt], + types.Typ[types.UntypedRune], + types.Typ[types.UntypedFloat], + types.Typ[types.UntypedComplex], + types.Typ[types.UntypedString], + types.Typ[types.UntypedNil], + + // package unsafe + types.Typ[types.UnsafePointer], + + // invalid type + types.Typ[types.Invalid], // only appears in packages with errors + + // used internally by gc; never used by this package or in .a files + anyType{}, + } + }) + return predecl +} + +type anyType struct{} + +func (t anyType) Underlying() types.Type { return t } +func (t anyType) String() string { return "any" } diff --git a/golang.org/x/tools/go/internal/gcimporter/gcimporter.go b/golang.org/x/tools/go/internal/gcimporter/gcimporter.go index 6a9265ea94..8dcd8bbb71 100644 --- a/golang.org/x/tools/go/internal/gcimporter/gcimporter.go +++ b/golang.org/x/tools/go/internal/gcimporter/gcimporter.go @@ -204,11 +204,14 @@ func Import(packages map[string]*types.Package, path, srcDir string, lookup func // Or, define a new standard go/types/gcexportdata package. fset := token.NewFileSet() - // The indexed export format starts with an 'i'. - if len(data) == 0 || data[0] != 'i' { - return nil, fmt.Errorf("unknown export data format") + // The indexed export format starts with an 'i'; the older + // binary export format starts with a 'c', 'd', or 'v' + // (from "version"). Select appropriate importer. + if len(data) > 0 && data[0] == 'i' { + _, pkg, err = IImportData(fset, packages, data[1:], id) + } else { + _, pkg, err = BImportData(fset, packages, data, id) } - _, pkg, err = IImportData(fset, packages, data[1:], id) default: err = fmt.Errorf("unknown export data header: %q", hdr) diff --git a/golang.org/x/tools/go/internal/gcimporter/iexport.go b/golang.org/x/tools/go/internal/gcimporter/iexport.go index 858eb9f456..4be32a2e55 100644 --- a/golang.org/x/tools/go/internal/gcimporter/iexport.go +++ b/golang.org/x/tools/go/internal/gcimporter/iexport.go @@ -11,7 +11,6 @@ package gcimporter import ( "bytes" "encoding/binary" - "fmt" "go/ast" "go/constant" "go/token" @@ -26,15 +25,6 @@ import ( // 0: Go1.11 encoding const iexportVersion = 0 -// internalError represents an error generated inside this package. -type internalError string - -func (e internalError) Error() string { return "gcimporter: " + string(e) } - -func internalErrorf(format string, args ...interface{}) error { - return internalError(fmt.Sprintf(format, args...)) -} - // IExportData returns the binary export data for pkg. // // If no file set is provided, position info will be missing. @@ -538,16 +528,6 @@ func constantToFloat(x constant.Value) *big.Float { return &f } -func valueToRat(x constant.Value) *big.Rat { - // Convert little-endian to big-endian. - // I can't believe this is necessary. - bytes := constant.Bytes(x) - for i := 0; i < len(bytes)/2; i++ { - bytes[i], bytes[len(bytes)-1-i] = bytes[len(bytes)-1-i], bytes[i] - } - return new(big.Rat).SetInt(new(big.Int).SetBytes(bytes)) -} - // mpint exports a multi-precision integer. // // For unsigned types, small values are written out as a single diff --git a/golang.org/x/tools/go/internal/gcimporter/iimport.go b/golang.org/x/tools/go/internal/gcimporter/iimport.go index fef8b30080..a31a880263 100644 --- a/golang.org/x/tools/go/internal/gcimporter/iimport.go +++ b/golang.org/x/tools/go/internal/gcimporter/iimport.go @@ -18,9 +18,6 @@ import ( "go/types" "io" "sort" - "sync" - "unicode" - "unicode/utf8" ) type intReader struct { @@ -28,10 +25,6 @@ type intReader struct { path string } -func errorf(format string, args ...interface{}) { - panic(fmt.Sprintf(format, args...)) -} - func (r *intReader) int64() int64 { i, err := binary.ReadVarint(r.Reader) if err != nil { @@ -635,166 +628,3 @@ func (r *importReader) byte() byte { } return x } - -const deltaNewFile = -64 // see cmd/compile/internal/gc/bexport.go - -// Synthesize a token.Pos -type fakeFileSet struct { - fset *token.FileSet - files map[string]*token.File -} - -func (s *fakeFileSet) pos(file string, line, column int) token.Pos { - // TODO(mdempsky): Make use of column. - - // Since we don't know the set of needed file positions, we - // reserve maxlines positions per file. - const maxlines = 64 * 1024 - f := s.files[file] - if f == nil { - f = s.fset.AddFile(file, -1, maxlines) - s.files[file] = f - // Allocate the fake linebreak indices on first use. - // TODO(adonovan): opt: save ~512KB using a more complex scheme? - fakeLinesOnce.Do(func() { - fakeLines = make([]int, maxlines) - for i := range fakeLines { - fakeLines[i] = i - } - }) - f.SetLines(fakeLines) - } - - if line > maxlines { - line = 1 - } - - // Treat the file as if it contained only newlines - // and column=1: use the line number as the offset. - return f.Pos(line - 1) -} - -var ( - fakeLines []int - fakeLinesOnce sync.Once -) - -func chanDir(d int) types.ChanDir { - // tag values must match the constants in cmd/compile/internal/gc/go.go - switch d { - case 1 /* Crecv */ : - return types.RecvOnly - case 2 /* Csend */ : - return types.SendOnly - case 3 /* Cboth */ : - return types.SendRecv - default: - errorf("unexpected channel dir %d", d) - return 0 - } -} - -func exported(name string) bool { - ch, _ := utf8.DecodeRuneInString(name) - return unicode.IsUpper(ch) -} - -// ---------------------------------------------------------------------------- -// Export format - -// Tags. Must be < 0. -const ( - // Objects - packageTag = -(iota + 1) - constTag - typeTag - varTag - funcTag - endTag - - // Types - namedTag - arrayTag - sliceTag - dddTag - structTag - pointerTag - signatureTag - interfaceTag - mapTag - chanTag - - // Values - falseTag - trueTag - int64Tag - floatTag - fractionTag // not used by gc - complexTag - stringTag - nilTag // only used by gc (appears in exported inlined function bodies) - unknownTag // not used by gc (only appears in packages with errors) - - // Type aliases - aliasTag -) - -var predeclOnce sync.Once -var predecl []types.Type // initialized lazily - -func predeclared() []types.Type { - predeclOnce.Do(func() { - // initialize lazily to be sure that all - // elements have been initialized before - predecl = []types.Type{ // basic types - types.Typ[types.Bool], - types.Typ[types.Int], - types.Typ[types.Int8], - types.Typ[types.Int16], - types.Typ[types.Int32], - types.Typ[types.Int64], - types.Typ[types.Uint], - types.Typ[types.Uint8], - types.Typ[types.Uint16], - types.Typ[types.Uint32], - types.Typ[types.Uint64], - types.Typ[types.Uintptr], - types.Typ[types.Float32], - types.Typ[types.Float64], - types.Typ[types.Complex64], - types.Typ[types.Complex128], - types.Typ[types.String], - - // basic type aliases - types.Universe.Lookup("byte").Type(), - types.Universe.Lookup("rune").Type(), - - // error - types.Universe.Lookup("error").Type(), - - // untyped types - types.Typ[types.UntypedBool], - types.Typ[types.UntypedInt], - types.Typ[types.UntypedRune], - types.Typ[types.UntypedFloat], - types.Typ[types.UntypedComplex], - types.Typ[types.UntypedString], - types.Typ[types.UntypedNil], - - // package unsafe - types.Typ[types.UnsafePointer], - - // invalid type - types.Typ[types.Invalid], // only appears in packages with errors - - // used internally by gc; never used by this package or in .a files - anyType{}, - } - }) - return predecl -} - -type anyType struct{} - -func (t anyType) Underlying() types.Type { return t } -func (t anyType) String() string { return "any" } diff --git a/golang.org/x/tools/go/packages/golist.go b/golang.org/x/tools/go/packages/golist.go index 88ca6691de..6e91391ce2 100644 --- a/golang.org/x/tools/go/packages/golist.go +++ b/golang.org/x/tools/go/packages/golist.go @@ -24,7 +24,6 @@ import ( "golang.org/x/tools/go/internal/packagesdriver" "golang.org/x/tools/internal/gocommand" - "golang.org/x/tools/internal/packagesinternal" "golang.org/x/xerrors" ) @@ -382,7 +381,7 @@ type jsonPackage struct { Imports []string ImportMap map[string]string Deps []string - Module *packagesinternal.Module + Module *Module TestGoFiles []string TestImports []string XTestGoFiles []string @@ -541,7 +540,26 @@ func (state *golistState) createDriverResponse(words ...string) (*driverResponse CompiledGoFiles: absJoin(p.Dir, p.CompiledGoFiles), OtherFiles: absJoin(p.Dir, otherFiles(p)...), forTest: p.ForTest, - module: p.Module, + Module: p.Module, + } + + if (state.cfg.Mode&typecheckCgo) != 0 && len(p.CgoFiles) != 0 { + if len(p.CompiledGoFiles) > len(p.GoFiles) { + // We need the cgo definitions, which are in the first + // CompiledGoFile after the non-cgo ones. This is a hack but there + // isn't currently a better way to find it. We also need the pure + // Go files and unprocessed cgo files, all of which are already + // in pkg.GoFiles. + cgoTypes := p.CompiledGoFiles[len(p.GoFiles)] + pkg.CompiledGoFiles = append([]string{cgoTypes}, pkg.GoFiles...) + } else { + // golang/go#38990: go list silently fails to do cgo processing + pkg.CompiledGoFiles = nil + pkg.Errors = append(pkg.Errors, Error{ + Msg: "go list failed to return CompiledGoFiles; https://golang.org/issue/38990?", + Kind: ListError, + }) + } } // Work around https://golang.org/issue/28749: diff --git a/golang.org/x/tools/go/packages/golist_overlay.go b/golang.org/x/tools/go/packages/golist_overlay.go index 3c99b6e48d..338d6f6232 100644 --- a/golang.org/x/tools/go/packages/golist_overlay.go +++ b/golang.org/x/tools/go/packages/golist_overlay.go @@ -70,9 +70,9 @@ func (state *golistState) processGolistOverlay(response *responseDeduper) (modif // to the overlay. continue } - // if all the overlay files belong to a different package, change the package - // name to that package. Otherwise leave it alone; there will be an error message. - maybeFixPackageName(pkgName, pkgOfDir, dir) + // If all the overlay files belong to a different package, change the + // package name to that package. + maybeFixPackageName(pkgName, isTestFile, pkgOfDir[dir]) nextPackage: for _, p := range response.dr.Packages { if pkgName != p.Name && p.ID != "command-line-arguments" { @@ -113,24 +113,35 @@ func (state *golistState) processGolistOverlay(response *responseDeduper) (modif if !ok { break } + var forTest string // only set for x tests isXTest := strings.HasSuffix(pkgName, "_test") if isXTest { + forTest = pkgPath pkgPath += "_test" } id := pkgPath - if isTestFile && !isXTest { - id = fmt.Sprintf("%s [%s.test]", pkgPath, pkgPath) + if isTestFile { + if isXTest { + id = fmt.Sprintf("%s [%s.test]", pkgPath, forTest) + } else { + id = fmt.Sprintf("%s [%s.test]", pkgPath, pkgPath) + } } - // Try to reclaim a package with the same id if it exists in the response. + // Try to reclaim a package with the same ID, if it exists in the response. for _, p := range response.dr.Packages { if reclaimPackage(p, id, opath, contents) { pkg = p break } } - // Otherwise, create a new package + // Otherwise, create a new package. if pkg == nil { - pkg = &Package{PkgPath: pkgPath, ID: id, Name: pkgName, Imports: make(map[string]*Package)} + pkg = &Package{ + PkgPath: pkgPath, + ID: id, + Name: pkgName, + Imports: make(map[string]*Package), + } response.addPackage(pkg) havePkgs[pkg.PkgPath] = id // Add the production package's sources for a test variant. @@ -143,6 +154,9 @@ func (state *golistState) processGolistOverlay(response *responseDeduper) (modif pkg.Imports[k] = &Package{ID: v.ID} } } + if isXTest { + pkg.forTest = forTest + } } } if !fileExists { @@ -158,6 +172,8 @@ func (state *golistState) processGolistOverlay(response *responseDeduper) (modif continue } for _, imp := range imports { + // TODO(rstambler): If the package is an x test and the import has + // a test variant, make sure to replace it. if _, found := pkg.Imports[imp]; found { continue } @@ -415,24 +431,35 @@ func commonDir(a []string) string { // package name, and they all have the same package name, then that name becomes // the package name. // It returns true if it changes the package name, false otherwise. -func maybeFixPackageName(newName string, pkgOfDir map[string][]*Package, dir string) bool { +func maybeFixPackageName(newName string, isTestFile bool, pkgsOfDir []*Package) { names := make(map[string]int) - for _, p := range pkgOfDir[dir] { + for _, p := range pkgsOfDir { names[p.Name]++ } if len(names) != 1 { // some files are in different packages - return false + return } - oldName := "" + var oldName string for k := range names { oldName = k } if newName == oldName { - return false - } - for _, p := range pkgOfDir[dir] { + return + } + // We might have a case where all of the package names in the directory are + // the same, but the overlay file is for an x test, which belongs to its + // own package. If the x test does not yet exist on disk, we may not yet + // have its package name on disk, but we should not rename the packages. + // + // We use a heuristic to determine if this file belongs to an x test: + // The test file should have a package name whose package name has a _test + // suffix or looks like "newName_test". + maybeXTest := strings.HasPrefix(oldName+"_test", newName) || strings.HasSuffix(newName, "_test") + if isTestFile && maybeXTest { + return + } + for _, p := range pkgsOfDir { p.Name = newName } - return true } diff --git a/golang.org/x/tools/go/packages/packages.go b/golang.org/x/tools/go/packages/packages.go index 03fd999c0c..04053f1e7d 100644 --- a/golang.org/x/tools/go/packages/packages.go +++ b/golang.org/x/tools/go/packages/packages.go @@ -21,10 +21,12 @@ import ( "path/filepath" "strings" "sync" + "time" "golang.org/x/tools/go/gcexportdata" "golang.org/x/tools/internal/gocommand" "golang.org/x/tools/internal/packagesinternal" + "golang.org/x/tools/internal/typesinternal" ) // A LoadMode controls the amount of detail to return when loading. @@ -70,6 +72,13 @@ const ( // NeedTypesSizes adds TypesSizes. NeedTypesSizes + + // typecheckCgo enables full support for type checking cgo. Requires Go 1.15+. + // Modifies CompiledGoFiles and Types, and has no effect on its own. + typecheckCgo + + // NeedModule adds Module. + NeedModule ) const ( @@ -182,6 +191,13 @@ type driver func(cfg *Config, patterns ...string) (*driverResponse, error) // driverResponse contains the results for a driver query. type driverResponse struct { + // NotHandled is returned if the request can't be handled by the current + // driver. If an external driver returns a response with NotHandled, the + // rest of the driverResponse is ignored, and go/packages will fallback + // to the next driver. If go/packages is extended in the future to support + // lists of multiple drivers, go/packages will fall back to the next driver. + NotHandled bool + // Sizes, if not nil, is the types.Sizes to use when type checking. Sizes *types.StdSizes @@ -223,14 +239,22 @@ func Load(cfg *Config, patterns ...string) ([]*Package, error) { return l.refine(response.Roots, response.Packages...) } -// defaultDriver is a driver that looks for an external driver binary, and if -// it does not find it falls back to the built in go list driver. +// defaultDriver is a driver that implements go/packages' fallback behavior. +// It will try to request to an external driver, if one exists. If there's +// no external driver, or the driver returns a response with NotHandled set, +// defaultDriver will fall back to the go list driver. func defaultDriver(cfg *Config, patterns ...string) (*driverResponse, error) { driver := findExternalDriver(cfg) if driver == nil { driver = goListDriver } - return driver(cfg, patterns...) + response, err := driver(cfg, patterns...) + if err != nil { + return response, err + } else if response.NotHandled { + return goListDriver(cfg, patterns...) + } + return response, nil } // A Package describes a loaded Go package. @@ -257,7 +281,7 @@ type Package struct { GoFiles []string // CompiledGoFiles lists the absolute file paths of the package's source - // files that were presented to the compiler. + // files that are suitable for type checking. // This may differ from GoFiles if files are processed before compilation. CompiledGoFiles []string @@ -305,22 +329,39 @@ type Package struct { forTest string // module is the module information for the package if it exists. - module *packagesinternal.Module + Module *Module +} + +// Module provides module information for a package. +type Module struct { + Path string // module path + Version string // module version + Replace *Module // replaced by this module + Time *time.Time // time version was created + Main bool // is this the main module? + Indirect bool // is this module only an indirect dependency of main module? + Dir string // directory holding files for this module, if any + GoMod string // path to go.mod file used when loading this module, if any + GoVersion string // go version used in module + Error *ModuleError // error loading module +} + +// ModuleError holds errors loading a module. +type ModuleError struct { + Err string // the error itself } func init() { packagesinternal.GetForTest = func(p interface{}) string { return p.(*Package).forTest } - packagesinternal.GetModule = func(p interface{}) *packagesinternal.Module { - return p.(*Package).module - } packagesinternal.GetGoCmdRunner = func(config interface{}) *gocommand.Runner { return config.(*Config).gocmdRunner } packagesinternal.SetGoCmdRunner = func(config interface{}, runner *gocommand.Runner) { config.(*Config).gocmdRunner = runner } + packagesinternal.TypecheckCgo = int(typecheckCgo) } // An Error describes a problem with a package's metadata, syntax, or types. @@ -703,6 +744,9 @@ func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) { if ld.requestedMode&NeedTypesSizes == 0 { ld.pkgs[i].TypesSizes = nil } + if ld.requestedMode&NeedModule == 0 { + ld.pkgs[i].Module = nil + } } return result, nil @@ -878,6 +922,15 @@ func (ld *loader) loadPackage(lpkg *loaderPackage) { Error: appendError, Sizes: ld.sizes, } + if (ld.Mode & typecheckCgo) != 0 { + if !typesinternal.SetUsesCgo(tc) { + appendError(Error{ + Msg: "typecheckCgo requires Go 1.15+", + Kind: ListError, + }) + return + } + } types.NewChecker(tc, ld.Fset, lpkg.Types, lpkg.TypesInfo).Files(lpkg.Syntax) lpkg.importErrors = nil // no longer needed diff --git a/golang.org/x/tools/internal/analysisinternal/analysis.go b/golang.org/x/tools/internal/analysisinternal/analysis.go index 26586810c7..14b96a79fa 100644 --- a/golang.org/x/tools/internal/analysisinternal/analysis.go +++ b/golang.org/x/tools/internal/analysisinternal/analysis.go @@ -48,7 +48,7 @@ func ZeroValue(fset *token.FileSet, f *ast.File, pkg *types.Package, typ types.T case *types.Chan, *types.Interface, *types.Map, *types.Pointer, *types.Signature, *types.Slice: return ast.NewIdent("nil") case *types.Struct: - texpr := typeExpr(fset, f, pkg, typ) // typ because we want the name here. + texpr := TypeExpr(fset, f, pkg, typ) // typ because we want the name here. if texpr == nil { return nil } @@ -56,7 +56,7 @@ func ZeroValue(fset *token.FileSet, f *ast.File, pkg *types.Package, typ types.T Type: texpr, } case *types.Array: - texpr := typeExpr(fset, f, pkg, u.Elem()) + texpr := TypeExpr(fset, f, pkg, u.Elem()) if texpr == nil { return nil } @@ -70,7 +70,7 @@ func ZeroValue(fset *token.FileSet, f *ast.File, pkg *types.Package, typ types.T return nil } -func typeExpr(fset *token.FileSet, f *ast.File, pkg *types.Package, typ types.Type) ast.Expr { +func TypeExpr(fset *token.FileSet, f *ast.File, pkg *types.Package, typ types.Type) ast.Expr { switch t := typ.(type) { case *types.Basic: switch t.Kind() { @@ -101,6 +101,11 @@ func typeExpr(fset *token.FileSet, f *ast.File, pkg *types.Package, typ types.Ty X: ast.NewIdent(pkgName), Sel: ast.NewIdent(t.Obj().Name()), } + case *types.Pointer: + return &ast.UnaryExpr{ + Op: token.MUL, + X: TypeExpr(fset, f, pkg, t.Elem()), + } default: return nil // TODO: anonymous structs, but who does that } diff --git a/golang.org/x/tools/internal/gocommand/invoke.go b/golang.org/x/tools/internal/gocommand/invoke.go index 9aa7984561..f516e17623 100644 --- a/golang.org/x/tools/internal/gocommand/invoke.go +++ b/golang.org/x/tools/internal/gocommand/invoke.go @@ -23,57 +23,106 @@ import ( // An Runner will run go command invocations and serialize // them if it sees a concurrency error. type Runner struct { - // LoadMu guards packages.Load calls and associated state. - loadMu sync.Mutex - serializeLoads int + // once guards the runner initialization. + once sync.Once + + // inFlight tracks available workers. + inFlight chan struct{} + + // serialized guards the ability to run a go command serially, + // to avoid deadlocks when claiming workers. + serialized chan struct{} +} + +const maxInFlight = 10 + +func (runner *Runner) initialize() { + runner.once.Do(func() { + runner.inFlight = make(chan struct{}, maxInFlight) + runner.serialized = make(chan struct{}, 1) + }) } // 1.13: go: updates to go.mod needed, but contents have changed // 1.14: go: updating go.mod: existing contents have changed since last read var modConcurrencyError = regexp.MustCompile(`go:.*go.mod.*contents have changed`) -// Run calls Runner.RunRaw, serializing requests if they fight over -// go.mod changes. +// Run is a convenience wrapper around RunRaw. +// It returns only stdout and a "friendly" error. func (runner *Runner) Run(ctx context.Context, inv Invocation) (*bytes.Buffer, error) { stdout, _, friendly, _ := runner.RunRaw(ctx, inv) return stdout, friendly } -// RunRaw calls Invocation.runRaw, serializing requests if they fight over +// RunPiped runs the invocation serially, always waiting for any concurrent +// invocations to complete first. +func (runner *Runner) RunPiped(ctx context.Context, inv Invocation, stdout, stderr io.Writer) error { + _, err := runner.runPiped(ctx, inv, stdout, stderr) + return err +} + +// RunRaw runs the invocation, serializing requests only if they fight over // go.mod changes. func (runner *Runner) RunRaw(ctx context.Context, inv Invocation) (*bytes.Buffer, *bytes.Buffer, error, error) { - // We want to run invocations concurrently as much as possible. However, - // if go.mod updates are needed, only one can make them and the others will - // fail. We need to retry in those cases, but we don't want to thrash so - // badly we never recover. To avoid that, once we've seen one concurrency - // error, start serializing everything until the backlog has cleared out. - runner.loadMu.Lock() - var locked bool // If true, we hold the mutex and have incremented. - if runner.serializeLoads == 0 { - runner.loadMu.Unlock() - } else { - locked = true - runner.serializeLoads++ + // Make sure the runner is always initialized. + runner.initialize() + + // First, try to run the go command concurrently. + stdout, stderr, friendlyErr, err := runner.runConcurrent(ctx, inv) + + // If we encounter a load concurrency error, we need to retry serially. + if friendlyErr == nil || !modConcurrencyError.MatchString(friendlyErr.Error()) { + return stdout, stderr, friendlyErr, err } - defer func() { - if locked { - runner.serializeLoads-- - runner.loadMu.Unlock() - } - }() + event.Error(ctx, "Load concurrency error, will retry serially", err) - for { - stdout, stderr, friendlyErr, err := inv.runRaw(ctx) - if friendlyErr == nil || !modConcurrencyError.MatchString(friendlyErr.Error()) { - return stdout, stderr, friendlyErr, err - } - event.Error(ctx, "Load concurrency error, will retry serially", err) - if !locked { - runner.loadMu.Lock() - runner.serializeLoads++ - locked = true + // Run serially by calling runPiped. + stdout.Reset() + stderr.Reset() + friendlyErr, err = runner.runPiped(ctx, inv, stdout, stderr) + return stdout, stderr, friendlyErr, err +} + +func (runner *Runner) runConcurrent(ctx context.Context, inv Invocation) (*bytes.Buffer, *bytes.Buffer, error, error) { + // Wait for 1 worker to become available. + select { + case <-ctx.Done(): + return nil, nil, nil, ctx.Err() + case runner.inFlight <- struct{}{}: + defer func() { <-runner.inFlight }() + } + + stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{} + friendlyErr, err := inv.runWithFriendlyError(ctx, stdout, stderr) + return stdout, stderr, friendlyErr, err +} + +func (runner *Runner) runPiped(ctx context.Context, inv Invocation, stdout, stderr io.Writer) (error, error) { + // Make sure the runner is always initialized. + runner.initialize() + + // Acquire the serialization lock. This avoids deadlocks between two + // runPiped commands. + select { + case <-ctx.Done(): + return nil, ctx.Err() + case runner.serialized <- struct{}{}: + defer func() { <-runner.serialized }() + } + + // Wait for all in-progress go commands to return before proceeding, + // to avoid load concurrency errors. + for i := 0; i < maxInFlight; i++ { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case runner.inFlight <- struct{}{}: + // Make sure we always "return" any workers we took. + defer func() { <-runner.inFlight }() } } + + return inv.runWithFriendlyError(ctx, stdout, stderr) } // An Invocation represents a call to the go command. @@ -86,12 +135,8 @@ type Invocation struct { Logf func(format string, args ...interface{}) } -// RunRaw is like RunPiped, but also returns the raw stderr and error for callers -// that want to do low-level error handling/recovery. -func (i *Invocation) runRaw(ctx context.Context) (stdout *bytes.Buffer, stderr *bytes.Buffer, friendlyError error, rawError error) { - stdout = &bytes.Buffer{} - stderr = &bytes.Buffer{} - rawError = i.RunPiped(ctx, stdout, stderr) +func (i *Invocation) runWithFriendlyError(ctx context.Context, stdout, stderr io.Writer) (friendlyError error, rawError error) { + rawError = i.run(ctx, stdout, stderr) if rawError != nil { friendlyError = rawError // Check for 'go' executable not being found. @@ -106,8 +151,7 @@ func (i *Invocation) runRaw(ctx context.Context) (stdout *bytes.Buffer, stderr * return } -// RunPiped is like Run, but relies on the given stdout/stderr -func (i *Invocation) RunPiped(ctx context.Context, stdout, stderr io.Writer) error { +func (i *Invocation) run(ctx context.Context, stdout, stderr io.Writer) error { log := i.Logf if log == nil { log = func(string, ...interface{}) {} @@ -141,7 +185,6 @@ func (i *Invocation) RunPiped(ctx context.Context, stdout, stderr io.Writer) err cmd.Env = append(cmd.Env, "PWD="+i.WorkingDir) cmd.Dir = i.WorkingDir } - defer func(start time.Time) { log("%s for %v", time.Since(start), cmdDebugStr(cmd)) }(time.Now()) return runCmdContext(ctx, cmd) diff --git a/golang.org/x/tools/internal/gocommand/vendor.go b/golang.org/x/tools/internal/gocommand/vendor.go new file mode 100644 index 0000000000..1cd8d8473e --- /dev/null +++ b/golang.org/x/tools/internal/gocommand/vendor.go @@ -0,0 +1,102 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gocommand + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + + "golang.org/x/mod/semver" +) + +// ModuleJSON holds information about a module. +type ModuleJSON struct { + Path string // module path + Replace *ModuleJSON // replaced by this module + Main bool // is this the main module? + Indirect bool // is this module only an indirect dependency of main module? + Dir string // directory holding files for this module, if any + GoMod string // path to go.mod file for this module, if any + GoVersion string // go version used in module +} + +var modFlagRegexp = regexp.MustCompile(`-mod[ =](\w+)`) + +// VendorEnabled reports whether vendoring is enabled. It takes a *Runner to execute Go commands +// with the supplied context.Context and Invocation. The Invocation can contain pre-defined fields, +// of which only Verb and Args are modified to run the appropriate Go command. +// Inspired by setDefaultBuildMod in modload/init.go +func VendorEnabled(ctx context.Context, inv Invocation, r *Runner) (*ModuleJSON, bool, error) { + mainMod, go114, err := getMainModuleAnd114(ctx, inv, r) + if err != nil { + return nil, false, err + } + + // We check the GOFLAGS to see if there is anything overridden or not. + inv.Verb = "env" + inv.Args = []string{"GOFLAGS"} + stdout, err := r.Run(ctx, inv) + if err != nil { + return nil, false, err + } + goflags := string(bytes.TrimSpace(stdout.Bytes())) + matches := modFlagRegexp.FindStringSubmatch(goflags) + var modFlag string + if len(matches) != 0 { + modFlag = matches[1] + } + if modFlag != "" { + // Don't override an explicit '-mod=' argument. + return mainMod, modFlag == "vendor", nil + } + if mainMod == nil || !go114 { + return mainMod, false, nil + } + // Check 1.14's automatic vendor mode. + if fi, err := os.Stat(filepath.Join(mainMod.Dir, "vendor")); err == nil && fi.IsDir() { + if mainMod.GoVersion != "" && semver.Compare("v"+mainMod.GoVersion, "v1.14") >= 0 { + // The Go version is at least 1.14, and a vendor directory exists. + // Set -mod=vendor by default. + return mainMod, true, nil + } + } + return mainMod, false, nil +} + +// getMainModuleAnd114 gets the main module's information and whether the +// go command in use is 1.14+. This is the information needed to figure out +// if vendoring should be enabled. +func getMainModuleAnd114(ctx context.Context, inv Invocation, r *Runner) (*ModuleJSON, bool, error) { + const format = `{{.Path}} +{{.Dir}} +{{.GoMod}} +{{.GoVersion}} +{{range context.ReleaseTags}}{{if eq . "go1.14"}}{{.}}{{end}}{{end}} +` + inv.Verb = "list" + inv.Args = []string{"-m", "-f", format} + stdout, err := r.Run(ctx, inv) + if err != nil { + return nil, false, err + } + + lines := strings.Split(stdout.String(), "\n") + if len(lines) < 5 { + return nil, false, fmt.Errorf("unexpected stdout: %q", stdout.String()) + } + mod := &ModuleJSON{ + Path: lines[0], + Dir: lines[1], + GoMod: lines[2], + GoVersion: lines[3], + Main: true, + } + return mod, lines[4] == "go1.14", nil +} diff --git a/golang.org/x/tools/internal/packagesinternal/packages.go b/golang.org/x/tools/internal/packagesinternal/packages.go index a88750be2b..2c4527f243 100644 --- a/golang.org/x/tools/internal/packagesinternal/packages.go +++ b/golang.org/x/tools/internal/packagesinternal/packages.go @@ -2,34 +2,13 @@ package packagesinternal import ( - "time" - "golang.org/x/tools/internal/gocommand" ) -// Fields must match go list; -type Module struct { - Path string // module path - Version string // module version - Versions []string // available module versions (with -versions) - Replace *Module // replaced by this module - Time *time.Time // time version was created - Update *Module // available update, if any (with -u) - Main bool // is this the main module? - Indirect bool // is this module only an indirect dependency of main module? - Dir string // directory holding files for this module, if any - GoMod string // path to go.mod file used when loading this module, if any - GoVersion string // go version used in module - Error *ModuleError // error loading module -} -type ModuleError struct { - Err string // the error itself -} - var GetForTest = func(p interface{}) string { return "" } -var GetModule = func(p interface{}) *Module { return nil } - var GetGoCmdRunner = func(config interface{}) *gocommand.Runner { return nil } var SetGoCmdRunner = func(config interface{}, runner *gocommand.Runner) {} + +var TypecheckCgo int diff --git a/golang.org/x/tools/internal/span/uri.go b/golang.org/x/tools/internal/span/uri.go index f9f7760076..78e71fe451 100644 --- a/golang.org/x/tools/internal/span/uri.go +++ b/golang.org/x/tools/internal/span/uri.go @@ -54,10 +54,14 @@ func filename(uri URI) (string, error) { } func URIFromURI(s string) URI { - if !strings.HasPrefix(s, "file:///") { + if !strings.HasPrefix(s, "file://") { return URI(s) } + if !strings.HasPrefix(s, "file:///") { + // VS Code sends URLs with only two slashes, which are invalid. golang/go#39789. + s = "file:///" + s[len("file://"):] + } // Even though the input is a URI, it may not be in canonical form. VS Code // in particular over-escapes :, @, etc. Unescape and re-encode to canonicalize. path, err := url.PathUnescape(s[len("file://"):]) diff --git a/golang.org/x/tools/internal/testenv/testenv.go b/golang.org/x/tools/internal/testenv/testenv.go index 6c5fb98c45..f725b959f8 100644 --- a/golang.org/x/tools/internal/testenv/testenv.go +++ b/golang.org/x/tools/internal/testenv/testenv.go @@ -9,6 +9,7 @@ package testenv import ( "bytes" "fmt" + "go/build" "io/ioutil" "os" "os/exec" @@ -40,6 +41,17 @@ var checkGoGoroot struct { } func hasTool(tool string) error { + if tool == "cgo" { + enabled, err := cgoEnabled(false) + if err != nil { + return fmt.Errorf("checking cgo: %v", err) + } + if !enabled { + return fmt.Errorf("cgo not enabled") + } + return nil + } + _, err := exec.LookPath(tool) if err != nil { return err @@ -94,6 +106,19 @@ func hasTool(tool string) error { return nil } +func cgoEnabled(bypassEnvironment bool) (bool, error) { + cmd := exec.Command("go", "env", "CGO_ENABLED") + if bypassEnvironment { + cmd.Env = append(append([]string(nil), os.Environ()...), "CGO_ENABLED=") + } + out, err := cmd.CombinedOutput() + if err != nil { + return false, err + } + enabled := strings.TrimSpace(string(out)) + return enabled == "1", nil +} + func allowMissingTool(tool string) bool { if runtime.GOOS == "android" { // Android builds generally run tests on a separate machine from the build, @@ -102,6 +127,15 @@ func allowMissingTool(tool string) bool { } switch tool { + case "cgo": + if strings.HasSuffix(os.Getenv("GO_BUILDER_NAME"), "-nocgo") { + // Explicitly disabled on -nocgo builders. + return true + } + if enabled, err := cgoEnabled(true); err == nil && !enabled { + // No platform support. + return true + } case "go": if os.Getenv("GO_BUILDER_NAME") == "illumos-amd64-joyent" { // Work around a misconfigured builder (see https://golang.org/issue/33950). @@ -125,6 +159,7 @@ func allowMissingTool(tool string) bool { } // NeedsTool skips t if the named tool is not present in the path. +// As a special case, "cgo" means "go" is present and can compile cgo programs. func NeedsTool(t Testing, tool string) { if t, ok := t.(helperer); ok { t.Helper() @@ -185,6 +220,27 @@ func NeedsGoPackagesEnv(t Testing, env []string) { NeedsGoPackages(t) } +// NeedsGoBuild skips t if the current system can't build programs with ``go build'' +// and then run them with os.StartProcess or exec.Command. +// android, and darwin/arm systems don't have the userspace go build needs to run, +// and js/wasm doesn't support running subprocesses. +func NeedsGoBuild(t Testing) { + if t, ok := t.(helperer); ok { + t.Helper() + } + + NeedsTool(t, "go") + + switch runtime.GOOS { + case "android", "js": + t.Skipf("skipping test: %v can't build and run Go binaries", runtime.GOOS) + case "darwin": + if strings.HasPrefix(runtime.GOARCH, "arm") { + t.Skipf("skipping test: darwin/arm can't build and run Go binaries") + } + } +} + // ExitIfSmallMachine emits a helpful diagnostic and calls os.Exit(0) if the // current machine is a builder known to have scarce resources. // @@ -199,3 +255,37 @@ func ExitIfSmallMachine() { os.Exit(0) } } + +// Go1Point returns the x in Go 1.x. +func Go1Point() int { + for i := len(build.Default.ReleaseTags) - 1; i >= 0; i-- { + var version int + if _, err := fmt.Sscanf(build.Default.ReleaseTags[i], "go1.%d", &version); err != nil { + continue + } + return version + } + panic("bad release tags") +} + +// NeedsGo1Point skips t if the Go version used to run the test is older than +// 1.x. +func NeedsGo1Point(t Testing, x int) { + if t, ok := t.(helperer); ok { + t.Helper() + } + if Go1Point() < x { + t.Skipf("running Go version %q is version 1.%d, older than required 1.%d", runtime.Version(), Go1Point(), x) + } +} + +// SkipAfterGo1Point skips t if the Go version used to run the test is newer than +// 1.x. +func SkipAfterGo1Point(t Testing, x int) { + if t, ok := t.(helperer); ok { + t.Helper() + } + if Go1Point() > x { + t.Skipf("running Go version %q is version 1.%d, newer than maximum 1.%d", runtime.Version(), Go1Point(), x) + } +} diff --git a/golang.org/x/tools/internal/typesinternal/types.go b/golang.org/x/tools/internal/typesinternal/types.go new file mode 100644 index 0000000000..a5bb408e2f --- /dev/null +++ b/golang.org/x/tools/internal/typesinternal/types.go @@ -0,0 +1,28 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package typesinternal + +import ( + "go/types" + "reflect" + "unsafe" +) + +func SetUsesCgo(conf *types.Config) bool { + v := reflect.ValueOf(conf).Elem() + + f := v.FieldByName("go115UsesCgo") + if !f.IsValid() { + f = v.FieldByName("UsesCgo") + if !f.IsValid() { + return false + } + } + + addr := unsafe.Pointer(f.UnsafeAddr()) + *(*bool)(addr) = true + + return true +} diff --git a/gopkg.in/yaml.v3/.travis.yml b/gopkg.in/yaml.v3/.travis.yml new file mode 100644 index 0000000000..04d4dae09c --- /dev/null +++ b/gopkg.in/yaml.v3/.travis.yml @@ -0,0 +1,16 @@ +language: go + +go: + - "1.4.x" + - "1.5.x" + - "1.6.x" + - "1.7.x" + - "1.8.x" + - "1.9.x" + - "1.10.x" + - "1.11.x" + - "1.12.x" + - "1.13.x" + - "tip" + +go_import_path: gopkg.in/yaml.v3 diff --git a/gopkg.in/yaml.v3/LICENSE b/gopkg.in/yaml.v3/LICENSE new file mode 100644 index 0000000000..2683e4bb1f --- /dev/null +++ b/gopkg.in/yaml.v3/LICENSE @@ -0,0 +1,50 @@ + +This project is covered by two different licenses: MIT and Apache. + +#### MIT License #### + +The following files were ported to Go from C files of libyaml, and thus +are still covered by their original MIT license, with the additional +copyright staring in 2011 when the project was ported over: + + apic.go emitterc.go parserc.go readerc.go scannerc.go + writerc.go yamlh.go yamlprivateh.go + +Copyright (c) 2006-2010 Kirill Simonov +Copyright (c) 2006-2011 Kirill Simonov + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +### Apache License ### + +All the remaining project files are covered by the Apache license: + +Copyright (c) 2011-2019 Canonical Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/gopkg.in/yaml.v3/NOTICE b/gopkg.in/yaml.v3/NOTICE new file mode 100644 index 0000000000..866d74a7ad --- /dev/null +++ b/gopkg.in/yaml.v3/NOTICE @@ -0,0 +1,13 @@ +Copyright 2011-2016 Canonical Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/gopkg.in/yaml.v3/README.md b/gopkg.in/yaml.v3/README.md new file mode 100644 index 0000000000..08eb1babdd --- /dev/null +++ b/gopkg.in/yaml.v3/README.md @@ -0,0 +1,150 @@ +# YAML support for the Go language + +Introduction +------------ + +The yaml package enables Go programs to comfortably encode and decode YAML +values. It was developed within [Canonical](https://www.canonical.com) as +part of the [juju](https://juju.ubuntu.com) project, and is based on a +pure Go port of the well-known [libyaml](http://pyyaml.org/wiki/LibYAML) +C library to parse and generate YAML data quickly and reliably. + +Compatibility +------------- + +The yaml package supports most of YAML 1.2, but preserves some behavior +from 1.1 for backwards compatibility. + +Specifically, as of v3 of the yaml package: + + - YAML 1.1 bools (_yes/no, on/off_) are supported as long as they are being + decoded into a typed bool value. Otherwise they behave as a string. Booleans + in YAML 1.2 are _true/false_ only. + - Octals encode and decode as _0777_ per YAML 1.1, rather than _0o777_ + as specified in YAML 1.2, because most parsers still use the old format. + Octals in the _0o777_ format are supported though, so new files work. + - Does not support base-60 floats. These are gone from YAML 1.2, and were + actually never supported by this package as it's clearly a poor choice. + +and offers backwards +compatibility with YAML 1.1 in some cases. +1.2, including support for +anchors, tags, map merging, etc. Multi-document unmarshalling is not yet +implemented, and base-60 floats from YAML 1.1 are purposefully not +supported since they're a poor design and are gone in YAML 1.2. + +Installation and usage +---------------------- + +The import path for the package is *gopkg.in/yaml.v3*. + +To install it, run: + + go get gopkg.in/yaml.v3 + +API documentation +----------------- + +If opened in a browser, the import path itself leads to the API documentation: + + - [https://gopkg.in/yaml.v3](https://gopkg.in/yaml.v3) + +API stability +------------- + +The package API for yaml v3 will remain stable as described in [gopkg.in](https://gopkg.in). + + +License +------- + +The yaml package is licensed under the MIT and Apache License 2.0 licenses. +Please see the LICENSE file for details. + + +Example +------- + +```Go +package main + +import ( + "fmt" + "log" + + "gopkg.in/yaml.v3" +) + +var data = ` +a: Easy! +b: + c: 2 + d: [3, 4] +` + +// Note: struct fields must be public in order for unmarshal to +// correctly populate the data. +type T struct { + A string + B struct { + RenamedC int `yaml:"c"` + D []int `yaml:",flow"` + } +} + +func main() { + t := T{} + + err := yaml.Unmarshal([]byte(data), &t) + if err != nil { + log.Fatalf("error: %v", err) + } + fmt.Printf("--- t:\n%v\n\n", t) + + d, err := yaml.Marshal(&t) + if err != nil { + log.Fatalf("error: %v", err) + } + fmt.Printf("--- t dump:\n%s\n\n", string(d)) + + m := make(map[interface{}]interface{}) + + err = yaml.Unmarshal([]byte(data), &m) + if err != nil { + log.Fatalf("error: %v", err) + } + fmt.Printf("--- m:\n%v\n\n", m) + + d, err = yaml.Marshal(&m) + if err != nil { + log.Fatalf("error: %v", err) + } + fmt.Printf("--- m dump:\n%s\n\n", string(d)) +} +``` + +This example will generate the following output: + +``` +--- t: +{Easy! {2 [3 4]}} + +--- t dump: +a: Easy! +b: + c: 2 + d: [3, 4] + + +--- m: +map[a:Easy! b:map[c:2 d:[3 4]]] + +--- m dump: +a: Easy! +b: + c: 2 + d: + - 3 + - 4 +``` + diff --git a/gopkg.in/yaml.v3/apic.go b/gopkg.in/yaml.v3/apic.go new file mode 100644 index 0000000000..65846e6749 --- /dev/null +++ b/gopkg.in/yaml.v3/apic.go @@ -0,0 +1,746 @@ +// +// Copyright (c) 2011-2019 Canonical Ltd +// Copyright (c) 2006-2010 Kirill Simonov +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +// of the Software, and to permit persons to whom the Software is furnished to do +// so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package yaml + +import ( + "io" +) + +func yaml_insert_token(parser *yaml_parser_t, pos int, token *yaml_token_t) { + //fmt.Println("yaml_insert_token", "pos:", pos, "typ:", token.typ, "head:", parser.tokens_head, "len:", len(parser.tokens)) + + // Check if we can move the queue at the beginning of the buffer. + if parser.tokens_head > 0 && len(parser.tokens) == cap(parser.tokens) { + if parser.tokens_head != len(parser.tokens) { + copy(parser.tokens, parser.tokens[parser.tokens_head:]) + } + parser.tokens = parser.tokens[:len(parser.tokens)-parser.tokens_head] + parser.tokens_head = 0 + } + parser.tokens = append(parser.tokens, *token) + if pos < 0 { + return + } + copy(parser.tokens[parser.tokens_head+pos+1:], parser.tokens[parser.tokens_head+pos:]) + parser.tokens[parser.tokens_head+pos] = *token +} + +// Create a new parser object. +func yaml_parser_initialize(parser *yaml_parser_t) bool { + *parser = yaml_parser_t{ + raw_buffer: make([]byte, 0, input_raw_buffer_size), + buffer: make([]byte, 0, input_buffer_size), + } + return true +} + +// Destroy a parser object. +func yaml_parser_delete(parser *yaml_parser_t) { + *parser = yaml_parser_t{} +} + +// String read handler. +func yaml_string_read_handler(parser *yaml_parser_t, buffer []byte) (n int, err error) { + if parser.input_pos == len(parser.input) { + return 0, io.EOF + } + n = copy(buffer, parser.input[parser.input_pos:]) + parser.input_pos += n + return n, nil +} + +// Reader read handler. +func yaml_reader_read_handler(parser *yaml_parser_t, buffer []byte) (n int, err error) { + return parser.input_reader.Read(buffer) +} + +// Set a string input. +func yaml_parser_set_input_string(parser *yaml_parser_t, input []byte) { + if parser.read_handler != nil { + panic("must set the input source only once") + } + parser.read_handler = yaml_string_read_handler + parser.input = input + parser.input_pos = 0 +} + +// Set a file input. +func yaml_parser_set_input_reader(parser *yaml_parser_t, r io.Reader) { + if parser.read_handler != nil { + panic("must set the input source only once") + } + parser.read_handler = yaml_reader_read_handler + parser.input_reader = r +} + +// Set the source encoding. +func yaml_parser_set_encoding(parser *yaml_parser_t, encoding yaml_encoding_t) { + if parser.encoding != yaml_ANY_ENCODING { + panic("must set the encoding only once") + } + parser.encoding = encoding +} + +// Create a new emitter object. +func yaml_emitter_initialize(emitter *yaml_emitter_t) { + *emitter = yaml_emitter_t{ + buffer: make([]byte, output_buffer_size), + raw_buffer: make([]byte, 0, output_raw_buffer_size), + states: make([]yaml_emitter_state_t, 0, initial_stack_size), + events: make([]yaml_event_t, 0, initial_queue_size), + } +} + +// Destroy an emitter object. +func yaml_emitter_delete(emitter *yaml_emitter_t) { + *emitter = yaml_emitter_t{} +} + +// String write handler. +func yaml_string_write_handler(emitter *yaml_emitter_t, buffer []byte) error { + *emitter.output_buffer = append(*emitter.output_buffer, buffer...) + return nil +} + +// yaml_writer_write_handler uses emitter.output_writer to write the +// emitted text. +func yaml_writer_write_handler(emitter *yaml_emitter_t, buffer []byte) error { + _, err := emitter.output_writer.Write(buffer) + return err +} + +// Set a string output. +func yaml_emitter_set_output_string(emitter *yaml_emitter_t, output_buffer *[]byte) { + if emitter.write_handler != nil { + panic("must set the output target only once") + } + emitter.write_handler = yaml_string_write_handler + emitter.output_buffer = output_buffer +} + +// Set a file output. +func yaml_emitter_set_output_writer(emitter *yaml_emitter_t, w io.Writer) { + if emitter.write_handler != nil { + panic("must set the output target only once") + } + emitter.write_handler = yaml_writer_write_handler + emitter.output_writer = w +} + +// Set the output encoding. +func yaml_emitter_set_encoding(emitter *yaml_emitter_t, encoding yaml_encoding_t) { + if emitter.encoding != yaml_ANY_ENCODING { + panic("must set the output encoding only once") + } + emitter.encoding = encoding +} + +// Set the canonical output style. +func yaml_emitter_set_canonical(emitter *yaml_emitter_t, canonical bool) { + emitter.canonical = canonical +} + +// Set the indentation increment. +func yaml_emitter_set_indent(emitter *yaml_emitter_t, indent int) { + if indent < 2 || indent > 9 { + indent = 2 + } + emitter.best_indent = indent +} + +// Set the preferred line width. +func yaml_emitter_set_width(emitter *yaml_emitter_t, width int) { + if width < 0 { + width = -1 + } + emitter.best_width = width +} + +// Set if unescaped non-ASCII characters are allowed. +func yaml_emitter_set_unicode(emitter *yaml_emitter_t, unicode bool) { + emitter.unicode = unicode +} + +// Set the preferred line break character. +func yaml_emitter_set_break(emitter *yaml_emitter_t, line_break yaml_break_t) { + emitter.line_break = line_break +} + +///* +// * Destroy a token object. +// */ +// +//YAML_DECLARE(void) +//yaml_token_delete(yaml_token_t *token) +//{ +// assert(token); // Non-NULL token object expected. +// +// switch (token.type) +// { +// case YAML_TAG_DIRECTIVE_TOKEN: +// yaml_free(token.data.tag_directive.handle); +// yaml_free(token.data.tag_directive.prefix); +// break; +// +// case YAML_ALIAS_TOKEN: +// yaml_free(token.data.alias.value); +// break; +// +// case YAML_ANCHOR_TOKEN: +// yaml_free(token.data.anchor.value); +// break; +// +// case YAML_TAG_TOKEN: +// yaml_free(token.data.tag.handle); +// yaml_free(token.data.tag.suffix); +// break; +// +// case YAML_SCALAR_TOKEN: +// yaml_free(token.data.scalar.value); +// break; +// +// default: +// break; +// } +// +// memset(token, 0, sizeof(yaml_token_t)); +//} +// +///* +// * Check if a string is a valid UTF-8 sequence. +// * +// * Check 'reader.c' for more details on UTF-8 encoding. +// */ +// +//static int +//yaml_check_utf8(yaml_char_t *start, size_t length) +//{ +// yaml_char_t *end = start+length; +// yaml_char_t *pointer = start; +// +// while (pointer < end) { +// unsigned char octet; +// unsigned int width; +// unsigned int value; +// size_t k; +// +// octet = pointer[0]; +// width = (octet & 0x80) == 0x00 ? 1 : +// (octet & 0xE0) == 0xC0 ? 2 : +// (octet & 0xF0) == 0xE0 ? 3 : +// (octet & 0xF8) == 0xF0 ? 4 : 0; +// value = (octet & 0x80) == 0x00 ? octet & 0x7F : +// (octet & 0xE0) == 0xC0 ? octet & 0x1F : +// (octet & 0xF0) == 0xE0 ? octet & 0x0F : +// (octet & 0xF8) == 0xF0 ? octet & 0x07 : 0; +// if (!width) return 0; +// if (pointer+width > end) return 0; +// for (k = 1; k < width; k ++) { +// octet = pointer[k]; +// if ((octet & 0xC0) != 0x80) return 0; +// value = (value << 6) + (octet & 0x3F); +// } +// if (!((width == 1) || +// (width == 2 && value >= 0x80) || +// (width == 3 && value >= 0x800) || +// (width == 4 && value >= 0x10000))) return 0; +// +// pointer += width; +// } +// +// return 1; +//} +// + +// Create STREAM-START. +func yaml_stream_start_event_initialize(event *yaml_event_t, encoding yaml_encoding_t) { + *event = yaml_event_t{ + typ: yaml_STREAM_START_EVENT, + encoding: encoding, + } +} + +// Create STREAM-END. +func yaml_stream_end_event_initialize(event *yaml_event_t) { + *event = yaml_event_t{ + typ: yaml_STREAM_END_EVENT, + } +} + +// Create DOCUMENT-START. +func yaml_document_start_event_initialize( + event *yaml_event_t, + version_directive *yaml_version_directive_t, + tag_directives []yaml_tag_directive_t, + implicit bool, +) { + *event = yaml_event_t{ + typ: yaml_DOCUMENT_START_EVENT, + version_directive: version_directive, + tag_directives: tag_directives, + implicit: implicit, + } +} + +// Create DOCUMENT-END. +func yaml_document_end_event_initialize(event *yaml_event_t, implicit bool) { + *event = yaml_event_t{ + typ: yaml_DOCUMENT_END_EVENT, + implicit: implicit, + } +} + +// Create ALIAS. +func yaml_alias_event_initialize(event *yaml_event_t, anchor []byte) bool { + *event = yaml_event_t{ + typ: yaml_ALIAS_EVENT, + anchor: anchor, + } + return true +} + +// Create SCALAR. +func yaml_scalar_event_initialize(event *yaml_event_t, anchor, tag, value []byte, plain_implicit, quoted_implicit bool, style yaml_scalar_style_t) bool { + *event = yaml_event_t{ + typ: yaml_SCALAR_EVENT, + anchor: anchor, + tag: tag, + value: value, + implicit: plain_implicit, + quoted_implicit: quoted_implicit, + style: yaml_style_t(style), + } + return true +} + +// Create SEQUENCE-START. +func yaml_sequence_start_event_initialize(event *yaml_event_t, anchor, tag []byte, implicit bool, style yaml_sequence_style_t) bool { + *event = yaml_event_t{ + typ: yaml_SEQUENCE_START_EVENT, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(style), + } + return true +} + +// Create SEQUENCE-END. +func yaml_sequence_end_event_initialize(event *yaml_event_t) bool { + *event = yaml_event_t{ + typ: yaml_SEQUENCE_END_EVENT, + } + return true +} + +// Create MAPPING-START. +func yaml_mapping_start_event_initialize(event *yaml_event_t, anchor, tag []byte, implicit bool, style yaml_mapping_style_t) { + *event = yaml_event_t{ + typ: yaml_MAPPING_START_EVENT, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(style), + } +} + +// Create MAPPING-END. +func yaml_mapping_end_event_initialize(event *yaml_event_t) { + *event = yaml_event_t{ + typ: yaml_MAPPING_END_EVENT, + } +} + +// Destroy an event object. +func yaml_event_delete(event *yaml_event_t) { + *event = yaml_event_t{} +} + +///* +// * Create a document object. +// */ +// +//YAML_DECLARE(int) +//yaml_document_initialize(document *yaml_document_t, +// version_directive *yaml_version_directive_t, +// tag_directives_start *yaml_tag_directive_t, +// tag_directives_end *yaml_tag_directive_t, +// start_implicit int, end_implicit int) +//{ +// struct { +// error yaml_error_type_t +// } context +// struct { +// start *yaml_node_t +// end *yaml_node_t +// top *yaml_node_t +// } nodes = { NULL, NULL, NULL } +// version_directive_copy *yaml_version_directive_t = NULL +// struct { +// start *yaml_tag_directive_t +// end *yaml_tag_directive_t +// top *yaml_tag_directive_t +// } tag_directives_copy = { NULL, NULL, NULL } +// value yaml_tag_directive_t = { NULL, NULL } +// mark yaml_mark_t = { 0, 0, 0 } +// +// assert(document) // Non-NULL document object is expected. +// assert((tag_directives_start && tag_directives_end) || +// (tag_directives_start == tag_directives_end)) +// // Valid tag directives are expected. +// +// if (!STACK_INIT(&context, nodes, INITIAL_STACK_SIZE)) goto error +// +// if (version_directive) { +// version_directive_copy = yaml_malloc(sizeof(yaml_version_directive_t)) +// if (!version_directive_copy) goto error +// version_directive_copy.major = version_directive.major +// version_directive_copy.minor = version_directive.minor +// } +// +// if (tag_directives_start != tag_directives_end) { +// tag_directive *yaml_tag_directive_t +// if (!STACK_INIT(&context, tag_directives_copy, INITIAL_STACK_SIZE)) +// goto error +// for (tag_directive = tag_directives_start +// tag_directive != tag_directives_end; tag_directive ++) { +// assert(tag_directive.handle) +// assert(tag_directive.prefix) +// if (!yaml_check_utf8(tag_directive.handle, +// strlen((char *)tag_directive.handle))) +// goto error +// if (!yaml_check_utf8(tag_directive.prefix, +// strlen((char *)tag_directive.prefix))) +// goto error +// value.handle = yaml_strdup(tag_directive.handle) +// value.prefix = yaml_strdup(tag_directive.prefix) +// if (!value.handle || !value.prefix) goto error +// if (!PUSH(&context, tag_directives_copy, value)) +// goto error +// value.handle = NULL +// value.prefix = NULL +// } +// } +// +// DOCUMENT_INIT(*document, nodes.start, nodes.end, version_directive_copy, +// tag_directives_copy.start, tag_directives_copy.top, +// start_implicit, end_implicit, mark, mark) +// +// return 1 +// +//error: +// STACK_DEL(&context, nodes) +// yaml_free(version_directive_copy) +// while (!STACK_EMPTY(&context, tag_directives_copy)) { +// value yaml_tag_directive_t = POP(&context, tag_directives_copy) +// yaml_free(value.handle) +// yaml_free(value.prefix) +// } +// STACK_DEL(&context, tag_directives_copy) +// yaml_free(value.handle) +// yaml_free(value.prefix) +// +// return 0 +//} +// +///* +// * Destroy a document object. +// */ +// +//YAML_DECLARE(void) +//yaml_document_delete(document *yaml_document_t) +//{ +// struct { +// error yaml_error_type_t +// } context +// tag_directive *yaml_tag_directive_t +// +// context.error = YAML_NO_ERROR // Eliminate a compiler warning. +// +// assert(document) // Non-NULL document object is expected. +// +// while (!STACK_EMPTY(&context, document.nodes)) { +// node yaml_node_t = POP(&context, document.nodes) +// yaml_free(node.tag) +// switch (node.type) { +// case YAML_SCALAR_NODE: +// yaml_free(node.data.scalar.value) +// break +// case YAML_SEQUENCE_NODE: +// STACK_DEL(&context, node.data.sequence.items) +// break +// case YAML_MAPPING_NODE: +// STACK_DEL(&context, node.data.mapping.pairs) +// break +// default: +// assert(0) // Should not happen. +// } +// } +// STACK_DEL(&context, document.nodes) +// +// yaml_free(document.version_directive) +// for (tag_directive = document.tag_directives.start +// tag_directive != document.tag_directives.end +// tag_directive++) { +// yaml_free(tag_directive.handle) +// yaml_free(tag_directive.prefix) +// } +// yaml_free(document.tag_directives.start) +// +// memset(document, 0, sizeof(yaml_document_t)) +//} +// +///** +// * Get a document node. +// */ +// +//YAML_DECLARE(yaml_node_t *) +//yaml_document_get_node(document *yaml_document_t, index int) +//{ +// assert(document) // Non-NULL document object is expected. +// +// if (index > 0 && document.nodes.start + index <= document.nodes.top) { +// return document.nodes.start + index - 1 +// } +// return NULL +//} +// +///** +// * Get the root object. +// */ +// +//YAML_DECLARE(yaml_node_t *) +//yaml_document_get_root_node(document *yaml_document_t) +//{ +// assert(document) // Non-NULL document object is expected. +// +// if (document.nodes.top != document.nodes.start) { +// return document.nodes.start +// } +// return NULL +//} +// +///* +// * Add a scalar node to a document. +// */ +// +//YAML_DECLARE(int) +//yaml_document_add_scalar(document *yaml_document_t, +// tag *yaml_char_t, value *yaml_char_t, length int, +// style yaml_scalar_style_t) +//{ +// struct { +// error yaml_error_type_t +// } context +// mark yaml_mark_t = { 0, 0, 0 } +// tag_copy *yaml_char_t = NULL +// value_copy *yaml_char_t = NULL +// node yaml_node_t +// +// assert(document) // Non-NULL document object is expected. +// assert(value) // Non-NULL value is expected. +// +// if (!tag) { +// tag = (yaml_char_t *)YAML_DEFAULT_SCALAR_TAG +// } +// +// if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error +// tag_copy = yaml_strdup(tag) +// if (!tag_copy) goto error +// +// if (length < 0) { +// length = strlen((char *)value) +// } +// +// if (!yaml_check_utf8(value, length)) goto error +// value_copy = yaml_malloc(length+1) +// if (!value_copy) goto error +// memcpy(value_copy, value, length) +// value_copy[length] = '\0' +// +// SCALAR_NODE_INIT(node, tag_copy, value_copy, length, style, mark, mark) +// if (!PUSH(&context, document.nodes, node)) goto error +// +// return document.nodes.top - document.nodes.start +// +//error: +// yaml_free(tag_copy) +// yaml_free(value_copy) +// +// return 0 +//} +// +///* +// * Add a sequence node to a document. +// */ +// +//YAML_DECLARE(int) +//yaml_document_add_sequence(document *yaml_document_t, +// tag *yaml_char_t, style yaml_sequence_style_t) +//{ +// struct { +// error yaml_error_type_t +// } context +// mark yaml_mark_t = { 0, 0, 0 } +// tag_copy *yaml_char_t = NULL +// struct { +// start *yaml_node_item_t +// end *yaml_node_item_t +// top *yaml_node_item_t +// } items = { NULL, NULL, NULL } +// node yaml_node_t +// +// assert(document) // Non-NULL document object is expected. +// +// if (!tag) { +// tag = (yaml_char_t *)YAML_DEFAULT_SEQUENCE_TAG +// } +// +// if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error +// tag_copy = yaml_strdup(tag) +// if (!tag_copy) goto error +// +// if (!STACK_INIT(&context, items, INITIAL_STACK_SIZE)) goto error +// +// SEQUENCE_NODE_INIT(node, tag_copy, items.start, items.end, +// style, mark, mark) +// if (!PUSH(&context, document.nodes, node)) goto error +// +// return document.nodes.top - document.nodes.start +// +//error: +// STACK_DEL(&context, items) +// yaml_free(tag_copy) +// +// return 0 +//} +// +///* +// * Add a mapping node to a document. +// */ +// +//YAML_DECLARE(int) +//yaml_document_add_mapping(document *yaml_document_t, +// tag *yaml_char_t, style yaml_mapping_style_t) +//{ +// struct { +// error yaml_error_type_t +// } context +// mark yaml_mark_t = { 0, 0, 0 } +// tag_copy *yaml_char_t = NULL +// struct { +// start *yaml_node_pair_t +// end *yaml_node_pair_t +// top *yaml_node_pair_t +// } pairs = { NULL, NULL, NULL } +// node yaml_node_t +// +// assert(document) // Non-NULL document object is expected. +// +// if (!tag) { +// tag = (yaml_char_t *)YAML_DEFAULT_MAPPING_TAG +// } +// +// if (!yaml_check_utf8(tag, strlen((char *)tag))) goto error +// tag_copy = yaml_strdup(tag) +// if (!tag_copy) goto error +// +// if (!STACK_INIT(&context, pairs, INITIAL_STACK_SIZE)) goto error +// +// MAPPING_NODE_INIT(node, tag_copy, pairs.start, pairs.end, +// style, mark, mark) +// if (!PUSH(&context, document.nodes, node)) goto error +// +// return document.nodes.top - document.nodes.start +// +//error: +// STACK_DEL(&context, pairs) +// yaml_free(tag_copy) +// +// return 0 +//} +// +///* +// * Append an item to a sequence node. +// */ +// +//YAML_DECLARE(int) +//yaml_document_append_sequence_item(document *yaml_document_t, +// sequence int, item int) +//{ +// struct { +// error yaml_error_type_t +// } context +// +// assert(document) // Non-NULL document is required. +// assert(sequence > 0 +// && document.nodes.start + sequence <= document.nodes.top) +// // Valid sequence id is required. +// assert(document.nodes.start[sequence-1].type == YAML_SEQUENCE_NODE) +// // A sequence node is required. +// assert(item > 0 && document.nodes.start + item <= document.nodes.top) +// // Valid item id is required. +// +// if (!PUSH(&context, +// document.nodes.start[sequence-1].data.sequence.items, item)) +// return 0 +// +// return 1 +//} +// +///* +// * Append a pair of a key and a value to a mapping node. +// */ +// +//YAML_DECLARE(int) +//yaml_document_append_mapping_pair(document *yaml_document_t, +// mapping int, key int, value int) +//{ +// struct { +// error yaml_error_type_t +// } context +// +// pair yaml_node_pair_t +// +// assert(document) // Non-NULL document is required. +// assert(mapping > 0 +// && document.nodes.start + mapping <= document.nodes.top) +// // Valid mapping id is required. +// assert(document.nodes.start[mapping-1].type == YAML_MAPPING_NODE) +// // A mapping node is required. +// assert(key > 0 && document.nodes.start + key <= document.nodes.top) +// // Valid key id is required. +// assert(value > 0 && document.nodes.start + value <= document.nodes.top) +// // Valid value id is required. +// +// pair.key = key +// pair.value = value +// +// if (!PUSH(&context, +// document.nodes.start[mapping-1].data.mapping.pairs, pair)) +// return 0 +// +// return 1 +//} +// +// diff --git a/gopkg.in/yaml.v3/decode.go b/gopkg.in/yaml.v3/decode.go new file mode 100644 index 0000000000..be63169b71 --- /dev/null +++ b/gopkg.in/yaml.v3/decode.go @@ -0,0 +1,931 @@ +// +// Copyright (c) 2011-2019 Canonical Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package yaml + +import ( + "encoding" + "encoding/base64" + "fmt" + "io" + "math" + "reflect" + "strconv" + "time" +) + +// ---------------------------------------------------------------------------- +// Parser, produces a node tree out of a libyaml event stream. + +type parser struct { + parser yaml_parser_t + event yaml_event_t + doc *Node + anchors map[string]*Node + doneInit bool +} + +func newParser(b []byte) *parser { + p := parser{} + if !yaml_parser_initialize(&p.parser) { + panic("failed to initialize YAML emitter") + } + if len(b) == 0 { + b = []byte{'\n'} + } + yaml_parser_set_input_string(&p.parser, b) + return &p +} + +func newParserFromReader(r io.Reader) *parser { + p := parser{} + if !yaml_parser_initialize(&p.parser) { + panic("failed to initialize YAML emitter") + } + yaml_parser_set_input_reader(&p.parser, r) + return &p +} + +func (p *parser) init() { + if p.doneInit { + return + } + p.anchors = make(map[string]*Node) + p.expect(yaml_STREAM_START_EVENT) + p.doneInit = true +} + +func (p *parser) destroy() { + if p.event.typ != yaml_NO_EVENT { + yaml_event_delete(&p.event) + } + yaml_parser_delete(&p.parser) +} + +// expect consumes an event from the event stream and +// checks that it's of the expected type. +func (p *parser) expect(e yaml_event_type_t) { + if p.event.typ == yaml_NO_EVENT { + if !yaml_parser_parse(&p.parser, &p.event) { + p.fail() + } + } + if p.event.typ == yaml_STREAM_END_EVENT { + failf("attempted to go past the end of stream; corrupted value?") + } + if p.event.typ != e { + p.parser.problem = fmt.Sprintf("expected %s event but got %s", e, p.event.typ) + p.fail() + } + yaml_event_delete(&p.event) + p.event.typ = yaml_NO_EVENT +} + +// peek peeks at the next event in the event stream, +// puts the results into p.event and returns the event type. +func (p *parser) peek() yaml_event_type_t { + if p.event.typ != yaml_NO_EVENT { + return p.event.typ + } + if !yaml_parser_parse(&p.parser, &p.event) { + p.fail() + } + return p.event.typ +} + +func (p *parser) fail() { + var where string + var line int + if p.parser.problem_mark.line != 0 { + line = p.parser.problem_mark.line + // Scanner errors don't iterate line before returning error + if p.parser.error == yaml_SCANNER_ERROR { + line++ + } + } else if p.parser.context_mark.line != 0 { + line = p.parser.context_mark.line + } + if line != 0 { + where = "line " + strconv.Itoa(line) + ": " + } + var msg string + if len(p.parser.problem) > 0 { + msg = p.parser.problem + } else { + msg = "unknown problem parsing YAML content" + } + failf("%s%s", where, msg) +} + +func (p *parser) anchor(n *Node, anchor []byte) { + if anchor != nil { + n.Anchor = string(anchor) + p.anchors[n.Anchor] = n + } +} + +func (p *parser) parse() *Node { + p.init() + switch p.peek() { + case yaml_SCALAR_EVENT: + return p.scalar() + case yaml_ALIAS_EVENT: + return p.alias() + case yaml_MAPPING_START_EVENT: + return p.mapping() + case yaml_SEQUENCE_START_EVENT: + return p.sequence() + case yaml_DOCUMENT_START_EVENT: + return p.document() + case yaml_STREAM_END_EVENT: + // Happens when attempting to decode an empty buffer. + return nil + case yaml_TAIL_COMMENT_EVENT: + panic("internal error: unexpected tail comment event (please report)") + default: + panic("internal error: attempted to parse unknown event (please report): " + p.event.typ.String()) + } +} + +func (p *parser) node(kind Kind, defaultTag, tag, value string) *Node { + var style Style + if tag != "" && tag != "!" { + tag = shortTag(tag) + style = TaggedStyle + } else if defaultTag != "" { + tag = defaultTag + } else if kind == ScalarNode { + tag, _ = resolve("", value) + } + return &Node{ + Kind: kind, + Tag: tag, + Value: value, + Style: style, + Line: p.event.start_mark.line + 1, + Column: p.event.start_mark.column + 1, + HeadComment: string(p.event.head_comment), + LineComment: string(p.event.line_comment), + FootComment: string(p.event.foot_comment), + } +} + +func (p *parser) parseChild(parent *Node) *Node { + child := p.parse() + parent.Content = append(parent.Content, child) + return child +} + +func (p *parser) document() *Node { + n := p.node(DocumentNode, "", "", "") + p.doc = n + p.expect(yaml_DOCUMENT_START_EVENT) + p.parseChild(n) + if p.peek() == yaml_DOCUMENT_END_EVENT { + n.FootComment = string(p.event.foot_comment) + } + p.expect(yaml_DOCUMENT_END_EVENT) + return n +} + +func (p *parser) alias() *Node { + n := p.node(AliasNode, "", "", string(p.event.anchor)) + n.Alias = p.anchors[n.Value] + if n.Alias == nil { + failf("unknown anchor '%s' referenced", n.Value) + } + p.expect(yaml_ALIAS_EVENT) + return n +} + +func (p *parser) scalar() *Node { + var parsedStyle = p.event.scalar_style() + var nodeStyle Style + switch { + case parsedStyle&yaml_DOUBLE_QUOTED_SCALAR_STYLE != 0: + nodeStyle = DoubleQuotedStyle + case parsedStyle&yaml_SINGLE_QUOTED_SCALAR_STYLE != 0: + nodeStyle = SingleQuotedStyle + case parsedStyle&yaml_LITERAL_SCALAR_STYLE != 0: + nodeStyle = LiteralStyle + case parsedStyle&yaml_FOLDED_SCALAR_STYLE != 0: + nodeStyle = FoldedStyle + } + var nodeValue = string(p.event.value) + var nodeTag = string(p.event.tag) + var defaultTag string + if nodeStyle == 0 { + if nodeValue == "<<" { + defaultTag = mergeTag + } + } else { + defaultTag = strTag + } + n := p.node(ScalarNode, defaultTag, nodeTag, nodeValue) + n.Style |= nodeStyle + p.anchor(n, p.event.anchor) + p.expect(yaml_SCALAR_EVENT) + return n +} + +func (p *parser) sequence() *Node { + n := p.node(SequenceNode, seqTag, string(p.event.tag), "") + if p.event.sequence_style()&yaml_FLOW_SEQUENCE_STYLE != 0 { + n.Style |= FlowStyle + } + p.anchor(n, p.event.anchor) + p.expect(yaml_SEQUENCE_START_EVENT) + for p.peek() != yaml_SEQUENCE_END_EVENT { + p.parseChild(n) + } + n.LineComment = string(p.event.line_comment) + n.FootComment = string(p.event.foot_comment) + p.expect(yaml_SEQUENCE_END_EVENT) + return n +} + +func (p *parser) mapping() *Node { + n := p.node(MappingNode, mapTag, string(p.event.tag), "") + block := true + if p.event.mapping_style()&yaml_FLOW_MAPPING_STYLE != 0 { + block = false + n.Style |= FlowStyle + } + p.anchor(n, p.event.anchor) + p.expect(yaml_MAPPING_START_EVENT) + for p.peek() != yaml_MAPPING_END_EVENT { + k := p.parseChild(n) + if block && k.FootComment != "" { + // Must be a foot comment for the prior value when being dedented. + if len(n.Content) > 2 { + n.Content[len(n.Content)-3].FootComment = k.FootComment + k.FootComment = "" + } + } + v := p.parseChild(n) + if k.FootComment == "" && v.FootComment != "" { + k.FootComment = v.FootComment + v.FootComment = "" + } + if p.peek() == yaml_TAIL_COMMENT_EVENT { + if k.FootComment == "" { + k.FootComment = string(p.event.foot_comment) + } + p.expect(yaml_TAIL_COMMENT_EVENT) + } + } + n.LineComment = string(p.event.line_comment) + n.FootComment = string(p.event.foot_comment) + if n.Style&FlowStyle == 0 && n.FootComment != "" && len(n.Content) > 1 { + n.Content[len(n.Content)-2].FootComment = n.FootComment + n.FootComment = "" + } + p.expect(yaml_MAPPING_END_EVENT) + return n +} + +// ---------------------------------------------------------------------------- +// Decoder, unmarshals a node into a provided value. + +type decoder struct { + doc *Node + aliases map[*Node]bool + terrors []string + + stringMapType reflect.Type + generalMapType reflect.Type + + knownFields bool + uniqueKeys bool + decodeCount int + aliasCount int + aliasDepth int +} + +var ( + nodeType = reflect.TypeOf(Node{}) + durationType = reflect.TypeOf(time.Duration(0)) + stringMapType = reflect.TypeOf(map[string]interface{}{}) + generalMapType = reflect.TypeOf(map[interface{}]interface{}{}) + ifaceType = generalMapType.Elem() + timeType = reflect.TypeOf(time.Time{}) + ptrTimeType = reflect.TypeOf(&time.Time{}) +) + +func newDecoder() *decoder { + d := &decoder{ + stringMapType: stringMapType, + generalMapType: generalMapType, + uniqueKeys: true, + } + d.aliases = make(map[*Node]bool) + return d +} + +func (d *decoder) terror(n *Node, tag string, out reflect.Value) { + if n.Tag != "" { + tag = n.Tag + } + value := n.Value + if tag != seqTag && tag != mapTag { + if len(value) > 10 { + value = " `" + value[:7] + "...`" + } else { + value = " `" + value + "`" + } + } + d.terrors = append(d.terrors, fmt.Sprintf("line %d: cannot unmarshal %s%s into %s", n.Line, shortTag(tag), value, out.Type())) +} + +func (d *decoder) callUnmarshaler(n *Node, u Unmarshaler) (good bool) { + err := u.UnmarshalYAML(n) + if e, ok := err.(*TypeError); ok { + d.terrors = append(d.terrors, e.Errors...) + return false + } + if err != nil { + fail(err) + } + return true +} + +func (d *decoder) callObsoleteUnmarshaler(n *Node, u obsoleteUnmarshaler) (good bool) { + terrlen := len(d.terrors) + err := u.UnmarshalYAML(func(v interface{}) (err error) { + defer handleErr(&err) + d.unmarshal(n, reflect.ValueOf(v)) + if len(d.terrors) > terrlen { + issues := d.terrors[terrlen:] + d.terrors = d.terrors[:terrlen] + return &TypeError{issues} + } + return nil + }) + if e, ok := err.(*TypeError); ok { + d.terrors = append(d.terrors, e.Errors...) + return false + } + if err != nil { + fail(err) + } + return true +} + +// d.prepare initializes and dereferences pointers and calls UnmarshalYAML +// if a value is found to implement it. +// It returns the initialized and dereferenced out value, whether +// unmarshalling was already done by UnmarshalYAML, and if so whether +// its types unmarshalled appropriately. +// +// If n holds a null value, prepare returns before doing anything. +func (d *decoder) prepare(n *Node, out reflect.Value) (newout reflect.Value, unmarshaled, good bool) { + if n.ShortTag() == nullTag { + return out, false, false + } + again := true + for again { + again = false + if out.Kind() == reflect.Ptr { + if out.IsNil() { + out.Set(reflect.New(out.Type().Elem())) + } + out = out.Elem() + again = true + } + if out.CanAddr() { + outi := out.Addr().Interface() + if u, ok := outi.(Unmarshaler); ok { + good = d.callUnmarshaler(n, u) + return out, true, good + } + if u, ok := outi.(obsoleteUnmarshaler); ok { + good = d.callObsoleteUnmarshaler(n, u) + return out, true, good + } + } + } + return out, false, false +} + +func (d *decoder) fieldByIndex(n *Node, v reflect.Value, index []int) (field reflect.Value) { + if n.ShortTag() == nullTag { + return reflect.Value{} + } + for _, num := range index { + for { + if v.Kind() == reflect.Ptr { + if v.IsNil() { + v.Set(reflect.New(v.Type().Elem())) + } + v = v.Elem() + continue + } + break + } + v = v.Field(num) + } + return v +} + +const ( + // 400,000 decode operations is ~500kb of dense object declarations, or + // ~5kb of dense object declarations with 10000% alias expansion + alias_ratio_range_low = 400000 + + // 4,000,000 decode operations is ~5MB of dense object declarations, or + // ~4.5MB of dense object declarations with 10% alias expansion + alias_ratio_range_high = 4000000 + + // alias_ratio_range is the range over which we scale allowed alias ratios + alias_ratio_range = float64(alias_ratio_range_high - alias_ratio_range_low) +) + +func allowedAliasRatio(decodeCount int) float64 { + switch { + case decodeCount <= alias_ratio_range_low: + // allow 99% to come from alias expansion for small-to-medium documents + return 0.99 + case decodeCount >= alias_ratio_range_high: + // allow 10% to come from alias expansion for very large documents + return 0.10 + default: + // scale smoothly from 99% down to 10% over the range. + // this maps to 396,000 - 400,000 allowed alias-driven decodes over the range. + // 400,000 decode operations is ~100MB of allocations in worst-case scenarios (single-item maps). + return 0.99 - 0.89*(float64(decodeCount-alias_ratio_range_low)/alias_ratio_range) + } +} + +func (d *decoder) unmarshal(n *Node, out reflect.Value) (good bool) { + d.decodeCount++ + if d.aliasDepth > 0 { + d.aliasCount++ + } + if d.aliasCount > 100 && d.decodeCount > 1000 && float64(d.aliasCount)/float64(d.decodeCount) > allowedAliasRatio(d.decodeCount) { + failf("document contains excessive aliasing") + } + if out.Type() == nodeType { + out.Set(reflect.ValueOf(n).Elem()) + return true + } + switch n.Kind { + case DocumentNode: + return d.document(n, out) + case AliasNode: + return d.alias(n, out) + } + out, unmarshaled, good := d.prepare(n, out) + if unmarshaled { + return good + } + switch n.Kind { + case ScalarNode: + good = d.scalar(n, out) + case MappingNode: + good = d.mapping(n, out) + case SequenceNode: + good = d.sequence(n, out) + default: + panic("internal error: unknown node kind: " + strconv.Itoa(int(n.Kind))) + } + return good +} + +func (d *decoder) document(n *Node, out reflect.Value) (good bool) { + if len(n.Content) == 1 { + d.doc = n + d.unmarshal(n.Content[0], out) + return true + } + return false +} + +func (d *decoder) alias(n *Node, out reflect.Value) (good bool) { + if d.aliases[n] { + // TODO this could actually be allowed in some circumstances. + failf("anchor '%s' value contains itself", n.Value) + } + d.aliases[n] = true + d.aliasDepth++ + good = d.unmarshal(n.Alias, out) + d.aliasDepth-- + delete(d.aliases, n) + return good +} + +var zeroValue reflect.Value + +func resetMap(out reflect.Value) { + for _, k := range out.MapKeys() { + out.SetMapIndex(k, zeroValue) + } +} + +func (d *decoder) scalar(n *Node, out reflect.Value) bool { + var tag string + var resolved interface{} + if n.indicatedString() { + tag = strTag + resolved = n.Value + } else { + tag, resolved = resolve(n.Tag, n.Value) + if tag == binaryTag { + data, err := base64.StdEncoding.DecodeString(resolved.(string)) + if err != nil { + failf("!!binary value contains invalid base64 data") + } + resolved = string(data) + } + } + if resolved == nil { + if out.CanAddr() { + switch out.Kind() { + case reflect.Interface, reflect.Ptr, reflect.Map, reflect.Slice: + out.Set(reflect.Zero(out.Type())) + return true + } + } + return false + } + if resolvedv := reflect.ValueOf(resolved); out.Type() == resolvedv.Type() { + // We've resolved to exactly the type we want, so use that. + out.Set(resolvedv) + return true + } + // Perhaps we can use the value as a TextUnmarshaler to + // set its value. + if out.CanAddr() { + u, ok := out.Addr().Interface().(encoding.TextUnmarshaler) + if ok { + var text []byte + if tag == binaryTag { + text = []byte(resolved.(string)) + } else { + // We let any value be unmarshaled into TextUnmarshaler. + // That might be more lax than we'd like, but the + // TextUnmarshaler itself should bowl out any dubious values. + text = []byte(n.Value) + } + err := u.UnmarshalText(text) + if err != nil { + fail(err) + } + return true + } + } + switch out.Kind() { + case reflect.String: + if tag == binaryTag { + out.SetString(resolved.(string)) + return true + } + out.SetString(n.Value) + return true + case reflect.Interface: + out.Set(reflect.ValueOf(resolved)) + return true + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + // This used to work in v2, but it's very unfriendly. + isDuration := out.Type() == durationType + + switch resolved := resolved.(type) { + case int: + if !isDuration && !out.OverflowInt(int64(resolved)) { + out.SetInt(int64(resolved)) + return true + } + case int64: + if !isDuration && !out.OverflowInt(resolved) { + out.SetInt(resolved) + return true + } + case uint64: + if !isDuration && resolved <= math.MaxInt64 && !out.OverflowInt(int64(resolved)) { + out.SetInt(int64(resolved)) + return true + } + case float64: + if !isDuration && resolved <= math.MaxInt64 && !out.OverflowInt(int64(resolved)) { + out.SetInt(int64(resolved)) + return true + } + case string: + if out.Type() == durationType { + d, err := time.ParseDuration(resolved) + if err == nil { + out.SetInt(int64(d)) + return true + } + } + } + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + switch resolved := resolved.(type) { + case int: + if resolved >= 0 && !out.OverflowUint(uint64(resolved)) { + out.SetUint(uint64(resolved)) + return true + } + case int64: + if resolved >= 0 && !out.OverflowUint(uint64(resolved)) { + out.SetUint(uint64(resolved)) + return true + } + case uint64: + if !out.OverflowUint(uint64(resolved)) { + out.SetUint(uint64(resolved)) + return true + } + case float64: + if resolved <= math.MaxUint64 && !out.OverflowUint(uint64(resolved)) { + out.SetUint(uint64(resolved)) + return true + } + } + case reflect.Bool: + switch resolved := resolved.(type) { + case bool: + out.SetBool(resolved) + return true + case string: + // This offers some compatibility with the 1.1 spec (https://yaml.org/type/bool.html). + // It only works if explicitly attempting to unmarshal into a typed bool value. + switch resolved { + case "y", "Y", "yes", "Yes", "YES", "on", "On", "ON": + out.SetBool(true) + return true + case "n", "N", "no", "No", "NO", "off", "Off", "OFF": + out.SetBool(false) + return true + } + } + case reflect.Float32, reflect.Float64: + switch resolved := resolved.(type) { + case int: + out.SetFloat(float64(resolved)) + return true + case int64: + out.SetFloat(float64(resolved)) + return true + case uint64: + out.SetFloat(float64(resolved)) + return true + case float64: + out.SetFloat(resolved) + return true + } + case reflect.Struct: + if resolvedv := reflect.ValueOf(resolved); out.Type() == resolvedv.Type() { + out.Set(resolvedv) + return true + } + case reflect.Ptr: + panic("yaml internal error: please report the issue") + } + d.terror(n, tag, out) + return false +} + +func settableValueOf(i interface{}) reflect.Value { + v := reflect.ValueOf(i) + sv := reflect.New(v.Type()).Elem() + sv.Set(v) + return sv +} + +func (d *decoder) sequence(n *Node, out reflect.Value) (good bool) { + l := len(n.Content) + + var iface reflect.Value + switch out.Kind() { + case reflect.Slice: + out.Set(reflect.MakeSlice(out.Type(), l, l)) + case reflect.Array: + if l != out.Len() { + failf("invalid array: want %d elements but got %d", out.Len(), l) + } + case reflect.Interface: + // No type hints. Will have to use a generic sequence. + iface = out + out = settableValueOf(make([]interface{}, l)) + default: + d.terror(n, seqTag, out) + return false + } + et := out.Type().Elem() + + j := 0 + for i := 0; i < l; i++ { + e := reflect.New(et).Elem() + if ok := d.unmarshal(n.Content[i], e); ok { + out.Index(j).Set(e) + j++ + } + } + if out.Kind() != reflect.Array { + out.Set(out.Slice(0, j)) + } + if iface.IsValid() { + iface.Set(out) + } + return true +} + +func (d *decoder) mapping(n *Node, out reflect.Value) (good bool) { + l := len(n.Content) + if d.uniqueKeys { + nerrs := len(d.terrors) + for i := 0; i < l; i += 2 { + ni := n.Content[i] + for j := i + 2; j < l; j += 2 { + nj := n.Content[j] + if ni.Kind == nj.Kind && ni.Value == nj.Value { + d.terrors = append(d.terrors, fmt.Sprintf("line %d: mapping key %#v already defined at line %d", nj.Line, nj.Value, ni.Line)) + } + } + } + if len(d.terrors) > nerrs { + return false + } + } + switch out.Kind() { + case reflect.Struct: + return d.mappingStruct(n, out) + case reflect.Map: + // okay + case reflect.Interface: + iface := out + if isStringMap(n) { + out = reflect.MakeMap(d.stringMapType) + } else { + out = reflect.MakeMap(d.generalMapType) + } + iface.Set(out) + default: + d.terror(n, mapTag, out) + return false + } + + outt := out.Type() + kt := outt.Key() + et := outt.Elem() + + stringMapType := d.stringMapType + generalMapType := d.generalMapType + if outt.Elem() == ifaceType { + if outt.Key().Kind() == reflect.String { + d.stringMapType = outt + } else if outt.Key() == ifaceType { + d.generalMapType = outt + } + } + + if out.IsNil() { + out.Set(reflect.MakeMap(outt)) + } + for i := 0; i < l; i += 2 { + if isMerge(n.Content[i]) { + d.merge(n.Content[i+1], out) + continue + } + k := reflect.New(kt).Elem() + if d.unmarshal(n.Content[i], k) { + kkind := k.Kind() + if kkind == reflect.Interface { + kkind = k.Elem().Kind() + } + if kkind == reflect.Map || kkind == reflect.Slice { + failf("invalid map key: %#v", k.Interface()) + } + e := reflect.New(et).Elem() + if d.unmarshal(n.Content[i+1], e) { + out.SetMapIndex(k, e) + } + } + } + d.stringMapType = stringMapType + d.generalMapType = generalMapType + return true +} + +func isStringMap(n *Node) bool { + if n.Kind != MappingNode { + return false + } + l := len(n.Content) + for i := 0; i < l; i += 2 { + if n.Content[i].ShortTag() != strTag { + return false + } + } + return true +} + +func (d *decoder) mappingStruct(n *Node, out reflect.Value) (good bool) { + sinfo, err := getStructInfo(out.Type()) + if err != nil { + panic(err) + } + + var inlineMap reflect.Value + var elemType reflect.Type + if sinfo.InlineMap != -1 { + inlineMap = out.Field(sinfo.InlineMap) + inlineMap.Set(reflect.New(inlineMap.Type()).Elem()) + elemType = inlineMap.Type().Elem() + } + + for _, index := range sinfo.InlineUnmarshalers { + field := d.fieldByIndex(n, out, index) + d.prepare(n, field) + } + + var doneFields []bool + if d.uniqueKeys { + doneFields = make([]bool, len(sinfo.FieldsList)) + } + name := settableValueOf("") + l := len(n.Content) + for i := 0; i < l; i += 2 { + ni := n.Content[i] + if isMerge(ni) { + d.merge(n.Content[i+1], out) + continue + } + if !d.unmarshal(ni, name) { + continue + } + if info, ok := sinfo.FieldsMap[name.String()]; ok { + if d.uniqueKeys { + if doneFields[info.Id] { + d.terrors = append(d.terrors, fmt.Sprintf("line %d: field %s already set in type %s", ni.Line, name.String(), out.Type())) + continue + } + doneFields[info.Id] = true + } + var field reflect.Value + if info.Inline == nil { + field = out.Field(info.Num) + } else { + field = d.fieldByIndex(n, out, info.Inline) + } + d.unmarshal(n.Content[i+1], field) + } else if sinfo.InlineMap != -1 { + if inlineMap.IsNil() { + inlineMap.Set(reflect.MakeMap(inlineMap.Type())) + } + value := reflect.New(elemType).Elem() + d.unmarshal(n.Content[i+1], value) + inlineMap.SetMapIndex(name, value) + } else if d.knownFields { + d.terrors = append(d.terrors, fmt.Sprintf("line %d: field %s not found in type %s", ni.Line, name.String(), out.Type())) + } + } + return true +} + +func failWantMap() { + failf("map merge requires map or sequence of maps as the value") +} + +func (d *decoder) merge(n *Node, out reflect.Value) { + switch n.Kind { + case MappingNode: + d.unmarshal(n, out) + case AliasNode: + if n.Alias != nil && n.Alias.Kind != MappingNode { + failWantMap() + } + d.unmarshal(n, out) + case SequenceNode: + // Step backwards as earlier nodes take precedence. + for i := len(n.Content) - 1; i >= 0; i-- { + ni := n.Content[i] + if ni.Kind == AliasNode { + if ni.Alias != nil && ni.Alias.Kind != MappingNode { + failWantMap() + } + } else if ni.Kind != MappingNode { + failWantMap() + } + d.unmarshal(ni, out) + } + default: + failWantMap() + } +} + +func isMerge(n *Node) bool { + return n.Kind == ScalarNode && n.Value == "<<" && (n.Tag == "" || n.Tag == "!" || shortTag(n.Tag) == mergeTag) +} diff --git a/gopkg.in/yaml.v3/emitterc.go b/gopkg.in/yaml.v3/emitterc.go new file mode 100644 index 0000000000..ab2a066194 --- /dev/null +++ b/gopkg.in/yaml.v3/emitterc.go @@ -0,0 +1,1992 @@ +// +// Copyright (c) 2011-2019 Canonical Ltd +// Copyright (c) 2006-2010 Kirill Simonov +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +// of the Software, and to permit persons to whom the Software is furnished to do +// so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package yaml + +import ( + "bytes" + "fmt" +) + +// Flush the buffer if needed. +func flush(emitter *yaml_emitter_t) bool { + if emitter.buffer_pos+5 >= len(emitter.buffer) { + return yaml_emitter_flush(emitter) + } + return true +} + +// Put a character to the output buffer. +func put(emitter *yaml_emitter_t, value byte) bool { + if emitter.buffer_pos+5 >= len(emitter.buffer) && !yaml_emitter_flush(emitter) { + return false + } + emitter.buffer[emitter.buffer_pos] = value + emitter.buffer_pos++ + emitter.column++ + return true +} + +// Put a line break to the output buffer. +func put_break(emitter *yaml_emitter_t) bool { + if emitter.buffer_pos+5 >= len(emitter.buffer) && !yaml_emitter_flush(emitter) { + return false + } + switch emitter.line_break { + case yaml_CR_BREAK: + emitter.buffer[emitter.buffer_pos] = '\r' + emitter.buffer_pos += 1 + case yaml_LN_BREAK: + emitter.buffer[emitter.buffer_pos] = '\n' + emitter.buffer_pos += 1 + case yaml_CRLN_BREAK: + emitter.buffer[emitter.buffer_pos+0] = '\r' + emitter.buffer[emitter.buffer_pos+1] = '\n' + emitter.buffer_pos += 2 + default: + panic("unknown line break setting") + } + if emitter.column == 0 { + emitter.space_above = true + } + emitter.column = 0 + emitter.line++ + // [Go] Do this here and below and drop from everywhere else (see commented lines). + emitter.indention = true + return true +} + +// Copy a character from a string into buffer. +func write(emitter *yaml_emitter_t, s []byte, i *int) bool { + if emitter.buffer_pos+5 >= len(emitter.buffer) && !yaml_emitter_flush(emitter) { + return false + } + p := emitter.buffer_pos + w := width(s[*i]) + switch w { + case 4: + emitter.buffer[p+3] = s[*i+3] + fallthrough + case 3: + emitter.buffer[p+2] = s[*i+2] + fallthrough + case 2: + emitter.buffer[p+1] = s[*i+1] + fallthrough + case 1: + emitter.buffer[p+0] = s[*i+0] + default: + panic("unknown character width") + } + emitter.column++ + emitter.buffer_pos += w + *i += w + return true +} + +// Write a whole string into buffer. +func write_all(emitter *yaml_emitter_t, s []byte) bool { + for i := 0; i < len(s); { + if !write(emitter, s, &i) { + return false + } + } + return true +} + +// Copy a line break character from a string into buffer. +func write_break(emitter *yaml_emitter_t, s []byte, i *int) bool { + if s[*i] == '\n' { + if !put_break(emitter) { + return false + } + *i++ + } else { + if !write(emitter, s, i) { + return false + } + if emitter.column == 0 { + emitter.space_above = true + } + emitter.column = 0 + emitter.line++ + // [Go] Do this here and above and drop from everywhere else (see commented lines). + emitter.indention = true + } + return true +} + +// Set an emitter error and return false. +func yaml_emitter_set_emitter_error(emitter *yaml_emitter_t, problem string) bool { + emitter.error = yaml_EMITTER_ERROR + emitter.problem = problem + return false +} + +// Emit an event. +func yaml_emitter_emit(emitter *yaml_emitter_t, event *yaml_event_t) bool { + emitter.events = append(emitter.events, *event) + for !yaml_emitter_need_more_events(emitter) { + event := &emitter.events[emitter.events_head] + if !yaml_emitter_analyze_event(emitter, event) { + return false + } + if !yaml_emitter_state_machine(emitter, event) { + return false + } + yaml_event_delete(event) + emitter.events_head++ + } + return true +} + +// Check if we need to accumulate more events before emitting. +// +// We accumulate extra +// - 1 event for DOCUMENT-START +// - 2 events for SEQUENCE-START +// - 3 events for MAPPING-START +// +func yaml_emitter_need_more_events(emitter *yaml_emitter_t) bool { + if emitter.events_head == len(emitter.events) { + return true + } + var accumulate int + switch emitter.events[emitter.events_head].typ { + case yaml_DOCUMENT_START_EVENT: + accumulate = 1 + break + case yaml_SEQUENCE_START_EVENT: + accumulate = 2 + break + case yaml_MAPPING_START_EVENT: + accumulate = 3 + break + default: + return false + } + if len(emitter.events)-emitter.events_head > accumulate { + return false + } + var level int + for i := emitter.events_head; i < len(emitter.events); i++ { + switch emitter.events[i].typ { + case yaml_STREAM_START_EVENT, yaml_DOCUMENT_START_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT: + level++ + case yaml_STREAM_END_EVENT, yaml_DOCUMENT_END_EVENT, yaml_SEQUENCE_END_EVENT, yaml_MAPPING_END_EVENT: + level-- + } + if level == 0 { + return false + } + } + return true +} + +// Append a directive to the directives stack. +func yaml_emitter_append_tag_directive(emitter *yaml_emitter_t, value *yaml_tag_directive_t, allow_duplicates bool) bool { + for i := 0; i < len(emitter.tag_directives); i++ { + if bytes.Equal(value.handle, emitter.tag_directives[i].handle) { + if allow_duplicates { + return true + } + return yaml_emitter_set_emitter_error(emitter, "duplicate %TAG directive") + } + } + + // [Go] Do we actually need to copy this given garbage collection + // and the lack of deallocating destructors? + tag_copy := yaml_tag_directive_t{ + handle: make([]byte, len(value.handle)), + prefix: make([]byte, len(value.prefix)), + } + copy(tag_copy.handle, value.handle) + copy(tag_copy.prefix, value.prefix) + emitter.tag_directives = append(emitter.tag_directives, tag_copy) + return true +} + +// Increase the indentation level. +func yaml_emitter_increase_indent(emitter *yaml_emitter_t, flow, indentless bool) bool { + emitter.indents = append(emitter.indents, emitter.indent) + if emitter.indent < 0 { + if flow { + emitter.indent = emitter.best_indent + } else { + emitter.indent = 0 + } + } else if !indentless { + emitter.indent += emitter.best_indent + // [Go] If inside a block sequence item, discount the space taken by the indicator. + if emitter.best_indent > 2 && emitter.states[len(emitter.states)-1] == yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE { + emitter.indent -= 2 + } + } + return true +} + +// State dispatcher. +func yaml_emitter_state_machine(emitter *yaml_emitter_t, event *yaml_event_t) bool { + switch emitter.state { + default: + case yaml_EMIT_STREAM_START_STATE: + return yaml_emitter_emit_stream_start(emitter, event) + + case yaml_EMIT_FIRST_DOCUMENT_START_STATE: + return yaml_emitter_emit_document_start(emitter, event, true) + + case yaml_EMIT_DOCUMENT_START_STATE: + return yaml_emitter_emit_document_start(emitter, event, false) + + case yaml_EMIT_DOCUMENT_CONTENT_STATE: + return yaml_emitter_emit_document_content(emitter, event) + + case yaml_EMIT_DOCUMENT_END_STATE: + return yaml_emitter_emit_document_end(emitter, event) + + case yaml_EMIT_FLOW_SEQUENCE_FIRST_ITEM_STATE: + return yaml_emitter_emit_flow_sequence_item(emitter, event, true, false) + + case yaml_EMIT_FLOW_SEQUENCE_TRAIL_ITEM_STATE: + return yaml_emitter_emit_flow_sequence_item(emitter, event, false, true) + + case yaml_EMIT_FLOW_SEQUENCE_ITEM_STATE: + return yaml_emitter_emit_flow_sequence_item(emitter, event, false, false) + + case yaml_EMIT_FLOW_MAPPING_FIRST_KEY_STATE: + return yaml_emitter_emit_flow_mapping_key(emitter, event, true, false) + + case yaml_EMIT_FLOW_MAPPING_TRAIL_KEY_STATE: + return yaml_emitter_emit_flow_mapping_key(emitter, event, false, true) + + case yaml_EMIT_FLOW_MAPPING_KEY_STATE: + return yaml_emitter_emit_flow_mapping_key(emitter, event, false, false) + + case yaml_EMIT_FLOW_MAPPING_SIMPLE_VALUE_STATE: + return yaml_emitter_emit_flow_mapping_value(emitter, event, true) + + case yaml_EMIT_FLOW_MAPPING_VALUE_STATE: + return yaml_emitter_emit_flow_mapping_value(emitter, event, false) + + case yaml_EMIT_BLOCK_SEQUENCE_FIRST_ITEM_STATE: + return yaml_emitter_emit_block_sequence_item(emitter, event, true) + + case yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE: + return yaml_emitter_emit_block_sequence_item(emitter, event, false) + + case yaml_EMIT_BLOCK_MAPPING_FIRST_KEY_STATE: + return yaml_emitter_emit_block_mapping_key(emitter, event, true) + + case yaml_EMIT_BLOCK_MAPPING_KEY_STATE: + return yaml_emitter_emit_block_mapping_key(emitter, event, false) + + case yaml_EMIT_BLOCK_MAPPING_SIMPLE_VALUE_STATE: + return yaml_emitter_emit_block_mapping_value(emitter, event, true) + + case yaml_EMIT_BLOCK_MAPPING_VALUE_STATE: + return yaml_emitter_emit_block_mapping_value(emitter, event, false) + + case yaml_EMIT_END_STATE: + return yaml_emitter_set_emitter_error(emitter, "expected nothing after STREAM-END") + } + panic("invalid emitter state") +} + +// Expect STREAM-START. +func yaml_emitter_emit_stream_start(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if event.typ != yaml_STREAM_START_EVENT { + return yaml_emitter_set_emitter_error(emitter, "expected STREAM-START") + } + if emitter.encoding == yaml_ANY_ENCODING { + emitter.encoding = event.encoding + if emitter.encoding == yaml_ANY_ENCODING { + emitter.encoding = yaml_UTF8_ENCODING + } + } + if emitter.best_indent < 2 || emitter.best_indent > 9 { + emitter.best_indent = 2 + } + if emitter.best_width >= 0 && emitter.best_width <= emitter.best_indent*2 { + emitter.best_width = 80 + } + if emitter.best_width < 0 { + emitter.best_width = 1<<31 - 1 + } + if emitter.line_break == yaml_ANY_BREAK { + emitter.line_break = yaml_LN_BREAK + } + + emitter.indent = -1 + emitter.line = 0 + emitter.column = 0 + emitter.whitespace = true + emitter.indention = true + emitter.space_above = true + emitter.foot_indent = -1 + + if emitter.encoding != yaml_UTF8_ENCODING { + if !yaml_emitter_write_bom(emitter) { + return false + } + } + emitter.state = yaml_EMIT_FIRST_DOCUMENT_START_STATE + return true +} + +// Expect DOCUMENT-START or STREAM-END. +func yaml_emitter_emit_document_start(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + + if event.typ == yaml_DOCUMENT_START_EVENT { + + if event.version_directive != nil { + if !yaml_emitter_analyze_version_directive(emitter, event.version_directive) { + return false + } + } + + for i := 0; i < len(event.tag_directives); i++ { + tag_directive := &event.tag_directives[i] + if !yaml_emitter_analyze_tag_directive(emitter, tag_directive) { + return false + } + if !yaml_emitter_append_tag_directive(emitter, tag_directive, false) { + return false + } + } + + for i := 0; i < len(default_tag_directives); i++ { + tag_directive := &default_tag_directives[i] + if !yaml_emitter_append_tag_directive(emitter, tag_directive, true) { + return false + } + } + + implicit := event.implicit + if !first || emitter.canonical { + implicit = false + } + + if emitter.open_ended && (event.version_directive != nil || len(event.tag_directives) > 0) { + if !yaml_emitter_write_indicator(emitter, []byte("..."), true, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + + if event.version_directive != nil { + implicit = false + if !yaml_emitter_write_indicator(emitter, []byte("%YAML"), true, false, false) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte("1.1"), true, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + + if len(event.tag_directives) > 0 { + implicit = false + for i := 0; i < len(event.tag_directives); i++ { + tag_directive := &event.tag_directives[i] + if !yaml_emitter_write_indicator(emitter, []byte("%TAG"), true, false, false) { + return false + } + if !yaml_emitter_write_tag_handle(emitter, tag_directive.handle) { + return false + } + if !yaml_emitter_write_tag_content(emitter, tag_directive.prefix, true) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + } + + if yaml_emitter_check_empty_document(emitter) { + implicit = false + } + if !implicit { + if !yaml_emitter_write_indent(emitter) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte("---"), true, false, false) { + return false + } + if emitter.canonical || true { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + } + + if len(emitter.head_comment) > 0 { + if !yaml_emitter_process_head_comment(emitter) { + return false + } + if !put_break(emitter) { + return false + } + } + + emitter.state = yaml_EMIT_DOCUMENT_CONTENT_STATE + return true + } + + if event.typ == yaml_STREAM_END_EVENT { + if emitter.open_ended { + if !yaml_emitter_write_indicator(emitter, []byte("..."), true, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_flush(emitter) { + return false + } + emitter.state = yaml_EMIT_END_STATE + return true + } + + return yaml_emitter_set_emitter_error(emitter, "expected DOCUMENT-START or STREAM-END") +} + +// Expect the root node. +func yaml_emitter_emit_document_content(emitter *yaml_emitter_t, event *yaml_event_t) bool { + emitter.states = append(emitter.states, yaml_EMIT_DOCUMENT_END_STATE) + + if !yaml_emitter_process_head_comment(emitter) { + return false + } + if !yaml_emitter_emit_node(emitter, event, true, false, false, false) { + return false + } + if !yaml_emitter_process_line_comment(emitter) { + return false + } + if !yaml_emitter_process_foot_comment(emitter) { + return false + } + return true +} + +// Expect DOCUMENT-END. +func yaml_emitter_emit_document_end(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if event.typ != yaml_DOCUMENT_END_EVENT { + return yaml_emitter_set_emitter_error(emitter, "expected DOCUMENT-END") + } + // [Go] Force document foot separation. + emitter.foot_indent = 0 + if !yaml_emitter_process_foot_comment(emitter) { + return false + } + emitter.foot_indent = -1 + if !yaml_emitter_write_indent(emitter) { + return false + } + if !event.implicit { + // [Go] Allocate the slice elsewhere. + if !yaml_emitter_write_indicator(emitter, []byte("..."), true, false, false) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_flush(emitter) { + return false + } + emitter.state = yaml_EMIT_DOCUMENT_START_STATE + emitter.tag_directives = emitter.tag_directives[:0] + return true +} + +// Expect a flow item node. +func yaml_emitter_emit_flow_sequence_item(emitter *yaml_emitter_t, event *yaml_event_t, first, trail bool) bool { + if first { + if !yaml_emitter_write_indicator(emitter, []byte{'['}, true, true, false) { + return false + } + if !yaml_emitter_increase_indent(emitter, true, false) { + return false + } + emitter.flow_level++ + } + + if event.typ == yaml_SEQUENCE_END_EVENT { + if emitter.canonical && !first && !trail { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + } + emitter.flow_level-- + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + if emitter.column == 0 || emitter.canonical && !first { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{']'}, false, false, false) { + return false + } + if !yaml_emitter_process_line_comment(emitter) { + return false + } + if !yaml_emitter_process_foot_comment(emitter) { + return false + } + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + + return true + } + + if !first && !trail { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + } + + if !yaml_emitter_process_head_comment(emitter) { + return false + } + if emitter.column == 0 { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + + if emitter.canonical || emitter.column > emitter.best_width { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if len(emitter.line_comment)+len(emitter.foot_comment)+len(emitter.tail_comment) > 0 { + emitter.states = append(emitter.states, yaml_EMIT_FLOW_SEQUENCE_TRAIL_ITEM_STATE) + } else { + emitter.states = append(emitter.states, yaml_EMIT_FLOW_SEQUENCE_ITEM_STATE) + } + if !yaml_emitter_emit_node(emitter, event, false, true, false, false) { + return false + } + if len(emitter.line_comment)+len(emitter.foot_comment)+len(emitter.tail_comment) > 0 { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + } + if !yaml_emitter_process_line_comment(emitter) { + return false + } + if !yaml_emitter_process_foot_comment(emitter) { + return false + } + return true +} + +// Expect a flow key node. +func yaml_emitter_emit_flow_mapping_key(emitter *yaml_emitter_t, event *yaml_event_t, first, trail bool) bool { + if first { + if !yaml_emitter_write_indicator(emitter, []byte{'{'}, true, true, false) { + return false + } + if !yaml_emitter_increase_indent(emitter, true, false) { + return false + } + emitter.flow_level++ + } + + if event.typ == yaml_MAPPING_END_EVENT { + if (emitter.canonical || len(emitter.head_comment)+len(emitter.foot_comment)+len(emitter.tail_comment) > 0) && !first && !trail { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + } + if !yaml_emitter_process_head_comment(emitter) { + return false + } + emitter.flow_level-- + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + if emitter.canonical && !first { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{'}'}, false, false, false) { + return false + } + if !yaml_emitter_process_line_comment(emitter) { + return false + } + if !yaml_emitter_process_foot_comment(emitter) { + return false + } + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true + } + + if !first && !trail { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + } + + if !yaml_emitter_process_head_comment(emitter) { + return false + } + + if emitter.column == 0 { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + + if emitter.canonical || emitter.column > emitter.best_width { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + + if !emitter.canonical && yaml_emitter_check_simple_key(emitter) { + emitter.states = append(emitter.states, yaml_EMIT_FLOW_MAPPING_SIMPLE_VALUE_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, true) + } + if !yaml_emitter_write_indicator(emitter, []byte{'?'}, true, false, false) { + return false + } + emitter.states = append(emitter.states, yaml_EMIT_FLOW_MAPPING_VALUE_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, false) +} + +// Expect a flow value node. +func yaml_emitter_emit_flow_mapping_value(emitter *yaml_emitter_t, event *yaml_event_t, simple bool) bool { + if simple { + if !yaml_emitter_write_indicator(emitter, []byte{':'}, false, false, false) { + return false + } + } else { + if emitter.canonical || emitter.column > emitter.best_width { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{':'}, true, false, false) { + return false + } + } + if len(emitter.line_comment)+len(emitter.foot_comment)+len(emitter.tail_comment) > 0 { + emitter.states = append(emitter.states, yaml_EMIT_FLOW_MAPPING_TRAIL_KEY_STATE) + } else { + emitter.states = append(emitter.states, yaml_EMIT_FLOW_MAPPING_KEY_STATE) + } + if !yaml_emitter_emit_node(emitter, event, false, false, true, false) { + return false + } + if len(emitter.line_comment)+len(emitter.foot_comment)+len(emitter.tail_comment) > 0 { + if !yaml_emitter_write_indicator(emitter, []byte{','}, false, false, false) { + return false + } + } + if !yaml_emitter_process_line_comment(emitter) { + return false + } + if !yaml_emitter_process_foot_comment(emitter) { + return false + } + return true +} + +// Expect a block item node. +func yaml_emitter_emit_block_sequence_item(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + if first { + // [Go] The original logic here would not indent the sequence when inside a mapping. + // In Go we always indent it, but take the sequence indicator out of the indentation. + indentless := emitter.best_indent == 2 && emitter.mapping_context && (emitter.column == 0 || !emitter.indention) + original := emitter.indent + if !yaml_emitter_increase_indent(emitter, false, indentless) { + return false + } + if emitter.indent > original+2 { + emitter.indent -= 2 + } + } + if event.typ == yaml_SEQUENCE_END_EVENT { + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true + } + if !yaml_emitter_process_head_comment(emitter) { + return false + } + if !yaml_emitter_write_indent(emitter) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte{'-'}, true, false, true) { + return false + } + emitter.states = append(emitter.states, yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE) + if !yaml_emitter_emit_node(emitter, event, false, true, false, false) { + return false + } + if !yaml_emitter_process_line_comment(emitter) { + return false + } + if !yaml_emitter_process_foot_comment(emitter) { + return false + } + return true +} + +// Expect a block key node. +func yaml_emitter_emit_block_mapping_key(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { + if first { + if !yaml_emitter_increase_indent(emitter, false, false) { + return false + } + } + if !yaml_emitter_process_head_comment(emitter) { + return false + } + if event.typ == yaml_MAPPING_END_EVENT { + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true + } + if !yaml_emitter_write_indent(emitter) { + return false + } + if yaml_emitter_check_simple_key(emitter) { + emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_SIMPLE_VALUE_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, true) + } + if !yaml_emitter_write_indicator(emitter, []byte{'?'}, true, false, true) { + return false + } + emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_VALUE_STATE) + return yaml_emitter_emit_node(emitter, event, false, false, true, false) +} + +// Expect a block value node. +func yaml_emitter_emit_block_mapping_value(emitter *yaml_emitter_t, event *yaml_event_t, simple bool) bool { + if simple { + if !yaml_emitter_write_indicator(emitter, []byte{':'}, false, false, false) { + return false + } + } else { + if !yaml_emitter_write_indent(emitter) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte{':'}, true, false, true) { + return false + } + } + emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_KEY_STATE) + if !yaml_emitter_emit_node(emitter, event, false, false, true, false) { + return false + } + if !yaml_emitter_process_line_comment(emitter) { + return false + } + if !yaml_emitter_process_foot_comment(emitter) { + return false + } + return true +} + +// Expect a node. +func yaml_emitter_emit_node(emitter *yaml_emitter_t, event *yaml_event_t, + root bool, sequence bool, mapping bool, simple_key bool) bool { + + emitter.root_context = root + emitter.sequence_context = sequence + emitter.mapping_context = mapping + emitter.simple_key_context = simple_key + + switch event.typ { + case yaml_ALIAS_EVENT: + return yaml_emitter_emit_alias(emitter, event) + case yaml_SCALAR_EVENT: + return yaml_emitter_emit_scalar(emitter, event) + case yaml_SEQUENCE_START_EVENT: + return yaml_emitter_emit_sequence_start(emitter, event) + case yaml_MAPPING_START_EVENT: + return yaml_emitter_emit_mapping_start(emitter, event) + default: + return yaml_emitter_set_emitter_error(emitter, + fmt.Sprintf("expected SCALAR, SEQUENCE-START, MAPPING-START, or ALIAS, but got %v", event.typ)) + } +} + +// Expect ALIAS. +func yaml_emitter_emit_alias(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if !yaml_emitter_process_anchor(emitter) { + return false + } + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true +} + +// Expect SCALAR. +func yaml_emitter_emit_scalar(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if !yaml_emitter_select_scalar_style(emitter, event) { + return false + } + if !yaml_emitter_process_anchor(emitter) { + return false + } + if !yaml_emitter_process_tag(emitter) { + return false + } + if !yaml_emitter_increase_indent(emitter, true, false) { + return false + } + if !yaml_emitter_process_scalar(emitter) { + return false + } + emitter.indent = emitter.indents[len(emitter.indents)-1] + emitter.indents = emitter.indents[:len(emitter.indents)-1] + emitter.state = emitter.states[len(emitter.states)-1] + emitter.states = emitter.states[:len(emitter.states)-1] + return true +} + +// Expect SEQUENCE-START. +func yaml_emitter_emit_sequence_start(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if !yaml_emitter_process_anchor(emitter) { + return false + } + if !yaml_emitter_process_tag(emitter) { + return false + } + if emitter.flow_level > 0 || emitter.canonical || event.sequence_style() == yaml_FLOW_SEQUENCE_STYLE || + yaml_emitter_check_empty_sequence(emitter) { + emitter.state = yaml_EMIT_FLOW_SEQUENCE_FIRST_ITEM_STATE + } else { + emitter.state = yaml_EMIT_BLOCK_SEQUENCE_FIRST_ITEM_STATE + } + return true +} + +// Expect MAPPING-START. +func yaml_emitter_emit_mapping_start(emitter *yaml_emitter_t, event *yaml_event_t) bool { + if !yaml_emitter_process_anchor(emitter) { + return false + } + if !yaml_emitter_process_tag(emitter) { + return false + } + if emitter.flow_level > 0 || emitter.canonical || event.mapping_style() == yaml_FLOW_MAPPING_STYLE || + yaml_emitter_check_empty_mapping(emitter) { + emitter.state = yaml_EMIT_FLOW_MAPPING_FIRST_KEY_STATE + } else { + emitter.state = yaml_EMIT_BLOCK_MAPPING_FIRST_KEY_STATE + } + return true +} + +// Check if the document content is an empty scalar. +func yaml_emitter_check_empty_document(emitter *yaml_emitter_t) bool { + return false // [Go] Huh? +} + +// Check if the next events represent an empty sequence. +func yaml_emitter_check_empty_sequence(emitter *yaml_emitter_t) bool { + if len(emitter.events)-emitter.events_head < 2 { + return false + } + return emitter.events[emitter.events_head].typ == yaml_SEQUENCE_START_EVENT && + emitter.events[emitter.events_head+1].typ == yaml_SEQUENCE_END_EVENT +} + +// Check if the next events represent an empty mapping. +func yaml_emitter_check_empty_mapping(emitter *yaml_emitter_t) bool { + if len(emitter.events)-emitter.events_head < 2 { + return false + } + return emitter.events[emitter.events_head].typ == yaml_MAPPING_START_EVENT && + emitter.events[emitter.events_head+1].typ == yaml_MAPPING_END_EVENT +} + +// Check if the next node can be expressed as a simple key. +func yaml_emitter_check_simple_key(emitter *yaml_emitter_t) bool { + length := 0 + switch emitter.events[emitter.events_head].typ { + case yaml_ALIAS_EVENT: + length += len(emitter.anchor_data.anchor) + case yaml_SCALAR_EVENT: + if emitter.scalar_data.multiline { + return false + } + length += len(emitter.anchor_data.anchor) + + len(emitter.tag_data.handle) + + len(emitter.tag_data.suffix) + + len(emitter.scalar_data.value) + case yaml_SEQUENCE_START_EVENT: + if !yaml_emitter_check_empty_sequence(emitter) { + return false + } + length += len(emitter.anchor_data.anchor) + + len(emitter.tag_data.handle) + + len(emitter.tag_data.suffix) + case yaml_MAPPING_START_EVENT: + if !yaml_emitter_check_empty_mapping(emitter) { + return false + } + length += len(emitter.anchor_data.anchor) + + len(emitter.tag_data.handle) + + len(emitter.tag_data.suffix) + default: + return false + } + return length <= 128 +} + +// Determine an acceptable scalar style. +func yaml_emitter_select_scalar_style(emitter *yaml_emitter_t, event *yaml_event_t) bool { + + no_tag := len(emitter.tag_data.handle) == 0 && len(emitter.tag_data.suffix) == 0 + if no_tag && !event.implicit && !event.quoted_implicit { + return yaml_emitter_set_emitter_error(emitter, "neither tag nor implicit flags are specified") + } + + style := event.scalar_style() + if style == yaml_ANY_SCALAR_STYLE { + style = yaml_PLAIN_SCALAR_STYLE + } + if emitter.canonical { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + if emitter.simple_key_context && emitter.scalar_data.multiline { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + + if style == yaml_PLAIN_SCALAR_STYLE { + if emitter.flow_level > 0 && !emitter.scalar_data.flow_plain_allowed || + emitter.flow_level == 0 && !emitter.scalar_data.block_plain_allowed { + style = yaml_SINGLE_QUOTED_SCALAR_STYLE + } + if len(emitter.scalar_data.value) == 0 && (emitter.flow_level > 0 || emitter.simple_key_context) { + style = yaml_SINGLE_QUOTED_SCALAR_STYLE + } + if no_tag && !event.implicit { + style = yaml_SINGLE_QUOTED_SCALAR_STYLE + } + } + if style == yaml_SINGLE_QUOTED_SCALAR_STYLE { + if !emitter.scalar_data.single_quoted_allowed { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + } + if style == yaml_LITERAL_SCALAR_STYLE || style == yaml_FOLDED_SCALAR_STYLE { + if !emitter.scalar_data.block_allowed || emitter.flow_level > 0 || emitter.simple_key_context { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + } + + if no_tag && !event.quoted_implicit && style != yaml_PLAIN_SCALAR_STYLE { + emitter.tag_data.handle = []byte{'!'} + } + emitter.scalar_data.style = style + return true +} + +// Write an anchor. +func yaml_emitter_process_anchor(emitter *yaml_emitter_t) bool { + if emitter.anchor_data.anchor == nil { + return true + } + c := []byte{'&'} + if emitter.anchor_data.alias { + c[0] = '*' + } + if !yaml_emitter_write_indicator(emitter, c, true, false, false) { + return false + } + return yaml_emitter_write_anchor(emitter, emitter.anchor_data.anchor) +} + +// Write a tag. +func yaml_emitter_process_tag(emitter *yaml_emitter_t) bool { + if len(emitter.tag_data.handle) == 0 && len(emitter.tag_data.suffix) == 0 { + return true + } + if len(emitter.tag_data.handle) > 0 { + if !yaml_emitter_write_tag_handle(emitter, emitter.tag_data.handle) { + return false + } + if len(emitter.tag_data.suffix) > 0 { + if !yaml_emitter_write_tag_content(emitter, emitter.tag_data.suffix, false) { + return false + } + } + } else { + // [Go] Allocate these slices elsewhere. + if !yaml_emitter_write_indicator(emitter, []byte("!<"), true, false, false) { + return false + } + if !yaml_emitter_write_tag_content(emitter, emitter.tag_data.suffix, false) { + return false + } + if !yaml_emitter_write_indicator(emitter, []byte{'>'}, false, false, false) { + return false + } + } + return true +} + +// Write a scalar. +func yaml_emitter_process_scalar(emitter *yaml_emitter_t) bool { + switch emitter.scalar_data.style { + case yaml_PLAIN_SCALAR_STYLE: + return yaml_emitter_write_plain_scalar(emitter, emitter.scalar_data.value, !emitter.simple_key_context) + + case yaml_SINGLE_QUOTED_SCALAR_STYLE: + return yaml_emitter_write_single_quoted_scalar(emitter, emitter.scalar_data.value, !emitter.simple_key_context) + + case yaml_DOUBLE_QUOTED_SCALAR_STYLE: + return yaml_emitter_write_double_quoted_scalar(emitter, emitter.scalar_data.value, !emitter.simple_key_context) + + case yaml_LITERAL_SCALAR_STYLE: + return yaml_emitter_write_literal_scalar(emitter, emitter.scalar_data.value) + + case yaml_FOLDED_SCALAR_STYLE: + return yaml_emitter_write_folded_scalar(emitter, emitter.scalar_data.value) + } + panic("unknown scalar style") +} + +// Write a head comment. +func yaml_emitter_process_head_comment(emitter *yaml_emitter_t) bool { + if len(emitter.tail_comment) > 0 { + if !yaml_emitter_write_indent(emitter) { + return false + } + if !yaml_emitter_write_comment(emitter, emitter.tail_comment) { + return false + } + emitter.tail_comment = emitter.tail_comment[:0] + emitter.foot_indent = emitter.indent + if emitter.foot_indent < 0 { + emitter.foot_indent = 0 + } + } + + if len(emitter.head_comment) == 0 { + return true + } + if !yaml_emitter_write_indent(emitter) { + return false + } + if !yaml_emitter_write_comment(emitter, emitter.head_comment) { + return false + } + emitter.head_comment = emitter.head_comment[:0] + return true +} + +// Write an line comment. +func yaml_emitter_process_line_comment(emitter *yaml_emitter_t) bool { + if len(emitter.line_comment) == 0 { + return true + } + if !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + if !yaml_emitter_write_comment(emitter, emitter.line_comment) { + return false + } + emitter.line_comment = emitter.line_comment[:0] + return true +} + +// Write a foot comment. +func yaml_emitter_process_foot_comment(emitter *yaml_emitter_t) bool { + if len(emitter.foot_comment) == 0 { + return true + } + if !yaml_emitter_write_indent(emitter) { + return false + } + if !yaml_emitter_write_comment(emitter, emitter.foot_comment) { + return false + } + emitter.foot_comment = emitter.foot_comment[:0] + emitter.foot_indent = emitter.indent + if emitter.foot_indent < 0 { + emitter.foot_indent = 0 + } + return true +} + +// Check if a %YAML directive is valid. +func yaml_emitter_analyze_version_directive(emitter *yaml_emitter_t, version_directive *yaml_version_directive_t) bool { + if version_directive.major != 1 || version_directive.minor != 1 { + return yaml_emitter_set_emitter_error(emitter, "incompatible %YAML directive") + } + return true +} + +// Check if a %TAG directive is valid. +func yaml_emitter_analyze_tag_directive(emitter *yaml_emitter_t, tag_directive *yaml_tag_directive_t) bool { + handle := tag_directive.handle + prefix := tag_directive.prefix + if len(handle) == 0 { + return yaml_emitter_set_emitter_error(emitter, "tag handle must not be empty") + } + if handle[0] != '!' { + return yaml_emitter_set_emitter_error(emitter, "tag handle must start with '!'") + } + if handle[len(handle)-1] != '!' { + return yaml_emitter_set_emitter_error(emitter, "tag handle must end with '!'") + } + for i := 1; i < len(handle)-1; i += width(handle[i]) { + if !is_alpha(handle, i) { + return yaml_emitter_set_emitter_error(emitter, "tag handle must contain alphanumerical characters only") + } + } + if len(prefix) == 0 { + return yaml_emitter_set_emitter_error(emitter, "tag prefix must not be empty") + } + return true +} + +// Check if an anchor is valid. +func yaml_emitter_analyze_anchor(emitter *yaml_emitter_t, anchor []byte, alias bool) bool { + if len(anchor) == 0 { + problem := "anchor value must not be empty" + if alias { + problem = "alias value must not be empty" + } + return yaml_emitter_set_emitter_error(emitter, problem) + } + for i := 0; i < len(anchor); i += width(anchor[i]) { + if !is_alpha(anchor, i) { + problem := "anchor value must contain alphanumerical characters only" + if alias { + problem = "alias value must contain alphanumerical characters only" + } + return yaml_emitter_set_emitter_error(emitter, problem) + } + } + emitter.anchor_data.anchor = anchor + emitter.anchor_data.alias = alias + return true +} + +// Check if a tag is valid. +func yaml_emitter_analyze_tag(emitter *yaml_emitter_t, tag []byte) bool { + if len(tag) == 0 { + return yaml_emitter_set_emitter_error(emitter, "tag value must not be empty") + } + for i := 0; i < len(emitter.tag_directives); i++ { + tag_directive := &emitter.tag_directives[i] + if bytes.HasPrefix(tag, tag_directive.prefix) { + emitter.tag_data.handle = tag_directive.handle + emitter.tag_data.suffix = tag[len(tag_directive.prefix):] + return true + } + } + emitter.tag_data.suffix = tag + return true +} + +// Check if a scalar is valid. +func yaml_emitter_analyze_scalar(emitter *yaml_emitter_t, value []byte) bool { + var ( + block_indicators = false + flow_indicators = false + line_breaks = false + special_characters = false + tab_characters = false + + leading_space = false + leading_break = false + trailing_space = false + trailing_break = false + break_space = false + space_break = false + + preceded_by_whitespace = false + followed_by_whitespace = false + previous_space = false + previous_break = false + ) + + emitter.scalar_data.value = value + + if len(value) == 0 { + emitter.scalar_data.multiline = false + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = true + emitter.scalar_data.single_quoted_allowed = true + emitter.scalar_data.block_allowed = false + return true + } + + if len(value) >= 3 && ((value[0] == '-' && value[1] == '-' && value[2] == '-') || (value[0] == '.' && value[1] == '.' && value[2] == '.')) { + block_indicators = true + flow_indicators = true + } + + preceded_by_whitespace = true + for i, w := 0, 0; i < len(value); i += w { + w = width(value[i]) + followed_by_whitespace = i+w >= len(value) || is_blank(value, i+w) + + if i == 0 { + switch value[i] { + case '#', ',', '[', ']', '{', '}', '&', '*', '!', '|', '>', '\'', '"', '%', '@', '`': + flow_indicators = true + block_indicators = true + case '?', ':': + flow_indicators = true + if followed_by_whitespace { + block_indicators = true + } + case '-': + if followed_by_whitespace { + flow_indicators = true + block_indicators = true + } + } + } else { + switch value[i] { + case ',', '?', '[', ']', '{', '}': + flow_indicators = true + case ':': + flow_indicators = true + if followed_by_whitespace { + block_indicators = true + } + case '#': + if preceded_by_whitespace { + flow_indicators = true + block_indicators = true + } + } + } + + if value[i] == '\t' { + tab_characters = true + } else if !is_printable(value, i) || !is_ascii(value, i) && !emitter.unicode { + special_characters = true + } + if is_space(value, i) { + if i == 0 { + leading_space = true + } + if i+width(value[i]) == len(value) { + trailing_space = true + } + if previous_break { + break_space = true + } + previous_space = true + previous_break = false + } else if is_break(value, i) { + line_breaks = true + if i == 0 { + leading_break = true + } + if i+width(value[i]) == len(value) { + trailing_break = true + } + if previous_space { + space_break = true + } + previous_space = false + previous_break = true + } else { + previous_space = false + previous_break = false + } + + // [Go]: Why 'z'? Couldn't be the end of the string as that's the loop condition. + preceded_by_whitespace = is_blankz(value, i) + } + + emitter.scalar_data.multiline = line_breaks + emitter.scalar_data.flow_plain_allowed = true + emitter.scalar_data.block_plain_allowed = true + emitter.scalar_data.single_quoted_allowed = true + emitter.scalar_data.block_allowed = true + + if leading_space || leading_break || trailing_space || trailing_break { + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = false + } + if trailing_space { + emitter.scalar_data.block_allowed = false + } + if break_space { + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = false + emitter.scalar_data.single_quoted_allowed = false + } + if space_break || tab_characters || special_characters { + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = false + emitter.scalar_data.single_quoted_allowed = false + } + if space_break || special_characters { + emitter.scalar_data.block_allowed = false + } + if line_breaks { + emitter.scalar_data.flow_plain_allowed = false + emitter.scalar_data.block_plain_allowed = false + } + if flow_indicators { + emitter.scalar_data.flow_plain_allowed = false + } + if block_indicators { + emitter.scalar_data.block_plain_allowed = false + } + return true +} + +// Check if the event data is valid. +func yaml_emitter_analyze_event(emitter *yaml_emitter_t, event *yaml_event_t) bool { + + emitter.anchor_data.anchor = nil + emitter.tag_data.handle = nil + emitter.tag_data.suffix = nil + emitter.scalar_data.value = nil + + if len(event.head_comment) > 0 { + emitter.head_comment = event.head_comment + } + if len(event.line_comment) > 0 { + emitter.line_comment = event.line_comment + } + if len(event.foot_comment) > 0 { + emitter.foot_comment = event.foot_comment + } + if len(event.tail_comment) > 0 { + emitter.tail_comment = event.tail_comment + } + + switch event.typ { + case yaml_ALIAS_EVENT: + if !yaml_emitter_analyze_anchor(emitter, event.anchor, true) { + return false + } + + case yaml_SCALAR_EVENT: + if len(event.anchor) > 0 { + if !yaml_emitter_analyze_anchor(emitter, event.anchor, false) { + return false + } + } + if len(event.tag) > 0 && (emitter.canonical || (!event.implicit && !event.quoted_implicit)) { + if !yaml_emitter_analyze_tag(emitter, event.tag) { + return false + } + } + if !yaml_emitter_analyze_scalar(emitter, event.value) { + return false + } + + case yaml_SEQUENCE_START_EVENT: + if len(event.anchor) > 0 { + if !yaml_emitter_analyze_anchor(emitter, event.anchor, false) { + return false + } + } + if len(event.tag) > 0 && (emitter.canonical || !event.implicit) { + if !yaml_emitter_analyze_tag(emitter, event.tag) { + return false + } + } + + case yaml_MAPPING_START_EVENT: + if len(event.anchor) > 0 { + if !yaml_emitter_analyze_anchor(emitter, event.anchor, false) { + return false + } + } + if len(event.tag) > 0 && (emitter.canonical || !event.implicit) { + if !yaml_emitter_analyze_tag(emitter, event.tag) { + return false + } + } + } + return true +} + +// Write the BOM character. +func yaml_emitter_write_bom(emitter *yaml_emitter_t) bool { + if !flush(emitter) { + return false + } + pos := emitter.buffer_pos + emitter.buffer[pos+0] = '\xEF' + emitter.buffer[pos+1] = '\xBB' + emitter.buffer[pos+2] = '\xBF' + emitter.buffer_pos += 3 + return true +} + +func yaml_emitter_write_indent(emitter *yaml_emitter_t) bool { + indent := emitter.indent + if indent < 0 { + indent = 0 + } + if !emitter.indention || emitter.column > indent || (emitter.column == indent && !emitter.whitespace) { + if !put_break(emitter) { + return false + } + } + if emitter.foot_indent == indent { + if !put_break(emitter) { + return false + } + } + for emitter.column < indent { + if !put(emitter, ' ') { + return false + } + } + emitter.whitespace = true + //emitter.indention = true + emitter.space_above = false + emitter.foot_indent = -1 + return true +} + +func yaml_emitter_write_indicator(emitter *yaml_emitter_t, indicator []byte, need_whitespace, is_whitespace, is_indention bool) bool { + if need_whitespace && !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + if !write_all(emitter, indicator) { + return false + } + emitter.whitespace = is_whitespace + emitter.indention = (emitter.indention && is_indention) + emitter.open_ended = false + return true +} + +func yaml_emitter_write_anchor(emitter *yaml_emitter_t, value []byte) bool { + if !write_all(emitter, value) { + return false + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_tag_handle(emitter *yaml_emitter_t, value []byte) bool { + if !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + if !write_all(emitter, value) { + return false + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_tag_content(emitter *yaml_emitter_t, value []byte, need_whitespace bool) bool { + if need_whitespace && !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + for i := 0; i < len(value); { + var must_write bool + switch value[i] { + case ';', '/', '?', ':', '@', '&', '=', '+', '$', ',', '_', '.', '~', '*', '\'', '(', ')', '[', ']': + must_write = true + default: + must_write = is_alpha(value, i) + } + if must_write { + if !write(emitter, value, &i) { + return false + } + } else { + w := width(value[i]) + for k := 0; k < w; k++ { + octet := value[i] + i++ + if !put(emitter, '%') { + return false + } + + c := octet >> 4 + if c < 10 { + c += '0' + } else { + c += 'A' - 10 + } + if !put(emitter, c) { + return false + } + + c = octet & 0x0f + if c < 10 { + c += '0' + } else { + c += 'A' - 10 + } + if !put(emitter, c) { + return false + } + } + } + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_plain_scalar(emitter *yaml_emitter_t, value []byte, allow_breaks bool) bool { + if len(value) > 0 && !emitter.whitespace { + if !put(emitter, ' ') { + return false + } + } + + spaces := false + breaks := false + for i := 0; i < len(value); { + if is_space(value, i) { + if allow_breaks && !spaces && emitter.column > emitter.best_width && !is_space(value, i+1) { + if !yaml_emitter_write_indent(emitter) { + return false + } + i += width(value[i]) + } else { + if !write(emitter, value, &i) { + return false + } + } + spaces = true + } else if is_break(value, i) { + if !breaks && value[i] == '\n' { + if !put_break(emitter) { + return false + } + } + if !write_break(emitter, value, &i) { + return false + } + //emitter.indention = true + breaks = true + } else { + if breaks { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !write(emitter, value, &i) { + return false + } + emitter.indention = false + spaces = false + breaks = false + } + } + + if len(value) > 0 { + emitter.whitespace = false + } + emitter.indention = false + if emitter.root_context { + emitter.open_ended = true + } + + return true +} + +func yaml_emitter_write_single_quoted_scalar(emitter *yaml_emitter_t, value []byte, allow_breaks bool) bool { + + if !yaml_emitter_write_indicator(emitter, []byte{'\''}, true, false, false) { + return false + } + + spaces := false + breaks := false + for i := 0; i < len(value); { + if is_space(value, i) { + if allow_breaks && !spaces && emitter.column > emitter.best_width && i > 0 && i < len(value)-1 && !is_space(value, i+1) { + if !yaml_emitter_write_indent(emitter) { + return false + } + i += width(value[i]) + } else { + if !write(emitter, value, &i) { + return false + } + } + spaces = true + } else if is_break(value, i) { + if !breaks && value[i] == '\n' { + if !put_break(emitter) { + return false + } + } + if !write_break(emitter, value, &i) { + return false + } + //emitter.indention = true + breaks = true + } else { + if breaks { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if value[i] == '\'' { + if !put(emitter, '\'') { + return false + } + } + if !write(emitter, value, &i) { + return false + } + emitter.indention = false + spaces = false + breaks = false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{'\''}, false, false, false) { + return false + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_double_quoted_scalar(emitter *yaml_emitter_t, value []byte, allow_breaks bool) bool { + spaces := false + if !yaml_emitter_write_indicator(emitter, []byte{'"'}, true, false, false) { + return false + } + + for i := 0; i < len(value); { + if !is_printable(value, i) || (!emitter.unicode && !is_ascii(value, i)) || + is_bom(value, i) || is_break(value, i) || + value[i] == '"' || value[i] == '\\' { + + octet := value[i] + + var w int + var v rune + switch { + case octet&0x80 == 0x00: + w, v = 1, rune(octet&0x7F) + case octet&0xE0 == 0xC0: + w, v = 2, rune(octet&0x1F) + case octet&0xF0 == 0xE0: + w, v = 3, rune(octet&0x0F) + case octet&0xF8 == 0xF0: + w, v = 4, rune(octet&0x07) + } + for k := 1; k < w; k++ { + octet = value[i+k] + v = (v << 6) + (rune(octet) & 0x3F) + } + i += w + + if !put(emitter, '\\') { + return false + } + + var ok bool + switch v { + case 0x00: + ok = put(emitter, '0') + case 0x07: + ok = put(emitter, 'a') + case 0x08: + ok = put(emitter, 'b') + case 0x09: + ok = put(emitter, 't') + case 0x0A: + ok = put(emitter, 'n') + case 0x0b: + ok = put(emitter, 'v') + case 0x0c: + ok = put(emitter, 'f') + case 0x0d: + ok = put(emitter, 'r') + case 0x1b: + ok = put(emitter, 'e') + case 0x22: + ok = put(emitter, '"') + case 0x5c: + ok = put(emitter, '\\') + case 0x85: + ok = put(emitter, 'N') + case 0xA0: + ok = put(emitter, '_') + case 0x2028: + ok = put(emitter, 'L') + case 0x2029: + ok = put(emitter, 'P') + default: + if v <= 0xFF { + ok = put(emitter, 'x') + w = 2 + } else if v <= 0xFFFF { + ok = put(emitter, 'u') + w = 4 + } else { + ok = put(emitter, 'U') + w = 8 + } + for k := (w - 1) * 4; ok && k >= 0; k -= 4 { + digit := byte((v >> uint(k)) & 0x0F) + if digit < 10 { + ok = put(emitter, digit+'0') + } else { + ok = put(emitter, digit+'A'-10) + } + } + } + if !ok { + return false + } + spaces = false + } else if is_space(value, i) { + if allow_breaks && !spaces && emitter.column > emitter.best_width && i > 0 && i < len(value)-1 { + if !yaml_emitter_write_indent(emitter) { + return false + } + if is_space(value, i+1) { + if !put(emitter, '\\') { + return false + } + } + i += width(value[i]) + } else if !write(emitter, value, &i) { + return false + } + spaces = true + } else { + if !write(emitter, value, &i) { + return false + } + spaces = false + } + } + if !yaml_emitter_write_indicator(emitter, []byte{'"'}, false, false, false) { + return false + } + emitter.whitespace = false + emitter.indention = false + return true +} + +func yaml_emitter_write_block_scalar_hints(emitter *yaml_emitter_t, value []byte) bool { + if is_space(value, 0) || is_break(value, 0) { + indent_hint := []byte{'0' + byte(emitter.best_indent)} + if !yaml_emitter_write_indicator(emitter, indent_hint, false, false, false) { + return false + } + } + + emitter.open_ended = false + + var chomp_hint [1]byte + if len(value) == 0 { + chomp_hint[0] = '-' + } else { + i := len(value) - 1 + for value[i]&0xC0 == 0x80 { + i-- + } + if !is_break(value, i) { + chomp_hint[0] = '-' + } else if i == 0 { + chomp_hint[0] = '+' + emitter.open_ended = true + } else { + i-- + for value[i]&0xC0 == 0x80 { + i-- + } + if is_break(value, i) { + chomp_hint[0] = '+' + emitter.open_ended = true + } + } + } + if chomp_hint[0] != 0 { + if !yaml_emitter_write_indicator(emitter, chomp_hint[:], false, false, false) { + return false + } + } + return true +} + +func yaml_emitter_write_literal_scalar(emitter *yaml_emitter_t, value []byte) bool { + if !yaml_emitter_write_indicator(emitter, []byte{'|'}, true, false, false) { + return false + } + if !yaml_emitter_write_block_scalar_hints(emitter, value) { + return false + } + if !put_break(emitter) { + return false + } + //emitter.indention = true + emitter.whitespace = true + breaks := true + for i := 0; i < len(value); { + if is_break(value, i) { + if !write_break(emitter, value, &i) { + return false + } + //emitter.indention = true + breaks = true + } else { + if breaks { + if !yaml_emitter_write_indent(emitter) { + return false + } + } + if !write(emitter, value, &i) { + return false + } + emitter.indention = false + breaks = false + } + } + + return true +} + +func yaml_emitter_write_folded_scalar(emitter *yaml_emitter_t, value []byte) bool { + if !yaml_emitter_write_indicator(emitter, []byte{'>'}, true, false, false) { + return false + } + if !yaml_emitter_write_block_scalar_hints(emitter, value) { + return false + } + + if !put_break(emitter) { + return false + } + //emitter.indention = true + emitter.whitespace = true + + breaks := true + leading_spaces := true + for i := 0; i < len(value); { + if is_break(value, i) { + if !breaks && !leading_spaces && value[i] == '\n' { + k := 0 + for is_break(value, k) { + k += width(value[k]) + } + if !is_blankz(value, k) { + if !put_break(emitter) { + return false + } + } + } + if !write_break(emitter, value, &i) { + return false + } + //emitter.indention = true + breaks = true + } else { + if breaks { + if !yaml_emitter_write_indent(emitter) { + return false + } + leading_spaces = is_blank(value, i) + } + if !breaks && is_space(value, i) && !is_space(value, i+1) && emitter.column > emitter.best_width { + if !yaml_emitter_write_indent(emitter) { + return false + } + i += width(value[i]) + } else { + if !write(emitter, value, &i) { + return false + } + } + emitter.indention = false + breaks = false + } + } + return true +} + +func yaml_emitter_write_comment(emitter *yaml_emitter_t, comment []byte) bool { + breaks := false + pound := false + for i := 0; i < len(comment); { + if is_break(comment, i) { + if !write_break(emitter, comment, &i) { + return false + } + //emitter.indention = true + breaks = true + pound = false + } else { + if breaks && !yaml_emitter_write_indent(emitter) { + return false + } + if !pound { + if comment[i] != '#' && (!put(emitter, '#') || !put(emitter, ' ')) { + return false + } + pound = true + } + if !write(emitter, comment, &i) { + return false + } + emitter.indention = false + breaks = false + } + } + if !breaks && !put_break(emitter) { + return false + } + + emitter.whitespace = true + //emitter.indention = true + return true +} diff --git a/gopkg.in/yaml.v3/encode.go b/gopkg.in/yaml.v3/encode.go new file mode 100644 index 0000000000..1f37271ce4 --- /dev/null +++ b/gopkg.in/yaml.v3/encode.go @@ -0,0 +1,561 @@ +// +// Copyright (c) 2011-2019 Canonical Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package yaml + +import ( + "encoding" + "fmt" + "io" + "reflect" + "regexp" + "sort" + "strconv" + "strings" + "time" + "unicode/utf8" +) + +type encoder struct { + emitter yaml_emitter_t + event yaml_event_t + out []byte + flow bool + indent int + doneInit bool +} + +func newEncoder() *encoder { + e := &encoder{} + yaml_emitter_initialize(&e.emitter) + yaml_emitter_set_output_string(&e.emitter, &e.out) + yaml_emitter_set_unicode(&e.emitter, true) + return e +} + +func newEncoderWithWriter(w io.Writer) *encoder { + e := &encoder{} + yaml_emitter_initialize(&e.emitter) + yaml_emitter_set_output_writer(&e.emitter, w) + yaml_emitter_set_unicode(&e.emitter, true) + return e +} + +func (e *encoder) init() { + if e.doneInit { + return + } + if e.indent == 0 { + e.indent = 4 + } + e.emitter.best_indent = e.indent + yaml_stream_start_event_initialize(&e.event, yaml_UTF8_ENCODING) + e.emit() + e.doneInit = true +} + +func (e *encoder) finish() { + e.emitter.open_ended = false + yaml_stream_end_event_initialize(&e.event) + e.emit() +} + +func (e *encoder) destroy() { + yaml_emitter_delete(&e.emitter) +} + +func (e *encoder) emit() { + // This will internally delete the e.event value. + e.must(yaml_emitter_emit(&e.emitter, &e.event)) +} + +func (e *encoder) must(ok bool) { + if !ok { + msg := e.emitter.problem + if msg == "" { + msg = "unknown problem generating YAML content" + } + failf("%s", msg) + } +} + +func (e *encoder) marshalDoc(tag string, in reflect.Value) { + e.init() + var node *Node + if in.IsValid() { + node, _ = in.Interface().(*Node) + } + if node != nil && node.Kind == DocumentNode { + e.nodev(in) + } else { + yaml_document_start_event_initialize(&e.event, nil, nil, true) + e.emit() + e.marshal(tag, in) + yaml_document_end_event_initialize(&e.event, true) + e.emit() + } +} + +func (e *encoder) marshal(tag string, in reflect.Value) { + tag = shortTag(tag) + if !in.IsValid() || in.Kind() == reflect.Ptr && in.IsNil() { + e.nilv() + return + } + iface := in.Interface() + switch value := iface.(type) { + case *Node: + e.nodev(in) + return + case time.Time: + e.timev(tag, in) + return + case *time.Time: + e.timev(tag, in.Elem()) + return + case time.Duration: + e.stringv(tag, reflect.ValueOf(value.String())) + return + case Marshaler: + v, err := value.MarshalYAML() + if err != nil { + fail(err) + } + if v == nil { + e.nilv() + return + } + e.marshal(tag, reflect.ValueOf(v)) + return + case encoding.TextMarshaler: + text, err := value.MarshalText() + if err != nil { + fail(err) + } + in = reflect.ValueOf(string(text)) + case nil: + e.nilv() + return + } + switch in.Kind() { + case reflect.Interface: + e.marshal(tag, in.Elem()) + case reflect.Map: + e.mapv(tag, in) + case reflect.Ptr: + e.marshal(tag, in.Elem()) + case reflect.Struct: + e.structv(tag, in) + case reflect.Slice, reflect.Array: + e.slicev(tag, in) + case reflect.String: + e.stringv(tag, in) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + e.intv(tag, in) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + e.uintv(tag, in) + case reflect.Float32, reflect.Float64: + e.floatv(tag, in) + case reflect.Bool: + e.boolv(tag, in) + default: + panic("cannot marshal type: " + in.Type().String()) + } +} + +func (e *encoder) mapv(tag string, in reflect.Value) { + e.mappingv(tag, func() { + keys := keyList(in.MapKeys()) + sort.Sort(keys) + for _, k := range keys { + e.marshal("", k) + e.marshal("", in.MapIndex(k)) + } + }) +} + +func (e *encoder) fieldByIndex(v reflect.Value, index []int) (field reflect.Value) { + for _, num := range index { + for { + if v.Kind() == reflect.Ptr { + if v.IsNil() { + return reflect.Value{} + } + v = v.Elem() + continue + } + break + } + v = v.Field(num) + } + return v +} + +func (e *encoder) structv(tag string, in reflect.Value) { + sinfo, err := getStructInfo(in.Type()) + if err != nil { + panic(err) + } + e.mappingv(tag, func() { + for _, info := range sinfo.FieldsList { + var value reflect.Value + if info.Inline == nil { + value = in.Field(info.Num) + } else { + value = e.fieldByIndex(in, info.Inline) + if !value.IsValid() { + continue + } + } + if info.OmitEmpty && isZero(value) { + continue + } + e.marshal("", reflect.ValueOf(info.Key)) + e.flow = info.Flow + e.marshal("", value) + } + if sinfo.InlineMap >= 0 { + m := in.Field(sinfo.InlineMap) + if m.Len() > 0 { + e.flow = false + keys := keyList(m.MapKeys()) + sort.Sort(keys) + for _, k := range keys { + if _, found := sinfo.FieldsMap[k.String()]; found { + panic(fmt.Sprintf("cannot have key %q in inlined map: conflicts with struct field", k.String())) + } + e.marshal("", k) + e.flow = false + e.marshal("", m.MapIndex(k)) + } + } + } + }) +} + +func (e *encoder) mappingv(tag string, f func()) { + implicit := tag == "" + style := yaml_BLOCK_MAPPING_STYLE + if e.flow { + e.flow = false + style = yaml_FLOW_MAPPING_STYLE + } + yaml_mapping_start_event_initialize(&e.event, nil, []byte(tag), implicit, style) + e.emit() + f() + yaml_mapping_end_event_initialize(&e.event) + e.emit() +} + +func (e *encoder) slicev(tag string, in reflect.Value) { + implicit := tag == "" + style := yaml_BLOCK_SEQUENCE_STYLE + if e.flow { + e.flow = false + style = yaml_FLOW_SEQUENCE_STYLE + } + e.must(yaml_sequence_start_event_initialize(&e.event, nil, []byte(tag), implicit, style)) + e.emit() + n := in.Len() + for i := 0; i < n; i++ { + e.marshal("", in.Index(i)) + } + e.must(yaml_sequence_end_event_initialize(&e.event)) + e.emit() +} + +// isBase60 returns whether s is in base 60 notation as defined in YAML 1.1. +// +// The base 60 float notation in YAML 1.1 is a terrible idea and is unsupported +// in YAML 1.2 and by this package, but these should be marshalled quoted for +// the time being for compatibility with other parsers. +func isBase60Float(s string) (result bool) { + // Fast path. + if s == "" { + return false + } + c := s[0] + if !(c == '+' || c == '-' || c >= '0' && c <= '9') || strings.IndexByte(s, ':') < 0 { + return false + } + // Do the full match. + return base60float.MatchString(s) +} + +// From http://yaml.org/type/float.html, except the regular expression there +// is bogus. In practice parsers do not enforce the "\.[0-9_]*" suffix. +var base60float = regexp.MustCompile(`^[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+(?:\.[0-9_]*)?$`) + +// isOldBool returns whether s is bool notation as defined in YAML 1.1. +// +// We continue to force strings that YAML 1.1 would interpret as booleans to be +// rendered as quotes strings so that the marshalled output valid for YAML 1.1 +// parsing. +func isOldBool(s string) (result bool) { + switch s { + case "y", "Y", "yes", "Yes", "YES", "on", "On", "ON", + "n", "N", "no", "No", "NO", "off", "Off", "OFF": + return true + default: + return false + } +} + +func (e *encoder) stringv(tag string, in reflect.Value) { + var style yaml_scalar_style_t + s := in.String() + canUsePlain := true + switch { + case !utf8.ValidString(s): + if tag == binaryTag { + failf("explicitly tagged !!binary data must be base64-encoded") + } + if tag != "" { + failf("cannot marshal invalid UTF-8 data as %s", shortTag(tag)) + } + // It can't be encoded directly as YAML so use a binary tag + // and encode it as base64. + tag = binaryTag + s = encodeBase64(s) + case tag == "": + // Check to see if it would resolve to a specific + // tag when encoded unquoted. If it doesn't, + // there's no need to quote it. + rtag, _ := resolve("", s) + canUsePlain = rtag == strTag && !(isBase60Float(s) || isOldBool(s)) + } + // Note: it's possible for user code to emit invalid YAML + // if they explicitly specify a tag and a string containing + // text that's incompatible with that tag. + switch { + case strings.Contains(s, "\n"): + if e.flow { + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } else { + style = yaml_LITERAL_SCALAR_STYLE + } + case canUsePlain: + style = yaml_PLAIN_SCALAR_STYLE + default: + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + e.emitScalar(s, "", tag, style, nil, nil, nil, nil) +} + +func (e *encoder) boolv(tag string, in reflect.Value) { + var s string + if in.Bool() { + s = "true" + } else { + s = "false" + } + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE, nil, nil, nil, nil) +} + +func (e *encoder) intv(tag string, in reflect.Value) { + s := strconv.FormatInt(in.Int(), 10) + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE, nil, nil, nil, nil) +} + +func (e *encoder) uintv(tag string, in reflect.Value) { + s := strconv.FormatUint(in.Uint(), 10) + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE, nil, nil, nil, nil) +} + +func (e *encoder) timev(tag string, in reflect.Value) { + t := in.Interface().(time.Time) + s := t.Format(time.RFC3339Nano) + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE, nil, nil, nil, nil) +} + +func (e *encoder) floatv(tag string, in reflect.Value) { + // Issue #352: When formatting, use the precision of the underlying value + precision := 64 + if in.Kind() == reflect.Float32 { + precision = 32 + } + + s := strconv.FormatFloat(in.Float(), 'g', -1, precision) + switch s { + case "+Inf": + s = ".inf" + case "-Inf": + s = "-.inf" + case "NaN": + s = ".nan" + } + e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE, nil, nil, nil, nil) +} + +func (e *encoder) nilv() { + e.emitScalar("null", "", "", yaml_PLAIN_SCALAR_STYLE, nil, nil, nil, nil) +} + +func (e *encoder) emitScalar(value, anchor, tag string, style yaml_scalar_style_t, head, line, foot, tail []byte) { + // TODO Kill this function. Replace all initialize calls by their underlining Go literals. + implicit := tag == "" + if !implicit { + tag = longTag(tag) + } + e.must(yaml_scalar_event_initialize(&e.event, []byte(anchor), []byte(tag), []byte(value), implicit, implicit, style)) + e.event.head_comment = head + e.event.line_comment = line + e.event.foot_comment = foot + e.event.tail_comment = tail + e.emit() +} + +func (e *encoder) nodev(in reflect.Value) { + e.node(in.Interface().(*Node), "") +} + +func (e *encoder) node(node *Node, tail string) { + // If the tag was not explicitly requested, and dropping it won't change the + // implicit tag of the value, don't include it in the presentation. + var tag = node.Tag + var stag = shortTag(tag) + var rtag string + var forceQuoting bool + if tag != "" && node.Style&TaggedStyle == 0 { + if node.Kind == ScalarNode { + if stag == strTag && node.Style&(SingleQuotedStyle|DoubleQuotedStyle|LiteralStyle|FoldedStyle) != 0 { + tag = "" + } else { + rtag, _ = resolve("", node.Value) + if rtag == stag { + tag = "" + } else if stag == strTag { + tag = "" + forceQuoting = true + } + } + } else { + switch node.Kind { + case MappingNode: + rtag = mapTag + case SequenceNode: + rtag = seqTag + } + if rtag == stag { + tag = "" + } + } + } + + switch node.Kind { + case DocumentNode: + yaml_document_start_event_initialize(&e.event, nil, nil, true) + e.event.head_comment = []byte(node.HeadComment) + e.emit() + for _, node := range node.Content { + e.node(node, "") + } + yaml_document_end_event_initialize(&e.event, true) + e.event.foot_comment = []byte(node.FootComment) + e.emit() + + case SequenceNode: + style := yaml_BLOCK_SEQUENCE_STYLE + if node.Style&FlowStyle != 0 { + style = yaml_FLOW_SEQUENCE_STYLE + } + e.must(yaml_sequence_start_event_initialize(&e.event, []byte(node.Anchor), []byte(tag), tag == "", style)) + e.event.head_comment = []byte(node.HeadComment) + e.emit() + for _, node := range node.Content { + e.node(node, "") + } + e.must(yaml_sequence_end_event_initialize(&e.event)) + e.event.line_comment = []byte(node.LineComment) + e.event.foot_comment = []byte(node.FootComment) + e.emit() + + case MappingNode: + style := yaml_BLOCK_MAPPING_STYLE + if node.Style&FlowStyle != 0 { + style = yaml_FLOW_MAPPING_STYLE + } + yaml_mapping_start_event_initialize(&e.event, []byte(node.Anchor), []byte(tag), tag == "", style) + e.event.tail_comment = []byte(tail) + e.event.head_comment = []byte(node.HeadComment) + e.emit() + + // The tail logic below moves the foot comment of prior keys to the following key, + // since the value for each key may be a nested structure and the foot needs to be + // processed only the entirety of the value is streamed. The last tail is processed + // with the mapping end event. + var tail string + for i := 0; i+1 < len(node.Content); i += 2 { + k := node.Content[i] + foot := k.FootComment + if foot != "" { + kopy := *k + kopy.FootComment = "" + k = &kopy + } + e.node(k, tail) + tail = foot + + v := node.Content[i+1] + e.node(v, "") + } + + yaml_mapping_end_event_initialize(&e.event) + e.event.tail_comment = []byte(tail) + e.event.line_comment = []byte(node.LineComment) + e.event.foot_comment = []byte(node.FootComment) + e.emit() + + case AliasNode: + yaml_alias_event_initialize(&e.event, []byte(node.Value)) + e.event.head_comment = []byte(node.HeadComment) + e.event.line_comment = []byte(node.LineComment) + e.event.foot_comment = []byte(node.FootComment) + e.emit() + + case ScalarNode: + value := node.Value + if !utf8.ValidString(value) { + if tag == binaryTag { + failf("explicitly tagged !!binary data must be base64-encoded") + } + if tag != "" { + failf("cannot marshal invalid UTF-8 data as %s", shortTag(tag)) + } + // It can't be encoded directly as YAML so use a binary tag + // and encode it as base64. + tag = binaryTag + value = encodeBase64(value) + } + + style := yaml_PLAIN_SCALAR_STYLE + switch { + case node.Style&DoubleQuotedStyle != 0: + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + case node.Style&SingleQuotedStyle != 0: + style = yaml_SINGLE_QUOTED_SCALAR_STYLE + case node.Style&LiteralStyle != 0: + style = yaml_LITERAL_SCALAR_STYLE + case node.Style&FoldedStyle != 0: + style = yaml_FOLDED_SCALAR_STYLE + case strings.Contains(value, "\n"): + style = yaml_LITERAL_SCALAR_STYLE + case forceQuoting: + style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + + e.emitScalar(value, node.Anchor, tag, style, []byte(node.HeadComment), []byte(node.LineComment), []byte(node.FootComment), []byte(tail)) + } +} diff --git a/gopkg.in/yaml.v3/go.mod b/gopkg.in/yaml.v3/go.mod new file mode 100644 index 0000000000..f407ea3213 --- /dev/null +++ b/gopkg.in/yaml.v3/go.mod @@ -0,0 +1,5 @@ +module "gopkg.in/yaml.v3" + +require ( + "gopkg.in/check.v1" v0.0.0-20161208181325-20d25e280405 +) diff --git a/gopkg.in/yaml.v3/parserc.go b/gopkg.in/yaml.v3/parserc.go new file mode 100644 index 0000000000..aea9050b83 --- /dev/null +++ b/gopkg.in/yaml.v3/parserc.go @@ -0,0 +1,1229 @@ +// +// Copyright (c) 2011-2019 Canonical Ltd +// Copyright (c) 2006-2010 Kirill Simonov +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +// of the Software, and to permit persons to whom the Software is furnished to do +// so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package yaml + +import ( + "bytes" +) + +// The parser implements the following grammar: +// +// stream ::= STREAM-START implicit_document? explicit_document* STREAM-END +// implicit_document ::= block_node DOCUMENT-END* +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// block_node_or_indentless_sequence ::= +// ALIAS +// | properties (block_content | indentless_block_sequence)? +// | block_content +// | indentless_block_sequence +// block_node ::= ALIAS +// | properties block_content? +// | block_content +// flow_node ::= ALIAS +// | properties flow_content? +// | flow_content +// properties ::= TAG ANCHOR? | ANCHOR TAG? +// block_content ::= block_collection | flow_collection | SCALAR +// flow_content ::= flow_collection | SCALAR +// block_collection ::= block_sequence | block_mapping +// flow_collection ::= flow_sequence | flow_mapping +// block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END +// indentless_sequence ::= (BLOCK-ENTRY block_node?)+ +// block_mapping ::= BLOCK-MAPPING_START +// ((KEY block_node_or_indentless_sequence?)? +// (VALUE block_node_or_indentless_sequence?)?)* +// BLOCK-END +// flow_sequence ::= FLOW-SEQUENCE-START +// (flow_sequence_entry FLOW-ENTRY)* +// flow_sequence_entry? +// FLOW-SEQUENCE-END +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// flow_mapping ::= FLOW-MAPPING-START +// (flow_mapping_entry FLOW-ENTRY)* +// flow_mapping_entry? +// FLOW-MAPPING-END +// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? + +// Peek the next token in the token queue. +func peek_token(parser *yaml_parser_t) *yaml_token_t { + if parser.token_available || yaml_parser_fetch_more_tokens(parser) { + token := &parser.tokens[parser.tokens_head] + yaml_parser_unfold_comments(parser, token) + return token + } + return nil +} + +// yaml_parser_unfold_comments walks through the comments queue and joins all +// comments behind the position of the provided token into the respective +// top-level comment slices in the parser. +func yaml_parser_unfold_comments(parser *yaml_parser_t, token *yaml_token_t) { + for parser.comments_head < len(parser.comments) && token.start_mark.index >= parser.comments[parser.comments_head].token_mark.index { + comment := &parser.comments[parser.comments_head] + if len(comment.head) > 0 { + if token.typ == yaml_BLOCK_END_TOKEN { + // No heads on ends, so keep comment.head for a follow up token. + break + } + if len(parser.head_comment) > 0 { + parser.head_comment = append(parser.head_comment, '\n') + } + parser.head_comment = append(parser.head_comment, comment.head...) + } + if len(comment.foot) > 0 { + if len(parser.foot_comment) > 0 { + parser.foot_comment = append(parser.foot_comment, '\n') + } + parser.foot_comment = append(parser.foot_comment, comment.foot...) + } + if len(comment.line) > 0 { + if len(parser.line_comment) > 0 { + parser.line_comment = append(parser.line_comment, '\n') + } + parser.line_comment = append(parser.line_comment, comment.line...) + } + *comment = yaml_comment_t{} + parser.comments_head++ + } +} + +// Remove the next token from the queue (must be called after peek_token). +func skip_token(parser *yaml_parser_t) { + parser.token_available = false + parser.tokens_parsed++ + parser.stream_end_produced = parser.tokens[parser.tokens_head].typ == yaml_STREAM_END_TOKEN + parser.tokens_head++ +} + +// Get the next event. +func yaml_parser_parse(parser *yaml_parser_t, event *yaml_event_t) bool { + // Erase the event object. + *event = yaml_event_t{} + + // No events after the end of the stream or error. + if parser.stream_end_produced || parser.error != yaml_NO_ERROR || parser.state == yaml_PARSE_END_STATE { + return true + } + + // Generate the next event. + return yaml_parser_state_machine(parser, event) +} + +// Set parser error. +func yaml_parser_set_parser_error(parser *yaml_parser_t, problem string, problem_mark yaml_mark_t) bool { + parser.error = yaml_PARSER_ERROR + parser.problem = problem + parser.problem_mark = problem_mark + return false +} + +func yaml_parser_set_parser_error_context(parser *yaml_parser_t, context string, context_mark yaml_mark_t, problem string, problem_mark yaml_mark_t) bool { + parser.error = yaml_PARSER_ERROR + parser.context = context + parser.context_mark = context_mark + parser.problem = problem + parser.problem_mark = problem_mark + return false +} + +// State dispatcher. +func yaml_parser_state_machine(parser *yaml_parser_t, event *yaml_event_t) bool { + //trace("yaml_parser_state_machine", "state:", parser.state.String()) + + switch parser.state { + case yaml_PARSE_STREAM_START_STATE: + return yaml_parser_parse_stream_start(parser, event) + + case yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE: + return yaml_parser_parse_document_start(parser, event, true) + + case yaml_PARSE_DOCUMENT_START_STATE: + return yaml_parser_parse_document_start(parser, event, false) + + case yaml_PARSE_DOCUMENT_CONTENT_STATE: + return yaml_parser_parse_document_content(parser, event) + + case yaml_PARSE_DOCUMENT_END_STATE: + return yaml_parser_parse_document_end(parser, event) + + case yaml_PARSE_BLOCK_NODE_STATE: + return yaml_parser_parse_node(parser, event, true, false) + + case yaml_PARSE_BLOCK_NODE_OR_INDENTLESS_SEQUENCE_STATE: + return yaml_parser_parse_node(parser, event, true, true) + + case yaml_PARSE_FLOW_NODE_STATE: + return yaml_parser_parse_node(parser, event, false, false) + + case yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE: + return yaml_parser_parse_block_sequence_entry(parser, event, true) + + case yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE: + return yaml_parser_parse_block_sequence_entry(parser, event, false) + + case yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE: + return yaml_parser_parse_indentless_sequence_entry(parser, event) + + case yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE: + return yaml_parser_parse_block_mapping_key(parser, event, true) + + case yaml_PARSE_BLOCK_MAPPING_KEY_STATE: + return yaml_parser_parse_block_mapping_key(parser, event, false) + + case yaml_PARSE_BLOCK_MAPPING_VALUE_STATE: + return yaml_parser_parse_block_mapping_value(parser, event) + + case yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE: + return yaml_parser_parse_flow_sequence_entry(parser, event, true) + + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE: + return yaml_parser_parse_flow_sequence_entry(parser, event, false) + + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE: + return yaml_parser_parse_flow_sequence_entry_mapping_key(parser, event) + + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE: + return yaml_parser_parse_flow_sequence_entry_mapping_value(parser, event) + + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE: + return yaml_parser_parse_flow_sequence_entry_mapping_end(parser, event) + + case yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE: + return yaml_parser_parse_flow_mapping_key(parser, event, true) + + case yaml_PARSE_FLOW_MAPPING_KEY_STATE: + return yaml_parser_parse_flow_mapping_key(parser, event, false) + + case yaml_PARSE_FLOW_MAPPING_VALUE_STATE: + return yaml_parser_parse_flow_mapping_value(parser, event, false) + + case yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE: + return yaml_parser_parse_flow_mapping_value(parser, event, true) + + default: + panic("invalid parser state") + } +} + +// Parse the production: +// stream ::= STREAM-START implicit_document? explicit_document* STREAM-END +// ************ +func yaml_parser_parse_stream_start(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_STREAM_START_TOKEN { + return yaml_parser_set_parser_error(parser, "did not find expected ", token.start_mark) + } + parser.state = yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE + *event = yaml_event_t{ + typ: yaml_STREAM_START_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + encoding: token.encoding, + } + skip_token(parser) + return true +} + +// Parse the productions: +// implicit_document ::= block_node DOCUMENT-END* +// * +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// ************************* +func yaml_parser_parse_document_start(parser *yaml_parser_t, event *yaml_event_t, implicit bool) bool { + + token := peek_token(parser) + if token == nil { + return false + } + + // Parse extra document end indicators. + if !implicit { + for token.typ == yaml_DOCUMENT_END_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } + } + + if implicit && token.typ != yaml_VERSION_DIRECTIVE_TOKEN && + token.typ != yaml_TAG_DIRECTIVE_TOKEN && + token.typ != yaml_DOCUMENT_START_TOKEN && + token.typ != yaml_STREAM_END_TOKEN { + // Parse an implicit document. + if !yaml_parser_process_directives(parser, nil, nil) { + return false + } + parser.states = append(parser.states, yaml_PARSE_DOCUMENT_END_STATE) + parser.state = yaml_PARSE_BLOCK_NODE_STATE + + var head_comment []byte + if len(parser.head_comment) > 0 { + // [Go] Scan the header comment backwards, and if an empty line is found, break + // the header so the part before the last empty line goes into the + // document header, while the bottom of it goes into a follow up event. + for i := len(parser.head_comment) - 1; i > 0; i-- { + if parser.head_comment[i] == '\n' { + if i == len(parser.head_comment)-1 { + head_comment = parser.head_comment[:i] + parser.head_comment = parser.head_comment[i+1:] + break + } else if parser.head_comment[i-1] == '\n' { + head_comment = parser.head_comment[:i-1] + parser.head_comment = parser.head_comment[i+1:] + break + } + } + } + } + + *event = yaml_event_t{ + typ: yaml_DOCUMENT_START_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + + head_comment: head_comment, + } + + } else if token.typ != yaml_STREAM_END_TOKEN { + // Parse an explicit document. + var version_directive *yaml_version_directive_t + var tag_directives []yaml_tag_directive_t + start_mark := token.start_mark + if !yaml_parser_process_directives(parser, &version_directive, &tag_directives) { + return false + } + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_DOCUMENT_START_TOKEN { + yaml_parser_set_parser_error(parser, + "did not find expected ", token.start_mark) + return false + } + parser.states = append(parser.states, yaml_PARSE_DOCUMENT_END_STATE) + parser.state = yaml_PARSE_DOCUMENT_CONTENT_STATE + end_mark := token.end_mark + + *event = yaml_event_t{ + typ: yaml_DOCUMENT_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + version_directive: version_directive, + tag_directives: tag_directives, + implicit: false, + } + skip_token(parser) + + } else { + // Parse the stream end. + parser.state = yaml_PARSE_END_STATE + *event = yaml_event_t{ + typ: yaml_STREAM_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + skip_token(parser) + } + + return true +} + +// Parse the productions: +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// *********** +// +func yaml_parser_parse_document_content(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ == yaml_VERSION_DIRECTIVE_TOKEN || + token.typ == yaml_TAG_DIRECTIVE_TOKEN || + token.typ == yaml_DOCUMENT_START_TOKEN || + token.typ == yaml_DOCUMENT_END_TOKEN || + token.typ == yaml_STREAM_END_TOKEN { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + return yaml_parser_process_empty_scalar(parser, event, + token.start_mark) + } + return yaml_parser_parse_node(parser, event, true, false) +} + +// Parse the productions: +// implicit_document ::= block_node DOCUMENT-END* +// ************* +// explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +// +func yaml_parser_parse_document_end(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + + start_mark := token.start_mark + end_mark := token.start_mark + + implicit := true + if token.typ == yaml_DOCUMENT_END_TOKEN { + end_mark = token.end_mark + skip_token(parser) + implicit = false + } + + parser.tag_directives = parser.tag_directives[:0] + + parser.state = yaml_PARSE_DOCUMENT_START_STATE + *event = yaml_event_t{ + typ: yaml_DOCUMENT_END_EVENT, + start_mark: start_mark, + end_mark: end_mark, + implicit: implicit, + } + yaml_parser_set_event_comments(parser, event) + if len(event.head_comment) > 0 && len(event.foot_comment) == 0 { + event.foot_comment = event.head_comment + event.head_comment = nil + } + return true +} + +func yaml_parser_set_event_comments(parser *yaml_parser_t, event *yaml_event_t) { + event.head_comment = parser.head_comment + event.line_comment = parser.line_comment + event.foot_comment = parser.foot_comment + parser.head_comment = nil + parser.line_comment = nil + parser.foot_comment = nil + parser.tail_comment = nil + parser.stem_comment = nil +} + +// Parse the productions: +// block_node_or_indentless_sequence ::= +// ALIAS +// ***** +// | properties (block_content | indentless_block_sequence)? +// ********** * +// | block_content | indentless_block_sequence +// * +// block_node ::= ALIAS +// ***** +// | properties block_content? +// ********** * +// | block_content +// * +// flow_node ::= ALIAS +// ***** +// | properties flow_content? +// ********** * +// | flow_content +// * +// properties ::= TAG ANCHOR? | ANCHOR TAG? +// ************************* +// block_content ::= block_collection | flow_collection | SCALAR +// ****** +// flow_content ::= flow_collection | SCALAR +// ****** +func yaml_parser_parse_node(parser *yaml_parser_t, event *yaml_event_t, block, indentless_sequence bool) bool { + //defer trace("yaml_parser_parse_node", "block:", block, "indentless_sequence:", indentless_sequence)() + + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ == yaml_ALIAS_TOKEN { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + *event = yaml_event_t{ + typ: yaml_ALIAS_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + anchor: token.value, + } + yaml_parser_set_event_comments(parser, event) + skip_token(parser) + return true + } + + start_mark := token.start_mark + end_mark := token.start_mark + + var tag_token bool + var tag_handle, tag_suffix, anchor []byte + var tag_mark yaml_mark_t + if token.typ == yaml_ANCHOR_TOKEN { + anchor = token.value + start_mark = token.start_mark + end_mark = token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_TAG_TOKEN { + tag_token = true + tag_handle = token.value + tag_suffix = token.suffix + tag_mark = token.start_mark + end_mark = token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } + } else if token.typ == yaml_TAG_TOKEN { + tag_token = true + tag_handle = token.value + tag_suffix = token.suffix + start_mark = token.start_mark + tag_mark = token.start_mark + end_mark = token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_ANCHOR_TOKEN { + anchor = token.value + end_mark = token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } + } + + var tag []byte + if tag_token { + if len(tag_handle) == 0 { + tag = tag_suffix + tag_suffix = nil + } else { + for i := range parser.tag_directives { + if bytes.Equal(parser.tag_directives[i].handle, tag_handle) { + tag = append([]byte(nil), parser.tag_directives[i].prefix...) + tag = append(tag, tag_suffix...) + break + } + } + if len(tag) == 0 { + yaml_parser_set_parser_error_context(parser, + "while parsing a node", start_mark, + "found undefined tag handle", tag_mark) + return false + } + } + } + + implicit := len(tag) == 0 + if indentless_sequence && token.typ == yaml_BLOCK_ENTRY_TOKEN { + end_mark = token.end_mark + parser.state = yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE + *event = yaml_event_t{ + typ: yaml_SEQUENCE_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_BLOCK_SEQUENCE_STYLE), + } + return true + } + if token.typ == yaml_SCALAR_TOKEN { + var plain_implicit, quoted_implicit bool + end_mark = token.end_mark + if (len(tag) == 0 && token.style == yaml_PLAIN_SCALAR_STYLE) || (len(tag) == 1 && tag[0] == '!') { + plain_implicit = true + } else if len(tag) == 0 { + quoted_implicit = true + } + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + + *event = yaml_event_t{ + typ: yaml_SCALAR_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + value: token.value, + implicit: plain_implicit, + quoted_implicit: quoted_implicit, + style: yaml_style_t(token.style), + } + yaml_parser_set_event_comments(parser, event) + skip_token(parser) + return true + } + if token.typ == yaml_FLOW_SEQUENCE_START_TOKEN { + // [Go] Some of the events below can be merged as they differ only on style. + end_mark = token.end_mark + parser.state = yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE + *event = yaml_event_t{ + typ: yaml_SEQUENCE_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_FLOW_SEQUENCE_STYLE), + } + yaml_parser_set_event_comments(parser, event) + return true + } + if token.typ == yaml_FLOW_MAPPING_START_TOKEN { + end_mark = token.end_mark + parser.state = yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE + *event = yaml_event_t{ + typ: yaml_MAPPING_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_FLOW_MAPPING_STYLE), + } + yaml_parser_set_event_comments(parser, event) + return true + } + if block && token.typ == yaml_BLOCK_SEQUENCE_START_TOKEN { + end_mark = token.end_mark + parser.state = yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE + *event = yaml_event_t{ + typ: yaml_SEQUENCE_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_BLOCK_SEQUENCE_STYLE), + } + if parser.stem_comment != nil { + event.head_comment = parser.stem_comment + parser.stem_comment = nil + } + return true + } + if block && token.typ == yaml_BLOCK_MAPPING_START_TOKEN { + end_mark = token.end_mark + parser.state = yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE + *event = yaml_event_t{ + typ: yaml_MAPPING_START_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + style: yaml_style_t(yaml_BLOCK_MAPPING_STYLE), + } + return true + } + if len(anchor) > 0 || len(tag) > 0 { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + + *event = yaml_event_t{ + typ: yaml_SCALAR_EVENT, + start_mark: start_mark, + end_mark: end_mark, + anchor: anchor, + tag: tag, + implicit: implicit, + quoted_implicit: false, + style: yaml_style_t(yaml_PLAIN_SCALAR_STYLE), + } + return true + } + + context := "while parsing a flow node" + if block { + context = "while parsing a block node" + } + yaml_parser_set_parser_error_context(parser, context, start_mark, + "did not find expected node content", token.start_mark) + return false +} + +// Parse the productions: +// block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END +// ******************** *********** * ********* +// +func yaml_parser_parse_block_sequence_entry(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { + if first { + token := peek_token(parser) + parser.marks = append(parser.marks, token.start_mark) + skip_token(parser) + } + + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ == yaml_BLOCK_ENTRY_TOKEN { + mark := token.end_mark + prior_head := len(parser.head_comment) + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if prior_head > 0 && token.typ == yaml_BLOCK_SEQUENCE_START_TOKEN { + // [Go] It's a sequence under a sequence entry, so the former head comment + // is for the list itself, not the first list item under it. + parser.stem_comment = parser.head_comment[:prior_head] + if len(parser.head_comment) == prior_head { + parser.head_comment = nil + } else { + // Copy suffix to prevent very strange bugs if someone ever appends + // further bytes to the prefix in the stem_comment slice above. + parser.head_comment = append([]byte(nil), parser.head_comment[prior_head+1:]...) + } + + } + if token.typ != yaml_BLOCK_ENTRY_TOKEN && token.typ != yaml_BLOCK_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE) + return yaml_parser_parse_node(parser, event, true, false) + } else { + parser.state = yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) + } + } + if token.typ == yaml_BLOCK_END_TOKEN { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + + *event = yaml_event_t{ + typ: yaml_SEQUENCE_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + + skip_token(parser) + return true + } + + context_mark := parser.marks[len(parser.marks)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + return yaml_parser_set_parser_error_context(parser, + "while parsing a block collection", context_mark, + "did not find expected '-' indicator", token.start_mark) +} + +// Parse the productions: +// indentless_sequence ::= (BLOCK-ENTRY block_node?)+ +// *********** * +func yaml_parser_parse_indentless_sequence_entry(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ == yaml_BLOCK_ENTRY_TOKEN { + mark := token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_BLOCK_ENTRY_TOKEN && + token.typ != yaml_KEY_TOKEN && + token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_BLOCK_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE) + return yaml_parser_parse_node(parser, event, true, false) + } + parser.state = yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) + } + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + + *event = yaml_event_t{ + typ: yaml_SEQUENCE_END_EVENT, + start_mark: token.start_mark, + end_mark: token.start_mark, // [Go] Shouldn't this be token.end_mark? + } + return true +} + +// Parse the productions: +// block_mapping ::= BLOCK-MAPPING_START +// ******************* +// ((KEY block_node_or_indentless_sequence?)? +// *** * +// (VALUE block_node_or_indentless_sequence?)?)* +// +// BLOCK-END +// ********* +// +func yaml_parser_parse_block_mapping_key(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { + if first { + token := peek_token(parser) + parser.marks = append(parser.marks, token.start_mark) + skip_token(parser) + } + + token := peek_token(parser) + if token == nil { + return false + } + + // [Go] A tail comment was left from the prior mapping value processed. Emit an event + // as it needs to be processed with that value and not the following key. + if len(parser.tail_comment) > 0 { + *event = yaml_event_t{ + typ: yaml_TAIL_COMMENT_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + foot_comment: parser.tail_comment, + } + parser.tail_comment = nil + return true + } + + if token.typ == yaml_KEY_TOKEN { + mark := token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_KEY_TOKEN && + token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_BLOCK_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_BLOCK_MAPPING_VALUE_STATE) + return yaml_parser_parse_node(parser, event, true, true) + } else { + parser.state = yaml_PARSE_BLOCK_MAPPING_VALUE_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) + } + } else if token.typ == yaml_BLOCK_END_TOKEN { + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + *event = yaml_event_t{ + typ: yaml_MAPPING_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + yaml_parser_set_event_comments(parser, event) + skip_token(parser) + return true + } + + context_mark := parser.marks[len(parser.marks)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + return yaml_parser_set_parser_error_context(parser, + "while parsing a block mapping", context_mark, + "did not find expected key", token.start_mark) +} + +// Parse the productions: +// block_mapping ::= BLOCK-MAPPING_START +// +// ((KEY block_node_or_indentless_sequence?)? +// +// (VALUE block_node_or_indentless_sequence?)?)* +// ***** * +// BLOCK-END +// +// +func yaml_parser_parse_block_mapping_value(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_VALUE_TOKEN { + mark := token.end_mark + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_KEY_TOKEN && + token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_BLOCK_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_BLOCK_MAPPING_KEY_STATE) + return yaml_parser_parse_node(parser, event, true, true) + } + parser.state = yaml_PARSE_BLOCK_MAPPING_KEY_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) + } + parser.state = yaml_PARSE_BLOCK_MAPPING_KEY_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) +} + +// Parse the productions: +// flow_sequence ::= FLOW-SEQUENCE-START +// ******************* +// (flow_sequence_entry FLOW-ENTRY)* +// * ********** +// flow_sequence_entry? +// * +// FLOW-SEQUENCE-END +// ***************** +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * +// +func yaml_parser_parse_flow_sequence_entry(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { + if first { + token := peek_token(parser) + parser.marks = append(parser.marks, token.start_mark) + skip_token(parser) + } + token := peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_FLOW_SEQUENCE_END_TOKEN { + if !first { + if token.typ == yaml_FLOW_ENTRY_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } else { + context_mark := parser.marks[len(parser.marks)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + return yaml_parser_set_parser_error_context(parser, + "while parsing a flow sequence", context_mark, + "did not find expected ',' or ']'", token.start_mark) + } + } + + if token.typ == yaml_KEY_TOKEN { + parser.state = yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE + *event = yaml_event_t{ + typ: yaml_MAPPING_START_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + implicit: true, + style: yaml_style_t(yaml_FLOW_MAPPING_STYLE), + } + skip_token(parser) + return true + } else if token.typ != yaml_FLOW_SEQUENCE_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + } + + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + + *event = yaml_event_t{ + typ: yaml_SEQUENCE_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + yaml_parser_set_event_comments(parser, event) + + skip_token(parser) + return true +} + +// +// Parse the productions: +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// *** * +// +func yaml_parser_parse_flow_sequence_entry_mapping_key(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_FLOW_ENTRY_TOKEN && + token.typ != yaml_FLOW_SEQUENCE_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + mark := token.end_mark + skip_token(parser) + parser.state = yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE + return yaml_parser_process_empty_scalar(parser, event, mark) +} + +// Parse the productions: +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// ***** * +// +func yaml_parser_parse_flow_sequence_entry_mapping_value(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + if token.typ == yaml_VALUE_TOKEN { + skip_token(parser) + token := peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_FLOW_ENTRY_TOKEN && token.typ != yaml_FLOW_SEQUENCE_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + } + parser.state = yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) +} + +// Parse the productions: +// flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * +// +func yaml_parser_parse_flow_sequence_entry_mapping_end(parser *yaml_parser_t, event *yaml_event_t) bool { + token := peek_token(parser) + if token == nil { + return false + } + parser.state = yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE + *event = yaml_event_t{ + typ: yaml_MAPPING_END_EVENT, + start_mark: token.start_mark, + end_mark: token.start_mark, // [Go] Shouldn't this be end_mark? + } + return true +} + +// Parse the productions: +// flow_mapping ::= FLOW-MAPPING-START +// ****************** +// (flow_mapping_entry FLOW-ENTRY)* +// * ********** +// flow_mapping_entry? +// ****************** +// FLOW-MAPPING-END +// **************** +// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * *** * +// +func yaml_parser_parse_flow_mapping_key(parser *yaml_parser_t, event *yaml_event_t, first bool) bool { + if first { + token := peek_token(parser) + parser.marks = append(parser.marks, token.start_mark) + skip_token(parser) + } + + token := peek_token(parser) + if token == nil { + return false + } + + if token.typ != yaml_FLOW_MAPPING_END_TOKEN { + if !first { + if token.typ == yaml_FLOW_ENTRY_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } else { + context_mark := parser.marks[len(parser.marks)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + return yaml_parser_set_parser_error_context(parser, + "while parsing a flow mapping", context_mark, + "did not find expected ',' or '}'", token.start_mark) + } + } + + if token.typ == yaml_KEY_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_VALUE_TOKEN && + token.typ != yaml_FLOW_ENTRY_TOKEN && + token.typ != yaml_FLOW_MAPPING_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_MAPPING_VALUE_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } else { + parser.state = yaml_PARSE_FLOW_MAPPING_VALUE_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) + } + } else if token.typ != yaml_FLOW_MAPPING_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + } + + parser.state = parser.states[len(parser.states)-1] + parser.states = parser.states[:len(parser.states)-1] + parser.marks = parser.marks[:len(parser.marks)-1] + *event = yaml_event_t{ + typ: yaml_MAPPING_END_EVENT, + start_mark: token.start_mark, + end_mark: token.end_mark, + } + yaml_parser_set_event_comments(parser, event) + skip_token(parser) + return true +} + +// Parse the productions: +// flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +// * ***** * +// +func yaml_parser_parse_flow_mapping_value(parser *yaml_parser_t, event *yaml_event_t, empty bool) bool { + token := peek_token(parser) + if token == nil { + return false + } + if empty { + parser.state = yaml_PARSE_FLOW_MAPPING_KEY_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) + } + if token.typ == yaml_VALUE_TOKEN { + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + if token.typ != yaml_FLOW_ENTRY_TOKEN && token.typ != yaml_FLOW_MAPPING_END_TOKEN { + parser.states = append(parser.states, yaml_PARSE_FLOW_MAPPING_KEY_STATE) + return yaml_parser_parse_node(parser, event, false, false) + } + } + parser.state = yaml_PARSE_FLOW_MAPPING_KEY_STATE + return yaml_parser_process_empty_scalar(parser, event, token.start_mark) +} + +// Generate an empty scalar event. +func yaml_parser_process_empty_scalar(parser *yaml_parser_t, event *yaml_event_t, mark yaml_mark_t) bool { + *event = yaml_event_t{ + typ: yaml_SCALAR_EVENT, + start_mark: mark, + end_mark: mark, + value: nil, // Empty + implicit: true, + style: yaml_style_t(yaml_PLAIN_SCALAR_STYLE), + } + return true +} + +var default_tag_directives = []yaml_tag_directive_t{ + {[]byte("!"), []byte("!")}, + {[]byte("!!"), []byte("tag:yaml.org,2002:")}, +} + +// Parse directives. +func yaml_parser_process_directives(parser *yaml_parser_t, + version_directive_ref **yaml_version_directive_t, + tag_directives_ref *[]yaml_tag_directive_t) bool { + + var version_directive *yaml_version_directive_t + var tag_directives []yaml_tag_directive_t + + token := peek_token(parser) + if token == nil { + return false + } + + for token.typ == yaml_VERSION_DIRECTIVE_TOKEN || token.typ == yaml_TAG_DIRECTIVE_TOKEN { + if token.typ == yaml_VERSION_DIRECTIVE_TOKEN { + if version_directive != nil { + yaml_parser_set_parser_error(parser, + "found duplicate %YAML directive", token.start_mark) + return false + } + if token.major != 1 || token.minor != 1 { + yaml_parser_set_parser_error(parser, + "found incompatible YAML document", token.start_mark) + return false + } + version_directive = &yaml_version_directive_t{ + major: token.major, + minor: token.minor, + } + } else if token.typ == yaml_TAG_DIRECTIVE_TOKEN { + value := yaml_tag_directive_t{ + handle: token.value, + prefix: token.prefix, + } + if !yaml_parser_append_tag_directive(parser, value, false, token.start_mark) { + return false + } + tag_directives = append(tag_directives, value) + } + + skip_token(parser) + token = peek_token(parser) + if token == nil { + return false + } + } + + for i := range default_tag_directives { + if !yaml_parser_append_tag_directive(parser, default_tag_directives[i], true, token.start_mark) { + return false + } + } + + if version_directive_ref != nil { + *version_directive_ref = version_directive + } + if tag_directives_ref != nil { + *tag_directives_ref = tag_directives + } + return true +} + +// Append a tag directive to the directives stack. +func yaml_parser_append_tag_directive(parser *yaml_parser_t, value yaml_tag_directive_t, allow_duplicates bool, mark yaml_mark_t) bool { + for i := range parser.tag_directives { + if bytes.Equal(value.handle, parser.tag_directives[i].handle) { + if allow_duplicates { + return true + } + return yaml_parser_set_parser_error(parser, "found duplicate %TAG directive", mark) + } + } + + // [Go] I suspect the copy is unnecessary. This was likely done + // because there was no way to track ownership of the data. + value_copy := yaml_tag_directive_t{ + handle: make([]byte, len(value.handle)), + prefix: make([]byte, len(value.prefix)), + } + copy(value_copy.handle, value.handle) + copy(value_copy.prefix, value.prefix) + parser.tag_directives = append(parser.tag_directives, value_copy) + return true +} diff --git a/gopkg.in/yaml.v3/readerc.go b/gopkg.in/yaml.v3/readerc.go new file mode 100644 index 0000000000..b7de0a89c4 --- /dev/null +++ b/gopkg.in/yaml.v3/readerc.go @@ -0,0 +1,434 @@ +// +// Copyright (c) 2011-2019 Canonical Ltd +// Copyright (c) 2006-2010 Kirill Simonov +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +// of the Software, and to permit persons to whom the Software is furnished to do +// so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package yaml + +import ( + "io" +) + +// Set the reader error and return 0. +func yaml_parser_set_reader_error(parser *yaml_parser_t, problem string, offset int, value int) bool { + parser.error = yaml_READER_ERROR + parser.problem = problem + parser.problem_offset = offset + parser.problem_value = value + return false +} + +// Byte order marks. +const ( + bom_UTF8 = "\xef\xbb\xbf" + bom_UTF16LE = "\xff\xfe" + bom_UTF16BE = "\xfe\xff" +) + +// Determine the input stream encoding by checking the BOM symbol. If no BOM is +// found, the UTF-8 encoding is assumed. Return 1 on success, 0 on failure. +func yaml_parser_determine_encoding(parser *yaml_parser_t) bool { + // Ensure that we had enough bytes in the raw buffer. + for !parser.eof && len(parser.raw_buffer)-parser.raw_buffer_pos < 3 { + if !yaml_parser_update_raw_buffer(parser) { + return false + } + } + + // Determine the encoding. + buf := parser.raw_buffer + pos := parser.raw_buffer_pos + avail := len(buf) - pos + if avail >= 2 && buf[pos] == bom_UTF16LE[0] && buf[pos+1] == bom_UTF16LE[1] { + parser.encoding = yaml_UTF16LE_ENCODING + parser.raw_buffer_pos += 2 + parser.offset += 2 + } else if avail >= 2 && buf[pos] == bom_UTF16BE[0] && buf[pos+1] == bom_UTF16BE[1] { + parser.encoding = yaml_UTF16BE_ENCODING + parser.raw_buffer_pos += 2 + parser.offset += 2 + } else if avail >= 3 && buf[pos] == bom_UTF8[0] && buf[pos+1] == bom_UTF8[1] && buf[pos+2] == bom_UTF8[2] { + parser.encoding = yaml_UTF8_ENCODING + parser.raw_buffer_pos += 3 + parser.offset += 3 + } else { + parser.encoding = yaml_UTF8_ENCODING + } + return true +} + +// Update the raw buffer. +func yaml_parser_update_raw_buffer(parser *yaml_parser_t) bool { + size_read := 0 + + // Return if the raw buffer is full. + if parser.raw_buffer_pos == 0 && len(parser.raw_buffer) == cap(parser.raw_buffer) { + return true + } + + // Return on EOF. + if parser.eof { + return true + } + + // Move the remaining bytes in the raw buffer to the beginning. + if parser.raw_buffer_pos > 0 && parser.raw_buffer_pos < len(parser.raw_buffer) { + copy(parser.raw_buffer, parser.raw_buffer[parser.raw_buffer_pos:]) + } + parser.raw_buffer = parser.raw_buffer[:len(parser.raw_buffer)-parser.raw_buffer_pos] + parser.raw_buffer_pos = 0 + + // Call the read handler to fill the buffer. + size_read, err := parser.read_handler(parser, parser.raw_buffer[len(parser.raw_buffer):cap(parser.raw_buffer)]) + parser.raw_buffer = parser.raw_buffer[:len(parser.raw_buffer)+size_read] + if err == io.EOF { + parser.eof = true + } else if err != nil { + return yaml_parser_set_reader_error(parser, "input error: "+err.Error(), parser.offset, -1) + } + return true +} + +// Ensure that the buffer contains at least `length` characters. +// Return true on success, false on failure. +// +// The length is supposed to be significantly less that the buffer size. +func yaml_parser_update_buffer(parser *yaml_parser_t, length int) bool { + if parser.read_handler == nil { + panic("read handler must be set") + } + + // [Go] This function was changed to guarantee the requested length size at EOF. + // The fact we need to do this is pretty awful, but the description above implies + // for that to be the case, and there are tests + + // If the EOF flag is set and the raw buffer is empty, do nothing. + if parser.eof && parser.raw_buffer_pos == len(parser.raw_buffer) { + // [Go] ACTUALLY! Read the documentation of this function above. + // This is just broken. To return true, we need to have the + // given length in the buffer. Not doing that means every single + // check that calls this function to make sure the buffer has a + // given length is Go) panicking; or C) accessing invalid memory. + //return true + } + + // Return if the buffer contains enough characters. + if parser.unread >= length { + return true + } + + // Determine the input encoding if it is not known yet. + if parser.encoding == yaml_ANY_ENCODING { + if !yaml_parser_determine_encoding(parser) { + return false + } + } + + // Move the unread characters to the beginning of the buffer. + buffer_len := len(parser.buffer) + if parser.buffer_pos > 0 && parser.buffer_pos < buffer_len { + copy(parser.buffer, parser.buffer[parser.buffer_pos:]) + buffer_len -= parser.buffer_pos + parser.buffer_pos = 0 + } else if parser.buffer_pos == buffer_len { + buffer_len = 0 + parser.buffer_pos = 0 + } + + // Open the whole buffer for writing, and cut it before returning. + parser.buffer = parser.buffer[:cap(parser.buffer)] + + // Fill the buffer until it has enough characters. + first := true + for parser.unread < length { + + // Fill the raw buffer if necessary. + if !first || parser.raw_buffer_pos == len(parser.raw_buffer) { + if !yaml_parser_update_raw_buffer(parser) { + parser.buffer = parser.buffer[:buffer_len] + return false + } + } + first = false + + // Decode the raw buffer. + inner: + for parser.raw_buffer_pos != len(parser.raw_buffer) { + var value rune + var width int + + raw_unread := len(parser.raw_buffer) - parser.raw_buffer_pos + + // Decode the next character. + switch parser.encoding { + case yaml_UTF8_ENCODING: + // Decode a UTF-8 character. Check RFC 3629 + // (http://www.ietf.org/rfc/rfc3629.txt) for more details. + // + // The following table (taken from the RFC) is used for + // decoding. + // + // Char. number range | UTF-8 octet sequence + // (hexadecimal) | (binary) + // --------------------+------------------------------------ + // 0000 0000-0000 007F | 0xxxxxxx + // 0000 0080-0000 07FF | 110xxxxx 10xxxxxx + // 0000 0800-0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx + // 0001 0000-0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + // + // Additionally, the characters in the range 0xD800-0xDFFF + // are prohibited as they are reserved for use with UTF-16 + // surrogate pairs. + + // Determine the length of the UTF-8 sequence. + octet := parser.raw_buffer[parser.raw_buffer_pos] + switch { + case octet&0x80 == 0x00: + width = 1 + case octet&0xE0 == 0xC0: + width = 2 + case octet&0xF0 == 0xE0: + width = 3 + case octet&0xF8 == 0xF0: + width = 4 + default: + // The leading octet is invalid. + return yaml_parser_set_reader_error(parser, + "invalid leading UTF-8 octet", + parser.offset, int(octet)) + } + + // Check if the raw buffer contains an incomplete character. + if width > raw_unread { + if parser.eof { + return yaml_parser_set_reader_error(parser, + "incomplete UTF-8 octet sequence", + parser.offset, -1) + } + break inner + } + + // Decode the leading octet. + switch { + case octet&0x80 == 0x00: + value = rune(octet & 0x7F) + case octet&0xE0 == 0xC0: + value = rune(octet & 0x1F) + case octet&0xF0 == 0xE0: + value = rune(octet & 0x0F) + case octet&0xF8 == 0xF0: + value = rune(octet & 0x07) + default: + value = 0 + } + + // Check and decode the trailing octets. + for k := 1; k < width; k++ { + octet = parser.raw_buffer[parser.raw_buffer_pos+k] + + // Check if the octet is valid. + if (octet & 0xC0) != 0x80 { + return yaml_parser_set_reader_error(parser, + "invalid trailing UTF-8 octet", + parser.offset+k, int(octet)) + } + + // Decode the octet. + value = (value << 6) + rune(octet&0x3F) + } + + // Check the length of the sequence against the value. + switch { + case width == 1: + case width == 2 && value >= 0x80: + case width == 3 && value >= 0x800: + case width == 4 && value >= 0x10000: + default: + return yaml_parser_set_reader_error(parser, + "invalid length of a UTF-8 sequence", + parser.offset, -1) + } + + // Check the range of the value. + if value >= 0xD800 && value <= 0xDFFF || value > 0x10FFFF { + return yaml_parser_set_reader_error(parser, + "invalid Unicode character", + parser.offset, int(value)) + } + + case yaml_UTF16LE_ENCODING, yaml_UTF16BE_ENCODING: + var low, high int + if parser.encoding == yaml_UTF16LE_ENCODING { + low, high = 0, 1 + } else { + low, high = 1, 0 + } + + // The UTF-16 encoding is not as simple as one might + // naively think. Check RFC 2781 + // (http://www.ietf.org/rfc/rfc2781.txt). + // + // Normally, two subsequent bytes describe a Unicode + // character. However a special technique (called a + // surrogate pair) is used for specifying character + // values larger than 0xFFFF. + // + // A surrogate pair consists of two pseudo-characters: + // high surrogate area (0xD800-0xDBFF) + // low surrogate area (0xDC00-0xDFFF) + // + // The following formulas are used for decoding + // and encoding characters using surrogate pairs: + // + // U = U' + 0x10000 (0x01 00 00 <= U <= 0x10 FF FF) + // U' = yyyyyyyyyyxxxxxxxxxx (0 <= U' <= 0x0F FF FF) + // W1 = 110110yyyyyyyyyy + // W2 = 110111xxxxxxxxxx + // + // where U is the character value, W1 is the high surrogate + // area, W2 is the low surrogate area. + + // Check for incomplete UTF-16 character. + if raw_unread < 2 { + if parser.eof { + return yaml_parser_set_reader_error(parser, + "incomplete UTF-16 character", + parser.offset, -1) + } + break inner + } + + // Get the character. + value = rune(parser.raw_buffer[parser.raw_buffer_pos+low]) + + (rune(parser.raw_buffer[parser.raw_buffer_pos+high]) << 8) + + // Check for unexpected low surrogate area. + if value&0xFC00 == 0xDC00 { + return yaml_parser_set_reader_error(parser, + "unexpected low surrogate area", + parser.offset, int(value)) + } + + // Check for a high surrogate area. + if value&0xFC00 == 0xD800 { + width = 4 + + // Check for incomplete surrogate pair. + if raw_unread < 4 { + if parser.eof { + return yaml_parser_set_reader_error(parser, + "incomplete UTF-16 surrogate pair", + parser.offset, -1) + } + break inner + } + + // Get the next character. + value2 := rune(parser.raw_buffer[parser.raw_buffer_pos+low+2]) + + (rune(parser.raw_buffer[parser.raw_buffer_pos+high+2]) << 8) + + // Check for a low surrogate area. + if value2&0xFC00 != 0xDC00 { + return yaml_parser_set_reader_error(parser, + "expected low surrogate area", + parser.offset+2, int(value2)) + } + + // Generate the value of the surrogate pair. + value = 0x10000 + ((value & 0x3FF) << 10) + (value2 & 0x3FF) + } else { + width = 2 + } + + default: + panic("impossible") + } + + // Check if the character is in the allowed range: + // #x9 | #xA | #xD | [#x20-#x7E] (8 bit) + // | #x85 | [#xA0-#xD7FF] | [#xE000-#xFFFD] (16 bit) + // | [#x10000-#x10FFFF] (32 bit) + switch { + case value == 0x09: + case value == 0x0A: + case value == 0x0D: + case value >= 0x20 && value <= 0x7E: + case value == 0x85: + case value >= 0xA0 && value <= 0xD7FF: + case value >= 0xE000 && value <= 0xFFFD: + case value >= 0x10000 && value <= 0x10FFFF: + default: + return yaml_parser_set_reader_error(parser, + "control characters are not allowed", + parser.offset, int(value)) + } + + // Move the raw pointers. + parser.raw_buffer_pos += width + parser.offset += width + + // Finally put the character into the buffer. + if value <= 0x7F { + // 0000 0000-0000 007F . 0xxxxxxx + parser.buffer[buffer_len+0] = byte(value) + buffer_len += 1 + } else if value <= 0x7FF { + // 0000 0080-0000 07FF . 110xxxxx 10xxxxxx + parser.buffer[buffer_len+0] = byte(0xC0 + (value >> 6)) + parser.buffer[buffer_len+1] = byte(0x80 + (value & 0x3F)) + buffer_len += 2 + } else if value <= 0xFFFF { + // 0000 0800-0000 FFFF . 1110xxxx 10xxxxxx 10xxxxxx + parser.buffer[buffer_len+0] = byte(0xE0 + (value >> 12)) + parser.buffer[buffer_len+1] = byte(0x80 + ((value >> 6) & 0x3F)) + parser.buffer[buffer_len+2] = byte(0x80 + (value & 0x3F)) + buffer_len += 3 + } else { + // 0001 0000-0010 FFFF . 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + parser.buffer[buffer_len+0] = byte(0xF0 + (value >> 18)) + parser.buffer[buffer_len+1] = byte(0x80 + ((value >> 12) & 0x3F)) + parser.buffer[buffer_len+2] = byte(0x80 + ((value >> 6) & 0x3F)) + parser.buffer[buffer_len+3] = byte(0x80 + (value & 0x3F)) + buffer_len += 4 + } + + parser.unread++ + } + + // On EOF, put NUL into the buffer and return. + if parser.eof { + parser.buffer[buffer_len] = 0 + buffer_len++ + parser.unread++ + break + } + } + // [Go] Read the documentation of this function above. To return true, + // we need to have the given length in the buffer. Not doing that means + // every single check that calls this function to make sure the buffer + // has a given length is Go) panicking; or C) accessing invalid memory. + // This happens here due to the EOF above breaking early. + for buffer_len < length { + parser.buffer[buffer_len] = 0 + buffer_len++ + } + parser.buffer = parser.buffer[:buffer_len] + return true +} diff --git a/gopkg.in/yaml.v3/resolve.go b/gopkg.in/yaml.v3/resolve.go new file mode 100644 index 0000000000..64ae888057 --- /dev/null +++ b/gopkg.in/yaml.v3/resolve.go @@ -0,0 +1,326 @@ +// +// Copyright (c) 2011-2019 Canonical Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package yaml + +import ( + "encoding/base64" + "math" + "regexp" + "strconv" + "strings" + "time" +) + +type resolveMapItem struct { + value interface{} + tag string +} + +var resolveTable = make([]byte, 256) +var resolveMap = make(map[string]resolveMapItem) + +func init() { + t := resolveTable + t[int('+')] = 'S' // Sign + t[int('-')] = 'S' + for _, c := range "0123456789" { + t[int(c)] = 'D' // Digit + } + for _, c := range "yYnNtTfFoO~" { + t[int(c)] = 'M' // In map + } + t[int('.')] = '.' // Float (potentially in map) + + var resolveMapList = []struct { + v interface{} + tag string + l []string + }{ + {true, boolTag, []string{"true", "True", "TRUE"}}, + {false, boolTag, []string{"false", "False", "FALSE"}}, + {nil, nullTag, []string{"", "~", "null", "Null", "NULL"}}, + {math.NaN(), floatTag, []string{".nan", ".NaN", ".NAN"}}, + {math.Inf(+1), floatTag, []string{".inf", ".Inf", ".INF"}}, + {math.Inf(+1), floatTag, []string{"+.inf", "+.Inf", "+.INF"}}, + {math.Inf(-1), floatTag, []string{"-.inf", "-.Inf", "-.INF"}}, + {"<<", mergeTag, []string{"<<"}}, + } + + m := resolveMap + for _, item := range resolveMapList { + for _, s := range item.l { + m[s] = resolveMapItem{item.v, item.tag} + } + } +} + +const ( + nullTag = "!!null" + boolTag = "!!bool" + strTag = "!!str" + intTag = "!!int" + floatTag = "!!float" + timestampTag = "!!timestamp" + seqTag = "!!seq" + mapTag = "!!map" + binaryTag = "!!binary" + mergeTag = "!!merge" +) + +var longTags = make(map[string]string) +var shortTags = make(map[string]string) + +func init() { + for _, stag := range []string{nullTag, boolTag, strTag, intTag, floatTag, timestampTag, seqTag, mapTag, binaryTag, mergeTag} { + ltag := longTag(stag) + longTags[stag] = ltag + shortTags[ltag] = stag + } +} + +const longTagPrefix = "tag:yaml.org,2002:" + +func shortTag(tag string) string { + if strings.HasPrefix(tag, longTagPrefix) { + if stag, ok := shortTags[tag]; ok { + return stag + } + return "!!" + tag[len(longTagPrefix):] + } + return tag +} + +func longTag(tag string) string { + if strings.HasPrefix(tag, "!!") { + if ltag, ok := longTags[tag]; ok { + return ltag + } + return longTagPrefix + tag[2:] + } + return tag +} + +func resolvableTag(tag string) bool { + switch tag { + case "", strTag, boolTag, intTag, floatTag, nullTag, timestampTag: + return true + } + return false +} + +var yamlStyleFloat = regexp.MustCompile(`^[-+]?(\.[0-9]+|[0-9]+(\.[0-9]*)?)([eE][-+]?[0-9]+)?$`) + +func resolve(tag string, in string) (rtag string, out interface{}) { + tag = shortTag(tag) + if !resolvableTag(tag) { + return tag, in + } + + defer func() { + switch tag { + case "", rtag, strTag, binaryTag: + return + case floatTag: + if rtag == intTag { + switch v := out.(type) { + case int64: + rtag = floatTag + out = float64(v) + return + case int: + rtag = floatTag + out = float64(v) + return + } + } + } + failf("cannot decode %s `%s` as a %s", shortTag(rtag), in, shortTag(tag)) + }() + + // Any data is accepted as a !!str or !!binary. + // Otherwise, the prefix is enough of a hint about what it might be. + hint := byte('N') + if in != "" { + hint = resolveTable[in[0]] + } + if hint != 0 && tag != strTag && tag != binaryTag { + // Handle things we can lookup in a map. + if item, ok := resolveMap[in]; ok { + return item.tag, item.value + } + + // Base 60 floats are a bad idea, were dropped in YAML 1.2, and + // are purposefully unsupported here. They're still quoted on + // the way out for compatibility with other parser, though. + + switch hint { + case 'M': + // We've already checked the map above. + + case '.': + // Not in the map, so maybe a normal float. + floatv, err := strconv.ParseFloat(in, 64) + if err == nil { + return floatTag, floatv + } + + case 'D', 'S': + // Int, float, or timestamp. + // Only try values as a timestamp if the value is unquoted or there's an explicit + // !!timestamp tag. + if tag == "" || tag == timestampTag { + t, ok := parseTimestamp(in) + if ok { + return timestampTag, t + } + } + + plain := strings.Replace(in, "_", "", -1) + intv, err := strconv.ParseInt(plain, 0, 64) + if err == nil { + if intv == int64(int(intv)) { + return intTag, int(intv) + } else { + return intTag, intv + } + } + uintv, err := strconv.ParseUint(plain, 0, 64) + if err == nil { + return intTag, uintv + } + if yamlStyleFloat.MatchString(plain) { + floatv, err := strconv.ParseFloat(plain, 64) + if err == nil { + return floatTag, floatv + } + } + if strings.HasPrefix(plain, "0b") { + intv, err := strconv.ParseInt(plain[2:], 2, 64) + if err == nil { + if intv == int64(int(intv)) { + return intTag, int(intv) + } else { + return intTag, intv + } + } + uintv, err := strconv.ParseUint(plain[2:], 2, 64) + if err == nil { + return intTag, uintv + } + } else if strings.HasPrefix(plain, "-0b") { + intv, err := strconv.ParseInt("-"+plain[3:], 2, 64) + if err == nil { + if true || intv == int64(int(intv)) { + return intTag, int(intv) + } else { + return intTag, intv + } + } + } + // Octals as introduced in version 1.2 of the spec. + // Octals from the 1.1 spec, spelled as 0777, are still + // decoded by default in v3 as well for compatibility. + // May be dropped in v4 depending on how usage evolves. + if strings.HasPrefix(plain, "0o") { + intv, err := strconv.ParseInt(plain[2:], 8, 64) + if err == nil { + if intv == int64(int(intv)) { + return intTag, int(intv) + } else { + return intTag, intv + } + } + uintv, err := strconv.ParseUint(plain[2:], 8, 64) + if err == nil { + return intTag, uintv + } + } else if strings.HasPrefix(plain, "-0o") { + intv, err := strconv.ParseInt("-"+plain[3:], 8, 64) + if err == nil { + if true || intv == int64(int(intv)) { + return intTag, int(intv) + } else { + return intTag, intv + } + } + } + default: + panic("internal error: missing handler for resolver table: " + string(rune(hint)) + " (with " + in + ")") + } + } + return strTag, in +} + +// encodeBase64 encodes s as base64 that is broken up into multiple lines +// as appropriate for the resulting length. +func encodeBase64(s string) string { + const lineLen = 70 + encLen := base64.StdEncoding.EncodedLen(len(s)) + lines := encLen/lineLen + 1 + buf := make([]byte, encLen*2+lines) + in := buf[0:encLen] + out := buf[encLen:] + base64.StdEncoding.Encode(in, []byte(s)) + k := 0 + for i := 0; i < len(in); i += lineLen { + j := i + lineLen + if j > len(in) { + j = len(in) + } + k += copy(out[k:], in[i:j]) + if lines > 1 { + out[k] = '\n' + k++ + } + } + return string(out[:k]) +} + +// This is a subset of the formats allowed by the regular expression +// defined at http://yaml.org/type/timestamp.html. +var allowedTimestampFormats = []string{ + "2006-1-2T15:4:5.999999999Z07:00", // RCF3339Nano with short date fields. + "2006-1-2t15:4:5.999999999Z07:00", // RFC3339Nano with short date fields and lower-case "t". + "2006-1-2 15:4:5.999999999", // space separated with no time zone + "2006-1-2", // date only + // Notable exception: time.Parse cannot handle: "2001-12-14 21:59:43.10 -5" + // from the set of examples. +} + +// parseTimestamp parses s as a timestamp string and +// returns the timestamp and reports whether it succeeded. +// Timestamp formats are defined at http://yaml.org/type/timestamp.html +func parseTimestamp(s string) (time.Time, bool) { + // TODO write code to check all the formats supported by + // http://yaml.org/type/timestamp.html instead of using time.Parse. + + // Quick check: all date formats start with YYYY-. + i := 0 + for ; i < len(s); i++ { + if c := s[i]; c < '0' || c > '9' { + break + } + } + if i != 4 || i == len(s) || s[i] != '-' { + return time.Time{}, false + } + for _, format := range allowedTimestampFormats { + if t, err := time.Parse(format, s); err == nil { + return t, true + } + } + return time.Time{}, false +} diff --git a/gopkg.in/yaml.v3/scannerc.go b/gopkg.in/yaml.v3/scannerc.go new file mode 100644 index 0000000000..57e954ca53 --- /dev/null +++ b/gopkg.in/yaml.v3/scannerc.go @@ -0,0 +1,3025 @@ +// +// Copyright (c) 2011-2019 Canonical Ltd +// Copyright (c) 2006-2010 Kirill Simonov +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +// of the Software, and to permit persons to whom the Software is furnished to do +// so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package yaml + +import ( + "bytes" + "fmt" +) + +// Introduction +// ************ +// +// The following notes assume that you are familiar with the YAML specification +// (http://yaml.org/spec/1.2/spec.html). We mostly follow it, although in +// some cases we are less restrictive that it requires. +// +// The process of transforming a YAML stream into a sequence of events is +// divided on two steps: Scanning and Parsing. +// +// The Scanner transforms the input stream into a sequence of tokens, while the +// parser transform the sequence of tokens produced by the Scanner into a +// sequence of parsing events. +// +// The Scanner is rather clever and complicated. The Parser, on the contrary, +// is a straightforward implementation of a recursive-descendant parser (or, +// LL(1) parser, as it is usually called). +// +// Actually there are two issues of Scanning that might be called "clever", the +// rest is quite straightforward. The issues are "block collection start" and +// "simple keys". Both issues are explained below in details. +// +// Here the Scanning step is explained and implemented. We start with the list +// of all the tokens produced by the Scanner together with short descriptions. +// +// Now, tokens: +// +// STREAM-START(encoding) # The stream start. +// STREAM-END # The stream end. +// VERSION-DIRECTIVE(major,minor) # The '%YAML' directive. +// TAG-DIRECTIVE(handle,prefix) # The '%TAG' directive. +// DOCUMENT-START # '---' +// DOCUMENT-END # '...' +// BLOCK-SEQUENCE-START # Indentation increase denoting a block +// BLOCK-MAPPING-START # sequence or a block mapping. +// BLOCK-END # Indentation decrease. +// FLOW-SEQUENCE-START # '[' +// FLOW-SEQUENCE-END # ']' +// BLOCK-SEQUENCE-START # '{' +// BLOCK-SEQUENCE-END # '}' +// BLOCK-ENTRY # '-' +// FLOW-ENTRY # ',' +// KEY # '?' or nothing (simple keys). +// VALUE # ':' +// ALIAS(anchor) # '*anchor' +// ANCHOR(anchor) # '&anchor' +// TAG(handle,suffix) # '!handle!suffix' +// SCALAR(value,style) # A scalar. +// +// The following two tokens are "virtual" tokens denoting the beginning and the +// end of the stream: +// +// STREAM-START(encoding) +// STREAM-END +// +// We pass the information about the input stream encoding with the +// STREAM-START token. +// +// The next two tokens are responsible for tags: +// +// VERSION-DIRECTIVE(major,minor) +// TAG-DIRECTIVE(handle,prefix) +// +// Example: +// +// %YAML 1.1 +// %TAG ! !foo +// %TAG !yaml! tag:yaml.org,2002: +// --- +// +// The correspoding sequence of tokens: +// +// STREAM-START(utf-8) +// VERSION-DIRECTIVE(1,1) +// TAG-DIRECTIVE("!","!foo") +// TAG-DIRECTIVE("!yaml","tag:yaml.org,2002:") +// DOCUMENT-START +// STREAM-END +// +// Note that the VERSION-DIRECTIVE and TAG-DIRECTIVE tokens occupy a whole +// line. +// +// The document start and end indicators are represented by: +// +// DOCUMENT-START +// DOCUMENT-END +// +// Note that if a YAML stream contains an implicit document (without '---' +// and '...' indicators), no DOCUMENT-START and DOCUMENT-END tokens will be +// produced. +// +// In the following examples, we present whole documents together with the +// produced tokens. +// +// 1. An implicit document: +// +// 'a scalar' +// +// Tokens: +// +// STREAM-START(utf-8) +// SCALAR("a scalar",single-quoted) +// STREAM-END +// +// 2. An explicit document: +// +// --- +// 'a scalar' +// ... +// +// Tokens: +// +// STREAM-START(utf-8) +// DOCUMENT-START +// SCALAR("a scalar",single-quoted) +// DOCUMENT-END +// STREAM-END +// +// 3. Several documents in a stream: +// +// 'a scalar' +// --- +// 'another scalar' +// --- +// 'yet another scalar' +// +// Tokens: +// +// STREAM-START(utf-8) +// SCALAR("a scalar",single-quoted) +// DOCUMENT-START +// SCALAR("another scalar",single-quoted) +// DOCUMENT-START +// SCALAR("yet another scalar",single-quoted) +// STREAM-END +// +// We have already introduced the SCALAR token above. The following tokens are +// used to describe aliases, anchors, tag, and scalars: +// +// ALIAS(anchor) +// ANCHOR(anchor) +// TAG(handle,suffix) +// SCALAR(value,style) +// +// The following series of examples illustrate the usage of these tokens: +// +// 1. A recursive sequence: +// +// &A [ *A ] +// +// Tokens: +// +// STREAM-START(utf-8) +// ANCHOR("A") +// FLOW-SEQUENCE-START +// ALIAS("A") +// FLOW-SEQUENCE-END +// STREAM-END +// +// 2. A tagged scalar: +// +// !!float "3.14" # A good approximation. +// +// Tokens: +// +// STREAM-START(utf-8) +// TAG("!!","float") +// SCALAR("3.14",double-quoted) +// STREAM-END +// +// 3. Various scalar styles: +// +// --- # Implicit empty plain scalars do not produce tokens. +// --- a plain scalar +// --- 'a single-quoted scalar' +// --- "a double-quoted scalar" +// --- |- +// a literal scalar +// --- >- +// a folded +// scalar +// +// Tokens: +// +// STREAM-START(utf-8) +// DOCUMENT-START +// DOCUMENT-START +// SCALAR("a plain scalar",plain) +// DOCUMENT-START +// SCALAR("a single-quoted scalar",single-quoted) +// DOCUMENT-START +// SCALAR("a double-quoted scalar",double-quoted) +// DOCUMENT-START +// SCALAR("a literal scalar",literal) +// DOCUMENT-START +// SCALAR("a folded scalar",folded) +// STREAM-END +// +// Now it's time to review collection-related tokens. We will start with +// flow collections: +// +// FLOW-SEQUENCE-START +// FLOW-SEQUENCE-END +// FLOW-MAPPING-START +// FLOW-MAPPING-END +// FLOW-ENTRY +// KEY +// VALUE +// +// The tokens FLOW-SEQUENCE-START, FLOW-SEQUENCE-END, FLOW-MAPPING-START, and +// FLOW-MAPPING-END represent the indicators '[', ']', '{', and '}' +// correspondingly. FLOW-ENTRY represent the ',' indicator. Finally the +// indicators '?' and ':', which are used for denoting mapping keys and values, +// are represented by the KEY and VALUE tokens. +// +// The following examples show flow collections: +// +// 1. A flow sequence: +// +// [item 1, item 2, item 3] +// +// Tokens: +// +// STREAM-START(utf-8) +// FLOW-SEQUENCE-START +// SCALAR("item 1",plain) +// FLOW-ENTRY +// SCALAR("item 2",plain) +// FLOW-ENTRY +// SCALAR("item 3",plain) +// FLOW-SEQUENCE-END +// STREAM-END +// +// 2. A flow mapping: +// +// { +// a simple key: a value, # Note that the KEY token is produced. +// ? a complex key: another value, +// } +// +// Tokens: +// +// STREAM-START(utf-8) +// FLOW-MAPPING-START +// KEY +// SCALAR("a simple key",plain) +// VALUE +// SCALAR("a value",plain) +// FLOW-ENTRY +// KEY +// SCALAR("a complex key",plain) +// VALUE +// SCALAR("another value",plain) +// FLOW-ENTRY +// FLOW-MAPPING-END +// STREAM-END +// +// A simple key is a key which is not denoted by the '?' indicator. Note that +// the Scanner still produce the KEY token whenever it encounters a simple key. +// +// For scanning block collections, the following tokens are used (note that we +// repeat KEY and VALUE here): +// +// BLOCK-SEQUENCE-START +// BLOCK-MAPPING-START +// BLOCK-END +// BLOCK-ENTRY +// KEY +// VALUE +// +// The tokens BLOCK-SEQUENCE-START and BLOCK-MAPPING-START denote indentation +// increase that precedes a block collection (cf. the INDENT token in Python). +// The token BLOCK-END denote indentation decrease that ends a block collection +// (cf. the DEDENT token in Python). However YAML has some syntax pecularities +// that makes detections of these tokens more complex. +// +// The tokens BLOCK-ENTRY, KEY, and VALUE are used to represent the indicators +// '-', '?', and ':' correspondingly. +// +// The following examples show how the tokens BLOCK-SEQUENCE-START, +// BLOCK-MAPPING-START, and BLOCK-END are emitted by the Scanner: +// +// 1. Block sequences: +// +// - item 1 +// - item 2 +// - +// - item 3.1 +// - item 3.2 +// - +// key 1: value 1 +// key 2: value 2 +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-ENTRY +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 3.1",plain) +// BLOCK-ENTRY +// SCALAR("item 3.2",plain) +// BLOCK-END +// BLOCK-ENTRY +// BLOCK-MAPPING-START +// KEY +// SCALAR("key 1",plain) +// VALUE +// SCALAR("value 1",plain) +// KEY +// SCALAR("key 2",plain) +// VALUE +// SCALAR("value 2",plain) +// BLOCK-END +// BLOCK-END +// STREAM-END +// +// 2. Block mappings: +// +// a simple key: a value # The KEY token is produced here. +// ? a complex key +// : another value +// a mapping: +// key 1: value 1 +// key 2: value 2 +// a sequence: +// - item 1 +// - item 2 +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-MAPPING-START +// KEY +// SCALAR("a simple key",plain) +// VALUE +// SCALAR("a value",plain) +// KEY +// SCALAR("a complex key",plain) +// VALUE +// SCALAR("another value",plain) +// KEY +// SCALAR("a mapping",plain) +// BLOCK-MAPPING-START +// KEY +// SCALAR("key 1",plain) +// VALUE +// SCALAR("value 1",plain) +// KEY +// SCALAR("key 2",plain) +// VALUE +// SCALAR("value 2",plain) +// BLOCK-END +// KEY +// SCALAR("a sequence",plain) +// VALUE +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-END +// BLOCK-END +// STREAM-END +// +// YAML does not always require to start a new block collection from a new +// line. If the current line contains only '-', '?', and ':' indicators, a new +// block collection may start at the current line. The following examples +// illustrate this case: +// +// 1. Collections in a sequence: +// +// - - item 1 +// - item 2 +// - key 1: value 1 +// key 2: value 2 +// - ? complex key +// : complex value +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-END +// BLOCK-ENTRY +// BLOCK-MAPPING-START +// KEY +// SCALAR("key 1",plain) +// VALUE +// SCALAR("value 1",plain) +// KEY +// SCALAR("key 2",plain) +// VALUE +// SCALAR("value 2",plain) +// BLOCK-END +// BLOCK-ENTRY +// BLOCK-MAPPING-START +// KEY +// SCALAR("complex key") +// VALUE +// SCALAR("complex value") +// BLOCK-END +// BLOCK-END +// STREAM-END +// +// 2. Collections in a mapping: +// +// ? a sequence +// : - item 1 +// - item 2 +// ? a mapping +// : key 1: value 1 +// key 2: value 2 +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-MAPPING-START +// KEY +// SCALAR("a sequence",plain) +// VALUE +// BLOCK-SEQUENCE-START +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-END +// KEY +// SCALAR("a mapping",plain) +// VALUE +// BLOCK-MAPPING-START +// KEY +// SCALAR("key 1",plain) +// VALUE +// SCALAR("value 1",plain) +// KEY +// SCALAR("key 2",plain) +// VALUE +// SCALAR("value 2",plain) +// BLOCK-END +// BLOCK-END +// STREAM-END +// +// YAML also permits non-indented sequences if they are included into a block +// mapping. In this case, the token BLOCK-SEQUENCE-START is not produced: +// +// key: +// - item 1 # BLOCK-SEQUENCE-START is NOT produced here. +// - item 2 +// +// Tokens: +// +// STREAM-START(utf-8) +// BLOCK-MAPPING-START +// KEY +// SCALAR("key",plain) +// VALUE +// BLOCK-ENTRY +// SCALAR("item 1",plain) +// BLOCK-ENTRY +// SCALAR("item 2",plain) +// BLOCK-END +// + +// Ensure that the buffer contains the required number of characters. +// Return true on success, false on failure (reader error or memory error). +func cache(parser *yaml_parser_t, length int) bool { + // [Go] This was inlined: !cache(A, B) -> unread < B && !update(A, B) + return parser.unread >= length || yaml_parser_update_buffer(parser, length) +} + +// Advance the buffer pointer. +func skip(parser *yaml_parser_t) { + if !is_blank(parser.buffer, parser.buffer_pos) { + parser.newlines = 0 + } + parser.mark.index++ + parser.mark.column++ + parser.unread-- + parser.buffer_pos += width(parser.buffer[parser.buffer_pos]) +} + +func skip_line(parser *yaml_parser_t) { + if is_crlf(parser.buffer, parser.buffer_pos) { + parser.mark.index += 2 + parser.mark.column = 0 + parser.mark.line++ + parser.unread -= 2 + parser.buffer_pos += 2 + parser.newlines++ + } else if is_break(parser.buffer, parser.buffer_pos) { + parser.mark.index++ + parser.mark.column = 0 + parser.mark.line++ + parser.unread-- + parser.buffer_pos += width(parser.buffer[parser.buffer_pos]) + parser.newlines++ + } +} + +// Copy a character to a string buffer and advance pointers. +func read(parser *yaml_parser_t, s []byte) []byte { + if !is_blank(parser.buffer, parser.buffer_pos) { + parser.newlines = 0 + } + w := width(parser.buffer[parser.buffer_pos]) + if w == 0 { + panic("invalid character sequence") + } + if len(s) == 0 { + s = make([]byte, 0, 32) + } + if w == 1 && len(s)+w <= cap(s) { + s = s[:len(s)+1] + s[len(s)-1] = parser.buffer[parser.buffer_pos] + parser.buffer_pos++ + } else { + s = append(s, parser.buffer[parser.buffer_pos:parser.buffer_pos+w]...) + parser.buffer_pos += w + } + parser.mark.index++ + parser.mark.column++ + parser.unread-- + return s +} + +// Copy a line break character to a string buffer and advance pointers. +func read_line(parser *yaml_parser_t, s []byte) []byte { + buf := parser.buffer + pos := parser.buffer_pos + switch { + case buf[pos] == '\r' && buf[pos+1] == '\n': + // CR LF . LF + s = append(s, '\n') + parser.buffer_pos += 2 + parser.mark.index++ + parser.unread-- + case buf[pos] == '\r' || buf[pos] == '\n': + // CR|LF . LF + s = append(s, '\n') + parser.buffer_pos += 1 + case buf[pos] == '\xC2' && buf[pos+1] == '\x85': + // NEL . LF + s = append(s, '\n') + parser.buffer_pos += 2 + case buf[pos] == '\xE2' && buf[pos+1] == '\x80' && (buf[pos+2] == '\xA8' || buf[pos+2] == '\xA9'): + // LS|PS . LS|PS + s = append(s, buf[parser.buffer_pos:pos+3]...) + parser.buffer_pos += 3 + default: + return s + } + parser.mark.index++ + parser.mark.column = 0 + parser.mark.line++ + parser.unread-- + parser.newlines++ + return s +} + +// Get the next token. +func yaml_parser_scan(parser *yaml_parser_t, token *yaml_token_t) bool { + // Erase the token object. + *token = yaml_token_t{} // [Go] Is this necessary? + + // No tokens after STREAM-END or error. + if parser.stream_end_produced || parser.error != yaml_NO_ERROR { + return true + } + + // Ensure that the tokens queue contains enough tokens. + if !parser.token_available { + if !yaml_parser_fetch_more_tokens(parser) { + return false + } + } + + // Fetch the next token from the queue. + *token = parser.tokens[parser.tokens_head] + parser.tokens_head++ + parser.tokens_parsed++ + parser.token_available = false + + if token.typ == yaml_STREAM_END_TOKEN { + parser.stream_end_produced = true + } + return true +} + +// Set the scanner error and return false. +func yaml_parser_set_scanner_error(parser *yaml_parser_t, context string, context_mark yaml_mark_t, problem string) bool { + parser.error = yaml_SCANNER_ERROR + parser.context = context + parser.context_mark = context_mark + parser.problem = problem + parser.problem_mark = parser.mark + return false +} + +func yaml_parser_set_scanner_tag_error(parser *yaml_parser_t, directive bool, context_mark yaml_mark_t, problem string) bool { + context := "while parsing a tag" + if directive { + context = "while parsing a %TAG directive" + } + return yaml_parser_set_scanner_error(parser, context, context_mark, problem) +} + +func trace(args ...interface{}) func() { + pargs := append([]interface{}{"+++"}, args...) + fmt.Println(pargs...) + pargs = append([]interface{}{"---"}, args...) + return func() { fmt.Println(pargs...) } +} + +// Ensure that the tokens queue contains at least one token which can be +// returned to the Parser. +func yaml_parser_fetch_more_tokens(parser *yaml_parser_t) bool { + // While we need more tokens to fetch, do it. + for { + // [Go] The comment parsing logic requires a lookahead of two tokens + // so that foot comments may be parsed in time of associating them + // with the tokens that are parsed before them, and also for line + // comments to be transformed into head comments in some edge cases. + if parser.tokens_head < len(parser.tokens)-2 { + // If a potential simple key is at the head position, we need to fetch + // the next token to disambiguate it. + head_tok_idx, ok := parser.simple_keys_by_tok[parser.tokens_parsed] + if !ok { + break + } else if valid, ok := yaml_simple_key_is_valid(parser, &parser.simple_keys[head_tok_idx]); !ok { + return false + } else if !valid { + break + } + } + // Fetch the next token. + if !yaml_parser_fetch_next_token(parser) { + return false + } + } + + parser.token_available = true + return true +} + +// The dispatcher for token fetchers. +func yaml_parser_fetch_next_token(parser *yaml_parser_t) (ok bool) { + // Ensure that the buffer is initialized. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + // Check if we just started scanning. Fetch STREAM-START then. + if !parser.stream_start_produced { + return yaml_parser_fetch_stream_start(parser) + } + + scan_mark := parser.mark + + // Eat whitespaces and comments until we reach the next token. + if !yaml_parser_scan_to_next_token(parser) { + return false + } + + // [Go] While unrolling indents, transform the head comments of prior + // indentation levels observed after scan_start into foot comments at + // the respective indexes. + + // Check the indentation level against the current column. + if !yaml_parser_unroll_indent(parser, parser.mark.column, scan_mark) { + return false + } + + // Ensure that the buffer contains at least 4 characters. 4 is the length + // of the longest indicators ('--- ' and '... '). + if parser.unread < 4 && !yaml_parser_update_buffer(parser, 4) { + return false + } + + // Is it the end of the stream? + if is_z(parser.buffer, parser.buffer_pos) { + return yaml_parser_fetch_stream_end(parser) + } + + // Is it a directive? + if parser.mark.column == 0 && parser.buffer[parser.buffer_pos] == '%' { + return yaml_parser_fetch_directive(parser) + } + + buf := parser.buffer + pos := parser.buffer_pos + + // Is it the document start indicator? + if parser.mark.column == 0 && buf[pos] == '-' && buf[pos+1] == '-' && buf[pos+2] == '-' && is_blankz(buf, pos+3) { + return yaml_parser_fetch_document_indicator(parser, yaml_DOCUMENT_START_TOKEN) + } + + // Is it the document end indicator? + if parser.mark.column == 0 && buf[pos] == '.' && buf[pos+1] == '.' && buf[pos+2] == '.' && is_blankz(buf, pos+3) { + return yaml_parser_fetch_document_indicator(parser, yaml_DOCUMENT_END_TOKEN) + } + + comment_mark := parser.mark + if len(parser.tokens) > 0 && (parser.flow_level == 0 && buf[pos] == ':' || parser.flow_level > 0 && buf[pos] == ',') { + // Associate any following comments with the prior token. + comment_mark = parser.tokens[len(parser.tokens)-1].start_mark + } + defer func() { + if !ok { + return + } + if !yaml_parser_scan_line_comment(parser, comment_mark) { + ok = false + return + } + }() + + // Is it the flow sequence start indicator? + if buf[pos] == '[' { + return yaml_parser_fetch_flow_collection_start(parser, yaml_FLOW_SEQUENCE_START_TOKEN) + } + + // Is it the flow mapping start indicator? + if parser.buffer[parser.buffer_pos] == '{' { + return yaml_parser_fetch_flow_collection_start(parser, yaml_FLOW_MAPPING_START_TOKEN) + } + + // Is it the flow sequence end indicator? + if parser.buffer[parser.buffer_pos] == ']' { + return yaml_parser_fetch_flow_collection_end(parser, + yaml_FLOW_SEQUENCE_END_TOKEN) + } + + // Is it the flow mapping end indicator? + if parser.buffer[parser.buffer_pos] == '}' { + return yaml_parser_fetch_flow_collection_end(parser, + yaml_FLOW_MAPPING_END_TOKEN) + } + + // Is it the flow entry indicator? + if parser.buffer[parser.buffer_pos] == ',' { + return yaml_parser_fetch_flow_entry(parser) + } + + // Is it the block entry indicator? + if parser.buffer[parser.buffer_pos] == '-' && is_blankz(parser.buffer, parser.buffer_pos+1) { + return yaml_parser_fetch_block_entry(parser) + } + + // Is it the key indicator? + if parser.buffer[parser.buffer_pos] == '?' && (parser.flow_level > 0 || is_blankz(parser.buffer, parser.buffer_pos+1)) { + return yaml_parser_fetch_key(parser) + } + + // Is it the value indicator? + if parser.buffer[parser.buffer_pos] == ':' && (parser.flow_level > 0 || is_blankz(parser.buffer, parser.buffer_pos+1)) { + return yaml_parser_fetch_value(parser) + } + + // Is it an alias? + if parser.buffer[parser.buffer_pos] == '*' { + return yaml_parser_fetch_anchor(parser, yaml_ALIAS_TOKEN) + } + + // Is it an anchor? + if parser.buffer[parser.buffer_pos] == '&' { + return yaml_parser_fetch_anchor(parser, yaml_ANCHOR_TOKEN) + } + + // Is it a tag? + if parser.buffer[parser.buffer_pos] == '!' { + return yaml_parser_fetch_tag(parser) + } + + // Is it a literal scalar? + if parser.buffer[parser.buffer_pos] == '|' && parser.flow_level == 0 { + return yaml_parser_fetch_block_scalar(parser, true) + } + + // Is it a folded scalar? + if parser.buffer[parser.buffer_pos] == '>' && parser.flow_level == 0 { + return yaml_parser_fetch_block_scalar(parser, false) + } + + // Is it a single-quoted scalar? + if parser.buffer[parser.buffer_pos] == '\'' { + return yaml_parser_fetch_flow_scalar(parser, true) + } + + // Is it a double-quoted scalar? + if parser.buffer[parser.buffer_pos] == '"' { + return yaml_parser_fetch_flow_scalar(parser, false) + } + + // Is it a plain scalar? + // + // A plain scalar may start with any non-blank characters except + // + // '-', '?', ':', ',', '[', ']', '{', '}', + // '#', '&', '*', '!', '|', '>', '\'', '\"', + // '%', '@', '`'. + // + // In the block context (and, for the '-' indicator, in the flow context + // too), it may also start with the characters + // + // '-', '?', ':' + // + // if it is followed by a non-space character. + // + // The last rule is more restrictive than the specification requires. + // [Go] TODO Make this logic more reasonable. + //switch parser.buffer[parser.buffer_pos] { + //case '-', '?', ':', ',', '?', '-', ',', ':', ']', '[', '}', '{', '&', '#', '!', '*', '>', '|', '"', '\'', '@', '%', '-', '`': + //} + if !(is_blankz(parser.buffer, parser.buffer_pos) || parser.buffer[parser.buffer_pos] == '-' || + parser.buffer[parser.buffer_pos] == '?' || parser.buffer[parser.buffer_pos] == ':' || + parser.buffer[parser.buffer_pos] == ',' || parser.buffer[parser.buffer_pos] == '[' || + parser.buffer[parser.buffer_pos] == ']' || parser.buffer[parser.buffer_pos] == '{' || + parser.buffer[parser.buffer_pos] == '}' || parser.buffer[parser.buffer_pos] == '#' || + parser.buffer[parser.buffer_pos] == '&' || parser.buffer[parser.buffer_pos] == '*' || + parser.buffer[parser.buffer_pos] == '!' || parser.buffer[parser.buffer_pos] == '|' || + parser.buffer[parser.buffer_pos] == '>' || parser.buffer[parser.buffer_pos] == '\'' || + parser.buffer[parser.buffer_pos] == '"' || parser.buffer[parser.buffer_pos] == '%' || + parser.buffer[parser.buffer_pos] == '@' || parser.buffer[parser.buffer_pos] == '`') || + (parser.buffer[parser.buffer_pos] == '-' && !is_blank(parser.buffer, parser.buffer_pos+1)) || + (parser.flow_level == 0 && + (parser.buffer[parser.buffer_pos] == '?' || parser.buffer[parser.buffer_pos] == ':') && + !is_blankz(parser.buffer, parser.buffer_pos+1)) { + return yaml_parser_fetch_plain_scalar(parser) + } + + // If we don't determine the token type so far, it is an error. + return yaml_parser_set_scanner_error(parser, + "while scanning for the next token", parser.mark, + "found character that cannot start any token") +} + +func yaml_simple_key_is_valid(parser *yaml_parser_t, simple_key *yaml_simple_key_t) (valid, ok bool) { + if !simple_key.possible { + return false, true + } + + // The 1.2 specification says: + // + // "If the ? indicator is omitted, parsing needs to see past the + // implicit key to recognize it as such. To limit the amount of + // lookahead required, the “:†indicator must appear at most 1024 + // Unicode characters beyond the start of the key. In addition, the key + // is restricted to a single line." + // + if simple_key.mark.line < parser.mark.line || simple_key.mark.index+1024 < parser.mark.index { + // Check if the potential simple key to be removed is required. + if simple_key.required { + return false, yaml_parser_set_scanner_error(parser, + "while scanning a simple key", simple_key.mark, + "could not find expected ':'") + } + simple_key.possible = false + return false, true + } + return true, true +} + +// Check if a simple key may start at the current position and add it if +// needed. +func yaml_parser_save_simple_key(parser *yaml_parser_t) bool { + // A simple key is required at the current position if the scanner is in + // the block context and the current column coincides with the indentation + // level. + + required := parser.flow_level == 0 && parser.indent == parser.mark.column + + // + // If the current position may start a simple key, save it. + // + if parser.simple_key_allowed { + simple_key := yaml_simple_key_t{ + possible: true, + required: required, + token_number: parser.tokens_parsed + (len(parser.tokens) - parser.tokens_head), + mark: parser.mark, + } + + if !yaml_parser_remove_simple_key(parser) { + return false + } + parser.simple_keys[len(parser.simple_keys)-1] = simple_key + parser.simple_keys_by_tok[simple_key.token_number] = len(parser.simple_keys) - 1 + } + return true +} + +// Remove a potential simple key at the current flow level. +func yaml_parser_remove_simple_key(parser *yaml_parser_t) bool { + i := len(parser.simple_keys) - 1 + if parser.simple_keys[i].possible { + // If the key is required, it is an error. + if parser.simple_keys[i].required { + return yaml_parser_set_scanner_error(parser, + "while scanning a simple key", parser.simple_keys[i].mark, + "could not find expected ':'") + } + // Remove the key from the stack. + parser.simple_keys[i].possible = false + delete(parser.simple_keys_by_tok, parser.simple_keys[i].token_number) + } + return true +} + +// max_flow_level limits the flow_level +const max_flow_level = 10000 + +// Increase the flow level and resize the simple key list if needed. +func yaml_parser_increase_flow_level(parser *yaml_parser_t) bool { + // Reset the simple key on the next level. + parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{ + possible: false, + required: false, + token_number: parser.tokens_parsed + (len(parser.tokens) - parser.tokens_head), + mark: parser.mark, + }) + + // Increase the flow level. + parser.flow_level++ + if parser.flow_level > max_flow_level { + return yaml_parser_set_scanner_error(parser, + "while increasing flow level", parser.simple_keys[len(parser.simple_keys)-1].mark, + fmt.Sprintf("exceeded max depth of %d", max_flow_level)) + } + return true +} + +// Decrease the flow level. +func yaml_parser_decrease_flow_level(parser *yaml_parser_t) bool { + if parser.flow_level > 0 { + parser.flow_level-- + last := len(parser.simple_keys) - 1 + delete(parser.simple_keys_by_tok, parser.simple_keys[last].token_number) + parser.simple_keys = parser.simple_keys[:last] + } + return true +} + +// max_indents limits the indents stack size +const max_indents = 10000 + +// Push the current indentation level to the stack and set the new level +// the current column is greater than the indentation level. In this case, +// append or insert the specified token into the token queue. +func yaml_parser_roll_indent(parser *yaml_parser_t, column, number int, typ yaml_token_type_t, mark yaml_mark_t) bool { + // In the flow context, do nothing. + if parser.flow_level > 0 { + return true + } + + if parser.indent < column { + // Push the current indentation level to the stack and set the new + // indentation level. + parser.indents = append(parser.indents, parser.indent) + parser.indent = column + if len(parser.indents) > max_indents { + return yaml_parser_set_scanner_error(parser, + "while increasing indent level", parser.simple_keys[len(parser.simple_keys)-1].mark, + fmt.Sprintf("exceeded max depth of %d", max_indents)) + } + + // Create a token and insert it into the queue. + token := yaml_token_t{ + typ: typ, + start_mark: mark, + end_mark: mark, + } + if number > -1 { + number -= parser.tokens_parsed + } + yaml_insert_token(parser, number, &token) + } + return true +} + +// Pop indentation levels from the indents stack until the current level +// becomes less or equal to the column. For each indentation level, append +// the BLOCK-END token. +func yaml_parser_unroll_indent(parser *yaml_parser_t, column int, scan_mark yaml_mark_t) bool { + // In the flow context, do nothing. + if parser.flow_level > 0 { + return true + } + + block_mark := scan_mark + block_mark.index-- + + // Loop through the indentation levels in the stack. + for parser.indent > column { + + // [Go] Reposition the end token before potential following + // foot comments of parent blocks. For that, search + // backwards for recent comments that were at the same + // indent as the block that is ending now. + stop_index := block_mark.index + for i := len(parser.comments) - 1; i >= 0; i-- { + comment := &parser.comments[i] + + if comment.end_mark.index < stop_index { + // Don't go back beyond the start of the comment/whitespace scan, unless column < 0. + // If requested indent column is < 0, then the document is over and everything else + // is a foot anyway. + break + } + if comment.start_mark.column == parser.indent+1 { + // This is a good match. But maybe there's a former comment + // at that same indent level, so keep searching. + block_mark = comment.start_mark + } + + // While the end of the former comment matches with + // the start of the following one, we know there's + // nothing in between and scanning is still safe. + stop_index = comment.scan_mark.index + } + + // Create a token and append it to the queue. + token := yaml_token_t{ + typ: yaml_BLOCK_END_TOKEN, + start_mark: block_mark, + end_mark: block_mark, + } + yaml_insert_token(parser, -1, &token) + + // Pop the indentation level. + parser.indent = parser.indents[len(parser.indents)-1] + parser.indents = parser.indents[:len(parser.indents)-1] + } + return true +} + +// Initialize the scanner and produce the STREAM-START token. +func yaml_parser_fetch_stream_start(parser *yaml_parser_t) bool { + + // Set the initial indentation. + parser.indent = -1 + + // Initialize the simple key stack. + parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{}) + + parser.simple_keys_by_tok = make(map[int]int) + + // A simple key is allowed at the beginning of the stream. + parser.simple_key_allowed = true + + // We have started. + parser.stream_start_produced = true + + // Create the STREAM-START token and append it to the queue. + token := yaml_token_t{ + typ: yaml_STREAM_START_TOKEN, + start_mark: parser.mark, + end_mark: parser.mark, + encoding: parser.encoding, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the STREAM-END token and shut down the scanner. +func yaml_parser_fetch_stream_end(parser *yaml_parser_t) bool { + + // Force new line. + if parser.mark.column != 0 { + parser.mark.column = 0 + parser.mark.line++ + } + + // Reset the indentation level. + if !yaml_parser_unroll_indent(parser, -1, parser.mark) { + return false + } + + // Reset simple keys. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + parser.simple_key_allowed = false + + // Create the STREAM-END token and append it to the queue. + token := yaml_token_t{ + typ: yaml_STREAM_END_TOKEN, + start_mark: parser.mark, + end_mark: parser.mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce a VERSION-DIRECTIVE or TAG-DIRECTIVE token. +func yaml_parser_fetch_directive(parser *yaml_parser_t) bool { + // Reset the indentation level. + if !yaml_parser_unroll_indent(parser, -1, parser.mark) { + return false + } + + // Reset simple keys. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + parser.simple_key_allowed = false + + // Create the YAML-DIRECTIVE or TAG-DIRECTIVE token. + token := yaml_token_t{} + if !yaml_parser_scan_directive(parser, &token) { + return false + } + // Append the token to the queue. + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the DOCUMENT-START or DOCUMENT-END token. +func yaml_parser_fetch_document_indicator(parser *yaml_parser_t, typ yaml_token_type_t) bool { + // Reset the indentation level. + if !yaml_parser_unroll_indent(parser, -1, parser.mark) { + return false + } + + // Reset simple keys. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + parser.simple_key_allowed = false + + // Consume the token. + start_mark := parser.mark + + skip(parser) + skip(parser) + skip(parser) + + end_mark := parser.mark + + // Create the DOCUMENT-START or DOCUMENT-END token. + token := yaml_token_t{ + typ: typ, + start_mark: start_mark, + end_mark: end_mark, + } + // Append the token to the queue. + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the FLOW-SEQUENCE-START or FLOW-MAPPING-START token. +func yaml_parser_fetch_flow_collection_start(parser *yaml_parser_t, typ yaml_token_type_t) bool { + + // The indicators '[' and '{' may start a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // Increase the flow level. + if !yaml_parser_increase_flow_level(parser) { + return false + } + + // A simple key may follow the indicators '[' and '{'. + parser.simple_key_allowed = true + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the FLOW-SEQUENCE-START of FLOW-MAPPING-START token. + token := yaml_token_t{ + typ: typ, + start_mark: start_mark, + end_mark: end_mark, + } + // Append the token to the queue. + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the FLOW-SEQUENCE-END or FLOW-MAPPING-END token. +func yaml_parser_fetch_flow_collection_end(parser *yaml_parser_t, typ yaml_token_type_t) bool { + // Reset any potential simple key on the current flow level. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // Decrease the flow level. + if !yaml_parser_decrease_flow_level(parser) { + return false + } + + // No simple keys after the indicators ']' and '}'. + parser.simple_key_allowed = false + + // Consume the token. + + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the FLOW-SEQUENCE-END of FLOW-MAPPING-END token. + token := yaml_token_t{ + typ: typ, + start_mark: start_mark, + end_mark: end_mark, + } + // Append the token to the queue. + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the FLOW-ENTRY token. +func yaml_parser_fetch_flow_entry(parser *yaml_parser_t) bool { + // Reset any potential simple keys on the current flow level. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // Simple keys are allowed after ','. + parser.simple_key_allowed = true + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the FLOW-ENTRY token and append it to the queue. + token := yaml_token_t{ + typ: yaml_FLOW_ENTRY_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the BLOCK-ENTRY token. +func yaml_parser_fetch_block_entry(parser *yaml_parser_t) bool { + // Check if the scanner is in the block context. + if parser.flow_level == 0 { + // Check if we are allowed to start a new entry. + if !parser.simple_key_allowed { + return yaml_parser_set_scanner_error(parser, "", parser.mark, + "block sequence entries are not allowed in this context") + } + // Add the BLOCK-SEQUENCE-START token if needed. + if !yaml_parser_roll_indent(parser, parser.mark.column, -1, yaml_BLOCK_SEQUENCE_START_TOKEN, parser.mark) { + return false + } + } else { + // It is an error for the '-' indicator to occur in the flow context, + // but we let the Parser detect and report about it because the Parser + // is able to point to the context. + } + + // Reset any potential simple keys on the current flow level. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // Simple keys are allowed after '-'. + parser.simple_key_allowed = true + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the BLOCK-ENTRY token and append it to the queue. + token := yaml_token_t{ + typ: yaml_BLOCK_ENTRY_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the KEY token. +func yaml_parser_fetch_key(parser *yaml_parser_t) bool { + + // In the block context, additional checks are required. + if parser.flow_level == 0 { + // Check if we are allowed to start a new key (not nessesary simple). + if !parser.simple_key_allowed { + return yaml_parser_set_scanner_error(parser, "", parser.mark, + "mapping keys are not allowed in this context") + } + // Add the BLOCK-MAPPING-START token if needed. + if !yaml_parser_roll_indent(parser, parser.mark.column, -1, yaml_BLOCK_MAPPING_START_TOKEN, parser.mark) { + return false + } + } + + // Reset any potential simple keys on the current flow level. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // Simple keys are allowed after '?' in the block context. + parser.simple_key_allowed = parser.flow_level == 0 + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the KEY token and append it to the queue. + token := yaml_token_t{ + typ: yaml_KEY_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the VALUE token. +func yaml_parser_fetch_value(parser *yaml_parser_t) bool { + + simple_key := &parser.simple_keys[len(parser.simple_keys)-1] + + // Have we found a simple key? + if valid, ok := yaml_simple_key_is_valid(parser, simple_key); !ok { + return false + + } else if valid { + + // Create the KEY token and insert it into the queue. + token := yaml_token_t{ + typ: yaml_KEY_TOKEN, + start_mark: simple_key.mark, + end_mark: simple_key.mark, + } + yaml_insert_token(parser, simple_key.token_number-parser.tokens_parsed, &token) + + // In the block context, we may need to add the BLOCK-MAPPING-START token. + if !yaml_parser_roll_indent(parser, simple_key.mark.column, + simple_key.token_number, + yaml_BLOCK_MAPPING_START_TOKEN, simple_key.mark) { + return false + } + + // Remove the simple key. + simple_key.possible = false + delete(parser.simple_keys_by_tok, simple_key.token_number) + + // A simple key cannot follow another simple key. + parser.simple_key_allowed = false + + } else { + // The ':' indicator follows a complex key. + + // In the block context, extra checks are required. + if parser.flow_level == 0 { + + // Check if we are allowed to start a complex value. + if !parser.simple_key_allowed { + return yaml_parser_set_scanner_error(parser, "", parser.mark, + "mapping values are not allowed in this context") + } + + // Add the BLOCK-MAPPING-START token if needed. + if !yaml_parser_roll_indent(parser, parser.mark.column, -1, yaml_BLOCK_MAPPING_START_TOKEN, parser.mark) { + return false + } + } + + // Simple keys after ':' are allowed in the block context. + parser.simple_key_allowed = parser.flow_level == 0 + } + + // Consume the token. + start_mark := parser.mark + skip(parser) + end_mark := parser.mark + + // Create the VALUE token and append it to the queue. + token := yaml_token_t{ + typ: yaml_VALUE_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the ALIAS or ANCHOR token. +func yaml_parser_fetch_anchor(parser *yaml_parser_t, typ yaml_token_type_t) bool { + // An anchor or an alias could be a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // A simple key cannot follow an anchor or an alias. + parser.simple_key_allowed = false + + // Create the ALIAS or ANCHOR token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_anchor(parser, &token, typ) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the TAG token. +func yaml_parser_fetch_tag(parser *yaml_parser_t) bool { + // A tag could be a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // A simple key cannot follow a tag. + parser.simple_key_allowed = false + + // Create the TAG token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_tag(parser, &token) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the SCALAR(...,literal) or SCALAR(...,folded) tokens. +func yaml_parser_fetch_block_scalar(parser *yaml_parser_t, literal bool) bool { + // Remove any potential simple keys. + if !yaml_parser_remove_simple_key(parser) { + return false + } + + // A simple key may follow a block scalar. + parser.simple_key_allowed = true + + // Create the SCALAR token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_block_scalar(parser, &token, literal) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the SCALAR(...,single-quoted) or SCALAR(...,double-quoted) tokens. +func yaml_parser_fetch_flow_scalar(parser *yaml_parser_t, single bool) bool { + // A plain scalar could be a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // A simple key cannot follow a flow scalar. + parser.simple_key_allowed = false + + // Create the SCALAR token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_flow_scalar(parser, &token, single) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Produce the SCALAR(...,plain) token. +func yaml_parser_fetch_plain_scalar(parser *yaml_parser_t) bool { + // A plain scalar could be a simple key. + if !yaml_parser_save_simple_key(parser) { + return false + } + + // A simple key cannot follow a flow scalar. + parser.simple_key_allowed = false + + // Create the SCALAR token and append it to the queue. + var token yaml_token_t + if !yaml_parser_scan_plain_scalar(parser, &token) { + return false + } + yaml_insert_token(parser, -1, &token) + return true +} + +// Eat whitespaces and comments until the next token is found. +func yaml_parser_scan_to_next_token(parser *yaml_parser_t) bool { + + scan_mark := parser.mark + + // Until the next token is not found. + for { + // Allow the BOM mark to start a line. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if parser.mark.column == 0 && is_bom(parser.buffer, parser.buffer_pos) { + skip(parser) + } + + // Eat whitespaces. + // Tabs are allowed: + // - in the flow context + // - in the block context, but not at the beginning of the line or + // after '-', '?', or ':' (complex value). + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for parser.buffer[parser.buffer_pos] == ' ' || ((parser.flow_level > 0 || !parser.simple_key_allowed) && parser.buffer[parser.buffer_pos] == '\t') { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check if we just had a line comment under a sequence entry that + // looks more like a header to the following content. Similar to this: + // + // - # The comment + // - Some data + // + // If so, transform the line comment to a head comment and reposition. + if len(parser.comments) > 0 && len(parser.tokens) > 1 { + tokenA := parser.tokens[len(parser.tokens)-2] + tokenB := parser.tokens[len(parser.tokens)-1] + comment := &parser.comments[len(parser.comments)-1] + if tokenA.typ == yaml_BLOCK_SEQUENCE_START_TOKEN && tokenB.typ == yaml_BLOCK_ENTRY_TOKEN && len(comment.line) > 0 && !is_break(parser.buffer, parser.buffer_pos) { + // If it was in the prior line, reposition so it becomes a + // header of the follow up token. Otherwise, keep it in place + // so it becomes a header of the former. + comment.head = comment.line + comment.line = nil + if comment.start_mark.line == parser.mark.line-1 { + comment.token_mark = parser.mark + } + } + } + + // Eat a comment until a line break. + if parser.buffer[parser.buffer_pos] == '#' { + if !yaml_parser_scan_comments(parser, scan_mark) { + return false + } + } + + // If it is a line break, eat it. + if is_break(parser.buffer, parser.buffer_pos) { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + skip_line(parser) + + // In the block context, a new line may start a simple key. + if parser.flow_level == 0 { + parser.simple_key_allowed = true + } + } else { + break // We have found a token. + } + } + + return true +} + +// Scan a YAML-DIRECTIVE or TAG-DIRECTIVE token. +// +// Scope: +// %YAML 1.1 # a comment \n +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +// %TAG !yaml! tag:yaml.org,2002: \n +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +// +func yaml_parser_scan_directive(parser *yaml_parser_t, token *yaml_token_t) bool { + // Eat '%'. + start_mark := parser.mark + skip(parser) + + // Scan the directive name. + var name []byte + if !yaml_parser_scan_directive_name(parser, start_mark, &name) { + return false + } + + // Is it a YAML directive? + if bytes.Equal(name, []byte("YAML")) { + // Scan the VERSION directive value. + var major, minor int8 + if !yaml_parser_scan_version_directive_value(parser, start_mark, &major, &minor) { + return false + } + end_mark := parser.mark + + // Create a VERSION-DIRECTIVE token. + *token = yaml_token_t{ + typ: yaml_VERSION_DIRECTIVE_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + major: major, + minor: minor, + } + + // Is it a TAG directive? + } else if bytes.Equal(name, []byte("TAG")) { + // Scan the TAG directive value. + var handle, prefix []byte + if !yaml_parser_scan_tag_directive_value(parser, start_mark, &handle, &prefix) { + return false + } + end_mark := parser.mark + + // Create a TAG-DIRECTIVE token. + *token = yaml_token_t{ + typ: yaml_TAG_DIRECTIVE_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: handle, + prefix: prefix, + } + + // Unknown directive. + } else { + yaml_parser_set_scanner_error(parser, "while scanning a directive", + start_mark, "found unknown directive name") + return false + } + + // Eat the rest of the line including any comments. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + if parser.buffer[parser.buffer_pos] == '#' { + // [Go] Discard this inline comment for the time being. + //if !yaml_parser_scan_line_comment(parser, start_mark) { + // return false + //} + for !is_breakz(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + } + + // Check if we are at the end of the line. + if !is_breakz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a directive", + start_mark, "did not find expected comment or line break") + return false + } + + // Eat a line break. + if is_break(parser.buffer, parser.buffer_pos) { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + skip_line(parser) + } + + return true +} + +// Scan the directive name. +// +// Scope: +// %YAML 1.1 # a comment \n +// ^^^^ +// %TAG !yaml! tag:yaml.org,2002: \n +// ^^^ +// +func yaml_parser_scan_directive_name(parser *yaml_parser_t, start_mark yaml_mark_t, name *[]byte) bool { + // Consume the directive name. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + var s []byte + for is_alpha(parser.buffer, parser.buffer_pos) { + s = read(parser, s) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check if the name is empty. + if len(s) == 0 { + yaml_parser_set_scanner_error(parser, "while scanning a directive", + start_mark, "could not find expected directive name") + return false + } + + // Check for an blank character after the name. + if !is_blankz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a directive", + start_mark, "found unexpected non-alphabetical character") + return false + } + *name = s + return true +} + +// Scan the value of VERSION-DIRECTIVE. +// +// Scope: +// %YAML 1.1 # a comment \n +// ^^^^^^ +func yaml_parser_scan_version_directive_value(parser *yaml_parser_t, start_mark yaml_mark_t, major, minor *int8) bool { + // Eat whitespaces. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Consume the major version number. + if !yaml_parser_scan_version_directive_number(parser, start_mark, major) { + return false + } + + // Eat '.'. + if parser.buffer[parser.buffer_pos] != '.' { + return yaml_parser_set_scanner_error(parser, "while scanning a %YAML directive", + start_mark, "did not find expected digit or '.' character") + } + + skip(parser) + + // Consume the minor version number. + if !yaml_parser_scan_version_directive_number(parser, start_mark, minor) { + return false + } + return true +} + +const max_number_length = 2 + +// Scan the version number of VERSION-DIRECTIVE. +// +// Scope: +// %YAML 1.1 # a comment \n +// ^ +// %YAML 1.1 # a comment \n +// ^ +func yaml_parser_scan_version_directive_number(parser *yaml_parser_t, start_mark yaml_mark_t, number *int8) bool { + + // Repeat while the next character is digit. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + var value, length int8 + for is_digit(parser.buffer, parser.buffer_pos) { + // Check if the number is too long. + length++ + if length > max_number_length { + return yaml_parser_set_scanner_error(parser, "while scanning a %YAML directive", + start_mark, "found extremely long version number") + } + value = value*10 + int8(as_digit(parser.buffer, parser.buffer_pos)) + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check if the number was present. + if length == 0 { + return yaml_parser_set_scanner_error(parser, "while scanning a %YAML directive", + start_mark, "did not find expected version number") + } + *number = value + return true +} + +// Scan the value of a TAG-DIRECTIVE token. +// +// Scope: +// %TAG !yaml! tag:yaml.org,2002: \n +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +// +func yaml_parser_scan_tag_directive_value(parser *yaml_parser_t, start_mark yaml_mark_t, handle, prefix *[]byte) bool { + var handle_value, prefix_value []byte + + // Eat whitespaces. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Scan a handle. + if !yaml_parser_scan_tag_handle(parser, true, start_mark, &handle_value) { + return false + } + + // Expect a whitespace. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if !is_blank(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a %TAG directive", + start_mark, "did not find expected whitespace") + return false + } + + // Eat whitespaces. + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Scan a prefix. + if !yaml_parser_scan_tag_uri(parser, true, nil, start_mark, &prefix_value) { + return false + } + + // Expect a whitespace or line break. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if !is_blankz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a %TAG directive", + start_mark, "did not find expected whitespace or line break") + return false + } + + *handle = handle_value + *prefix = prefix_value + return true +} + +func yaml_parser_scan_anchor(parser *yaml_parser_t, token *yaml_token_t, typ yaml_token_type_t) bool { + var s []byte + + // Eat the indicator character. + start_mark := parser.mark + skip(parser) + + // Consume the value. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_alpha(parser.buffer, parser.buffer_pos) { + s = read(parser, s) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + end_mark := parser.mark + + /* + * Check if length of the anchor is greater than 0 and it is followed by + * a whitespace character or one of the indicators: + * + * '?', ':', ',', ']', '}', '%', '@', '`'. + */ + + if len(s) == 0 || + !(is_blankz(parser.buffer, parser.buffer_pos) || parser.buffer[parser.buffer_pos] == '?' || + parser.buffer[parser.buffer_pos] == ':' || parser.buffer[parser.buffer_pos] == ',' || + parser.buffer[parser.buffer_pos] == ']' || parser.buffer[parser.buffer_pos] == '}' || + parser.buffer[parser.buffer_pos] == '%' || parser.buffer[parser.buffer_pos] == '@' || + parser.buffer[parser.buffer_pos] == '`') { + context := "while scanning an alias" + if typ == yaml_ANCHOR_TOKEN { + context = "while scanning an anchor" + } + yaml_parser_set_scanner_error(parser, context, start_mark, + "did not find expected alphabetic or numeric character") + return false + } + + // Create a token. + *token = yaml_token_t{ + typ: typ, + start_mark: start_mark, + end_mark: end_mark, + value: s, + } + + return true +} + +/* + * Scan a TAG token. + */ + +func yaml_parser_scan_tag(parser *yaml_parser_t, token *yaml_token_t) bool { + var handle, suffix []byte + + start_mark := parser.mark + + // Check if the tag is in the canonical form. + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + + if parser.buffer[parser.buffer_pos+1] == '<' { + // Keep the handle as '' + + // Eat '!<' + skip(parser) + skip(parser) + + // Consume the tag value. + if !yaml_parser_scan_tag_uri(parser, false, nil, start_mark, &suffix) { + return false + } + + // Check for '>' and eat it. + if parser.buffer[parser.buffer_pos] != '>' { + yaml_parser_set_scanner_error(parser, "while scanning a tag", + start_mark, "did not find the expected '>'") + return false + } + + skip(parser) + } else { + // The tag has either the '!suffix' or the '!handle!suffix' form. + + // First, try to scan a handle. + if !yaml_parser_scan_tag_handle(parser, false, start_mark, &handle) { + return false + } + + // Check if it is, indeed, handle. + if handle[0] == '!' && len(handle) > 1 && handle[len(handle)-1] == '!' { + // Scan the suffix now. + if !yaml_parser_scan_tag_uri(parser, false, nil, start_mark, &suffix) { + return false + } + } else { + // It wasn't a handle after all. Scan the rest of the tag. + if !yaml_parser_scan_tag_uri(parser, false, handle, start_mark, &suffix) { + return false + } + + // Set the handle to '!'. + handle = []byte{'!'} + + // A special case: the '!' tag. Set the handle to '' and the + // suffix to '!'. + if len(suffix) == 0 { + handle, suffix = suffix, handle + } + } + } + + // Check the character which ends the tag. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if !is_blankz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a tag", + start_mark, "did not find expected whitespace or line break") + return false + } + + end_mark := parser.mark + + // Create a token. + *token = yaml_token_t{ + typ: yaml_TAG_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: handle, + suffix: suffix, + } + return true +} + +// Scan a tag handle. +func yaml_parser_scan_tag_handle(parser *yaml_parser_t, directive bool, start_mark yaml_mark_t, handle *[]byte) bool { + // Check the initial '!' character. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if parser.buffer[parser.buffer_pos] != '!' { + yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "did not find expected '!'") + return false + } + + var s []byte + + // Copy the '!' character. + s = read(parser, s) + + // Copy all subsequent alphabetical and numerical characters. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + for is_alpha(parser.buffer, parser.buffer_pos) { + s = read(parser, s) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check if the trailing character is '!' and copy it. + if parser.buffer[parser.buffer_pos] == '!' { + s = read(parser, s) + } else { + // It's either the '!' tag or not really a tag handle. If it's a %TAG + // directive, it's an error. If it's a tag token, it must be a part of URI. + if directive && string(s) != "!" { + yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "did not find expected '!'") + return false + } + } + + *handle = s + return true +} + +// Scan a tag. +func yaml_parser_scan_tag_uri(parser *yaml_parser_t, directive bool, head []byte, start_mark yaml_mark_t, uri *[]byte) bool { + //size_t length = head ? strlen((char *)head) : 0 + var s []byte + hasTag := len(head) > 0 + + // Copy the head if needed. + // + // Note that we don't copy the leading '!' character. + if len(head) > 1 { + s = append(s, head[1:]...) + } + + // Scan the tag. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + // The set of characters that may appear in URI is as follows: + // + // '0'-'9', 'A'-'Z', 'a'-'z', '_', '-', ';', '/', '?', ':', '@', '&', + // '=', '+', '$', ',', '.', '!', '~', '*', '\'', '(', ')', '[', ']', + // '%'. + // [Go] TODO Convert this into more reasonable logic. + for is_alpha(parser.buffer, parser.buffer_pos) || parser.buffer[parser.buffer_pos] == ';' || + parser.buffer[parser.buffer_pos] == '/' || parser.buffer[parser.buffer_pos] == '?' || + parser.buffer[parser.buffer_pos] == ':' || parser.buffer[parser.buffer_pos] == '@' || + parser.buffer[parser.buffer_pos] == '&' || parser.buffer[parser.buffer_pos] == '=' || + parser.buffer[parser.buffer_pos] == '+' || parser.buffer[parser.buffer_pos] == '$' || + parser.buffer[parser.buffer_pos] == ',' || parser.buffer[parser.buffer_pos] == '.' || + parser.buffer[parser.buffer_pos] == '!' || parser.buffer[parser.buffer_pos] == '~' || + parser.buffer[parser.buffer_pos] == '*' || parser.buffer[parser.buffer_pos] == '\'' || + parser.buffer[parser.buffer_pos] == '(' || parser.buffer[parser.buffer_pos] == ')' || + parser.buffer[parser.buffer_pos] == '[' || parser.buffer[parser.buffer_pos] == ']' || + parser.buffer[parser.buffer_pos] == '%' { + // Check if it is a URI-escape sequence. + if parser.buffer[parser.buffer_pos] == '%' { + if !yaml_parser_scan_uri_escapes(parser, directive, start_mark, &s) { + return false + } + } else { + s = read(parser, s) + } + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + hasTag = true + } + + if !hasTag { + yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "did not find expected tag URI") + return false + } + *uri = s + return true +} + +// Decode an URI-escape sequence corresponding to a single UTF-8 character. +func yaml_parser_scan_uri_escapes(parser *yaml_parser_t, directive bool, start_mark yaml_mark_t, s *[]byte) bool { + + // Decode the required number of characters. + w := 1024 + for w > 0 { + // Check for a URI-escaped octet. + if parser.unread < 3 && !yaml_parser_update_buffer(parser, 3) { + return false + } + + if !(parser.buffer[parser.buffer_pos] == '%' && + is_hex(parser.buffer, parser.buffer_pos+1) && + is_hex(parser.buffer, parser.buffer_pos+2)) { + return yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "did not find URI escaped octet") + } + + // Get the octet. + octet := byte((as_hex(parser.buffer, parser.buffer_pos+1) << 4) + as_hex(parser.buffer, parser.buffer_pos+2)) + + // If it is the leading octet, determine the length of the UTF-8 sequence. + if w == 1024 { + w = width(octet) + if w == 0 { + return yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "found an incorrect leading UTF-8 octet") + } + } else { + // Check if the trailing octet is correct. + if octet&0xC0 != 0x80 { + return yaml_parser_set_scanner_tag_error(parser, directive, + start_mark, "found an incorrect trailing UTF-8 octet") + } + } + + // Copy the octet and move the pointers. + *s = append(*s, octet) + skip(parser) + skip(parser) + skip(parser) + w-- + } + return true +} + +// Scan a block scalar. +func yaml_parser_scan_block_scalar(parser *yaml_parser_t, token *yaml_token_t, literal bool) bool { + // Eat the indicator '|' or '>'. + start_mark := parser.mark + skip(parser) + + // Scan the additional block scalar indicators. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + // Check for a chomping indicator. + var chomping, increment int + if parser.buffer[parser.buffer_pos] == '+' || parser.buffer[parser.buffer_pos] == '-' { + // Set the chomping method and eat the indicator. + if parser.buffer[parser.buffer_pos] == '+' { + chomping = +1 + } else { + chomping = -1 + } + skip(parser) + + // Check for an indentation indicator. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if is_digit(parser.buffer, parser.buffer_pos) { + // Check that the indentation is greater than 0. + if parser.buffer[parser.buffer_pos] == '0' { + yaml_parser_set_scanner_error(parser, "while scanning a block scalar", + start_mark, "found an indentation indicator equal to 0") + return false + } + + // Get the indentation level and eat the indicator. + increment = as_digit(parser.buffer, parser.buffer_pos) + skip(parser) + } + + } else if is_digit(parser.buffer, parser.buffer_pos) { + // Do the same as above, but in the opposite order. + + if parser.buffer[parser.buffer_pos] == '0' { + yaml_parser_set_scanner_error(parser, "while scanning a block scalar", + start_mark, "found an indentation indicator equal to 0") + return false + } + increment = as_digit(parser.buffer, parser.buffer_pos) + skip(parser) + + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if parser.buffer[parser.buffer_pos] == '+' || parser.buffer[parser.buffer_pos] == '-' { + if parser.buffer[parser.buffer_pos] == '+' { + chomping = +1 + } else { + chomping = -1 + } + skip(parser) + } + } + + // Eat whitespaces and comments to the end of the line. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + for is_blank(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + if parser.buffer[parser.buffer_pos] == '#' { + // TODO Test this and then re-enable it. + //if !yaml_parser_scan_line_comment(parser, start_mark) { + // return false + //} + for !is_breakz(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + } + + // Check if we are at the end of the line. + if !is_breakz(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a block scalar", + start_mark, "did not find expected comment or line break") + return false + } + + // Eat a line break. + if is_break(parser.buffer, parser.buffer_pos) { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + skip_line(parser) + } + + end_mark := parser.mark + + // Set the indentation level if it was specified. + var indent int + if increment > 0 { + if parser.indent >= 0 { + indent = parser.indent + increment + } else { + indent = increment + } + } + + // Scan the leading line breaks and determine the indentation level if needed. + var s, leading_break, trailing_breaks []byte + if !yaml_parser_scan_block_scalar_breaks(parser, &indent, &trailing_breaks, start_mark, &end_mark) { + return false + } + + // Scan the block scalar content. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + var leading_blank, trailing_blank bool + for parser.mark.column == indent && !is_z(parser.buffer, parser.buffer_pos) { + // We are at the beginning of a non-empty line. + + // Is it a trailing whitespace? + trailing_blank = is_blank(parser.buffer, parser.buffer_pos) + + // Check if we need to fold the leading line break. + if !literal && !leading_blank && !trailing_blank && len(leading_break) > 0 && leading_break[0] == '\n' { + // Do we need to join the lines by space? + if len(trailing_breaks) == 0 { + s = append(s, ' ') + } + } else { + s = append(s, leading_break...) + } + leading_break = leading_break[:0] + + // Append the remaining line breaks. + s = append(s, trailing_breaks...) + trailing_breaks = trailing_breaks[:0] + + // Is it a leading whitespace? + leading_blank = is_blank(parser.buffer, parser.buffer_pos) + + // Consume the current line. + for !is_breakz(parser.buffer, parser.buffer_pos) { + s = read(parser, s) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Consume the line break. + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + + leading_break = read_line(parser, leading_break) + + // Eat the following indentation spaces and line breaks. + if !yaml_parser_scan_block_scalar_breaks(parser, &indent, &trailing_breaks, start_mark, &end_mark) { + return false + } + } + + // Chomp the tail. + if chomping != -1 { + s = append(s, leading_break...) + } + if chomping == 1 { + s = append(s, trailing_breaks...) + } + + // Create a token. + *token = yaml_token_t{ + typ: yaml_SCALAR_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: s, + style: yaml_LITERAL_SCALAR_STYLE, + } + if !literal { + token.style = yaml_FOLDED_SCALAR_STYLE + } + return true +} + +// Scan indentation spaces and line breaks for a block scalar. Determine the +// indentation level if needed. +func yaml_parser_scan_block_scalar_breaks(parser *yaml_parser_t, indent *int, breaks *[]byte, start_mark yaml_mark_t, end_mark *yaml_mark_t) bool { + *end_mark = parser.mark + + // Eat the indentation spaces and line breaks. + max_indent := 0 + for { + // Eat the indentation spaces. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + for (*indent == 0 || parser.mark.column < *indent) && is_space(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + if parser.mark.column > max_indent { + max_indent = parser.mark.column + } + + // Check for a tab character messing the indentation. + if (*indent == 0 || parser.mark.column < *indent) && is_tab(parser.buffer, parser.buffer_pos) { + return yaml_parser_set_scanner_error(parser, "while scanning a block scalar", + start_mark, "found a tab character where an indentation space is expected") + } + + // Have we found a non-empty line? + if !is_break(parser.buffer, parser.buffer_pos) { + break + } + + // Consume the line break. + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + // [Go] Should really be returning breaks instead. + *breaks = read_line(parser, *breaks) + *end_mark = parser.mark + } + + // Determine the indentation level if needed. + if *indent == 0 { + *indent = max_indent + if *indent < parser.indent+1 { + *indent = parser.indent + 1 + } + if *indent < 1 { + *indent = 1 + } + } + return true +} + +// Scan a quoted scalar. +func yaml_parser_scan_flow_scalar(parser *yaml_parser_t, token *yaml_token_t, single bool) bool { + // Eat the left quote. + start_mark := parser.mark + skip(parser) + + // Consume the content of the quoted scalar. + var s, leading_break, trailing_breaks, whitespaces []byte + for { + // Check that there are no document indicators at the beginning of the line. + if parser.unread < 4 && !yaml_parser_update_buffer(parser, 4) { + return false + } + + if parser.mark.column == 0 && + ((parser.buffer[parser.buffer_pos+0] == '-' && + parser.buffer[parser.buffer_pos+1] == '-' && + parser.buffer[parser.buffer_pos+2] == '-') || + (parser.buffer[parser.buffer_pos+0] == '.' && + parser.buffer[parser.buffer_pos+1] == '.' && + parser.buffer[parser.buffer_pos+2] == '.')) && + is_blankz(parser.buffer, parser.buffer_pos+3) { + yaml_parser_set_scanner_error(parser, "while scanning a quoted scalar", + start_mark, "found unexpected document indicator") + return false + } + + // Check for EOF. + if is_z(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a quoted scalar", + start_mark, "found unexpected end of stream") + return false + } + + // Consume non-blank characters. + leading_blanks := false + for !is_blankz(parser.buffer, parser.buffer_pos) { + if single && parser.buffer[parser.buffer_pos] == '\'' && parser.buffer[parser.buffer_pos+1] == '\'' { + // Is is an escaped single quote. + s = append(s, '\'') + skip(parser) + skip(parser) + + } else if single && parser.buffer[parser.buffer_pos] == '\'' { + // It is a right single quote. + break + } else if !single && parser.buffer[parser.buffer_pos] == '"' { + // It is a right double quote. + break + + } else if !single && parser.buffer[parser.buffer_pos] == '\\' && is_break(parser.buffer, parser.buffer_pos+1) { + // It is an escaped line break. + if parser.unread < 3 && !yaml_parser_update_buffer(parser, 3) { + return false + } + skip(parser) + skip_line(parser) + leading_blanks = true + break + + } else if !single && parser.buffer[parser.buffer_pos] == '\\' { + // It is an escape sequence. + code_length := 0 + + // Check the escape character. + switch parser.buffer[parser.buffer_pos+1] { + case '0': + s = append(s, 0) + case 'a': + s = append(s, '\x07') + case 'b': + s = append(s, '\x08') + case 't', '\t': + s = append(s, '\x09') + case 'n': + s = append(s, '\x0A') + case 'v': + s = append(s, '\x0B') + case 'f': + s = append(s, '\x0C') + case 'r': + s = append(s, '\x0D') + case 'e': + s = append(s, '\x1B') + case ' ': + s = append(s, '\x20') + case '"': + s = append(s, '"') + case '\'': + s = append(s, '\'') + case '\\': + s = append(s, '\\') + case 'N': // NEL (#x85) + s = append(s, '\xC2') + s = append(s, '\x85') + case '_': // #xA0 + s = append(s, '\xC2') + s = append(s, '\xA0') + case 'L': // LS (#x2028) + s = append(s, '\xE2') + s = append(s, '\x80') + s = append(s, '\xA8') + case 'P': // PS (#x2029) + s = append(s, '\xE2') + s = append(s, '\x80') + s = append(s, '\xA9') + case 'x': + code_length = 2 + case 'u': + code_length = 4 + case 'U': + code_length = 8 + default: + yaml_parser_set_scanner_error(parser, "while parsing a quoted scalar", + start_mark, "found unknown escape character") + return false + } + + skip(parser) + skip(parser) + + // Consume an arbitrary escape code. + if code_length > 0 { + var value int + + // Scan the character value. + if parser.unread < code_length && !yaml_parser_update_buffer(parser, code_length) { + return false + } + for k := 0; k < code_length; k++ { + if !is_hex(parser.buffer, parser.buffer_pos+k) { + yaml_parser_set_scanner_error(parser, "while parsing a quoted scalar", + start_mark, "did not find expected hexdecimal number") + return false + } + value = (value << 4) + as_hex(parser.buffer, parser.buffer_pos+k) + } + + // Check the value and write the character. + if (value >= 0xD800 && value <= 0xDFFF) || value > 0x10FFFF { + yaml_parser_set_scanner_error(parser, "while parsing a quoted scalar", + start_mark, "found invalid Unicode character escape code") + return false + } + if value <= 0x7F { + s = append(s, byte(value)) + } else if value <= 0x7FF { + s = append(s, byte(0xC0+(value>>6))) + s = append(s, byte(0x80+(value&0x3F))) + } else if value <= 0xFFFF { + s = append(s, byte(0xE0+(value>>12))) + s = append(s, byte(0x80+((value>>6)&0x3F))) + s = append(s, byte(0x80+(value&0x3F))) + } else { + s = append(s, byte(0xF0+(value>>18))) + s = append(s, byte(0x80+((value>>12)&0x3F))) + s = append(s, byte(0x80+((value>>6)&0x3F))) + s = append(s, byte(0x80+(value&0x3F))) + } + + // Advance the pointer. + for k := 0; k < code_length; k++ { + skip(parser) + } + } + } else { + // It is a non-escaped non-blank character. + s = read(parser, s) + } + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + } + + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + // Check if we are at the end of the scalar. + if single { + if parser.buffer[parser.buffer_pos] == '\'' { + break + } + } else { + if parser.buffer[parser.buffer_pos] == '"' { + break + } + } + + // Consume blank characters. + for is_blank(parser.buffer, parser.buffer_pos) || is_break(parser.buffer, parser.buffer_pos) { + if is_blank(parser.buffer, parser.buffer_pos) { + // Consume a space or a tab character. + if !leading_blanks { + whitespaces = read(parser, whitespaces) + } else { + skip(parser) + } + } else { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + + // Check if it is a first line break. + if !leading_blanks { + whitespaces = whitespaces[:0] + leading_break = read_line(parser, leading_break) + leading_blanks = true + } else { + trailing_breaks = read_line(parser, trailing_breaks) + } + } + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Join the whitespaces or fold line breaks. + if leading_blanks { + // Do we need to fold line breaks? + if len(leading_break) > 0 && leading_break[0] == '\n' { + if len(trailing_breaks) == 0 { + s = append(s, ' ') + } else { + s = append(s, trailing_breaks...) + } + } else { + s = append(s, leading_break...) + s = append(s, trailing_breaks...) + } + trailing_breaks = trailing_breaks[:0] + leading_break = leading_break[:0] + } else { + s = append(s, whitespaces...) + whitespaces = whitespaces[:0] + } + } + + // Eat the right quote. + skip(parser) + end_mark := parser.mark + + // Create a token. + *token = yaml_token_t{ + typ: yaml_SCALAR_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: s, + style: yaml_SINGLE_QUOTED_SCALAR_STYLE, + } + if !single { + token.style = yaml_DOUBLE_QUOTED_SCALAR_STYLE + } + return true +} + +// Scan a plain scalar. +func yaml_parser_scan_plain_scalar(parser *yaml_parser_t, token *yaml_token_t) bool { + + var s, leading_break, trailing_breaks, whitespaces []byte + var leading_blanks bool + var indent = parser.indent + 1 + + start_mark := parser.mark + end_mark := parser.mark + + // Consume the content of the plain scalar. + for { + // Check for a document indicator. + if parser.unread < 4 && !yaml_parser_update_buffer(parser, 4) { + return false + } + if parser.mark.column == 0 && + ((parser.buffer[parser.buffer_pos+0] == '-' && + parser.buffer[parser.buffer_pos+1] == '-' && + parser.buffer[parser.buffer_pos+2] == '-') || + (parser.buffer[parser.buffer_pos+0] == '.' && + parser.buffer[parser.buffer_pos+1] == '.' && + parser.buffer[parser.buffer_pos+2] == '.')) && + is_blankz(parser.buffer, parser.buffer_pos+3) { + break + } + + // Check for a comment. + if parser.buffer[parser.buffer_pos] == '#' { + break + } + + // Consume non-blank characters. + for !is_blankz(parser.buffer, parser.buffer_pos) { + + // Check for indicators that may end a plain scalar. + if (parser.buffer[parser.buffer_pos] == ':' && is_blankz(parser.buffer, parser.buffer_pos+1)) || + (parser.flow_level > 0 && + (parser.buffer[parser.buffer_pos] == ',' || + parser.buffer[parser.buffer_pos] == '?' || parser.buffer[parser.buffer_pos] == '[' || + parser.buffer[parser.buffer_pos] == ']' || parser.buffer[parser.buffer_pos] == '{' || + parser.buffer[parser.buffer_pos] == '}')) { + break + } + + // Check if we need to join whitespaces and breaks. + if leading_blanks || len(whitespaces) > 0 { + if leading_blanks { + // Do we need to fold line breaks? + if leading_break[0] == '\n' { + if len(trailing_breaks) == 0 { + s = append(s, ' ') + } else { + s = append(s, trailing_breaks...) + } + } else { + s = append(s, leading_break...) + s = append(s, trailing_breaks...) + } + trailing_breaks = trailing_breaks[:0] + leading_break = leading_break[:0] + leading_blanks = false + } else { + s = append(s, whitespaces...) + whitespaces = whitespaces[:0] + } + } + + // Copy the character. + s = read(parser, s) + + end_mark = parser.mark + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + } + + // Is it the end? + if !(is_blank(parser.buffer, parser.buffer_pos) || is_break(parser.buffer, parser.buffer_pos)) { + break + } + + // Consume blank characters. + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + + for is_blank(parser.buffer, parser.buffer_pos) || is_break(parser.buffer, parser.buffer_pos) { + if is_blank(parser.buffer, parser.buffer_pos) { + + // Check for tab characters that abuse indentation. + if leading_blanks && parser.mark.column < indent && is_tab(parser.buffer, parser.buffer_pos) { + yaml_parser_set_scanner_error(parser, "while scanning a plain scalar", + start_mark, "found a tab character that violates indentation") + return false + } + + // Consume a space or a tab character. + if !leading_blanks { + whitespaces = read(parser, whitespaces) + } else { + skip(parser) + } + } else { + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + + // Check if it is a first line break. + if !leading_blanks { + whitespaces = whitespaces[:0] + leading_break = read_line(parser, leading_break) + leading_blanks = true + } else { + trailing_breaks = read_line(parser, trailing_breaks) + } + } + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + } + + // Check indentation level. + if parser.flow_level == 0 && parser.mark.column < indent { + break + } + } + + // Create a token. + *token = yaml_token_t{ + typ: yaml_SCALAR_TOKEN, + start_mark: start_mark, + end_mark: end_mark, + value: s, + style: yaml_PLAIN_SCALAR_STYLE, + } + + // Note that we change the 'simple_key_allowed' flag. + if leading_blanks { + parser.simple_key_allowed = true + } + return true +} + +func yaml_parser_scan_line_comment(parser *yaml_parser_t, token_mark yaml_mark_t) bool { + if parser.newlines > 0 { + return true + } + + var start_mark yaml_mark_t + var text []byte + + for peek := 0; peek < 512; peek++ { + if parser.unread < peek+1 && !yaml_parser_update_buffer(parser, peek+1) { + break + } + if is_blank(parser.buffer, parser.buffer_pos+peek) { + continue + } + if parser.buffer[parser.buffer_pos+peek] == '#' { + seen := parser.mark.index+peek + for { + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if is_breakz(parser.buffer, parser.buffer_pos) { + if parser.mark.index >= seen { + break + } + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + skip_line(parser) + } else { + if parser.mark.index >= seen { + if len(text) == 0 { + start_mark = parser.mark + } + text = append(text, parser.buffer[parser.buffer_pos]) + } + skip(parser) + } + } + } + break + } + if len(text) > 0 { + parser.comments = append(parser.comments, yaml_comment_t{ + token_mark: token_mark, + start_mark: start_mark, + line: text, + }) + } + return true +} + +func yaml_parser_scan_comments(parser *yaml_parser_t, scan_mark yaml_mark_t) bool { + token := parser.tokens[len(parser.tokens)-1] + + if token.typ == yaml_FLOW_ENTRY_TOKEN && len(parser.tokens) > 1 { + token = parser.tokens[len(parser.tokens)-2] + } + + var token_mark = token.start_mark + var start_mark yaml_mark_t + + var recent_empty = false + var first_empty = parser.newlines <= 1 + + var line = parser.mark.line + var column = parser.mark.column + + var text []byte + + // The foot line is the place where a comment must start to + // still be considered as a foot of the prior content. + // If there's some content in the currently parsed line, then + // the foot is the line below it. + var foot_line = -1 + if scan_mark.line > 0 { + foot_line = parser.mark.line-parser.newlines+1 + if parser.newlines == 0 && parser.mark.column > 1 { + foot_line++ + } + } + + var peek = 0 + for ; peek < 512; peek++ { + if parser.unread < peek+1 && !yaml_parser_update_buffer(parser, peek+1) { + break + } + column++ + if is_blank(parser.buffer, parser.buffer_pos+peek) { + continue + } + c := parser.buffer[parser.buffer_pos+peek] + if is_breakz(parser.buffer, parser.buffer_pos+peek) || parser.flow_level > 0 && (c == ']' || c == '}') { + // Got line break or terminator. + if !recent_empty { + if first_empty && (start_mark.line == foot_line || start_mark.column-1 < parser.indent) { + // This is the first empty line and there were no empty lines before, + // so this initial part of the comment is a foot of the prior token + // instead of being a head for the following one. Split it up. + if len(text) > 0 { + if start_mark.column-1 < parser.indent { + // If dedented it's unrelated to the prior token. + token_mark = start_mark + } + parser.comments = append(parser.comments, yaml_comment_t{ + scan_mark: scan_mark, + token_mark: token_mark, + start_mark: start_mark, + end_mark: yaml_mark_t{parser.mark.index + peek, line, column}, + foot: text, + }) + scan_mark = yaml_mark_t{parser.mark.index + peek, line, column} + token_mark = scan_mark + text = nil + } + } else { + if len(text) > 0 && parser.buffer[parser.buffer_pos+peek] != 0 { + text = append(text, '\n') + } + } + } + if !is_break(parser.buffer, parser.buffer_pos+peek) { + break + } + first_empty = false + recent_empty = true + column = 0 + line++ + continue + } + + if len(text) > 0 && column < parser.indent+1 && column != start_mark.column { + // The comment at the different indentation is a foot of the + // preceding data rather than a head of the upcoming one. + parser.comments = append(parser.comments, yaml_comment_t{ + scan_mark: scan_mark, + token_mark: token_mark, + start_mark: start_mark, + end_mark: yaml_mark_t{parser.mark.index + peek, line, column}, + foot: text, + }) + scan_mark = yaml_mark_t{parser.mark.index + peek, line, column} + token_mark = scan_mark + text = nil + } + + if parser.buffer[parser.buffer_pos+peek] != '#' { + break + } + + if len(text) == 0 { + start_mark = yaml_mark_t{parser.mark.index + peek, line, column} + } else { + text = append(text, '\n') + } + + recent_empty = false + + // Consume until after the consumed comment line. + seen := parser.mark.index+peek + for { + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } + if is_breakz(parser.buffer, parser.buffer_pos) { + if parser.mark.index >= seen { + break + } + if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) { + return false + } + skip_line(parser) + } else { + if parser.mark.index >= seen { + text = append(text, parser.buffer[parser.buffer_pos]) + } + skip(parser) + } + } + + peek = 0 + column = 0 + line = parser.mark.line + } + + if len(text) > 0 { + parser.comments = append(parser.comments, yaml_comment_t{ + scan_mark: scan_mark, + token_mark: start_mark, + start_mark: start_mark, + end_mark: yaml_mark_t{parser.mark.index + peek - 1, line, column}, + head: text, + }) + } + return true +} diff --git a/gopkg.in/yaml.v3/sorter.go b/gopkg.in/yaml.v3/sorter.go new file mode 100644 index 0000000000..9210ece7e9 --- /dev/null +++ b/gopkg.in/yaml.v3/sorter.go @@ -0,0 +1,134 @@ +// +// Copyright (c) 2011-2019 Canonical Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package yaml + +import ( + "reflect" + "unicode" +) + +type keyList []reflect.Value + +func (l keyList) Len() int { return len(l) } +func (l keyList) Swap(i, j int) { l[i], l[j] = l[j], l[i] } +func (l keyList) Less(i, j int) bool { + a := l[i] + b := l[j] + ak := a.Kind() + bk := b.Kind() + for (ak == reflect.Interface || ak == reflect.Ptr) && !a.IsNil() { + a = a.Elem() + ak = a.Kind() + } + for (bk == reflect.Interface || bk == reflect.Ptr) && !b.IsNil() { + b = b.Elem() + bk = b.Kind() + } + af, aok := keyFloat(a) + bf, bok := keyFloat(b) + if aok && bok { + if af != bf { + return af < bf + } + if ak != bk { + return ak < bk + } + return numLess(a, b) + } + if ak != reflect.String || bk != reflect.String { + return ak < bk + } + ar, br := []rune(a.String()), []rune(b.String()) + digits := false + for i := 0; i < len(ar) && i < len(br); i++ { + if ar[i] == br[i] { + digits = unicode.IsDigit(ar[i]) + continue + } + al := unicode.IsLetter(ar[i]) + bl := unicode.IsLetter(br[i]) + if al && bl { + return ar[i] < br[i] + } + if al || bl { + if digits { + return al + } else { + return bl + } + } + var ai, bi int + var an, bn int64 + if ar[i] == '0' || br[i] == '0' { + for j := i - 1; j >= 0 && unicode.IsDigit(ar[j]); j-- { + if ar[j] != '0' { + an = 1 + bn = 1 + break + } + } + } + for ai = i; ai < len(ar) && unicode.IsDigit(ar[ai]); ai++ { + an = an*10 + int64(ar[ai]-'0') + } + for bi = i; bi < len(br) && unicode.IsDigit(br[bi]); bi++ { + bn = bn*10 + int64(br[bi]-'0') + } + if an != bn { + return an < bn + } + if ai != bi { + return ai < bi + } + return ar[i] < br[i] + } + return len(ar) < len(br) +} + +// keyFloat returns a float value for v if it is a number/bool +// and whether it is a number/bool or not. +func keyFloat(v reflect.Value) (f float64, ok bool) { + switch v.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return float64(v.Int()), true + case reflect.Float32, reflect.Float64: + return v.Float(), true + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return float64(v.Uint()), true + case reflect.Bool: + if v.Bool() { + return 1, true + } + return 0, true + } + return 0, false +} + +// numLess returns whether a < b. +// a and b must necessarily have the same kind. +func numLess(a, b reflect.Value) bool { + switch a.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return a.Int() < b.Int() + case reflect.Float32, reflect.Float64: + return a.Float() < b.Float() + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return a.Uint() < b.Uint() + case reflect.Bool: + return !a.Bool() && b.Bool() + } + panic("not a number") +} diff --git a/gopkg.in/yaml.v3/writerc.go b/gopkg.in/yaml.v3/writerc.go new file mode 100644 index 0000000000..b8a116bf9a --- /dev/null +++ b/gopkg.in/yaml.v3/writerc.go @@ -0,0 +1,48 @@ +// +// Copyright (c) 2011-2019 Canonical Ltd +// Copyright (c) 2006-2010 Kirill Simonov +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +// of the Software, and to permit persons to whom the Software is furnished to do +// so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package yaml + +// Set the writer error and return false. +func yaml_emitter_set_writer_error(emitter *yaml_emitter_t, problem string) bool { + emitter.error = yaml_WRITER_ERROR + emitter.problem = problem + return false +} + +// Flush the output buffer. +func yaml_emitter_flush(emitter *yaml_emitter_t) bool { + if emitter.write_handler == nil { + panic("write handler not set") + } + + // Check if the buffer is empty. + if emitter.buffer_pos == 0 { + return true + } + + if err := emitter.write_handler(emitter, emitter.buffer[:emitter.buffer_pos]); err != nil { + return yaml_emitter_set_writer_error(emitter, "write error: "+err.Error()) + } + emitter.buffer_pos = 0 + return true +} diff --git a/gopkg.in/yaml.v3/yaml.go b/gopkg.in/yaml.v3/yaml.go new file mode 100644 index 0000000000..b5d35a50de --- /dev/null +++ b/gopkg.in/yaml.v3/yaml.go @@ -0,0 +1,662 @@ +// +// Copyright (c) 2011-2019 Canonical Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package yaml implements YAML support for the Go language. +// +// Source code and other details for the project are available at GitHub: +// +// https://github.com/go-yaml/yaml +// +package yaml + +import ( + "errors" + "fmt" + "io" + "reflect" + "strings" + "sync" + "unicode/utf8" +) + +// The Unmarshaler interface may be implemented by types to customize their +// behavior when being unmarshaled from a YAML document. +type Unmarshaler interface { + UnmarshalYAML(value *Node) error +} + +type obsoleteUnmarshaler interface { + UnmarshalYAML(unmarshal func(interface{}) error) error +} + +// The Marshaler interface may be implemented by types to customize their +// behavior when being marshaled into a YAML document. The returned value +// is marshaled in place of the original value implementing Marshaler. +// +// If an error is returned by MarshalYAML, the marshaling procedure stops +// and returns with the provided error. +type Marshaler interface { + MarshalYAML() (interface{}, error) +} + +// Unmarshal decodes the first document found within the in byte slice +// and assigns decoded values into the out value. +// +// Maps and pointers (to a struct, string, int, etc) are accepted as out +// values. If an internal pointer within a struct is not initialized, +// the yaml package will initialize it if necessary for unmarshalling +// the provided data. The out parameter must not be nil. +// +// The type of the decoded values should be compatible with the respective +// values in out. If one or more values cannot be decoded due to a type +// mismatches, decoding continues partially until the end of the YAML +// content, and a *yaml.TypeError is returned with details for all +// missed values. +// +// Struct fields are only unmarshalled if they are exported (have an +// upper case first letter), and are unmarshalled using the field name +// lowercased as the default key. Custom keys may be defined via the +// "yaml" name in the field tag: the content preceding the first comma +// is used as the key, and the following comma-separated options are +// used to tweak the marshalling process (see Marshal). +// Conflicting names result in a runtime error. +// +// For example: +// +// type T struct { +// F int `yaml:"a,omitempty"` +// B int +// } +// var t T +// yaml.Unmarshal([]byte("a: 1\nb: 2"), &t) +// +// See the documentation of Marshal for the format of tags and a list of +// supported tag options. +// +func Unmarshal(in []byte, out interface{}) (err error) { + return unmarshal(in, out, false) +} + +// A Decorder reads and decodes YAML values from an input stream. +type Decoder struct { + parser *parser + knownFields bool +} + +// NewDecoder returns a new decoder that reads from r. +// +// The decoder introduces its own buffering and may read +// data from r beyond the YAML values requested. +func NewDecoder(r io.Reader) *Decoder { + return &Decoder{ + parser: newParserFromReader(r), + } +} + +// KnownFields ensures that the keys in decoded mappings to +// exist as fields in the struct being decoded into. +func (dec *Decoder) KnownFields(enable bool) { + dec.knownFields = enable +} + +// Decode reads the next YAML-encoded value from its input +// and stores it in the value pointed to by v. +// +// See the documentation for Unmarshal for details about the +// conversion of YAML into a Go value. +func (dec *Decoder) Decode(v interface{}) (err error) { + d := newDecoder() + d.knownFields = dec.knownFields + defer handleErr(&err) + node := dec.parser.parse() + if node == nil { + return io.EOF + } + out := reflect.ValueOf(v) + if out.Kind() == reflect.Ptr && !out.IsNil() { + out = out.Elem() + } + d.unmarshal(node, out) + if len(d.terrors) > 0 { + return &TypeError{d.terrors} + } + return nil +} + +// Decode decodes the node and stores its data into the value pointed to by v. +// +// See the documentation for Unmarshal for details about the +// conversion of YAML into a Go value. +func (n *Node) Decode(v interface{}) (err error) { + d := newDecoder() + defer handleErr(&err) + out := reflect.ValueOf(v) + if out.Kind() == reflect.Ptr && !out.IsNil() { + out = out.Elem() + } + d.unmarshal(n, out) + if len(d.terrors) > 0 { + return &TypeError{d.terrors} + } + return nil +} + +func unmarshal(in []byte, out interface{}, strict bool) (err error) { + defer handleErr(&err) + d := newDecoder() + p := newParser(in) + defer p.destroy() + node := p.parse() + if node != nil { + v := reflect.ValueOf(out) + if v.Kind() == reflect.Ptr && !v.IsNil() { + v = v.Elem() + } + d.unmarshal(node, v) + } + if len(d.terrors) > 0 { + return &TypeError{d.terrors} + } + return nil +} + +// Marshal serializes the value provided into a YAML document. The structure +// of the generated document will reflect the structure of the value itself. +// Maps and pointers (to struct, string, int, etc) are accepted as the in value. +// +// Struct fields are only marshalled if they are exported (have an upper case +// first letter), and are marshalled using the field name lowercased as the +// default key. Custom keys may be defined via the "yaml" name in the field +// tag: the content preceding the first comma is used as the key, and the +// following comma-separated options are used to tweak the marshalling process. +// Conflicting names result in a runtime error. +// +// The field tag format accepted is: +// +// `(...) yaml:"[][,[,]]" (...)` +// +// The following flags are currently supported: +// +// omitempty Only include the field if it's not set to the zero +// value for the type or to empty slices or maps. +// Zero valued structs will be omitted if all their public +// fields are zero, unless they implement an IsZero +// method (see the IsZeroer interface type), in which +// case the field will be included if that method returns true. +// +// flow Marshal using a flow style (useful for structs, +// sequences and maps). +// +// inline Inline the field, which must be a struct or a map, +// causing all of its fields or keys to be processed as if +// they were part of the outer struct. For maps, keys must +// not conflict with the yaml keys of other struct fields. +// +// In addition, if the key is "-", the field is ignored. +// +// For example: +// +// type T struct { +// F int `yaml:"a,omitempty"` +// B int +// } +// yaml.Marshal(&T{B: 2}) // Returns "b: 2\n" +// yaml.Marshal(&T{F: 1}} // Returns "a: 1\nb: 0\n" +// +func Marshal(in interface{}) (out []byte, err error) { + defer handleErr(&err) + e := newEncoder() + defer e.destroy() + e.marshalDoc("", reflect.ValueOf(in)) + e.finish() + out = e.out + return +} + +// An Encoder writes YAML values to an output stream. +type Encoder struct { + encoder *encoder +} + +// NewEncoder returns a new encoder that writes to w. +// The Encoder should be closed after use to flush all data +// to w. +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{ + encoder: newEncoderWithWriter(w), + } +} + +// Encode writes the YAML encoding of v to the stream. +// If multiple items are encoded to the stream, the +// second and subsequent document will be preceded +// with a "---" document separator, but the first will not. +// +// See the documentation for Marshal for details about the conversion of Go +// values to YAML. +func (e *Encoder) Encode(v interface{}) (err error) { + defer handleErr(&err) + e.encoder.marshalDoc("", reflect.ValueOf(v)) + return nil +} + +// SetIndent changes the used indentation used when encoding. +func (e *Encoder) SetIndent(spaces int) { + if spaces < 0 { + panic("yaml: cannot indent to a negative number of spaces") + } + e.encoder.indent = spaces +} + +// Close closes the encoder by writing any remaining data. +// It does not write a stream terminating string "...". +func (e *Encoder) Close() (err error) { + defer handleErr(&err) + e.encoder.finish() + return nil +} + +func handleErr(err *error) { + if v := recover(); v != nil { + if e, ok := v.(yamlError); ok { + *err = e.err + } else { + panic(v) + } + } +} + +type yamlError struct { + err error +} + +func fail(err error) { + panic(yamlError{err}) +} + +func failf(format string, args ...interface{}) { + panic(yamlError{fmt.Errorf("yaml: "+format, args...)}) +} + +// A TypeError is returned by Unmarshal when one or more fields in +// the YAML document cannot be properly decoded into the requested +// types. When this error is returned, the value is still +// unmarshaled partially. +type TypeError struct { + Errors []string +} + +func (e *TypeError) Error() string { + return fmt.Sprintf("yaml: unmarshal errors:\n %s", strings.Join(e.Errors, "\n ")) +} + +type Kind uint32 + +const ( + DocumentNode Kind = 1 << iota + SequenceNode + MappingNode + ScalarNode + AliasNode +) + +type Style uint32 + +const ( + TaggedStyle Style = 1 << iota + DoubleQuotedStyle + SingleQuotedStyle + LiteralStyle + FoldedStyle + FlowStyle +) + +// Node represents an element in the YAML document hierarchy. While documents +// are typically encoded and decoded into higher level types, such as structs +// and maps, Node is an intermediate representation that allows detailed +// control over the content being decoded or encoded. +// +// Values that make use of the Node type interact with the yaml package in the +// same way any other type would do, by encoding and decoding yaml data +// directly or indirectly into them. +// +// For example: +// +// var person struct { +// Name string +// Address yaml.Node +// } +// err := yaml.Unmarshal(data, &person) +// +// Or by itself: +// +// var person Node +// err := yaml.Unmarshal(data, &person) +// +type Node struct { + // Kind defines whether the node is a document, a mapping, a sequence, + // a scalar value, or an alias to another node. The specific data type of + // scalar nodes may be obtained via the ShortTag and LongTag methods. + Kind Kind + + // Style allows customizing the apperance of the node in the tree. + Style Style + + // Tag holds the YAML tag defining the data type for the value. + // When decoding, this field will always be set to the resolved tag, + // even when it wasn't explicitly provided in the YAML content. + // When encoding, if this field is unset the value type will be + // implied from the node properties, and if it is set, it will only + // be serialized into the representation if TaggedStyle is used or + // the implicit tag diverges from the provided one. + Tag string + + // Value holds the unescaped and unquoted represenation of the value. + Value string + + // Anchor holds the anchor name for this node, which allows aliases to point to it. + Anchor string + + // Alias holds the node that this alias points to. Only valid when Kind is AliasNode. + Alias *Node + + // Content holds contained nodes for documents, mappings, and sequences. + Content []*Node + + // HeadComment holds any comments in the lines preceding the node and + // not separated by an empty line. + HeadComment string + + // LineComment holds any comments at the end of the line where the node is in. + LineComment string + + // FootComment holds any comments following the node and before empty lines. + FootComment string + + // Line and Column hold the node position in the decoded YAML text. + // These fields are not respected when encoding the node. + Line int + Column int +} + +// LongTag returns the long form of the tag that indicates the data type for +// the node. If the Tag field isn't explicitly defined, one will be computed +// based on the node properties. +func (n *Node) LongTag() string { + return longTag(n.ShortTag()) +} + +// ShortTag returns the short form of the YAML tag that indicates data type for +// the node. If the Tag field isn't explicitly defined, one will be computed +// based on the node properties. +func (n *Node) ShortTag() string { + if n.indicatedString() { + return strTag + } + if n.Tag == "" || n.Tag == "!" { + switch n.Kind { + case MappingNode: + return mapTag + case SequenceNode: + return seqTag + case AliasNode: + if n.Alias != nil { + return n.Alias.ShortTag() + } + case ScalarNode: + tag, _ := resolve("", n.Value) + return tag + } + return "" + } + return shortTag(n.Tag) +} + +func (n *Node) indicatedString() bool { + return n.Kind == ScalarNode && + (shortTag(n.Tag) == strTag || + (n.Tag == "" || n.Tag == "!") && n.Style&(SingleQuotedStyle|DoubleQuotedStyle|LiteralStyle|FoldedStyle) != 0) +} + +// SetString is a convenience function that sets the node to a string value +// and defines its style in a pleasant way depending on its content. +func (n *Node) SetString(s string) { + n.Kind = ScalarNode + if utf8.ValidString(s) { + n.Value = s + n.Tag = strTag + } else { + n.Value = encodeBase64(s) + n.Tag = binaryTag + } + if strings.Contains(n.Value, "\n") { + n.Style = LiteralStyle + } +} + +// -------------------------------------------------------------------------- +// Maintain a mapping of keys to structure field indexes + +// The code in this section was copied from mgo/bson. + +// structInfo holds details for the serialization of fields of +// a given struct. +type structInfo struct { + FieldsMap map[string]fieldInfo + FieldsList []fieldInfo + + // InlineMap is the number of the field in the struct that + // contains an ,inline map, or -1 if there's none. + InlineMap int + + // InlineUnmarshalers holds indexes to inlined fields that + // contain unmarshaler values. + InlineUnmarshalers [][]int +} + +type fieldInfo struct { + Key string + Num int + OmitEmpty bool + Flow bool + // Id holds the unique field identifier, so we can cheaply + // check for field duplicates without maintaining an extra map. + Id int + + // Inline holds the field index if the field is part of an inlined struct. + Inline []int +} + +var structMap = make(map[reflect.Type]*structInfo) +var fieldMapMutex sync.RWMutex +var unmarshalerType reflect.Type + +func init() { + var v Unmarshaler + unmarshalerType = reflect.ValueOf(&v).Elem().Type() +} + +func getStructInfo(st reflect.Type) (*structInfo, error) { + fieldMapMutex.RLock() + sinfo, found := structMap[st] + fieldMapMutex.RUnlock() + if found { + return sinfo, nil + } + + n := st.NumField() + fieldsMap := make(map[string]fieldInfo) + fieldsList := make([]fieldInfo, 0, n) + inlineMap := -1 + inlineUnmarshalers := [][]int(nil) + for i := 0; i != n; i++ { + field := st.Field(i) + if field.PkgPath != "" && !field.Anonymous { + continue // Private field + } + + info := fieldInfo{Num: i} + + tag := field.Tag.Get("yaml") + if tag == "" && strings.Index(string(field.Tag), ":") < 0 { + tag = string(field.Tag) + } + if tag == "-" { + continue + } + + inline := false + fields := strings.Split(tag, ",") + if len(fields) > 1 { + for _, flag := range fields[1:] { + switch flag { + case "omitempty": + info.OmitEmpty = true + case "flow": + info.Flow = true + case "inline": + inline = true + default: + return nil, errors.New(fmt.Sprintf("unsupported flag %q in tag %q of type %s", flag, tag, st)) + } + } + tag = fields[0] + } + + if inline { + switch field.Type.Kind() { + case reflect.Map: + if inlineMap >= 0 { + return nil, errors.New("multiple ,inline maps in struct " + st.String()) + } + if field.Type.Key() != reflect.TypeOf("") { + return nil, errors.New("option ,inline needs a map with string keys in struct " + st.String()) + } + inlineMap = info.Num + case reflect.Struct, reflect.Ptr: + ftype := field.Type + for ftype.Kind() == reflect.Ptr { + ftype = ftype.Elem() + } + if ftype.Kind() != reflect.Struct { + return nil, errors.New("option ,inline may only be used on a struct or map field") + } + if reflect.PtrTo(ftype).Implements(unmarshalerType) { + inlineUnmarshalers = append(inlineUnmarshalers, []int{i}) + } else { + sinfo, err := getStructInfo(ftype) + if err != nil { + return nil, err + } + for _, index := range sinfo.InlineUnmarshalers { + inlineUnmarshalers = append(inlineUnmarshalers, append([]int{i}, index...)) + } + for _, finfo := range sinfo.FieldsList { + if _, found := fieldsMap[finfo.Key]; found { + msg := "duplicated key '" + finfo.Key + "' in struct " + st.String() + return nil, errors.New(msg) + } + if finfo.Inline == nil { + finfo.Inline = []int{i, finfo.Num} + } else { + finfo.Inline = append([]int{i}, finfo.Inline...) + } + finfo.Id = len(fieldsList) + fieldsMap[finfo.Key] = finfo + fieldsList = append(fieldsList, finfo) + } + } + default: + return nil, errors.New("option ,inline may only be used on a struct or map field") + } + continue + } + + if tag != "" { + info.Key = tag + } else { + info.Key = strings.ToLower(field.Name) + } + + if _, found = fieldsMap[info.Key]; found { + msg := "duplicated key '" + info.Key + "' in struct " + st.String() + return nil, errors.New(msg) + } + + info.Id = len(fieldsList) + fieldsList = append(fieldsList, info) + fieldsMap[info.Key] = info + } + + sinfo = &structInfo{ + FieldsMap: fieldsMap, + FieldsList: fieldsList, + InlineMap: inlineMap, + InlineUnmarshalers: inlineUnmarshalers, + } + + fieldMapMutex.Lock() + structMap[st] = sinfo + fieldMapMutex.Unlock() + return sinfo, nil +} + +// IsZeroer is used to check whether an object is zero to +// determine whether it should be omitted when marshaling +// with the omitempty flag. One notable implementation +// is time.Time. +type IsZeroer interface { + IsZero() bool +} + +func isZero(v reflect.Value) bool { + kind := v.Kind() + if z, ok := v.Interface().(IsZeroer); ok { + if (kind == reflect.Ptr || kind == reflect.Interface) && v.IsNil() { + return true + } + return z.IsZero() + } + switch kind { + case reflect.String: + return len(v.String()) == 0 + case reflect.Interface, reflect.Ptr: + return v.IsNil() + case reflect.Slice: + return v.Len() == 0 + case reflect.Map: + return v.Len() == 0 + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Bool: + return !v.Bool() + case reflect.Struct: + vt := v.Type() + for i := v.NumField() - 1; i >= 0; i-- { + if vt.Field(i).PkgPath != "" { + continue // Private field + } + if !isZero(v.Field(i)) { + return false + } + } + return true + } + return false +} diff --git a/gopkg.in/yaml.v3/yamlh.go b/gopkg.in/yaml.v3/yamlh.go new file mode 100644 index 0000000000..2719cfbb0b --- /dev/null +++ b/gopkg.in/yaml.v3/yamlh.go @@ -0,0 +1,805 @@ +// +// Copyright (c) 2011-2019 Canonical Ltd +// Copyright (c) 2006-2010 Kirill Simonov +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +// of the Software, and to permit persons to whom the Software is furnished to do +// so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package yaml + +import ( + "fmt" + "io" +) + +// The version directive data. +type yaml_version_directive_t struct { + major int8 // The major version number. + minor int8 // The minor version number. +} + +// The tag directive data. +type yaml_tag_directive_t struct { + handle []byte // The tag handle. + prefix []byte // The tag prefix. +} + +type yaml_encoding_t int + +// The stream encoding. +const ( + // Let the parser choose the encoding. + yaml_ANY_ENCODING yaml_encoding_t = iota + + yaml_UTF8_ENCODING // The default UTF-8 encoding. + yaml_UTF16LE_ENCODING // The UTF-16-LE encoding with BOM. + yaml_UTF16BE_ENCODING // The UTF-16-BE encoding with BOM. +) + +type yaml_break_t int + +// Line break types. +const ( + // Let the parser choose the break type. + yaml_ANY_BREAK yaml_break_t = iota + + yaml_CR_BREAK // Use CR for line breaks (Mac style). + yaml_LN_BREAK // Use LN for line breaks (Unix style). + yaml_CRLN_BREAK // Use CR LN for line breaks (DOS style). +) + +type yaml_error_type_t int + +// Many bad things could happen with the parser and emitter. +const ( + // No error is produced. + yaml_NO_ERROR yaml_error_type_t = iota + + yaml_MEMORY_ERROR // Cannot allocate or reallocate a block of memory. + yaml_READER_ERROR // Cannot read or decode the input stream. + yaml_SCANNER_ERROR // Cannot scan the input stream. + yaml_PARSER_ERROR // Cannot parse the input stream. + yaml_COMPOSER_ERROR // Cannot compose a YAML document. + yaml_WRITER_ERROR // Cannot write to the output stream. + yaml_EMITTER_ERROR // Cannot emit a YAML stream. +) + +// The pointer position. +type yaml_mark_t struct { + index int // The position index. + line int // The position line. + column int // The position column. +} + +// Node Styles + +type yaml_style_t int8 + +type yaml_scalar_style_t yaml_style_t + +// Scalar styles. +const ( + // Let the emitter choose the style. + yaml_ANY_SCALAR_STYLE yaml_scalar_style_t = 0 + + yaml_PLAIN_SCALAR_STYLE yaml_scalar_style_t = 1 << iota // The plain scalar style. + yaml_SINGLE_QUOTED_SCALAR_STYLE // The single-quoted scalar style. + yaml_DOUBLE_QUOTED_SCALAR_STYLE // The double-quoted scalar style. + yaml_LITERAL_SCALAR_STYLE // The literal scalar style. + yaml_FOLDED_SCALAR_STYLE // The folded scalar style. +) + +type yaml_sequence_style_t yaml_style_t + +// Sequence styles. +const ( + // Let the emitter choose the style. + yaml_ANY_SEQUENCE_STYLE yaml_sequence_style_t = iota + + yaml_BLOCK_SEQUENCE_STYLE // The block sequence style. + yaml_FLOW_SEQUENCE_STYLE // The flow sequence style. +) + +type yaml_mapping_style_t yaml_style_t + +// Mapping styles. +const ( + // Let the emitter choose the style. + yaml_ANY_MAPPING_STYLE yaml_mapping_style_t = iota + + yaml_BLOCK_MAPPING_STYLE // The block mapping style. + yaml_FLOW_MAPPING_STYLE // The flow mapping style. +) + +// Tokens + +type yaml_token_type_t int + +// Token types. +const ( + // An empty token. + yaml_NO_TOKEN yaml_token_type_t = iota + + yaml_STREAM_START_TOKEN // A STREAM-START token. + yaml_STREAM_END_TOKEN // A STREAM-END token. + + yaml_VERSION_DIRECTIVE_TOKEN // A VERSION-DIRECTIVE token. + yaml_TAG_DIRECTIVE_TOKEN // A TAG-DIRECTIVE token. + yaml_DOCUMENT_START_TOKEN // A DOCUMENT-START token. + yaml_DOCUMENT_END_TOKEN // A DOCUMENT-END token. + + yaml_BLOCK_SEQUENCE_START_TOKEN // A BLOCK-SEQUENCE-START token. + yaml_BLOCK_MAPPING_START_TOKEN // A BLOCK-SEQUENCE-END token. + yaml_BLOCK_END_TOKEN // A BLOCK-END token. + + yaml_FLOW_SEQUENCE_START_TOKEN // A FLOW-SEQUENCE-START token. + yaml_FLOW_SEQUENCE_END_TOKEN // A FLOW-SEQUENCE-END token. + yaml_FLOW_MAPPING_START_TOKEN // A FLOW-MAPPING-START token. + yaml_FLOW_MAPPING_END_TOKEN // A FLOW-MAPPING-END token. + + yaml_BLOCK_ENTRY_TOKEN // A BLOCK-ENTRY token. + yaml_FLOW_ENTRY_TOKEN // A FLOW-ENTRY token. + yaml_KEY_TOKEN // A KEY token. + yaml_VALUE_TOKEN // A VALUE token. + + yaml_ALIAS_TOKEN // An ALIAS token. + yaml_ANCHOR_TOKEN // An ANCHOR token. + yaml_TAG_TOKEN // A TAG token. + yaml_SCALAR_TOKEN // A SCALAR token. +) + +func (tt yaml_token_type_t) String() string { + switch tt { + case yaml_NO_TOKEN: + return "yaml_NO_TOKEN" + case yaml_STREAM_START_TOKEN: + return "yaml_STREAM_START_TOKEN" + case yaml_STREAM_END_TOKEN: + return "yaml_STREAM_END_TOKEN" + case yaml_VERSION_DIRECTIVE_TOKEN: + return "yaml_VERSION_DIRECTIVE_TOKEN" + case yaml_TAG_DIRECTIVE_TOKEN: + return "yaml_TAG_DIRECTIVE_TOKEN" + case yaml_DOCUMENT_START_TOKEN: + return "yaml_DOCUMENT_START_TOKEN" + case yaml_DOCUMENT_END_TOKEN: + return "yaml_DOCUMENT_END_TOKEN" + case yaml_BLOCK_SEQUENCE_START_TOKEN: + return "yaml_BLOCK_SEQUENCE_START_TOKEN" + case yaml_BLOCK_MAPPING_START_TOKEN: + return "yaml_BLOCK_MAPPING_START_TOKEN" + case yaml_BLOCK_END_TOKEN: + return "yaml_BLOCK_END_TOKEN" + case yaml_FLOW_SEQUENCE_START_TOKEN: + return "yaml_FLOW_SEQUENCE_START_TOKEN" + case yaml_FLOW_SEQUENCE_END_TOKEN: + return "yaml_FLOW_SEQUENCE_END_TOKEN" + case yaml_FLOW_MAPPING_START_TOKEN: + return "yaml_FLOW_MAPPING_START_TOKEN" + case yaml_FLOW_MAPPING_END_TOKEN: + return "yaml_FLOW_MAPPING_END_TOKEN" + case yaml_BLOCK_ENTRY_TOKEN: + return "yaml_BLOCK_ENTRY_TOKEN" + case yaml_FLOW_ENTRY_TOKEN: + return "yaml_FLOW_ENTRY_TOKEN" + case yaml_KEY_TOKEN: + return "yaml_KEY_TOKEN" + case yaml_VALUE_TOKEN: + return "yaml_VALUE_TOKEN" + case yaml_ALIAS_TOKEN: + return "yaml_ALIAS_TOKEN" + case yaml_ANCHOR_TOKEN: + return "yaml_ANCHOR_TOKEN" + case yaml_TAG_TOKEN: + return "yaml_TAG_TOKEN" + case yaml_SCALAR_TOKEN: + return "yaml_SCALAR_TOKEN" + } + return "" +} + +// The token structure. +type yaml_token_t struct { + // The token type. + typ yaml_token_type_t + + // The start/end of the token. + start_mark, end_mark yaml_mark_t + + // The stream encoding (for yaml_STREAM_START_TOKEN). + encoding yaml_encoding_t + + // The alias/anchor/scalar value or tag/tag directive handle + // (for yaml_ALIAS_TOKEN, yaml_ANCHOR_TOKEN, yaml_SCALAR_TOKEN, yaml_TAG_TOKEN, yaml_TAG_DIRECTIVE_TOKEN). + value []byte + + // The tag suffix (for yaml_TAG_TOKEN). + suffix []byte + + // The tag directive prefix (for yaml_TAG_DIRECTIVE_TOKEN). + prefix []byte + + // The scalar style (for yaml_SCALAR_TOKEN). + style yaml_scalar_style_t + + // The version directive major/minor (for yaml_VERSION_DIRECTIVE_TOKEN). + major, minor int8 +} + +// Events + +type yaml_event_type_t int8 + +// Event types. +const ( + // An empty event. + yaml_NO_EVENT yaml_event_type_t = iota + + yaml_STREAM_START_EVENT // A STREAM-START event. + yaml_STREAM_END_EVENT // A STREAM-END event. + yaml_DOCUMENT_START_EVENT // A DOCUMENT-START event. + yaml_DOCUMENT_END_EVENT // A DOCUMENT-END event. + yaml_ALIAS_EVENT // An ALIAS event. + yaml_SCALAR_EVENT // A SCALAR event. + yaml_SEQUENCE_START_EVENT // A SEQUENCE-START event. + yaml_SEQUENCE_END_EVENT // A SEQUENCE-END event. + yaml_MAPPING_START_EVENT // A MAPPING-START event. + yaml_MAPPING_END_EVENT // A MAPPING-END event. + yaml_TAIL_COMMENT_EVENT +) + +var eventStrings = []string{ + yaml_NO_EVENT: "none", + yaml_STREAM_START_EVENT: "stream start", + yaml_STREAM_END_EVENT: "stream end", + yaml_DOCUMENT_START_EVENT: "document start", + yaml_DOCUMENT_END_EVENT: "document end", + yaml_ALIAS_EVENT: "alias", + yaml_SCALAR_EVENT: "scalar", + yaml_SEQUENCE_START_EVENT: "sequence start", + yaml_SEQUENCE_END_EVENT: "sequence end", + yaml_MAPPING_START_EVENT: "mapping start", + yaml_MAPPING_END_EVENT: "mapping end", + yaml_TAIL_COMMENT_EVENT: "tail comment", +} + +func (e yaml_event_type_t) String() string { + if e < 0 || int(e) >= len(eventStrings) { + return fmt.Sprintf("unknown event %d", e) + } + return eventStrings[e] +} + +// The event structure. +type yaml_event_t struct { + + // The event type. + typ yaml_event_type_t + + // The start and end of the event. + start_mark, end_mark yaml_mark_t + + // The document encoding (for yaml_STREAM_START_EVENT). + encoding yaml_encoding_t + + // The version directive (for yaml_DOCUMENT_START_EVENT). + version_directive *yaml_version_directive_t + + // The list of tag directives (for yaml_DOCUMENT_START_EVENT). + tag_directives []yaml_tag_directive_t + + // The comments + head_comment []byte + line_comment []byte + foot_comment []byte + tail_comment []byte + + // The anchor (for yaml_SCALAR_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT, yaml_ALIAS_EVENT). + anchor []byte + + // The tag (for yaml_SCALAR_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT). + tag []byte + + // The scalar value (for yaml_SCALAR_EVENT). + value []byte + + // Is the document start/end indicator implicit, or the tag optional? + // (for yaml_DOCUMENT_START_EVENT, yaml_DOCUMENT_END_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT, yaml_SCALAR_EVENT). + implicit bool + + // Is the tag optional for any non-plain style? (for yaml_SCALAR_EVENT). + quoted_implicit bool + + // The style (for yaml_SCALAR_EVENT, yaml_SEQUENCE_START_EVENT, yaml_MAPPING_START_EVENT). + style yaml_style_t +} + +func (e *yaml_event_t) scalar_style() yaml_scalar_style_t { return yaml_scalar_style_t(e.style) } +func (e *yaml_event_t) sequence_style() yaml_sequence_style_t { return yaml_sequence_style_t(e.style) } +func (e *yaml_event_t) mapping_style() yaml_mapping_style_t { return yaml_mapping_style_t(e.style) } + +// Nodes + +const ( + yaml_NULL_TAG = "tag:yaml.org,2002:null" // The tag !!null with the only possible value: null. + yaml_BOOL_TAG = "tag:yaml.org,2002:bool" // The tag !!bool with the values: true and false. + yaml_STR_TAG = "tag:yaml.org,2002:str" // The tag !!str for string values. + yaml_INT_TAG = "tag:yaml.org,2002:int" // The tag !!int for integer values. + yaml_FLOAT_TAG = "tag:yaml.org,2002:float" // The tag !!float for float values. + yaml_TIMESTAMP_TAG = "tag:yaml.org,2002:timestamp" // The tag !!timestamp for date and time values. + + yaml_SEQ_TAG = "tag:yaml.org,2002:seq" // The tag !!seq is used to denote sequences. + yaml_MAP_TAG = "tag:yaml.org,2002:map" // The tag !!map is used to denote mapping. + + // Not in original libyaml. + yaml_BINARY_TAG = "tag:yaml.org,2002:binary" + yaml_MERGE_TAG = "tag:yaml.org,2002:merge" + + yaml_DEFAULT_SCALAR_TAG = yaml_STR_TAG // The default scalar tag is !!str. + yaml_DEFAULT_SEQUENCE_TAG = yaml_SEQ_TAG // The default sequence tag is !!seq. + yaml_DEFAULT_MAPPING_TAG = yaml_MAP_TAG // The default mapping tag is !!map. +) + +type yaml_node_type_t int + +// Node types. +const ( + // An empty node. + yaml_NO_NODE yaml_node_type_t = iota + + yaml_SCALAR_NODE // A scalar node. + yaml_SEQUENCE_NODE // A sequence node. + yaml_MAPPING_NODE // A mapping node. +) + +// An element of a sequence node. +type yaml_node_item_t int + +// An element of a mapping node. +type yaml_node_pair_t struct { + key int // The key of the element. + value int // The value of the element. +} + +// The node structure. +type yaml_node_t struct { + typ yaml_node_type_t // The node type. + tag []byte // The node tag. + + // The node data. + + // The scalar parameters (for yaml_SCALAR_NODE). + scalar struct { + value []byte // The scalar value. + length int // The length of the scalar value. + style yaml_scalar_style_t // The scalar style. + } + + // The sequence parameters (for YAML_SEQUENCE_NODE). + sequence struct { + items_data []yaml_node_item_t // The stack of sequence items. + style yaml_sequence_style_t // The sequence style. + } + + // The mapping parameters (for yaml_MAPPING_NODE). + mapping struct { + pairs_data []yaml_node_pair_t // The stack of mapping pairs (key, value). + pairs_start *yaml_node_pair_t // The beginning of the stack. + pairs_end *yaml_node_pair_t // The end of the stack. + pairs_top *yaml_node_pair_t // The top of the stack. + style yaml_mapping_style_t // The mapping style. + } + + start_mark yaml_mark_t // The beginning of the node. + end_mark yaml_mark_t // The end of the node. + +} + +// The document structure. +type yaml_document_t struct { + + // The document nodes. + nodes []yaml_node_t + + // The version directive. + version_directive *yaml_version_directive_t + + // The list of tag directives. + tag_directives_data []yaml_tag_directive_t + tag_directives_start int // The beginning of the tag directives list. + tag_directives_end int // The end of the tag directives list. + + start_implicit int // Is the document start indicator implicit? + end_implicit int // Is the document end indicator implicit? + + // The start/end of the document. + start_mark, end_mark yaml_mark_t +} + +// The prototype of a read handler. +// +// The read handler is called when the parser needs to read more bytes from the +// source. The handler should write not more than size bytes to the buffer. +// The number of written bytes should be set to the size_read variable. +// +// [in,out] data A pointer to an application data specified by +// yaml_parser_set_input(). +// [out] buffer The buffer to write the data from the source. +// [in] size The size of the buffer. +// [out] size_read The actual number of bytes read from the source. +// +// On success, the handler should return 1. If the handler failed, +// the returned value should be 0. On EOF, the handler should set the +// size_read to 0 and return 1. +type yaml_read_handler_t func(parser *yaml_parser_t, buffer []byte) (n int, err error) + +// This structure holds information about a potential simple key. +type yaml_simple_key_t struct { + possible bool // Is a simple key possible? + required bool // Is a simple key required? + token_number int // The number of the token. + mark yaml_mark_t // The position mark. +} + +// The states of the parser. +type yaml_parser_state_t int + +const ( + yaml_PARSE_STREAM_START_STATE yaml_parser_state_t = iota + + yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE // Expect the beginning of an implicit document. + yaml_PARSE_DOCUMENT_START_STATE // Expect DOCUMENT-START. + yaml_PARSE_DOCUMENT_CONTENT_STATE // Expect the content of a document. + yaml_PARSE_DOCUMENT_END_STATE // Expect DOCUMENT-END. + yaml_PARSE_BLOCK_NODE_STATE // Expect a block node. + yaml_PARSE_BLOCK_NODE_OR_INDENTLESS_SEQUENCE_STATE // Expect a block node or indentless sequence. + yaml_PARSE_FLOW_NODE_STATE // Expect a flow node. + yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE // Expect the first entry of a block sequence. + yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE // Expect an entry of a block sequence. + yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE // Expect an entry of an indentless sequence. + yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE // Expect the first key of a block mapping. + yaml_PARSE_BLOCK_MAPPING_KEY_STATE // Expect a block mapping key. + yaml_PARSE_BLOCK_MAPPING_VALUE_STATE // Expect a block mapping value. + yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE // Expect the first entry of a flow sequence. + yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE // Expect an entry of a flow sequence. + yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE // Expect a key of an ordered mapping. + yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE // Expect a value of an ordered mapping. + yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE // Expect the and of an ordered mapping entry. + yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE // Expect the first key of a flow mapping. + yaml_PARSE_FLOW_MAPPING_KEY_STATE // Expect a key of a flow mapping. + yaml_PARSE_FLOW_MAPPING_VALUE_STATE // Expect a value of a flow mapping. + yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE // Expect an empty value of a flow mapping. + yaml_PARSE_END_STATE // Expect nothing. +) + +func (ps yaml_parser_state_t) String() string { + switch ps { + case yaml_PARSE_STREAM_START_STATE: + return "yaml_PARSE_STREAM_START_STATE" + case yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE: + return "yaml_PARSE_IMPLICIT_DOCUMENT_START_STATE" + case yaml_PARSE_DOCUMENT_START_STATE: + return "yaml_PARSE_DOCUMENT_START_STATE" + case yaml_PARSE_DOCUMENT_CONTENT_STATE: + return "yaml_PARSE_DOCUMENT_CONTENT_STATE" + case yaml_PARSE_DOCUMENT_END_STATE: + return "yaml_PARSE_DOCUMENT_END_STATE" + case yaml_PARSE_BLOCK_NODE_STATE: + return "yaml_PARSE_BLOCK_NODE_STATE" + case yaml_PARSE_BLOCK_NODE_OR_INDENTLESS_SEQUENCE_STATE: + return "yaml_PARSE_BLOCK_NODE_OR_INDENTLESS_SEQUENCE_STATE" + case yaml_PARSE_FLOW_NODE_STATE: + return "yaml_PARSE_FLOW_NODE_STATE" + case yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE: + return "yaml_PARSE_BLOCK_SEQUENCE_FIRST_ENTRY_STATE" + case yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE: + return "yaml_PARSE_BLOCK_SEQUENCE_ENTRY_STATE" + case yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE: + return "yaml_PARSE_INDENTLESS_SEQUENCE_ENTRY_STATE" + case yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE: + return "yaml_PARSE_BLOCK_MAPPING_FIRST_KEY_STATE" + case yaml_PARSE_BLOCK_MAPPING_KEY_STATE: + return "yaml_PARSE_BLOCK_MAPPING_KEY_STATE" + case yaml_PARSE_BLOCK_MAPPING_VALUE_STATE: + return "yaml_PARSE_BLOCK_MAPPING_VALUE_STATE" + case yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_FIRST_ENTRY_STATE" + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_ENTRY_STATE" + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_KEY_STATE" + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE_STATE" + case yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE: + return "yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE" + case yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE: + return "yaml_PARSE_FLOW_MAPPING_FIRST_KEY_STATE" + case yaml_PARSE_FLOW_MAPPING_KEY_STATE: + return "yaml_PARSE_FLOW_MAPPING_KEY_STATE" + case yaml_PARSE_FLOW_MAPPING_VALUE_STATE: + return "yaml_PARSE_FLOW_MAPPING_VALUE_STATE" + case yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE: + return "yaml_PARSE_FLOW_MAPPING_EMPTY_VALUE_STATE" + case yaml_PARSE_END_STATE: + return "yaml_PARSE_END_STATE" + } + return "" +} + +// This structure holds aliases data. +type yaml_alias_data_t struct { + anchor []byte // The anchor. + index int // The node id. + mark yaml_mark_t // The anchor mark. +} + +// The parser structure. +// +// All members are internal. Manage the structure using the +// yaml_parser_ family of functions. +type yaml_parser_t struct { + + // Error handling + + error yaml_error_type_t // Error type. + + problem string // Error description. + + // The byte about which the problem occurred. + problem_offset int + problem_value int + problem_mark yaml_mark_t + + // The error context. + context string + context_mark yaml_mark_t + + // Reader stuff + + read_handler yaml_read_handler_t // Read handler. + + input_reader io.Reader // File input data. + input []byte // String input data. + input_pos int + + eof bool // EOF flag + + buffer []byte // The working buffer. + buffer_pos int // The current position of the buffer. + + unread int // The number of unread characters in the buffer. + + newlines int // The number of line breaks since last non-break/non-blank character + + raw_buffer []byte // The raw buffer. + raw_buffer_pos int // The current position of the buffer. + + encoding yaml_encoding_t // The input encoding. + + offset int // The offset of the current position (in bytes). + mark yaml_mark_t // The mark of the current position. + + // Comments + + head_comment []byte // The current head comments + line_comment []byte // The current line comments + foot_comment []byte // The current foot comments + tail_comment []byte // Foot comment that happens at the end of a block. + stem_comment []byte // Comment in item preceding a nested structure (list inside list item, etc) + + comments []yaml_comment_t // The folded comments for all parsed tokens + comments_head int + + // Scanner stuff + + stream_start_produced bool // Have we started to scan the input stream? + stream_end_produced bool // Have we reached the end of the input stream? + + flow_level int // The number of unclosed '[' and '{' indicators. + + tokens []yaml_token_t // The tokens queue. + tokens_head int // The head of the tokens queue. + tokens_parsed int // The number of tokens fetched from the queue. + token_available bool // Does the tokens queue contain a token ready for dequeueing. + + indent int // The current indentation level. + indents []int // The indentation levels stack. + + simple_key_allowed bool // May a simple key occur at the current position? + simple_keys []yaml_simple_key_t // The stack of simple keys. + simple_keys_by_tok map[int]int // possible simple_key indexes indexed by token_number + + // Parser stuff + + state yaml_parser_state_t // The current parser state. + states []yaml_parser_state_t // The parser states stack. + marks []yaml_mark_t // The stack of marks. + tag_directives []yaml_tag_directive_t // The list of TAG directives. + + // Dumper stuff + + aliases []yaml_alias_data_t // The alias data. + + document *yaml_document_t // The currently parsed document. +} + +type yaml_comment_t struct { + + scan_mark yaml_mark_t // Position where scanning for comments started + token_mark yaml_mark_t // Position after which tokens will be associated with this comment + start_mark yaml_mark_t // Position of '#' comment mark + end_mark yaml_mark_t // Position where comment terminated + + head []byte + line []byte + foot []byte +} + +// Emitter Definitions + +// The prototype of a write handler. +// +// The write handler is called when the emitter needs to flush the accumulated +// characters to the output. The handler should write @a size bytes of the +// @a buffer to the output. +// +// @param[in,out] data A pointer to an application data specified by +// yaml_emitter_set_output(). +// @param[in] buffer The buffer with bytes to be written. +// @param[in] size The size of the buffer. +// +// @returns On success, the handler should return @c 1. If the handler failed, +// the returned value should be @c 0. +// +type yaml_write_handler_t func(emitter *yaml_emitter_t, buffer []byte) error + +type yaml_emitter_state_t int + +// The emitter states. +const ( + // Expect STREAM-START. + yaml_EMIT_STREAM_START_STATE yaml_emitter_state_t = iota + + yaml_EMIT_FIRST_DOCUMENT_START_STATE // Expect the first DOCUMENT-START or STREAM-END. + yaml_EMIT_DOCUMENT_START_STATE // Expect DOCUMENT-START or STREAM-END. + yaml_EMIT_DOCUMENT_CONTENT_STATE // Expect the content of a document. + yaml_EMIT_DOCUMENT_END_STATE // Expect DOCUMENT-END. + yaml_EMIT_FLOW_SEQUENCE_FIRST_ITEM_STATE // Expect the first item of a flow sequence. + yaml_EMIT_FLOW_SEQUENCE_TRAIL_ITEM_STATE // Expect the next item of a flow sequence, with the comma already written out + yaml_EMIT_FLOW_SEQUENCE_ITEM_STATE // Expect an item of a flow sequence. + yaml_EMIT_FLOW_MAPPING_FIRST_KEY_STATE // Expect the first key of a flow mapping. + yaml_EMIT_FLOW_MAPPING_TRAIL_KEY_STATE // Expect the next key of a flow mapping, with the comma already written out + yaml_EMIT_FLOW_MAPPING_KEY_STATE // Expect a key of a flow mapping. + yaml_EMIT_FLOW_MAPPING_SIMPLE_VALUE_STATE // Expect a value for a simple key of a flow mapping. + yaml_EMIT_FLOW_MAPPING_VALUE_STATE // Expect a value of a flow mapping. + yaml_EMIT_BLOCK_SEQUENCE_FIRST_ITEM_STATE // Expect the first item of a block sequence. + yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE // Expect an item of a block sequence. + yaml_EMIT_BLOCK_MAPPING_FIRST_KEY_STATE // Expect the first key of a block mapping. + yaml_EMIT_BLOCK_MAPPING_KEY_STATE // Expect the key of a block mapping. + yaml_EMIT_BLOCK_MAPPING_SIMPLE_VALUE_STATE // Expect a value for a simple key of a block mapping. + yaml_EMIT_BLOCK_MAPPING_VALUE_STATE // Expect a value of a block mapping. + yaml_EMIT_END_STATE // Expect nothing. +) + +// The emitter structure. +// +// All members are internal. Manage the structure using the @c yaml_emitter_ +// family of functions. +type yaml_emitter_t struct { + + // Error handling + + error yaml_error_type_t // Error type. + problem string // Error description. + + // Writer stuff + + write_handler yaml_write_handler_t // Write handler. + + output_buffer *[]byte // String output data. + output_writer io.Writer // File output data. + + buffer []byte // The working buffer. + buffer_pos int // The current position of the buffer. + + raw_buffer []byte // The raw buffer. + raw_buffer_pos int // The current position of the buffer. + + encoding yaml_encoding_t // The stream encoding. + + // Emitter stuff + + canonical bool // If the output is in the canonical style? + best_indent int // The number of indentation spaces. + best_width int // The preferred width of the output lines. + unicode bool // Allow unescaped non-ASCII characters? + line_break yaml_break_t // The preferred line break. + + state yaml_emitter_state_t // The current emitter state. + states []yaml_emitter_state_t // The stack of states. + + events []yaml_event_t // The event queue. + events_head int // The head of the event queue. + + indents []int // The stack of indentation levels. + + tag_directives []yaml_tag_directive_t // The list of tag directives. + + indent int // The current indentation level. + + flow_level int // The current flow level. + + root_context bool // Is it the document root context? + sequence_context bool // Is it a sequence context? + mapping_context bool // Is it a mapping context? + simple_key_context bool // Is it a simple mapping key context? + + line int // The current line. + column int // The current column. + whitespace bool // If the last character was a whitespace? + indention bool // If the last character was an indentation character (' ', '-', '?', ':')? + open_ended bool // If an explicit document end is required? + + space_above bool // Is there's an empty line above? + foot_indent int // The indent used to write the foot comment above, or -1 if none. + + // Anchor analysis. + anchor_data struct { + anchor []byte // The anchor value. + alias bool // Is it an alias? + } + + // Tag analysis. + tag_data struct { + handle []byte // The tag handle. + suffix []byte // The tag suffix. + } + + // Scalar analysis. + scalar_data struct { + value []byte // The scalar value. + multiline bool // Does the scalar contain line breaks? + flow_plain_allowed bool // Can the scalar be expessed in the flow plain style? + block_plain_allowed bool // Can the scalar be expressed in the block plain style? + single_quoted_allowed bool // Can the scalar be expressed in the single quoted style? + block_allowed bool // Can the scalar be expressed in the literal or folded styles? + style yaml_scalar_style_t // The output style. + } + + // Comments + head_comment []byte + line_comment []byte + foot_comment []byte + tail_comment []byte + + // Dumper stuff + + opened bool // If the stream was already opened? + closed bool // If the stream was already closed? + + // The information associated with the document nodes. + anchors *struct { + references int // The number of references. + anchor int // The anchor id. + serialized bool // If the node has been emitted? + } + + last_anchor_id int // The last assigned anchor id. + + document *yaml_document_t // The currently emitted document. +} diff --git a/gopkg.in/yaml.v3/yamlprivateh.go b/gopkg.in/yaml.v3/yamlprivateh.go new file mode 100644 index 0000000000..e88f9c54ae --- /dev/null +++ b/gopkg.in/yaml.v3/yamlprivateh.go @@ -0,0 +1,198 @@ +// +// Copyright (c) 2011-2019 Canonical Ltd +// Copyright (c) 2006-2010 Kirill Simonov +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +// of the Software, and to permit persons to whom the Software is furnished to do +// so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package yaml + +const ( + // The size of the input raw buffer. + input_raw_buffer_size = 512 + + // The size of the input buffer. + // It should be possible to decode the whole raw buffer. + input_buffer_size = input_raw_buffer_size * 3 + + // The size of the output buffer. + output_buffer_size = 128 + + // The size of the output raw buffer. + // It should be possible to encode the whole output buffer. + output_raw_buffer_size = (output_buffer_size*2 + 2) + + // The size of other stacks and queues. + initial_stack_size = 16 + initial_queue_size = 16 + initial_string_size = 16 +) + +// Check if the character at the specified position is an alphabetical +// character, a digit, '_', or '-'. +func is_alpha(b []byte, i int) bool { + return b[i] >= '0' && b[i] <= '9' || b[i] >= 'A' && b[i] <= 'Z' || b[i] >= 'a' && b[i] <= 'z' || b[i] == '_' || b[i] == '-' +} + +// Check if the character at the specified position is a digit. +func is_digit(b []byte, i int) bool { + return b[i] >= '0' && b[i] <= '9' +} + +// Get the value of a digit. +func as_digit(b []byte, i int) int { + return int(b[i]) - '0' +} + +// Check if the character at the specified position is a hex-digit. +func is_hex(b []byte, i int) bool { + return b[i] >= '0' && b[i] <= '9' || b[i] >= 'A' && b[i] <= 'F' || b[i] >= 'a' && b[i] <= 'f' +} + +// Get the value of a hex-digit. +func as_hex(b []byte, i int) int { + bi := b[i] + if bi >= 'A' && bi <= 'F' { + return int(bi) - 'A' + 10 + } + if bi >= 'a' && bi <= 'f' { + return int(bi) - 'a' + 10 + } + return int(bi) - '0' +} + +// Check if the character is ASCII. +func is_ascii(b []byte, i int) bool { + return b[i] <= 0x7F +} + +// Check if the character at the start of the buffer can be printed unescaped. +func is_printable(b []byte, i int) bool { + return ((b[i] == 0x0A) || // . == #x0A + (b[i] >= 0x20 && b[i] <= 0x7E) || // #x20 <= . <= #x7E + (b[i] == 0xC2 && b[i+1] >= 0xA0) || // #0xA0 <= . <= #xD7FF + (b[i] > 0xC2 && b[i] < 0xED) || + (b[i] == 0xED && b[i+1] < 0xA0) || + (b[i] == 0xEE) || + (b[i] == 0xEF && // #xE000 <= . <= #xFFFD + !(b[i+1] == 0xBB && b[i+2] == 0xBF) && // && . != #xFEFF + !(b[i+1] == 0xBF && (b[i+2] == 0xBE || b[i+2] == 0xBF)))) +} + +// Check if the character at the specified position is NUL. +func is_z(b []byte, i int) bool { + return b[i] == 0x00 +} + +// Check if the beginning of the buffer is a BOM. +func is_bom(b []byte, i int) bool { + return b[0] == 0xEF && b[1] == 0xBB && b[2] == 0xBF +} + +// Check if the character at the specified position is space. +func is_space(b []byte, i int) bool { + return b[i] == ' ' +} + +// Check if the character at the specified position is tab. +func is_tab(b []byte, i int) bool { + return b[i] == '\t' +} + +// Check if the character at the specified position is blank (space or tab). +func is_blank(b []byte, i int) bool { + //return is_space(b, i) || is_tab(b, i) + return b[i] == ' ' || b[i] == '\t' +} + +// Check if the character at the specified position is a line break. +func is_break(b []byte, i int) bool { + return (b[i] == '\r' || // CR (#xD) + b[i] == '\n' || // LF (#xA) + b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9) // PS (#x2029) +} + +func is_crlf(b []byte, i int) bool { + return b[i] == '\r' && b[i+1] == '\n' +} + +// Check if the character is a line break or NUL. +func is_breakz(b []byte, i int) bool { + //return is_break(b, i) || is_z(b, i) + return ( + // is_break: + b[i] == '\r' || // CR (#xD) + b[i] == '\n' || // LF (#xA) + b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9 || // PS (#x2029) + // is_z: + b[i] == 0) +} + +// Check if the character is a line break, space, or NUL. +func is_spacez(b []byte, i int) bool { + //return is_space(b, i) || is_breakz(b, i) + return ( + // is_space: + b[i] == ' ' || + // is_breakz: + b[i] == '\r' || // CR (#xD) + b[i] == '\n' || // LF (#xA) + b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9 || // PS (#x2029) + b[i] == 0) +} + +// Check if the character is a line break, space, tab, or NUL. +func is_blankz(b []byte, i int) bool { + //return is_blank(b, i) || is_breakz(b, i) + return ( + // is_blank: + b[i] == ' ' || b[i] == '\t' || + // is_breakz: + b[i] == '\r' || // CR (#xD) + b[i] == '\n' || // LF (#xA) + b[i] == 0xC2 && b[i+1] == 0x85 || // NEL (#x85) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA8 || // LS (#x2028) + b[i] == 0xE2 && b[i+1] == 0x80 && b[i+2] == 0xA9 || // PS (#x2029) + b[i] == 0) +} + +// Determine the width of the character. +func width(b byte) int { + // Don't replace these by a switch without first + // confirming that it is being inlined. + if b&0x80 == 0x00 { + return 1 + } + if b&0xE0 == 0xC0 { + return 2 + } + if b&0xF0 == 0xE0 { + return 3 + } + if b&0xF8 == 0xF0 { + return 4 + } + return 0 + +} diff --git a/modules.txt b/modules.txt index 7071ed3265..6fe3fdc955 100644 --- a/modules.txt +++ b/modules.txt @@ -526,6 +526,9 @@ github.com/jcmturner/gofork/encoding/asn1 github.com/jcmturner/gofork/x/crypto/pbkdf2 # github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af github.com/jmespath/go-jmespath +# github.com/jordanlewis/gcassert v0.0.0-20200630054945-8ff878e72753 +## explicit +github.com/jordanlewis/gcassert # github.com/kevinburke/go-bindata v3.13.0+incompatible ## explicit github.com/kevinburke/go-bindata @@ -724,7 +727,7 @@ github.com/spf13/cobra/doc # github.com/spf13/pflag v1.0.5 ## explicit github.com/spf13/pflag -# github.com/stretchr/testify v1.5.1 +# github.com/stretchr/testify v1.6.1 ## explicit github.com/stretchr/testify/assert github.com/stretchr/testify/require @@ -799,6 +802,9 @@ golang.org/x/exp/rand ## explicit golang.org/x/lint golang.org/x/lint/golint +# golang.org/x/mod v0.3.0 +## explicit +golang.org/x/mod/semver # golang.org/x/net v0.0.0-20200602114024-627f9648deb9 ## explicit golang.org/x/net/context @@ -856,7 +862,7 @@ golang.org/x/text/width # golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 ## explicit golang.org/x/time/rate -# golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5 +# golang.org/x/tools v0.0.0-20200626171337-aa94e735be7f ## explicit golang.org/x/tools/cmd/goyacc golang.org/x/tools/cmd/stringer @@ -916,6 +922,7 @@ golang.org/x/tools/internal/lsp/diff/myers golang.org/x/tools/internal/packagesinternal golang.org/x/tools/internal/span golang.org/x/tools/internal/testenv +golang.org/x/tools/internal/typesinternal golang.org/x/tools/txtar # golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 golang.org/x/xerrors @@ -1077,6 +1084,8 @@ gopkg.in/jcmturner/rpc.v1/ndr # gopkg.in/yaml.v2 v2.3.0 => github.com/cockroachdb/yaml v0.0.0-20180705215940-0e2822948641 ## explicit gopkg.in/yaml.v2 +# gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c +gopkg.in/yaml.v3 # honnef.co/go/tools v0.0.0-20190530104931-1f0868a609b7 ## explicit honnef.co/go/tools/arg From e32fcdc83f80cfd8fe9973ce829af400ee129fcb Mon Sep 17 00:00:00 2001 From: Jackson Owens Date: Wed, 1 Jul 2020 13:35:25 -0400 Subject: [PATCH 05/49] bump Pebble to d845c8c2d870 7cbb7d6 internal/rate: deflake TestLongRunningQPS 06feeaa db: bump nextFileNum past obsolete file numbers on Open fc51635 build: rework build rules to remove redundancy 839ee18 build: replace GO111MODULE=off with -mod=vendor 133d4e3 cmd/pebble: output a Go-bench format summary line for `bench ycsb` daeb562 db: add Metrics.Total and Iterator.Metrics c459622 db: fix DeleteRangeFlushDelay for batches edited with SetRepr, Apply 931bab4 travis: build all variations of go1.14 --- github.com/cockroachdb/pebble/.travis.yml | 29 +++++++++++------------ github.com/cockroachdb/pebble/Makefile | 29 ++++++++++++----------- github.com/cockroachdb/pebble/batch.go | 9 ++++++- github.com/cockroachdb/pebble/go.sum | 11 --------- github.com/cockroachdb/pebble/iterator.go | 20 ++++++++++++++++ github.com/cockroachdb/pebble/metrics.go | 21 +++++++++++++++- github.com/cockroachdb/pebble/open.go | 7 ++++++ modules.txt | 2 +- 8 files changed, 85 insertions(+), 43 deletions(-) diff --git a/github.com/cockroachdb/pebble/.travis.yml b/github.com/cockroachdb/pebble/.travis.yml index 4fae68c91b..b5ff32883b 100644 --- a/github.com/cockroachdb/pebble/.travis.yml +++ b/github.com/cockroachdb/pebble/.travis.yml @@ -5,7 +5,6 @@ branches: - master install: - - go get -d -t -v ./... - go get -v golang.org/x/lint/golint matrix: @@ -13,32 +12,32 @@ matrix: - name: "go1.13.x-linux" go: 1.13.x os: linux + script: make test + - name: "go1.14.x-linux" + go: 1.14.x + os: linux script: make test generate - - name: "go1.13.x-linux-race" - go: 1.13.x + - name: "go1.14.x-linux-race" + go: 1.14.x os: linux script: make testrace TAGS= - - name: "go1.13.x-linux-no-invariants" - go: 1.13.x + - name: "go1.14.x-linux-no-invariants" + go: 1.14.x os: linux script: make test TAGS= - - name: "go1.13.x-darwin" - go: 1.13.x + - name: "go1.14.x-darwin" + go: 1.14.x os: osx script: make test - - name: "go1.13.x-windows" - go: 1.13.x + - name: "go1.14.x-windows" + go: 1.14.x os: windows script: go test ./... - - name: "go1.13.x-freebsd" - go: 1.13.x + - name: "go1.14.x-freebsd" + go: 1.14.x os: linux # NB: "env: GOOS=freebsd" does not have the desired effect. script: GOOS=freebsd go build -v ./... - - name: "go1.14.x-linux" - go: 1.14.x - os: linux - script: make test notifications: email: diff --git a/github.com/cockroachdb/pebble/Makefile b/github.com/cockroachdb/pebble/Makefile index 88192e7ee4..32d3dd4dfc 100644 --- a/github.com/cockroachdb/pebble/Makefile +++ b/github.com/cockroachdb/pebble/Makefile @@ -1,6 +1,6 @@ GO := go -GOFLAGS := PKG := ./... +GOFLAGS := STRESSFLAGS := TAGS := invariants TESTS := . @@ -16,29 +16,29 @@ all: @echo " make mod-update" @echo " make clean" +override testflags := .PHONY: test test: - GO111MODULE=off ${GO} test -tags '$(TAGS)' ${GOFLAGS} -run ${TESTS} ${PKG} + ${GO} test -mod=vendor -tags '$(TAGS)' ${testflags} -run ${TESTS} ${PKG} .PHONY: testrace -testrace: GOFLAGS += -race +testrace: testflags += -race testrace: test .PHONY: stress stressrace -stressrace: GOFLAGS += -race -stress stressrace: - GO111MODULE=off ${GO} test -v -tags '$(TAGS)' ${GOFLAGS} -exec 'stress ${STRESSFLAGS}' -run '${TESTS}' -timeout 0 ${PKG} +stressrace: testflags += -race +stress stressrace: testflags += -exec 'stress ${STRESSFLAGS}' -timeout 0 +stress stressrace: test .PHONY: stressmeta -stressmeta: PKG = ./internal/metamorphic -stressmeta: STRESSFLAGS += -p 1 -stressmeta: TESTS = TestMeta$$ -stressmeta: - GO111MODULE=off ${GO} test -v -tags '$(TAGS)' ${GOFLAGS} -exec 'stress ${STRESSFLAGS}' -run '${TESTS}' -timeout 0 ${PKG} +stressmeta: override PKG = ./internal/metamorphic +stressmeta: override STRESSFLAGS += -p 1 +stressmeta: override TESTS = TestMeta$$ +stressmeta: stress .PHONY: generate generate: - GO111MODULE=off ${GO} generate ${PKG} + ${GO} generate -mod=vendor ${PKG} # The cmd/pebble/{badger,boltdb,rocksdb}.go files causes various # cockroach dependencies to be pulled in which is undesirable. Hack @@ -46,8 +46,9 @@ generate: mod-update: mkdir -p cmd/pebble/_bak mv cmd/pebble/{badger,boltdb,rocksdb}.go cmd/pebble/_bak - GO111MODULE=on ${GO} get -u - GO111MODULE=on ${GO} mod vendor + ${GO} get -u + ${GO} mod tidy + ${GO} mod vendor mv cmd/pebble/_bak/* cmd/pebble && rmdir cmd/pebble/_bak .PHONY: clean diff --git a/github.com/cockroachdb/pebble/batch.go b/github.com/cockroachdb/pebble/batch.go index 8b415e79a8..3020ac8b39 100644 --- a/github.com/cockroachdb/pebble/batch.go +++ b/github.com/cockroachdb/pebble/batch.go @@ -300,12 +300,16 @@ func (b *Batch) refreshMemTableSize() { return } + b.countRangeDels = 0 for r := b.Reader(); ; { - _, key, value, ok := r.Next() + kind, key, value, ok := r.Next() if !ok { break } b.memTableSize += memTableEntrySize(len(key), len(value)) + if kind == InternalKeyKindRangeDelete { + b.countRangeDels++ + } } } @@ -338,6 +342,9 @@ func (b *Batch) Apply(batch *Batch, _ *WriteOptions) error { if !ok { break } + if kind == InternalKeyKindRangeDelete { + b.countRangeDels++ + } if b.index != nil { var err error if kind == InternalKeyKindRangeDelete { diff --git a/github.com/cockroachdb/pebble/go.sum b/github.com/cockroachdb/pebble/go.sum index b067e9e17c..d6d6767709 100644 --- a/github.com/cockroachdb/pebble/go.sum +++ b/github.com/cockroachdb/pebble/go.sum @@ -17,10 +17,6 @@ github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgoo github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf h1:gFVkHXmVAhEbxZVDln5V9GKrLaluNoFHDbrZwAWZgws= github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= @@ -45,30 +41,23 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190426190305-956cc1757749 h1:Bduxdpx1O6126WsH6F6NwKywZ/FPncphlTduoPxFG78= -golang.org/x/exp v0.0.0-20190426190305-956cc1757749/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20200513190911-00229845015e h1:rMqLP+9XLy+LdbCXHjJHAmTfXCr93W7oruWA6Hq1Alc= golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa h1:KIDDMLT1O0Nr7TSxp8xM5tJcdn8tgyAONntO829og1M= -golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/github.com/cockroachdb/pebble/iterator.go b/github.com/cockroachdb/pebble/iterator.go index 8839e9ea33..82ee35167a 100644 --- a/github.com/cockroachdb/pebble/iterator.go +++ b/github.com/cockroachdb/pebble/iterator.go @@ -21,6 +21,15 @@ const ( var errReversePrefixIteration = errors.New("pebble: unsupported reverse prefix iteration") +// IteratorMetrics holds per-iterator metrics. +type IteratorMetrics struct { + // The read amplification experienced by this iterator. This is the sum of + // the memtables, the L0 sublevels and the non-empty Ln levels. Higher read + // amplification generally results in slower reads, though allowing higher + // read amplification can also result in faster writes. + ReadAmp int +} + // Iterator iterates over a DB's key/value pairs in key order. // // An iterator must be closed after use, but it is not necessary to read an @@ -557,3 +566,14 @@ func (i *Iterator) SetBounds(lower, upper []byte) { i.opts.UpperBound = upper i.iter.SetBounds(lower, upper) } + +// Metrics returns per-iterator metrics. +func (i *Iterator) Metrics() IteratorMetrics { + m := IteratorMetrics{ + ReadAmp: 1, + } + if mi, ok := i.iter.(*mergingIter); ok { + m.ReadAmp = len(mi.levels) + } + return m +} diff --git a/github.com/cockroachdb/pebble/metrics.go b/github.com/cockroachdb/pebble/metrics.go index ada77c7b33..98f6e0d405 100644 --- a/github.com/cockroachdb/pebble/metrics.go +++ b/github.com/cockroachdb/pebble/metrics.go @@ -192,6 +192,25 @@ func (m *Metrics) ReadAmp() int { return int(ramp) } +// Total returns the sum of the per-level metrics and WAL metrics. +func (m *Metrics) Total() LevelMetrics { + var total LevelMetrics + for level := 0; level < numLevels; level++ { + l := &m.Levels[level] + total.Add(l) + total.Sublevels += l.Sublevels + total.NumFiles += l.NumFiles + total.Size += l.Size + } + // Compute total bytes-in as the bytes written to the WAL + bytes ingested. + total.BytesIn = m.WAL.BytesWritten + total.BytesIngested + // Add the total bytes-in to the total bytes-flushed. This is to account for + // the bytes written to the log and bytes written externally and then + // ingested. + total.BytesFlushed += total.BytesIn + return total +} + const notApplicable = "-" func (m *Metrics) formatWAL(buf *bytes.Buffer) { @@ -262,7 +281,7 @@ func (m *Metrics) String() string { total.NumFiles += l.NumFiles total.Size += l.Size } - // Compute total bytes-in as the bytes written to the WAL + bytes ingested + // Compute total bytes-in as the bytes written to the WAL + bytes ingested. total.BytesIn = m.WAL.BytesWritten + total.BytesIngested // Add the total bytes-in to the total bytes-flushed. This is to account for // the bytes written to the log and bytes written externally and then diff --git a/github.com/cockroachdb/pebble/open.go b/github.com/cockroachdb/pebble/open.go index 372bbd4c92..77f72a6c03 100644 --- a/github.com/cockroachdb/pebble/open.go +++ b/github.com/cockroachdb/pebble/open.go @@ -230,6 +230,13 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { if !ok { continue } + + // Don't reuse any obsolete file numbers to avoid modifying an + // ingested sstable's original external file. + if d.mu.versions.nextFileNum <= fn { + d.mu.versions.nextFileNum = fn + 1 + } + switch ft { case fileTypeLog: if fn >= d.mu.versions.minUnflushedLogNum { diff --git a/modules.txt b/modules.txt index 6fe3fdc955..195e33203f 100644 --- a/modules.txt +++ b/modules.txt @@ -224,7 +224,7 @@ github.com/cockroachdb/gostdlib/x/tools/internal/span # github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f ## explicit github.com/cockroachdb/logtags -# github.com/cockroachdb/pebble v0.0.0-20200626145914-3b86a4009c3b +# github.com/cockroachdb/pebble v0.0.0-20200701140535-d845c8c2d870 ## explicit github.com/cockroachdb/pebble github.com/cockroachdb/pebble/bloom From 5e2f2432b758b7273d37caece61c2d2edfad679a Mon Sep 17 00:00:00 2001 From: Oliver Tan Date: Thu, 2 Jul 2020 13:08:32 -0700 Subject: [PATCH 06/49] bump twpayne/go-geom to v1.3.2 --- .../go-geom/encoding/geojson/geojson.go | 47 ++++++++++++++----- github.com/twpayne/go-geom/go.mod | 1 - github.com/twpayne/go-geom/go.sum | 2 - github.com/twpayne/go-geom/point.go | 2 +- modules.txt | 2 +- 5 files changed, 36 insertions(+), 18 deletions(-) diff --git a/github.com/twpayne/go-geom/encoding/geojson/geojson.go b/github.com/twpayne/go-geom/encoding/geojson/geojson.go index 9244728425..dff51bc22c 100644 --- a/github.com/twpayne/go-geom/encoding/geojson/geojson.go +++ b/github.com/twpayne/go-geom/encoding/geojson/geojson.go @@ -44,7 +44,7 @@ type Geometry struct { BBox *json.RawMessage `json:"bbox,omitempty"` CRS *CRS `json:"crs,omitempty"` Coordinates *json.RawMessage `json:"coordinates,omitempty"` - Geometries []*Geometry `json:"geometries,omitempty"` + Geometries *json.RawMessage `json:"geometries,omitempty"` } // A Feature is a GeoJSON Feature. @@ -119,12 +119,15 @@ func (g *Geometry) Decode() (geom.T, error) { switch g.Type { case "Point": if g.Coordinates == nil { - return geom.NewPoint(geom.NoLayout), nil + return geom.NewPointEmpty(geom.NoLayout), nil } var coords geom.Coord if err := json.Unmarshal(*g.Coordinates, &coords); err != nil { return nil, err } + if len(coords) == 0 { + return geom.NewPointEmpty(DefaultLayout), nil + } layout, err := guessLayout0(coords) if err != nil { return nil, err @@ -196,8 +199,15 @@ func (g *Geometry) Decode() (geom.T, error) { } return geom.NewMultiPolygon(layout).SetCoords(coords) case "GeometryCollection": - geoms := make([]geom.T, len(g.Geometries)) - for i, subGeometry := range g.Geometries { + var geometries []Geometry + if g.Geometries != nil { + err := json.Unmarshal(*g.Geometries, &geometries) + if err != nil { + return nil, err + } + } + geoms := make([]geom.T, len(geometries)) + for i, subGeometry := range geometries { var err error geoms[i], err = subGeometry.Decode() if err != nil { @@ -345,13 +355,19 @@ func encode(g geom.T, opts ...EncodeGeometryOption) (*Geometry, error) { switch g := g.(type) { case *geom.Point: var coords json.RawMessage - var coordsIn interface{} = g.Coords() + var coordsIn interface{} + if !g.Empty() { + coordsIn = g.Coords() + } else { + coordsIn = []geom.Coord{} + } for _, opt := range opts { if opt.onFloat64Handler != nil { - coordsIn = opt.onFloat64Handler(g.Coords()) + coordsIn = opt.onFloat64Handler(coordsIn) } } - coords, err := json.Marshal(coordsIn) + var err error + coords, err = json.Marshal(coordsIn) if err != nil { return nil, err } @@ -364,7 +380,7 @@ func encode(g geom.T, opts ...EncodeGeometryOption) (*Geometry, error) { var coordsIn interface{} = g.Coords() for _, opt := range opts { if opt.onFloat64Handler != nil { - coordsIn = opt.onFloat64Handler(g.Coords()) + coordsIn = opt.onFloat64Handler(coordsIn) } } coords, err := json.Marshal(coordsIn) @@ -380,7 +396,7 @@ func encode(g geom.T, opts ...EncodeGeometryOption) (*Geometry, error) { var coordsIn interface{} = g.Coords() for _, opt := range opts { if opt.onFloat64Handler != nil { - coordsIn = opt.onFloat64Handler(g.Coords()) + coordsIn = opt.onFloat64Handler(coordsIn) } } coords, err := json.Marshal(coordsIn) @@ -396,7 +412,7 @@ func encode(g geom.T, opts ...EncodeGeometryOption) (*Geometry, error) { var coordsIn interface{} = g.Coords() for _, opt := range opts { if opt.onFloat64Handler != nil { - coordsIn = opt.onFloat64Handler(g.Coords()) + coordsIn = opt.onFloat64Handler(coordsIn) } } coords, err := json.Marshal(coordsIn) @@ -412,7 +428,7 @@ func encode(g geom.T, opts ...EncodeGeometryOption) (*Geometry, error) { var coordsIn interface{} = g.Coords() for _, opt := range opts { if opt.onFloat64Handler != nil { - coordsIn = opt.onFloat64Handler(g.Coords()) + coordsIn = opt.onFloat64Handler(coordsIn) } } coords, err := json.Marshal(coordsIn) @@ -428,7 +444,7 @@ func encode(g geom.T, opts ...EncodeGeometryOption) (*Geometry, error) { var coordsIn interface{} = g.Coords() for _, opt := range opts { if opt.onFloat64Handler != nil { - coordsIn = opt.onFloat64Handler(g.Coords()) + coordsIn = opt.onFloat64Handler(coordsIn) } } coords, err := json.Marshal(coordsIn) @@ -440,6 +456,7 @@ func encode(g geom.T, opts ...EncodeGeometryOption) (*Geometry, error) { Coordinates: &coords, }, nil case *geom.GeometryCollection: + var marshalledGeometries json.RawMessage geometries := make([]*Geometry, len(g.Geoms())) for i, subGeometry := range g.Geoms() { var err error @@ -448,9 +465,13 @@ func encode(g geom.T, opts ...EncodeGeometryOption) (*Geometry, error) { return nil, err } } + marshalledGeometries, err := json.Marshal(geometries) + if err != nil { + return nil, err + } return &Geometry{ Type: "GeometryCollection", - Geometries: geometries, + Geometries: &marshalledGeometries, }, nil default: return nil, geom.ErrUnsupportedType{Value: g} diff --git a/github.com/twpayne/go-geom/go.mod b/github.com/twpayne/go-geom/go.mod index 7a29d8fde5..3f10af2542 100644 --- a/github.com/twpayne/go-geom/go.mod +++ b/github.com/twpayne/go-geom/go.mod @@ -7,7 +7,6 @@ require ( github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/cenkalti/backoff v2.1.1+incompatible // indirect github.com/containerd/continuity v0.0.0-20181203112020-004b46473808 // indirect - github.com/d4l3k/messagediff v1.2.1 github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.3.3 // indirect github.com/google/go-cmp v0.4.0 // indirect diff --git a/github.com/twpayne/go-geom/go.sum b/github.com/twpayne/go-geom/go.sum index 6c0e9ae454..06df206554 100644 --- a/github.com/twpayne/go-geom/go.sum +++ b/github.com/twpayne/go-geom/go.sum @@ -13,8 +13,6 @@ github.com/cenkalti/backoff v2.1.1+incompatible h1:tKJnvO2kl0zmb/jA5UKAt4VoEVw1q github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/containerd/continuity v0.0.0-20181203112020-004b46473808 h1:4BX8f882bXEDKfWIf0wa8HRvpnBoPszJJXL+TVbBw4M= github.com/containerd/continuity v0.0.0-20181203112020-004b46473808/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U= -github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/github.com/twpayne/go-geom/point.go b/github.com/twpayne/go-geom/point.go index 546d88a128..69dc6d6c41 100644 --- a/github.com/twpayne/go-geom/point.go +++ b/github.com/twpayne/go-geom/point.go @@ -23,7 +23,7 @@ func NewPoint(l Layout) *Point { // NewPointEmpty allocates a new Point with no coordinates. func NewPointEmpty(l Layout) *Point { - return NewPointFlat(l, []float64{}) + return NewPointFlat(l, nil) } // NewPointFlat allocates a new Point with layout l and flat coordinates flatCoords. diff --git a/modules.txt b/modules.txt index 195e33203f..eae06716ac 100644 --- a/modules.txt +++ b/modules.txt @@ -731,7 +731,7 @@ github.com/spf13/pflag ## explicit github.com/stretchr/testify/assert github.com/stretchr/testify/require -# github.com/twpayne/go-geom v1.3.0 +# github.com/twpayne/go-geom v1.3.2 ## explicit github.com/twpayne/go-geom github.com/twpayne/go-geom/bigxy From 3af6a724782d811d922c811e9e9c52d69b22b643 Mon Sep 17 00:00:00 2001 From: Jordan Lewis Date: Mon, 6 Jul 2020 00:38:11 -0400 Subject: [PATCH 07/49] upgrade gcassert To get func-decl inline assertions and cross pkg checks. --- github.com/jordanlewis/gcassert/README.md | 19 ++- github.com/jordanlewis/gcassert/gcassert.go | 138 +++++++++++++++--- .../internal/analysisinternal/analysis.go | 3 + modules.txt | 4 +- 4 files changed, 139 insertions(+), 25 deletions(-) diff --git a/github.com/jordanlewis/gcassert/README.md b/github.com/jordanlewis/gcassert/README.md index 16e1af0191..fc2bf98585 100644 --- a/github.com/jordanlewis/gcassert/README.md +++ b/github.com/jordanlewis/gcassert/README.md @@ -14,12 +14,19 @@ func addOne(i int) int { return i+1 } +//gcassert:inline +func addTwo(i int) int { + return i+1 +} + func a(ints []int) int { var sum int for i := range ints { //gcassert:bce,inline sum += addOne(ints[i]) + sum += addTwo(ints[i]) //gcassert:bce + sum += ints[i] //gcassert:bce } return sum @@ -29,6 +36,9 @@ func a(ints []int) int { The inline `//gcassert` directive will cause `gcassert` to fail if the line `sum += addOne(ints[i])` is either not inlined or contains bounds checks. +A `//gcassert:inline` directive on a function will cause `gcassert` to fail +if any of the callers of that function do not get inlined. + `//gcassert` comments expect a comma-separated list of directives after `//gcassert:`. They can be included above the line in question or after, as an inline comment. @@ -70,9 +80,12 @@ that wasn't upheld when running the compiler on the package. //gcassert:inline ``` -The inline directive asserts that the following statement contains a function -that is inlined by the compiler. If the function does not get inlined, gcassert -will fail. +The inline directive on a CallExpr asserts that the following statement +contains a function that is inlined by the compiler. If the function does not +get inlined, gcassert will fail. + +The inline directive on a FuncDecl asserts that every caller of that function +is actually inlined by the compiler ``` //gcassert:bce diff --git a/github.com/jordanlewis/gcassert/gcassert.go b/github.com/jordanlewis/gcassert/gcassert.go index ceb93d58fd..b0d1ac8b0f 100644 --- a/github.com/jordanlewis/gcassert/gcassert.go +++ b/github.com/jordanlewis/gcassert/gcassert.go @@ -17,11 +17,13 @@ import ( "go/ast" "go/printer" "go/token" + "go/types" "io" "os" "os/exec" "path/filepath" "regexp" + "sort" "strconv" "strings" @@ -62,15 +64,31 @@ var gcAssertRegex = regexp.MustCompile(`//gcassert:([\w,]+)`) type assertVisitor struct { commentMap ast.CommentMap + // directiveMap is a map from line number in the source file to the AST node + // that the line number corresponded to, as well as any directives that we + // parsed. directiveMap map[int]lineInfo - fileSet *token.FileSet + + // mustInlineFuncs is a set of types.Objects that represent FuncDecls of + // some kind that were marked with //gcassert:inline by the user. + mustInlineFuncs map[types.Object]struct{} + fileSet *token.FileSet + + p *packages.Package } -func newAssertVisitor(commentMap ast.CommentMap, fileSet *token.FileSet) assertVisitor { +func newAssertVisitor( + commentMap ast.CommentMap, + fileSet *token.FileSet, + p *packages.Package, + mustInlineFuncs map[types.Object]struct{}, +) assertVisitor { return assertVisitor{ - commentMap: commentMap, - fileSet: fileSet, - directiveMap: make(map[int]lineInfo), + commentMap: commentMap, + fileSet: fileSet, + directiveMap: make(map[int]lineInfo), + mustInlineFuncs: mustInlineFuncs, + p: p, } } @@ -78,20 +96,21 @@ func (v assertVisitor) Visit(node ast.Node) (w ast.Visitor) { if node == nil { return w } + pos := node.Pos() + lineNumber := v.fileSet.Position(pos).Line + m := v.commentMap[node] -COMMENTLOOP: for _, g := range m { + COMMENT_LIST: for _, c := range g.List { matches := gcAssertRegex.FindStringSubmatch(c.Text) if len(matches) == 0 { - continue COMMENTLOOP + continue } // The 0th match is the whole string, and the 1st match is the // gcassert directive(s). directiveStrings := strings.Split(matches[1], ",") - pos := node.Pos() - lineNumber := v.fileSet.Position(pos).Line lineInfo := v.directiveMap[lineNumber] lineInfo.n = node for _, s := range directiveStrings { @@ -99,6 +118,18 @@ COMMENTLOOP: if err != nil { continue } + if directive == inline { + switch n := node.(type) { + case *ast.FuncDecl: + // Add the Object that this FuncDecl's ident is connected + // to to our map of must-inline functions. + obj := v.p.TypesInfo.Defs[n.Name] + if obj != nil { + v.mustInlineFuncs[obj] = struct{}{} + } + continue COMMENT_LIST + } + } lineInfo.directives = append(lineInfo.directives, directive) } v.directiveMap[lineNumber] = lineInfo @@ -109,12 +140,13 @@ COMMENTLOOP: // GCAssert searches through the packages at the input path and writes failures // to comply with //gcassert directives to the given io.Writer. -func GCAssert(path string, w io.Writer) error { +func GCAssert(w io.Writer, paths ...string) error { fileSet := token.NewFileSet() pkgs, err := packages.Load(&packages.Config{ - Mode: packages.NeedName | packages.NeedFiles | packages.NeedSyntax | packages.NeedCompiledGoFiles, + Mode: packages.NeedName | packages.NeedFiles | packages.NeedSyntax | packages.NeedCompiledGoFiles | + packages.NeedTypesInfo | packages.NeedTypes, Fset: fileSet, - }, path) + }, paths...) directiveMap, err := parseDirectives(pkgs, fileSet) if err != nil { return err @@ -123,7 +155,10 @@ func GCAssert(path string, w io.Writer) error { // Next: invoke Go compiler with -m flags to get the compiler to print // its optimization decisions. - args := append([]string{"build", "-gcflags=-m -m -d=ssa/check_bce/debug=1"}, "./"+path) + args := []string{"build", "-gcflags=-m -m -d=ssa/check_bce/debug=1"} + for i := range paths { + args = append(args, "./"+paths[i]) + } cmd := exec.Command("go", args...) cwd, err := os.Getwd() if err != nil { @@ -188,8 +223,22 @@ func GCAssert(path string, w io.Writer) error { } } - for _, lineToDirectives := range directiveMap { - for _, info := range lineToDirectives { + keys := make([]string, 0, len(directiveMap)) + for k := range directiveMap { + keys = append(keys, k) + } + sort.Strings(keys) + + var lines []int + for _, k := range keys { + lines = lines[:0] + lineToDirectives := directiveMap[k] + for line := range lineToDirectives { + lines = append(lines, line) + } + sort.Ints(lines) + for _, line := range lines { + info := lineToDirectives[line] for i, d := range info.directives { // An inlining directive passes if it has compiler output. For // each inlining directive, check if there was matching compiler @@ -224,23 +273,72 @@ type directiveMap map[string]map[int]lineInfo func parseDirectives(pkgs []*packages.Package, fileSet *token.FileSet) (directiveMap, error) { fileDirectiveMap := make(directiveMap) + mustInlineFuncs := make(map[types.Object]struct{}) for _, pkg := range pkgs { for i, file := range pkg.Syntax { commentMap := ast.NewCommentMap(fileSet, file, file.Comments) - v := newAssertVisitor(commentMap, fileSet) + v := newAssertVisitor(commentMap, fileSet, pkg, mustInlineFuncs) // First: find all lines of code annotated with our gcassert directives. ast.Walk(v, file) file := pkg.CompiledGoFiles[i] - idx := strings.LastIndex(file, pkg.PkgPath) - if idx == -1 { - return nil, fmt.Errorf("couldn't find file %s in package %s", file, pkg.PkgPath) - } if len(v.directiveMap) > 0 { fileDirectiveMap[file] = v.directiveMap } } } + + // Do another pass to find all callsites of funcs marked with inline. + for _, pkg := range pkgs { + for i, file := range pkg.Syntax { + v := &inlinedDeclVisitor{assertVisitor: newAssertVisitor(nil, fileSet, pkg, mustInlineFuncs)} + filePath := pkg.CompiledGoFiles[i] + v.directiveMap = fileDirectiveMap[filePath] + if v.directiveMap == nil { + v.directiveMap = make(map[int]lineInfo) + } + ast.Walk(v, file) + if len(v.directiveMap) > 0 { + fileDirectiveMap[filePath] = v.directiveMap + } + } + } return fileDirectiveMap, nil } + +type inlinedDeclVisitor struct { + assertVisitor +} + +func (v *inlinedDeclVisitor) Visit(node ast.Node) (w ast.Visitor) { + if node == nil { + return w + } + pos := node.Pos() + lineNumber := v.fileSet.Position(pos).Line + + // Search for all func callsites of functions that were marked with + // gcassert:inline and add inline directives to those callsites. + switch n := node.(type) { + case *ast.CallExpr: + var obj types.Object + switch n := n.Fun.(type) { + case *ast.Ident: + obj = v.p.TypesInfo.Uses[n] + case *ast.SelectorExpr: + sel := v.p.TypesInfo.Selections[n] + if sel == nil { + break + } + obj = sel.Obj() + } + if _, ok := v.mustInlineFuncs[obj]; ok { + lineInfo := v.directiveMap[lineNumber] + lineInfo.n = node + lineInfo.directives = append(lineInfo.directives, inline) + v.directiveMap[lineNumber] = lineInfo + } + } + return v +} diff --git a/golang.org/x/tools/internal/analysisinternal/analysis.go b/golang.org/x/tools/internal/analysisinternal/analysis.go index 14b96a79fa..c7f5cd54bb 100644 --- a/golang.org/x/tools/internal/analysisinternal/analysis.go +++ b/golang.org/x/tools/internal/analysisinternal/analysis.go @@ -80,6 +80,9 @@ func TypeExpr(fset *token.FileSet, f *ast.File, pkg *types.Package, typ types.Ty return ast.NewIdent(t.Name()) } case *types.Named: + if t.Obj().Pkg() == nil { + return nil + } if t.Obj().Pkg() == pkg { return ast.NewIdent(t.Obj().Name()) } diff --git a/modules.txt b/modules.txt index eae06716ac..335c3c9b05 100644 --- a/modules.txt +++ b/modules.txt @@ -526,7 +526,7 @@ github.com/jcmturner/gofork/encoding/asn1 github.com/jcmturner/gofork/x/crypto/pbkdf2 # github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af github.com/jmespath/go-jmespath -# github.com/jordanlewis/gcassert v0.0.0-20200630054945-8ff878e72753 +# github.com/jordanlewis/gcassert v0.0.0-20200706043056-bf61eb72ee48 ## explicit github.com/jordanlewis/gcassert # github.com/kevinburke/go-bindata v3.13.0+incompatible @@ -862,7 +862,7 @@ golang.org/x/text/width # golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 ## explicit golang.org/x/time/rate -# golang.org/x/tools v0.0.0-20200626171337-aa94e735be7f +# golang.org/x/tools v0.0.0-20200702044944-0cc1aa72b347 ## explicit golang.org/x/tools/cmd/goyacc golang.org/x/tools/cmd/stringer From a491c178c3b2144915989d1377ea20f1f56bad5c Mon Sep 17 00:00:00 2001 From: Raphael 'kena' Poss Date: Wed, 8 Jul 2020 15:44:46 +0200 Subject: [PATCH 08/49] bump datadriven --- .../cockroachdb/datadriven/datadriven.go | 138 ++++++++++++++---- github.com/cockroachdb/datadriven/go.mod | 1 + github.com/cockroachdb/datadriven/go.sum | 2 + .../datadriven/test_data_reader.go | 9 +- modules.txt | 2 +- 5 files changed, 121 insertions(+), 31 deletions(-) diff --git a/github.com/cockroachdb/datadriven/datadriven.go b/github.com/cockroachdb/datadriven/datadriven.go index ab1bb63dd8..db3fec6163 100644 --- a/github.com/cockroachdb/datadriven/datadriven.go +++ b/github.com/cockroachdb/datadriven/datadriven.go @@ -25,6 +25,9 @@ import ( "strconv" "strings" "testing" + + "github.com/cockroachdb/errors" + "github.com/pmezard/go-difflib/difflib" ) var ( @@ -34,8 +37,33 @@ var ( "run. Used to update tests when a change affects many cases; please verify the testfile "+ "diffs carefully!", ) + + quietLog = flag.Bool( + "datadriven-quiet", true, + "avoid echoing the directives and responses from test files.", + ) ) +// Verbose returns true iff -datadriven-quiet was not passed. +func Verbose() bool { + return !*quietLog +} + +// In CockroachDB we want to quiesce all the logs across all packages. +// If we had only a flag to work with, we'd get command line parsing +// errors on all packages that do not use datadriven. So +// we make do by also making a command line parameter available. +func init() { + const quietEnvVar = "DATADRIVEN_QUIET_LOG" + if str, ok := os.LookupEnv(quietEnvVar); ok { + v, err := strconv.ParseBool(str) + if err != nil { + panic(fmt.Sprintf("error parsing %s: %s", quietEnvVar, err)) + } + *quietLog = v + } +} + // RunTest invokes a data-driven test. The test cases are contained in a // separate test file and are dynamically loaded, parsed, and executed by this // testing framework. By convention, test files are typically located in a @@ -102,17 +130,28 @@ func RunTest(t *testing.T, path string, f func(t *testing.T, d *TestData) string if err != nil { t.Fatal(err) } else if finfo.IsDir() { - t.Fatalf("%s is a directly, not a file; consider using datadriven.Walk", path) + t.Fatalf("%s is a directory, not a file; consider using datadriven.Walk", path) } - runTestInternal(t, path, file, f, *rewriteTestFiles) + rewriteData := runTestInternal(t, path, file, f, *rewriteTestFiles) + if *rewriteTestFiles { + if _, err := file.WriteAt(rewriteData, 0); err != nil { + t.Fatal(err) + } + if err := file.Truncate(int64(len(rewriteData))); err != nil { + t.Fatal(err) + } + if err := file.Sync(); err != nil { + t.Fatal(err) + } + } } // RunTestFromString is a version of RunTest which takes the contents of a test // directly. func RunTestFromString(t *testing.T, input string, f func(t *testing.T, d *TestData) string) { t.Helper() - runTestInternal(t, "" /* optionalPath */, strings.NewReader(input), f, *rewriteTestFiles) + runTestInternal(t, "" /* sourceName */, strings.NewReader(input), f, *rewriteTestFiles) } func runTestInternal( @@ -121,7 +160,7 @@ func runTestInternal( reader io.Reader, f func(t *testing.T, d *TestData) string, rewrite bool, -) { +) (rewriteOutput []byte) { t.Helper() r := newTestDataReader(t, sourceName, reader, rewrite) @@ -131,23 +170,13 @@ func runTestInternal( if r.rewrite != nil { data := r.rewrite.Bytes() + // Remove any trailing blank line. if l := len(data); l > 2 && data[l-1] == '\n' && data[l-2] == '\n' { data = data[:l-1] } - if dest, ok := reader.(*os.File); ok { - if _, err := dest.WriteAt(data, 0); err != nil { - t.Fatal(err) - } - if err := dest.Truncate(int64(len(data))); err != nil { - t.Fatal(err) - } - if err := dest.Sync(); err != nil { - t.Fatal(err) - } - } else { - t.Logf("input is not a file; rewritten output is:\n%s", data) - } + return data } + return nil } // runDirectiveOrSubTest runs either a "subtest" directive or an @@ -284,7 +313,7 @@ func runDirective(t *testing.T, r *testDataReader, f func(*testing.T, *TestData) actual := func() string { defer func() { if r := recover(); r != nil { - fmt.Printf("\npanic during %s:\n%s\n", d.Pos, d.Input) + t.Logf("\npanic during %s:\n%s\n", d.Pos, d.Input) panic(r) } }() @@ -315,18 +344,35 @@ func runDirective(t *testing.T, r *testDataReader, f func(*testing.T, *TestData) r.rewrite.WriteString(actual) r.emit("----") r.emit("----") + r.emit("") } else { + // Here actual already ends in \n so emit adds a blank line. r.emit(actual) } } else if d.Expected != actual { + expectedLines := difflib.SplitLines(d.Expected) + actualLines := difflib.SplitLines(actual) + if len(expectedLines) > 5 { + // Print a unified diff if there is a lot of output to compare. + diff, err := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{ + Context: 5, + A: expectedLines, + B: actualLines, + }) + if err == nil { + t.Fatalf("output didn't match expected:\n%s", diff) + return + } + t.Logf("Failed to produce diff %v", err) + } t.Fatalf("\n%s: %s\nexpected:\n%s\nfound:\n%s", d.Pos, d.Input, d.Expected, actual) - } else if testing.Verbose() { + } else if !*quietLog { input := d.Input if input == "" { input = "" } // TODO(tbg): it's awkward to reproduce the args, but it would be helpful. - fmt.Printf("\n%s:\n%s [%d args]\n%s\n----\n%s", d.Pos, d.Cmd, len(d.CmdArgs), input, actual) + t.Logf("\n%s:\n%s [%d args]\n%s\n----\n%s", d.Pos, d.Cmd, len(d.CmdArgs), input, actual) } return } @@ -381,6 +427,32 @@ func Walk(t *testing.T, path string, f func(t *testing.T, path string)) { } } +func ClearResults(path string) error { + file, err := os.OpenFile(path, os.O_RDWR, 0644 /* irrelevant */) + if err != nil { + return err + } + defer func() { + _ = file.Close() + }() + finfo, err := file.Stat() + if err != nil { + return err + } + + if finfo.IsDir() { + return errors.Newf("%s is a directory, not a file", path) + } + + runTestInternal( + &testing.T{}, path, file, + func(t *testing.T, d *TestData) string { return "" }, + true, /* rewrite */ + ) + + return nil +} + // Ignore files named .XXXX, XXX~ or #XXX#. var tempFileRe = regexp.MustCompile(`(^\..*)|(.*~$)|(^#.*#$)`) @@ -444,14 +516,16 @@ func (td *TestData) ScanArgs(t *testing.T, key string, dests ...interface{}) { } } if arg.Key == "" { - t.Fatalf("missing argument: %s", key) + td.Fatalf(t, "missing argument: %s", key) } if len(dests) != len(arg.Vals) { - t.Fatalf("%s: got %d destinations, but %d values", arg.Key, len(dests), len(arg.Vals)) + td.Fatalf(t, "%s: got %d destinations, but %d values", arg.Key, len(dests), len(arg.Vals)) } for i := range dests { - arg.Scan(t, i, dests[i]) + if err := arg.scanErr(i, dests[i]); err != nil { + td.Fatalf(t, "%s: failed to scan argument %d: %v", arg.Key, i, err) + } } } @@ -480,8 +554,15 @@ func (arg CmdArg) String() string { // Scan attempts to parse the value at index i into the dest. func (arg CmdArg) Scan(t *testing.T, i int, dest interface{}) { + if err := arg.scanErr(i, dest); err != nil { + t.Fatal(err) + } +} + +// scanErr is like Scan but returns an error rather than taking a testing.T to fatal. +func (arg CmdArg) scanErr(i int, dest interface{}) error { if i < 0 || i >= len(arg.Vals) { - t.Fatalf("cannot scan index %d of key %s", i, arg.Key) + return errors.Errorf("cannot scan index %d of key %s", i, arg.Key) } val := arg.Vals[i] switch dest := dest.(type) { @@ -490,24 +571,25 @@ func (arg CmdArg) Scan(t *testing.T, i int, dest interface{}) { case *int: n, err := strconv.ParseInt(val, 10, 64) if err != nil { - t.Fatal(err) + return err } *dest = int(n) // assume 64bit ints case *uint64: n, err := strconv.ParseUint(val, 10, 64) if err != nil { - t.Fatal(err) + return err } *dest = n case *bool: b, err := strconv.ParseBool(val) if err != nil { - t.Fatal(err) + return err } *dest = b default: - t.Fatalf("unsupported type %T for destination #%d (might be easy to add it)", dest, i+1) + return errors.Errorf("unsupported type %T for destination #%d (might be easy to add it)", dest, i+1) } + return nil } // Fatalf wraps a fatal testing error with test file position information, so diff --git a/github.com/cockroachdb/datadriven/go.mod b/github.com/cockroachdb/datadriven/go.mod index 38b4e87a19..dd62d6aa75 100644 --- a/github.com/cockroachdb/datadriven/go.mod +++ b/github.com/cockroachdb/datadriven/go.mod @@ -10,4 +10,5 @@ require ( github.com/gogo/protobuf v1.3.1 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/pkg/errors v0.8.1 // indirect + github.com/pmezard/go-difflib v1.0.0 ) diff --git a/github.com/cockroachdb/datadriven/go.sum b/github.com/cockroachdb/datadriven/go.sum index cfda93a49b..4195c27b26 100644 --- a/github.com/cockroachdb/datadriven/go.sum +++ b/github.com/cockroachdb/datadriven/go.sum @@ -17,4 +17,6 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/github.com/cockroachdb/datadriven/test_data_reader.go b/github.com/cockroachdb/datadriven/test_data_reader.go index bfe5c64009..2f5397b1c4 100644 --- a/github.com/cockroachdb/datadriven/test_data_reader.go +++ b/github.com/cockroachdb/datadriven/test_data_reader.go @@ -112,14 +112,14 @@ func (r *testDataReader) Next(t *testing.T) bool { r.data.Input = strings.TrimSpace(buf.String()) if separator { - r.readExpected() + r.readExpected(t) } return true } return false } -func (r *testDataReader) readExpected() { +func (r *testDataReader) readExpected(t *testing.T) { var buf bytes.Buffer var line string var allowBlankLines bool @@ -140,6 +140,11 @@ func (r *testDataReader) readExpected() { if r.scanner.Scan() { line2 := r.scanner.Text() if line2 == "----" { + // Read the following blank line (if we don't do this, we will emit + // an extra blank line when rewriting). + if r.scanner.Scan() && r.scanner.Text() != "" { + t.Fatal("non-blank line after end of double ---- separator section") + } break } diff --git a/modules.txt b/modules.txt index 335c3c9b05..73eed3abe7 100644 --- a/modules.txt +++ b/modules.txt @@ -179,7 +179,7 @@ github.com/cockroachdb/cockroach-go/crdb github.com/cockroachdb/crlfmt github.com/cockroachdb/crlfmt/internal/parser github.com/cockroachdb/crlfmt/internal/render -# github.com/cockroachdb/datadriven v0.0.0-20200212141702-aca09668cb24 +# github.com/cockroachdb/datadriven v0.0.0-20200708133806-c52af14888bf ## explicit github.com/cockroachdb/datadriven # github.com/cockroachdb/errors v1.5.0 From ccd51fd3a377461f434acd619be0058ce1f425f9 Mon Sep 17 00:00:00 2001 From: Raphael 'kena' Poss Date: Wed, 8 Jul 2020 16:07:43 +0200 Subject: [PATCH 09/49] bump datadriven --- github.com/cockroachdb/datadriven/datadriven.go | 2 +- modules.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/github.com/cockroachdb/datadriven/datadriven.go b/github.com/cockroachdb/datadriven/datadriven.go index db3fec6163..b2e3fe81f3 100644 --- a/github.com/cockroachdb/datadriven/datadriven.go +++ b/github.com/cockroachdb/datadriven/datadriven.go @@ -39,7 +39,7 @@ var ( ) quietLog = flag.Bool( - "datadriven-quiet", true, + "datadriven-quiet", false, "avoid echoing the directives and responses from test files.", ) ) diff --git a/modules.txt b/modules.txt index 73eed3abe7..f62e9d7c11 100644 --- a/modules.txt +++ b/modules.txt @@ -179,7 +179,7 @@ github.com/cockroachdb/cockroach-go/crdb github.com/cockroachdb/crlfmt github.com/cockroachdb/crlfmt/internal/parser github.com/cockroachdb/crlfmt/internal/render -# github.com/cockroachdb/datadriven v0.0.0-20200708133806-c52af14888bf +# github.com/cockroachdb/datadriven v0.0.0-20200708140406-a8a63ef9e03e ## explicit github.com/cockroachdb/datadriven # github.com/cockroachdb/errors v1.5.0 From 81cbbf1bc1118ff3e08d48f491353a3ff6bdebf5 Mon Sep 17 00:00:00 2001 From: Tobias Schottdorf Date: Thu, 9 Jul 2020 22:03:16 +0200 Subject: [PATCH 10/49] Update vendor for sqlproxyccl package --- github.com/jackc/chunkreader/v2/.travis.yml | 9 + github.com/jackc/chunkreader/v2/LICENSE | 22 + github.com/jackc/chunkreader/v2/README.md | 8 + .../jackc/chunkreader/v2/chunkreader.go | 104 + github.com/jackc/chunkreader/v2/go.mod | 3 + github.com/jackc/pgconn/.gitignore | 3 + github.com/jackc/pgconn/.travis.yml | 49 + github.com/jackc/pgconn/CHANGELOG.md | 70 + github.com/jackc/pgconn/LICENSE | 22 + github.com/jackc/pgconn/README.md | 56 + github.com/jackc/pgconn/auth_scram.go | 266 + github.com/jackc/pgconn/config.go | 758 ++ github.com/jackc/pgconn/doc.go | 29 + github.com/jackc/pgconn/errors.go | 192 + github.com/jackc/pgconn/go.mod | 16 + github.com/jackc/pgconn/go.sum | 121 + .../internal/ctxwatch/context_watcher.go | 64 + github.com/jackc/pgconn/pgconn.go | 1592 ++++ github.com/jackc/pgconn/stmtcache/lru.go | 113 + .../jackc/pgconn/stmtcache/stmtcache.go | 52 + github.com/jackc/pgio/.travis.yml | 9 + github.com/jackc/pgio/LICENSE | 22 + github.com/jackc/pgio/README.md | 11 + github.com/jackc/pgio/doc.go | 6 + github.com/jackc/pgio/go.mod | 3 + github.com/jackc/pgio/write.go | 40 + github.com/jackc/pgpassfile/.travis.yml | 9 + github.com/jackc/pgpassfile/LICENSE | 22 + github.com/jackc/pgpassfile/README.md | 8 + github.com/jackc/pgpassfile/go.mod | 5 + github.com/jackc/pgpassfile/go.sum | 7 + github.com/jackc/pgpassfile/pgpass.go | 110 + github.com/jackc/pgproto3/v2/.travis.yml | 9 + github.com/jackc/pgproto3/v2/LICENSE | 22 + github.com/jackc/pgproto3/v2/README.md | 12 + .../v2/authentication_cleartext_password.go | 39 + .../v2/authentication_md5_password.go | 43 + .../jackc/pgproto3/v2/authentication_ok.go | 39 + .../jackc/pgproto3/v2/authentication_sasl.go | 60 + .../v2/authentication_sasl_continue.go | 49 + .../pgproto3/v2/authentication_sasl_final.go | 49 + github.com/jackc/pgproto3/v2/backend.go | 137 + .../jackc/pgproto3/v2/backend_key_data.go | 51 + github.com/jackc/pgproto3/v2/big_endian.go | 37 + github.com/jackc/pgproto3/v2/bind.go | 176 + github.com/jackc/pgproto3/v2/bind_complete.go | 34 + .../jackc/pgproto3/v2/cancel_request.go | 58 + github.com/jackc/pgproto3/v2/chunkreader.go | 19 + github.com/jackc/pgproto3/v2/close.go | 64 + .../jackc/pgproto3/v2/close_complete.go | 34 + .../jackc/pgproto3/v2/command_complete.go | 53 + .../jackc/pgproto3/v2/copy_both_response.go | 70 + github.com/jackc/pgproto3/v2/copy_data.go | 44 + github.com/jackc/pgproto3/v2/copy_done.go | 35 + github.com/jackc/pgproto3/v2/copy_fail.go | 53 + .../jackc/pgproto3/v2/copy_in_response.go | 70 + .../jackc/pgproto3/v2/copy_out_response.go | 71 + github.com/jackc/pgproto3/v2/data_row.go | 117 + github.com/jackc/pgproto3/v2/describe.go | 64 + github.com/jackc/pgproto3/v2/doc.go | 4 + .../jackc/pgproto3/v2/empty_query_response.go | 34 + .../jackc/pgproto3/v2/error_response.go | 218 + github.com/jackc/pgproto3/v2/execute.go | 65 + github.com/jackc/pgproto3/v2/flush.go | 34 + github.com/jackc/pgproto3/v2/frontend.go | 180 + .../pgproto3/v2/function_call_response.go | 83 + github.com/jackc/pgproto3/v2/go.mod | 9 + github.com/jackc/pgproto3/v2/go.sum | 14 + github.com/jackc/pgproto3/v2/no_data.go | 34 + .../jackc/pgproto3/v2/notice_response.go | 17 + .../pgproto3/v2/notification_response.go | 72 + .../pgproto3/v2/parameter_description.go | 66 + .../jackc/pgproto3/v2/parameter_status.go | 66 + github.com/jackc/pgproto3/v2/parse.go | 88 + .../jackc/pgproto3/v2/parse_complete.go | 34 + .../jackc/pgproto3/v2/password_message.go | 51 + github.com/jackc/pgproto3/v2/pgproto3.go | 42 + .../jackc/pgproto3/v2/portal_suspended.go | 34 + github.com/jackc/pgproto3/v2/query.go | 50 + .../jackc/pgproto3/v2/ready_for_query.go | 40 + .../jackc/pgproto3/v2/row_description.go | 134 + .../pgproto3/v2/sasl_initial_response.go | 69 + github.com/jackc/pgproto3/v2/sasl_response.go | 43 + github.com/jackc/pgproto3/v2/ssl_request.go | 49 + .../jackc/pgproto3/v2/startup_message.go | 96 + github.com/jackc/pgproto3/v2/sync.go | 34 + github.com/jackc/pgproto3/v2/terminate.go | 34 + github.com/jackc/pgservicefile/.travis.yml | 9 + github.com/jackc/pgservicefile/LICENSE | 22 + github.com/jackc/pgservicefile/README.md | 6 + github.com/jackc/pgservicefile/go.mod | 5 + github.com/jackc/pgservicefile/go.sum | 10 + .../jackc/pgservicefile/pgservicefile.go | 79 + github.com/jackc/pgtype/CHANGELOG.md | 32 + github.com/jackc/pgtype/LICENSE | 22 + github.com/jackc/pgtype/README.md | 7 + github.com/jackc/pgtype/aclitem.go | 139 + github.com/jackc/pgtype/aclitem_array.go | 232 + github.com/jackc/pgtype/array.go | 352 + github.com/jackc/pgtype/bit.go | 45 + github.com/jackc/pgtype/bool.go | 206 + github.com/jackc/pgtype/bool_array.go | 320 + github.com/jackc/pgtype/box.go | 166 + github.com/jackc/pgtype/bpchar.go | 68 + github.com/jackc/pgtype/bpchar_array.go | 320 + github.com/jackc/pgtype/bytea.go | 164 + github.com/jackc/pgtype/bytea_array.go | 320 + github.com/jackc/pgtype/cid.go | 61 + github.com/jackc/pgtype/cidr.go | 31 + github.com/jackc/pgtype/cidr_array.go | 349 + github.com/jackc/pgtype/circle.go | 151 + github.com/jackc/pgtype/convert.go | 453 + github.com/jackc/pgtype/database_sql.go | 42 + github.com/jackc/pgtype/date.go | 275 + github.com/jackc/pgtype/date_array.go | 321 + github.com/jackc/pgtype/daterange.go | 267 + github.com/jackc/pgtype/enum_array.go | 232 + github.com/jackc/pgtype/float4.go | 204 + github.com/jackc/pgtype/float4_array.go | 320 + github.com/jackc/pgtype/float8.go | 194 + github.com/jackc/pgtype/float8_array.go | 320 + github.com/jackc/pgtype/generic_binary.go | 39 + github.com/jackc/pgtype/generic_text.go | 39 + github.com/jackc/pgtype/go.mod | 14 + github.com/jackc/pgtype/go.sum | 120 + github.com/jackc/pgtype/hstore.go | 438 + github.com/jackc/pgtype/hstore_array.go | 320 + github.com/jackc/pgtype/inet.go | 223 + github.com/jackc/pgtype/inet_array.go | 349 + github.com/jackc/pgtype/int2.go | 216 + github.com/jackc/pgtype/int2_array.go | 516 ++ github.com/jackc/pgtype/int4.go | 224 + github.com/jackc/pgtype/int4_array.go | 516 ++ github.com/jackc/pgtype/int4range.go | 267 + github.com/jackc/pgtype/int8.go | 210 + github.com/jackc/pgtype/int8_array.go | 516 ++ github.com/jackc/pgtype/int8range.go | 267 + github.com/jackc/pgtype/interval.go | 258 + github.com/jackc/pgtype/json.go | 197 + github.com/jackc/pgtype/jsonb.go | 78 + github.com/jackc/pgtype/line.go | 149 + github.com/jackc/pgtype/lseg.go | 166 + github.com/jackc/pgtype/macaddr.go | 162 + github.com/jackc/pgtype/macaddr_array.go | 321 + github.com/jackc/pgtype/name.go | 58 + github.com/jackc/pgtype/numeric.go | 604 ++ github.com/jackc/pgtype/numeric_array.go | 404 + github.com/jackc/pgtype/numrange.go | 267 + github.com/jackc/pgtype/oid.go | 81 + github.com/jackc/pgtype/oid_value.go | 55 + github.com/jackc/pgtype/path.go | 196 + github.com/jackc/pgtype/pgtype.go | 493 ++ github.com/jackc/pgtype/pguint32.go | 162 + github.com/jackc/pgtype/point.go | 142 + github.com/jackc/pgtype/polygon.go | 177 + github.com/jackc/pgtype/qchar.go | 153 + github.com/jackc/pgtype/range.go | 278 + github.com/jackc/pgtype/record.go | 137 + github.com/jackc/pgtype/text.go | 175 + github.com/jackc/pgtype/text_array.go | 320 + github.com/jackc/pgtype/tid.go | 144 + github.com/jackc/pgtype/time.go | 226 + github.com/jackc/pgtype/timestamp.go | 233 + github.com/jackc/pgtype/timestamp_array.go | 321 + github.com/jackc/pgtype/timestamptz.go | 286 + github.com/jackc/pgtype/timestamptz_array.go | 321 + github.com/jackc/pgtype/tsrange.go | 267 + github.com/jackc/pgtype/tstzrange.go | 267 + github.com/jackc/pgtype/tstzrange_array.go | 301 + github.com/jackc/pgtype/typed_array.go.erb | 326 + github.com/jackc/pgtype/typed_array_gen.sh | 26 + github.com/jackc/pgtype/typed_range.go.erb | 269 + github.com/jackc/pgtype/typed_range_gen.sh | 7 + github.com/jackc/pgtype/unknown.go | 44 + github.com/jackc/pgtype/uuid.go | 193 + github.com/jackc/pgtype/uuid_array.go | 376 + github.com/jackc/pgtype/varbit.go | 133 + github.com/jackc/pgtype/varchar.go | 58 + github.com/jackc/pgtype/varchar_array.go | 320 + github.com/jackc/pgtype/xid.go | 64 + github.com/jackc/pgx/v4/.gitignore | 24 + github.com/jackc/pgx/v4/.travis.yml | 33 + github.com/jackc/pgx/v4/CHANGELOG.md | 102 + github.com/jackc/pgx/v4/LICENSE | 22 + github.com/jackc/pgx/v4/README.md | 186 + github.com/jackc/pgx/v4/batch.go | 181 + github.com/jackc/pgx/v4/conn.go | 797 ++ github.com/jackc/pgx/v4/copy_from.go | 169 + github.com/jackc/pgx/v4/doc.go | 250 + .../jackc/pgx/v4/extended_query_builder.go | 141 + github.com/jackc/pgx/v4/go.mod | 23 + github.com/jackc/pgx/v4/go.sum | 172 + github.com/jackc/pgx/v4/go_stdlib.go | 61 + .../pgx/v4/internal/sanitize/sanitize.go | 237 + github.com/jackc/pgx/v4/large_objects.go | 122 + github.com/jackc/pgx/v4/logger.go | 99 + github.com/jackc/pgx/v4/messages.go | 42 + github.com/jackc/pgx/v4/rows.go | 308 + github.com/jackc/pgx/v4/tx.go | 350 + github.com/jackc/pgx/v4/values.go | 261 + github.com/mattn/go-isatty/.travis.yml | 15 +- github.com/mattn/go-isatty/README.md | 2 +- github.com/mattn/go-isatty/go.mod | 4 +- github.com/mattn/go-isatty/go.sum | 4 +- github.com/mattn/go-isatty/go.test.sh | 12 + github.com/mattn/go-isatty/isatty_android.go | 23 - github.com/mattn/go-isatty/isatty_bsd.go | 12 +- github.com/mattn/go-isatty/isatty_plan9.go | 22 + github.com/mattn/go-isatty/isatty_tcgets.go | 1 - github.com/mattn/go-isatty/isatty_windows.go | 39 +- github.com/mattn/go-isatty/renovate.json | 8 + .../x/crypto/chacha20/chacha_generic.go | 119 +- golang.org/x/crypto/chacha20/xor.go | 17 +- golang.org/x/crypto/poly1305/mac_noasm.go | 4 +- golang.org/x/crypto/poly1305/poly1305.go | 26 +- golang.org/x/crypto/poly1305/sum_amd64.go | 11 - golang.org/x/crypto/poly1305/sum_generic.go | 21 +- golang.org/x/crypto/poly1305/sum_noasm.go | 13 - golang.org/x/crypto/poly1305/sum_ppc64le.go | 11 - golang.org/x/crypto/poly1305/sum_s390x.go | 72 +- golang.org/x/crypto/poly1305/sum_s390x.s | 667 +- golang.org/x/crypto/poly1305/sum_vmsl_s390x.s | 909 -- golang.org/x/crypto/ssh/agent/client.go | 28 +- golang.org/x/crypto/ssh/certs.go | 4 +- golang.org/x/crypto/ssh/cipher.go | 2 +- golang.org/x/crypto/ssh/client_auth.go | 22 +- golang.org/x/crypto/ssh/mux.go | 23 +- golang.org/x/crypto/ssh/terminal/terminal.go | 8 + golang.org/x/text/cases/tables11.0.0.go | 2 +- golang.org/x/text/cases/tables12.0.0.go | 2359 +++++ .../x/text/secure/precis/tables11.0.0.go | 2 +- .../x/text/secure/precis/tables12.0.0.go | 4118 +++++++++ golang.org/x/text/transform/transform.go | 6 +- golang.org/x/text/unicode/bidi/core.go | 8 +- .../x/text/unicode/bidi/tables11.0.0.go | 2 +- .../x/text/unicode/bidi/tables12.0.0.go | 1923 ++++ .../x/text/unicode/norm/tables11.0.0.go | 2 +- .../x/text/unicode/norm/tables12.0.0.go | 7710 +++++++++++++++++ golang.org/x/text/width/tables11.0.0.go | 2 +- golang.org/x/text/width/tables12.0.0.go | 1350 +++ modules.txt | 31 +- 241 files changed, 48034 insertions(+), 1397 deletions(-) create mode 100644 github.com/jackc/chunkreader/v2/.travis.yml create mode 100644 github.com/jackc/chunkreader/v2/LICENSE create mode 100644 github.com/jackc/chunkreader/v2/README.md create mode 100644 github.com/jackc/chunkreader/v2/chunkreader.go create mode 100644 github.com/jackc/chunkreader/v2/go.mod create mode 100644 github.com/jackc/pgconn/.gitignore create mode 100644 github.com/jackc/pgconn/.travis.yml create mode 100644 github.com/jackc/pgconn/CHANGELOG.md create mode 100644 github.com/jackc/pgconn/LICENSE create mode 100644 github.com/jackc/pgconn/README.md create mode 100644 github.com/jackc/pgconn/auth_scram.go create mode 100644 github.com/jackc/pgconn/config.go create mode 100644 github.com/jackc/pgconn/doc.go create mode 100644 github.com/jackc/pgconn/errors.go create mode 100644 github.com/jackc/pgconn/go.mod create mode 100644 github.com/jackc/pgconn/go.sum create mode 100644 github.com/jackc/pgconn/internal/ctxwatch/context_watcher.go create mode 100644 github.com/jackc/pgconn/pgconn.go create mode 100644 github.com/jackc/pgconn/stmtcache/lru.go create mode 100644 github.com/jackc/pgconn/stmtcache/stmtcache.go create mode 100644 github.com/jackc/pgio/.travis.yml create mode 100644 github.com/jackc/pgio/LICENSE create mode 100644 github.com/jackc/pgio/README.md create mode 100644 github.com/jackc/pgio/doc.go create mode 100644 github.com/jackc/pgio/go.mod create mode 100644 github.com/jackc/pgio/write.go create mode 100644 github.com/jackc/pgpassfile/.travis.yml create mode 100644 github.com/jackc/pgpassfile/LICENSE create mode 100644 github.com/jackc/pgpassfile/README.md create mode 100644 github.com/jackc/pgpassfile/go.mod create mode 100644 github.com/jackc/pgpassfile/go.sum create mode 100644 github.com/jackc/pgpassfile/pgpass.go create mode 100644 github.com/jackc/pgproto3/v2/.travis.yml create mode 100644 github.com/jackc/pgproto3/v2/LICENSE create mode 100644 github.com/jackc/pgproto3/v2/README.md create mode 100644 github.com/jackc/pgproto3/v2/authentication_cleartext_password.go create mode 100644 github.com/jackc/pgproto3/v2/authentication_md5_password.go create mode 100644 github.com/jackc/pgproto3/v2/authentication_ok.go create mode 100644 github.com/jackc/pgproto3/v2/authentication_sasl.go create mode 100644 github.com/jackc/pgproto3/v2/authentication_sasl_continue.go create mode 100644 github.com/jackc/pgproto3/v2/authentication_sasl_final.go create mode 100644 github.com/jackc/pgproto3/v2/backend.go create mode 100644 github.com/jackc/pgproto3/v2/backend_key_data.go create mode 100644 github.com/jackc/pgproto3/v2/big_endian.go create mode 100644 github.com/jackc/pgproto3/v2/bind.go create mode 100644 github.com/jackc/pgproto3/v2/bind_complete.go create mode 100644 github.com/jackc/pgproto3/v2/cancel_request.go create mode 100644 github.com/jackc/pgproto3/v2/chunkreader.go create mode 100644 github.com/jackc/pgproto3/v2/close.go create mode 100644 github.com/jackc/pgproto3/v2/close_complete.go create mode 100644 github.com/jackc/pgproto3/v2/command_complete.go create mode 100644 github.com/jackc/pgproto3/v2/copy_both_response.go create mode 100644 github.com/jackc/pgproto3/v2/copy_data.go create mode 100644 github.com/jackc/pgproto3/v2/copy_done.go create mode 100644 github.com/jackc/pgproto3/v2/copy_fail.go create mode 100644 github.com/jackc/pgproto3/v2/copy_in_response.go create mode 100644 github.com/jackc/pgproto3/v2/copy_out_response.go create mode 100644 github.com/jackc/pgproto3/v2/data_row.go create mode 100644 github.com/jackc/pgproto3/v2/describe.go create mode 100644 github.com/jackc/pgproto3/v2/doc.go create mode 100644 github.com/jackc/pgproto3/v2/empty_query_response.go create mode 100644 github.com/jackc/pgproto3/v2/error_response.go create mode 100644 github.com/jackc/pgproto3/v2/execute.go create mode 100644 github.com/jackc/pgproto3/v2/flush.go create mode 100644 github.com/jackc/pgproto3/v2/frontend.go create mode 100644 github.com/jackc/pgproto3/v2/function_call_response.go create mode 100644 github.com/jackc/pgproto3/v2/go.mod create mode 100644 github.com/jackc/pgproto3/v2/go.sum create mode 100644 github.com/jackc/pgproto3/v2/no_data.go create mode 100644 github.com/jackc/pgproto3/v2/notice_response.go create mode 100644 github.com/jackc/pgproto3/v2/notification_response.go create mode 100644 github.com/jackc/pgproto3/v2/parameter_description.go create mode 100644 github.com/jackc/pgproto3/v2/parameter_status.go create mode 100644 github.com/jackc/pgproto3/v2/parse.go create mode 100644 github.com/jackc/pgproto3/v2/parse_complete.go create mode 100644 github.com/jackc/pgproto3/v2/password_message.go create mode 100644 github.com/jackc/pgproto3/v2/pgproto3.go create mode 100644 github.com/jackc/pgproto3/v2/portal_suspended.go create mode 100644 github.com/jackc/pgproto3/v2/query.go create mode 100644 github.com/jackc/pgproto3/v2/ready_for_query.go create mode 100644 github.com/jackc/pgproto3/v2/row_description.go create mode 100644 github.com/jackc/pgproto3/v2/sasl_initial_response.go create mode 100644 github.com/jackc/pgproto3/v2/sasl_response.go create mode 100644 github.com/jackc/pgproto3/v2/ssl_request.go create mode 100644 github.com/jackc/pgproto3/v2/startup_message.go create mode 100644 github.com/jackc/pgproto3/v2/sync.go create mode 100644 github.com/jackc/pgproto3/v2/terminate.go create mode 100644 github.com/jackc/pgservicefile/.travis.yml create mode 100644 github.com/jackc/pgservicefile/LICENSE create mode 100644 github.com/jackc/pgservicefile/README.md create mode 100644 github.com/jackc/pgservicefile/go.mod create mode 100644 github.com/jackc/pgservicefile/go.sum create mode 100644 github.com/jackc/pgservicefile/pgservicefile.go create mode 100644 github.com/jackc/pgtype/CHANGELOG.md create mode 100644 github.com/jackc/pgtype/LICENSE create mode 100644 github.com/jackc/pgtype/README.md create mode 100644 github.com/jackc/pgtype/aclitem.go create mode 100644 github.com/jackc/pgtype/aclitem_array.go create mode 100644 github.com/jackc/pgtype/array.go create mode 100644 github.com/jackc/pgtype/bit.go create mode 100644 github.com/jackc/pgtype/bool.go create mode 100644 github.com/jackc/pgtype/bool_array.go create mode 100644 github.com/jackc/pgtype/box.go create mode 100644 github.com/jackc/pgtype/bpchar.go create mode 100644 github.com/jackc/pgtype/bpchar_array.go create mode 100644 github.com/jackc/pgtype/bytea.go create mode 100644 github.com/jackc/pgtype/bytea_array.go create mode 100644 github.com/jackc/pgtype/cid.go create mode 100644 github.com/jackc/pgtype/cidr.go create mode 100644 github.com/jackc/pgtype/cidr_array.go create mode 100644 github.com/jackc/pgtype/circle.go create mode 100644 github.com/jackc/pgtype/convert.go create mode 100644 github.com/jackc/pgtype/database_sql.go create mode 100644 github.com/jackc/pgtype/date.go create mode 100644 github.com/jackc/pgtype/date_array.go create mode 100644 github.com/jackc/pgtype/daterange.go create mode 100644 github.com/jackc/pgtype/enum_array.go create mode 100644 github.com/jackc/pgtype/float4.go create mode 100644 github.com/jackc/pgtype/float4_array.go create mode 100644 github.com/jackc/pgtype/float8.go create mode 100644 github.com/jackc/pgtype/float8_array.go create mode 100644 github.com/jackc/pgtype/generic_binary.go create mode 100644 github.com/jackc/pgtype/generic_text.go create mode 100644 github.com/jackc/pgtype/go.mod create mode 100644 github.com/jackc/pgtype/go.sum create mode 100644 github.com/jackc/pgtype/hstore.go create mode 100644 github.com/jackc/pgtype/hstore_array.go create mode 100644 github.com/jackc/pgtype/inet.go create mode 100644 github.com/jackc/pgtype/inet_array.go create mode 100644 github.com/jackc/pgtype/int2.go create mode 100644 github.com/jackc/pgtype/int2_array.go create mode 100644 github.com/jackc/pgtype/int4.go create mode 100644 github.com/jackc/pgtype/int4_array.go create mode 100644 github.com/jackc/pgtype/int4range.go create mode 100644 github.com/jackc/pgtype/int8.go create mode 100644 github.com/jackc/pgtype/int8_array.go create mode 100644 github.com/jackc/pgtype/int8range.go create mode 100644 github.com/jackc/pgtype/interval.go create mode 100644 github.com/jackc/pgtype/json.go create mode 100644 github.com/jackc/pgtype/jsonb.go create mode 100644 github.com/jackc/pgtype/line.go create mode 100644 github.com/jackc/pgtype/lseg.go create mode 100644 github.com/jackc/pgtype/macaddr.go create mode 100644 github.com/jackc/pgtype/macaddr_array.go create mode 100644 github.com/jackc/pgtype/name.go create mode 100644 github.com/jackc/pgtype/numeric.go create mode 100644 github.com/jackc/pgtype/numeric_array.go create mode 100644 github.com/jackc/pgtype/numrange.go create mode 100644 github.com/jackc/pgtype/oid.go create mode 100644 github.com/jackc/pgtype/oid_value.go create mode 100644 github.com/jackc/pgtype/path.go create mode 100644 github.com/jackc/pgtype/pgtype.go create mode 100644 github.com/jackc/pgtype/pguint32.go create mode 100644 github.com/jackc/pgtype/point.go create mode 100644 github.com/jackc/pgtype/polygon.go create mode 100644 github.com/jackc/pgtype/qchar.go create mode 100644 github.com/jackc/pgtype/range.go create mode 100644 github.com/jackc/pgtype/record.go create mode 100644 github.com/jackc/pgtype/text.go create mode 100644 github.com/jackc/pgtype/text_array.go create mode 100644 github.com/jackc/pgtype/tid.go create mode 100644 github.com/jackc/pgtype/time.go create mode 100644 github.com/jackc/pgtype/timestamp.go create mode 100644 github.com/jackc/pgtype/timestamp_array.go create mode 100644 github.com/jackc/pgtype/timestamptz.go create mode 100644 github.com/jackc/pgtype/timestamptz_array.go create mode 100644 github.com/jackc/pgtype/tsrange.go create mode 100644 github.com/jackc/pgtype/tstzrange.go create mode 100644 github.com/jackc/pgtype/tstzrange_array.go create mode 100644 github.com/jackc/pgtype/typed_array.go.erb create mode 100644 github.com/jackc/pgtype/typed_array_gen.sh create mode 100644 github.com/jackc/pgtype/typed_range.go.erb create mode 100644 github.com/jackc/pgtype/typed_range_gen.sh create mode 100644 github.com/jackc/pgtype/unknown.go create mode 100644 github.com/jackc/pgtype/uuid.go create mode 100644 github.com/jackc/pgtype/uuid_array.go create mode 100644 github.com/jackc/pgtype/varbit.go create mode 100644 github.com/jackc/pgtype/varchar.go create mode 100644 github.com/jackc/pgtype/varchar_array.go create mode 100644 github.com/jackc/pgtype/xid.go create mode 100644 github.com/jackc/pgx/v4/.gitignore create mode 100644 github.com/jackc/pgx/v4/.travis.yml create mode 100644 github.com/jackc/pgx/v4/CHANGELOG.md create mode 100644 github.com/jackc/pgx/v4/LICENSE create mode 100644 github.com/jackc/pgx/v4/README.md create mode 100644 github.com/jackc/pgx/v4/batch.go create mode 100644 github.com/jackc/pgx/v4/conn.go create mode 100644 github.com/jackc/pgx/v4/copy_from.go create mode 100644 github.com/jackc/pgx/v4/doc.go create mode 100644 github.com/jackc/pgx/v4/extended_query_builder.go create mode 100644 github.com/jackc/pgx/v4/go.mod create mode 100644 github.com/jackc/pgx/v4/go.sum create mode 100644 github.com/jackc/pgx/v4/go_stdlib.go create mode 100644 github.com/jackc/pgx/v4/internal/sanitize/sanitize.go create mode 100644 github.com/jackc/pgx/v4/large_objects.go create mode 100644 github.com/jackc/pgx/v4/logger.go create mode 100644 github.com/jackc/pgx/v4/messages.go create mode 100644 github.com/jackc/pgx/v4/rows.go create mode 100644 github.com/jackc/pgx/v4/tx.go create mode 100644 github.com/jackc/pgx/v4/values.go create mode 100644 github.com/mattn/go-isatty/go.test.sh delete mode 100644 github.com/mattn/go-isatty/isatty_android.go create mode 100644 github.com/mattn/go-isatty/isatty_plan9.go create mode 100644 github.com/mattn/go-isatty/renovate.json delete mode 100644 golang.org/x/crypto/poly1305/sum_noasm.go delete mode 100644 golang.org/x/crypto/poly1305/sum_vmsl_s390x.s create mode 100644 golang.org/x/text/cases/tables12.0.0.go create mode 100644 golang.org/x/text/secure/precis/tables12.0.0.go create mode 100644 golang.org/x/text/unicode/bidi/tables12.0.0.go create mode 100644 golang.org/x/text/unicode/norm/tables12.0.0.go create mode 100644 golang.org/x/text/width/tables12.0.0.go diff --git a/github.com/jackc/chunkreader/v2/.travis.yml b/github.com/jackc/chunkreader/v2/.travis.yml new file mode 100644 index 0000000000..e176228e8e --- /dev/null +++ b/github.com/jackc/chunkreader/v2/.travis.yml @@ -0,0 +1,9 @@ +language: go + +go: + - 1.x + - tip + +matrix: + allow_failures: + - go: tip diff --git a/github.com/jackc/chunkreader/v2/LICENSE b/github.com/jackc/chunkreader/v2/LICENSE new file mode 100644 index 0000000000..c1c4f50fc6 --- /dev/null +++ b/github.com/jackc/chunkreader/v2/LICENSE @@ -0,0 +1,22 @@ +Copyright (c) 2019 Jack Christensen + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/github.com/jackc/chunkreader/v2/README.md b/github.com/jackc/chunkreader/v2/README.md new file mode 100644 index 0000000000..01209bfa22 --- /dev/null +++ b/github.com/jackc/chunkreader/v2/README.md @@ -0,0 +1,8 @@ +[![](https://godoc.org/github.com/jackc/chunkreader?status.svg)](https://godoc.org/github.com/jackc/chunkreader) +[![Build Status](https://travis-ci.org/jackc/chunkreader.svg)](https://travis-ci.org/jackc/chunkreader) + +# chunkreader + +Package chunkreader provides an io.Reader wrapper that minimizes IO reads and memory allocations. + +Extracted from original implementation in https://github.com/jackc/pgx. diff --git a/github.com/jackc/chunkreader/v2/chunkreader.go b/github.com/jackc/chunkreader/v2/chunkreader.go new file mode 100644 index 0000000000..afea1c5207 --- /dev/null +++ b/github.com/jackc/chunkreader/v2/chunkreader.go @@ -0,0 +1,104 @@ +// Package chunkreader provides an io.Reader wrapper that minimizes IO reads and memory allocations. +package chunkreader + +import ( + "io" +) + +// ChunkReader is a io.Reader wrapper that minimizes IO reads and memory allocations. It allocates memory in chunks and +// will read as much as will fit in the current buffer in a single call regardless of how large a read is actually +// requested. The memory returned via Next is owned by the caller. This avoids the need for an additional copy. +// +// The downside of this approach is that a large buffer can be pinned in memory even if only a small slice is +// referenced. For example, an entire 4096 byte block could be pinned in memory by even a 1 byte slice. In these rare +// cases it would be advantageous to copy the bytes to another slice. +type ChunkReader struct { + r io.Reader + + buf []byte + rp, wp int // buf read position and write position + + config Config +} + +// Config contains configuration parameters for ChunkReader. +type Config struct { + MinBufLen int // Minimum buffer length +} + +// New creates and returns a new ChunkReader for r with default configuration. +func New(r io.Reader) *ChunkReader { + cr, err := NewConfig(r, Config{}) + if err != nil { + panic("default config can't be bad") + } + + return cr +} + +// NewConfig creates and a new ChunkReader for r configured by config. +func NewConfig(r io.Reader, config Config) (*ChunkReader, error) { + if config.MinBufLen == 0 { + // By historical reasons Postgres currently has 8KB send buffer inside, + // so here we want to have at least the same size buffer. + // @see https://github.com/postgres/postgres/blob/249d64999615802752940e017ee5166e726bc7cd/src/backend/libpq/pqcomm.c#L134 + // @see https://www.postgresql.org/message-id/0cdc5485-cb3c-5e16-4a46-e3b2f7a41322%40ya.ru + config.MinBufLen = 8192 + } + + return &ChunkReader{ + r: r, + buf: make([]byte, config.MinBufLen), + config: config, + }, nil +} + +// Next returns buf filled with the next n bytes. The caller gains ownership of buf. It is not necessary to make a copy +// of buf. If an error occurs, buf will be nil. +func (r *ChunkReader) Next(n int) (buf []byte, err error) { + // n bytes already in buf + if (r.wp - r.rp) >= n { + buf = r.buf[r.rp : r.rp+n] + r.rp += n + return buf, err + } + + // available space in buf is less than n + if len(r.buf) < n { + r.copyBufContents(r.newBuf(n)) + } + + // buf is large enough, but need to shift filled area to start to make enough contiguous space + minReadCount := n - (r.wp - r.rp) + if (len(r.buf) - r.wp) < minReadCount { + newBuf := r.newBuf(n) + r.copyBufContents(newBuf) + } + + if err := r.appendAtLeast(minReadCount); err != nil { + return nil, err + } + + buf = r.buf[r.rp : r.rp+n] + r.rp += n + return buf, nil +} + +func (r *ChunkReader) appendAtLeast(fillLen int) error { + n, err := io.ReadAtLeast(r.r, r.buf[r.wp:], fillLen) + r.wp += n + return err +} + +func (r *ChunkReader) newBuf(size int) []byte { + if size < r.config.MinBufLen { + size = r.config.MinBufLen + } + return make([]byte, size) +} + +func (r *ChunkReader) copyBufContents(dest []byte) { + r.wp = copy(dest, r.buf[r.rp:r.wp]) + r.rp = 0 + r.buf = dest +} diff --git a/github.com/jackc/chunkreader/v2/go.mod b/github.com/jackc/chunkreader/v2/go.mod new file mode 100644 index 0000000000..a1384b4075 --- /dev/null +++ b/github.com/jackc/chunkreader/v2/go.mod @@ -0,0 +1,3 @@ +module github.com/jackc/chunkreader/v2 + +go 1.12 diff --git a/github.com/jackc/pgconn/.gitignore b/github.com/jackc/pgconn/.gitignore new file mode 100644 index 0000000000..e980f5555e --- /dev/null +++ b/github.com/jackc/pgconn/.gitignore @@ -0,0 +1,3 @@ +.envrc +vendor/ +.vscode diff --git a/github.com/jackc/pgconn/.travis.yml b/github.com/jackc/pgconn/.travis.yml new file mode 100644 index 0000000000..0371101f1a --- /dev/null +++ b/github.com/jackc/pgconn/.travis.yml @@ -0,0 +1,49 @@ +language: go + +go: + - 1.14.x + - 1.13.x + - tip + +git: + depth: 1 + +# Derived from https://github.com/lib/pq/blob/master/.travis.yml +before_install: + - ./travis/before_install.bash + +env: + global: + - GO111MODULE=on + - GOPROXY=https://proxy.golang.org + - GOFLAGS=-mod=readonly + - PGX_TEST_CONN_STRING=postgres://pgx_md5:secret@127.0.0.1/pgx_test + - PGX_TEST_UNIX_SOCKET_CONN_STRING="host=/var/run/postgresql dbname=pgx_test" + - PGX_TEST_TCP_CONN_STRING=postgres://pgx_md5:secret@127.0.0.1/pgx_test + - PGX_TEST_TLS_CONN_STRING=postgres://pgx_md5:secret@127.0.0.1/pgx_test?sslmode=require + - PGX_TEST_MD5_PASSWORD_CONN_STRING=postgres://pgx_md5:secret@127.0.0.1/pgx_test + - PGX_TEST_PLAIN_PASSWORD_CONN_STRING=postgres://pgx_pw:secret@127.0.0.1/pgx_test + matrix: + - CRATEVERSION=2.1 PGX_TEST_CRATEDB_CONN_STRING="host=127.0.0.1 port=6543 user=pgx dbname=pgx_test" + - PGVERSION=12 + - PGVERSION=11 + - PGVERSION=10 + - PGVERSION=9.6 + - PGVERSION=9.5 + +cache: + directories: + - $HOME/.cache/go-build + - $HOME/gopath/pkg/mod + +before_script: + - ./travis/before_script.bash + +install: go mod download + +script: + - ./travis/script.bash + +matrix: + allow_failures: + - go: tip diff --git a/github.com/jackc/pgconn/CHANGELOG.md b/github.com/jackc/pgconn/CHANGELOG.md new file mode 100644 index 0000000000..253763010a --- /dev/null +++ b/github.com/jackc/pgconn/CHANGELOG.md @@ -0,0 +1,70 @@ +# 1.6.1 (June 27, 2020) + +* Update golang.org/x/crypto to latest +* Update golang.org/x/text to 0.3.3 +* Fix error handling for bad PGSERVICE definition +* Redact passwords in ParseConfig errors (Lukas Vogel) + +# 1.6.0 (June 6, 2020) + +* Fix panic when closing conn during cancellable query +* Fix behavior of sslmode=require with sslrootcert present (Petr Jediný) +* Fix field descriptions available after command concluded (Tobias Salzmann) +* Support connect_timeout (georgysavva) +* Handle IPv6 in connection URLs (Lukas Vogel) +* Fix ValidateConnect with cancelable context +* Improve CopyFrom performance +* Add Config.Copy (georgysavva) + +# 1.5.0 (March 30, 2020) + +* Update golang.org/x/crypto for security fix +* Implement "verify-ca" SSL mode (Greg Curtis) + +# 1.4.0 (March 7, 2020) + +* Fix ExecParams and ExecPrepared handling of empty query. +* Support reading config from PostgreSQL service files. + +# 1.3.2 (February 14, 2020) + +* Update chunkreader to v2.0.1 for optimized default buffer size. + +# 1.3.1 (February 5, 2020) + +* Fix CopyFrom deadlock when multiple NoticeResponse received during copy + +# 1.3.0 (January 23, 2020) + +* Add Hijack and Construct. +* Update pgproto3 to v2.0.1. + +# 1.2.1 (January 13, 2020) + +* Fix data race in context cancellation introduced in v1.2.0. + +# 1.2.0 (January 11, 2020) + +## Features + +* Add Insert(), Update(), Delete(), and Select() statement type query methods to CommandTag. +* Add PgError.SQLState method. This could be used for compatibility with other drivers and databases. + +## Performance + +* Improve performance when context.Background() is used. (bakape) +* CommandTag.RowsAffected is faster and does not allocate. + +## Fixes + +* Try to cancel any in-progress query when a conn is closed by ctx cancel. +* Handle NoticeResponse during CopyFrom. +* Ignore errors sending Terminate message while closing connection. This mimics the behavior of libpq PGfinish. + +# 1.1.0 (October 12, 2019) + +* Add PgConn.IsBusy() method. + +# 1.0.1 (September 19, 2019) + +* Fix statement cache not properly cleaning discarded statements. diff --git a/github.com/jackc/pgconn/LICENSE b/github.com/jackc/pgconn/LICENSE new file mode 100644 index 0000000000..c1c4f50fc6 --- /dev/null +++ b/github.com/jackc/pgconn/LICENSE @@ -0,0 +1,22 @@ +Copyright (c) 2019 Jack Christensen + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/github.com/jackc/pgconn/README.md b/github.com/jackc/pgconn/README.md new file mode 100644 index 0000000000..5d14e91404 --- /dev/null +++ b/github.com/jackc/pgconn/README.md @@ -0,0 +1,56 @@ +[![](https://godoc.org/github.com/jackc/pgconn?status.svg)](https://godoc.org/github.com/jackc/pgconn) +[![Build Status](https://travis-ci.org/jackc/pgconn.svg)](https://travis-ci.org/jackc/pgconn) + +# pgconn + +Package pgconn is a low-level PostgreSQL database driver. It operates at nearly the same level as the C library libpq. +It is primarily intended to serve as the foundation for higher level libraries such as https://github.com/jackc/pgx. +Applications should handle normal queries with a higher level library and only use pgconn directly when required for +low-level access to PostgreSQL functionality. + +## Example Usage + +```go +pgConn, err := pgconn.Connect(context.Background(), os.Getenv("DATABASE_URL")) +if err != nil { + log.Fatalln("pgconn failed to connect:", err) +} +defer pgConn.Close() + +result := pgConn.ExecParams(context.Background(), "SELECT email FROM users WHERE id=$1", [][]byte{[]byte("123")}, nil, nil, nil) +for result.NextRow() { + fmt.Println("User 123 has email:", string(result.Values()[0])) +} +_, err := result.Close() +if err != nil { + log.Fatalln("failed reading result:", err) +}) +``` + +## Testing + +The pgconn tests require a PostgreSQL database. It will connect to the database specified in the `PGX_TEST_CONN_STRING` +environment variable. The `PGX_TEST_CONN_STRING` environment variable can be a URL or DSN. In addition, the standard `PG*` +environment variables will be respected. Consider using [direnv](https://github.com/direnv/direnv) to simplify +environment variable handling. + +### Example Test Environment + +Connect to your PostgreSQL server and run: + +``` +create database pgx_test; +``` + +Now you can run the tests: + +```bash +PGX_TEST_CONN_STRING="host=/var/run/postgresql dbname=pgx_test" go test ./... +``` + +### Connection and Authentication Tests + +Pgconn supports multiple connection types and means of authentication. These tests are optional. They +will only run if the appropriate environment variable is set. Run `go test -v | grep SKIP` to see if any tests are being +skipped. Most developers will not need to enable these tests. See `travis.yml` for an example set up if you need change +authentication code. diff --git a/github.com/jackc/pgconn/auth_scram.go b/github.com/jackc/pgconn/auth_scram.go new file mode 100644 index 0000000000..665fc2c243 --- /dev/null +++ b/github.com/jackc/pgconn/auth_scram.go @@ -0,0 +1,266 @@ +// SCRAM-SHA-256 authentication +// +// Resources: +// https://tools.ietf.org/html/rfc5802 +// https://tools.ietf.org/html/rfc8265 +// https://www.postgresql.org/docs/current/sasl-authentication.html +// +// Inspiration drawn from other implementations: +// https://github.com/lib/pq/pull/608 +// https://github.com/lib/pq/pull/788 +// https://github.com/lib/pq/pull/833 + +package pgconn + +import ( + "bytes" + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "fmt" + "strconv" + + "github.com/jackc/pgproto3/v2" + "golang.org/x/crypto/pbkdf2" + "golang.org/x/text/secure/precis" + errors "golang.org/x/xerrors" +) + +const clientNonceLen = 18 + +// Perform SCRAM authentication. +func (c *PgConn) scramAuth(serverAuthMechanisms []string) error { + sc, err := newScramClient(serverAuthMechanisms, c.config.Password) + if err != nil { + return err + } + + // Send client-first-message in a SASLInitialResponse + saslInitialResponse := &pgproto3.SASLInitialResponse{ + AuthMechanism: "SCRAM-SHA-256", + Data: sc.clientFirstMessage(), + } + _, err = c.conn.Write(saslInitialResponse.Encode(nil)) + if err != nil { + return err + } + + // Receive server-first-message payload in a AuthenticationSASLContinue. + saslContinue, err := c.rxSASLContinue() + if err != nil { + return err + } + err = sc.recvServerFirstMessage(saslContinue.Data) + if err != nil { + return err + } + + // Send client-final-message in a SASLResponse + saslResponse := &pgproto3.SASLResponse{ + Data: []byte(sc.clientFinalMessage()), + } + _, err = c.conn.Write(saslResponse.Encode(nil)) + if err != nil { + return err + } + + // Receive server-final-message payload in a AuthenticationSASLFinal. + saslFinal, err := c.rxSASLFinal() + if err != nil { + return err + } + return sc.recvServerFinalMessage(saslFinal.Data) +} + +func (c *PgConn) rxSASLContinue() (*pgproto3.AuthenticationSASLContinue, error) { + msg, err := c.receiveMessage() + if err != nil { + return nil, err + } + saslContinue, ok := msg.(*pgproto3.AuthenticationSASLContinue) + if ok { + return saslContinue, nil + } + + return nil, errors.New("expected AuthenticationSASLContinue message but received unexpected message") +} + +func (c *PgConn) rxSASLFinal() (*pgproto3.AuthenticationSASLFinal, error) { + msg, err := c.receiveMessage() + if err != nil { + return nil, err + } + saslFinal, ok := msg.(*pgproto3.AuthenticationSASLFinal) + if ok { + return saslFinal, nil + } + + return nil, errors.New("expected AuthenticationSASLFinal message but received unexpected message") +} + +type scramClient struct { + serverAuthMechanisms []string + password []byte + clientNonce []byte + + clientFirstMessageBare []byte + + serverFirstMessage []byte + clientAndServerNonce []byte + salt []byte + iterations int + + saltedPassword []byte + authMessage []byte +} + +func newScramClient(serverAuthMechanisms []string, password string) (*scramClient, error) { + sc := &scramClient{ + serverAuthMechanisms: serverAuthMechanisms, + } + + // Ensure server supports SCRAM-SHA-256 + hasScramSHA256 := false + for _, mech := range sc.serverAuthMechanisms { + if mech == "SCRAM-SHA-256" { + hasScramSHA256 = true + break + } + } + if !hasScramSHA256 { + return nil, errors.New("server does not support SCRAM-SHA-256") + } + + // precis.OpaqueString is equivalent to SASLprep for password. + var err error + sc.password, err = precis.OpaqueString.Bytes([]byte(password)) + if err != nil { + // PostgreSQL allows passwords invalid according to SCRAM / SASLprep. + sc.password = []byte(password) + } + + buf := make([]byte, clientNonceLen) + _, err = rand.Read(buf) + if err != nil { + return nil, err + } + sc.clientNonce = make([]byte, base64.RawStdEncoding.EncodedLen(len(buf))) + base64.RawStdEncoding.Encode(sc.clientNonce, buf) + + return sc, nil +} + +func (sc *scramClient) clientFirstMessage() []byte { + sc.clientFirstMessageBare = []byte(fmt.Sprintf("n=,r=%s", sc.clientNonce)) + return []byte(fmt.Sprintf("n,,%s", sc.clientFirstMessageBare)) +} + +func (sc *scramClient) recvServerFirstMessage(serverFirstMessage []byte) error { + sc.serverFirstMessage = serverFirstMessage + buf := serverFirstMessage + if !bytes.HasPrefix(buf, []byte("r=")) { + return errors.New("invalid SCRAM server-first-message received from server: did not include r=") + } + buf = buf[2:] + + idx := bytes.IndexByte(buf, ',') + if idx == -1 { + return errors.New("invalid SCRAM server-first-message received from server: did not include s=") + } + sc.clientAndServerNonce = buf[:idx] + buf = buf[idx+1:] + + if !bytes.HasPrefix(buf, []byte("s=")) { + return errors.New("invalid SCRAM server-first-message received from server: did not include s=") + } + buf = buf[2:] + + idx = bytes.IndexByte(buf, ',') + if idx == -1 { + return errors.New("invalid SCRAM server-first-message received from server: did not include i=") + } + saltStr := buf[:idx] + buf = buf[idx+1:] + + if !bytes.HasPrefix(buf, []byte("i=")) { + return errors.New("invalid SCRAM server-first-message received from server: did not include i=") + } + buf = buf[2:] + iterationsStr := buf + + var err error + sc.salt, err = base64.StdEncoding.DecodeString(string(saltStr)) + if err != nil { + return errors.Errorf("invalid SCRAM salt received from server: %w", err) + } + + sc.iterations, err = strconv.Atoi(string(iterationsStr)) + if err != nil || sc.iterations <= 0 { + return errors.Errorf("invalid SCRAM iteration count received from server: %w", err) + } + + if !bytes.HasPrefix(sc.clientAndServerNonce, sc.clientNonce) { + return errors.New("invalid SCRAM nonce: did not start with client nonce") + } + + if len(sc.clientAndServerNonce) <= len(sc.clientNonce) { + return errors.New("invalid SCRAM nonce: did not include server nonce") + } + + return nil +} + +func (sc *scramClient) clientFinalMessage() string { + clientFinalMessageWithoutProof := []byte(fmt.Sprintf("c=biws,r=%s", sc.clientAndServerNonce)) + + sc.saltedPassword = pbkdf2.Key([]byte(sc.password), sc.salt, sc.iterations, 32, sha256.New) + sc.authMessage = bytes.Join([][]byte{sc.clientFirstMessageBare, sc.serverFirstMessage, clientFinalMessageWithoutProof}, []byte(",")) + + clientProof := computeClientProof(sc.saltedPassword, sc.authMessage) + + return fmt.Sprintf("%s,p=%s", clientFinalMessageWithoutProof, clientProof) +} + +func (sc *scramClient) recvServerFinalMessage(serverFinalMessage []byte) error { + if !bytes.HasPrefix(serverFinalMessage, []byte("v=")) { + return errors.New("invalid SCRAM server-final-message received from server") + } + + serverSignature := serverFinalMessage[2:] + + if !hmac.Equal(serverSignature, computeServerSignature(sc.saltedPassword, sc.authMessage)) { + return errors.New("invalid SCRAM ServerSignature received from server") + } + + return nil +} + +func computeHMAC(key, msg []byte) []byte { + mac := hmac.New(sha256.New, key) + mac.Write(msg) + return mac.Sum(nil) +} + +func computeClientProof(saltedPassword, authMessage []byte) []byte { + clientKey := computeHMAC(saltedPassword, []byte("Client Key")) + storedKey := sha256.Sum256(clientKey) + clientSignature := computeHMAC(storedKey[:], authMessage) + + clientProof := make([]byte, len(clientSignature)) + for i := 0; i < len(clientSignature); i++ { + clientProof[i] = clientKey[i] ^ clientSignature[i] + } + + buf := make([]byte, base64.StdEncoding.EncodedLen(len(clientProof))) + base64.StdEncoding.Encode(buf, clientProof) + return buf +} + +func computeServerSignature(saltedPassword []byte, authMessage []byte) []byte { + serverKey := computeHMAC(saltedPassword, []byte("Server Key")) + serverSignature := computeHMAC(serverKey, authMessage) + buf := make([]byte, base64.StdEncoding.EncodedLen(len(serverSignature))) + base64.StdEncoding.Encode(buf, serverSignature) + return buf +} diff --git a/github.com/jackc/pgconn/config.go b/github.com/jackc/pgconn/config.go new file mode 100644 index 0000000000..44953e0f0a --- /dev/null +++ b/github.com/jackc/pgconn/config.go @@ -0,0 +1,758 @@ +package pgconn + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "io" + "io/ioutil" + "math" + "net" + "net/url" + "os" + "os/user" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/jackc/chunkreader/v2" + "github.com/jackc/pgpassfile" + "github.com/jackc/pgproto3/v2" + "github.com/jackc/pgservicefile" + errors "golang.org/x/xerrors" +) + +type AfterConnectFunc func(ctx context.Context, pgconn *PgConn) error +type ValidateConnectFunc func(ctx context.Context, pgconn *PgConn) error + +// Config is the settings used to establish a connection to a PostgreSQL server. It must be created by ParseConfig. A +// manually initialized Config will cause ConnectConfig to panic. +type Config struct { + Host string // host (e.g. localhost) or absolute path to unix domain socket directory (e.g. /private/tmp) + Port uint16 + Database string + User string + Password string + TLSConfig *tls.Config // nil disables TLS + ConnectTimeout time.Duration + DialFunc DialFunc // e.g. net.Dialer.DialContext + LookupFunc LookupFunc // e.g. net.Resolver.LookupHost + BuildFrontend BuildFrontendFunc + RuntimeParams map[string]string // Run-time parameters to set on connection as session default values (e.g. search_path or application_name) + + Fallbacks []*FallbackConfig + + // ValidateConnect is called during a connection attempt after a successful authentication with the PostgreSQL server. + // It can be used to validate that the server is acceptable. If this returns an error the connection is closed and the next + // fallback config is tried. This allows implementing high availability behavior such as libpq does with target_session_attrs. + ValidateConnect ValidateConnectFunc + + // AfterConnect is called after ValidateConnect. It can be used to set up the connection (e.g. Set session variables + // or prepare statements). If this returns an error the connection attempt fails. + AfterConnect AfterConnectFunc + + // OnNotice is a callback function called when a notice response is received. + OnNotice NoticeHandler + + // OnNotification is a callback function called when a notification from the LISTEN/NOTIFY system is received. + OnNotification NotificationHandler + + createdByParseConfig bool // Used to enforce created by ParseConfig rule. +} + +// Copy returns a deep copy of the config that is safe to use and modify. +// The only exception is the TLSConfig field: +// according to the tls.Config docs it must not be modified after creation. +func (c *Config) Copy() *Config { + newConf := new(Config) + *newConf = *c + if newConf.TLSConfig != nil { + newConf.TLSConfig = c.TLSConfig.Clone() + } + if newConf.RuntimeParams != nil { + newConf.RuntimeParams = make(map[string]string, len(c.RuntimeParams)) + for k, v := range c.RuntimeParams { + newConf.RuntimeParams[k] = v + } + } + if newConf.Fallbacks != nil { + newConf.Fallbacks = make([]*FallbackConfig, len(c.Fallbacks)) + for i, fallback := range c.Fallbacks { + newFallback := new(FallbackConfig) + *newFallback = *fallback + if newFallback.TLSConfig != nil { + newFallback.TLSConfig = fallback.TLSConfig.Clone() + } + newConf.Fallbacks[i] = newFallback + } + } + return newConf +} + +// FallbackConfig is additional settings to attempt a connection with when the primary Config fails to establish a +// network connection. It is used for TLS fallback such as sslmode=prefer and high availability (HA) connections. +type FallbackConfig struct { + Host string // host (e.g. localhost) or path to unix domain socket directory (e.g. /private/tmp) + Port uint16 + TLSConfig *tls.Config // nil disables TLS +} + +// NetworkAddress converts a PostgreSQL host and port into network and address suitable for use with +// net.Dial. +func NetworkAddress(host string, port uint16) (network, address string) { + if strings.HasPrefix(host, "/") { + network = "unix" + address = filepath.Join(host, ".s.PGSQL.") + strconv.FormatInt(int64(port), 10) + } else { + network = "tcp" + address = net.JoinHostPort(host, strconv.Itoa(int(port))) + } + return network, address +} + +// ParseConfig builds a *Config with similar behavior to the PostgreSQL standard C library libpq. It uses the same +// defaults as libpq (e.g. port=5432) and understands most PG* environment variables. connString may be a URL or a DSN. +// It also may be empty to only read from the environment. If a password is not supplied it will attempt to read the +// .pgpass file. +// +// # Example DSN +// user=jack password=secret host=pg.example.com port=5432 dbname=mydb sslmode=verify-ca +// +// # Example URL +// postgres://jack:secret@pg.example.com:5432/mydb?sslmode=verify-ca +// +// The returned *Config may be modified. However, it is strongly recommended that any configuration that can be done +// through the connection string be done there. In particular the fields Host, Port, TLSConfig, and Fallbacks can be +// interdependent (e.g. TLSConfig needs knowledge of the host to validate the server certificate). These fields should +// not be modified individually. They should all be modified or all left unchanged. +// +// ParseConfig supports specifying multiple hosts in similar manner to libpq. Host and port may include comma separated +// values that will be tried in order. This can be used as part of a high availability system. See +// https://www.postgresql.org/docs/11/libpq-connect.html#LIBPQ-MULTIPLE-HOSTS for more information. +// +// # Example URL +// postgres://jack:secret@foo.example.com:5432,bar.example.com:5432/mydb +// +// ParseConfig currently recognizes the following environment variable and their parameter key word equivalents passed +// via database URL or DSN: +// +// PGHOST +// PGPORT +// PGDATABASE +// PGUSER +// PGPASSWORD +// PGPASSFILE +// PGSERVICE +// PGSERVICEFILE +// PGSSLMODE +// PGSSLCERT +// PGSSLKEY +// PGSSLROOTCERT +// PGAPPNAME +// PGCONNECT_TIMEOUT +// PGTARGETSESSIONATTRS +// +// See http://www.postgresql.org/docs/11/static/libpq-envars.html for details on the meaning of environment variables. +// +// See https://www.postgresql.org/docs/11/libpq-connect.html#LIBPQ-PARAMKEYWORDS for parameter key word names. They are +// usually but not always the environment variable name downcased and without the "PG" prefix. +// +// Important Security Notes: +// +// ParseConfig tries to match libpq behavior with regard to PGSSLMODE. This includes defaulting to "prefer" behavior if +// not set. +// +// See http://www.postgresql.org/docs/11/static/libpq-ssl.html#LIBPQ-SSL-PROTECTION for details on what level of +// security each sslmode provides. +// +// The sslmode "prefer" (the default), sslmode "allow", and multiple hosts are implemented via the Fallbacks field of +// the Config struct. If TLSConfig is manually changed it will not affect the fallbacks. For example, in the case of +// sslmode "prefer" this means it will first try the main Config settings which use TLS, then it will try the fallback +// which does not use TLS. This can lead to an unexpected unencrypted connection if the main TLS config is manually +// changed later but the unencrypted fallback is present. Ensure there are no stale fallbacks when manually setting +// TLCConfig. +// +// Other known differences with libpq: +// +// If a host name resolves into multiple addresses, libpq will try all addresses. pgconn will only try the first. +// +// When multiple hosts are specified, libpq allows them to have different passwords set via the .pgpass file. pgconn +// does not. +// +// In addition, ParseConfig accepts the following options: +// +// min_read_buffer_size +// The minimum size of the internal read buffer. Default 8192. +// servicefile +// libpq only reads servicefile from the PGSERVICEFILE environment variable. ParseConfig accepts servicefile as a +// part of the connection string. +func ParseConfig(connString string) (*Config, error) { + defaultSettings := defaultSettings() + envSettings := parseEnvSettings() + + connStringSettings := make(map[string]string) + if connString != "" { + var err error + // connString may be a database URL or a DSN + if strings.HasPrefix(connString, "postgres://") || strings.HasPrefix(connString, "postgresql://") { + connStringSettings, err = parseURLSettings(connString) + if err != nil { + return nil, &parseConfigError{connString: connString, msg: "failed to parse as URL", err: err} + } + } else { + connStringSettings, err = parseDSNSettings(connString) + if err != nil { + return nil, &parseConfigError{connString: connString, msg: "failed to parse as DSN", err: err} + } + } + } + + settings := mergeSettings(defaultSettings, envSettings, connStringSettings) + if service, present := settings["service"]; present { + serviceSettings, err := parseServiceSettings(settings["servicefile"], service) + if err != nil { + return nil, &parseConfigError{connString: connString, msg: "failed to read service", err: err} + } + + settings = mergeSettings(defaultSettings, envSettings, serviceSettings, connStringSettings) + } + + minReadBufferSize, err := strconv.ParseInt(settings["min_read_buffer_size"], 10, 32) + if err != nil { + return nil, &parseConfigError{connString: connString, msg: "cannot parse min_read_buffer_size", err: err} + } + + config := &Config{ + createdByParseConfig: true, + Database: settings["database"], + User: settings["user"], + Password: settings["password"], + RuntimeParams: make(map[string]string), + BuildFrontend: makeDefaultBuildFrontendFunc(int(minReadBufferSize)), + } + + if connectTimeoutSetting, present := settings["connect_timeout"]; present { + connectTimeout, err := parseConnectTimeoutSetting(connectTimeoutSetting) + if err != nil { + return nil, &parseConfigError{connString: connString, msg: "invalid connect_timeout", err: err} + } + config.ConnectTimeout = connectTimeout + config.DialFunc = makeConnectTimeoutDialFunc(connectTimeout) + } else { + defaultDialer := makeDefaultDialer() + config.DialFunc = defaultDialer.DialContext + } + + config.LookupFunc = makeDefaultResolver().LookupHost + + notRuntimeParams := map[string]struct{}{ + "host": struct{}{}, + "port": struct{}{}, + "database": struct{}{}, + "user": struct{}{}, + "password": struct{}{}, + "passfile": struct{}{}, + "connect_timeout": struct{}{}, + "sslmode": struct{}{}, + "sslkey": struct{}{}, + "sslcert": struct{}{}, + "sslrootcert": struct{}{}, + "target_session_attrs": struct{}{}, + "min_read_buffer_size": struct{}{}, + "service": struct{}{}, + "servicefile": struct{}{}, + } + + for k, v := range settings { + if _, present := notRuntimeParams[k]; present { + continue + } + config.RuntimeParams[k] = v + } + + fallbacks := []*FallbackConfig{} + + hosts := strings.Split(settings["host"], ",") + ports := strings.Split(settings["port"], ",") + + for i, host := range hosts { + var portStr string + if i < len(ports) { + portStr = ports[i] + } else { + portStr = ports[0] + } + + port, err := parsePort(portStr) + if err != nil { + return nil, &parseConfigError{connString: connString, msg: "invalid port", err: err} + } + + var tlsConfigs []*tls.Config + + // Ignore TLS settings if Unix domain socket like libpq + if network, _ := NetworkAddress(host, port); network == "unix" { + tlsConfigs = append(tlsConfigs, nil) + } else { + var err error + tlsConfigs, err = configTLS(settings) + if err != nil { + return nil, &parseConfigError{connString: connString, msg: "failed to configure TLS", err: err} + } + } + + for _, tlsConfig := range tlsConfigs { + fallbacks = append(fallbacks, &FallbackConfig{ + Host: host, + Port: port, + TLSConfig: tlsConfig, + }) + } + } + + config.Host = fallbacks[0].Host + config.Port = fallbacks[0].Port + config.TLSConfig = fallbacks[0].TLSConfig + config.Fallbacks = fallbacks[1:] + + passfile, err := pgpassfile.ReadPassfile(settings["passfile"]) + if err == nil { + if config.Password == "" { + host := config.Host + if network, _ := NetworkAddress(config.Host, config.Port); network == "unix" { + host = "localhost" + } + + config.Password = passfile.FindPassword(host, strconv.Itoa(int(config.Port)), config.Database, config.User) + } + } + + if settings["target_session_attrs"] == "read-write" { + config.ValidateConnect = ValidateConnectTargetSessionAttrsReadWrite + } else if settings["target_session_attrs"] != "any" { + return nil, &parseConfigError{connString: connString, msg: fmt.Sprintf("unknown target_session_attrs value: %v", settings["target_session_attrs"])} + } + + return config, nil +} + +func defaultSettings() map[string]string { + settings := make(map[string]string) + + settings["host"] = defaultHost() + settings["port"] = "5432" + + // Default to the OS user name. Purposely ignoring err getting user name from + // OS. The client application will simply have to specify the user in that + // case (which they typically will be doing anyway). + user, err := user.Current() + if err == nil { + settings["user"] = user.Username + settings["passfile"] = filepath.Join(user.HomeDir, ".pgpass") + settings["servicefile"] = filepath.Join(user.HomeDir, ".pg_service.conf") + } + + settings["target_session_attrs"] = "any" + + settings["min_read_buffer_size"] = "8192" + + return settings +} + +// defaultHost attempts to mimic libpq's default host. libpq uses the default unix socket location on *nix and localhost +// on Windows. The default socket location is compiled into libpq. Since pgx does not have access to that default it +// checks the existence of common locations. +func defaultHost() string { + candidatePaths := []string{ + "/var/run/postgresql", // Debian + "/private/tmp", // OSX - homebrew + "/tmp", // standard PostgreSQL + } + + for _, path := range candidatePaths { + if _, err := os.Stat(path); err == nil { + return path + } + } + + return "localhost" +} + +func mergeSettings(settingSets ...map[string]string) map[string]string { + settings := make(map[string]string) + + for _, s2 := range settingSets { + for k, v := range s2 { + settings[k] = v + } + } + + return settings +} + +func parseEnvSettings() map[string]string { + settings := make(map[string]string) + + nameMap := map[string]string{ + "PGHOST": "host", + "PGPORT": "port", + "PGDATABASE": "database", + "PGUSER": "user", + "PGPASSWORD": "password", + "PGPASSFILE": "passfile", + "PGAPPNAME": "application_name", + "PGCONNECT_TIMEOUT": "connect_timeout", + "PGSSLMODE": "sslmode", + "PGSSLKEY": "sslkey", + "PGSSLCERT": "sslcert", + "PGSSLROOTCERT": "sslrootcert", + "PGTARGETSESSIONATTRS": "target_session_attrs", + "PGSERVICE": "service", + "PGSERVICEFILE": "servicefile", + } + + for envname, realname := range nameMap { + value := os.Getenv(envname) + if value != "" { + settings[realname] = value + } + } + + return settings +} + +func parseURLSettings(connString string) (map[string]string, error) { + settings := make(map[string]string) + + url, err := url.Parse(connString) + if err != nil { + return nil, err + } + + if url.User != nil { + settings["user"] = url.User.Username() + if password, present := url.User.Password(); present { + settings["password"] = password + } + } + + // Handle multiple host:port's in url.Host by splitting them into host,host,host and port,port,port. + var hosts []string + var ports []string + for _, host := range strings.Split(url.Host, ",") { + if host == "" { + continue + } + if isIPOnly(host) { + hosts = append(hosts, strings.Trim(host, "[]")) + continue + } + h, p, err := net.SplitHostPort(host) + if err != nil { + return nil, errors.Errorf("failed to split host:port in '%s', err: %w", host, err) + } + hosts = append(hosts, h) + ports = append(ports, p) + } + if len(hosts) > 0 { + settings["host"] = strings.Join(hosts, ",") + } + if len(ports) > 0 { + settings["port"] = strings.Join(ports, ",") + } + + database := strings.TrimLeft(url.Path, "/") + if database != "" { + settings["database"] = database + } + + for k, v := range url.Query() { + settings[k] = v[0] + } + + return settings, nil +} + +func isIPOnly(host string) bool { + return net.ParseIP(strings.Trim(host, "[]")) != nil || !strings.Contains(host, ":") +} + +var asciiSpace = [256]uint8{'\t': 1, '\n': 1, '\v': 1, '\f': 1, '\r': 1, ' ': 1} + +func parseDSNSettings(s string) (map[string]string, error) { + settings := make(map[string]string) + + nameMap := map[string]string{ + "dbname": "database", + } + + for len(s) > 0 { + var key, val string + eqIdx := strings.IndexRune(s, '=') + if eqIdx < 0 { + return nil, errors.New("invalid dsn") + } + + key = strings.Trim(s[:eqIdx], " \t\n\r\v\f") + s = strings.TrimLeft(s[eqIdx+1:], " \t\n\r\v\f") + if s[0] != '\'' { + end := 0 + for ; end < len(s); end++ { + if asciiSpace[s[end]] == 1 { + break + } + if s[end] == '\\' { + end++ + } + } + val = strings.Replace(strings.Replace(s[:end], "\\\\", "\\", -1), "\\'", "'", -1) + if end == len(s) { + s = "" + } else { + s = s[end+1:] + } + } else { // quoted string + s = s[1:] + end := 0 + for ; end < len(s); end++ { + if s[end] == '\'' { + break + } + if s[end] == '\\' { + end++ + } + } + if end == len(s) { + return nil, errors.New("unterminated quoted string in connection info string") + } + val = strings.Replace(strings.Replace(s[:end], "\\\\", "\\", -1), "\\'", "'", -1) + if end == len(s) { + s = "" + } else { + s = s[end+1:] + } + } + + if k, ok := nameMap[key]; ok { + key = k + } + + settings[key] = val + } + + return settings, nil +} + +func parseServiceSettings(servicefilePath, serviceName string) (map[string]string, error) { + servicefile, err := pgservicefile.ReadServicefile(servicefilePath) + if err != nil { + return nil, fmt.Errorf("failed to read service file: %v", servicefilePath) + } + + service, err := servicefile.GetService(serviceName) + if err != nil { + return nil, fmt.Errorf("unable to find service: %v", serviceName) + } + + nameMap := map[string]string{ + "dbname": "database", + } + + settings := make(map[string]string, len(service.Settings)) + for k, v := range service.Settings { + if k2, present := nameMap[k]; present { + k = k2 + } + settings[k] = v + } + + return settings, nil +} + +type pgTLSArgs struct { + sslMode string + sslRootCert string + sslCert string + sslKey string +} + +// configTLS uses libpq's TLS parameters to construct []*tls.Config. It is +// necessary to allow returning multiple TLS configs as sslmode "allow" and +// "prefer" allow fallback. +func configTLS(settings map[string]string) ([]*tls.Config, error) { + host := settings["host"] + sslmode := settings["sslmode"] + sslrootcert := settings["sslrootcert"] + sslcert := settings["sslcert"] + sslkey := settings["sslkey"] + + // Match libpq default behavior + if sslmode == "" { + sslmode = "prefer" + } + + tlsConfig := &tls.Config{} + + switch sslmode { + case "disable": + return []*tls.Config{nil}, nil + case "allow", "prefer": + tlsConfig.InsecureSkipVerify = true + case "require": + // According to PostgreSQL documentation, if a root CA file exists, + // the behavior of sslmode=require should be the same as that of verify-ca + // + // See https://www.postgresql.org/docs/12/libpq-ssl.html + if sslrootcert != "" { + goto nextCase + } + tlsConfig.InsecureSkipVerify = true + break + nextCase: + fallthrough + case "verify-ca": + // Don't perform the default certificate verification because it + // will verify the hostname. Instead, verify the server's + // certificate chain ourselves in VerifyPeerCertificate and + // ignore the server name. This emulates libpq's verify-ca + // behavior. + // + // See https://github.com/golang/go/issues/21971#issuecomment-332693931 + // and https://pkg.go.dev/crypto/tls?tab=doc#example-Config-VerifyPeerCertificate + // for more info. + tlsConfig.InsecureSkipVerify = true + tlsConfig.VerifyPeerCertificate = func(certificates [][]byte, _ [][]*x509.Certificate) error { + certs := make([]*x509.Certificate, len(certificates)) + for i, asn1Data := range certificates { + cert, err := x509.ParseCertificate(asn1Data) + if err != nil { + return errors.New("failed to parse certificate from server: " + err.Error()) + } + certs[i] = cert + } + + // Leave DNSName empty to skip hostname verification. + opts := x509.VerifyOptions{ + Roots: tlsConfig.RootCAs, + Intermediates: x509.NewCertPool(), + } + // Skip the first cert because it's the leaf. All others + // are intermediates. + for _, cert := range certs[1:] { + opts.Intermediates.AddCert(cert) + } + _, err := certs[0].Verify(opts) + return err + } + case "verify-full": + tlsConfig.ServerName = host + default: + return nil, errors.New("sslmode is invalid") + } + + if sslrootcert != "" { + caCertPool := x509.NewCertPool() + + caPath := sslrootcert + caCert, err := ioutil.ReadFile(caPath) + if err != nil { + return nil, errors.Errorf("unable to read CA file: %w", err) + } + + if !caCertPool.AppendCertsFromPEM(caCert) { + return nil, errors.Errorf("unable to add CA to cert pool: %w", err) + } + + tlsConfig.RootCAs = caCertPool + tlsConfig.ClientCAs = caCertPool + } + + if (sslcert != "" && sslkey == "") || (sslcert == "" && sslkey != "") { + return nil, errors.New(`both "sslcert" and "sslkey" are required`) + } + + if sslcert != "" && sslkey != "" { + cert, err := tls.LoadX509KeyPair(sslcert, sslkey) + if err != nil { + return nil, errors.Errorf("unable to read cert: %w", err) + } + + tlsConfig.Certificates = []tls.Certificate{cert} + } + + switch sslmode { + case "allow": + return []*tls.Config{nil, tlsConfig}, nil + case "prefer": + return []*tls.Config{tlsConfig, nil}, nil + case "require", "verify-ca", "verify-full": + return []*tls.Config{tlsConfig}, nil + default: + panic("BUG: bad sslmode should already have been caught") + } +} + +func parsePort(s string) (uint16, error) { + port, err := strconv.ParseUint(s, 10, 16) + if err != nil { + return 0, err + } + if port < 1 || port > math.MaxUint16 { + return 0, errors.New("outside range") + } + return uint16(port), nil +} + +func makeDefaultDialer() *net.Dialer { + return &net.Dialer{KeepAlive: 5 * time.Minute} +} + +func makeDefaultResolver() *net.Resolver { + return net.DefaultResolver +} + +func makeDefaultBuildFrontendFunc(minBufferLen int) BuildFrontendFunc { + return func(r io.Reader, w io.Writer) Frontend { + cr, err := chunkreader.NewConfig(r, chunkreader.Config{MinBufLen: minBufferLen}) + if err != nil { + panic(fmt.Sprintf("BUG: chunkreader.NewConfig failed: %v", err)) + } + frontend := pgproto3.NewFrontend(cr, w) + + return frontend + } +} + +func parseConnectTimeoutSetting(s string) (time.Duration, error) { + timeout, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return 0, err + } + if timeout < 0 { + return 0, errors.New("negative timeout") + } + return time.Duration(timeout) * time.Second, nil +} + +func makeConnectTimeoutDialFunc(timeout time.Duration) DialFunc { + d := makeDefaultDialer() + d.Timeout = timeout + return d.DialContext +} + +// ValidateConnectTargetSessionAttrsReadWrite is an ValidateConnectFunc that implements libpq compatible +// target_session_attrs=read-write. +func ValidateConnectTargetSessionAttrsReadWrite(ctx context.Context, pgConn *PgConn) error { + result := pgConn.ExecParams(ctx, "show transaction_read_only", nil, nil, nil, nil).Read() + if result.Err != nil { + return result.Err + } + + if string(result.Rows[0][0]) == "on" { + return errors.New("read only connection") + } + + return nil +} diff --git a/github.com/jackc/pgconn/doc.go b/github.com/jackc/pgconn/doc.go new file mode 100644 index 0000000000..cde58cd89b --- /dev/null +++ b/github.com/jackc/pgconn/doc.go @@ -0,0 +1,29 @@ +// Package pgconn is a low-level PostgreSQL database driver. +/* +pgconn provides lower level access to a PostgreSQL connection than a database/sql or pgx connection. It operates at +nearly the same level is the C library libpq. + +Establishing a Connection + +Use Connect to establish a connection. It accepts a connection string in URL or DSN and will read the environment for +libpq style environment variables. + +Executing a Query + +ExecParams and ExecPrepared execute a single query. They return readers that iterate over each row. The Read method +reads all rows into memory. + +Executing Multiple Queries in a Single Round Trip + +Exec and ExecBatch can execute multiple queries in a single round trip. They return readers that iterate over each query +result. The ReadAll method reads all query results into memory. + +Context Support + +All potentially blocking operations take a context.Context. If a context is canceled while the method is in progress the +method immediately returns. In most circumstances, this will close the underlying connection. + +The CancelRequest method may be used to request the PostgreSQL server cancel an in-progress query without forcing the +client to abort. +*/ +package pgconn diff --git a/github.com/jackc/pgconn/errors.go b/github.com/jackc/pgconn/errors.go new file mode 100644 index 0000000000..b746c82523 --- /dev/null +++ b/github.com/jackc/pgconn/errors.go @@ -0,0 +1,192 @@ +package pgconn + +import ( + "context" + "fmt" + "net" + "net/url" + "regexp" + "strings" + + errors "golang.org/x/xerrors" +) + +// SafeToRetry checks if the err is guaranteed to have occurred before sending any data to the server. +func SafeToRetry(err error) bool { + if e, ok := err.(interface{ SafeToRetry() bool }); ok { + return e.SafeToRetry() + } + return false +} + +// Timeout checks if err was was caused by a timeout. To be specific, it is true if err is or was caused by a +// context.Canceled, context.Canceled or an implementer of net.Error where Timeout() is true. +func Timeout(err error) bool { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return true + } + + var netErr net.Error + return errors.As(err, &netErr) && netErr.Timeout() +} + +// PgError represents an error reported by the PostgreSQL server. See +// http://www.postgresql.org/docs/11/static/protocol-error-fields.html for +// detailed field description. +type PgError struct { + Severity string + Code string + Message string + Detail string + Hint string + Position int32 + InternalPosition int32 + InternalQuery string + Where string + SchemaName string + TableName string + ColumnName string + DataTypeName string + ConstraintName string + File string + Line int32 + Routine string +} + +func (pe *PgError) Error() string { + return pe.Severity + ": " + pe.Message + " (SQLSTATE " + pe.Code + ")" +} + +// SQLState returns the SQLState of the error. +func (pe *PgError) SQLState() string { + return pe.Code +} + +type connectError struct { + config *Config + msg string + err error +} + +func (e *connectError) Error() string { + sb := &strings.Builder{} + fmt.Fprintf(sb, "failed to connect to `host=%s user=%s database=%s`: %s", e.config.Host, e.config.User, e.config.Database, e.msg) + if e.err != nil { + fmt.Fprintf(sb, " (%s)", e.err.Error()) + } + return sb.String() +} + +func (e *connectError) Unwrap() error { + return e.err +} + +type connLockError struct { + status string +} + +func (e *connLockError) SafeToRetry() bool { + return true // a lock failure by definition happens before the connection is used. +} + +func (e *connLockError) Error() string { + return e.status +} + +type parseConfigError struct { + connString string + msg string + err error +} + +func (e *parseConfigError) Error() string { + connString := redactPW(e.connString) + if e.err == nil { + return fmt.Sprintf("cannot parse `%s`: %s", connString, e.msg) + } + return fmt.Sprintf("cannot parse `%s`: %s (%s)", connString, e.msg, e.err.Error()) +} + +func (e *parseConfigError) Unwrap() error { + return e.err +} + +type pgconnError struct { + msg string + err error + safeToRetry bool +} + +func (e *pgconnError) Error() string { + if e.msg == "" { + return e.err.Error() + } + if e.err == nil { + return e.msg + } + return fmt.Sprintf("%s: %s", e.msg, e.err.Error()) +} + +func (e *pgconnError) SafeToRetry() bool { + return e.safeToRetry +} + +func (e *pgconnError) Unwrap() error { + return e.err +} + +type contextAlreadyDoneError struct { + err error +} + +func (e *contextAlreadyDoneError) Error() string { + return fmt.Sprintf("context already done: %s", e.err.Error()) +} + +func (e *contextAlreadyDoneError) SafeToRetry() bool { + return true +} + +func (e *contextAlreadyDoneError) Unwrap() error { + return e.err +} + +type writeError struct { + err error + safeToRetry bool +} + +func (e *writeError) Error() string { + return fmt.Sprintf("write failed: %s", e.err.Error()) +} + +func (e *writeError) SafeToRetry() bool { + return e.safeToRetry +} + +func (e *writeError) Unwrap() error { + return e.err +} + +func redactPW(connString string) string { + if strings.HasPrefix(connString, "postgres://") || strings.HasPrefix(connString, "postgresql://") { + if u, err := url.Parse(connString); err == nil { + return redactURL(u) + } + } + quotedDSN := regexp.MustCompile(`password='[^']*'`) + connString = quotedDSN.ReplaceAllLiteralString(connString, "password=xxxxx") + plainDSN := regexp.MustCompile(`password=[^ ]*`) + connString = plainDSN.ReplaceAllLiteralString(connString, "password=xxxxx") + return connString +} + +func redactURL(u *url.URL) string { + if u == nil { + return "" + } + if _, pwSet := u.User.Password(); pwSet { + u.User = url.UserPassword(u.User.Username(), "xxxxx") + } + return u.String() +} diff --git a/github.com/jackc/pgconn/go.mod b/github.com/jackc/pgconn/go.mod new file mode 100644 index 0000000000..2487c271f2 --- /dev/null +++ b/github.com/jackc/pgconn/go.mod @@ -0,0 +1,16 @@ +module github.com/jackc/pgconn + +go 1.12 + +require ( + github.com/jackc/chunkreader/v2 v2.0.1 + github.com/jackc/pgio v1.0.0 + github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2 + github.com/jackc/pgpassfile v1.0.0 + github.com/jackc/pgproto3/v2 v2.0.2 + github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8 + github.com/stretchr/testify v1.5.1 + golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 + golang.org/x/text v0.3.3 + golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 +) diff --git a/github.com/jackc/pgconn/go.sum b/github.com/jackc/pgconn/go.sum new file mode 100644 index 0000000000..2440dd48b9 --- /dev/null +++ b/github.com/jackc/pgconn/go.sum @@ -0,0 +1,121 @@ +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0 h1:DUwgMQuuPnS0rhMXenUtZpqZqrR/30NWY+qQvTpSvEs= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2 h1:JVX6jT/XfzNqIjye4717ITLaNwV9mWbJx0dLCpcRzdA= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29 h1:f2HwOeI1NIJyNFVVeh1gUISyt57iw/fmI/IXJfH3ATE= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.1 h1:Rdjp4NFjwHnEslx2b66FfCI2S0LhO4itac3hXz6WX9M= +github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.0.2 h1:q1Hsy66zh4vuNsajBUF2PNqfAMMfxU5mk594lPE9vjY= +github.com/jackc/pgproto3/v2 v2.0.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8 h1:Q3tB+ExeflWUW7AFcAhXqk40s9mnNYLk1nOkKNZ5GnU= +github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 h1:7KByu05hhLed2MO29w7p1XfZvZ13m8mub3shuVftRs0= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 h1:3zb4D3T4G8jdExgVU/95+vQXfpEPiMdCaZgmGVxjNHM= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/github.com/jackc/pgconn/internal/ctxwatch/context_watcher.go b/github.com/jackc/pgconn/internal/ctxwatch/context_watcher.go new file mode 100644 index 0000000000..391f0b7910 --- /dev/null +++ b/github.com/jackc/pgconn/internal/ctxwatch/context_watcher.go @@ -0,0 +1,64 @@ +package ctxwatch + +import ( + "context" +) + +// ContextWatcher watches a context and performs an action when the context is canceled. It can watch one context at a +// time. +type ContextWatcher struct { + onCancel func() + onUnwatchAfterCancel func() + unwatchChan chan struct{} + watchInProgress bool + onCancelWasCalled bool +} + +// NewContextWatcher returns a ContextWatcher. onCancel will be called when a watched context is canceled. +// OnUnwatchAfterCancel will be called when Unwatch is called and the watched context had already been canceled and +// onCancel called. +func NewContextWatcher(onCancel func(), onUnwatchAfterCancel func()) *ContextWatcher { + cw := &ContextWatcher{ + onCancel: onCancel, + onUnwatchAfterCancel: onUnwatchAfterCancel, + unwatchChan: make(chan struct{}), + } + + return cw +} + +// Watch starts watching ctx. If ctx is canceled then the onCancel function passed to NewContextWatcher will be called. +func (cw *ContextWatcher) Watch(ctx context.Context) { + if cw.watchInProgress { + panic("Watch already in progress") + } + + cw.onCancelWasCalled = false + + if ctx.Done() != nil { + cw.watchInProgress = true + go func() { + select { + case <-ctx.Done(): + cw.onCancel() + cw.onCancelWasCalled = true + <-cw.unwatchChan + case <-cw.unwatchChan: + } + }() + } else { + cw.watchInProgress = false + } +} + +// Unwatch stops watching the previously watched context. If the onCancel function passed to NewContextWatcher was +// called then onUnwatchAfterCancel will also be called. +func (cw *ContextWatcher) Unwatch() { + if cw.watchInProgress { + cw.unwatchChan <- struct{}{} + if cw.onCancelWasCalled { + cw.onUnwatchAfterCancel() + } + cw.watchInProgress = false + } +} diff --git a/github.com/jackc/pgconn/pgconn.go b/github.com/jackc/pgconn/pgconn.go new file mode 100644 index 0000000000..5644904a35 --- /dev/null +++ b/github.com/jackc/pgconn/pgconn.go @@ -0,0 +1,1592 @@ +package pgconn + +import ( + "context" + "crypto/md5" + "crypto/tls" + "encoding/binary" + "encoding/hex" + "io" + "math" + "net" + "strings" + "sync" + "time" + + "github.com/jackc/pgconn/internal/ctxwatch" + "github.com/jackc/pgio" + "github.com/jackc/pgproto3/v2" + errors "golang.org/x/xerrors" +) + +const ( + connStatusUninitialized = iota + connStatusConnecting + connStatusClosed + connStatusIdle + connStatusBusy +) + +const wbufLen = 1024 + +// Notice represents a notice response message reported by the PostgreSQL server. Be aware that this is distinct from +// LISTEN/NOTIFY notification. +type Notice PgError + +// Notification is a message received from the PostgreSQL LISTEN/NOTIFY system +type Notification struct { + PID uint32 // backend pid that sent the notification + Channel string // channel from which notification was received + Payload string +} + +// DialFunc is a function that can be used to connect to a PostgreSQL server. +type DialFunc func(ctx context.Context, network, addr string) (net.Conn, error) + +// LookupFunc is a function that can be used to lookup IPs addrs from host. +type LookupFunc func(ctx context.Context, host string) (addrs []string, err error) + +// BuildFrontendFunc is a function that can be used to create Frontend implementation for connection. +type BuildFrontendFunc func(r io.Reader, w io.Writer) Frontend + +// NoticeHandler is a function that can handle notices received from the PostgreSQL server. Notices can be received at +// any time, usually during handling of a query response. The *PgConn is provided so the handler is aware of the origin +// of the notice, but it must not invoke any query method. Be aware that this is distinct from LISTEN/NOTIFY +// notification. +type NoticeHandler func(*PgConn, *Notice) + +// NotificationHandler is a function that can handle notifications received from the PostgreSQL server. Notifications +// can be received at any time, usually during handling of a query response. The *PgConn is provided so the handler is +// aware of the origin of the notice, but it must not invoke any query method. Be aware that this is distinct from a +// notice event. +type NotificationHandler func(*PgConn, *Notification) + +// Frontend used to receive messages from backend. +type Frontend interface { + Receive() (pgproto3.BackendMessage, error) +} + +// PgConn is a low-level PostgreSQL connection handle. It is not safe for concurrent usage. +type PgConn struct { + conn net.Conn // the underlying TCP or unix domain socket connection + pid uint32 // backend pid + secretKey uint32 // key to use to send a cancel query message to the server + parameterStatuses map[string]string // parameters that have been reported by the server + txStatus byte + frontend Frontend + + config *Config + + status byte // One of connStatus* constants + + bufferingReceive bool + bufferingReceiveMux sync.Mutex + bufferingReceiveMsg pgproto3.BackendMessage + bufferingReceiveErr error + + // Reusable / preallocated resources + wbuf []byte // write buffer + resultReader ResultReader + multiResultReader MultiResultReader + contextWatcher *ctxwatch.ContextWatcher +} + +// Connect establishes a connection to a PostgreSQL server using the environment and connString (in URL or DSN format) +// to provide configuration. See documention for ParseConfig for details. ctx can be used to cancel a connect attempt. +func Connect(ctx context.Context, connString string) (*PgConn, error) { + config, err := ParseConfig(connString) + if err != nil { + return nil, err + } + + return ConnectConfig(ctx, config) +} + +// Connect establishes a connection to a PostgreSQL server using config. config must have been constructed with +// ParseConfig. ctx can be used to cancel a connect attempt. +// +// If config.Fallbacks are present they will sequentially be tried in case of error establishing network connection. An +// authentication error will terminate the chain of attempts (like libpq: +// https://www.postgresql.org/docs/11/libpq-connect.html#LIBPQ-MULTIPLE-HOSTS) and be returned as the error. Otherwise, +// if all attempts fail the last error is returned. +func ConnectConfig(ctx context.Context, config *Config) (pgConn *PgConn, err error) { + // Default values are set in ParseConfig. Enforce initial creation by ParseConfig rather than setting defaults from + // zero values. + if !config.createdByParseConfig { + panic("config must be created by ParseConfig") + } + + // ConnectTimeout restricts the whole connection process. + if config.ConnectTimeout != 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, config.ConnectTimeout) + defer cancel() + } + // Simplify usage by treating primary config and fallbacks the same. + fallbackConfigs := []*FallbackConfig{ + { + Host: config.Host, + Port: config.Port, + TLSConfig: config.TLSConfig, + }, + } + fallbackConfigs = append(fallbackConfigs, config.Fallbacks...) + + fallbackConfigs, err = expandWithIPs(ctx, config.LookupFunc, fallbackConfigs) + if err != nil { + return nil, &connectError{config: config, msg: "hostname resolving error", err: err} + } + + if len(fallbackConfigs) == 0 { + return nil, &connectError{config: config, msg: "hostname resolving error", err: errors.New("ip addr wasn't found")} + } + + for _, fc := range fallbackConfigs { + pgConn, err = connect(ctx, config, fc) + if err == nil { + break + } else if err, ok := err.(*PgError); ok { + return nil, &connectError{config: config, msg: "server error", err: err} + } + } + + if err != nil { + return nil, err // no need to wrap in connectError because it will already be wrapped in all cases except PgError + } + + if config.AfterConnect != nil { + err := config.AfterConnect(ctx, pgConn) + if err != nil { + pgConn.conn.Close() + return nil, &connectError{config: config, msg: "AfterConnect error", err: err} + } + } + + return pgConn, nil +} + +func expandWithIPs(ctx context.Context, lookupFn LookupFunc, fallbacks []*FallbackConfig) ([]*FallbackConfig, error) { + var configs []*FallbackConfig + + for _, fb := range fallbacks { + // skip resolve for unix sockets + if strings.HasPrefix(fb.Host, "/") { + configs = append(configs, &FallbackConfig{ + Host: fb.Host, + Port: fb.Port, + TLSConfig: fb.TLSConfig, + }) + + continue + } + + ips, err := lookupFn(ctx, fb.Host) + if err != nil { + return nil, err + } + + for _, ip := range ips { + configs = append(configs, &FallbackConfig{ + Host: ip, + Port: fb.Port, + TLSConfig: fb.TLSConfig, + }) + } + } + + return configs, nil +} + +func connect(ctx context.Context, config *Config, fallbackConfig *FallbackConfig) (*PgConn, error) { + pgConn := new(PgConn) + pgConn.config = config + pgConn.wbuf = make([]byte, 0, wbufLen) + + var err error + network, address := NetworkAddress(fallbackConfig.Host, fallbackConfig.Port) + pgConn.conn, err = config.DialFunc(ctx, network, address) + if err != nil { + return nil, &connectError{config: config, msg: "dial error", err: err} + } + + pgConn.parameterStatuses = make(map[string]string) + + if fallbackConfig.TLSConfig != nil { + if err := pgConn.startTLS(fallbackConfig.TLSConfig); err != nil { + pgConn.conn.Close() + return nil, &connectError{config: config, msg: "tls error", err: err} + } + } + + pgConn.status = connStatusConnecting + pgConn.contextWatcher = ctxwatch.NewContextWatcher( + func() { pgConn.conn.SetDeadline(time.Date(1, 1, 1, 1, 1, 1, 1, time.UTC)) }, + func() { pgConn.conn.SetDeadline(time.Time{}) }, + ) + + pgConn.contextWatcher.Watch(ctx) + defer pgConn.contextWatcher.Unwatch() + + pgConn.frontend = config.BuildFrontend(pgConn.conn, pgConn.conn) + + startupMsg := pgproto3.StartupMessage{ + ProtocolVersion: pgproto3.ProtocolVersionNumber, + Parameters: make(map[string]string), + } + + // Copy default run-time params + for k, v := range config.RuntimeParams { + startupMsg.Parameters[k] = v + } + + startupMsg.Parameters["user"] = config.User + if config.Database != "" { + startupMsg.Parameters["database"] = config.Database + } + + if _, err := pgConn.conn.Write(startupMsg.Encode(pgConn.wbuf)); err != nil { + pgConn.conn.Close() + return nil, &connectError{config: config, msg: "failed to write startup message", err: err} + } + + for { + msg, err := pgConn.receiveMessage() + if err != nil { + pgConn.conn.Close() + if err, ok := err.(*PgError); ok { + return nil, err + } + return nil, &connectError{config: config, msg: "failed to receive message", err: err} + } + + switch msg := msg.(type) { + case *pgproto3.BackendKeyData: + pgConn.pid = msg.ProcessID + pgConn.secretKey = msg.SecretKey + + case *pgproto3.AuthenticationOk: + case *pgproto3.AuthenticationCleartextPassword: + err = pgConn.txPasswordMessage(pgConn.config.Password) + if err != nil { + pgConn.conn.Close() + return nil, &connectError{config: config, msg: "failed to write password message", err: err} + } + case *pgproto3.AuthenticationMD5Password: + digestedPassword := "md5" + hexMD5(hexMD5(pgConn.config.Password+pgConn.config.User)+string(msg.Salt[:])) + err = pgConn.txPasswordMessage(digestedPassword) + if err != nil { + pgConn.conn.Close() + return nil, &connectError{config: config, msg: "failed to write password message", err: err} + } + case *pgproto3.AuthenticationSASL: + err = pgConn.scramAuth(msg.AuthMechanisms) + if err != nil { + pgConn.conn.Close() + return nil, &connectError{config: config, msg: "failed SASL auth", err: err} + } + + case *pgproto3.ReadyForQuery: + pgConn.status = connStatusIdle + if config.ValidateConnect != nil { + // ValidateConnect may execute commands that cause the context to be watched again. Unwatch first to avoid + // the watch already in progress panic. This is that last thing done by this method so there is no need to + // restart the watch after ValidateConnect returns. + // + // See https://github.com/jackc/pgconn/issues/40. + pgConn.contextWatcher.Unwatch() + + err := config.ValidateConnect(ctx, pgConn) + if err != nil { + pgConn.conn.Close() + return nil, &connectError{config: config, msg: "ValidateConnect failed", err: err} + } + } + return pgConn, nil + case *pgproto3.ParameterStatus: + // handled by ReceiveMessage + case *pgproto3.ErrorResponse: + pgConn.conn.Close() + return nil, ErrorResponseToPgError(msg) + default: + pgConn.conn.Close() + return nil, &connectError{config: config, msg: "received unexpected message", err: err} + } + } +} + +func (pgConn *PgConn) startTLS(tlsConfig *tls.Config) (err error) { + err = binary.Write(pgConn.conn, binary.BigEndian, []int32{8, 80877103}) + if err != nil { + return + } + + response := make([]byte, 1) + if _, err = io.ReadFull(pgConn.conn, response); err != nil { + return + } + + if response[0] != 'S' { + return errors.New("server refused TLS connection") + } + + pgConn.conn = tls.Client(pgConn.conn, tlsConfig) + + return nil +} + +func (pgConn *PgConn) txPasswordMessage(password string) (err error) { + msg := &pgproto3.PasswordMessage{Password: password} + _, err = pgConn.conn.Write(msg.Encode(pgConn.wbuf)) + return err +} + +func hexMD5(s string) string { + hash := md5.New() + io.WriteString(hash, s) + return hex.EncodeToString(hash.Sum(nil)) +} + +func (pgConn *PgConn) signalMessage() chan struct{} { + if pgConn.bufferingReceive { + panic("BUG: signalMessage when already in progress") + } + + pgConn.bufferingReceive = true + pgConn.bufferingReceiveMux.Lock() + + ch := make(chan struct{}) + go func() { + pgConn.bufferingReceiveMsg, pgConn.bufferingReceiveErr = pgConn.frontend.Receive() + pgConn.bufferingReceiveMux.Unlock() + close(ch) + }() + + return ch +} + +// SendBytes sends buf to the PostgreSQL server. It must only be used when the connection is not busy. e.g. It is as +// error to call SendBytes while reading the result of a query. +// +// This is a very low level method that requires deep understanding of the PostgreSQL wire protocol to use correctly. +// See https://www.postgresql.org/docs/current/protocol.html. +func (pgConn *PgConn) SendBytes(ctx context.Context, buf []byte) error { + if err := pgConn.lock(); err != nil { + return err + } + defer pgConn.unlock() + + if ctx != context.Background() { + select { + case <-ctx.Done(): + return &contextAlreadyDoneError{err: ctx.Err()} + default: + } + pgConn.contextWatcher.Watch(ctx) + defer pgConn.contextWatcher.Unwatch() + } + + n, err := pgConn.conn.Write(buf) + if err != nil { + pgConn.asyncClose() + return &writeError{err: err, safeToRetry: n == 0} + } + + return nil +} + +// ReceiveMessage receives one wire protocol message from the PostgreSQL server. It must only be used when the +// connection is not busy. e.g. It is an error to call ReceiveMessage while reading the result of a query. The messages +// are still handled by the core pgconn message handling system so receiving a NotificationResponse will still trigger +// the OnNotification callback. +// +// This is a very low level method that requires deep understanding of the PostgreSQL wire protocol to use correctly. +// See https://www.postgresql.org/docs/current/protocol.html. +func (pgConn *PgConn) ReceiveMessage(ctx context.Context) (pgproto3.BackendMessage, error) { + if err := pgConn.lock(); err != nil { + return nil, err + } + defer pgConn.unlock() + + if ctx != context.Background() { + select { + case <-ctx.Done(): + return nil, &contextAlreadyDoneError{err: ctx.Err()} + default: + } + pgConn.contextWatcher.Watch(ctx) + defer pgConn.contextWatcher.Unwatch() + } + + msg, err := pgConn.receiveMessage() + if err != nil { + err = &pgconnError{msg: "receive message failed", err: err, safeToRetry: true} + } + return msg, err +} + +// receiveMessage receives a message without setting up context cancellation +func (pgConn *PgConn) receiveMessage() (pgproto3.BackendMessage, error) { + var msg pgproto3.BackendMessage + var err error + if pgConn.bufferingReceive { + pgConn.bufferingReceiveMux.Lock() + msg = pgConn.bufferingReceiveMsg + err = pgConn.bufferingReceiveErr + pgConn.bufferingReceiveMux.Unlock() + pgConn.bufferingReceive = false + + // If a timeout error happened in the background try the read again. + if netErr, ok := err.(net.Error); ok && netErr.Timeout() { + msg, err = pgConn.frontend.Receive() + } + } else { + msg, err = pgConn.frontend.Receive() + } + + if err != nil { + // Close on anything other than timeout error - everything else is fatal + if err, ok := err.(net.Error); !(ok && err.Timeout()) { + pgConn.asyncClose() + } + + return nil, err + } + + switch msg := msg.(type) { + case *pgproto3.ReadyForQuery: + pgConn.txStatus = msg.TxStatus + case *pgproto3.ParameterStatus: + pgConn.parameterStatuses[msg.Name] = msg.Value + case *pgproto3.ErrorResponse: + if msg.Severity == "FATAL" { + pgConn.asyncClose() + return nil, ErrorResponseToPgError(msg) + } + case *pgproto3.NoticeResponse: + if pgConn.config.OnNotice != nil { + pgConn.config.OnNotice(pgConn, noticeResponseToNotice(msg)) + } + case *pgproto3.NotificationResponse: + if pgConn.config.OnNotification != nil { + pgConn.config.OnNotification(pgConn, &Notification{PID: msg.PID, Channel: msg.Channel, Payload: msg.Payload}) + } + } + + return msg, nil +} + +// Conn returns the underlying net.Conn. +func (pgConn *PgConn) Conn() net.Conn { + return pgConn.conn +} + +// PID returns the backend PID. +func (pgConn *PgConn) PID() uint32 { + return pgConn.pid +} + +// TxStatus returns the current TxStatus as reported by the server. +func (pgConn *PgConn) TxStatus() byte { + return pgConn.txStatus +} + +// SecretKey returns the backend secret key used to send a cancel query message to the server. +func (pgConn *PgConn) SecretKey() uint32 { + return pgConn.secretKey +} + +// Close closes a connection. It is safe to call Close on a already closed connection. Close attempts a clean close by +// sending the exit message to PostgreSQL. However, this could block so ctx is available to limit the time to wait. The +// underlying net.Conn.Close() will always be called regardless of any other errors. +func (pgConn *PgConn) Close(ctx context.Context) error { + if pgConn.status == connStatusClosed { + return nil + } + pgConn.status = connStatusClosed + + defer pgConn.conn.Close() + + if ctx != context.Background() { + // Close may be called while a cancellable query is in progress. This will most often be triggered by panic when + // a defer closes the connection (possibly indirectly via a transaction or a connection pool). Unwatch to end any + // previous watch. It is safe to Unwatch regardless of whether a watch is already is progress. + // + // See https://github.com/jackc/pgconn/issues/29 + pgConn.contextWatcher.Unwatch() + + pgConn.contextWatcher.Watch(ctx) + defer pgConn.contextWatcher.Unwatch() + } + + // Ignore any errors sending Terminate message and waiting for server to close connection. + // This mimics the behavior of libpq PQfinish. It calls closePGconn which calls sendTerminateConn which purposefully + // ignores errors. + // + // See https://github.com/jackc/pgx/issues/637 + pgConn.conn.Write([]byte{'X', 0, 0, 0, 4}) + pgConn.conn.Read(make([]byte, 1)) + + return pgConn.conn.Close() +} + +// asyncClose marks the connection as closed and asynchronously sends a cancel query message and closes the underlying +// connection. +func (pgConn *PgConn) asyncClose() { + if pgConn.status == connStatusClosed { + return + } + pgConn.status = connStatusClosed + + go func() { + defer pgConn.conn.Close() + + deadline := time.Now().Add(time.Second * 15) + + ctx, cancel := context.WithDeadline(context.Background(), deadline) + defer cancel() + + pgConn.CancelRequest(ctx) + + pgConn.conn.SetDeadline(deadline) + + pgConn.conn.Write([]byte{'X', 0, 0, 0, 4}) + pgConn.conn.Read(make([]byte, 1)) + }() +} + +// IsClosed reports if the connection has been closed. +func (pgConn *PgConn) IsClosed() bool { + return pgConn.status < connStatusIdle +} + +// IsBusy reports if the connection is busy. +func (pgConn *PgConn) IsBusy() bool { + return pgConn.status == connStatusBusy +} + +// lock locks the connection. +func (pgConn *PgConn) lock() error { + switch pgConn.status { + case connStatusBusy: + return &connLockError{status: "conn busy"} // This only should be possible in case of an application bug. + case connStatusClosed: + return &connLockError{status: "conn closed"} + case connStatusUninitialized: + return &connLockError{status: "conn uninitialized"} + } + pgConn.status = connStatusBusy + return nil +} + +func (pgConn *PgConn) unlock() { + switch pgConn.status { + case connStatusBusy: + pgConn.status = connStatusIdle + case connStatusClosed: + default: + panic("BUG: cannot unlock unlocked connection") // This should only be possible if there is a bug in this package. + } +} + +// ParameterStatus returns the value of a parameter reported by the server (e.g. +// server_version). Returns an empty string for unknown parameters. +func (pgConn *PgConn) ParameterStatus(key string) string { + return pgConn.parameterStatuses[key] +} + +// CommandTag is the result of an Exec function +type CommandTag []byte + +// RowsAffected returns the number of rows affected. If the CommandTag was not +// for a row affecting command (e.g. "CREATE TABLE") then it returns 0. +func (ct CommandTag) RowsAffected() int64 { + // Find last non-digit + idx := -1 + for i := len(ct) - 1; i >= 0; i-- { + if ct[i] >= '0' && ct[i] <= '9' { + idx = i + } else { + break + } + } + + if idx == -1 { + return 0 + } + + var n int64 + for _, b := range ct[idx:] { + n = n*10 + int64(b-'0') + } + + return n +} + +func (ct CommandTag) String() string { + return string(ct) +} + +// Insert is true if the command tag starts with "INSERT". +func (ct CommandTag) Insert() bool { + return len(ct) >= 6 && + ct[0] == 'I' && + ct[1] == 'N' && + ct[2] == 'S' && + ct[3] == 'E' && + ct[4] == 'R' && + ct[5] == 'T' +} + +// Update is true if the command tag starts with "UPDATE". +func (ct CommandTag) Update() bool { + return len(ct) >= 6 && + ct[0] == 'U' && + ct[1] == 'P' && + ct[2] == 'D' && + ct[3] == 'A' && + ct[4] == 'T' && + ct[5] == 'E' +} + +// Delete is true if the command tag starts with "DELETE". +func (ct CommandTag) Delete() bool { + return len(ct) >= 6 && + ct[0] == 'D' && + ct[1] == 'E' && + ct[2] == 'L' && + ct[3] == 'E' && + ct[4] == 'T' && + ct[5] == 'E' +} + +// Select is true if the command tag starts with "SELECT". +func (ct CommandTag) Select() bool { + return len(ct) >= 6 && + ct[0] == 'S' && + ct[1] == 'E' && + ct[2] == 'L' && + ct[3] == 'E' && + ct[4] == 'C' && + ct[5] == 'T' +} + +type StatementDescription struct { + Name string + SQL string + ParamOIDs []uint32 + Fields []pgproto3.FieldDescription +} + +// Prepare creates a prepared statement. If the name is empty, the anonymous prepared statement will be used. This +// allows Prepare to also to describe statements without creating a server-side prepared statement. +func (pgConn *PgConn) Prepare(ctx context.Context, name, sql string, paramOIDs []uint32) (*StatementDescription, error) { + if err := pgConn.lock(); err != nil { + return nil, err + } + defer pgConn.unlock() + + if ctx != context.Background() { + select { + case <-ctx.Done(): + return nil, &contextAlreadyDoneError{err: ctx.Err()} + default: + } + pgConn.contextWatcher.Watch(ctx) + defer pgConn.contextWatcher.Unwatch() + } + + buf := pgConn.wbuf + buf = (&pgproto3.Parse{Name: name, Query: sql, ParameterOIDs: paramOIDs}).Encode(buf) + buf = (&pgproto3.Describe{ObjectType: 'S', Name: name}).Encode(buf) + buf = (&pgproto3.Sync{}).Encode(buf) + + n, err := pgConn.conn.Write(buf) + if err != nil { + pgConn.asyncClose() + return nil, &writeError{err: err, safeToRetry: n == 0} + } + + psd := &StatementDescription{Name: name, SQL: sql} + + var parseErr error + +readloop: + for { + msg, err := pgConn.receiveMessage() + if err != nil { + pgConn.asyncClose() + return nil, err + } + + switch msg := msg.(type) { + case *pgproto3.ParameterDescription: + psd.ParamOIDs = make([]uint32, len(msg.ParameterOIDs)) + copy(psd.ParamOIDs, msg.ParameterOIDs) + case *pgproto3.RowDescription: + psd.Fields = make([]pgproto3.FieldDescription, len(msg.Fields)) + copy(psd.Fields, msg.Fields) + case *pgproto3.ErrorResponse: + parseErr = ErrorResponseToPgError(msg) + case *pgproto3.ReadyForQuery: + break readloop + } + } + + if parseErr != nil { + return nil, parseErr + } + return psd, nil +} + +// ErrorResponseToPgError converts a wire protocol error message to a *PgError. +func ErrorResponseToPgError(msg *pgproto3.ErrorResponse) *PgError { + return &PgError{ + Severity: msg.Severity, + Code: string(msg.Code), + Message: string(msg.Message), + Detail: string(msg.Detail), + Hint: msg.Hint, + Position: msg.Position, + InternalPosition: msg.InternalPosition, + InternalQuery: string(msg.InternalQuery), + Where: string(msg.Where), + SchemaName: string(msg.SchemaName), + TableName: string(msg.TableName), + ColumnName: string(msg.ColumnName), + DataTypeName: string(msg.DataTypeName), + ConstraintName: msg.ConstraintName, + File: string(msg.File), + Line: msg.Line, + Routine: string(msg.Routine), + } +} + +func noticeResponseToNotice(msg *pgproto3.NoticeResponse) *Notice { + pgerr := ErrorResponseToPgError((*pgproto3.ErrorResponse)(msg)) + return (*Notice)(pgerr) +} + +// CancelRequest sends a cancel request to the PostgreSQL server. It returns an error if unable to deliver the cancel +// request, but lack of an error does not ensure that the query was canceled. As specified in the documentation, there +// is no way to be sure a query was canceled. See https://www.postgresql.org/docs/11/protocol-flow.html#id-1.10.5.7.9 +func (pgConn *PgConn) CancelRequest(ctx context.Context) error { + // Open a cancellation request to the same server. The address is taken from the net.Conn directly instead of reusing + // the connection config. This is important in high availability configurations where fallback connections may be + // specified or DNS may be used to load balance. + serverAddr := pgConn.conn.RemoteAddr() + cancelConn, err := pgConn.config.DialFunc(ctx, serverAddr.Network(), serverAddr.String()) + if err != nil { + return err + } + defer cancelConn.Close() + + if ctx != context.Background() { + contextWatcher := ctxwatch.NewContextWatcher( + func() { cancelConn.SetDeadline(time.Date(1, 1, 1, 1, 1, 1, 1, time.UTC)) }, + func() { cancelConn.SetDeadline(time.Time{}) }, + ) + contextWatcher.Watch(ctx) + defer contextWatcher.Unwatch() + } + + buf := make([]byte, 16) + binary.BigEndian.PutUint32(buf[0:4], 16) + binary.BigEndian.PutUint32(buf[4:8], 80877102) + binary.BigEndian.PutUint32(buf[8:12], uint32(pgConn.pid)) + binary.BigEndian.PutUint32(buf[12:16], uint32(pgConn.secretKey)) + _, err = cancelConn.Write(buf) + if err != nil { + return err + } + + _, err = cancelConn.Read(buf) + if err != io.EOF { + return err + } + + return nil +} + +// WaitForNotification waits for a LISTON/NOTIFY message to be received. It returns an error if a notification was not +// received. +func (pgConn *PgConn) WaitForNotification(ctx context.Context) error { + if err := pgConn.lock(); err != nil { + return err + } + defer pgConn.unlock() + + if ctx != context.Background() { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + pgConn.contextWatcher.Watch(ctx) + defer pgConn.contextWatcher.Unwatch() + } + + for { + msg, err := pgConn.receiveMessage() + if err != nil { + return err + } + + switch msg.(type) { + case *pgproto3.NotificationResponse: + return nil + } + } +} + +// Exec executes SQL via the PostgreSQL simple query protocol. SQL may contain multiple queries. Execution is +// implicitly wrapped in a transaction unless a transaction is already in progress or SQL contains transaction control +// statements. +// +// Prefer ExecParams unless executing arbitrary SQL that may contain multiple queries. +func (pgConn *PgConn) Exec(ctx context.Context, sql string) *MultiResultReader { + if err := pgConn.lock(); err != nil { + return &MultiResultReader{ + closed: true, + err: err, + } + } + + pgConn.multiResultReader = MultiResultReader{ + pgConn: pgConn, + ctx: ctx, + } + multiResult := &pgConn.multiResultReader + if ctx != context.Background() { + select { + case <-ctx.Done(): + multiResult.closed = true + multiResult.err = &contextAlreadyDoneError{err: ctx.Err()} + pgConn.unlock() + return multiResult + default: + } + pgConn.contextWatcher.Watch(ctx) + } + + buf := pgConn.wbuf + buf = (&pgproto3.Query{String: sql}).Encode(buf) + + n, err := pgConn.conn.Write(buf) + if err != nil { + pgConn.asyncClose() + pgConn.contextWatcher.Unwatch() + multiResult.closed = true + multiResult.err = &writeError{err: err, safeToRetry: n == 0} + pgConn.unlock() + return multiResult + } + + return multiResult +} + +// ExecParams executes a command via the PostgreSQL extended query protocol. +// +// sql is a SQL command string. It may only contain one query. Parameter substitution is positional using $1, $2, $3, +// etc. +// +// paramValues are the parameter values. It must be encoded in the format given by paramFormats. +// +// paramOIDs is a slice of data type OIDs for paramValues. If paramOIDs is nil, the server will infer the data type for +// all parameters. Any paramOID element that is 0 that will cause the server to infer the data type for that parameter. +// ExecParams will panic if len(paramOIDs) is not 0, 1, or len(paramValues). +// +// paramFormats is a slice of format codes determining for each paramValue column whether it is encoded in text or +// binary format. If paramFormats is nil all params are text format. ExecParams will panic if +// len(paramFormats) is not 0, 1, or len(paramValues). +// +// resultFormats is a slice of format codes determining for each result column whether it is encoded in text or +// binary format. If resultFormats is nil all results will be in text format. +// +// ResultReader must be closed before PgConn can be used again. +func (pgConn *PgConn) ExecParams(ctx context.Context, sql string, paramValues [][]byte, paramOIDs []uint32, paramFormats []int16, resultFormats []int16) *ResultReader { + result := pgConn.execExtendedPrefix(ctx, paramValues) + if result.closed { + return result + } + + buf := pgConn.wbuf + buf = (&pgproto3.Parse{Query: sql, ParameterOIDs: paramOIDs}).Encode(buf) + buf = (&pgproto3.Bind{ParameterFormatCodes: paramFormats, Parameters: paramValues, ResultFormatCodes: resultFormats}).Encode(buf) + + pgConn.execExtendedSuffix(buf, result) + + return result +} + +// ExecPrepared enqueues the execution of a prepared statement via the PostgreSQL extended query protocol. +// +// paramValues are the parameter values. It must be encoded in the format given by paramFormats. +// +// paramFormats is a slice of format codes determining for each paramValue column whether it is encoded in text or +// binary format. If paramFormats is nil all params are text format. ExecPrepared will panic if +// len(paramFormats) is not 0, 1, or len(paramValues). +// +// resultFormats is a slice of format codes determining for each result column whether it is encoded in text or +// binary format. If resultFormats is nil all results will be in text format. +// +// ResultReader must be closed before PgConn can be used again. +func (pgConn *PgConn) ExecPrepared(ctx context.Context, stmtName string, paramValues [][]byte, paramFormats []int16, resultFormats []int16) *ResultReader { + result := pgConn.execExtendedPrefix(ctx, paramValues) + if result.closed { + return result + } + + buf := pgConn.wbuf + buf = (&pgproto3.Bind{PreparedStatement: stmtName, ParameterFormatCodes: paramFormats, Parameters: paramValues, ResultFormatCodes: resultFormats}).Encode(buf) + + pgConn.execExtendedSuffix(buf, result) + + return result +} + +func (pgConn *PgConn) execExtendedPrefix(ctx context.Context, paramValues [][]byte) *ResultReader { + pgConn.resultReader = ResultReader{ + pgConn: pgConn, + ctx: ctx, + } + result := &pgConn.resultReader + + if err := pgConn.lock(); err != nil { + result.concludeCommand(nil, err) + result.closed = true + return result + } + + if len(paramValues) > math.MaxUint16 { + result.concludeCommand(nil, errors.Errorf("extended protocol limited to %v parameters", math.MaxUint16)) + result.closed = true + pgConn.unlock() + return result + } + + if ctx != context.Background() { + select { + case <-ctx.Done(): + result.concludeCommand(nil, &contextAlreadyDoneError{err: ctx.Err()}) + result.closed = true + pgConn.unlock() + return result + default: + } + pgConn.contextWatcher.Watch(ctx) + } + + return result +} + +func (pgConn *PgConn) execExtendedSuffix(buf []byte, result *ResultReader) { + buf = (&pgproto3.Describe{ObjectType: 'P'}).Encode(buf) + buf = (&pgproto3.Execute{}).Encode(buf) + buf = (&pgproto3.Sync{}).Encode(buf) + + n, err := pgConn.conn.Write(buf) + if err != nil { + pgConn.asyncClose() + result.concludeCommand(nil, &writeError{err: err, safeToRetry: n == 0}) + pgConn.contextWatcher.Unwatch() + result.closed = true + pgConn.unlock() + } +} + +// CopyTo executes the copy command sql and copies the results to w. +func (pgConn *PgConn) CopyTo(ctx context.Context, w io.Writer, sql string) (CommandTag, error) { + if err := pgConn.lock(); err != nil { + return nil, err + } + + if ctx != context.Background() { + select { + case <-ctx.Done(): + pgConn.unlock() + return nil, &contextAlreadyDoneError{err: ctx.Err()} + default: + } + pgConn.contextWatcher.Watch(ctx) + defer pgConn.contextWatcher.Unwatch() + } + + // Send copy to command + buf := pgConn.wbuf + buf = (&pgproto3.Query{String: sql}).Encode(buf) + + n, err := pgConn.conn.Write(buf) + if err != nil { + pgConn.asyncClose() + pgConn.unlock() + return nil, &writeError{err: err, safeToRetry: n == 0} + } + + // Read results + var commandTag CommandTag + var pgErr error + for { + msg, err := pgConn.receiveMessage() + if err != nil { + pgConn.asyncClose() + return nil, err + } + + switch msg := msg.(type) { + case *pgproto3.CopyDone: + case *pgproto3.CopyData: + _, err := w.Write(msg.Data) + if err != nil { + pgConn.asyncClose() + return nil, err + } + case *pgproto3.ReadyForQuery: + pgConn.unlock() + return commandTag, pgErr + case *pgproto3.CommandComplete: + commandTag = CommandTag(msg.CommandTag) + case *pgproto3.ErrorResponse: + pgErr = ErrorResponseToPgError(msg) + } + } +} + +// CopyFrom executes the copy command sql and copies all of r to the PostgreSQL server. +// +// Note: context cancellation will only interrupt operations on the underlying PostgreSQL network connection. Reads on r +// could still block. +func (pgConn *PgConn) CopyFrom(ctx context.Context, r io.Reader, sql string) (CommandTag, error) { + if err := pgConn.lock(); err != nil { + return nil, err + } + defer pgConn.unlock() + + if ctx != context.Background() { + select { + case <-ctx.Done(): + return nil, &contextAlreadyDoneError{err: ctx.Err()} + default: + } + pgConn.contextWatcher.Watch(ctx) + defer pgConn.contextWatcher.Unwatch() + } + + // Send copy to command + buf := pgConn.wbuf + buf = (&pgproto3.Query{String: sql}).Encode(buf) + + n, err := pgConn.conn.Write(buf) + if err != nil { + pgConn.asyncClose() + return nil, &writeError{err: err, safeToRetry: n == 0} + } + + // Read until copy in response or error. + var commandTag CommandTag + var pgErr error + pendingCopyInResponse := true + for pendingCopyInResponse { + msg, err := pgConn.receiveMessage() + if err != nil { + pgConn.asyncClose() + return nil, err + } + + switch msg := msg.(type) { + case *pgproto3.CopyInResponse: + pendingCopyInResponse = false + case *pgproto3.ErrorResponse: + pgErr = ErrorResponseToPgError(msg) + case *pgproto3.ReadyForQuery: + return commandTag, pgErr + } + } + + // Send copy data + abortCopyChan := make(chan struct{}) + copyErrChan := make(chan error, 1) + signalMessageChan := pgConn.signalMessage() + + go func() { + buf := make([]byte, 0, 65536) + buf = append(buf, 'd') + sp := len(buf) + + for { + n, readErr := r.Read(buf[5:cap(buf)]) + if n > 0 { + buf = buf[0 : n+5] + pgio.SetInt32(buf[sp:], int32(n+4)) + + _, writeErr := pgConn.conn.Write(buf) + if writeErr != nil { + // Write errors are always fatal, but we can't use asyncClose because we are in a different goroutine. + pgConn.conn.Close() + + copyErrChan <- writeErr + return + } + } + if readErr != nil { + copyErrChan <- readErr + return + } + + select { + case <-abortCopyChan: + return + default: + } + } + }() + + var copyErr error + for copyErr == nil && pgErr == nil { + select { + case copyErr = <-copyErrChan: + case <-signalMessageChan: + msg, err := pgConn.receiveMessage() + if err != nil { + pgConn.asyncClose() + return nil, err + } + + switch msg := msg.(type) { + case *pgproto3.ErrorResponse: + pgErr = ErrorResponseToPgError(msg) + default: + signalMessageChan = pgConn.signalMessage() + } + } + } + close(abortCopyChan) + + buf = buf[:0] + if copyErr == io.EOF || pgErr != nil { + copyDone := &pgproto3.CopyDone{} + buf = copyDone.Encode(buf) + } else { + copyFail := &pgproto3.CopyFail{Message: copyErr.Error()} + buf = copyFail.Encode(buf) + } + _, err = pgConn.conn.Write(buf) + if err != nil { + pgConn.asyncClose() + return nil, err + } + + // Read results + for { + msg, err := pgConn.receiveMessage() + if err != nil { + pgConn.asyncClose() + return nil, err + } + + switch msg := msg.(type) { + case *pgproto3.ReadyForQuery: + return commandTag, pgErr + case *pgproto3.CommandComplete: + commandTag = CommandTag(msg.CommandTag) + case *pgproto3.ErrorResponse: + pgErr = ErrorResponseToPgError(msg) + } + } +} + +// MultiResultReader is a reader for a command that could return multiple results such as Exec or ExecBatch. +type MultiResultReader struct { + pgConn *PgConn + ctx context.Context + + rr *ResultReader + + closed bool + err error +} + +// ReadAll reads all available results. Calling ReadAll is mutually exclusive with all other MultiResultReader methods. +func (mrr *MultiResultReader) ReadAll() ([]*Result, error) { + var results []*Result + + for mrr.NextResult() { + results = append(results, mrr.ResultReader().Read()) + } + err := mrr.Close() + + return results, err +} + +func (mrr *MultiResultReader) receiveMessage() (pgproto3.BackendMessage, error) { + msg, err := mrr.pgConn.receiveMessage() + + if err != nil { + mrr.pgConn.contextWatcher.Unwatch() + mrr.err = err + mrr.closed = true + mrr.pgConn.asyncClose() + return nil, mrr.err + } + + switch msg := msg.(type) { + case *pgproto3.ReadyForQuery: + mrr.pgConn.contextWatcher.Unwatch() + mrr.closed = true + mrr.pgConn.unlock() + case *pgproto3.ErrorResponse: + mrr.err = ErrorResponseToPgError(msg) + } + + return msg, nil +} + +// NextResult returns advances the MultiResultReader to the next result and returns true if a result is available. +func (mrr *MultiResultReader) NextResult() bool { + for !mrr.closed && mrr.err == nil { + msg, err := mrr.receiveMessage() + if err != nil { + return false + } + + switch msg := msg.(type) { + case *pgproto3.RowDescription: + mrr.pgConn.resultReader = ResultReader{ + pgConn: mrr.pgConn, + multiResultReader: mrr, + ctx: mrr.ctx, + fieldDescriptions: msg.Fields, + } + mrr.rr = &mrr.pgConn.resultReader + return true + case *pgproto3.CommandComplete: + mrr.pgConn.resultReader = ResultReader{ + commandTag: CommandTag(msg.CommandTag), + commandConcluded: true, + closed: true, + } + mrr.rr = &mrr.pgConn.resultReader + return true + case *pgproto3.EmptyQueryResponse: + return false + } + } + + return false +} + +// ResultReader returns the current ResultReader. +func (mrr *MultiResultReader) ResultReader() *ResultReader { + return mrr.rr +} + +// Close closes the MultiResultReader and returns the first error that occurred during the MultiResultReader's use. +func (mrr *MultiResultReader) Close() error { + for !mrr.closed { + _, err := mrr.receiveMessage() + if err != nil { + return mrr.err + } + } + + return mrr.err +} + +// ResultReader is a reader for the result of a single query. +type ResultReader struct { + pgConn *PgConn + multiResultReader *MultiResultReader + ctx context.Context + + fieldDescriptions []pgproto3.FieldDescription + rowValues [][]byte + commandTag CommandTag + commandConcluded bool + closed bool + err error +} + +// Result is the saved query response that is returned by calling Read on a ResultReader. +type Result struct { + FieldDescriptions []pgproto3.FieldDescription + Rows [][][]byte + CommandTag CommandTag + Err error +} + +// Read saves the query response to a Result. +func (rr *ResultReader) Read() *Result { + br := &Result{} + + for rr.NextRow() { + if br.FieldDescriptions == nil { + br.FieldDescriptions = make([]pgproto3.FieldDescription, len(rr.FieldDescriptions())) + copy(br.FieldDescriptions, rr.FieldDescriptions()) + } + + row := make([][]byte, len(rr.Values())) + copy(row, rr.Values()) + br.Rows = append(br.Rows, row) + } + + br.CommandTag, br.Err = rr.Close() + + return br +} + +// NextRow advances the ResultReader to the next row and returns true if a row is available. +func (rr *ResultReader) NextRow() bool { + for !rr.commandConcluded { + msg, err := rr.receiveMessage() + if err != nil { + return false + } + + switch msg := msg.(type) { + case *pgproto3.DataRow: + rr.rowValues = msg.Values + return true + } + } + + return false +} + +// FieldDescriptions returns the field descriptions for the current result set. The returned slice is only valid until +// the ResultReader is closed. +func (rr *ResultReader) FieldDescriptions() []pgproto3.FieldDescription { + return rr.fieldDescriptions +} + +// Values returns the current row data. NextRow must have been previously been called. The returned [][]byte is only +// valid until the next NextRow call or the ResultReader is closed. However, the underlying byte data is safe to +// retain a reference to and mutate. +func (rr *ResultReader) Values() [][]byte { + return rr.rowValues +} + +// Close consumes any remaining result data and returns the command tag or +// error. +func (rr *ResultReader) Close() (CommandTag, error) { + if rr.closed { + return rr.commandTag, rr.err + } + rr.closed = true + + for !rr.commandConcluded { + _, err := rr.receiveMessage() + if err != nil { + return nil, rr.err + } + } + + if rr.multiResultReader == nil { + for { + msg, err := rr.receiveMessage() + if err != nil { + return nil, rr.err + } + + switch msg := msg.(type) { + // Detect a deferred constraint violation where the ErrorResponse is sent after CommandComplete. + case *pgproto3.ErrorResponse: + rr.err = ErrorResponseToPgError(msg) + case *pgproto3.ReadyForQuery: + rr.pgConn.contextWatcher.Unwatch() + rr.pgConn.unlock() + return rr.commandTag, rr.err + } + } + } + + return rr.commandTag, rr.err +} + +func (rr *ResultReader) receiveMessage() (msg pgproto3.BackendMessage, err error) { + if rr.multiResultReader == nil { + msg, err = rr.pgConn.receiveMessage() + } else { + msg, err = rr.multiResultReader.receiveMessage() + } + + if err != nil { + rr.concludeCommand(nil, err) + rr.pgConn.contextWatcher.Unwatch() + rr.closed = true + if rr.multiResultReader == nil { + rr.pgConn.asyncClose() + } + + return nil, rr.err + } + + switch msg := msg.(type) { + case *pgproto3.RowDescription: + rr.fieldDescriptions = msg.Fields + case *pgproto3.CommandComplete: + rr.concludeCommand(CommandTag(msg.CommandTag), nil) + case *pgproto3.EmptyQueryResponse: + rr.concludeCommand(nil, nil) + case *pgproto3.ErrorResponse: + rr.concludeCommand(nil, ErrorResponseToPgError(msg)) + } + + return msg, nil +} + +func (rr *ResultReader) concludeCommand(commandTag CommandTag, err error) { + if rr.commandConcluded { + return + } + + rr.commandTag = commandTag + rr.err = err + rr.rowValues = nil + rr.commandConcluded = true +} + +// Batch is a collection of queries that can be sent to the PostgreSQL server in a single round-trip. +type Batch struct { + buf []byte +} + +// ExecParams appends an ExecParams command to the batch. See PgConn.ExecParams for parameter descriptions. +func (batch *Batch) ExecParams(sql string, paramValues [][]byte, paramOIDs []uint32, paramFormats []int16, resultFormats []int16) { + batch.buf = (&pgproto3.Parse{Query: sql, ParameterOIDs: paramOIDs}).Encode(batch.buf) + batch.ExecPrepared("", paramValues, paramFormats, resultFormats) +} + +// ExecPrepared appends an ExecPrepared e command to the batch. See PgConn.ExecPrepared for parameter descriptions. +func (batch *Batch) ExecPrepared(stmtName string, paramValues [][]byte, paramFormats []int16, resultFormats []int16) { + batch.buf = (&pgproto3.Bind{PreparedStatement: stmtName, ParameterFormatCodes: paramFormats, Parameters: paramValues, ResultFormatCodes: resultFormats}).Encode(batch.buf) + batch.buf = (&pgproto3.Describe{ObjectType: 'P'}).Encode(batch.buf) + batch.buf = (&pgproto3.Execute{}).Encode(batch.buf) +} + +// ExecBatch executes all the queries in batch in a single round-trip. Execution is implicitly transactional unless a +// transaction is already in progress or SQL contains transaction control statements. +func (pgConn *PgConn) ExecBatch(ctx context.Context, batch *Batch) *MultiResultReader { + if err := pgConn.lock(); err != nil { + return &MultiResultReader{ + closed: true, + err: err, + } + } + + pgConn.multiResultReader = MultiResultReader{ + pgConn: pgConn, + ctx: ctx, + } + multiResult := &pgConn.multiResultReader + + if ctx != context.Background() { + select { + case <-ctx.Done(): + multiResult.closed = true + multiResult.err = &contextAlreadyDoneError{err: ctx.Err()} + pgConn.unlock() + return multiResult + default: + } + pgConn.contextWatcher.Watch(ctx) + } + + batch.buf = (&pgproto3.Sync{}).Encode(batch.buf) + + // A large batch can deadlock without concurrent reading and writing. If the Write fails the underlying net.Conn is + // closed. This is all that can be done without introducing a race condition or adding a concurrent safe communication + // channel to relay the error back. The practical effect of this is that the underlying Write error is not reported. + // The error the code reading the batch results receives will be a closed connection error. + // + // See https://github.com/jackc/pgx/issues/374. + go func() { + _, err := pgConn.conn.Write(batch.buf) + if err != nil { + pgConn.conn.Close() + } + }() + + return multiResult +} + +// EscapeString escapes a string such that it can safely be interpolated into a SQL command string. It does not include +// the surrounding single quotes. +// +// The current implementation requires that standard_conforming_strings=on and client_encoding="UTF8". If these +// conditions are not met an error will be returned. It is possible these restrictions will be lifted in the future. +func (pgConn *PgConn) EscapeString(s string) (string, error) { + if pgConn.ParameterStatus("standard_conforming_strings") != "on" { + return "", errors.New("EscapeString must be run with standard_conforming_strings=on") + } + + if pgConn.ParameterStatus("client_encoding") != "UTF8" { + return "", errors.New("EscapeString must be run with client_encoding=UTF8") + } + + return strings.Replace(s, "'", "''", -1), nil +} + +// HijackedConn is the result of hijacking a connection. +// +// Due to the necessary exposure of internal implementation details, it is not covered by the semantic versioning +// compatibility. +type HijackedConn struct { + Conn net.Conn // the underlying TCP or unix domain socket connection + PID uint32 // backend pid + SecretKey uint32 // key to use to send a cancel query message to the server + ParameterStatuses map[string]string // parameters that have been reported by the server + TxStatus byte + Frontend Frontend + Config *Config +} + +// Hijack extracts the internal connection data. pgConn must be in an idle state. pgConn is unusable after hijacking. +// Hijacking is typically only useful when using pgconn to establish a connection, but taking complete control of the +// raw connection after that (e.g. a load balancer or proxy). +// +// Due to the necessary exposure of internal implementation details, it is not covered by the semantic versioning +// compatibility. +func (pgConn *PgConn) Hijack() (*HijackedConn, error) { + if err := pgConn.lock(); err != nil { + return nil, err + } + pgConn.status = connStatusClosed + + return &HijackedConn{ + Conn: pgConn.conn, + PID: pgConn.pid, + SecretKey: pgConn.secretKey, + ParameterStatuses: pgConn.parameterStatuses, + TxStatus: pgConn.txStatus, + Frontend: pgConn.frontend, + Config: pgConn.config, + }, nil +} + +// Construct created a PgConn from an already established connection to a PostgreSQL server. This is the inverse of +// PgConn.Hijack. The connection must be in an idle state. +// +// Due to the necessary exposure of internal implementation details, it is not covered by the semantic versioning +// compatibility. +func Construct(hc *HijackedConn) (*PgConn, error) { + pgConn := &PgConn{ + conn: hc.Conn, + pid: hc.PID, + secretKey: hc.SecretKey, + parameterStatuses: hc.ParameterStatuses, + txStatus: hc.TxStatus, + frontend: hc.Frontend, + config: hc.Config, + + status: connStatusIdle, + + wbuf: make([]byte, 0, wbufLen), + } + + pgConn.contextWatcher = ctxwatch.NewContextWatcher( + func() { pgConn.conn.SetDeadline(time.Date(1, 1, 1, 1, 1, 1, 1, time.UTC)) }, + func() { pgConn.conn.SetDeadline(time.Time{}) }, + ) + + return pgConn, nil +} diff --git a/github.com/jackc/pgconn/stmtcache/lru.go b/github.com/jackc/pgconn/stmtcache/lru.go new file mode 100644 index 0000000000..d82ced1999 --- /dev/null +++ b/github.com/jackc/pgconn/stmtcache/lru.go @@ -0,0 +1,113 @@ +package stmtcache + +import ( + "container/list" + "context" + "fmt" + "sync/atomic" + + "github.com/jackc/pgconn" +) + +var lruCount uint64 + +// LRU implements Cache with a Least Recently Used (LRU) cache. +type LRU struct { + conn *pgconn.PgConn + mode int + cap int + prepareCount int + m map[string]*list.Element + l *list.List + psNamePrefix string +} + +// NewLRU creates a new LRU. mode is either ModePrepare or ModeDescribe. cap is the maximum size of the cache. +func NewLRU(conn *pgconn.PgConn, mode int, cap int) *LRU { + mustBeValidMode(mode) + mustBeValidCap(cap) + + n := atomic.AddUint64(&lruCount, 1) + + return &LRU{ + conn: conn, + mode: mode, + cap: cap, + m: make(map[string]*list.Element), + l: list.New(), + psNamePrefix: fmt.Sprintf("lrupsc_%d", n), + } +} + +// Get returns the prepared statement description for sql preparing or describing the sql on the server as needed. +func (c *LRU) Get(ctx context.Context, sql string) (*pgconn.StatementDescription, error) { + if el, ok := c.m[sql]; ok { + c.l.MoveToFront(el) + return el.Value.(*pgconn.StatementDescription), nil + } + + if c.l.Len() == c.cap { + err := c.removeOldest(ctx) + if err != nil { + return nil, err + } + } + + psd, err := c.prepare(ctx, sql) + if err != nil { + return nil, err + } + + el := c.l.PushFront(psd) + c.m[sql] = el + + return psd, nil +} + +// Clear removes all entries in the cache. Any prepared statements will be deallocated from the PostgreSQL session. +func (c *LRU) Clear(ctx context.Context) error { + for c.l.Len() > 0 { + err := c.removeOldest(ctx) + if err != nil { + return err + } + } + + return nil +} + +// Len returns the number of cached prepared statement descriptions. +func (c *LRU) Len() int { + return c.l.Len() +} + +// Cap returns the maximum number of cached prepared statement descriptions. +func (c *LRU) Cap() int { + return c.cap +} + +// Mode returns the mode of the cache (ModePrepare or ModeDescribe) +func (c *LRU) Mode() int { + return c.mode +} + +func (c *LRU) prepare(ctx context.Context, sql string) (*pgconn.StatementDescription, error) { + var name string + if c.mode == ModePrepare { + name = fmt.Sprintf("%s_%d", c.psNamePrefix, c.prepareCount) + c.prepareCount += 1 + } + + return c.conn.Prepare(ctx, name, sql, nil) +} + +func (c *LRU) removeOldest(ctx context.Context) error { + oldest := c.l.Back() + c.l.Remove(oldest) + psd := oldest.Value.(*pgconn.StatementDescription) + delete(c.m, psd.SQL) + if c.mode == ModePrepare { + return c.conn.Exec(ctx, fmt.Sprintf("deallocate %s", psd.Name)).Close() + } + return nil +} diff --git a/github.com/jackc/pgconn/stmtcache/stmtcache.go b/github.com/jackc/pgconn/stmtcache/stmtcache.go new file mode 100644 index 0000000000..9621579919 --- /dev/null +++ b/github.com/jackc/pgconn/stmtcache/stmtcache.go @@ -0,0 +1,52 @@ +// Package stmtcache is a cache that can be used to implement lazy prepared statements. +package stmtcache + +import ( + "context" + + "github.com/jackc/pgconn" +) + +const ( + ModePrepare = iota // Cache should prepare named statements. + ModeDescribe // Cache should prepare the anonymous prepared statement to only fetch the description of the statement. +) + +// Cache prepares and caches prepared statement descriptions. +type Cache interface { + // Get returns the prepared statement description for sql preparing or describing the sql on the server as needed. + Get(ctx context.Context, sql string) (*pgconn.StatementDescription, error) + + // Clear removes all entries in the cache. Any prepared statements will be deallocated from the PostgreSQL session. + Clear(ctx context.Context) error + + // Len returns the number of cached prepared statement descriptions. + Len() int + + // Cap returns the maximum number of cached prepared statement descriptions. + Cap() int + + // Mode returns the mode of the cache (ModePrepare or ModeDescribe) + Mode() int +} + +// New returns the preferred cache implementation for mode and cap. mode is either ModePrepare or ModeDescribe. cap is +// the maximum size of the cache. +func New(conn *pgconn.PgConn, mode int, cap int) Cache { + mustBeValidMode(mode) + mustBeValidCap(cap) + + return NewLRU(conn, mode, cap) +} + +func mustBeValidMode(mode int) { + if mode != ModePrepare && mode != ModeDescribe { + panic("mode must be ModePrepare or ModeDescribe") + } +} + +func mustBeValidCap(cap int) { + if cap < 1 { + panic("cache must have cap of >= 1") + } +} diff --git a/github.com/jackc/pgio/.travis.yml b/github.com/jackc/pgio/.travis.yml new file mode 100644 index 0000000000..e176228e8e --- /dev/null +++ b/github.com/jackc/pgio/.travis.yml @@ -0,0 +1,9 @@ +language: go + +go: + - 1.x + - tip + +matrix: + allow_failures: + - go: tip diff --git a/github.com/jackc/pgio/LICENSE b/github.com/jackc/pgio/LICENSE new file mode 100644 index 0000000000..c1c4f50fc6 --- /dev/null +++ b/github.com/jackc/pgio/LICENSE @@ -0,0 +1,22 @@ +Copyright (c) 2019 Jack Christensen + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/github.com/jackc/pgio/README.md b/github.com/jackc/pgio/README.md new file mode 100644 index 0000000000..1952ed8621 --- /dev/null +++ b/github.com/jackc/pgio/README.md @@ -0,0 +1,11 @@ +[![](https://godoc.org/github.com/jackc/pgio?status.svg)](https://godoc.org/github.com/jackc/pgio) +[![Build Status](https://travis-ci.org/jackc/pgio.svg)](https://travis-ci.org/jackc/pgio) + +# pgio + +Package pgio is a low-level toolkit building messages in the PostgreSQL wire protocol. + +pgio provides functions for appending integers to a []byte while doing byte +order conversion. + +Extracted from original implementation in https://github.com/jackc/pgx. diff --git a/github.com/jackc/pgio/doc.go b/github.com/jackc/pgio/doc.go new file mode 100644 index 0000000000..ef2dcc7f72 --- /dev/null +++ b/github.com/jackc/pgio/doc.go @@ -0,0 +1,6 @@ +// Package pgio is a low-level toolkit building messages in the PostgreSQL wire protocol. +/* +pgio provides functions for appending integers to a []byte while doing byte +order conversion. +*/ +package pgio diff --git a/github.com/jackc/pgio/go.mod b/github.com/jackc/pgio/go.mod new file mode 100644 index 0000000000..c1efdddb6e --- /dev/null +++ b/github.com/jackc/pgio/go.mod @@ -0,0 +1,3 @@ +module github.com/jackc/pgio + +go 1.12 diff --git a/github.com/jackc/pgio/write.go b/github.com/jackc/pgio/write.go new file mode 100644 index 0000000000..96aedf9dd1 --- /dev/null +++ b/github.com/jackc/pgio/write.go @@ -0,0 +1,40 @@ +package pgio + +import "encoding/binary" + +func AppendUint16(buf []byte, n uint16) []byte { + wp := len(buf) + buf = append(buf, 0, 0) + binary.BigEndian.PutUint16(buf[wp:], n) + return buf +} + +func AppendUint32(buf []byte, n uint32) []byte { + wp := len(buf) + buf = append(buf, 0, 0, 0, 0) + binary.BigEndian.PutUint32(buf[wp:], n) + return buf +} + +func AppendUint64(buf []byte, n uint64) []byte { + wp := len(buf) + buf = append(buf, 0, 0, 0, 0, 0, 0, 0, 0) + binary.BigEndian.PutUint64(buf[wp:], n) + return buf +} + +func AppendInt16(buf []byte, n int16) []byte { + return AppendUint16(buf, uint16(n)) +} + +func AppendInt32(buf []byte, n int32) []byte { + return AppendUint32(buf, uint32(n)) +} + +func AppendInt64(buf []byte, n int64) []byte { + return AppendUint64(buf, uint64(n)) +} + +func SetInt32(buf []byte, n int32) { + binary.BigEndian.PutUint32(buf, uint32(n)) +} diff --git a/github.com/jackc/pgpassfile/.travis.yml b/github.com/jackc/pgpassfile/.travis.yml new file mode 100644 index 0000000000..e176228e8e --- /dev/null +++ b/github.com/jackc/pgpassfile/.travis.yml @@ -0,0 +1,9 @@ +language: go + +go: + - 1.x + - tip + +matrix: + allow_failures: + - go: tip diff --git a/github.com/jackc/pgpassfile/LICENSE b/github.com/jackc/pgpassfile/LICENSE new file mode 100644 index 0000000000..c1c4f50fc6 --- /dev/null +++ b/github.com/jackc/pgpassfile/LICENSE @@ -0,0 +1,22 @@ +Copyright (c) 2019 Jack Christensen + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/github.com/jackc/pgpassfile/README.md b/github.com/jackc/pgpassfile/README.md new file mode 100644 index 0000000000..661289ed88 --- /dev/null +++ b/github.com/jackc/pgpassfile/README.md @@ -0,0 +1,8 @@ +[![](https://godoc.org/github.com/jackc/pgpassfile?status.svg)](https://godoc.org/github.com/jackc/pgpassfile) +[![Build Status](https://travis-ci.org/jackc/pgpassfile.svg)](https://travis-ci.org/jackc/pgpassfile) + +# pgpassfile + +Package pgpassfile is a parser PostgreSQL .pgpass files. + +Extracted and rewritten from original implementation in https://github.com/jackc/pgx. diff --git a/github.com/jackc/pgpassfile/go.mod b/github.com/jackc/pgpassfile/go.mod new file mode 100644 index 0000000000..48d90e313a --- /dev/null +++ b/github.com/jackc/pgpassfile/go.mod @@ -0,0 +1,5 @@ +module github.com/jackc/pgpassfile + +go 1.12 + +require github.com/stretchr/testify v1.3.0 diff --git a/github.com/jackc/pgpassfile/go.sum b/github.com/jackc/pgpassfile/go.sum new file mode 100644 index 0000000000..4347755afe --- /dev/null +++ b/github.com/jackc/pgpassfile/go.sum @@ -0,0 +1,7 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= diff --git a/github.com/jackc/pgpassfile/pgpass.go b/github.com/jackc/pgpassfile/pgpass.go new file mode 100644 index 0000000000..f7eed3c841 --- /dev/null +++ b/github.com/jackc/pgpassfile/pgpass.go @@ -0,0 +1,110 @@ +// Package pgpassfile is a parser PostgreSQL .pgpass files. +package pgpassfile + +import ( + "bufio" + "io" + "os" + "regexp" + "strings" +) + +// Entry represents a line in a PG passfile. +type Entry struct { + Hostname string + Port string + Database string + Username string + Password string +} + +// Passfile is the in memory data structure representing a PG passfile. +type Passfile struct { + Entries []*Entry +} + +// ReadPassfile reads the file at path and parses it into a Passfile. +func ReadPassfile(path string) (*Passfile, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + return ParsePassfile(f) +} + +// ParsePassfile reads r and parses it into a Passfile. +func ParsePassfile(r io.Reader) (*Passfile, error) { + passfile := &Passfile{} + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + entry := parseLine(scanner.Text()) + if entry != nil { + passfile.Entries = append(passfile.Entries, entry) + } + } + + return passfile, scanner.Err() +} + +// Match (not colons or escaped colon or escaped backslash)+. Essentially gives a split on unescaped +// colon. +var colonSplitterRegexp = regexp.MustCompile("(([^:]|(\\:)))+") + +// var colonSplitterRegexp = regexp.MustCompile("((?:[^:]|(?:\\:)|(?:\\\\))+)") + +// parseLine parses a line into an *Entry. It returns nil on comment lines or any other unparsable +// line. +func parseLine(line string) *Entry { + const ( + tmpBackslash = "\r" + tmpColon = "\n" + ) + + line = strings.TrimSpace(line) + + if strings.HasPrefix(line, "#") { + return nil + } + + line = strings.Replace(line, `\\`, tmpBackslash, -1) + line = strings.Replace(line, `\:`, tmpColon, -1) + + parts := strings.Split(line, ":") + if len(parts) != 5 { + return nil + } + + // Unescape escaped colons and backslashes + for i := range parts { + parts[i] = strings.Replace(parts[i], tmpBackslash, `\`, -1) + parts[i] = strings.Replace(parts[i], tmpColon, `:`, -1) + } + + return &Entry{ + Hostname: parts[0], + Port: parts[1], + Database: parts[2], + Username: parts[3], + Password: parts[4], + } +} + +// FindPassword finds the password for the provided hostname, port, database, and username. For a +// Unix domain socket hostname must be set to "localhost". An empty string will be returned if no +// match is found. +// +// See https://www.postgresql.org/docs/current/libpq-pgpass.html for more password file information. +func (pf *Passfile) FindPassword(hostname, port, database, username string) (password string) { + for _, e := range pf.Entries { + if (e.Hostname == "*" || e.Hostname == hostname) && + (e.Port == "*" || e.Port == port) && + (e.Database == "*" || e.Database == database) && + (e.Username == "*" || e.Username == username) { + return e.Password + } + } + return "" +} diff --git a/github.com/jackc/pgproto3/v2/.travis.yml b/github.com/jackc/pgproto3/v2/.travis.yml new file mode 100644 index 0000000000..e176228e8e --- /dev/null +++ b/github.com/jackc/pgproto3/v2/.travis.yml @@ -0,0 +1,9 @@ +language: go + +go: + - 1.x + - tip + +matrix: + allow_failures: + - go: tip diff --git a/github.com/jackc/pgproto3/v2/LICENSE b/github.com/jackc/pgproto3/v2/LICENSE new file mode 100644 index 0000000000..c1c4f50fc6 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/LICENSE @@ -0,0 +1,22 @@ +Copyright (c) 2019 Jack Christensen + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/github.com/jackc/pgproto3/v2/README.md b/github.com/jackc/pgproto3/v2/README.md new file mode 100644 index 0000000000..565b3efd5b --- /dev/null +++ b/github.com/jackc/pgproto3/v2/README.md @@ -0,0 +1,12 @@ +[![](https://godoc.org/github.com/jackc/pgproto3?status.svg)](https://godoc.org/github.com/jackc/pgproto3) +[![Build Status](https://travis-ci.org/jackc/pgproto3.svg)](https://travis-ci.org/jackc/pgproto3) + +# pgproto3 + +Package pgproto3 is a encoder and decoder of the PostgreSQL wire protocol version 3. + +pgproto3 can be used as a foundation for PostgreSQL drivers, proxies, mock servers, load balancers and more. + +See example/pgfortune for a playful example of a fake PostgreSQL server. + +Extracted from original implementation in https://github.com/jackc/pgx. diff --git a/github.com/jackc/pgproto3/v2/authentication_cleartext_password.go b/github.com/jackc/pgproto3/v2/authentication_cleartext_password.go new file mode 100644 index 0000000000..dd82c7a770 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/authentication_cleartext_password.go @@ -0,0 +1,39 @@ +package pgproto3 + +import ( + "encoding/binary" + "errors" + + "github.com/jackc/pgio" +) + +// AuthenticationCleartextPassword is a message sent from the backend indicating that a clear-text password is required. +type AuthenticationCleartextPassword struct { +} + +// Backend identifies this message as sendable by the PostgreSQL backend. +func (*AuthenticationCleartextPassword) Backend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *AuthenticationCleartextPassword) Decode(src []byte) error { + if len(src) != 4 { + return errors.New("bad authentication message size") + } + + authType := binary.BigEndian.Uint32(src) + + if authType != AuthTypeCleartextPassword { + return errors.New("bad auth type") + } + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *AuthenticationCleartextPassword) Encode(dst []byte) []byte { + dst = append(dst, 'R') + dst = pgio.AppendInt32(dst, 8) + dst = pgio.AppendUint32(dst, AuthTypeCleartextPassword) + return dst +} diff --git a/github.com/jackc/pgproto3/v2/authentication_md5_password.go b/github.com/jackc/pgproto3/v2/authentication_md5_password.go new file mode 100644 index 0000000000..d505d26497 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/authentication_md5_password.go @@ -0,0 +1,43 @@ +package pgproto3 + +import ( + "encoding/binary" + "errors" + + "github.com/jackc/pgio" +) + +// AuthenticationMD5Password is a message sent from the backend indicating that an MD5 hashed password is required. +type AuthenticationMD5Password struct { + Salt [4]byte +} + +// Backend identifies this message as sendable by the PostgreSQL backend. +func (*AuthenticationMD5Password) Backend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *AuthenticationMD5Password) Decode(src []byte) error { + if len(src) != 8 { + return errors.New("bad authentication message size") + } + + authType := binary.BigEndian.Uint32(src) + + if authType != AuthTypeMD5Password { + return errors.New("bad auth type") + } + + copy(dst.Salt[:], src[4:8]) + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *AuthenticationMD5Password) Encode(dst []byte) []byte { + dst = append(dst, 'R') + dst = pgio.AppendInt32(dst, 12) + dst = pgio.AppendUint32(dst, AuthTypeMD5Password) + dst = append(dst, src.Salt[:]...) + return dst +} diff --git a/github.com/jackc/pgproto3/v2/authentication_ok.go b/github.com/jackc/pgproto3/v2/authentication_ok.go new file mode 100644 index 0000000000..7b13c6e010 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/authentication_ok.go @@ -0,0 +1,39 @@ +package pgproto3 + +import ( + "encoding/binary" + "errors" + + "github.com/jackc/pgio" +) + +// AuthenticationOk is a message sent from the backend indicating that authentication was successful. +type AuthenticationOk struct { +} + +// Backend identifies this message as sendable by the PostgreSQL backend. +func (*AuthenticationOk) Backend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *AuthenticationOk) Decode(src []byte) error { + if len(src) != 4 { + return errors.New("bad authentication message size") + } + + authType := binary.BigEndian.Uint32(src) + + if authType != AuthTypeOk { + return errors.New("bad auth type") + } + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *AuthenticationOk) Encode(dst []byte) []byte { + dst = append(dst, 'R') + dst = pgio.AppendInt32(dst, 8) + dst = pgio.AppendUint32(dst, AuthTypeOk) + return dst +} diff --git a/github.com/jackc/pgproto3/v2/authentication_sasl.go b/github.com/jackc/pgproto3/v2/authentication_sasl.go new file mode 100644 index 0000000000..c57ae32de3 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/authentication_sasl.go @@ -0,0 +1,60 @@ +package pgproto3 + +import ( + "bytes" + "encoding/binary" + "errors" + + "github.com/jackc/pgio" +) + +// AuthenticationSASL is a message sent from the backend indicating that SASL authentication is required. +type AuthenticationSASL struct { + AuthMechanisms []string +} + +// Backend identifies this message as sendable by the PostgreSQL backend. +func (*AuthenticationSASL) Backend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *AuthenticationSASL) Decode(src []byte) error { + if len(src) < 4 { + return errors.New("authentication message too short") + } + + authType := binary.BigEndian.Uint32(src) + + if authType != AuthTypeSASL { + return errors.New("bad auth type") + } + + authMechanisms := src[4:] + for len(authMechanisms) > 1 { + idx := bytes.IndexByte(authMechanisms, 0) + if idx > 0 { + dst.AuthMechanisms = append(dst.AuthMechanisms, string(authMechanisms[:idx])) + authMechanisms = authMechanisms[idx+1:] + } + } + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *AuthenticationSASL) Encode(dst []byte) []byte { + dst = append(dst, 'R') + sp := len(dst) + dst = pgio.AppendInt32(dst, -1) + dst = pgio.AppendUint32(dst, AuthTypeSASL) + + for _, s := range src.AuthMechanisms { + dst = append(dst, []byte(s)...) + dst = append(dst, 0) + } + dst = append(dst, 0) + + pgio.SetInt32(dst[sp:], int32(len(dst[sp:]))) + + return dst +} diff --git a/github.com/jackc/pgproto3/v2/authentication_sasl_continue.go b/github.com/jackc/pgproto3/v2/authentication_sasl_continue.go new file mode 100644 index 0000000000..a393ae10d0 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/authentication_sasl_continue.go @@ -0,0 +1,49 @@ +package pgproto3 + +import ( + "encoding/binary" + "errors" + + "github.com/jackc/pgio" +) + +// AuthenticationSASLContinue is a message sent from the backend containing a SASL challenge. +type AuthenticationSASLContinue struct { + Data []byte +} + +// Backend identifies this message as sendable by the PostgreSQL backend. +func (*AuthenticationSASLContinue) Backend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *AuthenticationSASLContinue) Decode(src []byte) error { + if len(src) < 4 { + return errors.New("authentication message too short") + } + + authType := binary.BigEndian.Uint32(src) + + if authType != AuthTypeSASLContinue { + return errors.New("bad auth type") + } + + dst.Data = src[4:] + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *AuthenticationSASLContinue) Encode(dst []byte) []byte { + dst = append(dst, 'R') + sp := len(dst) + dst = pgio.AppendInt32(dst, -1) + dst = pgio.AppendUint32(dst, AuthTypeSASLContinue) + + dst = pgio.AppendInt32(dst, int32(len(src.Data))) + dst = append(dst, src.Data...) + + pgio.SetInt32(dst[sp:], int32(len(dst[sp:]))) + + return dst +} diff --git a/github.com/jackc/pgproto3/v2/authentication_sasl_final.go b/github.com/jackc/pgproto3/v2/authentication_sasl_final.go new file mode 100644 index 0000000000..b8f89d5906 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/authentication_sasl_final.go @@ -0,0 +1,49 @@ +package pgproto3 + +import ( + "encoding/binary" + "errors" + + "github.com/jackc/pgio" +) + +// AuthenticationSASLFinal is a message sent from the backend indicating a SASL authentication has completed. +type AuthenticationSASLFinal struct { + Data []byte +} + +// Backend identifies this message as sendable by the PostgreSQL backend. +func (*AuthenticationSASLFinal) Backend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *AuthenticationSASLFinal) Decode(src []byte) error { + if len(src) < 4 { + return errors.New("authentication message too short") + } + + authType := binary.BigEndian.Uint32(src) + + if authType != AuthTypeSASLFinal { + return errors.New("bad auth type") + } + + dst.Data = src[4:] + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *AuthenticationSASLFinal) Encode(dst []byte) []byte { + dst = append(dst, 'R') + sp := len(dst) + dst = pgio.AppendInt32(dst, -1) + dst = pgio.AppendUint32(dst, AuthTypeSASLFinal) + + dst = pgio.AppendInt32(dst, int32(len(src.Data))) + dst = append(dst, src.Data...) + + pgio.SetInt32(dst[sp:], int32(len(dst[sp:]))) + + return dst +} diff --git a/github.com/jackc/pgproto3/v2/backend.go b/github.com/jackc/pgproto3/v2/backend.go new file mode 100644 index 0000000000..5741647f9c --- /dev/null +++ b/github.com/jackc/pgproto3/v2/backend.go @@ -0,0 +1,137 @@ +package pgproto3 + +import ( + "encoding/binary" + "fmt" + "io" +) + +// Backend acts as a server for the PostgreSQL wire protocol version 3. +type Backend struct { + cr ChunkReader + w io.Writer + + // Frontend message flyweights + bind Bind + cancelRequest CancelRequest + _close Close + copyFail CopyFail + describe Describe + execute Execute + flush Flush + parse Parse + passwordMessage PasswordMessage + query Query + sslRequest SSLRequest + startupMessage StartupMessage + sync Sync + terminate Terminate + + bodyLen int + msgType byte + partialMsg bool +} + +// NewBackend creates a new Backend. +func NewBackend(cr ChunkReader, w io.Writer) *Backend { + return &Backend{cr: cr, w: w} +} + +// Send sends a message to the frontend. +func (b *Backend) Send(msg BackendMessage) error { + _, err := b.w.Write(msg.Encode(nil)) + return err +} + +// ReceiveStartupMessage receives the initial connection message. This method is used of the normal Receive method +// because the initial connection message is "special" and does not include the message type as the first byte. This +// will return either a StartupMessage, SSLRequest, or CancelRequest. +func (b *Backend) ReceiveStartupMessage() (FrontendMessage, error) { + buf, err := b.cr.Next(4) + if err != nil { + return nil, err + } + msgSize := int(binary.BigEndian.Uint32(buf) - 4) + + buf, err = b.cr.Next(msgSize) + if err != nil { + return nil, err + } + + code := binary.BigEndian.Uint32(buf) + + switch code { + case ProtocolVersionNumber: + err = b.startupMessage.Decode(buf) + if err != nil { + return nil, err + } + return &b.startupMessage, nil + case sslRequestNumber: + err = b.sslRequest.Decode(buf) + if err != nil { + return nil, err + } + return &b.sslRequest, nil + case cancelRequestCode: + err = b.cancelRequest.Decode(buf) + if err != nil { + return nil, err + } + return &b.cancelRequest, nil + default: + return nil, fmt.Errorf("unknown startup message code: %d", code) + } +} + +// Receive receives a message from the frontend. +func (b *Backend) Receive() (FrontendMessage, error) { + if !b.partialMsg { + header, err := b.cr.Next(5) + if err != nil { + return nil, err + } + + b.msgType = header[0] + b.bodyLen = int(binary.BigEndian.Uint32(header[1:])) - 4 + b.partialMsg = true + } + + var msg FrontendMessage + switch b.msgType { + case 'B': + msg = &b.bind + case 'C': + msg = &b._close + case 'D': + msg = &b.describe + case 'E': + msg = &b.execute + case 'f': + msg = &b.copyFail + case 'H': + msg = &b.flush + case 'P': + msg = &b.parse + case 'p': + msg = &b.passwordMessage + case 'Q': + msg = &b.query + case 'S': + msg = &b.sync + case 'X': + msg = &b.terminate + default: + return nil, fmt.Errorf("unknown message type: %c", b.msgType) + } + + msgBody, err := b.cr.Next(b.bodyLen) + if err != nil { + return nil, err + } + + b.partialMsg = false + + err = msg.Decode(msgBody) + return msg, err +} diff --git a/github.com/jackc/pgproto3/v2/backend_key_data.go b/github.com/jackc/pgproto3/v2/backend_key_data.go new file mode 100644 index 0000000000..ca20dd259b --- /dev/null +++ b/github.com/jackc/pgproto3/v2/backend_key_data.go @@ -0,0 +1,51 @@ +package pgproto3 + +import ( + "encoding/binary" + "encoding/json" + + "github.com/jackc/pgio" +) + +type BackendKeyData struct { + ProcessID uint32 + SecretKey uint32 +} + +// Backend identifies this message as sendable by the PostgreSQL backend. +func (*BackendKeyData) Backend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *BackendKeyData) Decode(src []byte) error { + if len(src) != 8 { + return &invalidMessageLenErr{messageType: "BackendKeyData", expectedLen: 8, actualLen: len(src)} + } + + dst.ProcessID = binary.BigEndian.Uint32(src[:4]) + dst.SecretKey = binary.BigEndian.Uint32(src[4:]) + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *BackendKeyData) Encode(dst []byte) []byte { + dst = append(dst, 'K') + dst = pgio.AppendUint32(dst, 12) + dst = pgio.AppendUint32(dst, src.ProcessID) + dst = pgio.AppendUint32(dst, src.SecretKey) + return dst +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src BackendKeyData) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + ProcessID uint32 + SecretKey uint32 + }{ + Type: "BackendKeyData", + ProcessID: src.ProcessID, + SecretKey: src.SecretKey, + }) +} diff --git a/github.com/jackc/pgproto3/v2/big_endian.go b/github.com/jackc/pgproto3/v2/big_endian.go new file mode 100644 index 0000000000..f7bdb97eb7 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/big_endian.go @@ -0,0 +1,37 @@ +package pgproto3 + +import ( + "encoding/binary" +) + +type BigEndianBuf [8]byte + +func (b BigEndianBuf) Int16(n int16) []byte { + buf := b[0:2] + binary.BigEndian.PutUint16(buf, uint16(n)) + return buf +} + +func (b BigEndianBuf) Uint16(n uint16) []byte { + buf := b[0:2] + binary.BigEndian.PutUint16(buf, n) + return buf +} + +func (b BigEndianBuf) Int32(n int32) []byte { + buf := b[0:4] + binary.BigEndian.PutUint32(buf, uint32(n)) + return buf +} + +func (b BigEndianBuf) Uint32(n uint32) []byte { + buf := b[0:4] + binary.BigEndian.PutUint32(buf, n) + return buf +} + +func (b BigEndianBuf) Int64(n int64) []byte { + buf := b[0:8] + binary.BigEndian.PutUint64(buf, uint64(n)) + return buf +} diff --git a/github.com/jackc/pgproto3/v2/bind.go b/github.com/jackc/pgproto3/v2/bind.go new file mode 100644 index 0000000000..65b4c1d60c --- /dev/null +++ b/github.com/jackc/pgproto3/v2/bind.go @@ -0,0 +1,176 @@ +package pgproto3 + +import ( + "bytes" + "encoding/binary" + "encoding/hex" + "encoding/json" + + "github.com/jackc/pgio" +) + +type Bind struct { + DestinationPortal string + PreparedStatement string + ParameterFormatCodes []int16 + Parameters [][]byte + ResultFormatCodes []int16 +} + +// Frontend identifies this message as sendable by a PostgreSQL frontend. +func (*Bind) Frontend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *Bind) Decode(src []byte) error { + *dst = Bind{} + + idx := bytes.IndexByte(src, 0) + if idx < 0 { + return &invalidMessageFormatErr{messageType: "Bind"} + } + dst.DestinationPortal = string(src[:idx]) + rp := idx + 1 + + idx = bytes.IndexByte(src[rp:], 0) + if idx < 0 { + return &invalidMessageFormatErr{messageType: "Bind"} + } + dst.PreparedStatement = string(src[rp : rp+idx]) + rp += idx + 1 + + if len(src[rp:]) < 2 { + return &invalidMessageFormatErr{messageType: "Bind"} + } + parameterFormatCodeCount := int(binary.BigEndian.Uint16(src[rp:])) + rp += 2 + + if parameterFormatCodeCount > 0 { + dst.ParameterFormatCodes = make([]int16, parameterFormatCodeCount) + + if len(src[rp:]) < len(dst.ParameterFormatCodes)*2 { + return &invalidMessageFormatErr{messageType: "Bind"} + } + for i := 0; i < parameterFormatCodeCount; i++ { + dst.ParameterFormatCodes[i] = int16(binary.BigEndian.Uint16(src[rp:])) + rp += 2 + } + } + + if len(src[rp:]) < 2 { + return &invalidMessageFormatErr{messageType: "Bind"} + } + parameterCount := int(binary.BigEndian.Uint16(src[rp:])) + rp += 2 + + if parameterCount > 0 { + dst.Parameters = make([][]byte, parameterCount) + + for i := 0; i < parameterCount; i++ { + if len(src[rp:]) < 4 { + return &invalidMessageFormatErr{messageType: "Bind"} + } + + msgSize := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + + // null + if msgSize == -1 { + continue + } + + if len(src[rp:]) < msgSize { + return &invalidMessageFormatErr{messageType: "Bind"} + } + + dst.Parameters[i] = src[rp : rp+msgSize] + rp += msgSize + } + } + + if len(src[rp:]) < 2 { + return &invalidMessageFormatErr{messageType: "Bind"} + } + resultFormatCodeCount := int(binary.BigEndian.Uint16(src[rp:])) + rp += 2 + + dst.ResultFormatCodes = make([]int16, resultFormatCodeCount) + if len(src[rp:]) < len(dst.ResultFormatCodes)*2 { + return &invalidMessageFormatErr{messageType: "Bind"} + } + for i := 0; i < resultFormatCodeCount; i++ { + dst.ResultFormatCodes[i] = int16(binary.BigEndian.Uint16(src[rp:])) + rp += 2 + } + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *Bind) Encode(dst []byte) []byte { + dst = append(dst, 'B') + sp := len(dst) + dst = pgio.AppendInt32(dst, -1) + + dst = append(dst, src.DestinationPortal...) + dst = append(dst, 0) + dst = append(dst, src.PreparedStatement...) + dst = append(dst, 0) + + dst = pgio.AppendUint16(dst, uint16(len(src.ParameterFormatCodes))) + for _, fc := range src.ParameterFormatCodes { + dst = pgio.AppendInt16(dst, fc) + } + + dst = pgio.AppendUint16(dst, uint16(len(src.Parameters))) + for _, p := range src.Parameters { + if p == nil { + dst = pgio.AppendInt32(dst, -1) + continue + } + + dst = pgio.AppendInt32(dst, int32(len(p))) + dst = append(dst, p...) + } + + dst = pgio.AppendUint16(dst, uint16(len(src.ResultFormatCodes))) + for _, fc := range src.ResultFormatCodes { + dst = pgio.AppendInt16(dst, fc) + } + + pgio.SetInt32(dst[sp:], int32(len(dst[sp:]))) + + return dst +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src Bind) MarshalJSON() ([]byte, error) { + formattedParameters := make([]map[string]string, len(src.Parameters)) + for i, p := range src.Parameters { + if p == nil { + continue + } + + if src.ParameterFormatCodes[i] == 0 { + formattedParameters[i] = map[string]string{"text": string(p)} + } else { + formattedParameters[i] = map[string]string{"binary": hex.EncodeToString(p)} + } + } + + return json.Marshal(struct { + Type string + DestinationPortal string + PreparedStatement string + ParameterFormatCodes []int16 + Parameters []map[string]string + ResultFormatCodes []int16 + }{ + Type: "Bind", + DestinationPortal: src.DestinationPortal, + PreparedStatement: src.PreparedStatement, + ParameterFormatCodes: src.ParameterFormatCodes, + Parameters: formattedParameters, + ResultFormatCodes: src.ResultFormatCodes, + }) +} diff --git a/github.com/jackc/pgproto3/v2/bind_complete.go b/github.com/jackc/pgproto3/v2/bind_complete.go new file mode 100644 index 0000000000..3be256c897 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/bind_complete.go @@ -0,0 +1,34 @@ +package pgproto3 + +import ( + "encoding/json" +) + +type BindComplete struct{} + +// Backend identifies this message as sendable by the PostgreSQL backend. +func (*BindComplete) Backend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *BindComplete) Decode(src []byte) error { + if len(src) != 0 { + return &invalidMessageLenErr{messageType: "BindComplete", expectedLen: 0, actualLen: len(src)} + } + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *BindComplete) Encode(dst []byte) []byte { + return append(dst, '2', 0, 0, 0, 4) +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src BindComplete) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + }{ + Type: "BindComplete", + }) +} diff --git a/github.com/jackc/pgproto3/v2/cancel_request.go b/github.com/jackc/pgproto3/v2/cancel_request.go new file mode 100644 index 0000000000..942e404be4 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/cancel_request.go @@ -0,0 +1,58 @@ +package pgproto3 + +import ( + "encoding/binary" + "encoding/json" + "errors" + + "github.com/jackc/pgio" +) + +const cancelRequestCode = 80877102 + +type CancelRequest struct { + ProcessID uint32 + SecretKey uint32 +} + +// Frontend identifies this message as sendable by a PostgreSQL frontend. +func (*CancelRequest) Frontend() {} + +func (dst *CancelRequest) Decode(src []byte) error { + if len(src) != 12 { + return errors.New("bad cancel request size") + } + + requestCode := binary.BigEndian.Uint32(src) + + if requestCode != cancelRequestCode { + return errors.New("bad cancel request code") + } + + dst.ProcessID = binary.BigEndian.Uint32(src[4:]) + dst.SecretKey = binary.BigEndian.Uint32(src[8:]) + + return nil +} + +// Encode encodes src into dst. dst will include the 4 byte message length. +func (src *CancelRequest) Encode(dst []byte) []byte { + dst = pgio.AppendInt32(dst, 16) + dst = pgio.AppendInt32(dst, cancelRequestCode) + dst = pgio.AppendUint32(dst, src.ProcessID) + dst = pgio.AppendUint32(dst, src.SecretKey) + return dst +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src CancelRequest) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + ProcessID uint32 + SecretKey uint32 + }{ + Type: "CancelRequest", + ProcessID: src.ProcessID, + SecretKey: src.SecretKey, + }) +} diff --git a/github.com/jackc/pgproto3/v2/chunkreader.go b/github.com/jackc/pgproto3/v2/chunkreader.go new file mode 100644 index 0000000000..92206f358c --- /dev/null +++ b/github.com/jackc/pgproto3/v2/chunkreader.go @@ -0,0 +1,19 @@ +package pgproto3 + +import ( + "io" + + "github.com/jackc/chunkreader/v2" +) + +// ChunkReader is an interface to decouple github.com/jackc/chunkreader from this package. +type ChunkReader interface { + // Next returns buf filled with the next n bytes. If an error (including a partial read) occurs, + // buf must be nil. Next must preserve any partially read data. Next must not reuse buf. + Next(n int) (buf []byte, err error) +} + +// NewChunkReader creates and returns a new default ChunkReader. +func NewChunkReader(r io.Reader) ChunkReader { + return chunkreader.New(r) +} diff --git a/github.com/jackc/pgproto3/v2/close.go b/github.com/jackc/pgproto3/v2/close.go new file mode 100644 index 0000000000..382969093d --- /dev/null +++ b/github.com/jackc/pgproto3/v2/close.go @@ -0,0 +1,64 @@ +package pgproto3 + +import ( + "bytes" + "encoding/json" + + "github.com/jackc/pgio" +) + +type Close struct { + ObjectType byte // 'S' = prepared statement, 'P' = portal + Name string +} + +// Frontend identifies this message as sendable by a PostgreSQL frontend. +func (*Close) Frontend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *Close) Decode(src []byte) error { + if len(src) < 2 { + return &invalidMessageFormatErr{messageType: "Close"} + } + + dst.ObjectType = src[0] + rp := 1 + + idx := bytes.IndexByte(src[rp:], 0) + if idx != len(src[rp:])-1 { + return &invalidMessageFormatErr{messageType: "Close"} + } + + dst.Name = string(src[rp : len(src)-1]) + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *Close) Encode(dst []byte) []byte { + dst = append(dst, 'C') + sp := len(dst) + dst = pgio.AppendInt32(dst, -1) + + dst = append(dst, src.ObjectType) + dst = append(dst, src.Name...) + dst = append(dst, 0) + + pgio.SetInt32(dst[sp:], int32(len(dst[sp:]))) + + return dst +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src Close) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + ObjectType string + Name string + }{ + Type: "Close", + ObjectType: string(src.ObjectType), + Name: src.Name, + }) +} diff --git a/github.com/jackc/pgproto3/v2/close_complete.go b/github.com/jackc/pgproto3/v2/close_complete.go new file mode 100644 index 0000000000..1d7b8f085a --- /dev/null +++ b/github.com/jackc/pgproto3/v2/close_complete.go @@ -0,0 +1,34 @@ +package pgproto3 + +import ( + "encoding/json" +) + +type CloseComplete struct{} + +// Backend identifies this message as sendable by the PostgreSQL backend. +func (*CloseComplete) Backend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *CloseComplete) Decode(src []byte) error { + if len(src) != 0 { + return &invalidMessageLenErr{messageType: "CloseComplete", expectedLen: 0, actualLen: len(src)} + } + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *CloseComplete) Encode(dst []byte) []byte { + return append(dst, '3', 0, 0, 0, 4) +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src CloseComplete) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + }{ + Type: "CloseComplete", + }) +} diff --git a/github.com/jackc/pgproto3/v2/command_complete.go b/github.com/jackc/pgproto3/v2/command_complete.go new file mode 100644 index 0000000000..b5106fdaf4 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/command_complete.go @@ -0,0 +1,53 @@ +package pgproto3 + +import ( + "bytes" + "encoding/json" + + "github.com/jackc/pgio" +) + +type CommandComplete struct { + CommandTag []byte +} + +// Backend identifies this message as sendable by the PostgreSQL backend. +func (*CommandComplete) Backend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *CommandComplete) Decode(src []byte) error { + idx := bytes.IndexByte(src, 0) + if idx != len(src)-1 { + return &invalidMessageFormatErr{messageType: "CommandComplete"} + } + + dst.CommandTag = src[:idx] + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *CommandComplete) Encode(dst []byte) []byte { + dst = append(dst, 'C') + sp := len(dst) + dst = pgio.AppendInt32(dst, -1) + + dst = append(dst, src.CommandTag...) + dst = append(dst, 0) + + pgio.SetInt32(dst[sp:], int32(len(dst[sp:]))) + + return dst +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src CommandComplete) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + CommandTag string + }{ + Type: "CommandComplete", + CommandTag: string(src.CommandTag), + }) +} diff --git a/github.com/jackc/pgproto3/v2/copy_both_response.go b/github.com/jackc/pgproto3/v2/copy_both_response.go new file mode 100644 index 0000000000..2d58f820e3 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/copy_both_response.go @@ -0,0 +1,70 @@ +package pgproto3 + +import ( + "bytes" + "encoding/binary" + "encoding/json" + + "github.com/jackc/pgio" +) + +type CopyBothResponse struct { + OverallFormat byte + ColumnFormatCodes []uint16 +} + +// Backend identifies this message as sendable by the PostgreSQL backend. +func (*CopyBothResponse) Backend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *CopyBothResponse) Decode(src []byte) error { + buf := bytes.NewBuffer(src) + + if buf.Len() < 3 { + return &invalidMessageFormatErr{messageType: "CopyBothResponse"} + } + + overallFormat := buf.Next(1)[0] + + columnCount := int(binary.BigEndian.Uint16(buf.Next(2))) + if buf.Len() != columnCount*2 { + return &invalidMessageFormatErr{messageType: "CopyBothResponse"} + } + + columnFormatCodes := make([]uint16, columnCount) + for i := 0; i < columnCount; i++ { + columnFormatCodes[i] = binary.BigEndian.Uint16(buf.Next(2)) + } + + *dst = CopyBothResponse{OverallFormat: overallFormat, ColumnFormatCodes: columnFormatCodes} + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *CopyBothResponse) Encode(dst []byte) []byte { + dst = append(dst, 'W') + sp := len(dst) + dst = pgio.AppendInt32(dst, -1) + + dst = pgio.AppendUint16(dst, uint16(len(src.ColumnFormatCodes))) + for _, fc := range src.ColumnFormatCodes { + dst = pgio.AppendUint16(dst, fc) + } + + pgio.SetInt32(dst[sp:], int32(len(dst[sp:]))) + + return dst +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src CopyBothResponse) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + ColumnFormatCodes []uint16 + }{ + Type: "CopyBothResponse", + ColumnFormatCodes: src.ColumnFormatCodes, + }) +} diff --git a/github.com/jackc/pgproto3/v2/copy_data.go b/github.com/jackc/pgproto3/v2/copy_data.go new file mode 100644 index 0000000000..7d6002fe0f --- /dev/null +++ b/github.com/jackc/pgproto3/v2/copy_data.go @@ -0,0 +1,44 @@ +package pgproto3 + +import ( + "encoding/hex" + "encoding/json" + + "github.com/jackc/pgio" +) + +type CopyData struct { + Data []byte +} + +// Backend identifies this message as sendable by the PostgreSQL backend. +func (*CopyData) Backend() {} + +// Frontend identifies this message as sendable by a PostgreSQL frontend. +func (*CopyData) Frontend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *CopyData) Decode(src []byte) error { + dst.Data = src + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *CopyData) Encode(dst []byte) []byte { + dst = append(dst, 'd') + dst = pgio.AppendInt32(dst, int32(4+len(src.Data))) + dst = append(dst, src.Data...) + return dst +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src CopyData) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + Data string + }{ + Type: "CopyData", + Data: hex.EncodeToString(src.Data), + }) +} diff --git a/github.com/jackc/pgproto3/v2/copy_done.go b/github.com/jackc/pgproto3/v2/copy_done.go new file mode 100644 index 0000000000..d8b6e5d7d8 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/copy_done.go @@ -0,0 +1,35 @@ +package pgproto3 + +import ( + "encoding/json" +) + +type CopyDone struct { +} + +// Backend identifies this message as sendable by the PostgreSQL backend. +func (*CopyDone) Backend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *CopyDone) Decode(src []byte) error { + if len(src) != 0 { + return &invalidMessageLenErr{messageType: "CopyDone", expectedLen: 0, actualLen: len(src)} + } + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *CopyDone) Encode(dst []byte) []byte { + return append(dst, 'c', 0, 0, 0, 4) +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src CopyDone) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + }{ + Type: "CopyDone", + }) +} diff --git a/github.com/jackc/pgproto3/v2/copy_fail.go b/github.com/jackc/pgproto3/v2/copy_fail.go new file mode 100644 index 0000000000..78ff0b30be --- /dev/null +++ b/github.com/jackc/pgproto3/v2/copy_fail.go @@ -0,0 +1,53 @@ +package pgproto3 + +import ( + "bytes" + "encoding/json" + + "github.com/jackc/pgio" +) + +type CopyFail struct { + Message string +} + +// Frontend identifies this message as sendable by a PostgreSQL frontend. +func (*CopyFail) Frontend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *CopyFail) Decode(src []byte) error { + idx := bytes.IndexByte(src, 0) + if idx != len(src)-1 { + return &invalidMessageFormatErr{messageType: "CopyFail"} + } + + dst.Message = string(src[:idx]) + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *CopyFail) Encode(dst []byte) []byte { + dst = append(dst, 'f') + sp := len(dst) + dst = pgio.AppendInt32(dst, -1) + + dst = append(dst, src.Message...) + dst = append(dst, 0) + + pgio.SetInt32(dst[sp:], int32(len(dst[sp:]))) + + return dst +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src CopyFail) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + Message string + }{ + Type: "CopyFail", + Message: src.Message, + }) +} diff --git a/github.com/jackc/pgproto3/v2/copy_in_response.go b/github.com/jackc/pgproto3/v2/copy_in_response.go new file mode 100644 index 0000000000..4439a03284 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/copy_in_response.go @@ -0,0 +1,70 @@ +package pgproto3 + +import ( + "bytes" + "encoding/binary" + "encoding/json" + + "github.com/jackc/pgio" +) + +type CopyInResponse struct { + OverallFormat byte + ColumnFormatCodes []uint16 +} + +// Backend identifies this message as sendable by the PostgreSQL backend. +func (*CopyInResponse) Backend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *CopyInResponse) Decode(src []byte) error { + buf := bytes.NewBuffer(src) + + if buf.Len() < 3 { + return &invalidMessageFormatErr{messageType: "CopyInResponse"} + } + + overallFormat := buf.Next(1)[0] + + columnCount := int(binary.BigEndian.Uint16(buf.Next(2))) + if buf.Len() != columnCount*2 { + return &invalidMessageFormatErr{messageType: "CopyInResponse"} + } + + columnFormatCodes := make([]uint16, columnCount) + for i := 0; i < columnCount; i++ { + columnFormatCodes[i] = binary.BigEndian.Uint16(buf.Next(2)) + } + + *dst = CopyInResponse{OverallFormat: overallFormat, ColumnFormatCodes: columnFormatCodes} + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *CopyInResponse) Encode(dst []byte) []byte { + dst = append(dst, 'G') + sp := len(dst) + dst = pgio.AppendInt32(dst, -1) + + dst = pgio.AppendUint16(dst, uint16(len(src.ColumnFormatCodes))) + for _, fc := range src.ColumnFormatCodes { + dst = pgio.AppendUint16(dst, fc) + } + + pgio.SetInt32(dst[sp:], int32(len(dst[sp:]))) + + return dst +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src CopyInResponse) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + ColumnFormatCodes []uint16 + }{ + Type: "CopyInResponse", + ColumnFormatCodes: src.ColumnFormatCodes, + }) +} diff --git a/github.com/jackc/pgproto3/v2/copy_out_response.go b/github.com/jackc/pgproto3/v2/copy_out_response.go new file mode 100644 index 0000000000..8538dfc7d7 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/copy_out_response.go @@ -0,0 +1,71 @@ +package pgproto3 + +import ( + "bytes" + "encoding/binary" + "encoding/json" + + "github.com/jackc/pgio" +) + +type CopyOutResponse struct { + OverallFormat byte + ColumnFormatCodes []uint16 +} + +func (*CopyOutResponse) Backend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *CopyOutResponse) Decode(src []byte) error { + buf := bytes.NewBuffer(src) + + if buf.Len() < 3 { + return &invalidMessageFormatErr{messageType: "CopyOutResponse"} + } + + overallFormat := buf.Next(1)[0] + + columnCount := int(binary.BigEndian.Uint16(buf.Next(2))) + if buf.Len() != columnCount*2 { + return &invalidMessageFormatErr{messageType: "CopyOutResponse"} + } + + columnFormatCodes := make([]uint16, columnCount) + for i := 0; i < columnCount; i++ { + columnFormatCodes[i] = binary.BigEndian.Uint16(buf.Next(2)) + } + + *dst = CopyOutResponse{OverallFormat: overallFormat, ColumnFormatCodes: columnFormatCodes} + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *CopyOutResponse) Encode(dst []byte) []byte { + dst = append(dst, 'H') + sp := len(dst) + dst = pgio.AppendInt32(dst, -1) + + dst = append(dst, src.OverallFormat) + + dst = pgio.AppendUint16(dst, uint16(len(src.ColumnFormatCodes))) + for _, fc := range src.ColumnFormatCodes { + dst = pgio.AppendUint16(dst, fc) + } + + pgio.SetInt32(dst[sp:], int32(len(dst[sp:]))) + + return dst +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src CopyOutResponse) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + ColumnFormatCodes []uint16 + }{ + Type: "CopyOutResponse", + ColumnFormatCodes: src.ColumnFormatCodes, + }) +} diff --git a/github.com/jackc/pgproto3/v2/data_row.go b/github.com/jackc/pgproto3/v2/data_row.go new file mode 100644 index 0000000000..d908e7b2b4 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/data_row.go @@ -0,0 +1,117 @@ +package pgproto3 + +import ( + "encoding/binary" + "encoding/hex" + "encoding/json" + + "github.com/jackc/pgio" +) + +type DataRow struct { + Values [][]byte +} + +// Backend identifies this message as sendable by the PostgreSQL backend. +func (*DataRow) Backend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *DataRow) Decode(src []byte) error { + if len(src) < 2 { + return &invalidMessageFormatErr{messageType: "DataRow"} + } + rp := 0 + fieldCount := int(binary.BigEndian.Uint16(src[rp:])) + rp += 2 + + // If the capacity of the values slice is too small OR substantially too + // large reallocate. This is too avoid one row with many columns from + // permanently allocating memory. + if cap(dst.Values) < fieldCount || cap(dst.Values)-fieldCount > 32 { + newCap := 32 + if newCap < fieldCount { + newCap = fieldCount + } + dst.Values = make([][]byte, fieldCount, newCap) + } else { + dst.Values = dst.Values[:fieldCount] + } + + for i := 0; i < fieldCount; i++ { + if len(src[rp:]) < 4 { + return &invalidMessageFormatErr{messageType: "DataRow"} + } + + msgSize := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + + // null + if msgSize == -1 { + dst.Values[i] = nil + } else { + if len(src[rp:]) < msgSize { + return &invalidMessageFormatErr{messageType: "DataRow"} + } + + dst.Values[i] = src[rp : rp+msgSize] + rp += msgSize + } + } + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *DataRow) Encode(dst []byte) []byte { + dst = append(dst, 'D') + sp := len(dst) + dst = pgio.AppendInt32(dst, -1) + + dst = pgio.AppendUint16(dst, uint16(len(src.Values))) + for _, v := range src.Values { + if v == nil { + dst = pgio.AppendInt32(dst, -1) + continue + } + + dst = pgio.AppendInt32(dst, int32(len(v))) + dst = append(dst, v...) + } + + pgio.SetInt32(dst[sp:], int32(len(dst[sp:]))) + + return dst +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src DataRow) MarshalJSON() ([]byte, error) { + formattedValues := make([]map[string]string, len(src.Values)) + for i, v := range src.Values { + if v == nil { + continue + } + + var hasNonPrintable bool + for _, b := range v { + if b < 32 { + hasNonPrintable = true + break + } + } + + if hasNonPrintable { + formattedValues[i] = map[string]string{"binary": hex.EncodeToString(v)} + } else { + formattedValues[i] = map[string]string{"text": string(v)} + } + } + + return json.Marshal(struct { + Type string + Values []map[string]string + }{ + Type: "DataRow", + Values: formattedValues, + }) +} diff --git a/github.com/jackc/pgproto3/v2/describe.go b/github.com/jackc/pgproto3/v2/describe.go new file mode 100644 index 0000000000..308f582e71 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/describe.go @@ -0,0 +1,64 @@ +package pgproto3 + +import ( + "bytes" + "encoding/json" + + "github.com/jackc/pgio" +) + +type Describe struct { + ObjectType byte // 'S' = prepared statement, 'P' = portal + Name string +} + +// Frontend identifies this message as sendable by a PostgreSQL frontend. +func (*Describe) Frontend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *Describe) Decode(src []byte) error { + if len(src) < 2 { + return &invalidMessageFormatErr{messageType: "Describe"} + } + + dst.ObjectType = src[0] + rp := 1 + + idx := bytes.IndexByte(src[rp:], 0) + if idx != len(src[rp:])-1 { + return &invalidMessageFormatErr{messageType: "Describe"} + } + + dst.Name = string(src[rp : len(src)-1]) + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *Describe) Encode(dst []byte) []byte { + dst = append(dst, 'D') + sp := len(dst) + dst = pgio.AppendInt32(dst, -1) + + dst = append(dst, src.ObjectType) + dst = append(dst, src.Name...) + dst = append(dst, 0) + + pgio.SetInt32(dst[sp:], int32(len(dst[sp:]))) + + return dst +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src Describe) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + ObjectType string + Name string + }{ + Type: "Describe", + ObjectType: string(src.ObjectType), + Name: src.Name, + }) +} diff --git a/github.com/jackc/pgproto3/v2/doc.go b/github.com/jackc/pgproto3/v2/doc.go new file mode 100644 index 0000000000..8226dc983c --- /dev/null +++ b/github.com/jackc/pgproto3/v2/doc.go @@ -0,0 +1,4 @@ +// Package pgproto3 is a encoder and decoder of the PostgreSQL wire protocol version 3. +// +// See https://www.postgresql.org/docs/current/protocol-message-formats.html for meanings of the different messages. +package pgproto3 diff --git a/github.com/jackc/pgproto3/v2/empty_query_response.go b/github.com/jackc/pgproto3/v2/empty_query_response.go new file mode 100644 index 0000000000..2b85e744bc --- /dev/null +++ b/github.com/jackc/pgproto3/v2/empty_query_response.go @@ -0,0 +1,34 @@ +package pgproto3 + +import ( + "encoding/json" +) + +type EmptyQueryResponse struct{} + +// Backend identifies this message as sendable by the PostgreSQL backend. +func (*EmptyQueryResponse) Backend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *EmptyQueryResponse) Decode(src []byte) error { + if len(src) != 0 { + return &invalidMessageLenErr{messageType: "EmptyQueryResponse", expectedLen: 0, actualLen: len(src)} + } + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *EmptyQueryResponse) Encode(dst []byte) []byte { + return append(dst, 'I', 0, 0, 0, 4) +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src EmptyQueryResponse) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + }{ + Type: "EmptyQueryResponse", + }) +} diff --git a/github.com/jackc/pgproto3/v2/error_response.go b/github.com/jackc/pgproto3/v2/error_response.go new file mode 100644 index 0000000000..d444798bce --- /dev/null +++ b/github.com/jackc/pgproto3/v2/error_response.go @@ -0,0 +1,218 @@ +package pgproto3 + +import ( + "bytes" + "encoding/binary" + "strconv" +) + +type ErrorResponse struct { + Severity string + Code string + Message string + Detail string + Hint string + Position int32 + InternalPosition int32 + InternalQuery string + Where string + SchemaName string + TableName string + ColumnName string + DataTypeName string + ConstraintName string + File string + Line int32 + Routine string + + UnknownFields map[byte]string +} + +// Backend identifies this message as sendable by the PostgreSQL backend. +func (*ErrorResponse) Backend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *ErrorResponse) Decode(src []byte) error { + *dst = ErrorResponse{} + + buf := bytes.NewBuffer(src) + + for { + k, err := buf.ReadByte() + if err != nil { + return err + } + if k == 0 { + break + } + + vb, err := buf.ReadBytes(0) + if err != nil { + return err + } + v := string(vb[:len(vb)-1]) + + switch k { + case 'S': + dst.Severity = v + case 'C': + dst.Code = v + case 'M': + dst.Message = v + case 'D': + dst.Detail = v + case 'H': + dst.Hint = v + case 'P': + s := v + n, _ := strconv.ParseInt(s, 10, 32) + dst.Position = int32(n) + case 'p': + s := v + n, _ := strconv.ParseInt(s, 10, 32) + dst.InternalPosition = int32(n) + case 'q': + dst.InternalQuery = v + case 'W': + dst.Where = v + case 's': + dst.SchemaName = v + case 't': + dst.TableName = v + case 'c': + dst.ColumnName = v + case 'd': + dst.DataTypeName = v + case 'n': + dst.ConstraintName = v + case 'F': + dst.File = v + case 'L': + s := v + n, _ := strconv.ParseInt(s, 10, 32) + dst.Line = int32(n) + case 'R': + dst.Routine = v + + default: + if dst.UnknownFields == nil { + dst.UnknownFields = make(map[byte]string) + } + dst.UnknownFields[k] = v + } + } + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *ErrorResponse) Encode(dst []byte) []byte { + return append(dst, src.marshalBinary('E')...) +} + +func (src *ErrorResponse) marshalBinary(typeByte byte) []byte { + var bigEndian BigEndianBuf + buf := &bytes.Buffer{} + + buf.WriteByte(typeByte) + buf.Write(bigEndian.Uint32(0)) + + if src.Severity != "" { + buf.WriteByte('S') + buf.WriteString(src.Severity) + buf.WriteByte(0) + } + if src.Code != "" { + buf.WriteByte('C') + buf.WriteString(src.Code) + buf.WriteByte(0) + } + if src.Message != "" { + buf.WriteByte('M') + buf.WriteString(src.Message) + buf.WriteByte(0) + } + if src.Detail != "" { + buf.WriteByte('D') + buf.WriteString(src.Detail) + buf.WriteByte(0) + } + if src.Hint != "" { + buf.WriteByte('H') + buf.WriteString(src.Hint) + buf.WriteByte(0) + } + if src.Position != 0 { + buf.WriteByte('P') + buf.WriteString(strconv.Itoa(int(src.Position))) + buf.WriteByte(0) + } + if src.InternalPosition != 0 { + buf.WriteByte('p') + buf.WriteString(strconv.Itoa(int(src.InternalPosition))) + buf.WriteByte(0) + } + if src.InternalQuery != "" { + buf.WriteByte('q') + buf.WriteString(src.InternalQuery) + buf.WriteByte(0) + } + if src.Where != "" { + buf.WriteByte('W') + buf.WriteString(src.Where) + buf.WriteByte(0) + } + if src.SchemaName != "" { + buf.WriteByte('s') + buf.WriteString(src.SchemaName) + buf.WriteByte(0) + } + if src.TableName != "" { + buf.WriteByte('t') + buf.WriteString(src.TableName) + buf.WriteByte(0) + } + if src.ColumnName != "" { + buf.WriteByte('c') + buf.WriteString(src.ColumnName) + buf.WriteByte(0) + } + if src.DataTypeName != "" { + buf.WriteByte('d') + buf.WriteString(src.DataTypeName) + buf.WriteByte(0) + } + if src.ConstraintName != "" { + buf.WriteByte('n') + buf.WriteString(src.ConstraintName) + buf.WriteByte(0) + } + if src.File != "" { + buf.WriteByte('F') + buf.WriteString(src.File) + buf.WriteByte(0) + } + if src.Line != 0 { + buf.WriteByte('L') + buf.WriteString(strconv.Itoa(int(src.Line))) + buf.WriteByte(0) + } + if src.Routine != "" { + buf.WriteByte('R') + buf.WriteString(src.Routine) + buf.WriteByte(0) + } + + for k, v := range src.UnknownFields { + buf.WriteByte(k) + buf.WriteByte(0) + buf.WriteString(v) + buf.WriteByte(0) + } + buf.WriteByte(0) + + binary.BigEndian.PutUint32(buf.Bytes()[1:5], uint32(buf.Len()-1)) + + return buf.Bytes() +} diff --git a/github.com/jackc/pgproto3/v2/execute.go b/github.com/jackc/pgproto3/v2/execute.go new file mode 100644 index 0000000000..8bae613321 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/execute.go @@ -0,0 +1,65 @@ +package pgproto3 + +import ( + "bytes" + "encoding/binary" + "encoding/json" + + "github.com/jackc/pgio" +) + +type Execute struct { + Portal string + MaxRows uint32 +} + +// Frontend identifies this message as sendable by a PostgreSQL frontend. +func (*Execute) Frontend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *Execute) Decode(src []byte) error { + buf := bytes.NewBuffer(src) + + b, err := buf.ReadBytes(0) + if err != nil { + return err + } + dst.Portal = string(b[:len(b)-1]) + + if buf.Len() < 4 { + return &invalidMessageFormatErr{messageType: "Execute"} + } + dst.MaxRows = binary.BigEndian.Uint32(buf.Next(4)) + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *Execute) Encode(dst []byte) []byte { + dst = append(dst, 'E') + sp := len(dst) + dst = pgio.AppendInt32(dst, -1) + + dst = append(dst, src.Portal...) + dst = append(dst, 0) + + dst = pgio.AppendUint32(dst, src.MaxRows) + + pgio.SetInt32(dst[sp:], int32(len(dst[sp:]))) + + return dst +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src Execute) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + Portal string + MaxRows uint32 + }{ + Type: "Execute", + Portal: src.Portal, + MaxRows: src.MaxRows, + }) +} diff --git a/github.com/jackc/pgproto3/v2/flush.go b/github.com/jackc/pgproto3/v2/flush.go new file mode 100644 index 0000000000..2725f68942 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/flush.go @@ -0,0 +1,34 @@ +package pgproto3 + +import ( + "encoding/json" +) + +type Flush struct{} + +// Frontend identifies this message as sendable by a PostgreSQL frontend. +func (*Flush) Frontend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *Flush) Decode(src []byte) error { + if len(src) != 0 { + return &invalidMessageLenErr{messageType: "Flush", expectedLen: 0, actualLen: len(src)} + } + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *Flush) Encode(dst []byte) []byte { + return append(dst, 'H', 0, 0, 0, 4) +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src Flush) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + }{ + Type: "Flush", + }) +} diff --git a/github.com/jackc/pgproto3/v2/frontend.go b/github.com/jackc/pgproto3/v2/frontend.go new file mode 100644 index 0000000000..3298d7e6d5 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/frontend.go @@ -0,0 +1,180 @@ +package pgproto3 + +import ( + "encoding/binary" + "errors" + "fmt" + "io" +) + +// Frontend acts as a client for the PostgreSQL wire protocol version 3. +type Frontend struct { + cr ChunkReader + w io.Writer + + // Backend message flyweights + authenticationOk AuthenticationOk + authenticationCleartextPassword AuthenticationCleartextPassword + authenticationMD5Password AuthenticationMD5Password + authenticationSASL AuthenticationSASL + authenticationSASLContinue AuthenticationSASLContinue + authenticationSASLFinal AuthenticationSASLFinal + backendKeyData BackendKeyData + bindComplete BindComplete + closeComplete CloseComplete + commandComplete CommandComplete + copyBothResponse CopyBothResponse + copyData CopyData + copyInResponse CopyInResponse + copyOutResponse CopyOutResponse + copyDone CopyDone + dataRow DataRow + emptyQueryResponse EmptyQueryResponse + errorResponse ErrorResponse + functionCallResponse FunctionCallResponse + noData NoData + noticeResponse NoticeResponse + notificationResponse NotificationResponse + parameterDescription ParameterDescription + parameterStatus ParameterStatus + parseComplete ParseComplete + readyForQuery ReadyForQuery + rowDescription RowDescription + portalSuspended PortalSuspended + + bodyLen int + msgType byte + partialMsg bool +} + +// NewFrontend creates a new Frontend. +func NewFrontend(cr ChunkReader, w io.Writer) *Frontend { + return &Frontend{cr: cr, w: w} +} + +// Send sends a message to the backend. +func (f *Frontend) Send(msg FrontendMessage) error { + _, err := f.w.Write(msg.Encode(nil)) + return err +} + +func translateEOFtoErrUnexpectedEOF(err error) error { + if err == io.EOF { + return io.ErrUnexpectedEOF + } + return err +} + +// Receive receives a message from the backend. +func (f *Frontend) Receive() (BackendMessage, error) { + if !f.partialMsg { + header, err := f.cr.Next(5) + if err != nil { + return nil, translateEOFtoErrUnexpectedEOF(err) + } + + f.msgType = header[0] + f.bodyLen = int(binary.BigEndian.Uint32(header[1:])) - 4 + f.partialMsg = true + } + + msgBody, err := f.cr.Next(f.bodyLen) + if err != nil { + return nil, translateEOFtoErrUnexpectedEOF(err) + } + + f.partialMsg = false + + var msg BackendMessage + switch f.msgType { + case '1': + msg = &f.parseComplete + case '2': + msg = &f.bindComplete + case '3': + msg = &f.closeComplete + case 'A': + msg = &f.notificationResponse + case 'c': + msg = &f.copyDone + case 'C': + msg = &f.commandComplete + case 'd': + msg = &f.copyData + case 'D': + msg = &f.dataRow + case 'E': + msg = &f.errorResponse + case 'G': + msg = &f.copyInResponse + case 'H': + msg = &f.copyOutResponse + case 'I': + msg = &f.emptyQueryResponse + case 'K': + msg = &f.backendKeyData + case 'n': + msg = &f.noData + case 'N': + msg = &f.noticeResponse + case 'R': + var err error + msg, err = f.findAuthenticationMessageType(msgBody) + if err != nil { + return nil, err + } + case 's': + msg = &f.portalSuspended + case 'S': + msg = &f.parameterStatus + case 't': + msg = &f.parameterDescription + case 'T': + msg = &f.rowDescription + case 'V': + msg = &f.functionCallResponse + case 'W': + msg = &f.copyBothResponse + case 'Z': + msg = &f.readyForQuery + default: + return nil, fmt.Errorf("unknown message type: %c", f.msgType) + } + + err = msg.Decode(msgBody) + return msg, err +} + +// Authentication message type constants. +const ( + AuthTypeOk = 0 + AuthTypeCleartextPassword = 3 + AuthTypeMD5Password = 5 + AuthTypeSASL = 10 + AuthTypeSASLContinue = 11 + AuthTypeSASLFinal = 12 +) + +func (f *Frontend) findAuthenticationMessageType(src []byte) (BackendMessage, error) { + if len(src) < 4 { + return nil, errors.New("authentication message too short") + } + authType := binary.BigEndian.Uint32(src[:4]) + + switch authType { + case AuthTypeOk: + return &f.authenticationOk, nil + case AuthTypeCleartextPassword: + return &f.authenticationCleartextPassword, nil + case AuthTypeMD5Password: + return &f.authenticationMD5Password, nil + case AuthTypeSASL: + return &f.authenticationSASL, nil + case AuthTypeSASLContinue: + return &f.authenticationSASLContinue, nil + case AuthTypeSASLFinal: + return &f.authenticationSASLFinal, nil + default: + return nil, fmt.Errorf("unknown authentication type: %d", authType) + } +} diff --git a/github.com/jackc/pgproto3/v2/function_call_response.go b/github.com/jackc/pgproto3/v2/function_call_response.go new file mode 100644 index 0000000000..5cc2d4d29b --- /dev/null +++ b/github.com/jackc/pgproto3/v2/function_call_response.go @@ -0,0 +1,83 @@ +package pgproto3 + +import ( + "encoding/binary" + "encoding/hex" + "encoding/json" + + "github.com/jackc/pgio" +) + +type FunctionCallResponse struct { + Result []byte +} + +// Backend identifies this message as sendable by the PostgreSQL backend. +func (*FunctionCallResponse) Backend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *FunctionCallResponse) Decode(src []byte) error { + if len(src) < 4 { + return &invalidMessageFormatErr{messageType: "FunctionCallResponse"} + } + rp := 0 + resultSize := int(binary.BigEndian.Uint32(src[rp:])) + rp += 4 + + if resultSize == -1 { + dst.Result = nil + return nil + } + + if len(src[rp:]) != resultSize { + return &invalidMessageFormatErr{messageType: "FunctionCallResponse"} + } + + dst.Result = src[rp:] + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *FunctionCallResponse) Encode(dst []byte) []byte { + dst = append(dst, 'V') + sp := len(dst) + dst = pgio.AppendInt32(dst, -1) + + if src.Result == nil { + dst = pgio.AppendInt32(dst, -1) + } else { + dst = pgio.AppendInt32(dst, int32(len(src.Result))) + dst = append(dst, src.Result...) + } + + pgio.SetInt32(dst[sp:], int32(len(dst[sp:]))) + + return dst +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src FunctionCallResponse) MarshalJSON() ([]byte, error) { + var formattedValue map[string]string + var hasNonPrintable bool + for _, b := range src.Result { + if b < 32 { + hasNonPrintable = true + break + } + } + + if hasNonPrintable { + formattedValue = map[string]string{"binary": hex.EncodeToString(src.Result)} + } else { + formattedValue = map[string]string{"text": string(src.Result)} + } + + return json.Marshal(struct { + Type string + Result map[string]string + }{ + Type: "FunctionCallResponse", + Result: formattedValue, + }) +} diff --git a/github.com/jackc/pgproto3/v2/go.mod b/github.com/jackc/pgproto3/v2/go.mod new file mode 100644 index 0000000000..36041a94ad --- /dev/null +++ b/github.com/jackc/pgproto3/v2/go.mod @@ -0,0 +1,9 @@ +module github.com/jackc/pgproto3/v2 + +go 1.12 + +require ( + github.com/jackc/chunkreader/v2 v2.0.0 + github.com/jackc/pgio v1.0.0 + github.com/stretchr/testify v1.4.0 +) diff --git a/github.com/jackc/pgproto3/v2/go.sum b/github.com/jackc/pgproto3/v2/go.sum new file mode 100644 index 0000000000..dd9cd044f6 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/go.sum @@ -0,0 +1,14 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jackc/chunkreader/v2 v2.0.0 h1:DUwgMQuuPnS0rhMXenUtZpqZqrR/30NWY+qQvTpSvEs= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/github.com/jackc/pgproto3/v2/no_data.go b/github.com/jackc/pgproto3/v2/no_data.go new file mode 100644 index 0000000000..d8f85d38a7 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/no_data.go @@ -0,0 +1,34 @@ +package pgproto3 + +import ( + "encoding/json" +) + +type NoData struct{} + +// Backend identifies this message as sendable by the PostgreSQL backend. +func (*NoData) Backend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *NoData) Decode(src []byte) error { + if len(src) != 0 { + return &invalidMessageLenErr{messageType: "NoData", expectedLen: 0, actualLen: len(src)} + } + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *NoData) Encode(dst []byte) []byte { + return append(dst, 'n', 0, 0, 0, 4) +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src NoData) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + }{ + Type: "NoData", + }) +} diff --git a/github.com/jackc/pgproto3/v2/notice_response.go b/github.com/jackc/pgproto3/v2/notice_response.go new file mode 100644 index 0000000000..4ac28a7911 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/notice_response.go @@ -0,0 +1,17 @@ +package pgproto3 + +type NoticeResponse ErrorResponse + +// Backend identifies this message as sendable by the PostgreSQL backend. +func (*NoticeResponse) Backend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *NoticeResponse) Decode(src []byte) error { + return (*ErrorResponse)(dst).Decode(src) +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *NoticeResponse) Encode(dst []byte) []byte { + return append(dst, (*ErrorResponse)(src).marshalBinary('N')...) +} diff --git a/github.com/jackc/pgproto3/v2/notification_response.go b/github.com/jackc/pgproto3/v2/notification_response.go new file mode 100644 index 0000000000..cd83c5baed --- /dev/null +++ b/github.com/jackc/pgproto3/v2/notification_response.go @@ -0,0 +1,72 @@ +package pgproto3 + +import ( + "bytes" + "encoding/binary" + "encoding/json" + + "github.com/jackc/pgio" +) + +type NotificationResponse struct { + PID uint32 + Channel string + Payload string +} + +// Backend identifies this message as sendable by the PostgreSQL backend. +func (*NotificationResponse) Backend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *NotificationResponse) Decode(src []byte) error { + buf := bytes.NewBuffer(src) + + pid := binary.BigEndian.Uint32(buf.Next(4)) + + b, err := buf.ReadBytes(0) + if err != nil { + return err + } + channel := string(b[:len(b)-1]) + + b, err = buf.ReadBytes(0) + if err != nil { + return err + } + payload := string(b[:len(b)-1]) + + *dst = NotificationResponse{PID: pid, Channel: channel, Payload: payload} + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *NotificationResponse) Encode(dst []byte) []byte { + dst = append(dst, 'A') + sp := len(dst) + dst = pgio.AppendInt32(dst, -1) + + dst = append(dst, src.Channel...) + dst = append(dst, 0) + dst = append(dst, src.Payload...) + dst = append(dst, 0) + + pgio.SetInt32(dst[sp:], int32(len(dst[sp:]))) + + return dst +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src NotificationResponse) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + PID uint32 + Channel string + Payload string + }{ + Type: "NotificationResponse", + PID: src.PID, + Channel: src.Channel, + Payload: src.Payload, + }) +} diff --git a/github.com/jackc/pgproto3/v2/parameter_description.go b/github.com/jackc/pgproto3/v2/parameter_description.go new file mode 100644 index 0000000000..e28965c8ab --- /dev/null +++ b/github.com/jackc/pgproto3/v2/parameter_description.go @@ -0,0 +1,66 @@ +package pgproto3 + +import ( + "bytes" + "encoding/binary" + "encoding/json" + + "github.com/jackc/pgio" +) + +type ParameterDescription struct { + ParameterOIDs []uint32 +} + +// Backend identifies this message as sendable by the PostgreSQL backend. +func (*ParameterDescription) Backend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *ParameterDescription) Decode(src []byte) error { + buf := bytes.NewBuffer(src) + + if buf.Len() < 2 { + return &invalidMessageFormatErr{messageType: "ParameterDescription"} + } + + // Reported parameter count will be incorrect when number of args is greater than uint16 + buf.Next(2) + // Instead infer parameter count by remaining size of message + parameterCount := buf.Len() / 4 + + *dst = ParameterDescription{ParameterOIDs: make([]uint32, parameterCount)} + + for i := 0; i < parameterCount; i++ { + dst.ParameterOIDs[i] = binary.BigEndian.Uint32(buf.Next(4)) + } + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *ParameterDescription) Encode(dst []byte) []byte { + dst = append(dst, 't') + sp := len(dst) + dst = pgio.AppendInt32(dst, -1) + + dst = pgio.AppendUint16(dst, uint16(len(src.ParameterOIDs))) + for _, oid := range src.ParameterOIDs { + dst = pgio.AppendUint32(dst, oid) + } + + pgio.SetInt32(dst[sp:], int32(len(dst[sp:]))) + + return dst +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src ParameterDescription) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + ParameterOIDs []uint32 + }{ + Type: "ParameterDescription", + ParameterOIDs: src.ParameterOIDs, + }) +} diff --git a/github.com/jackc/pgproto3/v2/parameter_status.go b/github.com/jackc/pgproto3/v2/parameter_status.go new file mode 100644 index 0000000000..c4021d92f7 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/parameter_status.go @@ -0,0 +1,66 @@ +package pgproto3 + +import ( + "bytes" + "encoding/json" + + "github.com/jackc/pgio" +) + +type ParameterStatus struct { + Name string + Value string +} + +// Backend identifies this message as sendable by the PostgreSQL backend. +func (*ParameterStatus) Backend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *ParameterStatus) Decode(src []byte) error { + buf := bytes.NewBuffer(src) + + b, err := buf.ReadBytes(0) + if err != nil { + return err + } + name := string(b[:len(b)-1]) + + b, err = buf.ReadBytes(0) + if err != nil { + return err + } + value := string(b[:len(b)-1]) + + *dst = ParameterStatus{Name: name, Value: value} + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *ParameterStatus) Encode(dst []byte) []byte { + dst = append(dst, 'S') + sp := len(dst) + dst = pgio.AppendInt32(dst, -1) + + dst = append(dst, src.Name...) + dst = append(dst, 0) + dst = append(dst, src.Value...) + dst = append(dst, 0) + + pgio.SetInt32(dst[sp:], int32(len(dst[sp:]))) + + return dst +} + +// MarshalJSON implements encoding/json.Marshaler. +func (ps ParameterStatus) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + Name string + Value string + }{ + Type: "ParameterStatus", + Name: ps.Name, + Value: ps.Value, + }) +} diff --git a/github.com/jackc/pgproto3/v2/parse.go b/github.com/jackc/pgproto3/v2/parse.go new file mode 100644 index 0000000000..723885d41e --- /dev/null +++ b/github.com/jackc/pgproto3/v2/parse.go @@ -0,0 +1,88 @@ +package pgproto3 + +import ( + "bytes" + "encoding/binary" + "encoding/json" + + "github.com/jackc/pgio" +) + +type Parse struct { + Name string + Query string + ParameterOIDs []uint32 +} + +// Frontend identifies this message as sendable by a PostgreSQL frontend. +func (*Parse) Frontend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *Parse) Decode(src []byte) error { + *dst = Parse{} + + buf := bytes.NewBuffer(src) + + b, err := buf.ReadBytes(0) + if err != nil { + return err + } + dst.Name = string(b[:len(b)-1]) + + b, err = buf.ReadBytes(0) + if err != nil { + return err + } + dst.Query = string(b[:len(b)-1]) + + if buf.Len() < 2 { + return &invalidMessageFormatErr{messageType: "Parse"} + } + parameterOIDCount := int(binary.BigEndian.Uint16(buf.Next(2))) + + for i := 0; i < parameterOIDCount; i++ { + if buf.Len() < 4 { + return &invalidMessageFormatErr{messageType: "Parse"} + } + dst.ParameterOIDs = append(dst.ParameterOIDs, binary.BigEndian.Uint32(buf.Next(4))) + } + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *Parse) Encode(dst []byte) []byte { + dst = append(dst, 'P') + sp := len(dst) + dst = pgio.AppendInt32(dst, -1) + + dst = append(dst, src.Name...) + dst = append(dst, 0) + dst = append(dst, src.Query...) + dst = append(dst, 0) + + dst = pgio.AppendUint16(dst, uint16(len(src.ParameterOIDs))) + for _, oid := range src.ParameterOIDs { + dst = pgio.AppendUint32(dst, oid) + } + + pgio.SetInt32(dst[sp:], int32(len(dst[sp:]))) + + return dst +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src Parse) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + Name string + Query string + ParameterOIDs []uint32 + }{ + Type: "Parse", + Name: src.Name, + Query: src.Query, + ParameterOIDs: src.ParameterOIDs, + }) +} diff --git a/github.com/jackc/pgproto3/v2/parse_complete.go b/github.com/jackc/pgproto3/v2/parse_complete.go new file mode 100644 index 0000000000..92c9498b6d --- /dev/null +++ b/github.com/jackc/pgproto3/v2/parse_complete.go @@ -0,0 +1,34 @@ +package pgproto3 + +import ( + "encoding/json" +) + +type ParseComplete struct{} + +// Backend identifies this message as sendable by the PostgreSQL backend. +func (*ParseComplete) Backend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *ParseComplete) Decode(src []byte) error { + if len(src) != 0 { + return &invalidMessageLenErr{messageType: "ParseComplete", expectedLen: 0, actualLen: len(src)} + } + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *ParseComplete) Encode(dst []byte) []byte { + return append(dst, '1', 0, 0, 0, 4) +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src ParseComplete) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + }{ + Type: "ParseComplete", + }) +} diff --git a/github.com/jackc/pgproto3/v2/password_message.go b/github.com/jackc/pgproto3/v2/password_message.go new file mode 100644 index 0000000000..4b68b31a84 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/password_message.go @@ -0,0 +1,51 @@ +package pgproto3 + +import ( + "bytes" + "encoding/json" + + "github.com/jackc/pgio" +) + +type PasswordMessage struct { + Password string +} + +// Frontend identifies this message as sendable by a PostgreSQL frontend. +func (*PasswordMessage) Frontend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *PasswordMessage) Decode(src []byte) error { + buf := bytes.NewBuffer(src) + + b, err := buf.ReadBytes(0) + if err != nil { + return err + } + dst.Password = string(b[:len(b)-1]) + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *PasswordMessage) Encode(dst []byte) []byte { + dst = append(dst, 'p') + dst = pgio.AppendInt32(dst, int32(4+len(src.Password)+1)) + + dst = append(dst, src.Password...) + dst = append(dst, 0) + + return dst +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src PasswordMessage) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + Password string + }{ + Type: "PasswordMessage", + Password: src.Password, + }) +} diff --git a/github.com/jackc/pgproto3/v2/pgproto3.go b/github.com/jackc/pgproto3/v2/pgproto3.go new file mode 100644 index 0000000000..fe7b085bcb --- /dev/null +++ b/github.com/jackc/pgproto3/v2/pgproto3.go @@ -0,0 +1,42 @@ +package pgproto3 + +import "fmt" + +// Message is the interface implemented by an object that can decode and encode +// a particular PostgreSQL message. +type Message interface { + // Decode is allowed and expected to retain a reference to data after + // returning (unlike encoding.BinaryUnmarshaler). + Decode(data []byte) error + + // Encode appends itself to dst and returns the new buffer. + Encode(dst []byte) []byte +} + +type FrontendMessage interface { + Message + Frontend() // no-op method to distinguish frontend from backend methods +} + +type BackendMessage interface { + Message + Backend() // no-op method to distinguish frontend from backend methods +} + +type invalidMessageLenErr struct { + messageType string + expectedLen int + actualLen int +} + +func (e *invalidMessageLenErr) Error() string { + return fmt.Sprintf("%s body must have length of %d, but it is %d", e.messageType, e.expectedLen, e.actualLen) +} + +type invalidMessageFormatErr struct { + messageType string +} + +func (e *invalidMessageFormatErr) Error() string { + return fmt.Sprintf("%s body is invalid", e.messageType) +} diff --git a/github.com/jackc/pgproto3/v2/portal_suspended.go b/github.com/jackc/pgproto3/v2/portal_suspended.go new file mode 100644 index 0000000000..1a9e7bfb1a --- /dev/null +++ b/github.com/jackc/pgproto3/v2/portal_suspended.go @@ -0,0 +1,34 @@ +package pgproto3 + +import ( + "encoding/json" +) + +type PortalSuspended struct{} + +// Backend identifies this message as sendable by the PostgreSQL backend. +func (*PortalSuspended) Backend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *PortalSuspended) Decode(src []byte) error { + if len(src) != 0 { + return &invalidMessageLenErr{messageType: "PortalSuspended", expectedLen: 0, actualLen: len(src)} + } + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *PortalSuspended) Encode(dst []byte) []byte { + return append(dst, 's', 0, 0, 0, 4) +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src PortalSuspended) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + }{ + Type: "PortalSuspended", + }) +} diff --git a/github.com/jackc/pgproto3/v2/query.go b/github.com/jackc/pgproto3/v2/query.go new file mode 100644 index 0000000000..41c93b4a83 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/query.go @@ -0,0 +1,50 @@ +package pgproto3 + +import ( + "bytes" + "encoding/json" + + "github.com/jackc/pgio" +) + +type Query struct { + String string +} + +// Frontend identifies this message as sendable by a PostgreSQL frontend. +func (*Query) Frontend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *Query) Decode(src []byte) error { + i := bytes.IndexByte(src, 0) + if i != len(src)-1 { + return &invalidMessageFormatErr{messageType: "Query"} + } + + dst.String = string(src[:i]) + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *Query) Encode(dst []byte) []byte { + dst = append(dst, 'Q') + dst = pgio.AppendInt32(dst, int32(4+len(src.String)+1)) + + dst = append(dst, src.String...) + dst = append(dst, 0) + + return dst +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src Query) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + String string + }{ + Type: "Query", + String: src.String, + }) +} diff --git a/github.com/jackc/pgproto3/v2/ready_for_query.go b/github.com/jackc/pgproto3/v2/ready_for_query.go new file mode 100644 index 0000000000..879afe390d --- /dev/null +++ b/github.com/jackc/pgproto3/v2/ready_for_query.go @@ -0,0 +1,40 @@ +package pgproto3 + +import ( + "encoding/json" +) + +type ReadyForQuery struct { + TxStatus byte +} + +// Backend identifies this message as sendable by the PostgreSQL backend. +func (*ReadyForQuery) Backend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *ReadyForQuery) Decode(src []byte) error { + if len(src) != 1 { + return &invalidMessageLenErr{messageType: "ReadyForQuery", expectedLen: 1, actualLen: len(src)} + } + + dst.TxStatus = src[0] + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *ReadyForQuery) Encode(dst []byte) []byte { + return append(dst, 'Z', 0, 0, 0, 5, src.TxStatus) +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src ReadyForQuery) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + TxStatus string + }{ + Type: "ReadyForQuery", + TxStatus: string(src.TxStatus), + }) +} diff --git a/github.com/jackc/pgproto3/v2/row_description.go b/github.com/jackc/pgproto3/v2/row_description.go new file mode 100644 index 0000000000..d9b8c7c989 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/row_description.go @@ -0,0 +1,134 @@ +package pgproto3 + +import ( + "bytes" + "encoding/binary" + "encoding/json" + + "github.com/jackc/pgio" +) + +const ( + TextFormat = 0 + BinaryFormat = 1 +) + +type FieldDescription struct { + Name []byte + TableOID uint32 + TableAttributeNumber uint16 + DataTypeOID uint32 + DataTypeSize int16 + TypeModifier int32 + Format int16 +} + +// MarshalJSON implements encoding/json.Marshaler. +func (fd FieldDescription) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Name string + TableOID uint32 + TableAttributeNumber uint16 + DataTypeOID uint32 + DataTypeSize int16 + TypeModifier int32 + Format int16 + }{ + Name: string(fd.Name), + TableOID: fd.TableOID, + TableAttributeNumber: fd.TableAttributeNumber, + DataTypeOID: fd.DataTypeOID, + DataTypeSize: fd.DataTypeSize, + TypeModifier: fd.TypeModifier, + Format: fd.Format, + }) +} + +type RowDescription struct { + Fields []FieldDescription +} + +// Backend identifies this message as sendable by the PostgreSQL backend. +func (*RowDescription) Backend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *RowDescription) Decode(src []byte) error { + + if len(src) < 2 { + return &invalidMessageFormatErr{messageType: "RowDescription"} + } + fieldCount := int(binary.BigEndian.Uint16(src)) + rp := 2 + + dst.Fields = dst.Fields[0:0] + + for i := 0; i < fieldCount; i++ { + var fd FieldDescription + + idx := bytes.IndexByte(src[rp:], 0) + if idx < 0 { + return &invalidMessageFormatErr{messageType: "RowDescription"} + } + fd.Name = src[rp : rp+idx] + rp += idx + 1 + + // Since buf.Next() doesn't return an error if we hit the end of the buffer + // check Len ahead of time + if len(src[rp:]) < 18 { + return &invalidMessageFormatErr{messageType: "RowDescription"} + } + + fd.TableOID = binary.BigEndian.Uint32(src[rp:]) + rp += 4 + fd.TableAttributeNumber = binary.BigEndian.Uint16(src[rp:]) + rp += 2 + fd.DataTypeOID = binary.BigEndian.Uint32(src[rp:]) + rp += 4 + fd.DataTypeSize = int16(binary.BigEndian.Uint16(src[rp:])) + rp += 2 + fd.TypeModifier = int32(binary.BigEndian.Uint32(src[rp:])) + rp += 4 + fd.Format = int16(binary.BigEndian.Uint16(src[rp:])) + rp += 2 + + dst.Fields = append(dst.Fields, fd) + } + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *RowDescription) Encode(dst []byte) []byte { + dst = append(dst, 'T') + sp := len(dst) + dst = pgio.AppendInt32(dst, -1) + + dst = pgio.AppendUint16(dst, uint16(len(src.Fields))) + for _, fd := range src.Fields { + dst = append(dst, fd.Name...) + dst = append(dst, 0) + + dst = pgio.AppendUint32(dst, fd.TableOID) + dst = pgio.AppendUint16(dst, fd.TableAttributeNumber) + dst = pgio.AppendUint32(dst, fd.DataTypeOID) + dst = pgio.AppendInt16(dst, fd.DataTypeSize) + dst = pgio.AppendInt32(dst, fd.TypeModifier) + dst = pgio.AppendInt16(dst, fd.Format) + } + + pgio.SetInt32(dst[sp:], int32(len(dst[sp:]))) + + return dst +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src RowDescription) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + Fields []FieldDescription + }{ + Type: "RowDescription", + Fields: src.Fields, + }) +} diff --git a/github.com/jackc/pgproto3/v2/sasl_initial_response.go b/github.com/jackc/pgproto3/v2/sasl_initial_response.go new file mode 100644 index 0000000000..0bf8a9e56e --- /dev/null +++ b/github.com/jackc/pgproto3/v2/sasl_initial_response.go @@ -0,0 +1,69 @@ +package pgproto3 + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "errors" + + "github.com/jackc/pgio" +) + +type SASLInitialResponse struct { + AuthMechanism string + Data []byte +} + +// Frontend identifies this message as sendable by a PostgreSQL frontend. +func (*SASLInitialResponse) Frontend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *SASLInitialResponse) Decode(src []byte) error { + *dst = SASLInitialResponse{} + + rp := 0 + + idx := bytes.IndexByte(src, 0) + if idx < 0 { + return errors.New("invalid SASLInitialResponse") + } + + dst.AuthMechanism = string(src[rp:idx]) + rp = idx + 1 + + rp += 4 // The rest of the message is data so we can just skip the size + dst.Data = src[rp:] + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *SASLInitialResponse) Encode(dst []byte) []byte { + dst = append(dst, 'p') + sp := len(dst) + dst = pgio.AppendInt32(dst, -1) + + dst = append(dst, []byte(src.AuthMechanism)...) + dst = append(dst, 0) + + dst = pgio.AppendInt32(dst, int32(len(src.Data))) + dst = append(dst, src.Data...) + + pgio.SetInt32(dst[sp:], int32(len(dst[sp:]))) + + return dst +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src SASLInitialResponse) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + AuthMechanism string + Data string + }{ + Type: "SASLInitialResponse", + AuthMechanism: src.AuthMechanism, + Data: hex.EncodeToString(src.Data), + }) +} diff --git a/github.com/jackc/pgproto3/v2/sasl_response.go b/github.com/jackc/pgproto3/v2/sasl_response.go new file mode 100644 index 0000000000..21be6d7554 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/sasl_response.go @@ -0,0 +1,43 @@ +package pgproto3 + +import ( + "encoding/hex" + "encoding/json" + + "github.com/jackc/pgio" +) + +type SASLResponse struct { + Data []byte +} + +// Frontend identifies this message as sendable by a PostgreSQL frontend. +func (*SASLResponse) Frontend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *SASLResponse) Decode(src []byte) error { + *dst = SASLResponse{Data: src} + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *SASLResponse) Encode(dst []byte) []byte { + dst = append(dst, 'p') + dst = pgio.AppendInt32(dst, int32(4+len(src.Data))) + + dst = append(dst, src.Data...) + + return dst +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src SASLResponse) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + Data string + }{ + Type: "SASLResponse", + Data: hex.EncodeToString(src.Data), + }) +} diff --git a/github.com/jackc/pgproto3/v2/ssl_request.go b/github.com/jackc/pgproto3/v2/ssl_request.go new file mode 100644 index 0000000000..96ce489e5c --- /dev/null +++ b/github.com/jackc/pgproto3/v2/ssl_request.go @@ -0,0 +1,49 @@ +package pgproto3 + +import ( + "encoding/binary" + "encoding/json" + "errors" + + "github.com/jackc/pgio" +) + +const sslRequestNumber = 80877103 + +type SSLRequest struct { +} + +// Frontend identifies this message as sendable by a PostgreSQL frontend. +func (*SSLRequest) Frontend() {} + +func (dst *SSLRequest) Decode(src []byte) error { + if len(src) < 4 { + return errors.New("ssl request too short") + } + + requestCode := binary.BigEndian.Uint32(src) + + if requestCode != sslRequestNumber { + return errors.New("bad ssl request code") + } + + return nil +} + +// Encode encodes src into dst. dst will include the 4 byte message length. +func (src *SSLRequest) Encode(dst []byte) []byte { + dst = pgio.AppendInt32(dst, 8) + dst = pgio.AppendInt32(dst, sslRequestNumber) + return dst +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src SSLRequest) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + ProtocolVersion uint32 + Parameters map[string]string + }{ + Type: "SSLRequest", + }) +} diff --git a/github.com/jackc/pgproto3/v2/startup_message.go b/github.com/jackc/pgproto3/v2/startup_message.go new file mode 100644 index 0000000000..5f1cd24f77 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/startup_message.go @@ -0,0 +1,96 @@ +package pgproto3 + +import ( + "bytes" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + + "github.com/jackc/pgio" +) + +const ProtocolVersionNumber = 196608 // 3.0 + +type StartupMessage struct { + ProtocolVersion uint32 + Parameters map[string]string +} + +// Frontend identifies this message as sendable by a PostgreSQL frontend. +func (*StartupMessage) Frontend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *StartupMessage) Decode(src []byte) error { + if len(src) < 4 { + return errors.New("startup message too short") + } + + dst.ProtocolVersion = binary.BigEndian.Uint32(src) + rp := 4 + + if dst.ProtocolVersion != ProtocolVersionNumber { + return fmt.Errorf("Bad startup message version number. Expected %d, got %d", ProtocolVersionNumber, dst.ProtocolVersion) + } + + dst.Parameters = make(map[string]string) + for { + idx := bytes.IndexByte(src[rp:], 0) + if idx < 0 { + return &invalidMessageFormatErr{messageType: "StartupMesage"} + } + key := string(src[rp : rp+idx]) + rp += idx + 1 + + idx = bytes.IndexByte(src[rp:], 0) + if idx < 0 { + return &invalidMessageFormatErr{messageType: "StartupMesage"} + } + value := string(src[rp : rp+idx]) + rp += idx + 1 + + dst.Parameters[key] = value + + if len(src[rp:]) == 1 { + if src[rp] != 0 { + return fmt.Errorf("Bad startup message last byte. Expected 0, got %d", src[rp]) + } + break + } + } + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *StartupMessage) Encode(dst []byte) []byte { + sp := len(dst) + dst = pgio.AppendInt32(dst, -1) + + dst = pgio.AppendUint32(dst, src.ProtocolVersion) + for k, v := range src.Parameters { + dst = append(dst, k...) + dst = append(dst, 0) + dst = append(dst, v...) + dst = append(dst, 0) + } + dst = append(dst, 0) + + pgio.SetInt32(dst[sp:], int32(len(dst[sp:]))) + + return dst +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src StartupMessage) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + ProtocolVersion uint32 + Parameters map[string]string + }{ + Type: "StartupMessage", + ProtocolVersion: src.ProtocolVersion, + Parameters: src.Parameters, + }) +} diff --git a/github.com/jackc/pgproto3/v2/sync.go b/github.com/jackc/pgproto3/v2/sync.go new file mode 100644 index 0000000000..5db8e07ac1 --- /dev/null +++ b/github.com/jackc/pgproto3/v2/sync.go @@ -0,0 +1,34 @@ +package pgproto3 + +import ( + "encoding/json" +) + +type Sync struct{} + +// Frontend identifies this message as sendable by a PostgreSQL frontend. +func (*Sync) Frontend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *Sync) Decode(src []byte) error { + if len(src) != 0 { + return &invalidMessageLenErr{messageType: "Sync", expectedLen: 0, actualLen: len(src)} + } + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *Sync) Encode(dst []byte) []byte { + return append(dst, 'S', 0, 0, 0, 4) +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src Sync) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + }{ + Type: "Sync", + }) +} diff --git a/github.com/jackc/pgproto3/v2/terminate.go b/github.com/jackc/pgproto3/v2/terminate.go new file mode 100644 index 0000000000..135191eaee --- /dev/null +++ b/github.com/jackc/pgproto3/v2/terminate.go @@ -0,0 +1,34 @@ +package pgproto3 + +import ( + "encoding/json" +) + +type Terminate struct{} + +// Frontend identifies this message as sendable by a PostgreSQL frontend. +func (*Terminate) Frontend() {} + +// Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message +// type identifier and 4 byte message length. +func (dst *Terminate) Decode(src []byte) error { + if len(src) != 0 { + return &invalidMessageLenErr{messageType: "Terminate", expectedLen: 0, actualLen: len(src)} + } + + return nil +} + +// Encode encodes src into dst. dst will include the 1 byte message type identifier and the 4 byte message length. +func (src *Terminate) Encode(dst []byte) []byte { + return append(dst, 'X', 0, 0, 0, 4) +} + +// MarshalJSON implements encoding/json.Marshaler. +func (src Terminate) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Type string + }{ + Type: "Terminate", + }) +} diff --git a/github.com/jackc/pgservicefile/.travis.yml b/github.com/jackc/pgservicefile/.travis.yml new file mode 100644 index 0000000000..e176228e8e --- /dev/null +++ b/github.com/jackc/pgservicefile/.travis.yml @@ -0,0 +1,9 @@ +language: go + +go: + - 1.x + - tip + +matrix: + allow_failures: + - go: tip diff --git a/github.com/jackc/pgservicefile/LICENSE b/github.com/jackc/pgservicefile/LICENSE new file mode 100644 index 0000000000..f1b4c28923 --- /dev/null +++ b/github.com/jackc/pgservicefile/LICENSE @@ -0,0 +1,22 @@ +Copyright (c) 2020 Jack Christensen + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/github.com/jackc/pgservicefile/README.md b/github.com/jackc/pgservicefile/README.md new file mode 100644 index 0000000000..e50ca12627 --- /dev/null +++ b/github.com/jackc/pgservicefile/README.md @@ -0,0 +1,6 @@ +[![](https://godoc.org/github.com/jackc/pgservicefile?status.svg)](https://godoc.org/github.com/jackc/pgservicefile) +[![Build Status](https://travis-ci.org/jackc/pgservicefile.svg)](https://travis-ci.org/jackc/pgservicefile) + +# pgservicefile + +Package pgservicefile is a parser for PostgreSQL service files (e.g. `.pg_service.conf`). diff --git a/github.com/jackc/pgservicefile/go.mod b/github.com/jackc/pgservicefile/go.mod new file mode 100644 index 0000000000..051e9e0f4e --- /dev/null +++ b/github.com/jackc/pgservicefile/go.mod @@ -0,0 +1,5 @@ +module github.com/jackc/pgservicefile + +go 1.14 + +require github.com/stretchr/testify v1.5.1 diff --git a/github.com/jackc/pgservicefile/go.sum b/github.com/jackc/pgservicefile/go.sum new file mode 100644 index 0000000000..a80206ab1a --- /dev/null +++ b/github.com/jackc/pgservicefile/go.sum @@ -0,0 +1,10 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/github.com/jackc/pgservicefile/pgservicefile.go b/github.com/jackc/pgservicefile/pgservicefile.go new file mode 100644 index 0000000000..a5fae76bd6 --- /dev/null +++ b/github.com/jackc/pgservicefile/pgservicefile.go @@ -0,0 +1,79 @@ +// Package pgservicefile is a parser for PostgreSQL service files (e.g. .pg_service.conf). +package pgservicefile + +import ( + "bufio" + "errors" + "fmt" + "io" + "os" + "strings" +) + +type Service struct { + Name string + Settings map[string]string +} + +type Servicefile struct { + Services []*Service + servicesByName map[string]*Service +} + +// GetService returns the named service. +func (sf *Servicefile) GetService(name string) (*Service, error) { + service, present := sf.servicesByName[name] + if !present { + return nil, errors.New("not found") + } + return service, nil +} + +// ReadServicefile reads the file at path and parses it into a Servicefile. +func ReadServicefile(path string) (*Servicefile, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + return ParseServicefile(f) +} + +// ParseServicefile reads r and parses it into a Servicefile. +func ParseServicefile(r io.Reader) (*Servicefile, error) { + servicefile := &Servicefile{} + + var service *Service + scanner := bufio.NewScanner(r) + lineNum := 0 + for scanner.Scan() { + lineNum += 1 + line := scanner.Text() + line = strings.TrimSpace(line) + + if line == "" || strings.HasPrefix(line, "#") { + // ignore comments and empty lines + } else if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") { + service = &Service{Name: line[1 : len(line)-1], Settings: make(map[string]string)} + servicefile.Services = append(servicefile.Services, service) + } else { + parts := strings.SplitN(line, "=", 2) + if len(parts) != 2 { + fmt.Errorf("unable to parse line %d", lineNum) + } + + key := strings.TrimSpace(parts[0]) + value := strings.TrimSpace(parts[1]) + + service.Settings[key] = value + } + } + + servicefile.servicesByName = make(map[string]*Service, len(servicefile.Services)) + for _, service := range servicefile.Services { + servicefile.servicesByName[service.Name] = service + } + + return servicefile, scanner.Err() +} diff --git a/github.com/jackc/pgtype/CHANGELOG.md b/github.com/jackc/pgtype/CHANGELOG.md new file mode 100644 index 0000000000..560abff372 --- /dev/null +++ b/github.com/jackc/pgtype/CHANGELOG.md @@ -0,0 +1,32 @@ +# 1.3.0 (March 30, 2020) + +* Get implemented on T instead of *T +* Set will call Get on src if possible +* Range types Set method supports its own type, string, and nil +* Date.Set parses string +* Fix correct format verb for unknown type error (Robert Welin) +* Truncate nanoseconds in EncodeText for Timestamptz and Timestamp + +# 1.2.0 (February 5, 2020) + +* Add zeronull package for easier NULL <-> zero conversion +* Add JSON marshalling for shopspring-numeric extension +* Add JSON marshalling for Bool, Date, JSON/B, Timestamptz (Jeffrey Stiles) +* Fix null status in UnmarshalJSON for some types (Jeffrey Stiles) + +# 1.1.0 (January 11, 2020) + +* Add PostgreSQL time type support +* Add more automatic conversions of integer arrays of different types (Jean-Philippe Quéméner) + +# 1.0.3 (November 16, 2019) + +* Support initializing Array types from a slice of the value (Alex Gaynor) + +# 1.0.2 (October 22, 2019) + +* Fix scan into null into pointer to pointer implementing Decode* interface. (Jeremy Altavilla) + +# 1.0.1 (September 19, 2019) + +* Fix daterange OID diff --git a/github.com/jackc/pgtype/LICENSE b/github.com/jackc/pgtype/LICENSE new file mode 100644 index 0000000000..dd9e7be918 --- /dev/null +++ b/github.com/jackc/pgtype/LICENSE @@ -0,0 +1,22 @@ +Copyright (c) 2013 Jack Christensen + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/github.com/jackc/pgtype/README.md b/github.com/jackc/pgtype/README.md new file mode 100644 index 0000000000..6848acc57b --- /dev/null +++ b/github.com/jackc/pgtype/README.md @@ -0,0 +1,7 @@ +[![](https://godoc.org/github.com/jackc/pgtype?status.svg)](https://godoc.org/github.com/jackc/pgtype) + +# pgtype + +pgtype implements Go types for over 70 PostgreSQL types. pgtype is the type system underlying the +https://github.com/jackc/pgx PostgreSQL driver. These types support the binary format for enhanced performance with pgx. +They also support the database/sql `Scan` and `Value` interfaces and can be used with https://github.com/lib/pq. diff --git a/github.com/jackc/pgtype/aclitem.go b/github.com/jackc/pgtype/aclitem.go new file mode 100644 index 0000000000..d2fe75297c --- /dev/null +++ b/github.com/jackc/pgtype/aclitem.go @@ -0,0 +1,139 @@ +package pgtype + +import ( + "database/sql/driver" + + errors "golang.org/x/xerrors" +) + +// ACLItem is used for PostgreSQL's aclitem data type. A sample aclitem +// might look like this: +// +// postgres=arwdDxt/postgres +// +// Note, however, that because the user/role name part of an aclitem is +// an identifier, it follows all the usual formatting rules for SQL +// identifiers: if it contains spaces and other special characters, +// it should appear in double-quotes: +// +// postgres=arwdDxt/"role with spaces" +// +type ACLItem struct { + String string + Status Status +} + +func (dst *ACLItem) Set(src interface{}) error { + if src == nil { + *dst = ACLItem{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + case string: + *dst = ACLItem{String: value, Status: Present} + case *string: + if value == nil { + *dst = ACLItem{Status: Null} + } else { + *dst = ACLItem{String: *value, Status: Present} + } + default: + if originalSrc, ok := underlyingStringType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to ACLItem", value) + } + + return nil +} + +func (dst ACLItem) Get() interface{} { + switch dst.Status { + case Present: + return dst.String + case Null: + return nil + default: + return dst.Status + } +} + +func (src *ACLItem) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + case *string: + *v = src.String + return nil + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *ACLItem) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = ACLItem{Status: Null} + return nil + } + + *dst = ACLItem{String: string(src), Status: Present} + return nil +} + +func (src ACLItem) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + return append(buf, src.String...), nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *ACLItem) Scan(src interface{}) error { + if src == nil { + *dst = ACLItem{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src ACLItem) Value() (driver.Value, error) { + switch src.Status { + case Present: + return src.String, nil + case Null: + return nil, nil + default: + return nil, errUndefined + } +} diff --git a/github.com/jackc/pgtype/aclitem_array.go b/github.com/jackc/pgtype/aclitem_array.go new file mode 100644 index 0000000000..1d3de130bb --- /dev/null +++ b/github.com/jackc/pgtype/aclitem_array.go @@ -0,0 +1,232 @@ +package pgtype + +import ( + "database/sql/driver" + + errors "golang.org/x/xerrors" +) + +type ACLItemArray struct { + Elements []ACLItem + Dimensions []ArrayDimension + Status Status +} + +func (dst *ACLItemArray) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = ACLItemArray{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + + case []string: + if value == nil { + *dst = ACLItemArray{Status: Null} + } else if len(value) == 0 { + *dst = ACLItemArray{Status: Present} + } else { + elements := make([]ACLItem, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = ACLItemArray{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []ACLItem: + if value == nil { + *dst = ACLItemArray{Status: Null} + } else if len(value) == 0 { + *dst = ACLItemArray{Status: Present} + } else { + *dst = ACLItemArray{ + Elements: value, + Dimensions: []ArrayDimension{{Length: int32(len(value)), LowerBound: 1}}, + Status: Present, + } + } + default: + if originalSrc, ok := underlyingSliceType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to ACLItemArray", value) + } + + return nil +} + +func (dst ACLItemArray) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *ACLItemArray) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + + case *[]string: + *v = make([]string, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *ACLItemArray) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = ACLItemArray{Status: Null} + return nil + } + + uta, err := ParseUntypedTextArray(string(src)) + if err != nil { + return err + } + + var elements []ACLItem + + if len(uta.Elements) > 0 { + elements = make([]ACLItem, len(uta.Elements)) + + for i, s := range uta.Elements { + var elem ACLItem + var elemSrc []byte + if s != "NULL" { + elemSrc = []byte(s) + } + err = elem.DecodeText(ci, elemSrc) + if err != nil { + return err + } + + elements[i] = elem + } + } + + *dst = ACLItemArray{Elements: elements, Dimensions: uta.Dimensions, Status: Present} + + return nil +} + +func (src ACLItemArray) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + if len(src.Dimensions) == 0 { + return append(buf, '{', '}'), nil + } + + buf = EncodeTextArrayDimensions(buf, src.Dimensions) + + // dimElemCounts is the multiples of elements that each array lies on. For + // example, a single dimension array of length 4 would have a dimElemCounts of + // [4]. A multi-dimensional array of lengths [3,5,2] would have a + // dimElemCounts of [30,10,2]. This is used to simplify when to render a '{' + // or '}'. + dimElemCounts := make([]int, len(src.Dimensions)) + dimElemCounts[len(src.Dimensions)-1] = int(src.Dimensions[len(src.Dimensions)-1].Length) + for i := len(src.Dimensions) - 2; i > -1; i-- { + dimElemCounts[i] = int(src.Dimensions[i].Length) * dimElemCounts[i+1] + } + + inElemBuf := make([]byte, 0, 32) + for i, elem := range src.Elements { + if i > 0 { + buf = append(buf, ',') + } + + for _, dec := range dimElemCounts { + if i%dec == 0 { + buf = append(buf, '{') + } + } + + elemBuf, err := elem.EncodeText(ci, inElemBuf) + if err != nil { + return nil, err + } + if elemBuf == nil { + buf = append(buf, `NULL`...) + } else { + buf = append(buf, QuoteArrayElementIfNeeded(string(elemBuf))...) + } + + for _, dec := range dimElemCounts { + if (i+1)%dec == 0 { + buf = append(buf, '}') + } + } + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *ACLItemArray) Scan(src interface{}) error { + if src == nil { + return dst.DecodeText(nil, nil) + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src ACLItemArray) Value() (driver.Value, error) { + buf, err := src.EncodeText(nil, nil) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + + return string(buf), nil +} diff --git a/github.com/jackc/pgtype/array.go b/github.com/jackc/pgtype/array.go new file mode 100644 index 0000000000..bd3a993bdf --- /dev/null +++ b/github.com/jackc/pgtype/array.go @@ -0,0 +1,352 @@ +package pgtype + +import ( + "bytes" + "encoding/binary" + "io" + "strconv" + "strings" + "unicode" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +// Information on the internals of PostgreSQL arrays can be found in +// src/include/utils/array.h and src/backend/utils/adt/arrayfuncs.c. Of +// particular interest is the array_send function. + +type ArrayHeader struct { + ContainsNull bool + ElementOID int32 + Dimensions []ArrayDimension +} + +type ArrayDimension struct { + Length int32 + LowerBound int32 +} + +func (dst *ArrayHeader) DecodeBinary(ci *ConnInfo, src []byte) (int, error) { + if len(src) < 12 { + return 0, errors.Errorf("array header too short: %d", len(src)) + } + + rp := 0 + + numDims := int(binary.BigEndian.Uint32(src[rp:])) + rp += 4 + + dst.ContainsNull = binary.BigEndian.Uint32(src[rp:]) == 1 + rp += 4 + + dst.ElementOID = int32(binary.BigEndian.Uint32(src[rp:])) + rp += 4 + + if numDims > 0 { + dst.Dimensions = make([]ArrayDimension, numDims) + } + if len(src) < 12+numDims*8 { + return 0, errors.Errorf("array header too short for %d dimensions: %d", numDims, len(src)) + } + for i := range dst.Dimensions { + dst.Dimensions[i].Length = int32(binary.BigEndian.Uint32(src[rp:])) + rp += 4 + + dst.Dimensions[i].LowerBound = int32(binary.BigEndian.Uint32(src[rp:])) + rp += 4 + } + + return rp, nil +} + +func (src ArrayHeader) EncodeBinary(ci *ConnInfo, buf []byte) []byte { + buf = pgio.AppendInt32(buf, int32(len(src.Dimensions))) + + var containsNull int32 + if src.ContainsNull { + containsNull = 1 + } + buf = pgio.AppendInt32(buf, containsNull) + + buf = pgio.AppendInt32(buf, src.ElementOID) + + for i := range src.Dimensions { + buf = pgio.AppendInt32(buf, src.Dimensions[i].Length) + buf = pgio.AppendInt32(buf, src.Dimensions[i].LowerBound) + } + + return buf +} + +type UntypedTextArray struct { + Elements []string + Dimensions []ArrayDimension +} + +func ParseUntypedTextArray(src string) (*UntypedTextArray, error) { + dst := &UntypedTextArray{} + + buf := bytes.NewBufferString(src) + + skipWhitespace(buf) + + r, _, err := buf.ReadRune() + if err != nil { + return nil, errors.Errorf("invalid array: %v", err) + } + + var explicitDimensions []ArrayDimension + + // Array has explicit dimensions + if r == '[' { + buf.UnreadRune() + + for { + r, _, err = buf.ReadRune() + if err != nil { + return nil, errors.Errorf("invalid array: %v", err) + } + + if r == '=' { + break + } else if r != '[' { + return nil, errors.Errorf("invalid array, expected '[' or '=' got %v", r) + } + + lower, err := arrayParseInteger(buf) + if err != nil { + return nil, errors.Errorf("invalid array: %v", err) + } + + r, _, err = buf.ReadRune() + if err != nil { + return nil, errors.Errorf("invalid array: %v", err) + } + + if r != ':' { + return nil, errors.Errorf("invalid array, expected ':' got %v", r) + } + + upper, err := arrayParseInteger(buf) + if err != nil { + return nil, errors.Errorf("invalid array: %v", err) + } + + r, _, err = buf.ReadRune() + if err != nil { + return nil, errors.Errorf("invalid array: %v", err) + } + + if r != ']' { + return nil, errors.Errorf("invalid array, expected ']' got %v", r) + } + + explicitDimensions = append(explicitDimensions, ArrayDimension{LowerBound: lower, Length: upper - lower + 1}) + } + + r, _, err = buf.ReadRune() + if err != nil { + return nil, errors.Errorf("invalid array: %v", err) + } + } + + if r != '{' { + return nil, errors.Errorf("invalid array, expected '{': %v", err) + } + + implicitDimensions := []ArrayDimension{{LowerBound: 1, Length: 0}} + + // Consume all initial opening brackets. This provides number of dimensions. + for { + r, _, err = buf.ReadRune() + if err != nil { + return nil, errors.Errorf("invalid array: %v", err) + } + + if r == '{' { + implicitDimensions[len(implicitDimensions)-1].Length = 1 + implicitDimensions = append(implicitDimensions, ArrayDimension{LowerBound: 1}) + } else { + buf.UnreadRune() + break + } + } + currentDim := len(implicitDimensions) - 1 + counterDim := currentDim + + for { + r, _, err = buf.ReadRune() + if err != nil { + return nil, errors.Errorf("invalid array: %v", err) + } + + switch r { + case '{': + if currentDim == counterDim { + implicitDimensions[currentDim].Length++ + } + currentDim++ + case ',': + case '}': + currentDim-- + if currentDim < counterDim { + counterDim = currentDim + } + default: + buf.UnreadRune() + value, err := arrayParseValue(buf) + if err != nil { + return nil, errors.Errorf("invalid array value: %v", err) + } + if currentDim == counterDim { + implicitDimensions[currentDim].Length++ + } + dst.Elements = append(dst.Elements, value) + } + + if currentDim < 0 { + break + } + } + + skipWhitespace(buf) + + if buf.Len() > 0 { + return nil, errors.Errorf("unexpected trailing data: %v", buf.String()) + } + + if len(dst.Elements) == 0 { + dst.Dimensions = nil + } else if len(explicitDimensions) > 0 { + dst.Dimensions = explicitDimensions + } else { + dst.Dimensions = implicitDimensions + } + + return dst, nil +} + +func skipWhitespace(buf *bytes.Buffer) { + var r rune + var err error + for r, _, _ = buf.ReadRune(); unicode.IsSpace(r); r, _, _ = buf.ReadRune() { + } + + if err != io.EOF { + buf.UnreadRune() + } +} + +func arrayParseValue(buf *bytes.Buffer) (string, error) { + r, _, err := buf.ReadRune() + if err != nil { + return "", err + } + if r == '"' { + return arrayParseQuotedValue(buf) + } + buf.UnreadRune() + + s := &bytes.Buffer{} + + for { + r, _, err := buf.ReadRune() + if err != nil { + return "", err + } + + switch r { + case ',', '}': + buf.UnreadRune() + return s.String(), nil + } + + s.WriteRune(r) + } +} + +func arrayParseQuotedValue(buf *bytes.Buffer) (string, error) { + s := &bytes.Buffer{} + + for { + r, _, err := buf.ReadRune() + if err != nil { + return "", err + } + + switch r { + case '\\': + r, _, err = buf.ReadRune() + if err != nil { + return "", err + } + case '"': + r, _, err = buf.ReadRune() + if err != nil { + return "", err + } + buf.UnreadRune() + return s.String(), nil + } + s.WriteRune(r) + } +} + +func arrayParseInteger(buf *bytes.Buffer) (int32, error) { + s := &bytes.Buffer{} + + for { + r, _, err := buf.ReadRune() + if err != nil { + return 0, err + } + + if '0' <= r && r <= '9' { + s.WriteRune(r) + } else { + buf.UnreadRune() + n, err := strconv.ParseInt(s.String(), 10, 32) + if err != nil { + return 0, err + } + return int32(n), nil + } + } +} + +func EncodeTextArrayDimensions(buf []byte, dimensions []ArrayDimension) []byte { + var customDimensions bool + for _, dim := range dimensions { + if dim.LowerBound != 1 { + customDimensions = true + } + } + + if !customDimensions { + return buf + } + + for _, dim := range dimensions { + buf = append(buf, '[') + buf = append(buf, strconv.FormatInt(int64(dim.LowerBound), 10)...) + buf = append(buf, ':') + buf = append(buf, strconv.FormatInt(int64(dim.LowerBound+dim.Length-1), 10)...) + buf = append(buf, ']') + } + + return append(buf, '=') +} + +var quoteArrayReplacer = strings.NewReplacer(`\`, `\\`, `"`, `\"`) + +func quoteArrayElement(src string) string { + return `"` + quoteArrayReplacer.Replace(src) + `"` +} + +func QuoteArrayElementIfNeeded(src string) string { + if src == "" || (len(src) == 4 && strings.ToLower(src) == "null") || src[0] == ' ' || src[len(src)-1] == ' ' || strings.ContainsAny(src, `{},"\`) { + return quoteArrayElement(src) + } + return src +} diff --git a/github.com/jackc/pgtype/bit.go b/github.com/jackc/pgtype/bit.go new file mode 100644 index 0000000000..c1709e6b9d --- /dev/null +++ b/github.com/jackc/pgtype/bit.go @@ -0,0 +1,45 @@ +package pgtype + +import ( + "database/sql/driver" +) + +type Bit Varbit + +func (dst *Bit) Set(src interface{}) error { + return (*Varbit)(dst).Set(src) +} + +func (dst Bit) Get() interface{} { + return (Varbit)(dst).Get() +} + +func (src *Bit) AssignTo(dst interface{}) error { + return (*Varbit)(src).AssignTo(dst) +} + +func (dst *Bit) DecodeBinary(ci *ConnInfo, src []byte) error { + return (*Varbit)(dst).DecodeBinary(ci, src) +} + +func (src Bit) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + return (Varbit)(src).EncodeBinary(ci, buf) +} + +func (dst *Bit) DecodeText(ci *ConnInfo, src []byte) error { + return (*Varbit)(dst).DecodeText(ci, src) +} + +func (src Bit) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + return (Varbit)(src).EncodeText(ci, buf) +} + +// Scan implements the database/sql Scanner interface. +func (dst *Bit) Scan(src interface{}) error { + return (*Varbit)(dst).Scan(src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Bit) Value() (driver.Value, error) { + return (Varbit)(src).Value() +} diff --git a/github.com/jackc/pgtype/bool.go b/github.com/jackc/pgtype/bool.go new file mode 100644 index 0000000000..8b03a1af5d --- /dev/null +++ b/github.com/jackc/pgtype/bool.go @@ -0,0 +1,206 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/json" + "strconv" + + errors "golang.org/x/xerrors" +) + +type Bool struct { + Bool bool + Status Status +} + +func (dst *Bool) Set(src interface{}) error { + if src == nil { + *dst = Bool{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + case bool: + *dst = Bool{Bool: value, Status: Present} + case string: + bb, err := strconv.ParseBool(value) + if err != nil { + return err + } + *dst = Bool{Bool: bb, Status: Present} + default: + if originalSrc, ok := underlyingBoolType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to Bool", value) + } + + return nil +} + +func (dst Bool) Get() interface{} { + switch dst.Status { + case Present: + return dst.Bool + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Bool) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + case *bool: + *v = src.Bool + return nil + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *Bool) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Bool{Status: Null} + return nil + } + + if len(src) != 1 { + return errors.Errorf("invalid length for bool: %v", len(src)) + } + + *dst = Bool{Bool: src[0] == 't', Status: Present} + return nil +} + +func (dst *Bool) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Bool{Status: Null} + return nil + } + + if len(src) != 1 { + return errors.Errorf("invalid length for bool: %v", len(src)) + } + + *dst = Bool{Bool: src[0] == 1, Status: Present} + return nil +} + +func (src Bool) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + if src.Bool { + buf = append(buf, 't') + } else { + buf = append(buf, 'f') + } + + return buf, nil +} + +func (src Bool) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + if src.Bool { + buf = append(buf, 1) + } else { + buf = append(buf, 0) + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Bool) Scan(src interface{}) error { + if src == nil { + *dst = Bool{Status: Null} + return nil + } + + switch src := src.(type) { + case bool: + *dst = Bool{Bool: src, Status: Present} + return nil + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Bool) Value() (driver.Value, error) { + switch src.Status { + case Present: + return src.Bool, nil + case Null: + return nil, nil + default: + return nil, errUndefined + } +} + +func (src Bool) MarshalJSON() ([]byte, error) { + switch src.Status { + case Present: + if src.Bool { + return []byte("true"), nil + } else { + return []byte("false"), nil + } + case Null: + return []byte("null"), nil + case Undefined: + return nil, errUndefined + } + + return nil, errBadStatus +} + +func (dst *Bool) UnmarshalJSON(b []byte) error { + var v *bool + err := json.Unmarshal(b, &v) + if err != nil { + return err + } + + if v == nil { + *dst = Bool{Status: Null} + } else { + *dst = Bool{Bool: *v, Status: Present} + } + + return nil +} diff --git a/github.com/jackc/pgtype/bool_array.go b/github.com/jackc/pgtype/bool_array.go new file mode 100644 index 0000000000..c1af1e1f95 --- /dev/null +++ b/github.com/jackc/pgtype/bool_array.go @@ -0,0 +1,320 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type BoolArray struct { + Elements []Bool + Dimensions []ArrayDimension + Status Status +} + +func (dst *BoolArray) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = BoolArray{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + + case []bool: + if value == nil { + *dst = BoolArray{Status: Null} + } else if len(value) == 0 { + *dst = BoolArray{Status: Present} + } else { + elements := make([]Bool, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = BoolArray{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []Bool: + if value == nil { + *dst = BoolArray{Status: Null} + } else if len(value) == 0 { + *dst = BoolArray{Status: Present} + } else { + *dst = BoolArray{ + Elements: value, + Dimensions: []ArrayDimension{{Length: int32(len(value)), LowerBound: 1}}, + Status: Present, + } + } + default: + if originalSrc, ok := underlyingSliceType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to BoolArray", value) + } + + return nil +} + +func (dst BoolArray) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *BoolArray) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + + case *[]bool: + *v = make([]bool, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *BoolArray) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = BoolArray{Status: Null} + return nil + } + + uta, err := ParseUntypedTextArray(string(src)) + if err != nil { + return err + } + + var elements []Bool + + if len(uta.Elements) > 0 { + elements = make([]Bool, len(uta.Elements)) + + for i, s := range uta.Elements { + var elem Bool + var elemSrc []byte + if s != "NULL" { + elemSrc = []byte(s) + } + err = elem.DecodeText(ci, elemSrc) + if err != nil { + return err + } + + elements[i] = elem + } + } + + *dst = BoolArray{Elements: elements, Dimensions: uta.Dimensions, Status: Present} + + return nil +} + +func (dst *BoolArray) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = BoolArray{Status: Null} + return nil + } + + var arrayHeader ArrayHeader + rp, err := arrayHeader.DecodeBinary(ci, src) + if err != nil { + return err + } + + if len(arrayHeader.Dimensions) == 0 { + *dst = BoolArray{Dimensions: arrayHeader.Dimensions, Status: Present} + return nil + } + + elementCount := arrayHeader.Dimensions[0].Length + for _, d := range arrayHeader.Dimensions[1:] { + elementCount *= d.Length + } + + elements := make([]Bool, elementCount) + + for i := range elements { + elemLen := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + var elemSrc []byte + if elemLen >= 0 { + elemSrc = src[rp : rp+elemLen] + rp += elemLen + } + err = elements[i].DecodeBinary(ci, elemSrc) + if err != nil { + return err + } + } + + *dst = BoolArray{Elements: elements, Dimensions: arrayHeader.Dimensions, Status: Present} + return nil +} + +func (src BoolArray) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + if len(src.Dimensions) == 0 { + return append(buf, '{', '}'), nil + } + + buf = EncodeTextArrayDimensions(buf, src.Dimensions) + + // dimElemCounts is the multiples of elements that each array lies on. For + // example, a single dimension array of length 4 would have a dimElemCounts of + // [4]. A multi-dimensional array of lengths [3,5,2] would have a + // dimElemCounts of [30,10,2]. This is used to simplify when to render a '{' + // or '}'. + dimElemCounts := make([]int, len(src.Dimensions)) + dimElemCounts[len(src.Dimensions)-1] = int(src.Dimensions[len(src.Dimensions)-1].Length) + for i := len(src.Dimensions) - 2; i > -1; i-- { + dimElemCounts[i] = int(src.Dimensions[i].Length) * dimElemCounts[i+1] + } + + inElemBuf := make([]byte, 0, 32) + for i, elem := range src.Elements { + if i > 0 { + buf = append(buf, ',') + } + + for _, dec := range dimElemCounts { + if i%dec == 0 { + buf = append(buf, '{') + } + } + + elemBuf, err := elem.EncodeText(ci, inElemBuf) + if err != nil { + return nil, err + } + if elemBuf == nil { + buf = append(buf, `NULL`...) + } else { + buf = append(buf, QuoteArrayElementIfNeeded(string(elemBuf))...) + } + + for _, dec := range dimElemCounts { + if (i+1)%dec == 0 { + buf = append(buf, '}') + } + } + } + + return buf, nil +} + +func (src BoolArray) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + arrayHeader := ArrayHeader{ + Dimensions: src.Dimensions, + } + + if dt, ok := ci.DataTypeForName("bool"); ok { + arrayHeader.ElementOID = int32(dt.OID) + } else { + return nil, errors.Errorf("unable to find oid for type name %v", "bool") + } + + for i := range src.Elements { + if src.Elements[i].Status == Null { + arrayHeader.ContainsNull = true + break + } + } + + buf = arrayHeader.EncodeBinary(ci, buf) + + for i := range src.Elements { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + elemBuf, err := src.Elements[i].EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if elemBuf != nil { + buf = elemBuf + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *BoolArray) Scan(src interface{}) error { + if src == nil { + return dst.DecodeText(nil, nil) + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src BoolArray) Value() (driver.Value, error) { + buf, err := src.EncodeText(nil, nil) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + + return string(buf), nil +} diff --git a/github.com/jackc/pgtype/box.go b/github.com/jackc/pgtype/box.go new file mode 100644 index 0000000000..75d50f9829 --- /dev/null +++ b/github.com/jackc/pgtype/box.go @@ -0,0 +1,166 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + "fmt" + "math" + "strconv" + "strings" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type Box struct { + P [2]Vec2 + Status Status +} + +func (dst *Box) Set(src interface{}) error { + return errors.Errorf("cannot convert %v to Box", src) +} + +func (dst Box) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Box) AssignTo(dst interface{}) error { + return errors.Errorf("cannot assign %v to %T", src, dst) +} + +func (dst *Box) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Box{Status: Null} + return nil + } + + if len(src) < 11 { + return errors.Errorf("invalid length for Box: %v", len(src)) + } + + str := string(src[1:]) + + var end int + end = strings.IndexByte(str, ',') + + x1, err := strconv.ParseFloat(str[:end], 64) + if err != nil { + return err + } + + str = str[end+1:] + end = strings.IndexByte(str, ')') + + y1, err := strconv.ParseFloat(str[:end], 64) + if err != nil { + return err + } + + str = str[end+3:] + end = strings.IndexByte(str, ',') + + x2, err := strconv.ParseFloat(str[:end], 64) + if err != nil { + return err + } + + str = str[end+1 : len(str)-1] + + y2, err := strconv.ParseFloat(str, 64) + if err != nil { + return err + } + + *dst = Box{P: [2]Vec2{{x1, y1}, {x2, y2}}, Status: Present} + return nil +} + +func (dst *Box) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Box{Status: Null} + return nil + } + + if len(src) != 32 { + return errors.Errorf("invalid length for Box: %v", len(src)) + } + + x1 := binary.BigEndian.Uint64(src) + y1 := binary.BigEndian.Uint64(src[8:]) + x2 := binary.BigEndian.Uint64(src[16:]) + y2 := binary.BigEndian.Uint64(src[24:]) + + *dst = Box{ + P: [2]Vec2{ + {math.Float64frombits(x1), math.Float64frombits(y1)}, + {math.Float64frombits(x2), math.Float64frombits(y2)}, + }, + Status: Present, + } + return nil +} + +func (src Box) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + buf = append(buf, fmt.Sprintf(`(%s,%s),(%s,%s)`, + strconv.FormatFloat(src.P[0].X, 'f', -1, 64), + strconv.FormatFloat(src.P[0].Y, 'f', -1, 64), + strconv.FormatFloat(src.P[1].X, 'f', -1, 64), + strconv.FormatFloat(src.P[1].Y, 'f', -1, 64), + )...) + return buf, nil +} + +func (src Box) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + buf = pgio.AppendUint64(buf, math.Float64bits(src.P[0].X)) + buf = pgio.AppendUint64(buf, math.Float64bits(src.P[0].Y)) + buf = pgio.AppendUint64(buf, math.Float64bits(src.P[1].X)) + buf = pgio.AppendUint64(buf, math.Float64bits(src.P[1].Y)) + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Box) Scan(src interface{}) error { + if src == nil { + *dst = Box{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Box) Value() (driver.Value, error) { + return EncodeValueText(src) +} diff --git a/github.com/jackc/pgtype/bpchar.go b/github.com/jackc/pgtype/bpchar.go new file mode 100644 index 0000000000..f82e37243d --- /dev/null +++ b/github.com/jackc/pgtype/bpchar.go @@ -0,0 +1,68 @@ +package pgtype + +import ( + "database/sql/driver" +) + +// BPChar is fixed-length, blank padded char type +// character(n), char(n) +type BPChar Text + +// Set converts from src to dst. +func (dst *BPChar) Set(src interface{}) error { + return (*Text)(dst).Set(src) +} + +// Get returns underlying value +func (dst BPChar) Get() interface{} { + return (Text)(dst).Get() +} + +// AssignTo assigns from src to dst. +func (src *BPChar) AssignTo(dst interface{}) error { + if src.Status == Present { + switch v := dst.(type) { + case *rune: + runes := []rune(src.String) + if len(runes) == 1 { + *v = runes[0] + return nil + } + } + } + return (*Text)(src).AssignTo(dst) +} + +func (dst *BPChar) DecodeText(ci *ConnInfo, src []byte) error { + return (*Text)(dst).DecodeText(ci, src) +} + +func (dst *BPChar) DecodeBinary(ci *ConnInfo, src []byte) error { + return (*Text)(dst).DecodeBinary(ci, src) +} + +func (src BPChar) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + return (Text)(src).EncodeText(ci, buf) +} + +func (src BPChar) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + return (Text)(src).EncodeBinary(ci, buf) +} + +// Scan implements the database/sql Scanner interface. +func (dst *BPChar) Scan(src interface{}) error { + return (*Text)(dst).Scan(src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src BPChar) Value() (driver.Value, error) { + return (Text)(src).Value() +} + +func (src BPChar) MarshalJSON() ([]byte, error) { + return (Text)(src).MarshalJSON() +} + +func (dst *BPChar) UnmarshalJSON(b []byte) error { + return (*Text)(dst).UnmarshalJSON(b) +} diff --git a/github.com/jackc/pgtype/bpchar_array.go b/github.com/jackc/pgtype/bpchar_array.go new file mode 100644 index 0000000000..b6eeabd753 --- /dev/null +++ b/github.com/jackc/pgtype/bpchar_array.go @@ -0,0 +1,320 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type BPCharArray struct { + Elements []BPChar + Dimensions []ArrayDimension + Status Status +} + +func (dst *BPCharArray) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = BPCharArray{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + + case []string: + if value == nil { + *dst = BPCharArray{Status: Null} + } else if len(value) == 0 { + *dst = BPCharArray{Status: Present} + } else { + elements := make([]BPChar, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = BPCharArray{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []BPChar: + if value == nil { + *dst = BPCharArray{Status: Null} + } else if len(value) == 0 { + *dst = BPCharArray{Status: Present} + } else { + *dst = BPCharArray{ + Elements: value, + Dimensions: []ArrayDimension{{Length: int32(len(value)), LowerBound: 1}}, + Status: Present, + } + } + default: + if originalSrc, ok := underlyingSliceType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to BPCharArray", value) + } + + return nil +} + +func (dst BPCharArray) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *BPCharArray) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + + case *[]string: + *v = make([]string, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *BPCharArray) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = BPCharArray{Status: Null} + return nil + } + + uta, err := ParseUntypedTextArray(string(src)) + if err != nil { + return err + } + + var elements []BPChar + + if len(uta.Elements) > 0 { + elements = make([]BPChar, len(uta.Elements)) + + for i, s := range uta.Elements { + var elem BPChar + var elemSrc []byte + if s != "NULL" { + elemSrc = []byte(s) + } + err = elem.DecodeText(ci, elemSrc) + if err != nil { + return err + } + + elements[i] = elem + } + } + + *dst = BPCharArray{Elements: elements, Dimensions: uta.Dimensions, Status: Present} + + return nil +} + +func (dst *BPCharArray) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = BPCharArray{Status: Null} + return nil + } + + var arrayHeader ArrayHeader + rp, err := arrayHeader.DecodeBinary(ci, src) + if err != nil { + return err + } + + if len(arrayHeader.Dimensions) == 0 { + *dst = BPCharArray{Dimensions: arrayHeader.Dimensions, Status: Present} + return nil + } + + elementCount := arrayHeader.Dimensions[0].Length + for _, d := range arrayHeader.Dimensions[1:] { + elementCount *= d.Length + } + + elements := make([]BPChar, elementCount) + + for i := range elements { + elemLen := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + var elemSrc []byte + if elemLen >= 0 { + elemSrc = src[rp : rp+elemLen] + rp += elemLen + } + err = elements[i].DecodeBinary(ci, elemSrc) + if err != nil { + return err + } + } + + *dst = BPCharArray{Elements: elements, Dimensions: arrayHeader.Dimensions, Status: Present} + return nil +} + +func (src BPCharArray) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + if len(src.Dimensions) == 0 { + return append(buf, '{', '}'), nil + } + + buf = EncodeTextArrayDimensions(buf, src.Dimensions) + + // dimElemCounts is the multiples of elements that each array lies on. For + // example, a single dimension array of length 4 would have a dimElemCounts of + // [4]. A multi-dimensional array of lengths [3,5,2] would have a + // dimElemCounts of [30,10,2]. This is used to simplify when to render a '{' + // or '}'. + dimElemCounts := make([]int, len(src.Dimensions)) + dimElemCounts[len(src.Dimensions)-1] = int(src.Dimensions[len(src.Dimensions)-1].Length) + for i := len(src.Dimensions) - 2; i > -1; i-- { + dimElemCounts[i] = int(src.Dimensions[i].Length) * dimElemCounts[i+1] + } + + inElemBuf := make([]byte, 0, 32) + for i, elem := range src.Elements { + if i > 0 { + buf = append(buf, ',') + } + + for _, dec := range dimElemCounts { + if i%dec == 0 { + buf = append(buf, '{') + } + } + + elemBuf, err := elem.EncodeText(ci, inElemBuf) + if err != nil { + return nil, err + } + if elemBuf == nil { + buf = append(buf, `NULL`...) + } else { + buf = append(buf, QuoteArrayElementIfNeeded(string(elemBuf))...) + } + + for _, dec := range dimElemCounts { + if (i+1)%dec == 0 { + buf = append(buf, '}') + } + } + } + + return buf, nil +} + +func (src BPCharArray) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + arrayHeader := ArrayHeader{ + Dimensions: src.Dimensions, + } + + if dt, ok := ci.DataTypeForName("bpchar"); ok { + arrayHeader.ElementOID = int32(dt.OID) + } else { + return nil, errors.Errorf("unable to find oid for type name %v", "bpchar") + } + + for i := range src.Elements { + if src.Elements[i].Status == Null { + arrayHeader.ContainsNull = true + break + } + } + + buf = arrayHeader.EncodeBinary(ci, buf) + + for i := range src.Elements { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + elemBuf, err := src.Elements[i].EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if elemBuf != nil { + buf = elemBuf + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *BPCharArray) Scan(src interface{}) error { + if src == nil { + return dst.DecodeText(nil, nil) + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src BPCharArray) Value() (driver.Value, error) { + buf, err := src.EncodeText(nil, nil) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + + return string(buf), nil +} diff --git a/github.com/jackc/pgtype/bytea.go b/github.com/jackc/pgtype/bytea.go new file mode 100644 index 0000000000..b9e4d15a23 --- /dev/null +++ b/github.com/jackc/pgtype/bytea.go @@ -0,0 +1,164 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/hex" + + errors "golang.org/x/xerrors" +) + +type Bytea struct { + Bytes []byte + Status Status +} + +func (dst *Bytea) Set(src interface{}) error { + if src == nil { + *dst = Bytea{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + case []byte: + if value != nil { + *dst = Bytea{Bytes: value, Status: Present} + } else { + *dst = Bytea{Status: Null} + } + default: + if originalSrc, ok := underlyingBytesType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to Bytea", value) + } + + return nil +} + +func (dst Bytea) Get() interface{} { + switch dst.Status { + case Present: + return dst.Bytes + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Bytea) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + case *[]byte: + buf := make([]byte, len(src.Bytes)) + copy(buf, src.Bytes) + *v = buf + return nil + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +// DecodeText only supports the hex format. This has been the default since +// PostgreSQL 9.0. +func (dst *Bytea) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Bytea{Status: Null} + return nil + } + + if len(src) < 2 || src[0] != '\\' || src[1] != 'x' { + return errors.Errorf("invalid hex format") + } + + buf := make([]byte, (len(src)-2)/2) + _, err := hex.Decode(buf, src[2:]) + if err != nil { + return err + } + + *dst = Bytea{Bytes: buf, Status: Present} + return nil +} + +func (dst *Bytea) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Bytea{Status: Null} + return nil + } + + *dst = Bytea{Bytes: src, Status: Present} + return nil +} + +func (src Bytea) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + buf = append(buf, `\x`...) + buf = append(buf, hex.EncodeToString(src.Bytes)...) + return buf, nil +} + +func (src Bytea) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + return append(buf, src.Bytes...), nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Bytea) Scan(src interface{}) error { + if src == nil { + *dst = Bytea{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + buf := make([]byte, len(src)) + copy(buf, src) + *dst = Bytea{Bytes: buf, Status: Present} + return nil + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Bytea) Value() (driver.Value, error) { + switch src.Status { + case Present: + return src.Bytes, nil + case Null: + return nil, nil + default: + return nil, errUndefined + } +} diff --git a/github.com/jackc/pgtype/bytea_array.go b/github.com/jackc/pgtype/bytea_array.go new file mode 100644 index 0000000000..6a45e4da90 --- /dev/null +++ b/github.com/jackc/pgtype/bytea_array.go @@ -0,0 +1,320 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type ByteaArray struct { + Elements []Bytea + Dimensions []ArrayDimension + Status Status +} + +func (dst *ByteaArray) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = ByteaArray{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + + case [][]byte: + if value == nil { + *dst = ByteaArray{Status: Null} + } else if len(value) == 0 { + *dst = ByteaArray{Status: Present} + } else { + elements := make([]Bytea, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = ByteaArray{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []Bytea: + if value == nil { + *dst = ByteaArray{Status: Null} + } else if len(value) == 0 { + *dst = ByteaArray{Status: Present} + } else { + *dst = ByteaArray{ + Elements: value, + Dimensions: []ArrayDimension{{Length: int32(len(value)), LowerBound: 1}}, + Status: Present, + } + } + default: + if originalSrc, ok := underlyingSliceType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to ByteaArray", value) + } + + return nil +} + +func (dst ByteaArray) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *ByteaArray) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + + case *[][]byte: + *v = make([][]byte, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *ByteaArray) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = ByteaArray{Status: Null} + return nil + } + + uta, err := ParseUntypedTextArray(string(src)) + if err != nil { + return err + } + + var elements []Bytea + + if len(uta.Elements) > 0 { + elements = make([]Bytea, len(uta.Elements)) + + for i, s := range uta.Elements { + var elem Bytea + var elemSrc []byte + if s != "NULL" { + elemSrc = []byte(s) + } + err = elem.DecodeText(ci, elemSrc) + if err != nil { + return err + } + + elements[i] = elem + } + } + + *dst = ByteaArray{Elements: elements, Dimensions: uta.Dimensions, Status: Present} + + return nil +} + +func (dst *ByteaArray) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = ByteaArray{Status: Null} + return nil + } + + var arrayHeader ArrayHeader + rp, err := arrayHeader.DecodeBinary(ci, src) + if err != nil { + return err + } + + if len(arrayHeader.Dimensions) == 0 { + *dst = ByteaArray{Dimensions: arrayHeader.Dimensions, Status: Present} + return nil + } + + elementCount := arrayHeader.Dimensions[0].Length + for _, d := range arrayHeader.Dimensions[1:] { + elementCount *= d.Length + } + + elements := make([]Bytea, elementCount) + + for i := range elements { + elemLen := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + var elemSrc []byte + if elemLen >= 0 { + elemSrc = src[rp : rp+elemLen] + rp += elemLen + } + err = elements[i].DecodeBinary(ci, elemSrc) + if err != nil { + return err + } + } + + *dst = ByteaArray{Elements: elements, Dimensions: arrayHeader.Dimensions, Status: Present} + return nil +} + +func (src ByteaArray) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + if len(src.Dimensions) == 0 { + return append(buf, '{', '}'), nil + } + + buf = EncodeTextArrayDimensions(buf, src.Dimensions) + + // dimElemCounts is the multiples of elements that each array lies on. For + // example, a single dimension array of length 4 would have a dimElemCounts of + // [4]. A multi-dimensional array of lengths [3,5,2] would have a + // dimElemCounts of [30,10,2]. This is used to simplify when to render a '{' + // or '}'. + dimElemCounts := make([]int, len(src.Dimensions)) + dimElemCounts[len(src.Dimensions)-1] = int(src.Dimensions[len(src.Dimensions)-1].Length) + for i := len(src.Dimensions) - 2; i > -1; i-- { + dimElemCounts[i] = int(src.Dimensions[i].Length) * dimElemCounts[i+1] + } + + inElemBuf := make([]byte, 0, 32) + for i, elem := range src.Elements { + if i > 0 { + buf = append(buf, ',') + } + + for _, dec := range dimElemCounts { + if i%dec == 0 { + buf = append(buf, '{') + } + } + + elemBuf, err := elem.EncodeText(ci, inElemBuf) + if err != nil { + return nil, err + } + if elemBuf == nil { + buf = append(buf, `NULL`...) + } else { + buf = append(buf, QuoteArrayElementIfNeeded(string(elemBuf))...) + } + + for _, dec := range dimElemCounts { + if (i+1)%dec == 0 { + buf = append(buf, '}') + } + } + } + + return buf, nil +} + +func (src ByteaArray) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + arrayHeader := ArrayHeader{ + Dimensions: src.Dimensions, + } + + if dt, ok := ci.DataTypeForName("bytea"); ok { + arrayHeader.ElementOID = int32(dt.OID) + } else { + return nil, errors.Errorf("unable to find oid for type name %v", "bytea") + } + + for i := range src.Elements { + if src.Elements[i].Status == Null { + arrayHeader.ContainsNull = true + break + } + } + + buf = arrayHeader.EncodeBinary(ci, buf) + + for i := range src.Elements { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + elemBuf, err := src.Elements[i].EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if elemBuf != nil { + buf = elemBuf + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *ByteaArray) Scan(src interface{}) error { + if src == nil { + return dst.DecodeText(nil, nil) + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src ByteaArray) Value() (driver.Value, error) { + buf, err := src.EncodeText(nil, nil) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + + return string(buf), nil +} diff --git a/github.com/jackc/pgtype/cid.go b/github.com/jackc/pgtype/cid.go new file mode 100644 index 0000000000..b944748c7d --- /dev/null +++ b/github.com/jackc/pgtype/cid.go @@ -0,0 +1,61 @@ +package pgtype + +import ( + "database/sql/driver" +) + +// CID is PostgreSQL's Command Identifier type. +// +// When one does +// +// select cmin, cmax, * from some_table; +// +// it is the data type of the cmin and cmax hidden system columns. +// +// It is currently implemented as an unsigned four byte integer. +// Its definition can be found in src/include/c.h as CommandId +// in the PostgreSQL sources. +type CID pguint32 + +// Set converts from src to dst. Note that as CID is not a general +// number type Set does not do automatic type conversion as other number +// types do. +func (dst *CID) Set(src interface{}) error { + return (*pguint32)(dst).Set(src) +} + +func (dst CID) Get() interface{} { + return (pguint32)(dst).Get() +} + +// AssignTo assigns from src to dst. Note that as CID is not a general number +// type AssignTo does not do automatic type conversion as other number types do. +func (src *CID) AssignTo(dst interface{}) error { + return (*pguint32)(src).AssignTo(dst) +} + +func (dst *CID) DecodeText(ci *ConnInfo, src []byte) error { + return (*pguint32)(dst).DecodeText(ci, src) +} + +func (dst *CID) DecodeBinary(ci *ConnInfo, src []byte) error { + return (*pguint32)(dst).DecodeBinary(ci, src) +} + +func (src CID) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + return (pguint32)(src).EncodeText(ci, buf) +} + +func (src CID) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + return (pguint32)(src).EncodeBinary(ci, buf) +} + +// Scan implements the database/sql Scanner interface. +func (dst *CID) Scan(src interface{}) error { + return (*pguint32)(dst).Scan(src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src CID) Value() (driver.Value, error) { + return (pguint32)(src).Value() +} diff --git a/github.com/jackc/pgtype/cidr.go b/github.com/jackc/pgtype/cidr.go new file mode 100644 index 0000000000..2241ca1c05 --- /dev/null +++ b/github.com/jackc/pgtype/cidr.go @@ -0,0 +1,31 @@ +package pgtype + +type CIDR Inet + +func (dst *CIDR) Set(src interface{}) error { + return (*Inet)(dst).Set(src) +} + +func (dst CIDR) Get() interface{} { + return (Inet)(dst).Get() +} + +func (src *CIDR) AssignTo(dst interface{}) error { + return (*Inet)(src).AssignTo(dst) +} + +func (dst *CIDR) DecodeText(ci *ConnInfo, src []byte) error { + return (*Inet)(dst).DecodeText(ci, src) +} + +func (dst *CIDR) DecodeBinary(ci *ConnInfo, src []byte) error { + return (*Inet)(dst).DecodeBinary(ci, src) +} + +func (src CIDR) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + return (Inet)(src).EncodeText(ci, buf) +} + +func (src CIDR) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + return (Inet)(src).EncodeBinary(ci, buf) +} diff --git a/github.com/jackc/pgtype/cidr_array.go b/github.com/jackc/pgtype/cidr_array.go new file mode 100644 index 0000000000..4f3097a0cf --- /dev/null +++ b/github.com/jackc/pgtype/cidr_array.go @@ -0,0 +1,349 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + "net" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type CIDRArray struct { + Elements []CIDR + Dimensions []ArrayDimension + Status Status +} + +func (dst *CIDRArray) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = CIDRArray{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + + case []*net.IPNet: + if value == nil { + *dst = CIDRArray{Status: Null} + } else if len(value) == 0 { + *dst = CIDRArray{Status: Present} + } else { + elements := make([]CIDR, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = CIDRArray{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []net.IP: + if value == nil { + *dst = CIDRArray{Status: Null} + } else if len(value) == 0 { + *dst = CIDRArray{Status: Present} + } else { + elements := make([]CIDR, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = CIDRArray{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []CIDR: + if value == nil { + *dst = CIDRArray{Status: Null} + } else if len(value) == 0 { + *dst = CIDRArray{Status: Present} + } else { + *dst = CIDRArray{ + Elements: value, + Dimensions: []ArrayDimension{{Length: int32(len(value)), LowerBound: 1}}, + Status: Present, + } + } + default: + if originalSrc, ok := underlyingSliceType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to CIDRArray", value) + } + + return nil +} + +func (dst CIDRArray) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *CIDRArray) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + + case *[]*net.IPNet: + *v = make([]*net.IPNet, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[]net.IP: + *v = make([]net.IP, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *CIDRArray) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = CIDRArray{Status: Null} + return nil + } + + uta, err := ParseUntypedTextArray(string(src)) + if err != nil { + return err + } + + var elements []CIDR + + if len(uta.Elements) > 0 { + elements = make([]CIDR, len(uta.Elements)) + + for i, s := range uta.Elements { + var elem CIDR + var elemSrc []byte + if s != "NULL" { + elemSrc = []byte(s) + } + err = elem.DecodeText(ci, elemSrc) + if err != nil { + return err + } + + elements[i] = elem + } + } + + *dst = CIDRArray{Elements: elements, Dimensions: uta.Dimensions, Status: Present} + + return nil +} + +func (dst *CIDRArray) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = CIDRArray{Status: Null} + return nil + } + + var arrayHeader ArrayHeader + rp, err := arrayHeader.DecodeBinary(ci, src) + if err != nil { + return err + } + + if len(arrayHeader.Dimensions) == 0 { + *dst = CIDRArray{Dimensions: arrayHeader.Dimensions, Status: Present} + return nil + } + + elementCount := arrayHeader.Dimensions[0].Length + for _, d := range arrayHeader.Dimensions[1:] { + elementCount *= d.Length + } + + elements := make([]CIDR, elementCount) + + for i := range elements { + elemLen := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + var elemSrc []byte + if elemLen >= 0 { + elemSrc = src[rp : rp+elemLen] + rp += elemLen + } + err = elements[i].DecodeBinary(ci, elemSrc) + if err != nil { + return err + } + } + + *dst = CIDRArray{Elements: elements, Dimensions: arrayHeader.Dimensions, Status: Present} + return nil +} + +func (src CIDRArray) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + if len(src.Dimensions) == 0 { + return append(buf, '{', '}'), nil + } + + buf = EncodeTextArrayDimensions(buf, src.Dimensions) + + // dimElemCounts is the multiples of elements that each array lies on. For + // example, a single dimension array of length 4 would have a dimElemCounts of + // [4]. A multi-dimensional array of lengths [3,5,2] would have a + // dimElemCounts of [30,10,2]. This is used to simplify when to render a '{' + // or '}'. + dimElemCounts := make([]int, len(src.Dimensions)) + dimElemCounts[len(src.Dimensions)-1] = int(src.Dimensions[len(src.Dimensions)-1].Length) + for i := len(src.Dimensions) - 2; i > -1; i-- { + dimElemCounts[i] = int(src.Dimensions[i].Length) * dimElemCounts[i+1] + } + + inElemBuf := make([]byte, 0, 32) + for i, elem := range src.Elements { + if i > 0 { + buf = append(buf, ',') + } + + for _, dec := range dimElemCounts { + if i%dec == 0 { + buf = append(buf, '{') + } + } + + elemBuf, err := elem.EncodeText(ci, inElemBuf) + if err != nil { + return nil, err + } + if elemBuf == nil { + buf = append(buf, `NULL`...) + } else { + buf = append(buf, QuoteArrayElementIfNeeded(string(elemBuf))...) + } + + for _, dec := range dimElemCounts { + if (i+1)%dec == 0 { + buf = append(buf, '}') + } + } + } + + return buf, nil +} + +func (src CIDRArray) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + arrayHeader := ArrayHeader{ + Dimensions: src.Dimensions, + } + + if dt, ok := ci.DataTypeForName("cidr"); ok { + arrayHeader.ElementOID = int32(dt.OID) + } else { + return nil, errors.Errorf("unable to find oid for type name %v", "cidr") + } + + for i := range src.Elements { + if src.Elements[i].Status == Null { + arrayHeader.ContainsNull = true + break + } + } + + buf = arrayHeader.EncodeBinary(ci, buf) + + for i := range src.Elements { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + elemBuf, err := src.Elements[i].EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if elemBuf != nil { + buf = elemBuf + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *CIDRArray) Scan(src interface{}) error { + if src == nil { + return dst.DecodeText(nil, nil) + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src CIDRArray) Value() (driver.Value, error) { + buf, err := src.EncodeText(nil, nil) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + + return string(buf), nil +} diff --git a/github.com/jackc/pgtype/circle.go b/github.com/jackc/pgtype/circle.go new file mode 100644 index 0000000000..d3f8b38ae1 --- /dev/null +++ b/github.com/jackc/pgtype/circle.go @@ -0,0 +1,151 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + "fmt" + "math" + "strconv" + "strings" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type Circle struct { + P Vec2 + R float64 + Status Status +} + +func (dst *Circle) Set(src interface{}) error { + return errors.Errorf("cannot convert %v to Circle", src) +} + +func (dst Circle) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Circle) AssignTo(dst interface{}) error { + return errors.Errorf("cannot assign %v to %T", src, dst) +} + +func (dst *Circle) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Circle{Status: Null} + return nil + } + + if len(src) < 9 { + return errors.Errorf("invalid length for Circle: %v", len(src)) + } + + str := string(src[2:]) + end := strings.IndexByte(str, ',') + x, err := strconv.ParseFloat(str[:end], 64) + if err != nil { + return err + } + + str = str[end+1:] + end = strings.IndexByte(str, ')') + + y, err := strconv.ParseFloat(str[:end], 64) + if err != nil { + return err + } + + str = str[end+2 : len(str)-1] + + r, err := strconv.ParseFloat(str, 64) + if err != nil { + return err + } + + *dst = Circle{P: Vec2{x, y}, R: r, Status: Present} + return nil +} + +func (dst *Circle) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Circle{Status: Null} + return nil + } + + if len(src) != 24 { + return errors.Errorf("invalid length for Circle: %v", len(src)) + } + + x := binary.BigEndian.Uint64(src) + y := binary.BigEndian.Uint64(src[8:]) + r := binary.BigEndian.Uint64(src[16:]) + + *dst = Circle{ + P: Vec2{math.Float64frombits(x), math.Float64frombits(y)}, + R: math.Float64frombits(r), + Status: Present, + } + return nil +} + +func (src Circle) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + buf = append(buf, fmt.Sprintf(`<(%s,%s),%s>`, + strconv.FormatFloat(src.P.X, 'f', -1, 64), + strconv.FormatFloat(src.P.Y, 'f', -1, 64), + strconv.FormatFloat(src.R, 'f', -1, 64), + )...) + + return buf, nil +} + +func (src Circle) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + buf = pgio.AppendUint64(buf, math.Float64bits(src.P.X)) + buf = pgio.AppendUint64(buf, math.Float64bits(src.P.Y)) + buf = pgio.AppendUint64(buf, math.Float64bits(src.R)) + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Circle) Scan(src interface{}) error { + if src == nil { + *dst = Circle{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Circle) Value() (driver.Value, error) { + return EncodeValueText(src) +} diff --git a/github.com/jackc/pgtype/convert.go b/github.com/jackc/pgtype/convert.go new file mode 100644 index 0000000000..cc5c10abfe --- /dev/null +++ b/github.com/jackc/pgtype/convert.go @@ -0,0 +1,453 @@ +package pgtype + +import ( + "math" + "reflect" + "time" + + errors "golang.org/x/xerrors" +) + +const maxUint = ^uint(0) +const maxInt = int(maxUint >> 1) +const minInt = -maxInt - 1 + +// underlyingNumberType gets the underlying type that can be converted to Int2, Int4, Int8, Float4, or Float8 +func underlyingNumberType(val interface{}) (interface{}, bool) { + refVal := reflect.ValueOf(val) + + switch refVal.Kind() { + case reflect.Ptr: + if refVal.IsNil() { + return nil, false + } + convVal := refVal.Elem().Interface() + return convVal, true + case reflect.Int: + convVal := int(refVal.Int()) + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Int8: + convVal := int8(refVal.Int()) + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Int16: + convVal := int16(refVal.Int()) + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Int32: + convVal := int32(refVal.Int()) + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Int64: + convVal := int64(refVal.Int()) + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Uint: + convVal := uint(refVal.Uint()) + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Uint8: + convVal := uint8(refVal.Uint()) + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Uint16: + convVal := uint16(refVal.Uint()) + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Uint32: + convVal := uint32(refVal.Uint()) + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Uint64: + convVal := uint64(refVal.Uint()) + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Float32: + convVal := float32(refVal.Float()) + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.Float64: + convVal := refVal.Float() + return convVal, reflect.TypeOf(convVal) != refVal.Type() + case reflect.String: + convVal := refVal.String() + return convVal, reflect.TypeOf(convVal) != refVal.Type() + } + + return nil, false +} + +// underlyingBoolType gets the underlying type that can be converted to Bool +func underlyingBoolType(val interface{}) (interface{}, bool) { + refVal := reflect.ValueOf(val) + + switch refVal.Kind() { + case reflect.Ptr: + if refVal.IsNil() { + return nil, false + } + convVal := refVal.Elem().Interface() + return convVal, true + case reflect.Bool: + convVal := refVal.Bool() + return convVal, reflect.TypeOf(convVal) != refVal.Type() + } + + return nil, false +} + +// underlyingBytesType gets the underlying type that can be converted to []byte +func underlyingBytesType(val interface{}) (interface{}, bool) { + refVal := reflect.ValueOf(val) + + switch refVal.Kind() { + case reflect.Ptr: + if refVal.IsNil() { + return nil, false + } + convVal := refVal.Elem().Interface() + return convVal, true + case reflect.Slice: + if refVal.Type().Elem().Kind() == reflect.Uint8 { + convVal := refVal.Bytes() + return convVal, reflect.TypeOf(convVal) != refVal.Type() + } + } + + return nil, false +} + +// underlyingStringType gets the underlying type that can be converted to String +func underlyingStringType(val interface{}) (interface{}, bool) { + refVal := reflect.ValueOf(val) + + switch refVal.Kind() { + case reflect.Ptr: + if refVal.IsNil() { + return nil, false + } + convVal := refVal.Elem().Interface() + return convVal, true + case reflect.String: + convVal := refVal.String() + return convVal, reflect.TypeOf(convVal) != refVal.Type() + } + + return nil, false +} + +// underlyingPtrType dereferences a pointer +func underlyingPtrType(val interface{}) (interface{}, bool) { + refVal := reflect.ValueOf(val) + + switch refVal.Kind() { + case reflect.Ptr: + if refVal.IsNil() { + return nil, false + } + convVal := refVal.Elem().Interface() + return convVal, true + } + + return nil, false +} + +// underlyingTimeType gets the underlying type that can be converted to time.Time +func underlyingTimeType(val interface{}) (interface{}, bool) { + refVal := reflect.ValueOf(val) + + switch refVal.Kind() { + case reflect.Ptr: + if refVal.IsNil() { + return nil, false + } + convVal := refVal.Elem().Interface() + return convVal, true + } + + timeType := reflect.TypeOf(time.Time{}) + if refVal.Type().ConvertibleTo(timeType) { + return refVal.Convert(timeType).Interface(), true + } + + return nil, false +} + +// underlyingUUIDType gets the underlying type that can be converted to [16]byte +func underlyingUUIDType(val interface{}) (interface{}, bool) { + refVal := reflect.ValueOf(val) + + switch refVal.Kind() { + case reflect.Ptr: + if refVal.IsNil() { + return time.Time{}, false + } + convVal := refVal.Elem().Interface() + return convVal, true + } + + uuidType := reflect.TypeOf([16]byte{}) + if refVal.Type().ConvertibleTo(uuidType) { + return refVal.Convert(uuidType).Interface(), true + } + + return nil, false +} + +// underlyingSliceType gets the underlying slice type +func underlyingSliceType(val interface{}) (interface{}, bool) { + refVal := reflect.ValueOf(val) + + switch refVal.Kind() { + case reflect.Ptr: + if refVal.IsNil() { + return nil, false + } + convVal := refVal.Elem().Interface() + return convVal, true + case reflect.Slice: + baseSliceType := reflect.SliceOf(refVal.Type().Elem()) + if refVal.Type().ConvertibleTo(baseSliceType) { + convVal := refVal.Convert(baseSliceType) + return convVal.Interface(), reflect.TypeOf(convVal.Interface()) != refVal.Type() + } + } + + return nil, false +} + +func int64AssignTo(srcVal int64, srcStatus Status, dst interface{}) error { + if srcStatus == Present { + switch v := dst.(type) { + case *int: + if srcVal < int64(minInt) { + return errors.Errorf("%d is less than minimum value for int", srcVal) + } else if srcVal > int64(maxInt) { + return errors.Errorf("%d is greater than maximum value for int", srcVal) + } + *v = int(srcVal) + case *int8: + if srcVal < math.MinInt8 { + return errors.Errorf("%d is less than minimum value for int8", srcVal) + } else if srcVal > math.MaxInt8 { + return errors.Errorf("%d is greater than maximum value for int8", srcVal) + } + *v = int8(srcVal) + case *int16: + if srcVal < math.MinInt16 { + return errors.Errorf("%d is less than minimum value for int16", srcVal) + } else if srcVal > math.MaxInt16 { + return errors.Errorf("%d is greater than maximum value for int16", srcVal) + } + *v = int16(srcVal) + case *int32: + if srcVal < math.MinInt32 { + return errors.Errorf("%d is less than minimum value for int32", srcVal) + } else if srcVal > math.MaxInt32 { + return errors.Errorf("%d is greater than maximum value for int32", srcVal) + } + *v = int32(srcVal) + case *int64: + if srcVal < math.MinInt64 { + return errors.Errorf("%d is less than minimum value for int64", srcVal) + } else if srcVal > math.MaxInt64 { + return errors.Errorf("%d is greater than maximum value for int64", srcVal) + } + *v = int64(srcVal) + case *uint: + if srcVal < 0 { + return errors.Errorf("%d is less than zero for uint", srcVal) + } else if uint64(srcVal) > uint64(maxUint) { + return errors.Errorf("%d is greater than maximum value for uint", srcVal) + } + *v = uint(srcVal) + case *uint8: + if srcVal < 0 { + return errors.Errorf("%d is less than zero for uint8", srcVal) + } else if srcVal > math.MaxUint8 { + return errors.Errorf("%d is greater than maximum value for uint8", srcVal) + } + *v = uint8(srcVal) + case *uint16: + if srcVal < 0 { + return errors.Errorf("%d is less than zero for uint32", srcVal) + } else if srcVal > math.MaxUint16 { + return errors.Errorf("%d is greater than maximum value for uint16", srcVal) + } + *v = uint16(srcVal) + case *uint32: + if srcVal < 0 { + return errors.Errorf("%d is less than zero for uint32", srcVal) + } else if srcVal > math.MaxUint32 { + return errors.Errorf("%d is greater than maximum value for uint32", srcVal) + } + *v = uint32(srcVal) + case *uint64: + if srcVal < 0 { + return errors.Errorf("%d is less than zero for uint64", srcVal) + } + *v = uint64(srcVal) + default: + if v := reflect.ValueOf(dst); v.Kind() == reflect.Ptr { + el := v.Elem() + switch el.Kind() { + // if dst is a pointer to pointer, strip the pointer and try again + case reflect.Ptr: + if el.IsNil() { + // allocate destination + el.Set(reflect.New(el.Type().Elem())) + } + return int64AssignTo(srcVal, srcStatus, el.Interface()) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + if el.OverflowInt(int64(srcVal)) { + return errors.Errorf("cannot put %d into %T", srcVal, dst) + } + el.SetInt(int64(srcVal)) + return nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + if srcVal < 0 { + return errors.Errorf("%d is less than zero for %T", srcVal, dst) + } + if el.OverflowUint(uint64(srcVal)) { + return errors.Errorf("cannot put %d into %T", srcVal, dst) + } + el.SetUint(uint64(srcVal)) + return nil + } + } + return errors.Errorf("cannot assign %v into %T", srcVal, dst) + } + return nil + } + + // if dst is a pointer to pointer and srcStatus is not Present, nil it out + if v := reflect.ValueOf(dst); v.Kind() == reflect.Ptr { + el := v.Elem() + if el.Kind() == reflect.Ptr { + el.Set(reflect.Zero(el.Type())) + return nil + } + } + + return errors.Errorf("cannot assign %v %v into %T", srcVal, srcStatus, dst) +} + +func float64AssignTo(srcVal float64, srcStatus Status, dst interface{}) error { + if srcStatus == Present { + switch v := dst.(type) { + case *float32: + *v = float32(srcVal) + case *float64: + *v = srcVal + default: + if v := reflect.ValueOf(dst); v.Kind() == reflect.Ptr { + el := v.Elem() + switch el.Kind() { + // if dst is a pointer to pointer, strip the pointer and try again + case reflect.Ptr: + if el.IsNil() { + // allocate destination + el.Set(reflect.New(el.Type().Elem())) + } + return float64AssignTo(srcVal, srcStatus, el.Interface()) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + i64 := int64(srcVal) + if float64(i64) == srcVal { + return int64AssignTo(i64, srcStatus, dst) + } + } + } + return errors.Errorf("cannot assign %v into %T", srcVal, dst) + } + return nil + } + + // if dst is a pointer to pointer and srcStatus is not Present, nil it out + if v := reflect.ValueOf(dst); v.Kind() == reflect.Ptr { + el := v.Elem() + if el.Kind() == reflect.Ptr { + el.Set(reflect.Zero(el.Type())) + return nil + } + } + + return errors.Errorf("cannot assign %v %v into %T", srcVal, srcStatus, dst) +} + +func NullAssignTo(dst interface{}) error { + dstPtr := reflect.ValueOf(dst) + + // AssignTo dst must always be a pointer + if dstPtr.Kind() != reflect.Ptr { + return errors.Errorf("cannot assign NULL to %T", dst) + } + + dstVal := dstPtr.Elem() + + switch dstVal.Kind() { + case reflect.Ptr, reflect.Slice, reflect.Map: + dstVal.Set(reflect.Zero(dstVal.Type())) + return nil + } + + return errors.Errorf("cannot assign NULL to %T", dst) +} + +var kindTypes map[reflect.Kind]reflect.Type + +// GetAssignToDstType attempts to convert dst to something AssignTo can assign +// to. If dst is a pointer to pointer it allocates a value and returns the +// dereferences pointer. If dst is a named type such as *Foo where Foo is type +// Foo int16, it converts dst to *int16. +// +// GetAssignToDstType returns the converted dst and a bool representing if any +// change was made. +func GetAssignToDstType(dst interface{}) (interface{}, bool) { + dstPtr := reflect.ValueOf(dst) + + // AssignTo dst must always be a pointer + if dstPtr.Kind() != reflect.Ptr { + return nil, false + } + + dstVal := dstPtr.Elem() + + // if dst is a pointer to pointer, allocate space try again with the dereferenced pointer + if dstVal.Kind() == reflect.Ptr { + dstVal.Set(reflect.New(dstVal.Type().Elem())) + return dstVal.Interface(), true + } + + // if dst is pointer to a base type that has been renamed + if baseValType, ok := kindTypes[dstVal.Kind()]; ok { + nextDst := dstPtr.Convert(reflect.PtrTo(baseValType)) + return nextDst.Interface(), dstPtr.Type() != nextDst.Type() + } + + if dstVal.Kind() == reflect.Slice { + if baseElemType, ok := kindTypes[dstVal.Type().Elem().Kind()]; ok { + baseSliceType := reflect.PtrTo(reflect.SliceOf(baseElemType)) + nextDst := dstPtr.Convert(baseSliceType) + return nextDst.Interface(), dstPtr.Type() != nextDst.Type() + } + } + + if dstVal.Kind() == reflect.Array { + if baseElemType, ok := kindTypes[dstVal.Type().Elem().Kind()]; ok { + baseArrayType := reflect.PtrTo(reflect.ArrayOf(dstVal.Len(), baseElemType)) + nextDst := dstPtr.Convert(baseArrayType) + return nextDst.Interface(), dstPtr.Type() != nextDst.Type() + } + } + + return nil, false +} + +func init() { + kindTypes = map[reflect.Kind]reflect.Type{ + reflect.Bool: reflect.TypeOf(false), + reflect.Float32: reflect.TypeOf(float32(0)), + reflect.Float64: reflect.TypeOf(float64(0)), + reflect.Int: reflect.TypeOf(int(0)), + reflect.Int8: reflect.TypeOf(int8(0)), + reflect.Int16: reflect.TypeOf(int16(0)), + reflect.Int32: reflect.TypeOf(int32(0)), + reflect.Int64: reflect.TypeOf(int64(0)), + reflect.Uint: reflect.TypeOf(uint(0)), + reflect.Uint8: reflect.TypeOf(uint8(0)), + reflect.Uint16: reflect.TypeOf(uint16(0)), + reflect.Uint32: reflect.TypeOf(uint32(0)), + reflect.Uint64: reflect.TypeOf(uint64(0)), + reflect.String: reflect.TypeOf(""), + } +} diff --git a/github.com/jackc/pgtype/database_sql.go b/github.com/jackc/pgtype/database_sql.go new file mode 100644 index 0000000000..f54a750de8 --- /dev/null +++ b/github.com/jackc/pgtype/database_sql.go @@ -0,0 +1,42 @@ +package pgtype + +import ( + "database/sql/driver" + + errors "golang.org/x/xerrors" +) + +func DatabaseSQLValue(ci *ConnInfo, src Value) (interface{}, error) { + if valuer, ok := src.(driver.Valuer); ok { + return valuer.Value() + } + + if textEncoder, ok := src.(TextEncoder); ok { + buf, err := textEncoder.EncodeText(ci, nil) + if err != nil { + return nil, err + } + return string(buf), nil + } + + if binaryEncoder, ok := src.(BinaryEncoder); ok { + buf, err := binaryEncoder.EncodeBinary(ci, nil) + if err != nil { + return nil, err + } + return buf, nil + } + + return nil, errors.New("cannot convert to database/sql compatible value") +} + +func EncodeValueText(src TextEncoder) (interface{}, error) { + buf, err := src.EncodeText(nil, make([]byte, 0, 32)) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + return string(buf), err +} diff --git a/github.com/jackc/pgtype/date.go b/github.com/jackc/pgtype/date.go new file mode 100644 index 0000000000..37fb83021c --- /dev/null +++ b/github.com/jackc/pgtype/date.go @@ -0,0 +1,275 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + "encoding/json" + "time" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type Date struct { + Time time.Time + Status Status + InfinityModifier InfinityModifier +} + +const ( + negativeInfinityDayOffset = -2147483648 + infinityDayOffset = 2147483647 +) + +func (dst *Date) Set(src interface{}) error { + if src == nil { + *dst = Date{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + case time.Time: + *dst = Date{Time: value, Status: Present} + case string: + return dst.DecodeText(nil, []byte(value)) + default: + if originalSrc, ok := underlyingTimeType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to Date", value) + } + + return nil +} + +func (dst Date) Get() interface{} { + switch dst.Status { + case Present: + if dst.InfinityModifier != None { + return dst.InfinityModifier + } + return dst.Time + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Date) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + case *time.Time: + if src.InfinityModifier != None { + return errors.Errorf("cannot assign %v to %T", src, dst) + } + *v = src.Time + return nil + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *Date) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Date{Status: Null} + return nil + } + + sbuf := string(src) + switch sbuf { + case "infinity": + *dst = Date{Status: Present, InfinityModifier: Infinity} + case "-infinity": + *dst = Date{Status: Present, InfinityModifier: -Infinity} + default: + t, err := time.ParseInLocation("2006-01-02", sbuf, time.UTC) + if err != nil { + return err + } + + *dst = Date{Time: t, Status: Present} + } + + return nil +} + +func (dst *Date) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Date{Status: Null} + return nil + } + + if len(src) != 4 { + return errors.Errorf("invalid length for date: %v", len(src)) + } + + dayOffset := int32(binary.BigEndian.Uint32(src)) + + switch dayOffset { + case infinityDayOffset: + *dst = Date{Status: Present, InfinityModifier: Infinity} + case negativeInfinityDayOffset: + *dst = Date{Status: Present, InfinityModifier: -Infinity} + default: + t := time.Date(2000, 1, int(1+dayOffset), 0, 0, 0, 0, time.UTC) + *dst = Date{Time: t, Status: Present} + } + + return nil +} + +func (src Date) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + var s string + + switch src.InfinityModifier { + case None: + s = src.Time.Format("2006-01-02") + case Infinity: + s = "infinity" + case NegativeInfinity: + s = "-infinity" + } + + return append(buf, s...), nil +} + +func (src Date) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + var daysSinceDateEpoch int32 + switch src.InfinityModifier { + case None: + tUnix := time.Date(src.Time.Year(), src.Time.Month(), src.Time.Day(), 0, 0, 0, 0, time.UTC).Unix() + dateEpoch := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC).Unix() + + secSinceDateEpoch := tUnix - dateEpoch + daysSinceDateEpoch = int32(secSinceDateEpoch / 86400) + case Infinity: + daysSinceDateEpoch = infinityDayOffset + case NegativeInfinity: + daysSinceDateEpoch = negativeInfinityDayOffset + } + + return pgio.AppendInt32(buf, daysSinceDateEpoch), nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Date) Scan(src interface{}) error { + if src == nil { + *dst = Date{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + case time.Time: + *dst = Date{Time: src, Status: Present} + return nil + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Date) Value() (driver.Value, error) { + switch src.Status { + case Present: + if src.InfinityModifier != None { + return src.InfinityModifier.String(), nil + } + return src.Time, nil + case Null: + return nil, nil + default: + return nil, errUndefined + } +} + +func (src Date) MarshalJSON() ([]byte, error) { + switch src.Status { + case Null: + return []byte("null"), nil + case Undefined: + return nil, errUndefined + } + + if src.Status != Present { + return nil, errBadStatus + } + + var s string + + switch src.InfinityModifier { + case None: + s = src.Time.Format("2006-01-02") + case Infinity: + s = "infinity" + case NegativeInfinity: + s = "-infinity" + } + + return json.Marshal(s) +} + +func (dst *Date) UnmarshalJSON(b []byte) error { + var s *string + err := json.Unmarshal(b, &s) + if err != nil { + return err + } + + if s == nil { + *dst = Date{Status: Null} + return nil + } + + switch *s { + case "infinity": + *dst = Date{Status: Present, InfinityModifier: Infinity} + case "-infinity": + *dst = Date{Status: Present, InfinityModifier: -Infinity} + default: + t, err := time.ParseInLocation("2006-01-02", *s, time.UTC) + if err != nil { + return err + } + + *dst = Date{Time: t, Status: Present} + } + + return nil +} diff --git a/github.com/jackc/pgtype/date_array.go b/github.com/jackc/pgtype/date_array.go new file mode 100644 index 0000000000..644e78fec3 --- /dev/null +++ b/github.com/jackc/pgtype/date_array.go @@ -0,0 +1,321 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + "time" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type DateArray struct { + Elements []Date + Dimensions []ArrayDimension + Status Status +} + +func (dst *DateArray) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = DateArray{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + + case []time.Time: + if value == nil { + *dst = DateArray{Status: Null} + } else if len(value) == 0 { + *dst = DateArray{Status: Present} + } else { + elements := make([]Date, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = DateArray{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []Date: + if value == nil { + *dst = DateArray{Status: Null} + } else if len(value) == 0 { + *dst = DateArray{Status: Present} + } else { + *dst = DateArray{ + Elements: value, + Dimensions: []ArrayDimension{{Length: int32(len(value)), LowerBound: 1}}, + Status: Present, + } + } + default: + if originalSrc, ok := underlyingSliceType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to DateArray", value) + } + + return nil +} + +func (dst DateArray) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *DateArray) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + + case *[]time.Time: + *v = make([]time.Time, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *DateArray) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = DateArray{Status: Null} + return nil + } + + uta, err := ParseUntypedTextArray(string(src)) + if err != nil { + return err + } + + var elements []Date + + if len(uta.Elements) > 0 { + elements = make([]Date, len(uta.Elements)) + + for i, s := range uta.Elements { + var elem Date + var elemSrc []byte + if s != "NULL" { + elemSrc = []byte(s) + } + err = elem.DecodeText(ci, elemSrc) + if err != nil { + return err + } + + elements[i] = elem + } + } + + *dst = DateArray{Elements: elements, Dimensions: uta.Dimensions, Status: Present} + + return nil +} + +func (dst *DateArray) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = DateArray{Status: Null} + return nil + } + + var arrayHeader ArrayHeader + rp, err := arrayHeader.DecodeBinary(ci, src) + if err != nil { + return err + } + + if len(arrayHeader.Dimensions) == 0 { + *dst = DateArray{Dimensions: arrayHeader.Dimensions, Status: Present} + return nil + } + + elementCount := arrayHeader.Dimensions[0].Length + for _, d := range arrayHeader.Dimensions[1:] { + elementCount *= d.Length + } + + elements := make([]Date, elementCount) + + for i := range elements { + elemLen := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + var elemSrc []byte + if elemLen >= 0 { + elemSrc = src[rp : rp+elemLen] + rp += elemLen + } + err = elements[i].DecodeBinary(ci, elemSrc) + if err != nil { + return err + } + } + + *dst = DateArray{Elements: elements, Dimensions: arrayHeader.Dimensions, Status: Present} + return nil +} + +func (src DateArray) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + if len(src.Dimensions) == 0 { + return append(buf, '{', '}'), nil + } + + buf = EncodeTextArrayDimensions(buf, src.Dimensions) + + // dimElemCounts is the multiples of elements that each array lies on. For + // example, a single dimension array of length 4 would have a dimElemCounts of + // [4]. A multi-dimensional array of lengths [3,5,2] would have a + // dimElemCounts of [30,10,2]. This is used to simplify when to render a '{' + // or '}'. + dimElemCounts := make([]int, len(src.Dimensions)) + dimElemCounts[len(src.Dimensions)-1] = int(src.Dimensions[len(src.Dimensions)-1].Length) + for i := len(src.Dimensions) - 2; i > -1; i-- { + dimElemCounts[i] = int(src.Dimensions[i].Length) * dimElemCounts[i+1] + } + + inElemBuf := make([]byte, 0, 32) + for i, elem := range src.Elements { + if i > 0 { + buf = append(buf, ',') + } + + for _, dec := range dimElemCounts { + if i%dec == 0 { + buf = append(buf, '{') + } + } + + elemBuf, err := elem.EncodeText(ci, inElemBuf) + if err != nil { + return nil, err + } + if elemBuf == nil { + buf = append(buf, `NULL`...) + } else { + buf = append(buf, QuoteArrayElementIfNeeded(string(elemBuf))...) + } + + for _, dec := range dimElemCounts { + if (i+1)%dec == 0 { + buf = append(buf, '}') + } + } + } + + return buf, nil +} + +func (src DateArray) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + arrayHeader := ArrayHeader{ + Dimensions: src.Dimensions, + } + + if dt, ok := ci.DataTypeForName("date"); ok { + arrayHeader.ElementOID = int32(dt.OID) + } else { + return nil, errors.Errorf("unable to find oid for type name %v", "date") + } + + for i := range src.Elements { + if src.Elements[i].Status == Null { + arrayHeader.ContainsNull = true + break + } + } + + buf = arrayHeader.EncodeBinary(ci, buf) + + for i := range src.Elements { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + elemBuf, err := src.Elements[i].EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if elemBuf != nil { + buf = elemBuf + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *DateArray) Scan(src interface{}) error { + if src == nil { + return dst.DecodeText(nil, nil) + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src DateArray) Value() (driver.Value, error) { + buf, err := src.EncodeText(nil, nil) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + + return string(buf), nil +} diff --git a/github.com/jackc/pgtype/daterange.go b/github.com/jackc/pgtype/daterange.go new file mode 100644 index 0000000000..7b9af79571 --- /dev/null +++ b/github.com/jackc/pgtype/daterange.go @@ -0,0 +1,267 @@ +package pgtype + +import ( + "database/sql/driver" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type Daterange struct { + Lower Date + Upper Date + LowerType BoundType + UpperType BoundType + Status Status +} + +func (dst *Daterange) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = Daterange{Status: Null} + return nil + } + + switch value := src.(type) { + case Daterange: + *dst = value + case *Daterange: + *dst = *value + case string: + return dst.DecodeText(nil, []byte(value)) + default: + return errors.Errorf("cannot convert %v to Daterange", src) + } + + return nil +} + +func (dst Daterange) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Daterange) AssignTo(dst interface{}) error { + return errors.Errorf("cannot assign %v to %T", src, dst) +} + +func (dst *Daterange) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Daterange{Status: Null} + return nil + } + + utr, err := ParseUntypedTextRange(string(src)) + if err != nil { + return err + } + + *dst = Daterange{Status: Present} + + dst.LowerType = utr.LowerType + dst.UpperType = utr.UpperType + + if dst.LowerType == Empty { + return nil + } + + if dst.LowerType == Inclusive || dst.LowerType == Exclusive { + if err := dst.Lower.DecodeText(ci, []byte(utr.Lower)); err != nil { + return err + } + } + + if dst.UpperType == Inclusive || dst.UpperType == Exclusive { + if err := dst.Upper.DecodeText(ci, []byte(utr.Upper)); err != nil { + return err + } + } + + return nil +} + +func (dst *Daterange) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Daterange{Status: Null} + return nil + } + + ubr, err := ParseUntypedBinaryRange(src) + if err != nil { + return err + } + + *dst = Daterange{Status: Present} + + dst.LowerType = ubr.LowerType + dst.UpperType = ubr.UpperType + + if dst.LowerType == Empty { + return nil + } + + if dst.LowerType == Inclusive || dst.LowerType == Exclusive { + if err := dst.Lower.DecodeBinary(ci, ubr.Lower); err != nil { + return err + } + } + + if dst.UpperType == Inclusive || dst.UpperType == Exclusive { + if err := dst.Upper.DecodeBinary(ci, ubr.Upper); err != nil { + return err + } + } + + return nil +} + +func (src Daterange) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + switch src.LowerType { + case Exclusive, Unbounded: + buf = append(buf, '(') + case Inclusive: + buf = append(buf, '[') + case Empty: + return append(buf, "empty"...), nil + default: + return nil, errors.Errorf("unknown lower bound type %v", src.LowerType) + } + + var err error + + if src.LowerType != Unbounded { + buf, err = src.Lower.EncodeText(ci, buf) + if err != nil { + return nil, err + } else if buf == nil { + return nil, errors.Errorf("Lower cannot be null unless LowerType is Unbounded") + } + } + + buf = append(buf, ',') + + if src.UpperType != Unbounded { + buf, err = src.Upper.EncodeText(ci, buf) + if err != nil { + return nil, err + } else if buf == nil { + return nil, errors.Errorf("Upper cannot be null unless UpperType is Unbounded") + } + } + + switch src.UpperType { + case Exclusive, Unbounded: + buf = append(buf, ')') + case Inclusive: + buf = append(buf, ']') + default: + return nil, errors.Errorf("unknown upper bound type %v", src.UpperType) + } + + return buf, nil +} + +func (src Daterange) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + var rangeType byte + switch src.LowerType { + case Inclusive: + rangeType |= lowerInclusiveMask + case Unbounded: + rangeType |= lowerUnboundedMask + case Exclusive: + case Empty: + return append(buf, emptyMask), nil + default: + return nil, errors.Errorf("unknown LowerType: %v", src.LowerType) + } + + switch src.UpperType { + case Inclusive: + rangeType |= upperInclusiveMask + case Unbounded: + rangeType |= upperUnboundedMask + case Exclusive: + default: + return nil, errors.Errorf("unknown UpperType: %v", src.UpperType) + } + + buf = append(buf, rangeType) + + var err error + + if src.LowerType != Unbounded { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + buf, err = src.Lower.EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if buf == nil { + return nil, errors.Errorf("Lower cannot be null unless LowerType is Unbounded") + } + + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + + if src.UpperType != Unbounded { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + buf, err = src.Upper.EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if buf == nil { + return nil, errors.Errorf("Upper cannot be null unless UpperType is Unbounded") + } + + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Daterange) Scan(src interface{}) error { + if src == nil { + *dst = Daterange{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Daterange) Value() (driver.Value, error) { + return EncodeValueText(src) +} diff --git a/github.com/jackc/pgtype/enum_array.go b/github.com/jackc/pgtype/enum_array.go new file mode 100644 index 0000000000..a31916dce0 --- /dev/null +++ b/github.com/jackc/pgtype/enum_array.go @@ -0,0 +1,232 @@ +package pgtype + +import ( + "database/sql/driver" + + errors "golang.org/x/xerrors" +) + +type EnumArray struct { + Elements []GenericText + Dimensions []ArrayDimension + Status Status +} + +func (dst *EnumArray) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = EnumArray{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + + case []string: + if value == nil { + *dst = EnumArray{Status: Null} + } else if len(value) == 0 { + *dst = EnumArray{Status: Present} + } else { + elements := make([]GenericText, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = EnumArray{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []GenericText: + if value == nil { + *dst = EnumArray{Status: Null} + } else if len(value) == 0 { + *dst = EnumArray{Status: Present} + } else { + *dst = EnumArray{ + Elements: value, + Dimensions: []ArrayDimension{{Length: int32(len(value)), LowerBound: 1}}, + Status: Present, + } + } + default: + if originalSrc, ok := underlyingSliceType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to EnumArray", value) + } + + return nil +} + +func (dst EnumArray) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *EnumArray) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + + case *[]string: + *v = make([]string, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *EnumArray) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = EnumArray{Status: Null} + return nil + } + + uta, err := ParseUntypedTextArray(string(src)) + if err != nil { + return err + } + + var elements []GenericText + + if len(uta.Elements) > 0 { + elements = make([]GenericText, len(uta.Elements)) + + for i, s := range uta.Elements { + var elem GenericText + var elemSrc []byte + if s != "NULL" { + elemSrc = []byte(s) + } + err = elem.DecodeText(ci, elemSrc) + if err != nil { + return err + } + + elements[i] = elem + } + } + + *dst = EnumArray{Elements: elements, Dimensions: uta.Dimensions, Status: Present} + + return nil +} + +func (src EnumArray) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + if len(src.Dimensions) == 0 { + return append(buf, '{', '}'), nil + } + + buf = EncodeTextArrayDimensions(buf, src.Dimensions) + + // dimElemCounts is the multiples of elements that each array lies on. For + // example, a single dimension array of length 4 would have a dimElemCounts of + // [4]. A multi-dimensional array of lengths [3,5,2] would have a + // dimElemCounts of [30,10,2]. This is used to simplify when to render a '{' + // or '}'. + dimElemCounts := make([]int, len(src.Dimensions)) + dimElemCounts[len(src.Dimensions)-1] = int(src.Dimensions[len(src.Dimensions)-1].Length) + for i := len(src.Dimensions) - 2; i > -1; i-- { + dimElemCounts[i] = int(src.Dimensions[i].Length) * dimElemCounts[i+1] + } + + inElemBuf := make([]byte, 0, 32) + for i, elem := range src.Elements { + if i > 0 { + buf = append(buf, ',') + } + + for _, dec := range dimElemCounts { + if i%dec == 0 { + buf = append(buf, '{') + } + } + + elemBuf, err := elem.EncodeText(ci, inElemBuf) + if err != nil { + return nil, err + } + if elemBuf == nil { + buf = append(buf, `NULL`...) + } else { + buf = append(buf, QuoteArrayElementIfNeeded(string(elemBuf))...) + } + + for _, dec := range dimElemCounts { + if (i+1)%dec == 0 { + buf = append(buf, '}') + } + } + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *EnumArray) Scan(src interface{}) error { + if src == nil { + return dst.DecodeText(nil, nil) + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src EnumArray) Value() (driver.Value, error) { + buf, err := src.EncodeText(nil, nil) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + + return string(buf), nil +} diff --git a/github.com/jackc/pgtype/float4.go b/github.com/jackc/pgtype/float4.go new file mode 100644 index 0000000000..e33dfc7500 --- /dev/null +++ b/github.com/jackc/pgtype/float4.go @@ -0,0 +1,204 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + "math" + "strconv" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type Float4 struct { + Float float32 + Status Status +} + +func (dst *Float4) Set(src interface{}) error { + if src == nil { + *dst = Float4{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + case float32: + *dst = Float4{Float: value, Status: Present} + case float64: + *dst = Float4{Float: float32(value), Status: Present} + case int8: + *dst = Float4{Float: float32(value), Status: Present} + case uint8: + *dst = Float4{Float: float32(value), Status: Present} + case int16: + *dst = Float4{Float: float32(value), Status: Present} + case uint16: + *dst = Float4{Float: float32(value), Status: Present} + case int32: + f32 := float32(value) + if int32(f32) == value { + *dst = Float4{Float: f32, Status: Present} + } else { + return errors.Errorf("%v cannot be exactly represented as float32", value) + } + case uint32: + f32 := float32(value) + if uint32(f32) == value { + *dst = Float4{Float: f32, Status: Present} + } else { + return errors.Errorf("%v cannot be exactly represented as float32", value) + } + case int64: + f32 := float32(value) + if int64(f32) == value { + *dst = Float4{Float: f32, Status: Present} + } else { + return errors.Errorf("%v cannot be exactly represented as float32", value) + } + case uint64: + f32 := float32(value) + if uint64(f32) == value { + *dst = Float4{Float: f32, Status: Present} + } else { + return errors.Errorf("%v cannot be exactly represented as float32", value) + } + case int: + f32 := float32(value) + if int(f32) == value { + *dst = Float4{Float: f32, Status: Present} + } else { + return errors.Errorf("%v cannot be exactly represented as float32", value) + } + case uint: + f32 := float32(value) + if uint(f32) == value { + *dst = Float4{Float: f32, Status: Present} + } else { + return errors.Errorf("%v cannot be exactly represented as float32", value) + } + case string: + num, err := strconv.ParseFloat(value, 32) + if err != nil { + return err + } + *dst = Float4{Float: float32(num), Status: Present} + default: + if originalSrc, ok := underlyingNumberType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to Float8", value) + } + + return nil +} + +func (dst Float4) Get() interface{} { + switch dst.Status { + case Present: + return dst.Float + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Float4) AssignTo(dst interface{}) error { + return float64AssignTo(float64(src.Float), src.Status, dst) +} + +func (dst *Float4) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Float4{Status: Null} + return nil + } + + n, err := strconv.ParseFloat(string(src), 32) + if err != nil { + return err + } + + *dst = Float4{Float: float32(n), Status: Present} + return nil +} + +func (dst *Float4) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Float4{Status: Null} + return nil + } + + if len(src) != 4 { + return errors.Errorf("invalid length for float4: %v", len(src)) + } + + n := int32(binary.BigEndian.Uint32(src)) + + *dst = Float4{Float: math.Float32frombits(uint32(n)), Status: Present} + return nil +} + +func (src Float4) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + buf = append(buf, strconv.FormatFloat(float64(src.Float), 'f', -1, 32)...) + return buf, nil +} + +func (src Float4) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + buf = pgio.AppendUint32(buf, math.Float32bits(src.Float)) + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Float4) Scan(src interface{}) error { + if src == nil { + *dst = Float4{Status: Null} + return nil + } + + switch src := src.(type) { + case float64: + *dst = Float4{Float: float32(src), Status: Present} + return nil + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Float4) Value() (driver.Value, error) { + switch src.Status { + case Present: + return float64(src.Float), nil + case Null: + return nil, nil + default: + return nil, errUndefined + } +} diff --git a/github.com/jackc/pgtype/float4_array.go b/github.com/jackc/pgtype/float4_array.go new file mode 100644 index 0000000000..ccd718a151 --- /dev/null +++ b/github.com/jackc/pgtype/float4_array.go @@ -0,0 +1,320 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type Float4Array struct { + Elements []Float4 + Dimensions []ArrayDimension + Status Status +} + +func (dst *Float4Array) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = Float4Array{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + + case []float32: + if value == nil { + *dst = Float4Array{Status: Null} + } else if len(value) == 0 { + *dst = Float4Array{Status: Present} + } else { + elements := make([]Float4, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = Float4Array{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []Float4: + if value == nil { + *dst = Float4Array{Status: Null} + } else if len(value) == 0 { + *dst = Float4Array{Status: Present} + } else { + *dst = Float4Array{ + Elements: value, + Dimensions: []ArrayDimension{{Length: int32(len(value)), LowerBound: 1}}, + Status: Present, + } + } + default: + if originalSrc, ok := underlyingSliceType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to Float4Array", value) + } + + return nil +} + +func (dst Float4Array) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Float4Array) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + + case *[]float32: + *v = make([]float32, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *Float4Array) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Float4Array{Status: Null} + return nil + } + + uta, err := ParseUntypedTextArray(string(src)) + if err != nil { + return err + } + + var elements []Float4 + + if len(uta.Elements) > 0 { + elements = make([]Float4, len(uta.Elements)) + + for i, s := range uta.Elements { + var elem Float4 + var elemSrc []byte + if s != "NULL" { + elemSrc = []byte(s) + } + err = elem.DecodeText(ci, elemSrc) + if err != nil { + return err + } + + elements[i] = elem + } + } + + *dst = Float4Array{Elements: elements, Dimensions: uta.Dimensions, Status: Present} + + return nil +} + +func (dst *Float4Array) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Float4Array{Status: Null} + return nil + } + + var arrayHeader ArrayHeader + rp, err := arrayHeader.DecodeBinary(ci, src) + if err != nil { + return err + } + + if len(arrayHeader.Dimensions) == 0 { + *dst = Float4Array{Dimensions: arrayHeader.Dimensions, Status: Present} + return nil + } + + elementCount := arrayHeader.Dimensions[0].Length + for _, d := range arrayHeader.Dimensions[1:] { + elementCount *= d.Length + } + + elements := make([]Float4, elementCount) + + for i := range elements { + elemLen := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + var elemSrc []byte + if elemLen >= 0 { + elemSrc = src[rp : rp+elemLen] + rp += elemLen + } + err = elements[i].DecodeBinary(ci, elemSrc) + if err != nil { + return err + } + } + + *dst = Float4Array{Elements: elements, Dimensions: arrayHeader.Dimensions, Status: Present} + return nil +} + +func (src Float4Array) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + if len(src.Dimensions) == 0 { + return append(buf, '{', '}'), nil + } + + buf = EncodeTextArrayDimensions(buf, src.Dimensions) + + // dimElemCounts is the multiples of elements that each array lies on. For + // example, a single dimension array of length 4 would have a dimElemCounts of + // [4]. A multi-dimensional array of lengths [3,5,2] would have a + // dimElemCounts of [30,10,2]. This is used to simplify when to render a '{' + // or '}'. + dimElemCounts := make([]int, len(src.Dimensions)) + dimElemCounts[len(src.Dimensions)-1] = int(src.Dimensions[len(src.Dimensions)-1].Length) + for i := len(src.Dimensions) - 2; i > -1; i-- { + dimElemCounts[i] = int(src.Dimensions[i].Length) * dimElemCounts[i+1] + } + + inElemBuf := make([]byte, 0, 32) + for i, elem := range src.Elements { + if i > 0 { + buf = append(buf, ',') + } + + for _, dec := range dimElemCounts { + if i%dec == 0 { + buf = append(buf, '{') + } + } + + elemBuf, err := elem.EncodeText(ci, inElemBuf) + if err != nil { + return nil, err + } + if elemBuf == nil { + buf = append(buf, `NULL`...) + } else { + buf = append(buf, QuoteArrayElementIfNeeded(string(elemBuf))...) + } + + for _, dec := range dimElemCounts { + if (i+1)%dec == 0 { + buf = append(buf, '}') + } + } + } + + return buf, nil +} + +func (src Float4Array) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + arrayHeader := ArrayHeader{ + Dimensions: src.Dimensions, + } + + if dt, ok := ci.DataTypeForName("float4"); ok { + arrayHeader.ElementOID = int32(dt.OID) + } else { + return nil, errors.Errorf("unable to find oid for type name %v", "float4") + } + + for i := range src.Elements { + if src.Elements[i].Status == Null { + arrayHeader.ContainsNull = true + break + } + } + + buf = arrayHeader.EncodeBinary(ci, buf) + + for i := range src.Elements { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + elemBuf, err := src.Elements[i].EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if elemBuf != nil { + buf = elemBuf + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Float4Array) Scan(src interface{}) error { + if src == nil { + return dst.DecodeText(nil, nil) + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Float4Array) Value() (driver.Value, error) { + buf, err := src.EncodeText(nil, nil) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + + return string(buf), nil +} diff --git a/github.com/jackc/pgtype/float8.go b/github.com/jackc/pgtype/float8.go new file mode 100644 index 0000000000..41d0fe70dc --- /dev/null +++ b/github.com/jackc/pgtype/float8.go @@ -0,0 +1,194 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + "math" + "strconv" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type Float8 struct { + Float float64 + Status Status +} + +func (dst *Float8) Set(src interface{}) error { + if src == nil { + *dst = Float8{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + case float32: + *dst = Float8{Float: float64(value), Status: Present} + case float64: + *dst = Float8{Float: value, Status: Present} + case int8: + *dst = Float8{Float: float64(value), Status: Present} + case uint8: + *dst = Float8{Float: float64(value), Status: Present} + case int16: + *dst = Float8{Float: float64(value), Status: Present} + case uint16: + *dst = Float8{Float: float64(value), Status: Present} + case int32: + *dst = Float8{Float: float64(value), Status: Present} + case uint32: + *dst = Float8{Float: float64(value), Status: Present} + case int64: + f64 := float64(value) + if int64(f64) == value { + *dst = Float8{Float: f64, Status: Present} + } else { + return errors.Errorf("%v cannot be exactly represented as float64", value) + } + case uint64: + f64 := float64(value) + if uint64(f64) == value { + *dst = Float8{Float: f64, Status: Present} + } else { + return errors.Errorf("%v cannot be exactly represented as float64", value) + } + case int: + f64 := float64(value) + if int(f64) == value { + *dst = Float8{Float: f64, Status: Present} + } else { + return errors.Errorf("%v cannot be exactly represented as float64", value) + } + case uint: + f64 := float64(value) + if uint(f64) == value { + *dst = Float8{Float: f64, Status: Present} + } else { + return errors.Errorf("%v cannot be exactly represented as float64", value) + } + case string: + num, err := strconv.ParseFloat(value, 64) + if err != nil { + return err + } + *dst = Float8{Float: float64(num), Status: Present} + default: + if originalSrc, ok := underlyingNumberType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to Float8", value) + } + + return nil +} + +func (dst Float8) Get() interface{} { + switch dst.Status { + case Present: + return dst.Float + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Float8) AssignTo(dst interface{}) error { + return float64AssignTo(src.Float, src.Status, dst) +} + +func (dst *Float8) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Float8{Status: Null} + return nil + } + + n, err := strconv.ParseFloat(string(src), 64) + if err != nil { + return err + } + + *dst = Float8{Float: n, Status: Present} + return nil +} + +func (dst *Float8) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Float8{Status: Null} + return nil + } + + if len(src) != 8 { + return errors.Errorf("invalid length for float4: %v", len(src)) + } + + n := int64(binary.BigEndian.Uint64(src)) + + *dst = Float8{Float: math.Float64frombits(uint64(n)), Status: Present} + return nil +} + +func (src Float8) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + buf = append(buf, strconv.FormatFloat(float64(src.Float), 'f', -1, 64)...) + return buf, nil +} + +func (src Float8) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + buf = pgio.AppendUint64(buf, math.Float64bits(src.Float)) + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Float8) Scan(src interface{}) error { + if src == nil { + *dst = Float8{Status: Null} + return nil + } + + switch src := src.(type) { + case float64: + *dst = Float8{Float: src, Status: Present} + return nil + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Float8) Value() (driver.Value, error) { + switch src.Status { + case Present: + return src.Float, nil + case Null: + return nil, nil + default: + return nil, errUndefined + } +} diff --git a/github.com/jackc/pgtype/float8_array.go b/github.com/jackc/pgtype/float8_array.go new file mode 100644 index 0000000000..740e85580b --- /dev/null +++ b/github.com/jackc/pgtype/float8_array.go @@ -0,0 +1,320 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type Float8Array struct { + Elements []Float8 + Dimensions []ArrayDimension + Status Status +} + +func (dst *Float8Array) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = Float8Array{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + + case []float64: + if value == nil { + *dst = Float8Array{Status: Null} + } else if len(value) == 0 { + *dst = Float8Array{Status: Present} + } else { + elements := make([]Float8, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = Float8Array{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []Float8: + if value == nil { + *dst = Float8Array{Status: Null} + } else if len(value) == 0 { + *dst = Float8Array{Status: Present} + } else { + *dst = Float8Array{ + Elements: value, + Dimensions: []ArrayDimension{{Length: int32(len(value)), LowerBound: 1}}, + Status: Present, + } + } + default: + if originalSrc, ok := underlyingSliceType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to Float8Array", value) + } + + return nil +} + +func (dst Float8Array) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Float8Array) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + + case *[]float64: + *v = make([]float64, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *Float8Array) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Float8Array{Status: Null} + return nil + } + + uta, err := ParseUntypedTextArray(string(src)) + if err != nil { + return err + } + + var elements []Float8 + + if len(uta.Elements) > 0 { + elements = make([]Float8, len(uta.Elements)) + + for i, s := range uta.Elements { + var elem Float8 + var elemSrc []byte + if s != "NULL" { + elemSrc = []byte(s) + } + err = elem.DecodeText(ci, elemSrc) + if err != nil { + return err + } + + elements[i] = elem + } + } + + *dst = Float8Array{Elements: elements, Dimensions: uta.Dimensions, Status: Present} + + return nil +} + +func (dst *Float8Array) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Float8Array{Status: Null} + return nil + } + + var arrayHeader ArrayHeader + rp, err := arrayHeader.DecodeBinary(ci, src) + if err != nil { + return err + } + + if len(arrayHeader.Dimensions) == 0 { + *dst = Float8Array{Dimensions: arrayHeader.Dimensions, Status: Present} + return nil + } + + elementCount := arrayHeader.Dimensions[0].Length + for _, d := range arrayHeader.Dimensions[1:] { + elementCount *= d.Length + } + + elements := make([]Float8, elementCount) + + for i := range elements { + elemLen := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + var elemSrc []byte + if elemLen >= 0 { + elemSrc = src[rp : rp+elemLen] + rp += elemLen + } + err = elements[i].DecodeBinary(ci, elemSrc) + if err != nil { + return err + } + } + + *dst = Float8Array{Elements: elements, Dimensions: arrayHeader.Dimensions, Status: Present} + return nil +} + +func (src Float8Array) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + if len(src.Dimensions) == 0 { + return append(buf, '{', '}'), nil + } + + buf = EncodeTextArrayDimensions(buf, src.Dimensions) + + // dimElemCounts is the multiples of elements that each array lies on. For + // example, a single dimension array of length 4 would have a dimElemCounts of + // [4]. A multi-dimensional array of lengths [3,5,2] would have a + // dimElemCounts of [30,10,2]. This is used to simplify when to render a '{' + // or '}'. + dimElemCounts := make([]int, len(src.Dimensions)) + dimElemCounts[len(src.Dimensions)-1] = int(src.Dimensions[len(src.Dimensions)-1].Length) + for i := len(src.Dimensions) - 2; i > -1; i-- { + dimElemCounts[i] = int(src.Dimensions[i].Length) * dimElemCounts[i+1] + } + + inElemBuf := make([]byte, 0, 32) + for i, elem := range src.Elements { + if i > 0 { + buf = append(buf, ',') + } + + for _, dec := range dimElemCounts { + if i%dec == 0 { + buf = append(buf, '{') + } + } + + elemBuf, err := elem.EncodeText(ci, inElemBuf) + if err != nil { + return nil, err + } + if elemBuf == nil { + buf = append(buf, `NULL`...) + } else { + buf = append(buf, QuoteArrayElementIfNeeded(string(elemBuf))...) + } + + for _, dec := range dimElemCounts { + if (i+1)%dec == 0 { + buf = append(buf, '}') + } + } + } + + return buf, nil +} + +func (src Float8Array) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + arrayHeader := ArrayHeader{ + Dimensions: src.Dimensions, + } + + if dt, ok := ci.DataTypeForName("float8"); ok { + arrayHeader.ElementOID = int32(dt.OID) + } else { + return nil, errors.Errorf("unable to find oid for type name %v", "float8") + } + + for i := range src.Elements { + if src.Elements[i].Status == Null { + arrayHeader.ContainsNull = true + break + } + } + + buf = arrayHeader.EncodeBinary(ci, buf) + + for i := range src.Elements { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + elemBuf, err := src.Elements[i].EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if elemBuf != nil { + buf = elemBuf + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Float8Array) Scan(src interface{}) error { + if src == nil { + return dst.DecodeText(nil, nil) + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Float8Array) Value() (driver.Value, error) { + buf, err := src.EncodeText(nil, nil) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + + return string(buf), nil +} diff --git a/github.com/jackc/pgtype/generic_binary.go b/github.com/jackc/pgtype/generic_binary.go new file mode 100644 index 0000000000..76a1d3511a --- /dev/null +++ b/github.com/jackc/pgtype/generic_binary.go @@ -0,0 +1,39 @@ +package pgtype + +import ( + "database/sql/driver" +) + +// GenericBinary is a placeholder for binary format values that no other type exists +// to handle. +type GenericBinary Bytea + +func (dst *GenericBinary) Set(src interface{}) error { + return (*Bytea)(dst).Set(src) +} + +func (dst GenericBinary) Get() interface{} { + return (Bytea)(dst).Get() +} + +func (src *GenericBinary) AssignTo(dst interface{}) error { + return (*Bytea)(src).AssignTo(dst) +} + +func (dst *GenericBinary) DecodeBinary(ci *ConnInfo, src []byte) error { + return (*Bytea)(dst).DecodeBinary(ci, src) +} + +func (src GenericBinary) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + return (Bytea)(src).EncodeBinary(ci, buf) +} + +// Scan implements the database/sql Scanner interface. +func (dst *GenericBinary) Scan(src interface{}) error { + return (*Bytea)(dst).Scan(src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src GenericBinary) Value() (driver.Value, error) { + return (Bytea)(src).Value() +} diff --git a/github.com/jackc/pgtype/generic_text.go b/github.com/jackc/pgtype/generic_text.go new file mode 100644 index 0000000000..dbf5b47e80 --- /dev/null +++ b/github.com/jackc/pgtype/generic_text.go @@ -0,0 +1,39 @@ +package pgtype + +import ( + "database/sql/driver" +) + +// GenericText is a placeholder for text format values that no other type exists +// to handle. +type GenericText Text + +func (dst *GenericText) Set(src interface{}) error { + return (*Text)(dst).Set(src) +} + +func (dst GenericText) Get() interface{} { + return (Text)(dst).Get() +} + +func (src *GenericText) AssignTo(dst interface{}) error { + return (*Text)(src).AssignTo(dst) +} + +func (dst *GenericText) DecodeText(ci *ConnInfo, src []byte) error { + return (*Text)(dst).DecodeText(ci, src) +} + +func (src GenericText) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + return (Text)(src).EncodeText(ci, buf) +} + +// Scan implements the database/sql Scanner interface. +func (dst *GenericText) Scan(src interface{}) error { + return (*Text)(dst).Scan(src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src GenericText) Value() (driver.Value, error) { + return (Text)(src).Value() +} diff --git a/github.com/jackc/pgtype/go.mod b/github.com/jackc/pgtype/go.mod new file mode 100644 index 0000000000..920f4b2195 --- /dev/null +++ b/github.com/jackc/pgtype/go.mod @@ -0,0 +1,14 @@ +module github.com/jackc/pgtype + +go 1.12 + +require ( + github.com/gofrs/uuid v3.2.0+incompatible + github.com/jackc/pgio v1.0.0 + github.com/jackc/pgx v3.6.2+incompatible + github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186 + github.com/lib/pq v1.2.0 + github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 + github.com/stretchr/testify v1.4.0 + golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 +) diff --git a/github.com/jackc/pgtype/go.sum b/github.com/jackc/pgtype/go.sum new file mode 100644 index 0000000000..7a3fc71e5e --- /dev/null +++ b/github.com/jackc/pgtype/go.sum @@ -0,0 +1,120 @@ +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0 h1:DUwgMQuuPnS0rhMXenUtZpqZqrR/30NWY+qQvTpSvEs= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3 h1:ZFYpB74Kq8xE9gmfxCmXD6QxZ27ja+j3HwGFc+YurhQ= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb h1:d6GP9szHvXVopAOAnZ7WhRnF3Xdxrylmm/9jnfmW4Ag= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db h1:UpaKn/gYxzH6/zWyRQH1S260zvKqwJJ4h8+Kf09ooh0= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711 h1:vZp4bYotXUkFx7JUSm7U8KV/7Q0AOdrQxxBBj0ZmZsg= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgx v3.6.2+incompatible h1:2zP5OD7kiyR3xzRYMhOcXVvkDZsImVXfj+yIyTQf3/o= +github.com/jackc/pgx v3.6.2+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96 h1:ylEAOd688Duev/fxTmGdupsbyZfxNMdngIG14DoBKTM= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912 h1:YuOWGsSK5L4Fz81Olx5TNlZftmDuNrfv4ip0Yos77Tw= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186 h1:ZQM8qLT/E/CGD6XX0E6q9FAwxJYmWpJufzmLMaFuzgQ= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0 h1:/5u4a+KGJptBRqGzPvYQL9p0d/tPR4S31+Tnzj9lEO4= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a h1:Igim7XhdOpBnWPuYJ70XcNpq8q3BCACtVgNfoJxOV7g= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 h1:7KByu05hhLed2MO29w7p1XfZvZ13m8mub3shuVftRs0= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373 h1:PPwnA7z1Pjf7XYaBP9GL1VAMZmcIWyFz7QCMSIIa3Bg= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/github.com/jackc/pgtype/hstore.go b/github.com/jackc/pgtype/hstore.go new file mode 100644 index 0000000000..3fe50ae58d --- /dev/null +++ b/github.com/jackc/pgtype/hstore.go @@ -0,0 +1,438 @@ +package pgtype + +import ( + "bytes" + "database/sql/driver" + "encoding/binary" + "strings" + "unicode" + "unicode/utf8" + + errors "golang.org/x/xerrors" + + "github.com/jackc/pgio" +) + +// Hstore represents an hstore column that can be null or have null values +// associated with its keys. +type Hstore struct { + Map map[string]Text + Status Status +} + +func (dst *Hstore) Set(src interface{}) error { + if src == nil { + *dst = Hstore{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + case map[string]string: + m := make(map[string]Text, len(value)) + for k, v := range value { + m[k] = Text{String: v, Status: Present} + } + *dst = Hstore{Map: m, Status: Present} + default: + return errors.Errorf("cannot convert %v to Hstore", src) + } + + return nil +} + +func (dst Hstore) Get() interface{} { + switch dst.Status { + case Present: + return dst.Map + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Hstore) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + case *map[string]string: + *v = make(map[string]string, len(src.Map)) + for k, val := range src.Map { + if val.Status != Present { + return errors.Errorf("cannot decode %#v into %T", src, dst) + } + (*v)[k] = val.String + } + return nil + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *Hstore) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Hstore{Status: Null} + return nil + } + + keys, values, err := parseHstore(string(src)) + if err != nil { + return err + } + + m := make(map[string]Text, len(keys)) + for i := range keys { + m[keys[i]] = values[i] + } + + *dst = Hstore{Map: m, Status: Present} + return nil +} + +func (dst *Hstore) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Hstore{Status: Null} + return nil + } + + rp := 0 + + if len(src[rp:]) < 4 { + return errors.Errorf("hstore incomplete %v", src) + } + pairCount := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + + m := make(map[string]Text, pairCount) + + for i := 0; i < pairCount; i++ { + if len(src[rp:]) < 4 { + return errors.Errorf("hstore incomplete %v", src) + } + keyLen := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + + if len(src[rp:]) < keyLen { + return errors.Errorf("hstore incomplete %v", src) + } + key := string(src[rp : rp+keyLen]) + rp += keyLen + + if len(src[rp:]) < 4 { + return errors.Errorf("hstore incomplete %v", src) + } + valueLen := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + + var valueBuf []byte + if valueLen >= 0 { + valueBuf = src[rp : rp+valueLen] + } + rp += valueLen + + var value Text + err := value.DecodeBinary(ci, valueBuf) + if err != nil { + return err + } + m[key] = value + } + + *dst = Hstore{Map: m, Status: Present} + + return nil +} + +func (src Hstore) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + firstPair := true + + for k, v := range src.Map { + if firstPair { + firstPair = false + } else { + buf = append(buf, ',') + } + + buf = append(buf, quoteHstoreElementIfNeeded(k)...) + buf = append(buf, "=>"...) + + elemBuf, err := v.EncodeText(ci, nil) + if err != nil { + return nil, err + } + + if elemBuf == nil { + buf = append(buf, "NULL"...) + } else { + buf = append(buf, quoteHstoreElementIfNeeded(string(elemBuf))...) + } + } + + return buf, nil +} + +func (src Hstore) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + buf = pgio.AppendInt32(buf, int32(len(src.Map))) + + var err error + for k, v := range src.Map { + buf = pgio.AppendInt32(buf, int32(len(k))) + buf = append(buf, k...) + + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + elemBuf, err := v.EncodeText(ci, buf) + if err != nil { + return nil, err + } + if elemBuf != nil { + buf = elemBuf + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + } + + return buf, err +} + +var quoteHstoreReplacer = strings.NewReplacer(`\`, `\\`, `"`, `\"`) + +func quoteHstoreElement(src string) string { + return `"` + quoteArrayReplacer.Replace(src) + `"` +} + +func quoteHstoreElementIfNeeded(src string) string { + if src == "" || (len(src) == 4 && strings.ToLower(src) == "null") || strings.ContainsAny(src, ` {},"\=>`) { + return quoteArrayElement(src) + } + return src +} + +const ( + hsPre = iota + hsKey + hsSep + hsVal + hsNul + hsNext +) + +type hstoreParser struct { + str string + pos int +} + +func newHSP(in string) *hstoreParser { + return &hstoreParser{ + pos: 0, + str: in, + } +} + +func (p *hstoreParser) Consume() (r rune, end bool) { + if p.pos >= len(p.str) { + end = true + return + } + r, w := utf8.DecodeRuneInString(p.str[p.pos:]) + p.pos += w + return +} + +func (p *hstoreParser) Peek() (r rune, end bool) { + if p.pos >= len(p.str) { + end = true + return + } + r, _ = utf8.DecodeRuneInString(p.str[p.pos:]) + return +} + +// parseHstore parses the string representation of an hstore column (the same +// you would get from an ordinary SELECT) into two slices of keys and values. it +// is used internally in the default parsing of hstores. +func parseHstore(s string) (k []string, v []Text, err error) { + if s == "" { + return + } + + buf := bytes.Buffer{} + keys := []string{} + values := []Text{} + p := newHSP(s) + + r, end := p.Consume() + state := hsPre + + for !end { + switch state { + case hsPre: + if r == '"' { + state = hsKey + } else { + err = errors.New("String does not begin with \"") + } + case hsKey: + switch r { + case '"': //End of the key + keys = append(keys, buf.String()) + buf = bytes.Buffer{} + state = hsSep + case '\\': //Potential escaped character + n, end := p.Consume() + switch { + case end: + err = errors.New("Found EOS in key, expecting character or \"") + case n == '"', n == '\\': + buf.WriteRune(n) + default: + buf.WriteRune(r) + buf.WriteRune(n) + } + default: //Any other character + buf.WriteRune(r) + } + case hsSep: + if r == '=' { + r, end = p.Consume() + switch { + case end: + err = errors.New("Found EOS after '=', expecting '>'") + case r == '>': + r, end = p.Consume() + switch { + case end: + err = errors.New("Found EOS after '=>', expecting '\"' or 'NULL'") + case r == '"': + state = hsVal + case r == 'N': + state = hsNul + default: + err = errors.Errorf("Invalid character '%c' after '=>', expecting '\"' or 'NULL'", r) + } + default: + err = errors.Errorf("Invalid character after '=', expecting '>'") + } + } else { + err = errors.Errorf("Invalid character '%c' after value, expecting '='", r) + } + case hsVal: + switch r { + case '"': //End of the value + values = append(values, Text{String: buf.String(), Status: Present}) + buf = bytes.Buffer{} + state = hsNext + case '\\': //Potential escaped character + n, end := p.Consume() + switch { + case end: + err = errors.New("Found EOS in key, expecting character or \"") + case n == '"', n == '\\': + buf.WriteRune(n) + default: + buf.WriteRune(r) + buf.WriteRune(n) + } + default: //Any other character + buf.WriteRune(r) + } + case hsNul: + nulBuf := make([]rune, 3) + nulBuf[0] = r + for i := 1; i < 3; i++ { + r, end = p.Consume() + if end { + err = errors.New("Found EOS in NULL value") + return + } + nulBuf[i] = r + } + if nulBuf[0] == 'U' && nulBuf[1] == 'L' && nulBuf[2] == 'L' { + values = append(values, Text{Status: Null}) + state = hsNext + } else { + err = errors.Errorf("Invalid NULL value: 'N%s'", string(nulBuf)) + } + case hsNext: + if r == ',' { + r, end = p.Consume() + switch { + case end: + err = errors.New("Found EOS after ',', expcting space") + case (unicode.IsSpace(r)): + r, end = p.Consume() + state = hsKey + default: + err = errors.Errorf("Invalid character '%c' after ', ', expecting \"", r) + } + } else { + err = errors.Errorf("Invalid character '%c' after value, expecting ','", r) + } + } + + if err != nil { + return + } + r, end = p.Consume() + } + if state != hsNext { + err = errors.New("Improperly formatted hstore") + return + } + k = keys + v = values + return +} + +// Scan implements the database/sql Scanner interface. +func (dst *Hstore) Scan(src interface{}) error { + if src == nil { + *dst = Hstore{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Hstore) Value() (driver.Value, error) { + return EncodeValueText(src) +} diff --git a/github.com/jackc/pgtype/hstore_array.go b/github.com/jackc/pgtype/hstore_array.go new file mode 100644 index 0000000000..54909e42b4 --- /dev/null +++ b/github.com/jackc/pgtype/hstore_array.go @@ -0,0 +1,320 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type HstoreArray struct { + Elements []Hstore + Dimensions []ArrayDimension + Status Status +} + +func (dst *HstoreArray) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = HstoreArray{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + + case []map[string]string: + if value == nil { + *dst = HstoreArray{Status: Null} + } else if len(value) == 0 { + *dst = HstoreArray{Status: Present} + } else { + elements := make([]Hstore, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = HstoreArray{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []Hstore: + if value == nil { + *dst = HstoreArray{Status: Null} + } else if len(value) == 0 { + *dst = HstoreArray{Status: Present} + } else { + *dst = HstoreArray{ + Elements: value, + Dimensions: []ArrayDimension{{Length: int32(len(value)), LowerBound: 1}}, + Status: Present, + } + } + default: + if originalSrc, ok := underlyingSliceType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to HstoreArray", value) + } + + return nil +} + +func (dst HstoreArray) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *HstoreArray) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + + case *[]map[string]string: + *v = make([]map[string]string, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *HstoreArray) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = HstoreArray{Status: Null} + return nil + } + + uta, err := ParseUntypedTextArray(string(src)) + if err != nil { + return err + } + + var elements []Hstore + + if len(uta.Elements) > 0 { + elements = make([]Hstore, len(uta.Elements)) + + for i, s := range uta.Elements { + var elem Hstore + var elemSrc []byte + if s != "NULL" { + elemSrc = []byte(s) + } + err = elem.DecodeText(ci, elemSrc) + if err != nil { + return err + } + + elements[i] = elem + } + } + + *dst = HstoreArray{Elements: elements, Dimensions: uta.Dimensions, Status: Present} + + return nil +} + +func (dst *HstoreArray) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = HstoreArray{Status: Null} + return nil + } + + var arrayHeader ArrayHeader + rp, err := arrayHeader.DecodeBinary(ci, src) + if err != nil { + return err + } + + if len(arrayHeader.Dimensions) == 0 { + *dst = HstoreArray{Dimensions: arrayHeader.Dimensions, Status: Present} + return nil + } + + elementCount := arrayHeader.Dimensions[0].Length + for _, d := range arrayHeader.Dimensions[1:] { + elementCount *= d.Length + } + + elements := make([]Hstore, elementCount) + + for i := range elements { + elemLen := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + var elemSrc []byte + if elemLen >= 0 { + elemSrc = src[rp : rp+elemLen] + rp += elemLen + } + err = elements[i].DecodeBinary(ci, elemSrc) + if err != nil { + return err + } + } + + *dst = HstoreArray{Elements: elements, Dimensions: arrayHeader.Dimensions, Status: Present} + return nil +} + +func (src HstoreArray) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + if len(src.Dimensions) == 0 { + return append(buf, '{', '}'), nil + } + + buf = EncodeTextArrayDimensions(buf, src.Dimensions) + + // dimElemCounts is the multiples of elements that each array lies on. For + // example, a single dimension array of length 4 would have a dimElemCounts of + // [4]. A multi-dimensional array of lengths [3,5,2] would have a + // dimElemCounts of [30,10,2]. This is used to simplify when to render a '{' + // or '}'. + dimElemCounts := make([]int, len(src.Dimensions)) + dimElemCounts[len(src.Dimensions)-1] = int(src.Dimensions[len(src.Dimensions)-1].Length) + for i := len(src.Dimensions) - 2; i > -1; i-- { + dimElemCounts[i] = int(src.Dimensions[i].Length) * dimElemCounts[i+1] + } + + inElemBuf := make([]byte, 0, 32) + for i, elem := range src.Elements { + if i > 0 { + buf = append(buf, ',') + } + + for _, dec := range dimElemCounts { + if i%dec == 0 { + buf = append(buf, '{') + } + } + + elemBuf, err := elem.EncodeText(ci, inElemBuf) + if err != nil { + return nil, err + } + if elemBuf == nil { + buf = append(buf, `NULL`...) + } else { + buf = append(buf, QuoteArrayElementIfNeeded(string(elemBuf))...) + } + + for _, dec := range dimElemCounts { + if (i+1)%dec == 0 { + buf = append(buf, '}') + } + } + } + + return buf, nil +} + +func (src HstoreArray) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + arrayHeader := ArrayHeader{ + Dimensions: src.Dimensions, + } + + if dt, ok := ci.DataTypeForName("hstore"); ok { + arrayHeader.ElementOID = int32(dt.OID) + } else { + return nil, errors.Errorf("unable to find oid for type name %v", "hstore") + } + + for i := range src.Elements { + if src.Elements[i].Status == Null { + arrayHeader.ContainsNull = true + break + } + } + + buf = arrayHeader.EncodeBinary(ci, buf) + + for i := range src.Elements { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + elemBuf, err := src.Elements[i].EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if elemBuf != nil { + buf = elemBuf + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *HstoreArray) Scan(src interface{}) error { + if src == nil { + return dst.DecodeText(nil, nil) + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src HstoreArray) Value() (driver.Value, error) { + buf, err := src.EncodeText(nil, nil) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + + return string(buf), nil +} diff --git a/github.com/jackc/pgtype/inet.go b/github.com/jackc/pgtype/inet.go new file mode 100644 index 0000000000..7ab78bdf9b --- /dev/null +++ b/github.com/jackc/pgtype/inet.go @@ -0,0 +1,223 @@ +package pgtype + +import ( + "database/sql/driver" + "net" + + errors "golang.org/x/xerrors" +) + +// Network address family is dependent on server socket.h value for AF_INET. +// In practice, all platforms appear to have the same value. See +// src/include/utils/inet.h for more information. +const ( + defaultAFInet = 2 + defaultAFInet6 = 3 +) + +// Inet represents both inet and cidr PostgreSQL types. +type Inet struct { + IPNet *net.IPNet + Status Status +} + +func (dst *Inet) Set(src interface{}) error { + if src == nil { + *dst = Inet{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + case net.IPNet: + *dst = Inet{IPNet: &value, Status: Present} + case *net.IPNet: + *dst = Inet{IPNet: value, Status: Present} + case net.IP: + bitCount := len(value) * 8 + mask := net.CIDRMask(bitCount, bitCount) + *dst = Inet{IPNet: &net.IPNet{Mask: mask, IP: value}, Status: Present} + case string: + _, ipnet, err := net.ParseCIDR(value) + if err != nil { + return err + } + *dst = Inet{IPNet: ipnet, Status: Present} + default: + if originalSrc, ok := underlyingPtrType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to Inet", value) + } + + return nil +} + +func (dst Inet) Get() interface{} { + switch dst.Status { + case Present: + return dst.IPNet + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Inet) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + case *net.IPNet: + *v = net.IPNet{ + IP: make(net.IP, len(src.IPNet.IP)), + Mask: make(net.IPMask, len(src.IPNet.Mask)), + } + copy(v.IP, src.IPNet.IP) + copy(v.Mask, src.IPNet.Mask) + return nil + case *net.IP: + if oneCount, bitCount := src.IPNet.Mask.Size(); oneCount != bitCount { + return errors.Errorf("cannot assign %v to %T", src, dst) + } + *v = make(net.IP, len(src.IPNet.IP)) + copy(*v, src.IPNet.IP) + return nil + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *Inet) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Inet{Status: Null} + return nil + } + + var ipnet *net.IPNet + var err error + + if ip := net.ParseIP(string(src)); ip != nil { + ipv4 := ip.To4() + if ipv4 != nil { + ip = ipv4 + } + bitCount := len(ip) * 8 + mask := net.CIDRMask(bitCount, bitCount) + ipnet = &net.IPNet{Mask: mask, IP: ip} + } else { + _, ipnet, err = net.ParseCIDR(string(src)) + if err != nil { + return err + } + } + + *dst = Inet{IPNet: ipnet, Status: Present} + return nil +} + +func (dst *Inet) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Inet{Status: Null} + return nil + } + + if len(src) != 8 && len(src) != 20 { + return errors.Errorf("Received an invalid size for a inet: %d", len(src)) + } + + // ignore family + bits := src[1] + // ignore is_cidr + addressLength := src[3] + + var ipnet net.IPNet + ipnet.IP = make(net.IP, int(addressLength)) + copy(ipnet.IP, src[4:]) + ipnet.Mask = net.CIDRMask(int(bits), int(addressLength)*8) + + *dst = Inet{IPNet: &ipnet, Status: Present} + + return nil +} + +func (src Inet) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + return append(buf, src.IPNet.String()...), nil +} + +// EncodeBinary encodes src into w. +func (src Inet) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + var family byte + switch len(src.IPNet.IP) { + case net.IPv4len: + family = defaultAFInet + case net.IPv6len: + family = defaultAFInet6 + default: + return nil, errors.Errorf("Unexpected IP length: %v", len(src.IPNet.IP)) + } + + buf = append(buf, family) + + ones, _ := src.IPNet.Mask.Size() + buf = append(buf, byte(ones)) + + // is_cidr is ignored on server + buf = append(buf, 0) + + buf = append(buf, byte(len(src.IPNet.IP))) + + return append(buf, src.IPNet.IP...), nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Inet) Scan(src interface{}) error { + if src == nil { + *dst = Inet{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Inet) Value() (driver.Value, error) { + return EncodeValueText(src) +} diff --git a/github.com/jackc/pgtype/inet_array.go b/github.com/jackc/pgtype/inet_array.go new file mode 100644 index 0000000000..a663d51dab --- /dev/null +++ b/github.com/jackc/pgtype/inet_array.go @@ -0,0 +1,349 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + "net" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type InetArray struct { + Elements []Inet + Dimensions []ArrayDimension + Status Status +} + +func (dst *InetArray) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = InetArray{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + + case []*net.IPNet: + if value == nil { + *dst = InetArray{Status: Null} + } else if len(value) == 0 { + *dst = InetArray{Status: Present} + } else { + elements := make([]Inet, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = InetArray{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []net.IP: + if value == nil { + *dst = InetArray{Status: Null} + } else if len(value) == 0 { + *dst = InetArray{Status: Present} + } else { + elements := make([]Inet, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = InetArray{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []Inet: + if value == nil { + *dst = InetArray{Status: Null} + } else if len(value) == 0 { + *dst = InetArray{Status: Present} + } else { + *dst = InetArray{ + Elements: value, + Dimensions: []ArrayDimension{{Length: int32(len(value)), LowerBound: 1}}, + Status: Present, + } + } + default: + if originalSrc, ok := underlyingSliceType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to InetArray", value) + } + + return nil +} + +func (dst InetArray) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *InetArray) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + + case *[]*net.IPNet: + *v = make([]*net.IPNet, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[]net.IP: + *v = make([]net.IP, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *InetArray) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = InetArray{Status: Null} + return nil + } + + uta, err := ParseUntypedTextArray(string(src)) + if err != nil { + return err + } + + var elements []Inet + + if len(uta.Elements) > 0 { + elements = make([]Inet, len(uta.Elements)) + + for i, s := range uta.Elements { + var elem Inet + var elemSrc []byte + if s != "NULL" { + elemSrc = []byte(s) + } + err = elem.DecodeText(ci, elemSrc) + if err != nil { + return err + } + + elements[i] = elem + } + } + + *dst = InetArray{Elements: elements, Dimensions: uta.Dimensions, Status: Present} + + return nil +} + +func (dst *InetArray) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = InetArray{Status: Null} + return nil + } + + var arrayHeader ArrayHeader + rp, err := arrayHeader.DecodeBinary(ci, src) + if err != nil { + return err + } + + if len(arrayHeader.Dimensions) == 0 { + *dst = InetArray{Dimensions: arrayHeader.Dimensions, Status: Present} + return nil + } + + elementCount := arrayHeader.Dimensions[0].Length + for _, d := range arrayHeader.Dimensions[1:] { + elementCount *= d.Length + } + + elements := make([]Inet, elementCount) + + for i := range elements { + elemLen := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + var elemSrc []byte + if elemLen >= 0 { + elemSrc = src[rp : rp+elemLen] + rp += elemLen + } + err = elements[i].DecodeBinary(ci, elemSrc) + if err != nil { + return err + } + } + + *dst = InetArray{Elements: elements, Dimensions: arrayHeader.Dimensions, Status: Present} + return nil +} + +func (src InetArray) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + if len(src.Dimensions) == 0 { + return append(buf, '{', '}'), nil + } + + buf = EncodeTextArrayDimensions(buf, src.Dimensions) + + // dimElemCounts is the multiples of elements that each array lies on. For + // example, a single dimension array of length 4 would have a dimElemCounts of + // [4]. A multi-dimensional array of lengths [3,5,2] would have a + // dimElemCounts of [30,10,2]. This is used to simplify when to render a '{' + // or '}'. + dimElemCounts := make([]int, len(src.Dimensions)) + dimElemCounts[len(src.Dimensions)-1] = int(src.Dimensions[len(src.Dimensions)-1].Length) + for i := len(src.Dimensions) - 2; i > -1; i-- { + dimElemCounts[i] = int(src.Dimensions[i].Length) * dimElemCounts[i+1] + } + + inElemBuf := make([]byte, 0, 32) + for i, elem := range src.Elements { + if i > 0 { + buf = append(buf, ',') + } + + for _, dec := range dimElemCounts { + if i%dec == 0 { + buf = append(buf, '{') + } + } + + elemBuf, err := elem.EncodeText(ci, inElemBuf) + if err != nil { + return nil, err + } + if elemBuf == nil { + buf = append(buf, `NULL`...) + } else { + buf = append(buf, QuoteArrayElementIfNeeded(string(elemBuf))...) + } + + for _, dec := range dimElemCounts { + if (i+1)%dec == 0 { + buf = append(buf, '}') + } + } + } + + return buf, nil +} + +func (src InetArray) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + arrayHeader := ArrayHeader{ + Dimensions: src.Dimensions, + } + + if dt, ok := ci.DataTypeForName("inet"); ok { + arrayHeader.ElementOID = int32(dt.OID) + } else { + return nil, errors.Errorf("unable to find oid for type name %v", "inet") + } + + for i := range src.Elements { + if src.Elements[i].Status == Null { + arrayHeader.ContainsNull = true + break + } + } + + buf = arrayHeader.EncodeBinary(ci, buf) + + for i := range src.Elements { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + elemBuf, err := src.Elements[i].EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if elemBuf != nil { + buf = elemBuf + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *InetArray) Scan(src interface{}) error { + if src == nil { + return dst.DecodeText(nil, nil) + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src InetArray) Value() (driver.Value, error) { + buf, err := src.EncodeText(nil, nil) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + + return string(buf), nil +} diff --git a/github.com/jackc/pgtype/int2.go b/github.com/jackc/pgtype/int2.go new file mode 100644 index 0000000000..54bab272a7 --- /dev/null +++ b/github.com/jackc/pgtype/int2.go @@ -0,0 +1,216 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + "math" + "strconv" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type Int2 struct { + Int int16 + Status Status +} + +func (dst *Int2) Set(src interface{}) error { + if src == nil { + *dst = Int2{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + case int8: + *dst = Int2{Int: int16(value), Status: Present} + case uint8: + *dst = Int2{Int: int16(value), Status: Present} + case int16: + *dst = Int2{Int: int16(value), Status: Present} + case uint16: + if value > math.MaxInt16 { + return errors.Errorf("%d is greater than maximum value for Int2", value) + } + *dst = Int2{Int: int16(value), Status: Present} + case int32: + if value < math.MinInt16 { + return errors.Errorf("%d is greater than maximum value for Int2", value) + } + if value > math.MaxInt16 { + return errors.Errorf("%d is greater than maximum value for Int2", value) + } + *dst = Int2{Int: int16(value), Status: Present} + case uint32: + if value > math.MaxInt16 { + return errors.Errorf("%d is greater than maximum value for Int2", value) + } + *dst = Int2{Int: int16(value), Status: Present} + case int64: + if value < math.MinInt16 { + return errors.Errorf("%d is greater than maximum value for Int2", value) + } + if value > math.MaxInt16 { + return errors.Errorf("%d is greater than maximum value for Int2", value) + } + *dst = Int2{Int: int16(value), Status: Present} + case uint64: + if value > math.MaxInt16 { + return errors.Errorf("%d is greater than maximum value for Int2", value) + } + *dst = Int2{Int: int16(value), Status: Present} + case int: + if value < math.MinInt16 { + return errors.Errorf("%d is greater than maximum value for Int2", value) + } + if value > math.MaxInt16 { + return errors.Errorf("%d is greater than maximum value for Int2", value) + } + *dst = Int2{Int: int16(value), Status: Present} + case uint: + if value > math.MaxInt16 { + return errors.Errorf("%d is greater than maximum value for Int2", value) + } + *dst = Int2{Int: int16(value), Status: Present} + case string: + num, err := strconv.ParseInt(value, 10, 16) + if err != nil { + return err + } + *dst = Int2{Int: int16(num), Status: Present} + default: + if originalSrc, ok := underlyingNumberType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to Int2", value) + } + + return nil +} + +func (dst Int2) Get() interface{} { + switch dst.Status { + case Present: + return dst.Int + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Int2) AssignTo(dst interface{}) error { + return int64AssignTo(int64(src.Int), src.Status, dst) +} + +func (dst *Int2) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Int2{Status: Null} + return nil + } + + n, err := strconv.ParseInt(string(src), 10, 16) + if err != nil { + return err + } + + *dst = Int2{Int: int16(n), Status: Present} + return nil +} + +func (dst *Int2) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Int2{Status: Null} + return nil + } + + if len(src) != 2 { + return errors.Errorf("invalid length for int2: %v", len(src)) + } + + n := int16(binary.BigEndian.Uint16(src)) + *dst = Int2{Int: n, Status: Present} + return nil +} + +func (src Int2) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + return append(buf, strconv.FormatInt(int64(src.Int), 10)...), nil +} + +func (src Int2) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + return pgio.AppendInt16(buf, src.Int), nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Int2) Scan(src interface{}) error { + if src == nil { + *dst = Int2{Status: Null} + return nil + } + + switch src := src.(type) { + case int64: + if src < math.MinInt16 { + return errors.Errorf("%d is greater than maximum value for Int2", src) + } + if src > math.MaxInt16 { + return errors.Errorf("%d is greater than maximum value for Int2", src) + } + *dst = Int2{Int: int16(src), Status: Present} + return nil + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Int2) Value() (driver.Value, error) { + switch src.Status { + case Present: + return int64(src.Int), nil + case Null: + return nil, nil + default: + return nil, errUndefined + } +} + +func (src Int2) MarshalJSON() ([]byte, error) { + switch src.Status { + case Present: + return []byte(strconv.FormatInt(int64(src.Int), 10)), nil + case Null: + return []byte("null"), nil + case Undefined: + return nil, errUndefined + } + + return nil, errBadStatus +} diff --git a/github.com/jackc/pgtype/int2_array.go b/github.com/jackc/pgtype/int2_array.go new file mode 100644 index 0000000000..9855217195 --- /dev/null +++ b/github.com/jackc/pgtype/int2_array.go @@ -0,0 +1,516 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type Int2Array struct { + Elements []Int2 + Dimensions []ArrayDimension + Status Status +} + +func (dst *Int2Array) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = Int2Array{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + + case []int16: + if value == nil { + *dst = Int2Array{Status: Null} + } else if len(value) == 0 { + *dst = Int2Array{Status: Present} + } else { + elements := make([]Int2, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = Int2Array{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []uint16: + if value == nil { + *dst = Int2Array{Status: Null} + } else if len(value) == 0 { + *dst = Int2Array{Status: Present} + } else { + elements := make([]Int2, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = Int2Array{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []int32: + if value == nil { + *dst = Int2Array{Status: Null} + } else if len(value) == 0 { + *dst = Int2Array{Status: Present} + } else { + elements := make([]Int2, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = Int2Array{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []uint32: + if value == nil { + *dst = Int2Array{Status: Null} + } else if len(value) == 0 { + *dst = Int2Array{Status: Present} + } else { + elements := make([]Int2, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = Int2Array{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []int64: + if value == nil { + *dst = Int2Array{Status: Null} + } else if len(value) == 0 { + *dst = Int2Array{Status: Present} + } else { + elements := make([]Int2, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = Int2Array{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []uint64: + if value == nil { + *dst = Int2Array{Status: Null} + } else if len(value) == 0 { + *dst = Int2Array{Status: Present} + } else { + elements := make([]Int2, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = Int2Array{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []int: + if value == nil { + *dst = Int2Array{Status: Null} + } else if len(value) == 0 { + *dst = Int2Array{Status: Present} + } else { + elements := make([]Int2, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = Int2Array{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []uint: + if value == nil { + *dst = Int2Array{Status: Null} + } else if len(value) == 0 { + *dst = Int2Array{Status: Present} + } else { + elements := make([]Int2, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = Int2Array{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []Int2: + if value == nil { + *dst = Int2Array{Status: Null} + } else if len(value) == 0 { + *dst = Int2Array{Status: Present} + } else { + *dst = Int2Array{ + Elements: value, + Dimensions: []ArrayDimension{{Length: int32(len(value)), LowerBound: 1}}, + Status: Present, + } + } + default: + if originalSrc, ok := underlyingSliceType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to Int2Array", value) + } + + return nil +} + +func (dst Int2Array) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Int2Array) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + + case *[]int16: + *v = make([]int16, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[]uint16: + *v = make([]uint16, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[]int32: + *v = make([]int32, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[]uint32: + *v = make([]uint32, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[]int64: + *v = make([]int64, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[]uint64: + *v = make([]uint64, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[]int: + *v = make([]int, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[]uint: + *v = make([]uint, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *Int2Array) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Int2Array{Status: Null} + return nil + } + + uta, err := ParseUntypedTextArray(string(src)) + if err != nil { + return err + } + + var elements []Int2 + + if len(uta.Elements) > 0 { + elements = make([]Int2, len(uta.Elements)) + + for i, s := range uta.Elements { + var elem Int2 + var elemSrc []byte + if s != "NULL" { + elemSrc = []byte(s) + } + err = elem.DecodeText(ci, elemSrc) + if err != nil { + return err + } + + elements[i] = elem + } + } + + *dst = Int2Array{Elements: elements, Dimensions: uta.Dimensions, Status: Present} + + return nil +} + +func (dst *Int2Array) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Int2Array{Status: Null} + return nil + } + + var arrayHeader ArrayHeader + rp, err := arrayHeader.DecodeBinary(ci, src) + if err != nil { + return err + } + + if len(arrayHeader.Dimensions) == 0 { + *dst = Int2Array{Dimensions: arrayHeader.Dimensions, Status: Present} + return nil + } + + elementCount := arrayHeader.Dimensions[0].Length + for _, d := range arrayHeader.Dimensions[1:] { + elementCount *= d.Length + } + + elements := make([]Int2, elementCount) + + for i := range elements { + elemLen := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + var elemSrc []byte + if elemLen >= 0 { + elemSrc = src[rp : rp+elemLen] + rp += elemLen + } + err = elements[i].DecodeBinary(ci, elemSrc) + if err != nil { + return err + } + } + + *dst = Int2Array{Elements: elements, Dimensions: arrayHeader.Dimensions, Status: Present} + return nil +} + +func (src Int2Array) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + if len(src.Dimensions) == 0 { + return append(buf, '{', '}'), nil + } + + buf = EncodeTextArrayDimensions(buf, src.Dimensions) + + // dimElemCounts is the multiples of elements that each array lies on. For + // example, a single dimension array of length 4 would have a dimElemCounts of + // [4]. A multi-dimensional array of lengths [3,5,2] would have a + // dimElemCounts of [30,10,2]. This is used to simplify when to render a '{' + // or '}'. + dimElemCounts := make([]int, len(src.Dimensions)) + dimElemCounts[len(src.Dimensions)-1] = int(src.Dimensions[len(src.Dimensions)-1].Length) + for i := len(src.Dimensions) - 2; i > -1; i-- { + dimElemCounts[i] = int(src.Dimensions[i].Length) * dimElemCounts[i+1] + } + + inElemBuf := make([]byte, 0, 32) + for i, elem := range src.Elements { + if i > 0 { + buf = append(buf, ',') + } + + for _, dec := range dimElemCounts { + if i%dec == 0 { + buf = append(buf, '{') + } + } + + elemBuf, err := elem.EncodeText(ci, inElemBuf) + if err != nil { + return nil, err + } + if elemBuf == nil { + buf = append(buf, `NULL`...) + } else { + buf = append(buf, QuoteArrayElementIfNeeded(string(elemBuf))...) + } + + for _, dec := range dimElemCounts { + if (i+1)%dec == 0 { + buf = append(buf, '}') + } + } + } + + return buf, nil +} + +func (src Int2Array) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + arrayHeader := ArrayHeader{ + Dimensions: src.Dimensions, + } + + if dt, ok := ci.DataTypeForName("int2"); ok { + arrayHeader.ElementOID = int32(dt.OID) + } else { + return nil, errors.Errorf("unable to find oid for type name %v", "int2") + } + + for i := range src.Elements { + if src.Elements[i].Status == Null { + arrayHeader.ContainsNull = true + break + } + } + + buf = arrayHeader.EncodeBinary(ci, buf) + + for i := range src.Elements { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + elemBuf, err := src.Elements[i].EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if elemBuf != nil { + buf = elemBuf + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Int2Array) Scan(src interface{}) error { + if src == nil { + return dst.DecodeText(nil, nil) + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Int2Array) Value() (driver.Value, error) { + buf, err := src.EncodeText(nil, nil) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + + return string(buf), nil +} diff --git a/github.com/jackc/pgtype/int4.go b/github.com/jackc/pgtype/int4.go new file mode 100644 index 0000000000..66fe91551d --- /dev/null +++ b/github.com/jackc/pgtype/int4.go @@ -0,0 +1,224 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + "encoding/json" + "math" + "strconv" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type Int4 struct { + Int int32 + Status Status +} + +func (dst *Int4) Set(src interface{}) error { + if src == nil { + *dst = Int4{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + case int8: + *dst = Int4{Int: int32(value), Status: Present} + case uint8: + *dst = Int4{Int: int32(value), Status: Present} + case int16: + *dst = Int4{Int: int32(value), Status: Present} + case uint16: + *dst = Int4{Int: int32(value), Status: Present} + case int32: + *dst = Int4{Int: int32(value), Status: Present} + case uint32: + if value > math.MaxInt32 { + return errors.Errorf("%d is greater than maximum value for Int4", value) + } + *dst = Int4{Int: int32(value), Status: Present} + case int64: + if value < math.MinInt32 { + return errors.Errorf("%d is greater than maximum value for Int4", value) + } + if value > math.MaxInt32 { + return errors.Errorf("%d is greater than maximum value for Int4", value) + } + *dst = Int4{Int: int32(value), Status: Present} + case uint64: + if value > math.MaxInt32 { + return errors.Errorf("%d is greater than maximum value for Int4", value) + } + *dst = Int4{Int: int32(value), Status: Present} + case int: + if value < math.MinInt32 { + return errors.Errorf("%d is greater than maximum value for Int4", value) + } + if value > math.MaxInt32 { + return errors.Errorf("%d is greater than maximum value for Int4", value) + } + *dst = Int4{Int: int32(value), Status: Present} + case uint: + if value > math.MaxInt32 { + return errors.Errorf("%d is greater than maximum value for Int4", value) + } + *dst = Int4{Int: int32(value), Status: Present} + case string: + num, err := strconv.ParseInt(value, 10, 32) + if err != nil { + return err + } + *dst = Int4{Int: int32(num), Status: Present} + default: + if originalSrc, ok := underlyingNumberType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to Int4", value) + } + + return nil +} + +func (dst Int4) Get() interface{} { + switch dst.Status { + case Present: + return dst.Int + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Int4) AssignTo(dst interface{}) error { + return int64AssignTo(int64(src.Int), src.Status, dst) +} + +func (dst *Int4) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Int4{Status: Null} + return nil + } + + n, err := strconv.ParseInt(string(src), 10, 32) + if err != nil { + return err + } + + *dst = Int4{Int: int32(n), Status: Present} + return nil +} + +func (dst *Int4) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Int4{Status: Null} + return nil + } + + if len(src) != 4 { + return errors.Errorf("invalid length for int4: %v", len(src)) + } + + n := int32(binary.BigEndian.Uint32(src)) + *dst = Int4{Int: n, Status: Present} + return nil +} + +func (src Int4) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + return append(buf, strconv.FormatInt(int64(src.Int), 10)...), nil +} + +func (src Int4) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + return pgio.AppendInt32(buf, src.Int), nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Int4) Scan(src interface{}) error { + if src == nil { + *dst = Int4{Status: Null} + return nil + } + + switch src := src.(type) { + case int64: + if src < math.MinInt32 { + return errors.Errorf("%d is greater than maximum value for Int4", src) + } + if src > math.MaxInt32 { + return errors.Errorf("%d is greater than maximum value for Int4", src) + } + *dst = Int4{Int: int32(src), Status: Present} + return nil + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Int4) Value() (driver.Value, error) { + switch src.Status { + case Present: + return int64(src.Int), nil + case Null: + return nil, nil + default: + return nil, errUndefined + } +} + +func (src Int4) MarshalJSON() ([]byte, error) { + switch src.Status { + case Present: + return []byte(strconv.FormatInt(int64(src.Int), 10)), nil + case Null: + return []byte("null"), nil + case Undefined: + return nil, errUndefined + } + + return nil, errBadStatus +} + +func (dst *Int4) UnmarshalJSON(b []byte) error { + var n *int32 + err := json.Unmarshal(b, &n) + if err != nil { + return err + } + + if n == nil { + *dst = Int4{Status: Null} + } else { + *dst = Int4{Int: *n, Status: Present} + } + + return nil +} diff --git a/github.com/jackc/pgtype/int4_array.go b/github.com/jackc/pgtype/int4_array.go new file mode 100644 index 0000000000..a52ab437fd --- /dev/null +++ b/github.com/jackc/pgtype/int4_array.go @@ -0,0 +1,516 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type Int4Array struct { + Elements []Int4 + Dimensions []ArrayDimension + Status Status +} + +func (dst *Int4Array) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = Int4Array{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + + case []int16: + if value == nil { + *dst = Int4Array{Status: Null} + } else if len(value) == 0 { + *dst = Int4Array{Status: Present} + } else { + elements := make([]Int4, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = Int4Array{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []uint16: + if value == nil { + *dst = Int4Array{Status: Null} + } else if len(value) == 0 { + *dst = Int4Array{Status: Present} + } else { + elements := make([]Int4, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = Int4Array{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []int32: + if value == nil { + *dst = Int4Array{Status: Null} + } else if len(value) == 0 { + *dst = Int4Array{Status: Present} + } else { + elements := make([]Int4, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = Int4Array{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []uint32: + if value == nil { + *dst = Int4Array{Status: Null} + } else if len(value) == 0 { + *dst = Int4Array{Status: Present} + } else { + elements := make([]Int4, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = Int4Array{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []int64: + if value == nil { + *dst = Int4Array{Status: Null} + } else if len(value) == 0 { + *dst = Int4Array{Status: Present} + } else { + elements := make([]Int4, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = Int4Array{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []uint64: + if value == nil { + *dst = Int4Array{Status: Null} + } else if len(value) == 0 { + *dst = Int4Array{Status: Present} + } else { + elements := make([]Int4, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = Int4Array{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []int: + if value == nil { + *dst = Int4Array{Status: Null} + } else if len(value) == 0 { + *dst = Int4Array{Status: Present} + } else { + elements := make([]Int4, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = Int4Array{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []uint: + if value == nil { + *dst = Int4Array{Status: Null} + } else if len(value) == 0 { + *dst = Int4Array{Status: Present} + } else { + elements := make([]Int4, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = Int4Array{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []Int4: + if value == nil { + *dst = Int4Array{Status: Null} + } else if len(value) == 0 { + *dst = Int4Array{Status: Present} + } else { + *dst = Int4Array{ + Elements: value, + Dimensions: []ArrayDimension{{Length: int32(len(value)), LowerBound: 1}}, + Status: Present, + } + } + default: + if originalSrc, ok := underlyingSliceType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to Int4Array", value) + } + + return nil +} + +func (dst Int4Array) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Int4Array) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + + case *[]int16: + *v = make([]int16, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[]uint16: + *v = make([]uint16, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[]int32: + *v = make([]int32, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[]uint32: + *v = make([]uint32, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[]int64: + *v = make([]int64, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[]uint64: + *v = make([]uint64, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[]int: + *v = make([]int, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[]uint: + *v = make([]uint, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *Int4Array) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Int4Array{Status: Null} + return nil + } + + uta, err := ParseUntypedTextArray(string(src)) + if err != nil { + return err + } + + var elements []Int4 + + if len(uta.Elements) > 0 { + elements = make([]Int4, len(uta.Elements)) + + for i, s := range uta.Elements { + var elem Int4 + var elemSrc []byte + if s != "NULL" { + elemSrc = []byte(s) + } + err = elem.DecodeText(ci, elemSrc) + if err != nil { + return err + } + + elements[i] = elem + } + } + + *dst = Int4Array{Elements: elements, Dimensions: uta.Dimensions, Status: Present} + + return nil +} + +func (dst *Int4Array) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Int4Array{Status: Null} + return nil + } + + var arrayHeader ArrayHeader + rp, err := arrayHeader.DecodeBinary(ci, src) + if err != nil { + return err + } + + if len(arrayHeader.Dimensions) == 0 { + *dst = Int4Array{Dimensions: arrayHeader.Dimensions, Status: Present} + return nil + } + + elementCount := arrayHeader.Dimensions[0].Length + for _, d := range arrayHeader.Dimensions[1:] { + elementCount *= d.Length + } + + elements := make([]Int4, elementCount) + + for i := range elements { + elemLen := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + var elemSrc []byte + if elemLen >= 0 { + elemSrc = src[rp : rp+elemLen] + rp += elemLen + } + err = elements[i].DecodeBinary(ci, elemSrc) + if err != nil { + return err + } + } + + *dst = Int4Array{Elements: elements, Dimensions: arrayHeader.Dimensions, Status: Present} + return nil +} + +func (src Int4Array) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + if len(src.Dimensions) == 0 { + return append(buf, '{', '}'), nil + } + + buf = EncodeTextArrayDimensions(buf, src.Dimensions) + + // dimElemCounts is the multiples of elements that each array lies on. For + // example, a single dimension array of length 4 would have a dimElemCounts of + // [4]. A multi-dimensional array of lengths [3,5,2] would have a + // dimElemCounts of [30,10,2]. This is used to simplify when to render a '{' + // or '}'. + dimElemCounts := make([]int, len(src.Dimensions)) + dimElemCounts[len(src.Dimensions)-1] = int(src.Dimensions[len(src.Dimensions)-1].Length) + for i := len(src.Dimensions) - 2; i > -1; i-- { + dimElemCounts[i] = int(src.Dimensions[i].Length) * dimElemCounts[i+1] + } + + inElemBuf := make([]byte, 0, 32) + for i, elem := range src.Elements { + if i > 0 { + buf = append(buf, ',') + } + + for _, dec := range dimElemCounts { + if i%dec == 0 { + buf = append(buf, '{') + } + } + + elemBuf, err := elem.EncodeText(ci, inElemBuf) + if err != nil { + return nil, err + } + if elemBuf == nil { + buf = append(buf, `NULL`...) + } else { + buf = append(buf, QuoteArrayElementIfNeeded(string(elemBuf))...) + } + + for _, dec := range dimElemCounts { + if (i+1)%dec == 0 { + buf = append(buf, '}') + } + } + } + + return buf, nil +} + +func (src Int4Array) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + arrayHeader := ArrayHeader{ + Dimensions: src.Dimensions, + } + + if dt, ok := ci.DataTypeForName("int4"); ok { + arrayHeader.ElementOID = int32(dt.OID) + } else { + return nil, errors.Errorf("unable to find oid for type name %v", "int4") + } + + for i := range src.Elements { + if src.Elements[i].Status == Null { + arrayHeader.ContainsNull = true + break + } + } + + buf = arrayHeader.EncodeBinary(ci, buf) + + for i := range src.Elements { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + elemBuf, err := src.Elements[i].EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if elemBuf != nil { + buf = elemBuf + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Int4Array) Scan(src interface{}) error { + if src == nil { + return dst.DecodeText(nil, nil) + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Int4Array) Value() (driver.Value, error) { + buf, err := src.EncodeText(nil, nil) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + + return string(buf), nil +} diff --git a/github.com/jackc/pgtype/int4range.go b/github.com/jackc/pgtype/int4range.go new file mode 100644 index 0000000000..442f2501b1 --- /dev/null +++ b/github.com/jackc/pgtype/int4range.go @@ -0,0 +1,267 @@ +package pgtype + +import ( + "database/sql/driver" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type Int4range struct { + Lower Int4 + Upper Int4 + LowerType BoundType + UpperType BoundType + Status Status +} + +func (dst *Int4range) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = Int4range{Status: Null} + return nil + } + + switch value := src.(type) { + case Int4range: + *dst = value + case *Int4range: + *dst = *value + case string: + return dst.DecodeText(nil, []byte(value)) + default: + return errors.Errorf("cannot convert %v to Int4range", src) + } + + return nil +} + +func (dst Int4range) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Int4range) AssignTo(dst interface{}) error { + return errors.Errorf("cannot assign %v to %T", src, dst) +} + +func (dst *Int4range) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Int4range{Status: Null} + return nil + } + + utr, err := ParseUntypedTextRange(string(src)) + if err != nil { + return err + } + + *dst = Int4range{Status: Present} + + dst.LowerType = utr.LowerType + dst.UpperType = utr.UpperType + + if dst.LowerType == Empty { + return nil + } + + if dst.LowerType == Inclusive || dst.LowerType == Exclusive { + if err := dst.Lower.DecodeText(ci, []byte(utr.Lower)); err != nil { + return err + } + } + + if dst.UpperType == Inclusive || dst.UpperType == Exclusive { + if err := dst.Upper.DecodeText(ci, []byte(utr.Upper)); err != nil { + return err + } + } + + return nil +} + +func (dst *Int4range) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Int4range{Status: Null} + return nil + } + + ubr, err := ParseUntypedBinaryRange(src) + if err != nil { + return err + } + + *dst = Int4range{Status: Present} + + dst.LowerType = ubr.LowerType + dst.UpperType = ubr.UpperType + + if dst.LowerType == Empty { + return nil + } + + if dst.LowerType == Inclusive || dst.LowerType == Exclusive { + if err := dst.Lower.DecodeBinary(ci, ubr.Lower); err != nil { + return err + } + } + + if dst.UpperType == Inclusive || dst.UpperType == Exclusive { + if err := dst.Upper.DecodeBinary(ci, ubr.Upper); err != nil { + return err + } + } + + return nil +} + +func (src Int4range) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + switch src.LowerType { + case Exclusive, Unbounded: + buf = append(buf, '(') + case Inclusive: + buf = append(buf, '[') + case Empty: + return append(buf, "empty"...), nil + default: + return nil, errors.Errorf("unknown lower bound type %v", src.LowerType) + } + + var err error + + if src.LowerType != Unbounded { + buf, err = src.Lower.EncodeText(ci, buf) + if err != nil { + return nil, err + } else if buf == nil { + return nil, errors.Errorf("Lower cannot be null unless LowerType is Unbounded") + } + } + + buf = append(buf, ',') + + if src.UpperType != Unbounded { + buf, err = src.Upper.EncodeText(ci, buf) + if err != nil { + return nil, err + } else if buf == nil { + return nil, errors.Errorf("Upper cannot be null unless UpperType is Unbounded") + } + } + + switch src.UpperType { + case Exclusive, Unbounded: + buf = append(buf, ')') + case Inclusive: + buf = append(buf, ']') + default: + return nil, errors.Errorf("unknown upper bound type %v", src.UpperType) + } + + return buf, nil +} + +func (src Int4range) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + var rangeType byte + switch src.LowerType { + case Inclusive: + rangeType |= lowerInclusiveMask + case Unbounded: + rangeType |= lowerUnboundedMask + case Exclusive: + case Empty: + return append(buf, emptyMask), nil + default: + return nil, errors.Errorf("unknown LowerType: %v", src.LowerType) + } + + switch src.UpperType { + case Inclusive: + rangeType |= upperInclusiveMask + case Unbounded: + rangeType |= upperUnboundedMask + case Exclusive: + default: + return nil, errors.Errorf("unknown UpperType: %v", src.UpperType) + } + + buf = append(buf, rangeType) + + var err error + + if src.LowerType != Unbounded { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + buf, err = src.Lower.EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if buf == nil { + return nil, errors.Errorf("Lower cannot be null unless LowerType is Unbounded") + } + + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + + if src.UpperType != Unbounded { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + buf, err = src.Upper.EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if buf == nil { + return nil, errors.Errorf("Upper cannot be null unless UpperType is Unbounded") + } + + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Int4range) Scan(src interface{}) error { + if src == nil { + *dst = Int4range{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Int4range) Value() (driver.Value, error) { + return EncodeValueText(src) +} diff --git a/github.com/jackc/pgtype/int8.go b/github.com/jackc/pgtype/int8.go new file mode 100644 index 0000000000..fd72114207 --- /dev/null +++ b/github.com/jackc/pgtype/int8.go @@ -0,0 +1,210 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + "encoding/json" + "math" + "strconv" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type Int8 struct { + Int int64 + Status Status +} + +func (dst *Int8) Set(src interface{}) error { + if src == nil { + *dst = Int8{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + case int8: + *dst = Int8{Int: int64(value), Status: Present} + case uint8: + *dst = Int8{Int: int64(value), Status: Present} + case int16: + *dst = Int8{Int: int64(value), Status: Present} + case uint16: + *dst = Int8{Int: int64(value), Status: Present} + case int32: + *dst = Int8{Int: int64(value), Status: Present} + case uint32: + *dst = Int8{Int: int64(value), Status: Present} + case int64: + *dst = Int8{Int: int64(value), Status: Present} + case uint64: + if value > math.MaxInt64 { + return errors.Errorf("%d is greater than maximum value for Int8", value) + } + *dst = Int8{Int: int64(value), Status: Present} + case int: + if int64(value) < math.MinInt64 { + return errors.Errorf("%d is greater than maximum value for Int8", value) + } + if int64(value) > math.MaxInt64 { + return errors.Errorf("%d is greater than maximum value for Int8", value) + } + *dst = Int8{Int: int64(value), Status: Present} + case uint: + if uint64(value) > math.MaxInt64 { + return errors.Errorf("%d is greater than maximum value for Int8", value) + } + *dst = Int8{Int: int64(value), Status: Present} + case string: + num, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return err + } + *dst = Int8{Int: num, Status: Present} + default: + if originalSrc, ok := underlyingNumberType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to Int8", value) + } + + return nil +} + +func (dst Int8) Get() interface{} { + switch dst.Status { + case Present: + return dst.Int + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Int8) AssignTo(dst interface{}) error { + return int64AssignTo(int64(src.Int), src.Status, dst) +} + +func (dst *Int8) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Int8{Status: Null} + return nil + } + + n, err := strconv.ParseInt(string(src), 10, 64) + if err != nil { + return err + } + + *dst = Int8{Int: n, Status: Present} + return nil +} + +func (dst *Int8) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Int8{Status: Null} + return nil + } + + if len(src) != 8 { + return errors.Errorf("invalid length for int8: %v", len(src)) + } + + n := int64(binary.BigEndian.Uint64(src)) + + *dst = Int8{Int: n, Status: Present} + return nil +} + +func (src Int8) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + return append(buf, strconv.FormatInt(src.Int, 10)...), nil +} + +func (src Int8) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + return pgio.AppendInt64(buf, src.Int), nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Int8) Scan(src interface{}) error { + if src == nil { + *dst = Int8{Status: Null} + return nil + } + + switch src := src.(type) { + case int64: + *dst = Int8{Int: src, Status: Present} + return nil + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Int8) Value() (driver.Value, error) { + switch src.Status { + case Present: + return int64(src.Int), nil + case Null: + return nil, nil + default: + return nil, errUndefined + } +} + +func (src Int8) MarshalJSON() ([]byte, error) { + switch src.Status { + case Present: + return []byte(strconv.FormatInt(src.Int, 10)), nil + case Null: + return []byte("null"), nil + case Undefined: + return nil, errUndefined + } + + return nil, errBadStatus +} + +func (dst *Int8) UnmarshalJSON(b []byte) error { + var n *int64 + err := json.Unmarshal(b, &n) + if err != nil { + return err + } + + if n == nil { + *dst = Int8{Status: Null} + } else { + *dst = Int8{Int: *n, Status: Present} + } + + return nil +} diff --git a/github.com/jackc/pgtype/int8_array.go b/github.com/jackc/pgtype/int8_array.go new file mode 100644 index 0000000000..f6d577f0c1 --- /dev/null +++ b/github.com/jackc/pgtype/int8_array.go @@ -0,0 +1,516 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type Int8Array struct { + Elements []Int8 + Dimensions []ArrayDimension + Status Status +} + +func (dst *Int8Array) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = Int8Array{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + + case []int16: + if value == nil { + *dst = Int8Array{Status: Null} + } else if len(value) == 0 { + *dst = Int8Array{Status: Present} + } else { + elements := make([]Int8, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = Int8Array{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []uint16: + if value == nil { + *dst = Int8Array{Status: Null} + } else if len(value) == 0 { + *dst = Int8Array{Status: Present} + } else { + elements := make([]Int8, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = Int8Array{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []int32: + if value == nil { + *dst = Int8Array{Status: Null} + } else if len(value) == 0 { + *dst = Int8Array{Status: Present} + } else { + elements := make([]Int8, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = Int8Array{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []uint32: + if value == nil { + *dst = Int8Array{Status: Null} + } else if len(value) == 0 { + *dst = Int8Array{Status: Present} + } else { + elements := make([]Int8, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = Int8Array{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []int64: + if value == nil { + *dst = Int8Array{Status: Null} + } else if len(value) == 0 { + *dst = Int8Array{Status: Present} + } else { + elements := make([]Int8, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = Int8Array{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []uint64: + if value == nil { + *dst = Int8Array{Status: Null} + } else if len(value) == 0 { + *dst = Int8Array{Status: Present} + } else { + elements := make([]Int8, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = Int8Array{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []int: + if value == nil { + *dst = Int8Array{Status: Null} + } else if len(value) == 0 { + *dst = Int8Array{Status: Present} + } else { + elements := make([]Int8, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = Int8Array{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []uint: + if value == nil { + *dst = Int8Array{Status: Null} + } else if len(value) == 0 { + *dst = Int8Array{Status: Present} + } else { + elements := make([]Int8, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = Int8Array{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []Int8: + if value == nil { + *dst = Int8Array{Status: Null} + } else if len(value) == 0 { + *dst = Int8Array{Status: Present} + } else { + *dst = Int8Array{ + Elements: value, + Dimensions: []ArrayDimension{{Length: int32(len(value)), LowerBound: 1}}, + Status: Present, + } + } + default: + if originalSrc, ok := underlyingSliceType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to Int8Array", value) + } + + return nil +} + +func (dst Int8Array) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Int8Array) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + + case *[]int16: + *v = make([]int16, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[]uint16: + *v = make([]uint16, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[]int32: + *v = make([]int32, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[]uint32: + *v = make([]uint32, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[]int64: + *v = make([]int64, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[]uint64: + *v = make([]uint64, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[]int: + *v = make([]int, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[]uint: + *v = make([]uint, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *Int8Array) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Int8Array{Status: Null} + return nil + } + + uta, err := ParseUntypedTextArray(string(src)) + if err != nil { + return err + } + + var elements []Int8 + + if len(uta.Elements) > 0 { + elements = make([]Int8, len(uta.Elements)) + + for i, s := range uta.Elements { + var elem Int8 + var elemSrc []byte + if s != "NULL" { + elemSrc = []byte(s) + } + err = elem.DecodeText(ci, elemSrc) + if err != nil { + return err + } + + elements[i] = elem + } + } + + *dst = Int8Array{Elements: elements, Dimensions: uta.Dimensions, Status: Present} + + return nil +} + +func (dst *Int8Array) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Int8Array{Status: Null} + return nil + } + + var arrayHeader ArrayHeader + rp, err := arrayHeader.DecodeBinary(ci, src) + if err != nil { + return err + } + + if len(arrayHeader.Dimensions) == 0 { + *dst = Int8Array{Dimensions: arrayHeader.Dimensions, Status: Present} + return nil + } + + elementCount := arrayHeader.Dimensions[0].Length + for _, d := range arrayHeader.Dimensions[1:] { + elementCount *= d.Length + } + + elements := make([]Int8, elementCount) + + for i := range elements { + elemLen := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + var elemSrc []byte + if elemLen >= 0 { + elemSrc = src[rp : rp+elemLen] + rp += elemLen + } + err = elements[i].DecodeBinary(ci, elemSrc) + if err != nil { + return err + } + } + + *dst = Int8Array{Elements: elements, Dimensions: arrayHeader.Dimensions, Status: Present} + return nil +} + +func (src Int8Array) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + if len(src.Dimensions) == 0 { + return append(buf, '{', '}'), nil + } + + buf = EncodeTextArrayDimensions(buf, src.Dimensions) + + // dimElemCounts is the multiples of elements that each array lies on. For + // example, a single dimension array of length 4 would have a dimElemCounts of + // [4]. A multi-dimensional array of lengths [3,5,2] would have a + // dimElemCounts of [30,10,2]. This is used to simplify when to render a '{' + // or '}'. + dimElemCounts := make([]int, len(src.Dimensions)) + dimElemCounts[len(src.Dimensions)-1] = int(src.Dimensions[len(src.Dimensions)-1].Length) + for i := len(src.Dimensions) - 2; i > -1; i-- { + dimElemCounts[i] = int(src.Dimensions[i].Length) * dimElemCounts[i+1] + } + + inElemBuf := make([]byte, 0, 32) + for i, elem := range src.Elements { + if i > 0 { + buf = append(buf, ',') + } + + for _, dec := range dimElemCounts { + if i%dec == 0 { + buf = append(buf, '{') + } + } + + elemBuf, err := elem.EncodeText(ci, inElemBuf) + if err != nil { + return nil, err + } + if elemBuf == nil { + buf = append(buf, `NULL`...) + } else { + buf = append(buf, QuoteArrayElementIfNeeded(string(elemBuf))...) + } + + for _, dec := range dimElemCounts { + if (i+1)%dec == 0 { + buf = append(buf, '}') + } + } + } + + return buf, nil +} + +func (src Int8Array) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + arrayHeader := ArrayHeader{ + Dimensions: src.Dimensions, + } + + if dt, ok := ci.DataTypeForName("int8"); ok { + arrayHeader.ElementOID = int32(dt.OID) + } else { + return nil, errors.Errorf("unable to find oid for type name %v", "int8") + } + + for i := range src.Elements { + if src.Elements[i].Status == Null { + arrayHeader.ContainsNull = true + break + } + } + + buf = arrayHeader.EncodeBinary(ci, buf) + + for i := range src.Elements { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + elemBuf, err := src.Elements[i].EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if elemBuf != nil { + buf = elemBuf + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Int8Array) Scan(src interface{}) error { + if src == nil { + return dst.DecodeText(nil, nil) + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Int8Array) Value() (driver.Value, error) { + buf, err := src.EncodeText(nil, nil) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + + return string(buf), nil +} diff --git a/github.com/jackc/pgtype/int8range.go b/github.com/jackc/pgtype/int8range.go new file mode 100644 index 0000000000..92fcb13649 --- /dev/null +++ b/github.com/jackc/pgtype/int8range.go @@ -0,0 +1,267 @@ +package pgtype + +import ( + "database/sql/driver" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type Int8range struct { + Lower Int8 + Upper Int8 + LowerType BoundType + UpperType BoundType + Status Status +} + +func (dst *Int8range) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = Int8range{Status: Null} + return nil + } + + switch value := src.(type) { + case Int8range: + *dst = value + case *Int8range: + *dst = *value + case string: + return dst.DecodeText(nil, []byte(value)) + default: + return errors.Errorf("cannot convert %v to Int8range", src) + } + + return nil +} + +func (dst Int8range) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Int8range) AssignTo(dst interface{}) error { + return errors.Errorf("cannot assign %v to %T", src, dst) +} + +func (dst *Int8range) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Int8range{Status: Null} + return nil + } + + utr, err := ParseUntypedTextRange(string(src)) + if err != nil { + return err + } + + *dst = Int8range{Status: Present} + + dst.LowerType = utr.LowerType + dst.UpperType = utr.UpperType + + if dst.LowerType == Empty { + return nil + } + + if dst.LowerType == Inclusive || dst.LowerType == Exclusive { + if err := dst.Lower.DecodeText(ci, []byte(utr.Lower)); err != nil { + return err + } + } + + if dst.UpperType == Inclusive || dst.UpperType == Exclusive { + if err := dst.Upper.DecodeText(ci, []byte(utr.Upper)); err != nil { + return err + } + } + + return nil +} + +func (dst *Int8range) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Int8range{Status: Null} + return nil + } + + ubr, err := ParseUntypedBinaryRange(src) + if err != nil { + return err + } + + *dst = Int8range{Status: Present} + + dst.LowerType = ubr.LowerType + dst.UpperType = ubr.UpperType + + if dst.LowerType == Empty { + return nil + } + + if dst.LowerType == Inclusive || dst.LowerType == Exclusive { + if err := dst.Lower.DecodeBinary(ci, ubr.Lower); err != nil { + return err + } + } + + if dst.UpperType == Inclusive || dst.UpperType == Exclusive { + if err := dst.Upper.DecodeBinary(ci, ubr.Upper); err != nil { + return err + } + } + + return nil +} + +func (src Int8range) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + switch src.LowerType { + case Exclusive, Unbounded: + buf = append(buf, '(') + case Inclusive: + buf = append(buf, '[') + case Empty: + return append(buf, "empty"...), nil + default: + return nil, errors.Errorf("unknown lower bound type %v", src.LowerType) + } + + var err error + + if src.LowerType != Unbounded { + buf, err = src.Lower.EncodeText(ci, buf) + if err != nil { + return nil, err + } else if buf == nil { + return nil, errors.Errorf("Lower cannot be null unless LowerType is Unbounded") + } + } + + buf = append(buf, ',') + + if src.UpperType != Unbounded { + buf, err = src.Upper.EncodeText(ci, buf) + if err != nil { + return nil, err + } else if buf == nil { + return nil, errors.Errorf("Upper cannot be null unless UpperType is Unbounded") + } + } + + switch src.UpperType { + case Exclusive, Unbounded: + buf = append(buf, ')') + case Inclusive: + buf = append(buf, ']') + default: + return nil, errors.Errorf("unknown upper bound type %v", src.UpperType) + } + + return buf, nil +} + +func (src Int8range) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + var rangeType byte + switch src.LowerType { + case Inclusive: + rangeType |= lowerInclusiveMask + case Unbounded: + rangeType |= lowerUnboundedMask + case Exclusive: + case Empty: + return append(buf, emptyMask), nil + default: + return nil, errors.Errorf("unknown LowerType: %v", src.LowerType) + } + + switch src.UpperType { + case Inclusive: + rangeType |= upperInclusiveMask + case Unbounded: + rangeType |= upperUnboundedMask + case Exclusive: + default: + return nil, errors.Errorf("unknown UpperType: %v", src.UpperType) + } + + buf = append(buf, rangeType) + + var err error + + if src.LowerType != Unbounded { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + buf, err = src.Lower.EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if buf == nil { + return nil, errors.Errorf("Lower cannot be null unless LowerType is Unbounded") + } + + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + + if src.UpperType != Unbounded { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + buf, err = src.Upper.EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if buf == nil { + return nil, errors.Errorf("Upper cannot be null unless UpperType is Unbounded") + } + + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Int8range) Scan(src interface{}) error { + if src == nil { + *dst = Int8range{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Int8range) Value() (driver.Value, error) { + return EncodeValueText(src) +} diff --git a/github.com/jackc/pgtype/interval.go b/github.com/jackc/pgtype/interval.go new file mode 100644 index 0000000000..3a91c595d8 --- /dev/null +++ b/github.com/jackc/pgtype/interval.go @@ -0,0 +1,258 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + "fmt" + "strconv" + "strings" + "time" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +const ( + microsecondsPerSecond = 1000000 + microsecondsPerMinute = 60 * microsecondsPerSecond + microsecondsPerHour = 60 * microsecondsPerMinute +) + +type Interval struct { + Microseconds int64 + Days int32 + Months int32 + Status Status +} + +func (dst *Interval) Set(src interface{}) error { + if src == nil { + *dst = Interval{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + case time.Duration: + *dst = Interval{Microseconds: int64(value) / 1000, Status: Present} + default: + if originalSrc, ok := underlyingPtrType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to Interval", value) + } + + return nil +} + +func (dst Interval) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Interval) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + case *time.Duration: + if src.Days > 0 || src.Months > 0 { + return errors.Errorf("interval with months or days cannot be decoded into %T", dst) + } + *v = time.Duration(src.Microseconds) * time.Microsecond + return nil + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *Interval) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Interval{Status: Null} + return nil + } + + var microseconds int64 + var days int32 + var months int32 + + parts := strings.Split(string(src), " ") + + for i := 0; i < len(parts)-1; i += 2 { + scalar, err := strconv.ParseInt(parts[i], 10, 64) + if err != nil { + return errors.Errorf("bad interval format") + } + + switch parts[i+1] { + case "year", "years": + months += int32(scalar * 12) + case "mon", "mons": + months += int32(scalar) + case "day", "days": + days = int32(scalar) + } + } + + if len(parts)%2 == 1 { + timeParts := strings.SplitN(parts[len(parts)-1], ":", 3) + if len(timeParts) != 3 { + return errors.Errorf("bad interval format") + } + + var negative bool + if timeParts[0][0] == '-' { + negative = true + timeParts[0] = timeParts[0][1:] + } + + hours, err := strconv.ParseInt(timeParts[0], 10, 64) + if err != nil { + return errors.Errorf("bad interval hour format: %s", timeParts[0]) + } + + minutes, err := strconv.ParseInt(timeParts[1], 10, 64) + if err != nil { + return errors.Errorf("bad interval minute format: %s", timeParts[1]) + } + + secondParts := strings.SplitN(timeParts[2], ".", 2) + + seconds, err := strconv.ParseInt(secondParts[0], 10, 64) + if err != nil { + return errors.Errorf("bad interval second format: %s", secondParts[0]) + } + + var uSeconds int64 + if len(secondParts) == 2 { + uSeconds, err = strconv.ParseInt(secondParts[1], 10, 64) + if err != nil { + return errors.Errorf("bad interval decimal format: %s", secondParts[1]) + } + + for i := 0; i < 6-len(secondParts[1]); i++ { + uSeconds *= 10 + } + } + + microseconds = hours * microsecondsPerHour + microseconds += minutes * microsecondsPerMinute + microseconds += seconds * microsecondsPerSecond + microseconds += uSeconds + + if negative { + microseconds = -microseconds + } + } + + *dst = Interval{Months: months, Days: days, Microseconds: microseconds, Status: Present} + return nil +} + +func (dst *Interval) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Interval{Status: Null} + return nil + } + + if len(src) != 16 { + return errors.Errorf("Received an invalid size for a interval: %d", len(src)) + } + + microseconds := int64(binary.BigEndian.Uint64(src)) + days := int32(binary.BigEndian.Uint32(src[8:])) + months := int32(binary.BigEndian.Uint32(src[12:])) + + *dst = Interval{Microseconds: microseconds, Days: days, Months: months, Status: Present} + return nil +} + +func (src Interval) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + if src.Months != 0 { + buf = append(buf, strconv.FormatInt(int64(src.Months), 10)...) + buf = append(buf, " mon "...) + } + + if src.Days != 0 { + buf = append(buf, strconv.FormatInt(int64(src.Days), 10)...) + buf = append(buf, " day "...) + } + + absMicroseconds := src.Microseconds + if absMicroseconds < 0 { + absMicroseconds = -absMicroseconds + buf = append(buf, '-') + } + + hours := absMicroseconds / microsecondsPerHour + minutes := (absMicroseconds % microsecondsPerHour) / microsecondsPerMinute + seconds := (absMicroseconds % microsecondsPerMinute) / microsecondsPerSecond + microseconds := absMicroseconds % microsecondsPerSecond + + timeStr := fmt.Sprintf("%02d:%02d:%02d.%06d", hours, minutes, seconds, microseconds) + return append(buf, timeStr...), nil +} + +// EncodeBinary encodes src into w. +func (src Interval) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + buf = pgio.AppendInt64(buf, src.Microseconds) + buf = pgio.AppendInt32(buf, src.Days) + return pgio.AppendInt32(buf, src.Months), nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Interval) Scan(src interface{}) error { + if src == nil { + *dst = Interval{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Interval) Value() (driver.Value, error) { + return EncodeValueText(src) +} diff --git a/github.com/jackc/pgtype/json.go b/github.com/jackc/pgtype/json.go new file mode 100644 index 0000000000..c642c72762 --- /dev/null +++ b/github.com/jackc/pgtype/json.go @@ -0,0 +1,197 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/json" + + errors "golang.org/x/xerrors" +) + +type JSON struct { + Bytes []byte + Status Status +} + +func (dst *JSON) Set(src interface{}) error { + if src == nil { + *dst = JSON{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + case string: + *dst = JSON{Bytes: []byte(value), Status: Present} + case *string: + if value == nil { + *dst = JSON{Status: Null} + } else { + *dst = JSON{Bytes: []byte(*value), Status: Present} + } + case []byte: + if value == nil { + *dst = JSON{Status: Null} + } else { + *dst = JSON{Bytes: value, Status: Present} + } + // Encode* methods are defined on *JSON. If JSON is passed directly then the + // struct itself would be encoded instead of Bytes. This is clearly a footgun + // so detect and return an error. See https://github.com/jackc/pgx/issues/350. + case JSON: + return errors.New("use pointer to pgtype.JSON instead of value") + // Same as above but for JSONB (because they share implementation) + case JSONB: + return errors.New("use pointer to pgtype.JSONB instead of value") + + default: + buf, err := json.Marshal(value) + if err != nil { + return err + } + *dst = JSON{Bytes: buf, Status: Present} + } + + return nil +} + +func (dst JSON) Get() interface{} { + switch dst.Status { + case Present: + var i interface{} + err := json.Unmarshal(dst.Bytes, &i) + if err != nil { + return dst + } + return i + case Null: + return nil + default: + return dst.Status + } +} + +func (src *JSON) AssignTo(dst interface{}) error { + switch v := dst.(type) { + case *string: + if src.Status == Present { + *v = string(src.Bytes) + } else { + return errors.Errorf("cannot assign non-present status to %T", dst) + } + case **string: + if src.Status == Present { + s := string(src.Bytes) + *v = &s + return nil + } else { + *v = nil + return nil + } + case *[]byte: + if src.Status != Present { + *v = nil + } else { + buf := make([]byte, len(src.Bytes)) + copy(buf, src.Bytes) + *v = buf + } + default: + data := src.Bytes + if data == nil || src.Status != Present { + data = []byte("null") + } + + return json.Unmarshal(data, dst) + } + + return nil +} + +func (dst *JSON) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = JSON{Status: Null} + return nil + } + + *dst = JSON{Bytes: src, Status: Present} + return nil +} + +func (dst *JSON) DecodeBinary(ci *ConnInfo, src []byte) error { + return dst.DecodeText(ci, src) +} + +func (src JSON) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + return append(buf, src.Bytes...), nil +} + +func (src JSON) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + return src.EncodeText(ci, buf) +} + +// Scan implements the database/sql Scanner interface. +func (dst *JSON) Scan(src interface{}) error { + if src == nil { + *dst = JSON{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src JSON) Value() (driver.Value, error) { + switch src.Status { + case Present: + return src.Bytes, nil + case Null: + return nil, nil + default: + return nil, errUndefined + } +} + +func (src JSON) MarshalJSON() ([]byte, error) { + switch src.Status { + case Present: + return src.Bytes, nil + case Null: + return []byte("null"), nil + case Undefined: + return nil, errUndefined + } + + return nil, errBadStatus +} + +func (dst *JSON) UnmarshalJSON(b []byte) error { + if b == nil || string(b) == "null" { + *dst = JSON{Status: Null} + } else { + *dst = JSON{Bytes: b, Status: Present} + } + return nil + +} diff --git a/github.com/jackc/pgtype/jsonb.go b/github.com/jackc/pgtype/jsonb.go new file mode 100644 index 0000000000..984c09734f --- /dev/null +++ b/github.com/jackc/pgtype/jsonb.go @@ -0,0 +1,78 @@ +package pgtype + +import ( + "database/sql/driver" + + errors "golang.org/x/xerrors" +) + +type JSONB JSON + +func (dst *JSONB) Set(src interface{}) error { + return (*JSON)(dst).Set(src) +} + +func (dst JSONB) Get() interface{} { + return (JSON)(dst).Get() +} + +func (src *JSONB) AssignTo(dst interface{}) error { + return (*JSON)(src).AssignTo(dst) +} + +func (dst *JSONB) DecodeText(ci *ConnInfo, src []byte) error { + return (*JSON)(dst).DecodeText(ci, src) +} + +func (dst *JSONB) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = JSONB{Status: Null} + return nil + } + + if len(src) == 0 { + return errors.Errorf("jsonb too short") + } + + if src[0] != 1 { + return errors.Errorf("unknown jsonb version number %d", src[0]) + } + + *dst = JSONB{Bytes: src[1:], Status: Present} + return nil + +} + +func (src JSONB) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + return (JSON)(src).EncodeText(ci, buf) +} + +func (src JSONB) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + buf = append(buf, 1) + return append(buf, src.Bytes...), nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *JSONB) Scan(src interface{}) error { + return (*JSON)(dst).Scan(src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src JSONB) Value() (driver.Value, error) { + return (JSON)(src).Value() +} + +func (src JSONB) MarshalJSON() ([]byte, error) { + return (JSON)(src).MarshalJSON() +} + +func (dst *JSONB) UnmarshalJSON(b []byte) error { + return (*JSON)(dst).UnmarshalJSON(b) +} diff --git a/github.com/jackc/pgtype/line.go b/github.com/jackc/pgtype/line.go new file mode 100644 index 0000000000..737f5d867b --- /dev/null +++ b/github.com/jackc/pgtype/line.go @@ -0,0 +1,149 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + "fmt" + "math" + "strconv" + "strings" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type Line struct { + A, B, C float64 + Status Status +} + +func (dst *Line) Set(src interface{}) error { + return errors.Errorf("cannot convert %v to Line", src) +} + +func (dst Line) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Line) AssignTo(dst interface{}) error { + return errors.Errorf("cannot assign %v to %T", src, dst) +} + +func (dst *Line) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Line{Status: Null} + return nil + } + + if len(src) < 7 { + return errors.Errorf("invalid length for Line: %v", len(src)) + } + + parts := strings.SplitN(string(src[1:len(src)-1]), ",", 3) + if len(parts) < 3 { + return errors.Errorf("invalid format for line") + } + + a, err := strconv.ParseFloat(parts[0], 64) + if err != nil { + return err + } + + b, err := strconv.ParseFloat(parts[1], 64) + if err != nil { + return err + } + + c, err := strconv.ParseFloat(parts[2], 64) + if err != nil { + return err + } + + *dst = Line{A: a, B: b, C: c, Status: Present} + return nil +} + +func (dst *Line) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Line{Status: Null} + return nil + } + + if len(src) != 24 { + return errors.Errorf("invalid length for Line: %v", len(src)) + } + + a := binary.BigEndian.Uint64(src) + b := binary.BigEndian.Uint64(src[8:]) + c := binary.BigEndian.Uint64(src[16:]) + + *dst = Line{ + A: math.Float64frombits(a), + B: math.Float64frombits(b), + C: math.Float64frombits(c), + Status: Present, + } + return nil +} + +func (src Line) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + buf = append(buf, fmt.Sprintf(`{%s,%s,%s}`, + strconv.FormatFloat(src.A, 'f', -1, 64), + strconv.FormatFloat(src.B, 'f', -1, 64), + strconv.FormatFloat(src.C, 'f', -1, 64), + )...) + + return buf, nil +} + +func (src Line) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + buf = pgio.AppendUint64(buf, math.Float64bits(src.A)) + buf = pgio.AppendUint64(buf, math.Float64bits(src.B)) + buf = pgio.AppendUint64(buf, math.Float64bits(src.C)) + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Line) Scan(src interface{}) error { + if src == nil { + *dst = Line{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Line) Value() (driver.Value, error) { + return EncodeValueText(src) +} diff --git a/github.com/jackc/pgtype/lseg.go b/github.com/jackc/pgtype/lseg.go new file mode 100644 index 0000000000..a16dcea331 --- /dev/null +++ b/github.com/jackc/pgtype/lseg.go @@ -0,0 +1,166 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + "fmt" + "math" + "strconv" + "strings" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type Lseg struct { + P [2]Vec2 + Status Status +} + +func (dst *Lseg) Set(src interface{}) error { + return errors.Errorf("cannot convert %v to Lseg", src) +} + +func (dst Lseg) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Lseg) AssignTo(dst interface{}) error { + return errors.Errorf("cannot assign %v to %T", src, dst) +} + +func (dst *Lseg) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Lseg{Status: Null} + return nil + } + + if len(src) < 11 { + return errors.Errorf("invalid length for Lseg: %v", len(src)) + } + + str := string(src[2:]) + + var end int + end = strings.IndexByte(str, ',') + + x1, err := strconv.ParseFloat(str[:end], 64) + if err != nil { + return err + } + + str = str[end+1:] + end = strings.IndexByte(str, ')') + + y1, err := strconv.ParseFloat(str[:end], 64) + if err != nil { + return err + } + + str = str[end+3:] + end = strings.IndexByte(str, ',') + + x2, err := strconv.ParseFloat(str[:end], 64) + if err != nil { + return err + } + + str = str[end+1 : len(str)-2] + + y2, err := strconv.ParseFloat(str, 64) + if err != nil { + return err + } + + *dst = Lseg{P: [2]Vec2{{x1, y1}, {x2, y2}}, Status: Present} + return nil +} + +func (dst *Lseg) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Lseg{Status: Null} + return nil + } + + if len(src) != 32 { + return errors.Errorf("invalid length for Lseg: %v", len(src)) + } + + x1 := binary.BigEndian.Uint64(src) + y1 := binary.BigEndian.Uint64(src[8:]) + x2 := binary.BigEndian.Uint64(src[16:]) + y2 := binary.BigEndian.Uint64(src[24:]) + + *dst = Lseg{ + P: [2]Vec2{ + {math.Float64frombits(x1), math.Float64frombits(y1)}, + {math.Float64frombits(x2), math.Float64frombits(y2)}, + }, + Status: Present, + } + return nil +} + +func (src Lseg) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + buf = append(buf, fmt.Sprintf(`(%s,%s),(%s,%s)`, + strconv.FormatFloat(src.P[0].X, 'f', -1, 64), + strconv.FormatFloat(src.P[0].Y, 'f', -1, 64), + strconv.FormatFloat(src.P[1].X, 'f', -1, 64), + strconv.FormatFloat(src.P[1].Y, 'f', -1, 64), + )...) + + return buf, nil +} + +func (src Lseg) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + buf = pgio.AppendUint64(buf, math.Float64bits(src.P[0].X)) + buf = pgio.AppendUint64(buf, math.Float64bits(src.P[0].Y)) + buf = pgio.AppendUint64(buf, math.Float64bits(src.P[1].X)) + buf = pgio.AppendUint64(buf, math.Float64bits(src.P[1].Y)) + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Lseg) Scan(src interface{}) error { + if src == nil { + *dst = Lseg{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Lseg) Value() (driver.Value, error) { + return EncodeValueText(src) +} diff --git a/github.com/jackc/pgtype/macaddr.go b/github.com/jackc/pgtype/macaddr.go new file mode 100644 index 0000000000..af0901b026 --- /dev/null +++ b/github.com/jackc/pgtype/macaddr.go @@ -0,0 +1,162 @@ +package pgtype + +import ( + "database/sql/driver" + "net" + + errors "golang.org/x/xerrors" +) + +type Macaddr struct { + Addr net.HardwareAddr + Status Status +} + +func (dst *Macaddr) Set(src interface{}) error { + if src == nil { + *dst = Macaddr{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + case net.HardwareAddr: + addr := make(net.HardwareAddr, len(value)) + copy(addr, value) + *dst = Macaddr{Addr: addr, Status: Present} + case string: + addr, err := net.ParseMAC(value) + if err != nil { + return err + } + *dst = Macaddr{Addr: addr, Status: Present} + default: + if originalSrc, ok := underlyingPtrType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to Macaddr", value) + } + + return nil +} + +func (dst Macaddr) Get() interface{} { + switch dst.Status { + case Present: + return dst.Addr + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Macaddr) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + case *net.HardwareAddr: + *v = make(net.HardwareAddr, len(src.Addr)) + copy(*v, src.Addr) + return nil + case *string: + *v = src.Addr.String() + return nil + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *Macaddr) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Macaddr{Status: Null} + return nil + } + + addr, err := net.ParseMAC(string(src)) + if err != nil { + return err + } + + *dst = Macaddr{Addr: addr, Status: Present} + return nil +} + +func (dst *Macaddr) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Macaddr{Status: Null} + return nil + } + + if len(src) != 6 { + return errors.Errorf("Received an invalid size for a macaddr: %d", len(src)) + } + + addr := make(net.HardwareAddr, 6) + copy(addr, src) + + *dst = Macaddr{Addr: addr, Status: Present} + + return nil +} + +func (src Macaddr) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + return append(buf, src.Addr.String()...), nil +} + +// EncodeBinary encodes src into w. +func (src Macaddr) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + return append(buf, src.Addr...), nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Macaddr) Scan(src interface{}) error { + if src == nil { + *dst = Macaddr{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Macaddr) Value() (driver.Value, error) { + return EncodeValueText(src) +} diff --git a/github.com/jackc/pgtype/macaddr_array.go b/github.com/jackc/pgtype/macaddr_array.go new file mode 100644 index 0000000000..97b1353745 --- /dev/null +++ b/github.com/jackc/pgtype/macaddr_array.go @@ -0,0 +1,321 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + "net" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type MacaddrArray struct { + Elements []Macaddr + Dimensions []ArrayDimension + Status Status +} + +func (dst *MacaddrArray) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = MacaddrArray{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + + case []net.HardwareAddr: + if value == nil { + *dst = MacaddrArray{Status: Null} + } else if len(value) == 0 { + *dst = MacaddrArray{Status: Present} + } else { + elements := make([]Macaddr, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = MacaddrArray{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []Macaddr: + if value == nil { + *dst = MacaddrArray{Status: Null} + } else if len(value) == 0 { + *dst = MacaddrArray{Status: Present} + } else { + *dst = MacaddrArray{ + Elements: value, + Dimensions: []ArrayDimension{{Length: int32(len(value)), LowerBound: 1}}, + Status: Present, + } + } + default: + if originalSrc, ok := underlyingSliceType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to MacaddrArray", value) + } + + return nil +} + +func (dst MacaddrArray) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *MacaddrArray) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + + case *[]net.HardwareAddr: + *v = make([]net.HardwareAddr, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *MacaddrArray) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = MacaddrArray{Status: Null} + return nil + } + + uta, err := ParseUntypedTextArray(string(src)) + if err != nil { + return err + } + + var elements []Macaddr + + if len(uta.Elements) > 0 { + elements = make([]Macaddr, len(uta.Elements)) + + for i, s := range uta.Elements { + var elem Macaddr + var elemSrc []byte + if s != "NULL" { + elemSrc = []byte(s) + } + err = elem.DecodeText(ci, elemSrc) + if err != nil { + return err + } + + elements[i] = elem + } + } + + *dst = MacaddrArray{Elements: elements, Dimensions: uta.Dimensions, Status: Present} + + return nil +} + +func (dst *MacaddrArray) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = MacaddrArray{Status: Null} + return nil + } + + var arrayHeader ArrayHeader + rp, err := arrayHeader.DecodeBinary(ci, src) + if err != nil { + return err + } + + if len(arrayHeader.Dimensions) == 0 { + *dst = MacaddrArray{Dimensions: arrayHeader.Dimensions, Status: Present} + return nil + } + + elementCount := arrayHeader.Dimensions[0].Length + for _, d := range arrayHeader.Dimensions[1:] { + elementCount *= d.Length + } + + elements := make([]Macaddr, elementCount) + + for i := range elements { + elemLen := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + var elemSrc []byte + if elemLen >= 0 { + elemSrc = src[rp : rp+elemLen] + rp += elemLen + } + err = elements[i].DecodeBinary(ci, elemSrc) + if err != nil { + return err + } + } + + *dst = MacaddrArray{Elements: elements, Dimensions: arrayHeader.Dimensions, Status: Present} + return nil +} + +func (src MacaddrArray) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + if len(src.Dimensions) == 0 { + return append(buf, '{', '}'), nil + } + + buf = EncodeTextArrayDimensions(buf, src.Dimensions) + + // dimElemCounts is the multiples of elements that each array lies on. For + // example, a single dimension array of length 4 would have a dimElemCounts of + // [4]. A multi-dimensional array of lengths [3,5,2] would have a + // dimElemCounts of [30,10,2]. This is used to simplify when to render a '{' + // or '}'. + dimElemCounts := make([]int, len(src.Dimensions)) + dimElemCounts[len(src.Dimensions)-1] = int(src.Dimensions[len(src.Dimensions)-1].Length) + for i := len(src.Dimensions) - 2; i > -1; i-- { + dimElemCounts[i] = int(src.Dimensions[i].Length) * dimElemCounts[i+1] + } + + inElemBuf := make([]byte, 0, 32) + for i, elem := range src.Elements { + if i > 0 { + buf = append(buf, ',') + } + + for _, dec := range dimElemCounts { + if i%dec == 0 { + buf = append(buf, '{') + } + } + + elemBuf, err := elem.EncodeText(ci, inElemBuf) + if err != nil { + return nil, err + } + if elemBuf == nil { + buf = append(buf, `NULL`...) + } else { + buf = append(buf, QuoteArrayElementIfNeeded(string(elemBuf))...) + } + + for _, dec := range dimElemCounts { + if (i+1)%dec == 0 { + buf = append(buf, '}') + } + } + } + + return buf, nil +} + +func (src MacaddrArray) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + arrayHeader := ArrayHeader{ + Dimensions: src.Dimensions, + } + + if dt, ok := ci.DataTypeForName("macaddr"); ok { + arrayHeader.ElementOID = int32(dt.OID) + } else { + return nil, errors.Errorf("unable to find oid for type name %v", "macaddr") + } + + for i := range src.Elements { + if src.Elements[i].Status == Null { + arrayHeader.ContainsNull = true + break + } + } + + buf = arrayHeader.EncodeBinary(ci, buf) + + for i := range src.Elements { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + elemBuf, err := src.Elements[i].EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if elemBuf != nil { + buf = elemBuf + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *MacaddrArray) Scan(src interface{}) error { + if src == nil { + return dst.DecodeText(nil, nil) + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src MacaddrArray) Value() (driver.Value, error) { + buf, err := src.EncodeText(nil, nil) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + + return string(buf), nil +} diff --git a/github.com/jackc/pgtype/name.go b/github.com/jackc/pgtype/name.go new file mode 100644 index 0000000000..7ce8d25e98 --- /dev/null +++ b/github.com/jackc/pgtype/name.go @@ -0,0 +1,58 @@ +package pgtype + +import ( + "database/sql/driver" +) + +// Name is a type used for PostgreSQL's special 63-byte +// name data type, used for identifiers like table names. +// The pg_class.relname column is a good example of where the +// name data type is used. +// +// Note that the underlying Go data type of pgx.Name is string, +// so there is no way to enforce the 63-byte length. Inputting +// a longer name into PostgreSQL will result in silent truncation +// to 63 bytes. +// +// Also, if you have custom-compiled PostgreSQL and set +// NAMEDATALEN to a different value, obviously that number of +// bytes applies, rather than the default 63. +type Name Text + +func (dst *Name) Set(src interface{}) error { + return (*Text)(dst).Set(src) +} + +func (dst Name) Get() interface{} { + return (Text)(dst).Get() +} + +func (src *Name) AssignTo(dst interface{}) error { + return (*Text)(src).AssignTo(dst) +} + +func (dst *Name) DecodeText(ci *ConnInfo, src []byte) error { + return (*Text)(dst).DecodeText(ci, src) +} + +func (dst *Name) DecodeBinary(ci *ConnInfo, src []byte) error { + return (*Text)(dst).DecodeBinary(ci, src) +} + +func (src Name) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + return (Text)(src).EncodeText(ci, buf) +} + +func (src Name) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + return (Text)(src).EncodeBinary(ci, buf) +} + +// Scan implements the database/sql Scanner interface. +func (dst *Name) Scan(src interface{}) error { + return (*Text)(dst).Scan(src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Name) Value() (driver.Value, error) { + return (Text)(src).Value() +} diff --git a/github.com/jackc/pgtype/numeric.go b/github.com/jackc/pgtype/numeric.go new file mode 100644 index 0000000000..e6c58391d7 --- /dev/null +++ b/github.com/jackc/pgtype/numeric.go @@ -0,0 +1,604 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + "math" + "math/big" + "strconv" + "strings" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +// PostgreSQL internal numeric storage uses 16-bit "digits" with base of 10,000 +const nbase = 10000 + +var big0 *big.Int = big.NewInt(0) +var big1 *big.Int = big.NewInt(1) +var big10 *big.Int = big.NewInt(10) +var big100 *big.Int = big.NewInt(100) +var big1000 *big.Int = big.NewInt(1000) + +var bigMaxInt8 *big.Int = big.NewInt(math.MaxInt8) +var bigMinInt8 *big.Int = big.NewInt(math.MinInt8) +var bigMaxInt16 *big.Int = big.NewInt(math.MaxInt16) +var bigMinInt16 *big.Int = big.NewInt(math.MinInt16) +var bigMaxInt32 *big.Int = big.NewInt(math.MaxInt32) +var bigMinInt32 *big.Int = big.NewInt(math.MinInt32) +var bigMaxInt64 *big.Int = big.NewInt(math.MaxInt64) +var bigMinInt64 *big.Int = big.NewInt(math.MinInt64) +var bigMaxInt *big.Int = big.NewInt(int64(maxInt)) +var bigMinInt *big.Int = big.NewInt(int64(minInt)) + +var bigMaxUint8 *big.Int = big.NewInt(math.MaxUint8) +var bigMaxUint16 *big.Int = big.NewInt(math.MaxUint16) +var bigMaxUint32 *big.Int = big.NewInt(math.MaxUint32) +var bigMaxUint64 *big.Int = (&big.Int{}).SetUint64(uint64(math.MaxUint64)) +var bigMaxUint *big.Int = (&big.Int{}).SetUint64(uint64(maxUint)) + +var bigNBase *big.Int = big.NewInt(nbase) +var bigNBaseX2 *big.Int = big.NewInt(nbase * nbase) +var bigNBaseX3 *big.Int = big.NewInt(nbase * nbase * nbase) +var bigNBaseX4 *big.Int = big.NewInt(nbase * nbase * nbase * nbase) + +type Numeric struct { + Int *big.Int + Exp int32 + Status Status +} + +func (dst *Numeric) Set(src interface{}) error { + if src == nil { + *dst = Numeric{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + case float32: + num, exp, err := parseNumericString(strconv.FormatFloat(float64(value), 'f', -1, 64)) + if err != nil { + return err + } + *dst = Numeric{Int: num, Exp: exp, Status: Present} + case float64: + num, exp, err := parseNumericString(strconv.FormatFloat(value, 'f', -1, 64)) + if err != nil { + return err + } + *dst = Numeric{Int: num, Exp: exp, Status: Present} + case int8: + *dst = Numeric{Int: big.NewInt(int64(value)), Status: Present} + case uint8: + *dst = Numeric{Int: big.NewInt(int64(value)), Status: Present} + case int16: + *dst = Numeric{Int: big.NewInt(int64(value)), Status: Present} + case uint16: + *dst = Numeric{Int: big.NewInt(int64(value)), Status: Present} + case int32: + *dst = Numeric{Int: big.NewInt(int64(value)), Status: Present} + case uint32: + *dst = Numeric{Int: big.NewInt(int64(value)), Status: Present} + case int64: + *dst = Numeric{Int: big.NewInt(value), Status: Present} + case uint64: + *dst = Numeric{Int: (&big.Int{}).SetUint64(value), Status: Present} + case int: + *dst = Numeric{Int: big.NewInt(int64(value)), Status: Present} + case uint: + *dst = Numeric{Int: (&big.Int{}).SetUint64(uint64(value)), Status: Present} + case string: + num, exp, err := parseNumericString(value) + if err != nil { + return err + } + *dst = Numeric{Int: num, Exp: exp, Status: Present} + default: + if originalSrc, ok := underlyingNumberType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to Numeric", value) + } + + return nil +} + +func (dst Numeric) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Numeric) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + case *float32: + f, err := src.toFloat64() + if err != nil { + return err + } + return float64AssignTo(f, src.Status, dst) + case *float64: + f, err := src.toFloat64() + if err != nil { + return err + } + return float64AssignTo(f, src.Status, dst) + case *int: + normalizedInt, err := src.toBigInt() + if err != nil { + return err + } + if normalizedInt.Cmp(bigMaxInt) > 0 { + return errors.Errorf("%v is greater than maximum value for %T", normalizedInt, *v) + } + if normalizedInt.Cmp(bigMinInt) < 0 { + return errors.Errorf("%v is less than minimum value for %T", normalizedInt, *v) + } + *v = int(normalizedInt.Int64()) + case *int8: + normalizedInt, err := src.toBigInt() + if err != nil { + return err + } + if normalizedInt.Cmp(bigMaxInt8) > 0 { + return errors.Errorf("%v is greater than maximum value for %T", normalizedInt, *v) + } + if normalizedInt.Cmp(bigMinInt8) < 0 { + return errors.Errorf("%v is less than minimum value for %T", normalizedInt, *v) + } + *v = int8(normalizedInt.Int64()) + case *int16: + normalizedInt, err := src.toBigInt() + if err != nil { + return err + } + if normalizedInt.Cmp(bigMaxInt16) > 0 { + return errors.Errorf("%v is greater than maximum value for %T", normalizedInt, *v) + } + if normalizedInt.Cmp(bigMinInt16) < 0 { + return errors.Errorf("%v is less than minimum value for %T", normalizedInt, *v) + } + *v = int16(normalizedInt.Int64()) + case *int32: + normalizedInt, err := src.toBigInt() + if err != nil { + return err + } + if normalizedInt.Cmp(bigMaxInt32) > 0 { + return errors.Errorf("%v is greater than maximum value for %T", normalizedInt, *v) + } + if normalizedInt.Cmp(bigMinInt32) < 0 { + return errors.Errorf("%v is less than minimum value for %T", normalizedInt, *v) + } + *v = int32(normalizedInt.Int64()) + case *int64: + normalizedInt, err := src.toBigInt() + if err != nil { + return err + } + if normalizedInt.Cmp(bigMaxInt64) > 0 { + return errors.Errorf("%v is greater than maximum value for %T", normalizedInt, *v) + } + if normalizedInt.Cmp(bigMinInt64) < 0 { + return errors.Errorf("%v is less than minimum value for %T", normalizedInt, *v) + } + *v = normalizedInt.Int64() + case *uint: + normalizedInt, err := src.toBigInt() + if err != nil { + return err + } + if normalizedInt.Cmp(big0) < 0 { + return errors.Errorf("%d is less than zero for %T", normalizedInt, *v) + } else if normalizedInt.Cmp(bigMaxUint) > 0 { + return errors.Errorf("%d is greater than maximum value for %T", normalizedInt, *v) + } + *v = uint(normalizedInt.Uint64()) + case *uint8: + normalizedInt, err := src.toBigInt() + if err != nil { + return err + } + if normalizedInt.Cmp(big0) < 0 { + return errors.Errorf("%d is less than zero for %T", normalizedInt, *v) + } else if normalizedInt.Cmp(bigMaxUint8) > 0 { + return errors.Errorf("%d is greater than maximum value for %T", normalizedInt, *v) + } + *v = uint8(normalizedInt.Uint64()) + case *uint16: + normalizedInt, err := src.toBigInt() + if err != nil { + return err + } + if normalizedInt.Cmp(big0) < 0 { + return errors.Errorf("%d is less than zero for %T", normalizedInt, *v) + } else if normalizedInt.Cmp(bigMaxUint16) > 0 { + return errors.Errorf("%d is greater than maximum value for %T", normalizedInt, *v) + } + *v = uint16(normalizedInt.Uint64()) + case *uint32: + normalizedInt, err := src.toBigInt() + if err != nil { + return err + } + if normalizedInt.Cmp(big0) < 0 { + return errors.Errorf("%d is less than zero for %T", normalizedInt, *v) + } else if normalizedInt.Cmp(bigMaxUint32) > 0 { + return errors.Errorf("%d is greater than maximum value for %T", normalizedInt, *v) + } + *v = uint32(normalizedInt.Uint64()) + case *uint64: + normalizedInt, err := src.toBigInt() + if err != nil { + return err + } + if normalizedInt.Cmp(big0) < 0 { + return errors.Errorf("%d is less than zero for %T", normalizedInt, *v) + } else if normalizedInt.Cmp(bigMaxUint64) > 0 { + return errors.Errorf("%d is greater than maximum value for %T", normalizedInt, *v) + } + *v = normalizedInt.Uint64() + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return nil +} + +func (dst *Numeric) toBigInt() (*big.Int, error) { + if dst.Exp == 0 { + return dst.Int, nil + } + + num := &big.Int{} + num.Set(dst.Int) + if dst.Exp > 0 { + mul := &big.Int{} + mul.Exp(big10, big.NewInt(int64(dst.Exp)), nil) + num.Mul(num, mul) + return num, nil + } + + div := &big.Int{} + div.Exp(big10, big.NewInt(int64(-dst.Exp)), nil) + remainder := &big.Int{} + num.DivMod(num, div, remainder) + if remainder.Cmp(big0) != 0 { + return nil, errors.Errorf("cannot convert %v to integer", dst) + } + return num, nil +} + +func (src *Numeric) toFloat64() (float64, error) { + f, err := strconv.ParseFloat(src.Int.String(), 64) + if err != nil { + return 0, err + } + if src.Exp > 0 { + for i := 0; i < int(src.Exp); i++ { + f *= 10 + } + } else if src.Exp < 0 { + for i := 0; i > int(src.Exp); i-- { + f /= 10 + } + } + return f, nil +} + +func (dst *Numeric) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Numeric{Status: Null} + return nil + } + + num, exp, err := parseNumericString(string(src)) + if err != nil { + return err + } + + *dst = Numeric{Int: num, Exp: exp, Status: Present} + return nil +} + +func parseNumericString(str string) (n *big.Int, exp int32, err error) { + parts := strings.SplitN(str, ".", 2) + digits := strings.Join(parts, "") + + if len(parts) > 1 { + exp = int32(-len(parts[1])) + } else { + for len(digits) > 1 && digits[len(digits)-1] == '0' && digits[len(digits)-2] != '-' { + digits = digits[:len(digits)-1] + exp++ + } + } + + accum := &big.Int{} + if _, ok := accum.SetString(digits, 10); !ok { + return nil, 0, errors.Errorf("%s is not a number", str) + } + + return accum, exp, nil +} + +func (dst *Numeric) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Numeric{Status: Null} + return nil + } + + if len(src) < 8 { + return errors.Errorf("numeric incomplete %v", src) + } + + rp := 0 + ndigits := int16(binary.BigEndian.Uint16(src[rp:])) + rp += 2 + + if ndigits == 0 { + *dst = Numeric{Int: big.NewInt(0), Status: Present} + return nil + } + + weight := int16(binary.BigEndian.Uint16(src[rp:])) + rp += 2 + sign := int16(binary.BigEndian.Uint16(src[rp:])) + rp += 2 + dscale := int16(binary.BigEndian.Uint16(src[rp:])) + rp += 2 + + if len(src[rp:]) < int(ndigits)*2 { + return errors.Errorf("numeric incomplete %v", src) + } + + accum := &big.Int{} + + for i := 0; i < int(ndigits+3)/4; i++ { + int64accum, bytesRead, digitsRead := nbaseDigitsToInt64(src[rp:]) + rp += bytesRead + + if i > 0 { + var mul *big.Int + switch digitsRead { + case 1: + mul = bigNBase + case 2: + mul = bigNBaseX2 + case 3: + mul = bigNBaseX3 + case 4: + mul = bigNBaseX4 + default: + return errors.Errorf("invalid digitsRead: %d (this can't happen)", digitsRead) + } + accum.Mul(accum, mul) + } + + accum.Add(accum, big.NewInt(int64accum)) + } + + exp := (int32(weight) - int32(ndigits) + 1) * 4 + + if dscale > 0 { + fracNBaseDigits := ndigits - weight - 1 + fracDecimalDigits := fracNBaseDigits * 4 + + if dscale > fracDecimalDigits { + multCount := int(dscale - fracDecimalDigits) + for i := 0; i < multCount; i++ { + accum.Mul(accum, big10) + exp-- + } + } else if dscale < fracDecimalDigits { + divCount := int(fracDecimalDigits - dscale) + for i := 0; i < divCount; i++ { + accum.Div(accum, big10) + exp++ + } + } + } + + reduced := &big.Int{} + remainder := &big.Int{} + if exp >= 0 { + for { + reduced.DivMod(accum, big10, remainder) + if remainder.Cmp(big0) != 0 { + break + } + accum.Set(reduced) + exp++ + } + } + + if sign != 0 { + accum.Neg(accum) + } + + *dst = Numeric{Int: accum, Exp: exp, Status: Present} + + return nil + +} + +func nbaseDigitsToInt64(src []byte) (accum int64, bytesRead, digitsRead int) { + digits := len(src) / 2 + if digits > 4 { + digits = 4 + } + + rp := 0 + + for i := 0; i < digits; i++ { + if i > 0 { + accum *= nbase + } + accum += int64(binary.BigEndian.Uint16(src[rp:])) + rp += 2 + } + + return accum, rp, digits +} + +func (src Numeric) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + buf = append(buf, src.Int.String()...) + buf = append(buf, 'e') + buf = append(buf, strconv.FormatInt(int64(src.Exp), 10)...) + return buf, nil +} + +func (src Numeric) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + var sign int16 + if src.Int.Cmp(big0) < 0 { + sign = 16384 + } + + absInt := &big.Int{} + wholePart := &big.Int{} + fracPart := &big.Int{} + remainder := &big.Int{} + absInt.Abs(src.Int) + + // Normalize absInt and exp to where exp is always a multiple of 4. This makes + // converting to 16-bit base 10,000 digits easier. + var exp int32 + switch src.Exp % 4 { + case 1, -3: + exp = src.Exp - 1 + absInt.Mul(absInt, big10) + case 2, -2: + exp = src.Exp - 2 + absInt.Mul(absInt, big100) + case 3, -1: + exp = src.Exp - 3 + absInt.Mul(absInt, big1000) + default: + exp = src.Exp + } + + if exp < 0 { + divisor := &big.Int{} + divisor.Exp(big10, big.NewInt(int64(-exp)), nil) + wholePart.DivMod(absInt, divisor, fracPart) + fracPart.Add(fracPart, divisor) + } else { + wholePart = absInt + } + + var wholeDigits, fracDigits []int16 + + for wholePart.Cmp(big0) != 0 { + wholePart.DivMod(wholePart, bigNBase, remainder) + wholeDigits = append(wholeDigits, int16(remainder.Int64())) + } + + if fracPart.Cmp(big0) != 0 { + for fracPart.Cmp(big1) != 0 { + fracPart.DivMod(fracPart, bigNBase, remainder) + fracDigits = append(fracDigits, int16(remainder.Int64())) + } + } + + buf = pgio.AppendInt16(buf, int16(len(wholeDigits)+len(fracDigits))) + + var weight int16 + if len(wholeDigits) > 0 { + weight = int16(len(wholeDigits) - 1) + if exp > 0 { + weight += int16(exp / 4) + } + } else { + weight = int16(exp/4) - 1 + int16(len(fracDigits)) + } + buf = pgio.AppendInt16(buf, weight) + + buf = pgio.AppendInt16(buf, sign) + + var dscale int16 + if src.Exp < 0 { + dscale = int16(-src.Exp) + } + buf = pgio.AppendInt16(buf, dscale) + + for i := len(wholeDigits) - 1; i >= 0; i-- { + buf = pgio.AppendInt16(buf, wholeDigits[i]) + } + + for i := len(fracDigits) - 1; i >= 0; i-- { + buf = pgio.AppendInt16(buf, fracDigits[i]) + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Numeric) Scan(src interface{}) error { + if src == nil { + *dst = Numeric{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Numeric) Value() (driver.Value, error) { + switch src.Status { + case Present: + buf, err := src.EncodeText(nil, nil) + if err != nil { + return nil, err + } + + return string(buf), nil + case Null: + return nil, nil + default: + return nil, errUndefined + } +} diff --git a/github.com/jackc/pgtype/numeric_array.go b/github.com/jackc/pgtype/numeric_array.go new file mode 100644 index 0000000000..3cec9fea94 --- /dev/null +++ b/github.com/jackc/pgtype/numeric_array.go @@ -0,0 +1,404 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type NumericArray struct { + Elements []Numeric + Dimensions []ArrayDimension + Status Status +} + +func (dst *NumericArray) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = NumericArray{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + + case []float32: + if value == nil { + *dst = NumericArray{Status: Null} + } else if len(value) == 0 { + *dst = NumericArray{Status: Present} + } else { + elements := make([]Numeric, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = NumericArray{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []float64: + if value == nil { + *dst = NumericArray{Status: Null} + } else if len(value) == 0 { + *dst = NumericArray{Status: Present} + } else { + elements := make([]Numeric, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = NumericArray{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []int64: + if value == nil { + *dst = NumericArray{Status: Null} + } else if len(value) == 0 { + *dst = NumericArray{Status: Present} + } else { + elements := make([]Numeric, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = NumericArray{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []uint64: + if value == nil { + *dst = NumericArray{Status: Null} + } else if len(value) == 0 { + *dst = NumericArray{Status: Present} + } else { + elements := make([]Numeric, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = NumericArray{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []Numeric: + if value == nil { + *dst = NumericArray{Status: Null} + } else if len(value) == 0 { + *dst = NumericArray{Status: Present} + } else { + *dst = NumericArray{ + Elements: value, + Dimensions: []ArrayDimension{{Length: int32(len(value)), LowerBound: 1}}, + Status: Present, + } + } + default: + if originalSrc, ok := underlyingSliceType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to NumericArray", value) + } + + return nil +} + +func (dst NumericArray) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *NumericArray) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + + case *[]float32: + *v = make([]float32, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[]float64: + *v = make([]float64, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[]int64: + *v = make([]int64, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[]uint64: + *v = make([]uint64, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *NumericArray) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = NumericArray{Status: Null} + return nil + } + + uta, err := ParseUntypedTextArray(string(src)) + if err != nil { + return err + } + + var elements []Numeric + + if len(uta.Elements) > 0 { + elements = make([]Numeric, len(uta.Elements)) + + for i, s := range uta.Elements { + var elem Numeric + var elemSrc []byte + if s != "NULL" { + elemSrc = []byte(s) + } + err = elem.DecodeText(ci, elemSrc) + if err != nil { + return err + } + + elements[i] = elem + } + } + + *dst = NumericArray{Elements: elements, Dimensions: uta.Dimensions, Status: Present} + + return nil +} + +func (dst *NumericArray) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = NumericArray{Status: Null} + return nil + } + + var arrayHeader ArrayHeader + rp, err := arrayHeader.DecodeBinary(ci, src) + if err != nil { + return err + } + + if len(arrayHeader.Dimensions) == 0 { + *dst = NumericArray{Dimensions: arrayHeader.Dimensions, Status: Present} + return nil + } + + elementCount := arrayHeader.Dimensions[0].Length + for _, d := range arrayHeader.Dimensions[1:] { + elementCount *= d.Length + } + + elements := make([]Numeric, elementCount) + + for i := range elements { + elemLen := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + var elemSrc []byte + if elemLen >= 0 { + elemSrc = src[rp : rp+elemLen] + rp += elemLen + } + err = elements[i].DecodeBinary(ci, elemSrc) + if err != nil { + return err + } + } + + *dst = NumericArray{Elements: elements, Dimensions: arrayHeader.Dimensions, Status: Present} + return nil +} + +func (src NumericArray) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + if len(src.Dimensions) == 0 { + return append(buf, '{', '}'), nil + } + + buf = EncodeTextArrayDimensions(buf, src.Dimensions) + + // dimElemCounts is the multiples of elements that each array lies on. For + // example, a single dimension array of length 4 would have a dimElemCounts of + // [4]. A multi-dimensional array of lengths [3,5,2] would have a + // dimElemCounts of [30,10,2]. This is used to simplify when to render a '{' + // or '}'. + dimElemCounts := make([]int, len(src.Dimensions)) + dimElemCounts[len(src.Dimensions)-1] = int(src.Dimensions[len(src.Dimensions)-1].Length) + for i := len(src.Dimensions) - 2; i > -1; i-- { + dimElemCounts[i] = int(src.Dimensions[i].Length) * dimElemCounts[i+1] + } + + inElemBuf := make([]byte, 0, 32) + for i, elem := range src.Elements { + if i > 0 { + buf = append(buf, ',') + } + + for _, dec := range dimElemCounts { + if i%dec == 0 { + buf = append(buf, '{') + } + } + + elemBuf, err := elem.EncodeText(ci, inElemBuf) + if err != nil { + return nil, err + } + if elemBuf == nil { + buf = append(buf, `NULL`...) + } else { + buf = append(buf, QuoteArrayElementIfNeeded(string(elemBuf))...) + } + + for _, dec := range dimElemCounts { + if (i+1)%dec == 0 { + buf = append(buf, '}') + } + } + } + + return buf, nil +} + +func (src NumericArray) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + arrayHeader := ArrayHeader{ + Dimensions: src.Dimensions, + } + + if dt, ok := ci.DataTypeForName("numeric"); ok { + arrayHeader.ElementOID = int32(dt.OID) + } else { + return nil, errors.Errorf("unable to find oid for type name %v", "numeric") + } + + for i := range src.Elements { + if src.Elements[i].Status == Null { + arrayHeader.ContainsNull = true + break + } + } + + buf = arrayHeader.EncodeBinary(ci, buf) + + for i := range src.Elements { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + elemBuf, err := src.Elements[i].EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if elemBuf != nil { + buf = elemBuf + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *NumericArray) Scan(src interface{}) error { + if src == nil { + return dst.DecodeText(nil, nil) + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src NumericArray) Value() (driver.Value, error) { + buf, err := src.EncodeText(nil, nil) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + + return string(buf), nil +} diff --git a/github.com/jackc/pgtype/numrange.go b/github.com/jackc/pgtype/numrange.go new file mode 100644 index 0000000000..404676860e --- /dev/null +++ b/github.com/jackc/pgtype/numrange.go @@ -0,0 +1,267 @@ +package pgtype + +import ( + "database/sql/driver" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type Numrange struct { + Lower Numeric + Upper Numeric + LowerType BoundType + UpperType BoundType + Status Status +} + +func (dst *Numrange) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = Numrange{Status: Null} + return nil + } + + switch value := src.(type) { + case Numrange: + *dst = value + case *Numrange: + *dst = *value + case string: + return dst.DecodeText(nil, []byte(value)) + default: + return errors.Errorf("cannot convert %v to Numrange", src) + } + + return nil +} + +func (dst Numrange) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Numrange) AssignTo(dst interface{}) error { + return errors.Errorf("cannot assign %v to %T", src, dst) +} + +func (dst *Numrange) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Numrange{Status: Null} + return nil + } + + utr, err := ParseUntypedTextRange(string(src)) + if err != nil { + return err + } + + *dst = Numrange{Status: Present} + + dst.LowerType = utr.LowerType + dst.UpperType = utr.UpperType + + if dst.LowerType == Empty { + return nil + } + + if dst.LowerType == Inclusive || dst.LowerType == Exclusive { + if err := dst.Lower.DecodeText(ci, []byte(utr.Lower)); err != nil { + return err + } + } + + if dst.UpperType == Inclusive || dst.UpperType == Exclusive { + if err := dst.Upper.DecodeText(ci, []byte(utr.Upper)); err != nil { + return err + } + } + + return nil +} + +func (dst *Numrange) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Numrange{Status: Null} + return nil + } + + ubr, err := ParseUntypedBinaryRange(src) + if err != nil { + return err + } + + *dst = Numrange{Status: Present} + + dst.LowerType = ubr.LowerType + dst.UpperType = ubr.UpperType + + if dst.LowerType == Empty { + return nil + } + + if dst.LowerType == Inclusive || dst.LowerType == Exclusive { + if err := dst.Lower.DecodeBinary(ci, ubr.Lower); err != nil { + return err + } + } + + if dst.UpperType == Inclusive || dst.UpperType == Exclusive { + if err := dst.Upper.DecodeBinary(ci, ubr.Upper); err != nil { + return err + } + } + + return nil +} + +func (src Numrange) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + switch src.LowerType { + case Exclusive, Unbounded: + buf = append(buf, '(') + case Inclusive: + buf = append(buf, '[') + case Empty: + return append(buf, "empty"...), nil + default: + return nil, errors.Errorf("unknown lower bound type %v", src.LowerType) + } + + var err error + + if src.LowerType != Unbounded { + buf, err = src.Lower.EncodeText(ci, buf) + if err != nil { + return nil, err + } else if buf == nil { + return nil, errors.Errorf("Lower cannot be null unless LowerType is Unbounded") + } + } + + buf = append(buf, ',') + + if src.UpperType != Unbounded { + buf, err = src.Upper.EncodeText(ci, buf) + if err != nil { + return nil, err + } else if buf == nil { + return nil, errors.Errorf("Upper cannot be null unless UpperType is Unbounded") + } + } + + switch src.UpperType { + case Exclusive, Unbounded: + buf = append(buf, ')') + case Inclusive: + buf = append(buf, ']') + default: + return nil, errors.Errorf("unknown upper bound type %v", src.UpperType) + } + + return buf, nil +} + +func (src Numrange) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + var rangeType byte + switch src.LowerType { + case Inclusive: + rangeType |= lowerInclusiveMask + case Unbounded: + rangeType |= lowerUnboundedMask + case Exclusive: + case Empty: + return append(buf, emptyMask), nil + default: + return nil, errors.Errorf("unknown LowerType: %v", src.LowerType) + } + + switch src.UpperType { + case Inclusive: + rangeType |= upperInclusiveMask + case Unbounded: + rangeType |= upperUnboundedMask + case Exclusive: + default: + return nil, errors.Errorf("unknown UpperType: %v", src.UpperType) + } + + buf = append(buf, rangeType) + + var err error + + if src.LowerType != Unbounded { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + buf, err = src.Lower.EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if buf == nil { + return nil, errors.Errorf("Lower cannot be null unless LowerType is Unbounded") + } + + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + + if src.UpperType != Unbounded { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + buf, err = src.Upper.EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if buf == nil { + return nil, errors.Errorf("Upper cannot be null unless UpperType is Unbounded") + } + + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Numrange) Scan(src interface{}) error { + if src == nil { + *dst = Numrange{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Numrange) Value() (driver.Value, error) { + return EncodeValueText(src) +} diff --git a/github.com/jackc/pgtype/oid.go b/github.com/jackc/pgtype/oid.go new file mode 100644 index 0000000000..593a526118 --- /dev/null +++ b/github.com/jackc/pgtype/oid.go @@ -0,0 +1,81 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + "strconv" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +// OID (Object Identifier Type) is, according to +// https://www.postgresql.org/docs/current/static/datatype-oid.html, used +// internally by PostgreSQL as a primary key for various system tables. It is +// currently implemented as an unsigned four-byte integer. Its definition can be +// found in src/include/postgres_ext.h in the PostgreSQL sources. Because it is +// so frequently required to be in a NOT NULL condition OID cannot be NULL. To +// allow for NULL OIDs use OIDValue. +type OID uint32 + +func (dst *OID) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + return errors.Errorf("cannot decode nil into OID") + } + + n, err := strconv.ParseUint(string(src), 10, 32) + if err != nil { + return err + } + + *dst = OID(n) + return nil +} + +func (dst *OID) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + return errors.Errorf("cannot decode nil into OID") + } + + if len(src) != 4 { + return errors.Errorf("invalid length: %v", len(src)) + } + + n := binary.BigEndian.Uint32(src) + *dst = OID(n) + return nil +} + +func (src OID) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + return append(buf, strconv.FormatUint(uint64(src), 10)...), nil +} + +func (src OID) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + return pgio.AppendUint32(buf, uint32(src)), nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *OID) Scan(src interface{}) error { + if src == nil { + return errors.Errorf("cannot scan NULL into %T", src) + } + + switch src := src.(type) { + case int64: + *dst = OID(src) + return nil + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src OID) Value() (driver.Value, error) { + return int64(src), nil +} diff --git a/github.com/jackc/pgtype/oid_value.go b/github.com/jackc/pgtype/oid_value.go new file mode 100644 index 0000000000..5dc9136cba --- /dev/null +++ b/github.com/jackc/pgtype/oid_value.go @@ -0,0 +1,55 @@ +package pgtype + +import ( + "database/sql/driver" +) + +// OIDValue (Object Identifier Type) is, according to +// https://www.postgresql.org/docs/current/static/datatype-OIDValue.html, used +// internally by PostgreSQL as a primary key for various system tables. It is +// currently implemented as an unsigned four-byte integer. Its definition can be +// found in src/include/postgres_ext.h in the PostgreSQL sources. +type OIDValue pguint32 + +// Set converts from src to dst. Note that as OIDValue is not a general +// number type Set does not do automatic type conversion as other number +// types do. +func (dst *OIDValue) Set(src interface{}) error { + return (*pguint32)(dst).Set(src) +} + +func (dst OIDValue) Get() interface{} { + return (pguint32)(dst).Get() +} + +// AssignTo assigns from src to dst. Note that as OIDValue is not a general number +// type AssignTo does not do automatic type conversion as other number types do. +func (src *OIDValue) AssignTo(dst interface{}) error { + return (*pguint32)(src).AssignTo(dst) +} + +func (dst *OIDValue) DecodeText(ci *ConnInfo, src []byte) error { + return (*pguint32)(dst).DecodeText(ci, src) +} + +func (dst *OIDValue) DecodeBinary(ci *ConnInfo, src []byte) error { + return (*pguint32)(dst).DecodeBinary(ci, src) +} + +func (src OIDValue) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + return (pguint32)(src).EncodeText(ci, buf) +} + +func (src OIDValue) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + return (pguint32)(src).EncodeBinary(ci, buf) +} + +// Scan implements the database/sql Scanner interface. +func (dst *OIDValue) Scan(src interface{}) error { + return (*pguint32)(dst).Scan(src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src OIDValue) Value() (driver.Value, error) { + return (pguint32)(src).Value() +} diff --git a/github.com/jackc/pgtype/path.go b/github.com/jackc/pgtype/path.go new file mode 100644 index 0000000000..c50313306b --- /dev/null +++ b/github.com/jackc/pgtype/path.go @@ -0,0 +1,196 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + "fmt" + "math" + "strconv" + "strings" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type Path struct { + P []Vec2 + Closed bool + Status Status +} + +func (dst *Path) Set(src interface{}) error { + return errors.Errorf("cannot convert %v to Path", src) +} + +func (dst Path) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Path) AssignTo(dst interface{}) error { + return errors.Errorf("cannot assign %v to %T", src, dst) +} + +func (dst *Path) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Path{Status: Null} + return nil + } + + if len(src) < 7 { + return errors.Errorf("invalid length for Path: %v", len(src)) + } + + closed := src[0] == '(' + points := make([]Vec2, 0) + + str := string(src[2:]) + + for { + end := strings.IndexByte(str, ',') + x, err := strconv.ParseFloat(str[:end], 64) + if err != nil { + return err + } + + str = str[end+1:] + end = strings.IndexByte(str, ')') + + y, err := strconv.ParseFloat(str[:end], 64) + if err != nil { + return err + } + + points = append(points, Vec2{x, y}) + + if end+3 < len(str) { + str = str[end+3:] + } else { + break + } + } + + *dst = Path{P: points, Closed: closed, Status: Present} + return nil +} + +func (dst *Path) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Path{Status: Null} + return nil + } + + if len(src) < 5 { + return errors.Errorf("invalid length for Path: %v", len(src)) + } + + closed := src[0] == 1 + pointCount := int(binary.BigEndian.Uint32(src[1:])) + + rp := 5 + + if 5+pointCount*16 != len(src) { + return errors.Errorf("invalid length for Path with %d points: %v", pointCount, len(src)) + } + + points := make([]Vec2, pointCount) + for i := 0; i < len(points); i++ { + x := binary.BigEndian.Uint64(src[rp:]) + rp += 8 + y := binary.BigEndian.Uint64(src[rp:]) + rp += 8 + points[i] = Vec2{math.Float64frombits(x), math.Float64frombits(y)} + } + + *dst = Path{ + P: points, + Closed: closed, + Status: Present, + } + return nil +} + +func (src Path) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + var startByte, endByte byte + if src.Closed { + startByte = '(' + endByte = ')' + } else { + startByte = '[' + endByte = ']' + } + buf = append(buf, startByte) + + for i, p := range src.P { + if i > 0 { + buf = append(buf, ',') + } + buf = append(buf, fmt.Sprintf(`(%s,%s)`, + strconv.FormatFloat(p.X, 'f', -1, 64), + strconv.FormatFloat(p.Y, 'f', -1, 64), + )...) + } + + return append(buf, endByte), nil +} + +func (src Path) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + var closeByte byte + if src.Closed { + closeByte = 1 + } + buf = append(buf, closeByte) + + buf = pgio.AppendInt32(buf, int32(len(src.P))) + + for _, p := range src.P { + buf = pgio.AppendUint64(buf, math.Float64bits(p.X)) + buf = pgio.AppendUint64(buf, math.Float64bits(p.Y)) + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Path) Scan(src interface{}) error { + if src == nil { + *dst = Path{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Path) Value() (driver.Value, error) { + return EncodeValueText(src) +} diff --git a/github.com/jackc/pgtype/pgtype.go b/github.com/jackc/pgtype/pgtype.go new file mode 100644 index 0000000000..914e02d2cc --- /dev/null +++ b/github.com/jackc/pgtype/pgtype.go @@ -0,0 +1,493 @@ +package pgtype + +import ( + "database/sql" + "reflect" + + errors "golang.org/x/xerrors" +) + +// PostgreSQL oids for common types +const ( + BoolOID = 16 + ByteaOID = 17 + QCharOID = 18 + NameOID = 19 + Int8OID = 20 + Int2OID = 21 + Int4OID = 23 + TextOID = 25 + OIDOID = 26 + TIDOID = 27 + XIDOID = 28 + CIDOID = 29 + JSONOID = 114 + PointOID = 600 + LsegOID = 601 + PathOID = 602 + BoxOID = 603 + PolygonOID = 604 + LineOID = 628 + CIDROID = 650 + CIDRArrayOID = 651 + Float4OID = 700 + Float8OID = 701 + CircleOID = 718 + UnknownOID = 705 + MacaddrOID = 829 + InetOID = 869 + BoolArrayOID = 1000 + Int2ArrayOID = 1005 + Int4ArrayOID = 1007 + TextArrayOID = 1009 + ByteaArrayOID = 1001 + BPCharArrayOID = 1014 + VarcharArrayOID = 1015 + Int8ArrayOID = 1016 + Float4ArrayOID = 1021 + Float8ArrayOID = 1022 + ACLItemOID = 1033 + ACLItemArrayOID = 1034 + InetArrayOID = 1041 + BPCharOID = 1042 + VarcharOID = 1043 + DateOID = 1082 + TimeOID = 1083 + TimestampOID = 1114 + TimestampArrayOID = 1115 + DateArrayOID = 1182 + TimestamptzOID = 1184 + TimestamptzArrayOID = 1185 + IntervalOID = 1186 + NumericArrayOID = 1231 + BitOID = 1560 + VarbitOID = 1562 + NumericOID = 1700 + RecordOID = 2249 + UUIDOID = 2950 + UUIDArrayOID = 2951 + JSONBOID = 3802 + DaterangeOID = 3912 + Int4rangeOID = 3904 + NumrangeOID = 3906 + TsrangeOID = 3908 + TstzrangeOID = 3910 + Int8rangeOID = 3926 +) + +type Status byte + +const ( + Undefined Status = iota + Null + Present +) + +type InfinityModifier int8 + +const ( + Infinity InfinityModifier = 1 + None InfinityModifier = 0 + NegativeInfinity InfinityModifier = -Infinity +) + +func (im InfinityModifier) String() string { + switch im { + case None: + return "none" + case Infinity: + return "infinity" + case NegativeInfinity: + return "-infinity" + default: + return "invalid" + } +} + +// PostgreSQL format codes +const ( + TextFormatCode = 0 + BinaryFormatCode = 1 +) + +type Value interface { + // Set converts and assigns src to itself. + Set(src interface{}) error + + // Get returns the simplest representation of Value. If the Value is Null or + // Undefined that is the return value. If no simpler representation is + // possible, then Get() returns Value. + Get() interface{} + + // AssignTo converts and assigns the Value to dst. It MUST make a deep copy of + // any reference types. + AssignTo(dst interface{}) error +} + +type BinaryDecoder interface { + // DecodeBinary decodes src into BinaryDecoder. If src is nil then the + // original SQL value is NULL. BinaryDecoder takes ownership of src. The + // caller MUST not use it again. + DecodeBinary(ci *ConnInfo, src []byte) error +} + +type TextDecoder interface { + // DecodeText decodes src into TextDecoder. If src is nil then the original + // SQL value is NULL. TextDecoder takes ownership of src. The caller MUST not + // use it again. + DecodeText(ci *ConnInfo, src []byte) error +} + +// BinaryEncoder is implemented by types that can encode themselves into the +// PostgreSQL binary wire format. +type BinaryEncoder interface { + // EncodeBinary should append the binary format of self to buf. If self is the + // SQL value NULL then append nothing and return (nil, nil). The caller of + // EncodeBinary is responsible for writing the correct NULL value or the + // length of the data written. + EncodeBinary(ci *ConnInfo, buf []byte) (newBuf []byte, err error) +} + +// TextEncoder is implemented by types that can encode themselves into the +// PostgreSQL text wire format. +type TextEncoder interface { + // EncodeText should append the text format of self to buf. If self is the + // SQL value NULL then append nothing and return (nil, nil). The caller of + // EncodeText is responsible for writing the correct NULL value or the + // length of the data written. + EncodeText(ci *ConnInfo, buf []byte) (newBuf []byte, err error) +} + +var errUndefined = errors.New("cannot encode status undefined") +var errBadStatus = errors.New("invalid status") + +type DataType struct { + Value Value + Name string + OID uint32 +} + +type ConnInfo struct { + oidToDataType map[uint32]*DataType + nameToDataType map[string]*DataType + reflectTypeToDataType map[reflect.Type]*DataType + oidToParamFormatCode map[uint32]int16 + oidToResultFormatCode map[uint32]int16 +} + +func NewConnInfo() *ConnInfo { + ci := &ConnInfo{ + oidToDataType: make(map[uint32]*DataType, 128), + nameToDataType: make(map[string]*DataType, 128), + reflectTypeToDataType: make(map[reflect.Type]*DataType, 128), + oidToParamFormatCode: make(map[uint32]int16, 128), + oidToResultFormatCode: make(map[uint32]int16, 128), + } + + ci.RegisterDataType(DataType{Value: &ACLItemArray{}, Name: "_aclitem", OID: ACLItemArrayOID}) + ci.RegisterDataType(DataType{Value: &BoolArray{}, Name: "_bool", OID: BoolArrayOID}) + ci.RegisterDataType(DataType{Value: &BPCharArray{}, Name: "_bpchar", OID: BPCharArrayOID}) + ci.RegisterDataType(DataType{Value: &ByteaArray{}, Name: "_bytea", OID: ByteaArrayOID}) + ci.RegisterDataType(DataType{Value: &CIDRArray{}, Name: "_cidr", OID: CIDRArrayOID}) + ci.RegisterDataType(DataType{Value: &DateArray{}, Name: "_date", OID: DateArrayOID}) + ci.RegisterDataType(DataType{Value: &Float4Array{}, Name: "_float4", OID: Float4ArrayOID}) + ci.RegisterDataType(DataType{Value: &Float8Array{}, Name: "_float8", OID: Float8ArrayOID}) + ci.RegisterDataType(DataType{Value: &InetArray{}, Name: "_inet", OID: InetArrayOID}) + ci.RegisterDataType(DataType{Value: &Int2Array{}, Name: "_int2", OID: Int2ArrayOID}) + ci.RegisterDataType(DataType{Value: &Int4Array{}, Name: "_int4", OID: Int4ArrayOID}) + ci.RegisterDataType(DataType{Value: &Int8Array{}, Name: "_int8", OID: Int8ArrayOID}) + ci.RegisterDataType(DataType{Value: &NumericArray{}, Name: "_numeric", OID: NumericArrayOID}) + ci.RegisterDataType(DataType{Value: &TextArray{}, Name: "_text", OID: TextArrayOID}) + ci.RegisterDataType(DataType{Value: &TimestampArray{}, Name: "_timestamp", OID: TimestampArrayOID}) + ci.RegisterDataType(DataType{Value: &TimestamptzArray{}, Name: "_timestamptz", OID: TimestamptzArrayOID}) + ci.RegisterDataType(DataType{Value: &UUIDArray{}, Name: "_uuid", OID: UUIDArrayOID}) + ci.RegisterDataType(DataType{Value: &VarcharArray{}, Name: "_varchar", OID: VarcharArrayOID}) + ci.RegisterDataType(DataType{Value: &ACLItem{}, Name: "aclitem", OID: ACLItemOID}) + ci.RegisterDataType(DataType{Value: &Bit{}, Name: "bit", OID: BitOID}) + ci.RegisterDataType(DataType{Value: &Bool{}, Name: "bool", OID: BoolOID}) + ci.RegisterDataType(DataType{Value: &Box{}, Name: "box", OID: BoxOID}) + ci.RegisterDataType(DataType{Value: &BPChar{}, Name: "bpchar", OID: BPCharOID}) + ci.RegisterDataType(DataType{Value: &Bytea{}, Name: "bytea", OID: ByteaOID}) + ci.RegisterDataType(DataType{Value: &QChar{}, Name: "char", OID: QCharOID}) + ci.RegisterDataType(DataType{Value: &CID{}, Name: "cid", OID: CIDOID}) + ci.RegisterDataType(DataType{Value: &CIDR{}, Name: "cidr", OID: CIDROID}) + ci.RegisterDataType(DataType{Value: &Circle{}, Name: "circle", OID: CircleOID}) + ci.RegisterDataType(DataType{Value: &Date{}, Name: "date", OID: DateOID}) + ci.RegisterDataType(DataType{Value: &Daterange{}, Name: "daterange", OID: DaterangeOID}) + ci.RegisterDataType(DataType{Value: &Float4{}, Name: "float4", OID: Float4OID}) + ci.RegisterDataType(DataType{Value: &Float8{}, Name: "float8", OID: Float8OID}) + ci.RegisterDataType(DataType{Value: &Inet{}, Name: "inet", OID: InetOID}) + ci.RegisterDataType(DataType{Value: &Int2{}, Name: "int2", OID: Int2OID}) + ci.RegisterDataType(DataType{Value: &Int4{}, Name: "int4", OID: Int4OID}) + ci.RegisterDataType(DataType{Value: &Int4range{}, Name: "int4range", OID: Int4rangeOID}) + ci.RegisterDataType(DataType{Value: &Int8{}, Name: "int8", OID: Int8OID}) + ci.RegisterDataType(DataType{Value: &Int8range{}, Name: "int8range", OID: Int8rangeOID}) + ci.RegisterDataType(DataType{Value: &Interval{}, Name: "interval", OID: IntervalOID}) + ci.RegisterDataType(DataType{Value: &JSON{}, Name: "json", OID: JSONOID}) + ci.RegisterDataType(DataType{Value: &JSONB{}, Name: "jsonb", OID: JSONBOID}) + ci.RegisterDataType(DataType{Value: &Line{}, Name: "line", OID: LineOID}) + ci.RegisterDataType(DataType{Value: &Lseg{}, Name: "lseg", OID: LsegOID}) + ci.RegisterDataType(DataType{Value: &Macaddr{}, Name: "macaddr", OID: MacaddrOID}) + ci.RegisterDataType(DataType{Value: &Name{}, Name: "name", OID: NameOID}) + ci.RegisterDataType(DataType{Value: &Numeric{}, Name: "numeric", OID: NumericOID}) + ci.RegisterDataType(DataType{Value: &Numrange{}, Name: "numrange", OID: NumrangeOID}) + ci.RegisterDataType(DataType{Value: &OIDValue{}, Name: "oid", OID: OIDOID}) + ci.RegisterDataType(DataType{Value: &Path{}, Name: "path", OID: PathOID}) + ci.RegisterDataType(DataType{Value: &Point{}, Name: "point", OID: PointOID}) + ci.RegisterDataType(DataType{Value: &Polygon{}, Name: "polygon", OID: PolygonOID}) + ci.RegisterDataType(DataType{Value: &Record{}, Name: "record", OID: RecordOID}) + ci.RegisterDataType(DataType{Value: &Text{}, Name: "text", OID: TextOID}) + ci.RegisterDataType(DataType{Value: &TID{}, Name: "tid", OID: TIDOID}) + ci.RegisterDataType(DataType{Value: &Time{}, Name: "time", OID: TimeOID}) + ci.RegisterDataType(DataType{Value: &Timestamp{}, Name: "timestamp", OID: TimestampOID}) + ci.RegisterDataType(DataType{Value: &Timestamptz{}, Name: "timestamptz", OID: TimestamptzOID}) + ci.RegisterDataType(DataType{Value: &Tsrange{}, Name: "tsrange", OID: TsrangeOID}) + ci.RegisterDataType(DataType{Value: &Tstzrange{}, Name: "tstzrange", OID: TstzrangeOID}) + ci.RegisterDataType(DataType{Value: &Unknown{}, Name: "unknown", OID: UnknownOID}) + ci.RegisterDataType(DataType{Value: &UUID{}, Name: "uuid", OID: UUIDOID}) + ci.RegisterDataType(DataType{Value: &Varbit{}, Name: "varbit", OID: VarbitOID}) + ci.RegisterDataType(DataType{Value: &Varchar{}, Name: "varchar", OID: VarcharOID}) + ci.RegisterDataType(DataType{Value: &XID{}, Name: "xid", OID: XIDOID}) + + return ci +} + +func (ci *ConnInfo) InitializeDataTypes(nameOIDs map[string]uint32) { + for name, oid := range nameOIDs { + var value Value + if t, ok := nameValues[name]; ok { + value = reflect.New(reflect.ValueOf(t).Elem().Type()).Interface().(Value) + } else { + value = &GenericText{} + } + ci.RegisterDataType(DataType{Value: value, Name: name, OID: oid}) + } +} + +func (ci *ConnInfo) RegisterDataType(t DataType) { + ci.oidToDataType[t.OID] = &t + ci.nameToDataType[t.Name] = &t + ci.reflectTypeToDataType[reflect.ValueOf(t.Value).Type()] = &t + + { + var formatCode int16 + if _, ok := t.Value.(BinaryEncoder); ok { + formatCode = BinaryFormatCode + } + ci.oidToParamFormatCode[t.OID] = formatCode + } + + { + var formatCode int16 + if _, ok := t.Value.(BinaryDecoder); ok { + formatCode = BinaryFormatCode + } + ci.oidToResultFormatCode[t.OID] = formatCode + } +} + +func (ci *ConnInfo) DataTypeForOID(oid uint32) (*DataType, bool) { + dt, ok := ci.oidToDataType[oid] + return dt, ok +} + +func (ci *ConnInfo) DataTypeForName(name string) (*DataType, bool) { + dt, ok := ci.nameToDataType[name] + return dt, ok +} + +func (ci *ConnInfo) DataTypeForValue(v Value) (*DataType, bool) { + dt, ok := ci.reflectTypeToDataType[reflect.ValueOf(v).Type()] + return dt, ok +} + +func (ci *ConnInfo) ParamFormatCodeForOID(oid uint32) int16 { + fc, ok := ci.oidToParamFormatCode[oid] + if ok { + return fc + } + return TextFormatCode +} + +func (ci *ConnInfo) ResultFormatCodeForOID(oid uint32) int16 { + fc, ok := ci.oidToResultFormatCode[oid] + if ok { + return fc + } + return TextFormatCode +} + +// DeepCopy makes a deep copy of the ConnInfo. +func (ci *ConnInfo) DeepCopy() *ConnInfo { + ci2 := &ConnInfo{ + oidToDataType: make(map[uint32]*DataType, len(ci.oidToDataType)), + nameToDataType: make(map[string]*DataType, len(ci.nameToDataType)), + reflectTypeToDataType: make(map[reflect.Type]*DataType, len(ci.reflectTypeToDataType)), + } + + for _, dt := range ci.oidToDataType { + ci2.RegisterDataType(DataType{ + Value: reflect.New(reflect.ValueOf(dt.Value).Elem().Type()).Interface().(Value), + Name: dt.Name, + OID: dt.OID, + }) + } + + return ci2 +} + +func (ci *ConnInfo) Scan(oid uint32, formatCode int16, buf []byte, dest interface{}) error { + if dest, ok := dest.(BinaryDecoder); ok && formatCode == BinaryFormatCode { + return dest.DecodeBinary(ci, buf) + } + + if dest, ok := dest.(TextDecoder); ok && formatCode == TextFormatCode { + return dest.DecodeText(ci, buf) + } + + if dt, ok := ci.DataTypeForOID(oid); ok { + value := dt.Value + switch formatCode { + case TextFormatCode: + if textDecoder, ok := value.(TextDecoder); ok { + err := textDecoder.DecodeText(ci, buf) + if err != nil { + return err + } + } else { + return errors.Errorf("%T is not a pgtype.TextDecoder", value) + } + case BinaryFormatCode: + if binaryDecoder, ok := value.(BinaryDecoder); ok { + err := binaryDecoder.DecodeBinary(ci, buf) + if err != nil { + return err + } + } else { + return errors.Errorf("%T is not a pgtype.BinaryDecoder", value) + } + default: + return errors.Errorf("unknown format code: %v", formatCode) + } + + if scanner, ok := dest.(sql.Scanner); ok { + sqlSrc, err := DatabaseSQLValue(ci, value) + if err != nil { + return err + } + return scanner.Scan(sqlSrc) + } else { + return value.AssignTo(dest) + } + } + + // We might be given a pointer to something that implements the decoder interface(s), + // even though the pointer itself doesn't. + refVal := reflect.ValueOf(dest) + if refVal.Kind() == reflect.Ptr && refVal.Type().Elem().Kind() == reflect.Ptr { + // If the database returned NULL, then we set dest as nil to indicate that. + if buf == nil { + nilPtr := reflect.Zero(refVal.Type().Elem()) + refVal.Elem().Set(nilPtr) + return nil + } + + // We need to allocate an element, and set the destination to it + // Then we can retry as that element. + elemPtr := reflect.New(refVal.Type().Elem().Elem()) + refVal.Elem().Set(elemPtr) + return ci.Scan(oid, formatCode, buf, elemPtr.Interface()) + } + + return scanUnknownType(oid, formatCode, buf, dest) +} + +func scanUnknownType(oid uint32, formatCode int16, buf []byte, dest interface{}) error { + switch dest := dest.(type) { + case *string: + if formatCode == BinaryFormatCode { + return errors.Errorf("unknown oid %d in binary format cannot be scanned into %T", oid, dest) + } + *dest = string(buf) + return nil + case *[]byte: + *dest = buf + return nil + default: + if nextDst, retry := GetAssignToDstType(dest); retry { + return scanUnknownType(oid, formatCode, buf, nextDst) + } + return errors.Errorf("unknown oid %d cannot be scanned into %T", oid, dest) + } +} + +var nameValues map[string]Value + +func init() { + nameValues = map[string]Value{ + "_aclitem": &ACLItemArray{}, + "_bool": &BoolArray{}, + "_bpchar": &BPCharArray{}, + "_bytea": &ByteaArray{}, + "_cidr": &CIDRArray{}, + "_date": &DateArray{}, + "_float4": &Float4Array{}, + "_float8": &Float8Array{}, + "_inet": &InetArray{}, + "_int2": &Int2Array{}, + "_int4": &Int4Array{}, + "_int8": &Int8Array{}, + "_numeric": &NumericArray{}, + "_text": &TextArray{}, + "_timestamp": &TimestampArray{}, + "_timestamptz": &TimestamptzArray{}, + "_uuid": &UUIDArray{}, + "_varchar": &VarcharArray{}, + "aclitem": &ACLItem{}, + "bit": &Bit{}, + "bool": &Bool{}, + "box": &Box{}, + "bpchar": &BPChar{}, + "bytea": &Bytea{}, + "char": &QChar{}, + "cid": &CID{}, + "cidr": &CIDR{}, + "circle": &Circle{}, + "date": &Date{}, + "daterange": &Daterange{}, + "float4": &Float4{}, + "float8": &Float8{}, + "hstore": &Hstore{}, + "inet": &Inet{}, + "int2": &Int2{}, + "int4": &Int4{}, + "int4range": &Int4range{}, + "int8": &Int8{}, + "int8range": &Int8range{}, + "interval": &Interval{}, + "json": &JSON{}, + "jsonb": &JSONB{}, + "line": &Line{}, + "lseg": &Lseg{}, + "macaddr": &Macaddr{}, + "name": &Name{}, + "numeric": &Numeric{}, + "numrange": &Numrange{}, + "oid": &OIDValue{}, + "path": &Path{}, + "point": &Point{}, + "polygon": &Polygon{}, + "record": &Record{}, + "text": &Text{}, + "tid": &TID{}, + "timestamp": &Timestamp{}, + "timestamptz": &Timestamptz{}, + "tsrange": &Tsrange{}, + "tstzrange": &Tstzrange{}, + "unknown": &Unknown{}, + "uuid": &UUID{}, + "varbit": &Varbit{}, + "varchar": &Varchar{}, + "xid": &XID{}, + } +} diff --git a/github.com/jackc/pgtype/pguint32.go b/github.com/jackc/pgtype/pguint32.go new file mode 100644 index 0000000000..a245d2c904 --- /dev/null +++ b/github.com/jackc/pgtype/pguint32.go @@ -0,0 +1,162 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + "math" + "strconv" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +// pguint32 is the core type that is used to implement PostgreSQL types such as +// CID and XID. +type pguint32 struct { + Uint uint32 + Status Status +} + +// Set converts from src to dst. Note that as pguint32 is not a general +// number type Set does not do automatic type conversion as other number +// types do. +func (dst *pguint32) Set(src interface{}) error { + switch value := src.(type) { + case int64: + if value < 0 { + return errors.Errorf("%d is less than minimum value for pguint32", value) + } + if value > math.MaxUint32 { + return errors.Errorf("%d is greater than maximum value for pguint32", value) + } + *dst = pguint32{Uint: uint32(value), Status: Present} + case uint32: + *dst = pguint32{Uint: value, Status: Present} + default: + return errors.Errorf("cannot convert %v to pguint32", value) + } + + return nil +} + +func (dst pguint32) Get() interface{} { + switch dst.Status { + case Present: + return dst.Uint + case Null: + return nil + default: + return dst.Status + } +} + +// AssignTo assigns from src to dst. Note that as pguint32 is not a general number +// type AssignTo does not do automatic type conversion as other number types do. +func (src *pguint32) AssignTo(dst interface{}) error { + switch v := dst.(type) { + case *uint32: + if src.Status == Present { + *v = src.Uint + } else { + return errors.Errorf("cannot assign %v into %T", src, dst) + } + case **uint32: + if src.Status == Present { + n := src.Uint + *v = &n + } else { + *v = nil + } + } + + return nil +} + +func (dst *pguint32) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = pguint32{Status: Null} + return nil + } + + n, err := strconv.ParseUint(string(src), 10, 32) + if err != nil { + return err + } + + *dst = pguint32{Uint: uint32(n), Status: Present} + return nil +} + +func (dst *pguint32) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = pguint32{Status: Null} + return nil + } + + if len(src) != 4 { + return errors.Errorf("invalid length: %v", len(src)) + } + + n := binary.BigEndian.Uint32(src) + *dst = pguint32{Uint: n, Status: Present} + return nil +} + +func (src pguint32) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + return append(buf, strconv.FormatUint(uint64(src.Uint), 10)...), nil +} + +func (src pguint32) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + return pgio.AppendUint32(buf, src.Uint), nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *pguint32) Scan(src interface{}) error { + if src == nil { + *dst = pguint32{Status: Null} + return nil + } + + switch src := src.(type) { + case uint32: + *dst = pguint32{Uint: src, Status: Present} + return nil + case int64: + *dst = pguint32{Uint: uint32(src), Status: Present} + return nil + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src pguint32) Value() (driver.Value, error) { + switch src.Status { + case Present: + return int64(src.Uint), nil + case Null: + return nil, nil + default: + return nil, errUndefined + } +} diff --git a/github.com/jackc/pgtype/point.go b/github.com/jackc/pgtype/point.go new file mode 100644 index 0000000000..87993656c2 --- /dev/null +++ b/github.com/jackc/pgtype/point.go @@ -0,0 +1,142 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + "fmt" + "math" + "strconv" + "strings" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type Vec2 struct { + X float64 + Y float64 +} + +type Point struct { + P Vec2 + Status Status +} + +func (dst *Point) Set(src interface{}) error { + return errors.Errorf("cannot convert %v to Point", src) +} + +func (dst Point) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Point) AssignTo(dst interface{}) error { + return errors.Errorf("cannot assign %v to %T", src, dst) +} + +func (dst *Point) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Point{Status: Null} + return nil + } + + if len(src) < 5 { + return errors.Errorf("invalid length for point: %v", len(src)) + } + + parts := strings.SplitN(string(src[1:len(src)-1]), ",", 2) + if len(parts) < 2 { + return errors.Errorf("invalid format for point") + } + + x, err := strconv.ParseFloat(parts[0], 64) + if err != nil { + return err + } + + y, err := strconv.ParseFloat(parts[1], 64) + if err != nil { + return err + } + + *dst = Point{P: Vec2{x, y}, Status: Present} + return nil +} + +func (dst *Point) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Point{Status: Null} + return nil + } + + if len(src) != 16 { + return errors.Errorf("invalid length for point: %v", len(src)) + } + + x := binary.BigEndian.Uint64(src) + y := binary.BigEndian.Uint64(src[8:]) + + *dst = Point{ + P: Vec2{math.Float64frombits(x), math.Float64frombits(y)}, + Status: Present, + } + return nil +} + +func (src Point) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + return append(buf, fmt.Sprintf(`(%s,%s)`, + strconv.FormatFloat(src.P.X, 'f', -1, 64), + strconv.FormatFloat(src.P.Y, 'f', -1, 64), + )...), nil +} + +func (src Point) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + buf = pgio.AppendUint64(buf, math.Float64bits(src.P.X)) + buf = pgio.AppendUint64(buf, math.Float64bits(src.P.Y)) + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Point) Scan(src interface{}) error { + if src == nil { + *dst = Point{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Point) Value() (driver.Value, error) { + return EncodeValueText(src) +} diff --git a/github.com/jackc/pgtype/polygon.go b/github.com/jackc/pgtype/polygon.go new file mode 100644 index 0000000000..653b04c15f --- /dev/null +++ b/github.com/jackc/pgtype/polygon.go @@ -0,0 +1,177 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + "fmt" + "math" + "strconv" + "strings" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type Polygon struct { + P []Vec2 + Status Status +} + +func (dst *Polygon) Set(src interface{}) error { + return errors.Errorf("cannot convert %v to Polygon", src) +} + +func (dst Polygon) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Polygon) AssignTo(dst interface{}) error { + return errors.Errorf("cannot assign %v to %T", src, dst) +} + +func (dst *Polygon) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Polygon{Status: Null} + return nil + } + + if len(src) < 7 { + return errors.Errorf("invalid length for Polygon: %v", len(src)) + } + + points := make([]Vec2, 0) + + str := string(src[2:]) + + for { + end := strings.IndexByte(str, ',') + x, err := strconv.ParseFloat(str[:end], 64) + if err != nil { + return err + } + + str = str[end+1:] + end = strings.IndexByte(str, ')') + + y, err := strconv.ParseFloat(str[:end], 64) + if err != nil { + return err + } + + points = append(points, Vec2{x, y}) + + if end+3 < len(str) { + str = str[end+3:] + } else { + break + } + } + + *dst = Polygon{P: points, Status: Present} + return nil +} + +func (dst *Polygon) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Polygon{Status: Null} + return nil + } + + if len(src) < 5 { + return errors.Errorf("invalid length for Polygon: %v", len(src)) + } + + pointCount := int(binary.BigEndian.Uint32(src)) + rp := 4 + + if 4+pointCount*16 != len(src) { + return errors.Errorf("invalid length for Polygon with %d points: %v", pointCount, len(src)) + } + + points := make([]Vec2, pointCount) + for i := 0; i < len(points); i++ { + x := binary.BigEndian.Uint64(src[rp:]) + rp += 8 + y := binary.BigEndian.Uint64(src[rp:]) + rp += 8 + points[i] = Vec2{math.Float64frombits(x), math.Float64frombits(y)} + } + + *dst = Polygon{ + P: points, + Status: Present, + } + return nil +} + +func (src Polygon) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + buf = append(buf, '(') + + for i, p := range src.P { + if i > 0 { + buf = append(buf, ',') + } + buf = append(buf, fmt.Sprintf(`(%s,%s)`, + strconv.FormatFloat(p.X, 'f', -1, 64), + strconv.FormatFloat(p.Y, 'f', -1, 64), + )...) + } + + return append(buf, ')'), nil +} + +func (src Polygon) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + buf = pgio.AppendInt32(buf, int32(len(src.P))) + + for _, p := range src.P { + buf = pgio.AppendUint64(buf, math.Float64bits(p.X)) + buf = pgio.AppendUint64(buf, math.Float64bits(p.Y)) + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Polygon) Scan(src interface{}) error { + if src == nil { + *dst = Polygon{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Polygon) Value() (driver.Value, error) { + return EncodeValueText(src) +} diff --git a/github.com/jackc/pgtype/qchar.go b/github.com/jackc/pgtype/qchar.go new file mode 100644 index 0000000000..93964058ab --- /dev/null +++ b/github.com/jackc/pgtype/qchar.go @@ -0,0 +1,153 @@ +package pgtype + +import ( + "math" + "strconv" + + errors "golang.org/x/xerrors" +) + +// QChar is for PostgreSQL's special 8-bit-only "char" type more akin to the C +// language's char type, or Go's byte type. (Note that the name in PostgreSQL +// itself is "char", in double-quotes, and not char.) It gets used a lot in +// PostgreSQL's system tables to hold a single ASCII character value (eg +// pg_class.relkind). It is named Qchar for quoted char to disambiguate from SQL +// standard type char. +// +// Not all possible values of QChar are representable in the text format. +// Therefore, QChar does not implement TextEncoder and TextDecoder. In +// addition, database/sql Scanner and database/sql/driver Value are not +// implemented. +type QChar struct { + Int int8 + Status Status +} + +func (dst *QChar) Set(src interface{}) error { + if src == nil { + *dst = QChar{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + case int8: + *dst = QChar{Int: value, Status: Present} + case uint8: + if value > math.MaxInt8 { + return errors.Errorf("%d is greater than maximum value for QChar", value) + } + *dst = QChar{Int: int8(value), Status: Present} + case int16: + if value < math.MinInt8 { + return errors.Errorf("%d is greater than maximum value for QChar", value) + } + if value > math.MaxInt8 { + return errors.Errorf("%d is greater than maximum value for QChar", value) + } + *dst = QChar{Int: int8(value), Status: Present} + case uint16: + if value > math.MaxInt8 { + return errors.Errorf("%d is greater than maximum value for QChar", value) + } + *dst = QChar{Int: int8(value), Status: Present} + case int32: + if value < math.MinInt8 { + return errors.Errorf("%d is greater than maximum value for QChar", value) + } + if value > math.MaxInt8 { + return errors.Errorf("%d is greater than maximum value for QChar", value) + } + *dst = QChar{Int: int8(value), Status: Present} + case uint32: + if value > math.MaxInt8 { + return errors.Errorf("%d is greater than maximum value for QChar", value) + } + *dst = QChar{Int: int8(value), Status: Present} + case int64: + if value < math.MinInt8 { + return errors.Errorf("%d is greater than maximum value for QChar", value) + } + if value > math.MaxInt8 { + return errors.Errorf("%d is greater than maximum value for QChar", value) + } + *dst = QChar{Int: int8(value), Status: Present} + case uint64: + if value > math.MaxInt8 { + return errors.Errorf("%d is greater than maximum value for QChar", value) + } + *dst = QChar{Int: int8(value), Status: Present} + case int: + if value < math.MinInt8 { + return errors.Errorf("%d is greater than maximum value for QChar", value) + } + if value > math.MaxInt8 { + return errors.Errorf("%d is greater than maximum value for QChar", value) + } + *dst = QChar{Int: int8(value), Status: Present} + case uint: + if value > math.MaxInt8 { + return errors.Errorf("%d is greater than maximum value for QChar", value) + } + *dst = QChar{Int: int8(value), Status: Present} + case string: + num, err := strconv.ParseInt(value, 10, 8) + if err != nil { + return err + } + *dst = QChar{Int: int8(num), Status: Present} + default: + if originalSrc, ok := underlyingNumberType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to QChar", value) + } + + return nil +} + +func (dst QChar) Get() interface{} { + switch dst.Status { + case Present: + return dst.Int + case Null: + return nil + default: + return dst.Status + } +} + +func (src *QChar) AssignTo(dst interface{}) error { + return int64AssignTo(int64(src.Int), src.Status, dst) +} + +func (dst *QChar) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = QChar{Status: Null} + return nil + } + + if len(src) != 1 { + return errors.Errorf(`invalid length for "char": %v`, len(src)) + } + + *dst = QChar{Int: int8(src[0]), Status: Present} + return nil +} + +func (src QChar) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + return append(buf, byte(src.Int)), nil +} diff --git a/github.com/jackc/pgtype/range.go b/github.com/jackc/pgtype/range.go new file mode 100644 index 0000000000..35b80ced27 --- /dev/null +++ b/github.com/jackc/pgtype/range.go @@ -0,0 +1,278 @@ +package pgtype + +import ( + "bytes" + "encoding/binary" + + errors "golang.org/x/xerrors" +) + +type BoundType byte + +const ( + Inclusive = BoundType('i') + Exclusive = BoundType('e') + Unbounded = BoundType('U') + Empty = BoundType('E') +) + +func (bt BoundType) String() string { + return string(bt) +} + +type UntypedTextRange struct { + Lower string + Upper string + LowerType BoundType + UpperType BoundType +} + +func ParseUntypedTextRange(src string) (*UntypedTextRange, error) { + utr := &UntypedTextRange{} + if src == "empty" { + utr.LowerType = Empty + utr.UpperType = Empty + return utr, nil + } + + buf := bytes.NewBufferString(src) + + skipWhitespace(buf) + + r, _, err := buf.ReadRune() + if err != nil { + return nil, errors.Errorf("invalid lower bound: %v", err) + } + switch r { + case '(': + utr.LowerType = Exclusive + case '[': + utr.LowerType = Inclusive + default: + return nil, errors.Errorf("missing lower bound, instead got: %v", string(r)) + } + + r, _, err = buf.ReadRune() + if err != nil { + return nil, errors.Errorf("invalid lower value: %v", err) + } + buf.UnreadRune() + + if r == ',' { + utr.LowerType = Unbounded + } else { + utr.Lower, err = rangeParseValue(buf) + if err != nil { + return nil, errors.Errorf("invalid lower value: %v", err) + } + } + + r, _, err = buf.ReadRune() + if err != nil { + return nil, errors.Errorf("missing range separator: %v", err) + } + if r != ',' { + return nil, errors.Errorf("missing range separator: %v", r) + } + + r, _, err = buf.ReadRune() + if err != nil { + return nil, errors.Errorf("invalid upper value: %v", err) + } + + if r == ')' || r == ']' { + utr.UpperType = Unbounded + } else { + buf.UnreadRune() + utr.Upper, err = rangeParseValue(buf) + if err != nil { + return nil, errors.Errorf("invalid upper value: %v", err) + } + + r, _, err = buf.ReadRune() + if err != nil { + return nil, errors.Errorf("missing upper bound: %v", err) + } + switch r { + case ')': + utr.UpperType = Exclusive + case ']': + utr.UpperType = Inclusive + default: + return nil, errors.Errorf("missing upper bound, instead got: %v", string(r)) + } + } + + skipWhitespace(buf) + + if buf.Len() > 0 { + return nil, errors.Errorf("unexpected trailing data: %v", buf.String()) + } + + return utr, nil +} + +func rangeParseValue(buf *bytes.Buffer) (string, error) { + r, _, err := buf.ReadRune() + if err != nil { + return "", err + } + if r == '"' { + return rangeParseQuotedValue(buf) + } + buf.UnreadRune() + + s := &bytes.Buffer{} + + for { + r, _, err := buf.ReadRune() + if err != nil { + return "", err + } + + switch r { + case '\\': + r, _, err = buf.ReadRune() + if err != nil { + return "", err + } + case ',', '[', ']', '(', ')': + buf.UnreadRune() + return s.String(), nil + } + + s.WriteRune(r) + } +} + +func rangeParseQuotedValue(buf *bytes.Buffer) (string, error) { + s := &bytes.Buffer{} + + for { + r, _, err := buf.ReadRune() + if err != nil { + return "", err + } + + switch r { + case '\\': + r, _, err = buf.ReadRune() + if err != nil { + return "", err + } + case '"': + r, _, err = buf.ReadRune() + if err != nil { + return "", err + } + if r != '"' { + buf.UnreadRune() + return s.String(), nil + } + } + s.WriteRune(r) + } +} + +type UntypedBinaryRange struct { + Lower []byte + Upper []byte + LowerType BoundType + UpperType BoundType +} + +// 0 = () = 00000 +// 1 = empty = 00001 +// 2 = [) = 00010 +// 4 = (] = 00100 +// 6 = [] = 00110 +// 8 = ) = 01000 +// 12 = ] = 01100 +// 16 = ( = 10000 +// 18 = [ = 10010 +// 24 = = 11000 + +const emptyMask = 1 +const lowerInclusiveMask = 2 +const upperInclusiveMask = 4 +const lowerUnboundedMask = 8 +const upperUnboundedMask = 16 + +func ParseUntypedBinaryRange(src []byte) (*UntypedBinaryRange, error) { + ubr := &UntypedBinaryRange{} + + if len(src) == 0 { + return nil, errors.Errorf("range too short: %v", len(src)) + } + + rangeType := src[0] + rp := 1 + + if rangeType&emptyMask > 0 { + if len(src[rp:]) > 0 { + return nil, errors.Errorf("unexpected trailing bytes parsing empty range: %v", len(src[rp:])) + } + ubr.LowerType = Empty + ubr.UpperType = Empty + return ubr, nil + } + + if rangeType&lowerInclusiveMask > 0 { + ubr.LowerType = Inclusive + } else if rangeType&lowerUnboundedMask > 0 { + ubr.LowerType = Unbounded + } else { + ubr.LowerType = Exclusive + } + + if rangeType&upperInclusiveMask > 0 { + ubr.UpperType = Inclusive + } else if rangeType&upperUnboundedMask > 0 { + ubr.UpperType = Unbounded + } else { + ubr.UpperType = Exclusive + } + + if ubr.LowerType == Unbounded && ubr.UpperType == Unbounded { + if len(src[rp:]) > 0 { + return nil, errors.Errorf("unexpected trailing bytes parsing unbounded range: %v", len(src[rp:])) + } + return ubr, nil + } + + if len(src[rp:]) < 4 { + return nil, errors.Errorf("too few bytes for size: %v", src[rp:]) + } + valueLen := int(binary.BigEndian.Uint32(src[rp:])) + rp += 4 + + val := src[rp : rp+valueLen] + rp += valueLen + + if ubr.LowerType != Unbounded { + ubr.Lower = val + } else { + ubr.Upper = val + if len(src[rp:]) > 0 { + return nil, errors.Errorf("unexpected trailing bytes parsing range: %v", len(src[rp:])) + } + return ubr, nil + } + + if ubr.UpperType != Unbounded { + if len(src[rp:]) < 4 { + return nil, errors.Errorf("too few bytes for size: %v", src[rp:]) + } + valueLen := int(binary.BigEndian.Uint32(src[rp:])) + rp += 4 + ubr.Upper = src[rp : rp+valueLen] + rp += valueLen + } + + if len(src[rp:]) > 0 { + return nil, errors.Errorf("unexpected trailing bytes parsing range: %v", len(src[rp:])) + } + + return ubr, nil + +} diff --git a/github.com/jackc/pgtype/record.go b/github.com/jackc/pgtype/record.go new file mode 100644 index 0000000000..5c9d7a02f8 --- /dev/null +++ b/github.com/jackc/pgtype/record.go @@ -0,0 +1,137 @@ +package pgtype + +import ( + "encoding/binary" + "reflect" + + errors "golang.org/x/xerrors" +) + +// Record is the generic PostgreSQL record type such as is created with the +// "row" function. Record only implements BinaryEncoder and Value. The text +// format output format from PostgreSQL does not include type information and is +// therefore impossible to decode. No encoders are implemented because +// PostgreSQL does not support input of generic records. +type Record struct { + Fields []Value + Status Status +} + +func (dst *Record) Set(src interface{}) error { + if src == nil { + *dst = Record{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + case []Value: + *dst = Record{Fields: value, Status: Present} + default: + return errors.Errorf("cannot convert %v to Record", src) + } + + return nil +} + +func (dst Record) Get() interface{} { + switch dst.Status { + case Present: + return dst.Fields + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Record) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + case *[]Value: + *v = make([]Value, len(src.Fields)) + copy(*v, src.Fields) + return nil + case *[]interface{}: + *v = make([]interface{}, len(src.Fields)) + for i := range *v { + (*v)[i] = src.Fields[i].Get() + } + return nil + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *Record) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Record{Status: Null} + return nil + } + + rp := 0 + + if len(src[rp:]) < 4 { + return errors.Errorf("Record incomplete %v", src) + } + fieldCount := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + + fields := make([]Value, fieldCount) + + for i := 0; i < fieldCount; i++ { + if len(src[rp:]) < 8 { + return errors.Errorf("Record incomplete %v", src) + } + fieldOID := binary.BigEndian.Uint32(src[rp:]) + rp += 4 + + fieldLen := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + + var binaryDecoder BinaryDecoder + if dt, ok := ci.DataTypeForOID(fieldOID); ok { + binaryDecoder, _ = dt.Value.(BinaryDecoder) + } + if binaryDecoder == nil { + return errors.Errorf("unknown oid while decoding record: %v", fieldOID) + } + + var fieldBytes []byte + if fieldLen >= 0 { + if len(src[rp:]) < fieldLen { + return errors.Errorf("Record incomplete %v", src) + } + fieldBytes = src[rp : rp+fieldLen] + rp += fieldLen + } + + // Duplicate struct to scan into + binaryDecoder = reflect.New(reflect.ValueOf(binaryDecoder).Elem().Type()).Interface().(BinaryDecoder) + + if err := binaryDecoder.DecodeBinary(ci, fieldBytes); err != nil { + return err + } + + fields[i] = binaryDecoder.(Value) + } + + *dst = Record{Fields: fields, Status: Present} + + return nil +} diff --git a/github.com/jackc/pgtype/text.go b/github.com/jackc/pgtype/text.go new file mode 100644 index 0000000000..1f5d2a375b --- /dev/null +++ b/github.com/jackc/pgtype/text.go @@ -0,0 +1,175 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/json" + + errors "golang.org/x/xerrors" +) + +type Text struct { + String string + Status Status +} + +func (dst *Text) Set(src interface{}) error { + if src == nil { + *dst = Text{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + case string: + *dst = Text{String: value, Status: Present} + case *string: + if value == nil { + *dst = Text{Status: Null} + } else { + *dst = Text{String: *value, Status: Present} + } + case []byte: + if value == nil { + *dst = Text{Status: Null} + } else { + *dst = Text{String: string(value), Status: Present} + } + default: + if originalSrc, ok := underlyingStringType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to Text", value) + } + + return nil +} + +func (dst Text) Get() interface{} { + switch dst.Status { + case Present: + return dst.String + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Text) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + case *string: + *v = src.String + return nil + case *[]byte: + *v = make([]byte, len(src.String)) + copy(*v, src.String) + return nil + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *Text) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Text{Status: Null} + return nil + } + + *dst = Text{String: string(src), Status: Present} + return nil +} + +func (dst *Text) DecodeBinary(ci *ConnInfo, src []byte) error { + return dst.DecodeText(ci, src) +} + +func (src Text) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + return append(buf, src.String...), nil +} + +func (src Text) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + return src.EncodeText(ci, buf) +} + +// Scan implements the database/sql Scanner interface. +func (dst *Text) Scan(src interface{}) error { + if src == nil { + *dst = Text{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Text) Value() (driver.Value, error) { + switch src.Status { + case Present: + return src.String, nil + case Null: + return nil, nil + default: + return nil, errUndefined + } +} + +func (src Text) MarshalJSON() ([]byte, error) { + switch src.Status { + case Present: + return json.Marshal(src.String) + case Null: + return []byte("null"), nil + case Undefined: + return nil, errUndefined + } + + return nil, errBadStatus +} + +func (dst *Text) UnmarshalJSON(b []byte) error { + var s *string + err := json.Unmarshal(b, &s) + if err != nil { + return err + } + + if s == nil { + *dst = Text{Status: Null} + } else { + *dst = Text{String: *s, Status: Present} + } + + return nil +} diff --git a/github.com/jackc/pgtype/text_array.go b/github.com/jackc/pgtype/text_array.go new file mode 100644 index 0000000000..2130af8448 --- /dev/null +++ b/github.com/jackc/pgtype/text_array.go @@ -0,0 +1,320 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type TextArray struct { + Elements []Text + Dimensions []ArrayDimension + Status Status +} + +func (dst *TextArray) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = TextArray{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + + case []string: + if value == nil { + *dst = TextArray{Status: Null} + } else if len(value) == 0 { + *dst = TextArray{Status: Present} + } else { + elements := make([]Text, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = TextArray{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []Text: + if value == nil { + *dst = TextArray{Status: Null} + } else if len(value) == 0 { + *dst = TextArray{Status: Present} + } else { + *dst = TextArray{ + Elements: value, + Dimensions: []ArrayDimension{{Length: int32(len(value)), LowerBound: 1}}, + Status: Present, + } + } + default: + if originalSrc, ok := underlyingSliceType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to TextArray", value) + } + + return nil +} + +func (dst TextArray) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *TextArray) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + + case *[]string: + *v = make([]string, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *TextArray) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = TextArray{Status: Null} + return nil + } + + uta, err := ParseUntypedTextArray(string(src)) + if err != nil { + return err + } + + var elements []Text + + if len(uta.Elements) > 0 { + elements = make([]Text, len(uta.Elements)) + + for i, s := range uta.Elements { + var elem Text + var elemSrc []byte + if s != "NULL" { + elemSrc = []byte(s) + } + err = elem.DecodeText(ci, elemSrc) + if err != nil { + return err + } + + elements[i] = elem + } + } + + *dst = TextArray{Elements: elements, Dimensions: uta.Dimensions, Status: Present} + + return nil +} + +func (dst *TextArray) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = TextArray{Status: Null} + return nil + } + + var arrayHeader ArrayHeader + rp, err := arrayHeader.DecodeBinary(ci, src) + if err != nil { + return err + } + + if len(arrayHeader.Dimensions) == 0 { + *dst = TextArray{Dimensions: arrayHeader.Dimensions, Status: Present} + return nil + } + + elementCount := arrayHeader.Dimensions[0].Length + for _, d := range arrayHeader.Dimensions[1:] { + elementCount *= d.Length + } + + elements := make([]Text, elementCount) + + for i := range elements { + elemLen := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + var elemSrc []byte + if elemLen >= 0 { + elemSrc = src[rp : rp+elemLen] + rp += elemLen + } + err = elements[i].DecodeBinary(ci, elemSrc) + if err != nil { + return err + } + } + + *dst = TextArray{Elements: elements, Dimensions: arrayHeader.Dimensions, Status: Present} + return nil +} + +func (src TextArray) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + if len(src.Dimensions) == 0 { + return append(buf, '{', '}'), nil + } + + buf = EncodeTextArrayDimensions(buf, src.Dimensions) + + // dimElemCounts is the multiples of elements that each array lies on. For + // example, a single dimension array of length 4 would have a dimElemCounts of + // [4]. A multi-dimensional array of lengths [3,5,2] would have a + // dimElemCounts of [30,10,2]. This is used to simplify when to render a '{' + // or '}'. + dimElemCounts := make([]int, len(src.Dimensions)) + dimElemCounts[len(src.Dimensions)-1] = int(src.Dimensions[len(src.Dimensions)-1].Length) + for i := len(src.Dimensions) - 2; i > -1; i-- { + dimElemCounts[i] = int(src.Dimensions[i].Length) * dimElemCounts[i+1] + } + + inElemBuf := make([]byte, 0, 32) + for i, elem := range src.Elements { + if i > 0 { + buf = append(buf, ',') + } + + for _, dec := range dimElemCounts { + if i%dec == 0 { + buf = append(buf, '{') + } + } + + elemBuf, err := elem.EncodeText(ci, inElemBuf) + if err != nil { + return nil, err + } + if elemBuf == nil { + buf = append(buf, `NULL`...) + } else { + buf = append(buf, QuoteArrayElementIfNeeded(string(elemBuf))...) + } + + for _, dec := range dimElemCounts { + if (i+1)%dec == 0 { + buf = append(buf, '}') + } + } + } + + return buf, nil +} + +func (src TextArray) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + arrayHeader := ArrayHeader{ + Dimensions: src.Dimensions, + } + + if dt, ok := ci.DataTypeForName("text"); ok { + arrayHeader.ElementOID = int32(dt.OID) + } else { + return nil, errors.Errorf("unable to find oid for type name %v", "text") + } + + for i := range src.Elements { + if src.Elements[i].Status == Null { + arrayHeader.ContainsNull = true + break + } + } + + buf = arrayHeader.EncodeBinary(ci, buf) + + for i := range src.Elements { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + elemBuf, err := src.Elements[i].EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if elemBuf != nil { + buf = elemBuf + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *TextArray) Scan(src interface{}) error { + if src == nil { + return dst.DecodeText(nil, nil) + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src TextArray) Value() (driver.Value, error) { + buf, err := src.EncodeText(nil, nil) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + + return string(buf), nil +} diff --git a/github.com/jackc/pgtype/tid.go b/github.com/jackc/pgtype/tid.go new file mode 100644 index 0000000000..98b95e2a55 --- /dev/null +++ b/github.com/jackc/pgtype/tid.go @@ -0,0 +1,144 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + "fmt" + "strconv" + "strings" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +// TID is PostgreSQL's Tuple Identifier type. +// +// When one does +// +// select ctid, * from some_table; +// +// it is the data type of the ctid hidden system column. +// +// It is currently implemented as a pair unsigned two byte integers. +// Its conversion functions can be found in src/backend/utils/adt/tid.c +// in the PostgreSQL sources. +type TID struct { + BlockNumber uint32 + OffsetNumber uint16 + Status Status +} + +func (dst *TID) Set(src interface{}) error { + return errors.Errorf("cannot convert %v to TID", src) +} + +func (dst TID) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *TID) AssignTo(dst interface{}) error { + return errors.Errorf("cannot assign %v to %T", src, dst) +} + +func (dst *TID) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = TID{Status: Null} + return nil + } + + if len(src) < 5 { + return errors.Errorf("invalid length for tid: %v", len(src)) + } + + parts := strings.SplitN(string(src[1:len(src)-1]), ",", 2) + if len(parts) < 2 { + return errors.Errorf("invalid format for tid") + } + + blockNumber, err := strconv.ParseUint(parts[0], 10, 32) + if err != nil { + return err + } + + offsetNumber, err := strconv.ParseUint(parts[1], 10, 16) + if err != nil { + return err + } + + *dst = TID{BlockNumber: uint32(blockNumber), OffsetNumber: uint16(offsetNumber), Status: Present} + return nil +} + +func (dst *TID) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = TID{Status: Null} + return nil + } + + if len(src) != 6 { + return errors.Errorf("invalid length for tid: %v", len(src)) + } + + *dst = TID{ + BlockNumber: binary.BigEndian.Uint32(src), + OffsetNumber: binary.BigEndian.Uint16(src[4:]), + Status: Present, + } + return nil +} + +func (src TID) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + buf = append(buf, fmt.Sprintf(`(%d,%d)`, src.BlockNumber, src.OffsetNumber)...) + return buf, nil +} + +func (src TID) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + buf = pgio.AppendUint32(buf, src.BlockNumber) + buf = pgio.AppendUint16(buf, src.OffsetNumber) + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *TID) Scan(src interface{}) error { + if src == nil { + *dst = TID{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src TID) Value() (driver.Value, error) { + return EncodeValueText(src) +} diff --git a/github.com/jackc/pgtype/time.go b/github.com/jackc/pgtype/time.go new file mode 100644 index 0000000000..16a2a39391 --- /dev/null +++ b/github.com/jackc/pgtype/time.go @@ -0,0 +1,226 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + "fmt" + "strconv" + "time" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +// Time represents the PostgreSQL time type. The PostgreSQL time is a time of day without time zone. +// +// Time is represented as the number of microseconds since midnight in the same way that PostgreSQL does. Other time +// and date types in pgtype can use time.Time as the underlying representation. However, pgtype.Time type cannot due +// to needing to handle 24:00:00. time.Time converts that to 00:00:00 on the following day. +type Time struct { + Microseconds int64 // Number of microseconds since midnight + Status Status +} + +// Set converts src into a Time and stores in dst. +func (dst *Time) Set(src interface{}) error { + if src == nil { + *dst = Time{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + case time.Time: + usec := int64(value.Hour())*microsecondsPerHour + + int64(value.Minute())*microsecondsPerMinute + + int64(value.Second())*microsecondsPerSecond + + int64(value.Nanosecond())/1000 + *dst = Time{Microseconds: usec, Status: Present} + default: + if originalSrc, ok := underlyingTimeType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to Time", value) + } + + return nil +} + +func (dst Time) Get() interface{} { + switch dst.Status { + case Present: + return dst.Microseconds + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Time) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + case *time.Time: + // 24:00:00 is max allowed time in PostgreSQL, but time.Time will normalize that to 00:00:00 the next day. + var maxRepresentableByTime int64 = 24*60*60*1000000 - 1 + if src.Microseconds > maxRepresentableByTime { + return errors.Errorf("%d microseconds cannot be represented as time.Time", src.Microseconds) + } + + usec := src.Microseconds + hours := usec / microsecondsPerHour + usec -= hours * microsecondsPerHour + minutes := usec / microsecondsPerMinute + usec -= minutes * microsecondsPerMinute + seconds := usec / microsecondsPerSecond + usec -= seconds * microsecondsPerSecond + ns := usec * 1000 + *v = time.Date(2000, 1, 1, int(hours), int(minutes), int(seconds), int(ns), time.UTC) + return nil + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +// DecodeText decodes from src into dst. +func (dst *Time) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Time{Status: Null} + return nil + } + + s := string(src) + + if len(s) < 8 { + return errors.Errorf("cannot decode %v into Time", s) + } + + hours, err := strconv.ParseInt(s[0:2], 10, 64) + if err != nil { + return errors.Errorf("cannot decode %v into Time", s) + } + usec := hours * microsecondsPerHour + + minutes, err := strconv.ParseInt(s[3:5], 10, 64) + if err != nil { + return errors.Errorf("cannot decode %v into Time", s) + } + usec += minutes * microsecondsPerMinute + + seconds, err := strconv.ParseInt(s[6:8], 10, 64) + if err != nil { + return errors.Errorf("cannot decode %v into Time", s) + } + usec += seconds * microsecondsPerSecond + + if len(s) > 9 { + fraction := s[9:] + n, err := strconv.ParseInt(fraction, 10, 64) + if err != nil { + return errors.Errorf("cannot decode %v into Time", s) + } + + for i := len(fraction); i < 6; i++ { + n *= 10 + } + + usec += n + } + + *dst = Time{Microseconds: usec, Status: Present} + + return nil +} + +// DecodeBinary decodes from src into dst. +func (dst *Time) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Time{Status: Null} + return nil + } + + if len(src) != 8 { + return errors.Errorf("invalid length for time: %v", len(src)) + } + + usec := int64(binary.BigEndian.Uint64(src)) + *dst = Time{Microseconds: usec, Status: Present} + + return nil +} + +// EncodeText writes the text encoding of src into w. +func (src Time) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + usec := src.Microseconds + hours := usec / microsecondsPerHour + usec -= hours * microsecondsPerHour + minutes := usec / microsecondsPerMinute + usec -= minutes * microsecondsPerMinute + seconds := usec / microsecondsPerSecond + usec -= seconds * microsecondsPerSecond + + s := fmt.Sprintf("%02d:%02d:%02d.%06d", hours, minutes, seconds, usec) + + return append(buf, s...), nil +} + +// EncodeBinary writes the binary encoding of src into w. If src.Time is not in +// the UTC time zone it returns an error. +func (src Time) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + return pgio.AppendInt64(buf, src.Microseconds), nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Time) Scan(src interface{}) error { + if src == nil { + *dst = Time{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + case time.Time: + return dst.Set(src) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Time) Value() (driver.Value, error) { + return EncodeValueText(src) +} diff --git a/github.com/jackc/pgtype/timestamp.go b/github.com/jackc/pgtype/timestamp.go new file mode 100644 index 0000000000..de059f7e9d --- /dev/null +++ b/github.com/jackc/pgtype/timestamp.go @@ -0,0 +1,233 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + "time" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +const pgTimestampFormat = "2006-01-02 15:04:05.999999999" + +// Timestamp represents the PostgreSQL timestamp type. The PostgreSQL +// timestamp does not have a time zone. This presents a problem when +// translating to and from time.Time which requires a time zone. It is highly +// recommended to use timestamptz whenever possible. Timestamp methods either +// convert to UTC or return an error on non-UTC times. +type Timestamp struct { + Time time.Time // Time must always be in UTC. + Status Status + InfinityModifier InfinityModifier +} + +// Set converts src into a Timestamp and stores in dst. If src is a +// time.Time in a non-UTC time zone, the time zone is discarded. +func (dst *Timestamp) Set(src interface{}) error { + if src == nil { + *dst = Timestamp{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + case time.Time: + *dst = Timestamp{Time: time.Date(value.Year(), value.Month(), value.Day(), value.Hour(), value.Minute(), value.Second(), value.Nanosecond(), time.UTC), Status: Present} + default: + if originalSrc, ok := underlyingTimeType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to Timestamp", value) + } + + return nil +} + +func (dst Timestamp) Get() interface{} { + switch dst.Status { + case Present: + if dst.InfinityModifier != None { + return dst.InfinityModifier + } + return dst.Time + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Timestamp) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + case *time.Time: + if src.InfinityModifier != None { + return errors.Errorf("cannot assign %v to %T", src, dst) + } + *v = src.Time + return nil + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +// DecodeText decodes from src into dst. The decoded time is considered to +// be in UTC. +func (dst *Timestamp) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Timestamp{Status: Null} + return nil + } + + sbuf := string(src) + switch sbuf { + case "infinity": + *dst = Timestamp{Status: Present, InfinityModifier: Infinity} + case "-infinity": + *dst = Timestamp{Status: Present, InfinityModifier: -Infinity} + default: + tim, err := time.Parse(pgTimestampFormat, sbuf) + if err != nil { + return err + } + + *dst = Timestamp{Time: tim, Status: Present} + } + + return nil +} + +// DecodeBinary decodes from src into dst. The decoded time is considered to +// be in UTC. +func (dst *Timestamp) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Timestamp{Status: Null} + return nil + } + + if len(src) != 8 { + return errors.Errorf("invalid length for timestamp: %v", len(src)) + } + + microsecSinceY2K := int64(binary.BigEndian.Uint64(src)) + + switch microsecSinceY2K { + case infinityMicrosecondOffset: + *dst = Timestamp{Status: Present, InfinityModifier: Infinity} + case negativeInfinityMicrosecondOffset: + *dst = Timestamp{Status: Present, InfinityModifier: -Infinity} + default: + microsecSinceUnixEpoch := microsecFromUnixEpochToY2K + microsecSinceY2K + tim := time.Unix(microsecSinceUnixEpoch/1000000, (microsecSinceUnixEpoch%1000000)*1000).UTC() + *dst = Timestamp{Time: tim, Status: Present} + } + + return nil +} + +// EncodeText writes the text encoding of src into w. If src.Time is not in +// the UTC time zone it returns an error. +func (src Timestamp) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + if src.Time.Location() != time.UTC { + return nil, errors.Errorf("cannot encode non-UTC time into timestamp") + } + + var s string + + switch src.InfinityModifier { + case None: + s = src.Time.Truncate(time.Microsecond).Format(pgTimestampFormat) + case Infinity: + s = "infinity" + case NegativeInfinity: + s = "-infinity" + } + + return append(buf, s...), nil +} + +// EncodeBinary writes the binary encoding of src into w. If src.Time is not in +// the UTC time zone it returns an error. +func (src Timestamp) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + if src.Time.Location() != time.UTC { + return nil, errors.Errorf("cannot encode non-UTC time into timestamp") + } + + var microsecSinceY2K int64 + switch src.InfinityModifier { + case None: + microsecSinceUnixEpoch := src.Time.Unix()*1000000 + int64(src.Time.Nanosecond())/1000 + microsecSinceY2K = microsecSinceUnixEpoch - microsecFromUnixEpochToY2K + case Infinity: + microsecSinceY2K = infinityMicrosecondOffset + case NegativeInfinity: + microsecSinceY2K = negativeInfinityMicrosecondOffset + } + + return pgio.AppendInt64(buf, microsecSinceY2K), nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Timestamp) Scan(src interface{}) error { + if src == nil { + *dst = Timestamp{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + case time.Time: + *dst = Timestamp{Time: src, Status: Present} + return nil + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Timestamp) Value() (driver.Value, error) { + switch src.Status { + case Present: + if src.InfinityModifier != None { + return src.InfinityModifier.String(), nil + } + return src.Time, nil + case Null: + return nil, nil + default: + return nil, errUndefined + } +} diff --git a/github.com/jackc/pgtype/timestamp_array.go b/github.com/jackc/pgtype/timestamp_array.go new file mode 100644 index 0000000000..49ac98fdb4 --- /dev/null +++ b/github.com/jackc/pgtype/timestamp_array.go @@ -0,0 +1,321 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + "time" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type TimestampArray struct { + Elements []Timestamp + Dimensions []ArrayDimension + Status Status +} + +func (dst *TimestampArray) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = TimestampArray{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + + case []time.Time: + if value == nil { + *dst = TimestampArray{Status: Null} + } else if len(value) == 0 { + *dst = TimestampArray{Status: Present} + } else { + elements := make([]Timestamp, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = TimestampArray{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []Timestamp: + if value == nil { + *dst = TimestampArray{Status: Null} + } else if len(value) == 0 { + *dst = TimestampArray{Status: Present} + } else { + *dst = TimestampArray{ + Elements: value, + Dimensions: []ArrayDimension{{Length: int32(len(value)), LowerBound: 1}}, + Status: Present, + } + } + default: + if originalSrc, ok := underlyingSliceType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to TimestampArray", value) + } + + return nil +} + +func (dst TimestampArray) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *TimestampArray) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + + case *[]time.Time: + *v = make([]time.Time, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *TimestampArray) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = TimestampArray{Status: Null} + return nil + } + + uta, err := ParseUntypedTextArray(string(src)) + if err != nil { + return err + } + + var elements []Timestamp + + if len(uta.Elements) > 0 { + elements = make([]Timestamp, len(uta.Elements)) + + for i, s := range uta.Elements { + var elem Timestamp + var elemSrc []byte + if s != "NULL" { + elemSrc = []byte(s) + } + err = elem.DecodeText(ci, elemSrc) + if err != nil { + return err + } + + elements[i] = elem + } + } + + *dst = TimestampArray{Elements: elements, Dimensions: uta.Dimensions, Status: Present} + + return nil +} + +func (dst *TimestampArray) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = TimestampArray{Status: Null} + return nil + } + + var arrayHeader ArrayHeader + rp, err := arrayHeader.DecodeBinary(ci, src) + if err != nil { + return err + } + + if len(arrayHeader.Dimensions) == 0 { + *dst = TimestampArray{Dimensions: arrayHeader.Dimensions, Status: Present} + return nil + } + + elementCount := arrayHeader.Dimensions[0].Length + for _, d := range arrayHeader.Dimensions[1:] { + elementCount *= d.Length + } + + elements := make([]Timestamp, elementCount) + + for i := range elements { + elemLen := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + var elemSrc []byte + if elemLen >= 0 { + elemSrc = src[rp : rp+elemLen] + rp += elemLen + } + err = elements[i].DecodeBinary(ci, elemSrc) + if err != nil { + return err + } + } + + *dst = TimestampArray{Elements: elements, Dimensions: arrayHeader.Dimensions, Status: Present} + return nil +} + +func (src TimestampArray) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + if len(src.Dimensions) == 0 { + return append(buf, '{', '}'), nil + } + + buf = EncodeTextArrayDimensions(buf, src.Dimensions) + + // dimElemCounts is the multiples of elements that each array lies on. For + // example, a single dimension array of length 4 would have a dimElemCounts of + // [4]. A multi-dimensional array of lengths [3,5,2] would have a + // dimElemCounts of [30,10,2]. This is used to simplify when to render a '{' + // or '}'. + dimElemCounts := make([]int, len(src.Dimensions)) + dimElemCounts[len(src.Dimensions)-1] = int(src.Dimensions[len(src.Dimensions)-1].Length) + for i := len(src.Dimensions) - 2; i > -1; i-- { + dimElemCounts[i] = int(src.Dimensions[i].Length) * dimElemCounts[i+1] + } + + inElemBuf := make([]byte, 0, 32) + for i, elem := range src.Elements { + if i > 0 { + buf = append(buf, ',') + } + + for _, dec := range dimElemCounts { + if i%dec == 0 { + buf = append(buf, '{') + } + } + + elemBuf, err := elem.EncodeText(ci, inElemBuf) + if err != nil { + return nil, err + } + if elemBuf == nil { + buf = append(buf, `NULL`...) + } else { + buf = append(buf, QuoteArrayElementIfNeeded(string(elemBuf))...) + } + + for _, dec := range dimElemCounts { + if (i+1)%dec == 0 { + buf = append(buf, '}') + } + } + } + + return buf, nil +} + +func (src TimestampArray) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + arrayHeader := ArrayHeader{ + Dimensions: src.Dimensions, + } + + if dt, ok := ci.DataTypeForName("timestamp"); ok { + arrayHeader.ElementOID = int32(dt.OID) + } else { + return nil, errors.Errorf("unable to find oid for type name %v", "timestamp") + } + + for i := range src.Elements { + if src.Elements[i].Status == Null { + arrayHeader.ContainsNull = true + break + } + } + + buf = arrayHeader.EncodeBinary(ci, buf) + + for i := range src.Elements { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + elemBuf, err := src.Elements[i].EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if elemBuf != nil { + buf = elemBuf + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *TimestampArray) Scan(src interface{}) error { + if src == nil { + return dst.DecodeText(nil, nil) + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src TimestampArray) Value() (driver.Value, error) { + buf, err := src.EncodeText(nil, nil) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + + return string(buf), nil +} diff --git a/github.com/jackc/pgtype/timestamptz.go b/github.com/jackc/pgtype/timestamptz.go new file mode 100644 index 0000000000..100f44a57b --- /dev/null +++ b/github.com/jackc/pgtype/timestamptz.go @@ -0,0 +1,286 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + "encoding/json" + "time" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +const pgTimestamptzHourFormat = "2006-01-02 15:04:05.999999999Z07" +const pgTimestamptzMinuteFormat = "2006-01-02 15:04:05.999999999Z07:00" +const pgTimestamptzSecondFormat = "2006-01-02 15:04:05.999999999Z07:00:00" +const microsecFromUnixEpochToY2K = 946684800 * 1000000 + +const ( + negativeInfinityMicrosecondOffset = -9223372036854775808 + infinityMicrosecondOffset = 9223372036854775807 +) + +type Timestamptz struct { + Time time.Time + Status Status + InfinityModifier InfinityModifier +} + +func (dst *Timestamptz) Set(src interface{}) error { + if src == nil { + *dst = Timestamptz{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + case time.Time: + *dst = Timestamptz{Time: value, Status: Present} + default: + if originalSrc, ok := underlyingTimeType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to Timestamptz", value) + } + + return nil +} + +func (dst Timestamptz) Get() interface{} { + switch dst.Status { + case Present: + if dst.InfinityModifier != None { + return dst.InfinityModifier + } + return dst.Time + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Timestamptz) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + case *time.Time: + if src.InfinityModifier != None { + return errors.Errorf("cannot assign %v to %T", src, dst) + } + *v = src.Time + return nil + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *Timestamptz) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Timestamptz{Status: Null} + return nil + } + + sbuf := string(src) + switch sbuf { + case "infinity": + *dst = Timestamptz{Status: Present, InfinityModifier: Infinity} + case "-infinity": + *dst = Timestamptz{Status: Present, InfinityModifier: -Infinity} + default: + var format string + if sbuf[len(sbuf)-9] == '-' || sbuf[len(sbuf)-9] == '+' { + format = pgTimestamptzSecondFormat + } else if sbuf[len(sbuf)-6] == '-' || sbuf[len(sbuf)-6] == '+' { + format = pgTimestamptzMinuteFormat + } else { + format = pgTimestamptzHourFormat + } + + tim, err := time.Parse(format, sbuf) + if err != nil { + return err + } + + *dst = Timestamptz{Time: tim, Status: Present} + } + + return nil +} + +func (dst *Timestamptz) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Timestamptz{Status: Null} + return nil + } + + if len(src) != 8 { + return errors.Errorf("invalid length for timestamptz: %v", len(src)) + } + + microsecSinceY2K := int64(binary.BigEndian.Uint64(src)) + + switch microsecSinceY2K { + case infinityMicrosecondOffset: + *dst = Timestamptz{Status: Present, InfinityModifier: Infinity} + case negativeInfinityMicrosecondOffset: + *dst = Timestamptz{Status: Present, InfinityModifier: -Infinity} + default: + microsecSinceUnixEpoch := microsecFromUnixEpochToY2K + microsecSinceY2K + tim := time.Unix(microsecSinceUnixEpoch/1000000, (microsecSinceUnixEpoch%1000000)*1000) + *dst = Timestamptz{Time: tim, Status: Present} + } + + return nil +} + +func (src Timestamptz) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + var s string + + switch src.InfinityModifier { + case None: + s = src.Time.UTC().Truncate(time.Microsecond).Format(pgTimestamptzSecondFormat) + case Infinity: + s = "infinity" + case NegativeInfinity: + s = "-infinity" + } + + return append(buf, s...), nil +} + +func (src Timestamptz) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + var microsecSinceY2K int64 + switch src.InfinityModifier { + case None: + microsecSinceUnixEpoch := src.Time.Unix()*1000000 + int64(src.Time.Nanosecond())/1000 + microsecSinceY2K = microsecSinceUnixEpoch - microsecFromUnixEpochToY2K + case Infinity: + microsecSinceY2K = infinityMicrosecondOffset + case NegativeInfinity: + microsecSinceY2K = negativeInfinityMicrosecondOffset + } + + return pgio.AppendInt64(buf, microsecSinceY2K), nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Timestamptz) Scan(src interface{}) error { + if src == nil { + *dst = Timestamptz{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + case time.Time: + *dst = Timestamptz{Time: src, Status: Present} + return nil + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Timestamptz) Value() (driver.Value, error) { + switch src.Status { + case Present: + if src.InfinityModifier != None { + return src.InfinityModifier.String(), nil + } + return src.Time, nil + case Null: + return nil, nil + default: + return nil, errUndefined + } +} + +func (src Timestamptz) MarshalJSON() ([]byte, error) { + switch src.Status { + case Null: + return []byte("null"), nil + case Undefined: + return nil, errUndefined + } + + if src.Status != Present { + return nil, errBadStatus + } + + var s string + + switch src.InfinityModifier { + case None: + s = src.Time.Format(time.RFC3339Nano) + case Infinity: + s = "infinity" + case NegativeInfinity: + s = "-infinity" + } + + return json.Marshal(s) +} + +func (dst *Timestamptz) UnmarshalJSON(b []byte) error { + var s *string + err := json.Unmarshal(b, &s) + if err != nil { + return err + } + + if s == nil { + *dst = Timestamptz{Status: Null} + return nil + } + + switch *s { + case "infinity": + *dst = Timestamptz{Status: Present, InfinityModifier: Infinity} + case "-infinity": + *dst = Timestamptz{Status: Present, InfinityModifier: -Infinity} + default: + // PostgreSQL uses ISO 8601 for to_json function and casting from a string to timestamptz + tim, err := time.Parse(time.RFC3339Nano, *s) + if err != nil { + return err + } + + *dst = Timestamptz{Time: tim, Status: Present} + } + + return nil +} diff --git a/github.com/jackc/pgtype/timestamptz_array.go b/github.com/jackc/pgtype/timestamptz_array.go new file mode 100644 index 0000000000..2e26692bdc --- /dev/null +++ b/github.com/jackc/pgtype/timestamptz_array.go @@ -0,0 +1,321 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + "time" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type TimestamptzArray struct { + Elements []Timestamptz + Dimensions []ArrayDimension + Status Status +} + +func (dst *TimestamptzArray) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = TimestamptzArray{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + + case []time.Time: + if value == nil { + *dst = TimestamptzArray{Status: Null} + } else if len(value) == 0 { + *dst = TimestamptzArray{Status: Present} + } else { + elements := make([]Timestamptz, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = TimestamptzArray{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []Timestamptz: + if value == nil { + *dst = TimestamptzArray{Status: Null} + } else if len(value) == 0 { + *dst = TimestamptzArray{Status: Present} + } else { + *dst = TimestamptzArray{ + Elements: value, + Dimensions: []ArrayDimension{{Length: int32(len(value)), LowerBound: 1}}, + Status: Present, + } + } + default: + if originalSrc, ok := underlyingSliceType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to TimestamptzArray", value) + } + + return nil +} + +func (dst TimestamptzArray) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *TimestamptzArray) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + + case *[]time.Time: + *v = make([]time.Time, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *TimestamptzArray) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = TimestamptzArray{Status: Null} + return nil + } + + uta, err := ParseUntypedTextArray(string(src)) + if err != nil { + return err + } + + var elements []Timestamptz + + if len(uta.Elements) > 0 { + elements = make([]Timestamptz, len(uta.Elements)) + + for i, s := range uta.Elements { + var elem Timestamptz + var elemSrc []byte + if s != "NULL" { + elemSrc = []byte(s) + } + err = elem.DecodeText(ci, elemSrc) + if err != nil { + return err + } + + elements[i] = elem + } + } + + *dst = TimestamptzArray{Elements: elements, Dimensions: uta.Dimensions, Status: Present} + + return nil +} + +func (dst *TimestamptzArray) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = TimestamptzArray{Status: Null} + return nil + } + + var arrayHeader ArrayHeader + rp, err := arrayHeader.DecodeBinary(ci, src) + if err != nil { + return err + } + + if len(arrayHeader.Dimensions) == 0 { + *dst = TimestamptzArray{Dimensions: arrayHeader.Dimensions, Status: Present} + return nil + } + + elementCount := arrayHeader.Dimensions[0].Length + for _, d := range arrayHeader.Dimensions[1:] { + elementCount *= d.Length + } + + elements := make([]Timestamptz, elementCount) + + for i := range elements { + elemLen := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + var elemSrc []byte + if elemLen >= 0 { + elemSrc = src[rp : rp+elemLen] + rp += elemLen + } + err = elements[i].DecodeBinary(ci, elemSrc) + if err != nil { + return err + } + } + + *dst = TimestamptzArray{Elements: elements, Dimensions: arrayHeader.Dimensions, Status: Present} + return nil +} + +func (src TimestamptzArray) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + if len(src.Dimensions) == 0 { + return append(buf, '{', '}'), nil + } + + buf = EncodeTextArrayDimensions(buf, src.Dimensions) + + // dimElemCounts is the multiples of elements that each array lies on. For + // example, a single dimension array of length 4 would have a dimElemCounts of + // [4]. A multi-dimensional array of lengths [3,5,2] would have a + // dimElemCounts of [30,10,2]. This is used to simplify when to render a '{' + // or '}'. + dimElemCounts := make([]int, len(src.Dimensions)) + dimElemCounts[len(src.Dimensions)-1] = int(src.Dimensions[len(src.Dimensions)-1].Length) + for i := len(src.Dimensions) - 2; i > -1; i-- { + dimElemCounts[i] = int(src.Dimensions[i].Length) * dimElemCounts[i+1] + } + + inElemBuf := make([]byte, 0, 32) + for i, elem := range src.Elements { + if i > 0 { + buf = append(buf, ',') + } + + for _, dec := range dimElemCounts { + if i%dec == 0 { + buf = append(buf, '{') + } + } + + elemBuf, err := elem.EncodeText(ci, inElemBuf) + if err != nil { + return nil, err + } + if elemBuf == nil { + buf = append(buf, `NULL`...) + } else { + buf = append(buf, QuoteArrayElementIfNeeded(string(elemBuf))...) + } + + for _, dec := range dimElemCounts { + if (i+1)%dec == 0 { + buf = append(buf, '}') + } + } + } + + return buf, nil +} + +func (src TimestamptzArray) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + arrayHeader := ArrayHeader{ + Dimensions: src.Dimensions, + } + + if dt, ok := ci.DataTypeForName("timestamptz"); ok { + arrayHeader.ElementOID = int32(dt.OID) + } else { + return nil, errors.Errorf("unable to find oid for type name %v", "timestamptz") + } + + for i := range src.Elements { + if src.Elements[i].Status == Null { + arrayHeader.ContainsNull = true + break + } + } + + buf = arrayHeader.EncodeBinary(ci, buf) + + for i := range src.Elements { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + elemBuf, err := src.Elements[i].EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if elemBuf != nil { + buf = elemBuf + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *TimestamptzArray) Scan(src interface{}) error { + if src == nil { + return dst.DecodeText(nil, nil) + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src TimestamptzArray) Value() (driver.Value, error) { + buf, err := src.EncodeText(nil, nil) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + + return string(buf), nil +} diff --git a/github.com/jackc/pgtype/tsrange.go b/github.com/jackc/pgtype/tsrange.go new file mode 100644 index 0000000000..6ca12aed48 --- /dev/null +++ b/github.com/jackc/pgtype/tsrange.go @@ -0,0 +1,267 @@ +package pgtype + +import ( + "database/sql/driver" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type Tsrange struct { + Lower Timestamp + Upper Timestamp + LowerType BoundType + UpperType BoundType + Status Status +} + +func (dst *Tsrange) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = Tsrange{Status: Null} + return nil + } + + switch value := src.(type) { + case Tsrange: + *dst = value + case *Tsrange: + *dst = *value + case string: + return dst.DecodeText(nil, []byte(value)) + default: + return errors.Errorf("cannot convert %v to Tsrange", src) + } + + return nil +} + +func (dst Tsrange) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Tsrange) AssignTo(dst interface{}) error { + return errors.Errorf("cannot assign %v to %T", src, dst) +} + +func (dst *Tsrange) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Tsrange{Status: Null} + return nil + } + + utr, err := ParseUntypedTextRange(string(src)) + if err != nil { + return err + } + + *dst = Tsrange{Status: Present} + + dst.LowerType = utr.LowerType + dst.UpperType = utr.UpperType + + if dst.LowerType == Empty { + return nil + } + + if dst.LowerType == Inclusive || dst.LowerType == Exclusive { + if err := dst.Lower.DecodeText(ci, []byte(utr.Lower)); err != nil { + return err + } + } + + if dst.UpperType == Inclusive || dst.UpperType == Exclusive { + if err := dst.Upper.DecodeText(ci, []byte(utr.Upper)); err != nil { + return err + } + } + + return nil +} + +func (dst *Tsrange) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Tsrange{Status: Null} + return nil + } + + ubr, err := ParseUntypedBinaryRange(src) + if err != nil { + return err + } + + *dst = Tsrange{Status: Present} + + dst.LowerType = ubr.LowerType + dst.UpperType = ubr.UpperType + + if dst.LowerType == Empty { + return nil + } + + if dst.LowerType == Inclusive || dst.LowerType == Exclusive { + if err := dst.Lower.DecodeBinary(ci, ubr.Lower); err != nil { + return err + } + } + + if dst.UpperType == Inclusive || dst.UpperType == Exclusive { + if err := dst.Upper.DecodeBinary(ci, ubr.Upper); err != nil { + return err + } + } + + return nil +} + +func (src Tsrange) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + switch src.LowerType { + case Exclusive, Unbounded: + buf = append(buf, '(') + case Inclusive: + buf = append(buf, '[') + case Empty: + return append(buf, "empty"...), nil + default: + return nil, errors.Errorf("unknown lower bound type %v", src.LowerType) + } + + var err error + + if src.LowerType != Unbounded { + buf, err = src.Lower.EncodeText(ci, buf) + if err != nil { + return nil, err + } else if buf == nil { + return nil, errors.Errorf("Lower cannot be null unless LowerType is Unbounded") + } + } + + buf = append(buf, ',') + + if src.UpperType != Unbounded { + buf, err = src.Upper.EncodeText(ci, buf) + if err != nil { + return nil, err + } else if buf == nil { + return nil, errors.Errorf("Upper cannot be null unless UpperType is Unbounded") + } + } + + switch src.UpperType { + case Exclusive, Unbounded: + buf = append(buf, ')') + case Inclusive: + buf = append(buf, ']') + default: + return nil, errors.Errorf("unknown upper bound type %v", src.UpperType) + } + + return buf, nil +} + +func (src Tsrange) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + var rangeType byte + switch src.LowerType { + case Inclusive: + rangeType |= lowerInclusiveMask + case Unbounded: + rangeType |= lowerUnboundedMask + case Exclusive: + case Empty: + return append(buf, emptyMask), nil + default: + return nil, errors.Errorf("unknown LowerType: %v", src.LowerType) + } + + switch src.UpperType { + case Inclusive: + rangeType |= upperInclusiveMask + case Unbounded: + rangeType |= upperUnboundedMask + case Exclusive: + default: + return nil, errors.Errorf("unknown UpperType: %v", src.UpperType) + } + + buf = append(buf, rangeType) + + var err error + + if src.LowerType != Unbounded { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + buf, err = src.Lower.EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if buf == nil { + return nil, errors.Errorf("Lower cannot be null unless LowerType is Unbounded") + } + + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + + if src.UpperType != Unbounded { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + buf, err = src.Upper.EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if buf == nil { + return nil, errors.Errorf("Upper cannot be null unless UpperType is Unbounded") + } + + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Tsrange) Scan(src interface{}) error { + if src == nil { + *dst = Tsrange{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Tsrange) Value() (driver.Value, error) { + return EncodeValueText(src) +} diff --git a/github.com/jackc/pgtype/tstzrange.go b/github.com/jackc/pgtype/tstzrange.go new file mode 100644 index 0000000000..1b05c3ea60 --- /dev/null +++ b/github.com/jackc/pgtype/tstzrange.go @@ -0,0 +1,267 @@ +package pgtype + +import ( + "database/sql/driver" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type Tstzrange struct { + Lower Timestamptz + Upper Timestamptz + LowerType BoundType + UpperType BoundType + Status Status +} + +func (dst *Tstzrange) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = Tstzrange{Status: Null} + return nil + } + + switch value := src.(type) { + case Tstzrange: + *dst = value + case *Tstzrange: + *dst = *value + case string: + return dst.DecodeText(nil, []byte(value)) + default: + return errors.Errorf("cannot convert %v to Tstzrange", src) + } + + return nil +} + +func (dst Tstzrange) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Tstzrange) AssignTo(dst interface{}) error { + return errors.Errorf("cannot assign %v to %T", src, dst) +} + +func (dst *Tstzrange) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Tstzrange{Status: Null} + return nil + } + + utr, err := ParseUntypedTextRange(string(src)) + if err != nil { + return err + } + + *dst = Tstzrange{Status: Present} + + dst.LowerType = utr.LowerType + dst.UpperType = utr.UpperType + + if dst.LowerType == Empty { + return nil + } + + if dst.LowerType == Inclusive || dst.LowerType == Exclusive { + if err := dst.Lower.DecodeText(ci, []byte(utr.Lower)); err != nil { + return err + } + } + + if dst.UpperType == Inclusive || dst.UpperType == Exclusive { + if err := dst.Upper.DecodeText(ci, []byte(utr.Upper)); err != nil { + return err + } + } + + return nil +} + +func (dst *Tstzrange) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Tstzrange{Status: Null} + return nil + } + + ubr, err := ParseUntypedBinaryRange(src) + if err != nil { + return err + } + + *dst = Tstzrange{Status: Present} + + dst.LowerType = ubr.LowerType + dst.UpperType = ubr.UpperType + + if dst.LowerType == Empty { + return nil + } + + if dst.LowerType == Inclusive || dst.LowerType == Exclusive { + if err := dst.Lower.DecodeBinary(ci, ubr.Lower); err != nil { + return err + } + } + + if dst.UpperType == Inclusive || dst.UpperType == Exclusive { + if err := dst.Upper.DecodeBinary(ci, ubr.Upper); err != nil { + return err + } + } + + return nil +} + +func (src Tstzrange) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + switch src.LowerType { + case Exclusive, Unbounded: + buf = append(buf, '(') + case Inclusive: + buf = append(buf, '[') + case Empty: + return append(buf, "empty"...), nil + default: + return nil, errors.Errorf("unknown lower bound type %v", src.LowerType) + } + + var err error + + if src.LowerType != Unbounded { + buf, err = src.Lower.EncodeText(ci, buf) + if err != nil { + return nil, err + } else if buf == nil { + return nil, errors.Errorf("Lower cannot be null unless LowerType is Unbounded") + } + } + + buf = append(buf, ',') + + if src.UpperType != Unbounded { + buf, err = src.Upper.EncodeText(ci, buf) + if err != nil { + return nil, err + } else if buf == nil { + return nil, errors.Errorf("Upper cannot be null unless UpperType is Unbounded") + } + } + + switch src.UpperType { + case Exclusive, Unbounded: + buf = append(buf, ')') + case Inclusive: + buf = append(buf, ']') + default: + return nil, errors.Errorf("unknown upper bound type %v", src.UpperType) + } + + return buf, nil +} + +func (src Tstzrange) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + var rangeType byte + switch src.LowerType { + case Inclusive: + rangeType |= lowerInclusiveMask + case Unbounded: + rangeType |= lowerUnboundedMask + case Exclusive: + case Empty: + return append(buf, emptyMask), nil + default: + return nil, errors.Errorf("unknown LowerType: %v", src.LowerType) + } + + switch src.UpperType { + case Inclusive: + rangeType |= upperInclusiveMask + case Unbounded: + rangeType |= upperUnboundedMask + case Exclusive: + default: + return nil, errors.Errorf("unknown UpperType: %v", src.UpperType) + } + + buf = append(buf, rangeType) + + var err error + + if src.LowerType != Unbounded { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + buf, err = src.Lower.EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if buf == nil { + return nil, errors.Errorf("Lower cannot be null unless LowerType is Unbounded") + } + + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + + if src.UpperType != Unbounded { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + buf, err = src.Upper.EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if buf == nil { + return nil, errors.Errorf("Upper cannot be null unless UpperType is Unbounded") + } + + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Tstzrange) Scan(src interface{}) error { + if src == nil { + *dst = Tstzrange{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Tstzrange) Value() (driver.Value, error) { + return EncodeValueText(src) +} diff --git a/github.com/jackc/pgtype/tstzrange_array.go b/github.com/jackc/pgtype/tstzrange_array.go new file mode 100644 index 0000000000..2c3656459a --- /dev/null +++ b/github.com/jackc/pgtype/tstzrange_array.go @@ -0,0 +1,301 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type TstzrangeArray struct { + Elements []Tstzrange + Dimensions []ArrayDimension + Status Status +} + +func (dst *TstzrangeArray) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = TstzrangeArray{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + + case []Tstzrange: + if value == nil { + *dst = TstzrangeArray{Status: Null} + } else if len(value) == 0 { + *dst = TstzrangeArray{Status: Present} + } else { + *dst = TstzrangeArray{ + Elements: value, + Dimensions: []ArrayDimension{{Length: int32(len(value)), LowerBound: 1}}, + Status: Present, + } + } + default: + if originalSrc, ok := underlyingSliceType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to TstzrangeArray", value) + } + + return nil +} + +func (dst TstzrangeArray) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *TstzrangeArray) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + + case *[]Tstzrange: + *v = make([]Tstzrange, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *TstzrangeArray) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = TstzrangeArray{Status: Null} + return nil + } + + uta, err := ParseUntypedTextArray(string(src)) + if err != nil { + return err + } + + var elements []Tstzrange + + if len(uta.Elements) > 0 { + elements = make([]Tstzrange, len(uta.Elements)) + + for i, s := range uta.Elements { + var elem Tstzrange + var elemSrc []byte + if s != "NULL" { + elemSrc = []byte(s) + } + err = elem.DecodeText(ci, elemSrc) + if err != nil { + return err + } + + elements[i] = elem + } + } + + *dst = TstzrangeArray{Elements: elements, Dimensions: uta.Dimensions, Status: Present} + + return nil +} + +func (dst *TstzrangeArray) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = TstzrangeArray{Status: Null} + return nil + } + + var arrayHeader ArrayHeader + rp, err := arrayHeader.DecodeBinary(ci, src) + if err != nil { + return err + } + + if len(arrayHeader.Dimensions) == 0 { + *dst = TstzrangeArray{Dimensions: arrayHeader.Dimensions, Status: Present} + return nil + } + + elementCount := arrayHeader.Dimensions[0].Length + for _, d := range arrayHeader.Dimensions[1:] { + elementCount *= d.Length + } + + elements := make([]Tstzrange, elementCount) + + for i := range elements { + elemLen := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + var elemSrc []byte + if elemLen >= 0 { + elemSrc = src[rp : rp+elemLen] + rp += elemLen + } + err = elements[i].DecodeBinary(ci, elemSrc) + if err != nil { + return err + } + } + + *dst = TstzrangeArray{Elements: elements, Dimensions: arrayHeader.Dimensions, Status: Present} + return nil +} + +func (src TstzrangeArray) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + if len(src.Dimensions) == 0 { + return append(buf, '{', '}'), nil + } + + buf = EncodeTextArrayDimensions(buf, src.Dimensions) + + // dimElemCounts is the multiples of elements that each array lies on. For + // example, a single dimension array of length 4 would have a dimElemCounts of + // [4]. A multi-dimensional array of lengths [3,5,2] would have a + // dimElemCounts of [30,10,2]. This is used to simplify when to render a '{' + // or '}'. + dimElemCounts := make([]int, len(src.Dimensions)) + dimElemCounts[len(src.Dimensions)-1] = int(src.Dimensions[len(src.Dimensions)-1].Length) + for i := len(src.Dimensions) - 2; i > -1; i-- { + dimElemCounts[i] = int(src.Dimensions[i].Length) * dimElemCounts[i+1] + } + + inElemBuf := make([]byte, 0, 32) + for i, elem := range src.Elements { + if i > 0 { + buf = append(buf, ',') + } + + for _, dec := range dimElemCounts { + if i%dec == 0 { + buf = append(buf, '{') + } + } + + elemBuf, err := elem.EncodeText(ci, inElemBuf) + if err != nil { + return nil, err + } + if elemBuf == nil { + buf = append(buf, `NULL`...) + } else { + buf = append(buf, QuoteArrayElementIfNeeded(string(elemBuf))...) + } + + for _, dec := range dimElemCounts { + if (i+1)%dec == 0 { + buf = append(buf, '}') + } + } + } + + return buf, nil +} + +func (src TstzrangeArray) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + arrayHeader := ArrayHeader{ + Dimensions: src.Dimensions, + } + + if dt, ok := ci.DataTypeForName("tstzrange"); ok { + arrayHeader.ElementOID = int32(dt.OID) + } else { + return nil, errors.Errorf("unable to find oid for type name %v", "tstzrange") + } + + for i := range src.Elements { + if src.Elements[i].Status == Null { + arrayHeader.ContainsNull = true + break + } + } + + buf = arrayHeader.EncodeBinary(ci, buf) + + for i := range src.Elements { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + elemBuf, err := src.Elements[i].EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if elemBuf != nil { + buf = elemBuf + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *TstzrangeArray) Scan(src interface{}) error { + if src == nil { + return dst.DecodeText(nil, nil) + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src TstzrangeArray) Value() (driver.Value, error) { + buf, err := src.EncodeText(nil, nil) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + + return string(buf), nil +} diff --git a/github.com/jackc/pgtype/typed_array.go.erb b/github.com/jackc/pgtype/typed_array.go.erb new file mode 100644 index 0000000000..d8ae97dd20 --- /dev/null +++ b/github.com/jackc/pgtype/typed_array.go.erb @@ -0,0 +1,326 @@ +package pgtype + +import ( + "bytes" + "fmt" + "io" + + "github.com/jackc/pgio" +) + +type <%= pgtype_array_type %> struct { + Elements []<%= pgtype_element_type %> + Dimensions []ArrayDimension + Status Status +} + +func (dst *<%= pgtype_array_type %>) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = <%= pgtype_array_type %>{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + <% go_array_types.split(",").each do |t| %> + <% if t != "[]#{pgtype_element_type}" %> + case <%= t %>: + if value == nil { + *dst = <%= pgtype_array_type %>{Status: Null} + } else if len(value) == 0 { + *dst = <%= pgtype_array_type %>{Status: Present} + } else { + elements := make([]<%= pgtype_element_type %>, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = <%= pgtype_array_type %>{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + <% end %> + <% end %> + case []<%= pgtype_element_type %>: + if value == nil { + *dst = <%= pgtype_array_type %>{Status: Null} + } else if len(value) == 0 { + *dst = <%= pgtype_array_type %>{Status: Present} + } else { + *dst = <%= pgtype_array_type %>{ + Elements: value, + Dimensions: []ArrayDimension{{Length: int32(len(value)), LowerBound: 1}}, + Status : Present, + } + } + default: + if originalSrc, ok := underlyingSliceType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to <%= pgtype_array_type %>", value) + } + + return nil +} + +func (dst <%= pgtype_array_type %>) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *<%= pgtype_array_type %>) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + <% go_array_types.split(",").each do |t| %> + case *<%= t %>: + *v = make(<%= t %>, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + <% end %> + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *<%= pgtype_array_type %>) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = <%= pgtype_array_type %>{Status: Null} + return nil + } + + uta, err := ParseUntypedTextArray(string(src)) + if err != nil { + return err + } + + var elements []<%= pgtype_element_type %> + + if len(uta.Elements) > 0 { + elements = make([]<%= pgtype_element_type %>, len(uta.Elements)) + + for i, s := range uta.Elements { + var elem <%= pgtype_element_type %> + var elemSrc []byte + if s != "NULL" { + elemSrc = []byte(s) + } + err = elem.DecodeText(ci, elemSrc) + if err != nil { + return err + } + + elements[i] = elem + } + } + + *dst = <%= pgtype_array_type %>{Elements: elements, Dimensions: uta.Dimensions, Status: Present} + + return nil +} + +<% if binary_format == "true" %> +func (dst *<%= pgtype_array_type %>) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = <%= pgtype_array_type %>{Status: Null} + return nil + } + + var arrayHeader ArrayHeader + rp, err := arrayHeader.DecodeBinary(ci, src) + if err != nil { + return err + } + + if len(arrayHeader.Dimensions) == 0 { + *dst = <%= pgtype_array_type %>{Dimensions: arrayHeader.Dimensions, Status: Present} + return nil + } + + elementCount := arrayHeader.Dimensions[0].Length + for _, d := range arrayHeader.Dimensions[1:] { + elementCount *= d.Length + } + + elements := make([]<%= pgtype_element_type %>, elementCount) + + for i := range elements { + elemLen := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + var elemSrc []byte + if elemLen >= 0 { + elemSrc = src[rp:rp+elemLen] + rp += elemLen + } + err = elements[i].DecodeBinary(ci, elemSrc) + if err != nil { + return err + } + } + + *dst = <%= pgtype_array_type %>{Elements: elements, Dimensions: arrayHeader.Dimensions, Status: Present} + return nil +} +<% end %> + +func (src <%= pgtype_array_type %>) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + if len(src.Dimensions) == 0 { + return append(buf, '{', '}'), nil + } + + buf = EncodeTextArrayDimensions(buf, src.Dimensions) + + // dimElemCounts is the multiples of elements that each array lies on. For + // example, a single dimension array of length 4 would have a dimElemCounts of + // [4]. A multi-dimensional array of lengths [3,5,2] would have a + // dimElemCounts of [30,10,2]. This is used to simplify when to render a '{' + // or '}'. + dimElemCounts := make([]int, len(src.Dimensions)) + dimElemCounts[len(src.Dimensions)-1] = int(src.Dimensions[len(src.Dimensions)-1].Length) + for i := len(src.Dimensions) - 2; i > -1; i-- { + dimElemCounts[i] = int(src.Dimensions[i].Length) * dimElemCounts[i+1] + } + + inElemBuf := make([]byte, 0, 32) + for i, elem := range src.Elements { + if i > 0 { + buf = append(buf, ',') + } + + for _, dec := range dimElemCounts { + if i%dec == 0 { + buf = append(buf, '{') + } + } + + elemBuf, err := elem.EncodeText(ci, inElemBuf) + if err != nil { + return nil, err + } + if elemBuf == nil { + buf = append(buf, `<%= text_null %>`...) + } else { + buf = append(buf, QuoteArrayElementIfNeeded(string(elemBuf))...) + } + + for _, dec := range dimElemCounts { + if (i+1)%dec == 0 { + buf = append(buf, '}') + } + } + } + + return buf, nil +} + +<% if binary_format == "true" %> + func (src <%= pgtype_array_type %>) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + arrayHeader := ArrayHeader{ + Dimensions: src.Dimensions, + } + + if dt, ok := ci.DataTypeForName("<%= element_type_name %>"); ok { + arrayHeader.ElementOID = int32(dt.OID) + } else { + return nil, errors.Errorf("unable to find oid for type name %v", "<%= element_type_name %>") + } + + for i := range src.Elements { + if src.Elements[i].Status == Null { + arrayHeader.ContainsNull = true + break + } + } + + buf = arrayHeader.EncodeBinary(ci, buf) + + for i := range src.Elements { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + elemBuf, err := src.Elements[i].EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if elemBuf != nil { + buf = elemBuf + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + } + + return buf, nil + } +<% end %> + +// Scan implements the database/sql Scanner interface. +func (dst *<%= pgtype_array_type %>) Scan(src interface{}) error { + if src == nil { + return dst.DecodeText(nil, nil) + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src <%= pgtype_array_type %>) Value() (driver.Value, error) { + buf, err := src.EncodeText(nil, nil) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + + return string(buf), nil +} diff --git a/github.com/jackc/pgtype/typed_array_gen.sh b/github.com/jackc/pgtype/typed_array_gen.sh new file mode 100644 index 0000000000..6fd4926483 --- /dev/null +++ b/github.com/jackc/pgtype/typed_array_gen.sh @@ -0,0 +1,26 @@ +erb pgtype_array_type=Int2Array pgtype_element_type=Int2 go_array_types=[]int16,[]uint16,[]int32,[]uint32,[]int64,[]uint64,[]int,[]uint element_type_name=int2 text_null=NULL binary_format=true typed_array.go.erb > int2_array.go +erb pgtype_array_type=Int4Array pgtype_element_type=Int4 go_array_types=[]int16,[]uint16,[]int32,[]uint32,[]int64,[]uint64,[]int,[]uint element_type_name=int4 text_null=NULL binary_format=true typed_array.go.erb > int4_array.go +erb pgtype_array_type=Int8Array pgtype_element_type=Int8 go_array_types=[]int16,[]uint16,[]int32,[]uint32,[]int64,[]uint64,[]int,[]uint element_type_name=int8 text_null=NULL binary_format=true typed_array.go.erb > int8_array.go +erb pgtype_array_type=BoolArray pgtype_element_type=Bool go_array_types=[]bool element_type_name=bool text_null=NULL binary_format=true typed_array.go.erb > bool_array.go +erb pgtype_array_type=DateArray pgtype_element_type=Date go_array_types=[]time.Time element_type_name=date text_null=NULL binary_format=true typed_array.go.erb > date_array.go +erb pgtype_array_type=TimestamptzArray pgtype_element_type=Timestamptz go_array_types=[]time.Time element_type_name=timestamptz text_null=NULL binary_format=true typed_array.go.erb > timestamptz_array.go +erb pgtype_array_type=TstzrangeArray pgtype_element_type=Tstzrange go_array_types=[]Tstzrange element_type_name=tstzrange text_null=NULL binary_format=true typed_array.go.erb > tstzrange_array.go +erb pgtype_array_type=TimestampArray pgtype_element_type=Timestamp go_array_types=[]time.Time element_type_name=timestamp text_null=NULL binary_format=true typed_array.go.erb > timestamp_array.go +erb pgtype_array_type=Float4Array pgtype_element_type=Float4 go_array_types=[]float32 element_type_name=float4 text_null=NULL binary_format=true typed_array.go.erb > float4_array.go +erb pgtype_array_type=Float8Array pgtype_element_type=Float8 go_array_types=[]float64 element_type_name=float8 text_null=NULL binary_format=true typed_array.go.erb > float8_array.go +erb pgtype_array_type=InetArray pgtype_element_type=Inet go_array_types=[]*net.IPNet,[]net.IP element_type_name=inet text_null=NULL binary_format=true typed_array.go.erb > inet_array.go +erb pgtype_array_type=MacaddrArray pgtype_element_type=Macaddr go_array_types=[]net.HardwareAddr element_type_name=macaddr text_null=NULL binary_format=true typed_array.go.erb > macaddr_array.go +erb pgtype_array_type=CIDRArray pgtype_element_type=CIDR go_array_types=[]*net.IPNet,[]net.IP element_type_name=cidr text_null=NULL binary_format=true typed_array.go.erb > cidr_array.go +erb pgtype_array_type=TextArray pgtype_element_type=Text go_array_types=[]string element_type_name=text text_null=NULL binary_format=true typed_array.go.erb > text_array.go +erb pgtype_array_type=VarcharArray pgtype_element_type=Varchar go_array_types=[]string element_type_name=varchar text_null=NULL binary_format=true typed_array.go.erb > varchar_array.go +erb pgtype_array_type=BPCharArray pgtype_element_type=BPChar go_array_types=[]string element_type_name=bpchar text_null=NULL binary_format=true typed_array.go.erb > bpchar_array.go +erb pgtype_array_type=ByteaArray pgtype_element_type=Bytea go_array_types=[][]byte element_type_name=bytea text_null=NULL binary_format=true typed_array.go.erb > bytea_array.go +erb pgtype_array_type=ACLItemArray pgtype_element_type=ACLItem go_array_types=[]string element_type_name=aclitem text_null=NULL binary_format=false typed_array.go.erb > aclitem_array.go +erb pgtype_array_type=HstoreArray pgtype_element_type=Hstore go_array_types=[]map[string]string element_type_name=hstore text_null=NULL binary_format=true typed_array.go.erb > hstore_array.go +erb pgtype_array_type=NumericArray pgtype_element_type=Numeric go_array_types=[]float32,[]float64,[]int64,[]uint64 element_type_name=numeric text_null=NULL binary_format=true typed_array.go.erb > numeric_array.go +erb pgtype_array_type=UUIDArray pgtype_element_type=UUID go_array_types=[][16]byte,[][]byte,[]string element_type_name=uuid text_null=NULL binary_format=true typed_array.go.erb > uuid_array.go + +# While the binary format is theoretically possible it is only practical to use the text format. +erb pgtype_array_type=EnumArray pgtype_element_type=GenericText go_array_types=[]string text_null=NULL binary_format=false typed_array.go.erb > enum_array.go + +goimports -w *_array.go diff --git a/github.com/jackc/pgtype/typed_range.go.erb b/github.com/jackc/pgtype/typed_range.go.erb new file mode 100644 index 0000000000..e21b6cdabd --- /dev/null +++ b/github.com/jackc/pgtype/typed_range.go.erb @@ -0,0 +1,269 @@ +package pgtype + +import ( + "bytes" + "database/sql/driver" + "fmt" + "io" + + "github.com/jackc/pgio" +) + +type <%= range_type %> struct { + Lower <%= element_type %> + Upper <%= element_type %> + LowerType BoundType + UpperType BoundType + Status Status +} + +func (dst *<%= range_type %>) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = <%= range_type %>{Status: Null} + return nil + } + + switch value := src.(type) { + case <%= range_type %>: + *dst = value + case *<%= range_type %>: + *dst = *value + case string: + return dst.DecodeText(nil, []byte(value)) + default: + return errors.Errorf("cannot convert %v to <%= range_type %>", src) + } + + return nil +} + +func (dst <%= range_type %>) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *<%= range_type %>) AssignTo(dst interface{}) error { + return errors.Errorf("cannot assign %v to %T", src, dst) +} + +func (dst *<%= range_type %>) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = <%= range_type %>{Status: Null} + return nil + } + + utr, err := ParseUntypedTextRange(string(src)) + if err != nil { + return err + } + + *dst = <%= range_type %>{Status: Present} + + dst.LowerType = utr.LowerType + dst.UpperType = utr.UpperType + + if dst.LowerType == Empty { + return nil + } + + if dst.LowerType == Inclusive || dst.LowerType == Exclusive { + if err := dst.Lower.DecodeText(ci, []byte(utr.Lower)); err != nil { + return err + } + } + + if dst.UpperType == Inclusive || dst.UpperType == Exclusive { + if err := dst.Upper.DecodeText(ci, []byte(utr.Upper)); err != nil { + return err + } + } + + return nil +} + +func (dst *<%= range_type %>) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = <%= range_type %>{Status: Null} + return nil + } + + ubr, err := ParseUntypedBinaryRange(src) + if err != nil { + return err + } + + *dst = <%= range_type %>{Status: Present} + + dst.LowerType = ubr.LowerType + dst.UpperType = ubr.UpperType + + if dst.LowerType == Empty { + return nil + } + + if dst.LowerType == Inclusive || dst.LowerType == Exclusive { + if err := dst.Lower.DecodeBinary(ci, ubr.Lower); err != nil { + return err + } + } + + if dst.UpperType == Inclusive || dst.UpperType == Exclusive { + if err := dst.Upper.DecodeBinary(ci, ubr.Upper); err != nil { + return err + } + } + + return nil +} + +func (src <%= range_type %>) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + switch src.LowerType { + case Exclusive, Unbounded: + buf = append(buf, '(') + case Inclusive: + buf = append(buf, '[') + case Empty: + return append(buf, "empty"...), nil + default: + return nil, errors.Errorf("unknown lower bound type %v", src.LowerType) + } + + var err error + + if src.LowerType != Unbounded { + buf, err = src.Lower.EncodeText(ci, buf) + if err != nil { + return nil, err + } else if buf == nil { + return nil, errors.Errorf("Lower cannot be null unless LowerType is Unbounded") + } + } + + buf = append(buf, ',') + + if src.UpperType != Unbounded { + buf, err = src.Upper.EncodeText(ci, buf) + if err != nil { + return nil, err + } else if buf == nil { + return nil, errors.Errorf("Upper cannot be null unless UpperType is Unbounded") + } + } + + switch src.UpperType { + case Exclusive, Unbounded: + buf = append(buf, ')') + case Inclusive: + buf = append(buf, ']') + default: + return nil, errors.Errorf("unknown upper bound type %v", src.UpperType) + } + + return buf, nil +} + +func (src <%= range_type %>) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + var rangeType byte + switch src.LowerType { + case Inclusive: + rangeType |= lowerInclusiveMask + case Unbounded: + rangeType |= lowerUnboundedMask + case Exclusive: + case Empty: + return append(buf, emptyMask), nil + default: + return nil, errors.Errorf("unknown LowerType: %v", src.LowerType) + } + + switch src.UpperType { + case Inclusive: + rangeType |= upperInclusiveMask + case Unbounded: + rangeType |= upperUnboundedMask + case Exclusive: + default: + return nil, errors.Errorf("unknown UpperType: %v", src.UpperType) + } + + buf = append(buf, rangeType) + + var err error + + if src.LowerType != Unbounded { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + buf, err = src.Lower.EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if buf == nil { + return nil, errors.Errorf("Lower cannot be null unless LowerType is Unbounded") + } + + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + + if src.UpperType != Unbounded { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + buf, err = src.Upper.EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if buf == nil { + return nil, errors.Errorf("Upper cannot be null unless UpperType is Unbounded") + } + + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *<%= range_type %>) Scan(src interface{}) error { + if src == nil { + *dst = <%= range_type %>{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src <%= range_type %>) Value() (driver.Value, error) { + return EncodeValueText(src) +} diff --git a/github.com/jackc/pgtype/typed_range_gen.sh b/github.com/jackc/pgtype/typed_range_gen.sh new file mode 100644 index 0000000000..bedda29258 --- /dev/null +++ b/github.com/jackc/pgtype/typed_range_gen.sh @@ -0,0 +1,7 @@ +erb range_type=Int4range element_type=Int4 typed_range.go.erb > int4range.go +erb range_type=Int8range element_type=Int8 typed_range.go.erb > int8range.go +erb range_type=Tsrange element_type=Timestamp typed_range.go.erb > tsrange.go +erb range_type=Tstzrange element_type=Timestamptz typed_range.go.erb > tstzrange.go +erb range_type=Daterange element_type=Date typed_range.go.erb > daterange.go +erb range_type=Numrange element_type=Numeric typed_range.go.erb > numrange.go +goimports -w *range.go diff --git a/github.com/jackc/pgtype/unknown.go b/github.com/jackc/pgtype/unknown.go new file mode 100644 index 0000000000..c591b70835 --- /dev/null +++ b/github.com/jackc/pgtype/unknown.go @@ -0,0 +1,44 @@ +package pgtype + +import "database/sql/driver" + +// Unknown represents the PostgreSQL unknown type. It is either a string literal +// or NULL. It is used when PostgreSQL does not know the type of a value. In +// general, this will only be used in pgx when selecting a null value without +// type information. e.g. SELECT NULL; +type Unknown struct { + String string + Status Status +} + +func (dst *Unknown) Set(src interface{}) error { + return (*Text)(dst).Set(src) +} + +func (dst Unknown) Get() interface{} { + return (Text)(dst).Get() +} + +// AssignTo assigns from src to dst. Note that as Unknown is not a general number +// type AssignTo does not do automatic type conversion as other number types do. +func (src *Unknown) AssignTo(dst interface{}) error { + return (*Text)(src).AssignTo(dst) +} + +func (dst *Unknown) DecodeText(ci *ConnInfo, src []byte) error { + return (*Text)(dst).DecodeText(ci, src) +} + +func (dst *Unknown) DecodeBinary(ci *ConnInfo, src []byte) error { + return (*Text)(dst).DecodeBinary(ci, src) +} + +// Scan implements the database/sql Scanner interface. +func (dst *Unknown) Scan(src interface{}) error { + return (*Text)(dst).Scan(src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Unknown) Value() (driver.Value, error) { + return (Text)(src).Value() +} diff --git a/github.com/jackc/pgtype/uuid.go b/github.com/jackc/pgtype/uuid.go new file mode 100644 index 0000000000..bdbe17e426 --- /dev/null +++ b/github.com/jackc/pgtype/uuid.go @@ -0,0 +1,193 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/hex" + "fmt" + + errors "golang.org/x/xerrors" +) + +type UUID struct { + Bytes [16]byte + Status Status +} + +func (dst *UUID) Set(src interface{}) error { + if src == nil { + *dst = UUID{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + case [16]byte: + *dst = UUID{Bytes: value, Status: Present} + case []byte: + if value != nil { + if len(value) != 16 { + return errors.Errorf("[]byte must be 16 bytes to convert to UUID: %d", len(value)) + } + *dst = UUID{Status: Present} + copy(dst.Bytes[:], value) + } else { + *dst = UUID{Status: Null} + } + case string: + uuid, err := parseUUID(value) + if err != nil { + return err + } + *dst = UUID{Bytes: uuid, Status: Present} + default: + if originalSrc, ok := underlyingUUIDType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to UUID", value) + } + + return nil +} + +func (dst UUID) Get() interface{} { + switch dst.Status { + case Present: + return dst.Bytes + case Null: + return nil + default: + return dst.Status + } +} + +func (src *UUID) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + case *[16]byte: + *v = src.Bytes + return nil + case *[]byte: + *v = make([]byte, 16) + copy(*v, src.Bytes[:]) + return nil + case *string: + *v = encodeUUID(src.Bytes) + return nil + default: + if nextDst, retry := GetAssignToDstType(v); retry { + return src.AssignTo(nextDst) + } + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot assign %v into %T", src, dst) +} + +// parseUUID converts a string UUID in standard form to a byte array. +func parseUUID(src string) (dst [16]byte, err error) { + if len(src) < 36 { + return dst, errors.Errorf("cannot parse UUID %v", src) + } + src = src[0:8] + src[9:13] + src[14:18] + src[19:23] + src[24:] + buf, err := hex.DecodeString(src) + if err != nil { + return dst, err + } + + copy(dst[:], buf) + return dst, err +} + +// encodeUUID converts a uuid byte array to UUID standard string form. +func encodeUUID(src [16]byte) string { + return fmt.Sprintf("%x-%x-%x-%x-%x", src[0:4], src[4:6], src[6:8], src[8:10], src[10:16]) +} + +func (dst *UUID) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = UUID{Status: Null} + return nil + } + + if len(src) != 36 { + return errors.Errorf("invalid length for UUID: %v", len(src)) + } + + buf, err := parseUUID(string(src)) + if err != nil { + return err + } + + *dst = UUID{Bytes: buf, Status: Present} + return nil +} + +func (dst *UUID) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = UUID{Status: Null} + return nil + } + + if len(src) != 16 { + return errors.Errorf("invalid length for UUID: %v", len(src)) + } + + *dst = UUID{Status: Present} + copy(dst.Bytes[:], src) + return nil +} + +func (src UUID) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + return append(buf, encodeUUID(src.Bytes)...), nil +} + +func (src UUID) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + return append(buf, src.Bytes[:]...), nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *UUID) Scan(src interface{}) error { + if src == nil { + *dst = UUID{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src UUID) Value() (driver.Value, error) { + return EncodeValueText(src) +} diff --git a/github.com/jackc/pgtype/uuid_array.go b/github.com/jackc/pgtype/uuid_array.go new file mode 100644 index 0000000000..4cd6501783 --- /dev/null +++ b/github.com/jackc/pgtype/uuid_array.go @@ -0,0 +1,376 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type UUIDArray struct { + Elements []UUID + Dimensions []ArrayDimension + Status Status +} + +func (dst *UUIDArray) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = UUIDArray{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + + case [][16]byte: + if value == nil { + *dst = UUIDArray{Status: Null} + } else if len(value) == 0 { + *dst = UUIDArray{Status: Present} + } else { + elements := make([]UUID, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = UUIDArray{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case [][]byte: + if value == nil { + *dst = UUIDArray{Status: Null} + } else if len(value) == 0 { + *dst = UUIDArray{Status: Present} + } else { + elements := make([]UUID, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = UUIDArray{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []string: + if value == nil { + *dst = UUIDArray{Status: Null} + } else if len(value) == 0 { + *dst = UUIDArray{Status: Present} + } else { + elements := make([]UUID, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = UUIDArray{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []UUID: + if value == nil { + *dst = UUIDArray{Status: Null} + } else if len(value) == 0 { + *dst = UUIDArray{Status: Present} + } else { + *dst = UUIDArray{ + Elements: value, + Dimensions: []ArrayDimension{{Length: int32(len(value)), LowerBound: 1}}, + Status: Present, + } + } + default: + if originalSrc, ok := underlyingSliceType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to UUIDArray", value) + } + + return nil +} + +func (dst UUIDArray) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *UUIDArray) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + + case *[][16]byte: + *v = make([][16]byte, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[][]byte: + *v = make([][]byte, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + case *[]string: + *v = make([]string, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *UUIDArray) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = UUIDArray{Status: Null} + return nil + } + + uta, err := ParseUntypedTextArray(string(src)) + if err != nil { + return err + } + + var elements []UUID + + if len(uta.Elements) > 0 { + elements = make([]UUID, len(uta.Elements)) + + for i, s := range uta.Elements { + var elem UUID + var elemSrc []byte + if s != "NULL" { + elemSrc = []byte(s) + } + err = elem.DecodeText(ci, elemSrc) + if err != nil { + return err + } + + elements[i] = elem + } + } + + *dst = UUIDArray{Elements: elements, Dimensions: uta.Dimensions, Status: Present} + + return nil +} + +func (dst *UUIDArray) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = UUIDArray{Status: Null} + return nil + } + + var arrayHeader ArrayHeader + rp, err := arrayHeader.DecodeBinary(ci, src) + if err != nil { + return err + } + + if len(arrayHeader.Dimensions) == 0 { + *dst = UUIDArray{Dimensions: arrayHeader.Dimensions, Status: Present} + return nil + } + + elementCount := arrayHeader.Dimensions[0].Length + for _, d := range arrayHeader.Dimensions[1:] { + elementCount *= d.Length + } + + elements := make([]UUID, elementCount) + + for i := range elements { + elemLen := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + var elemSrc []byte + if elemLen >= 0 { + elemSrc = src[rp : rp+elemLen] + rp += elemLen + } + err = elements[i].DecodeBinary(ci, elemSrc) + if err != nil { + return err + } + } + + *dst = UUIDArray{Elements: elements, Dimensions: arrayHeader.Dimensions, Status: Present} + return nil +} + +func (src UUIDArray) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + if len(src.Dimensions) == 0 { + return append(buf, '{', '}'), nil + } + + buf = EncodeTextArrayDimensions(buf, src.Dimensions) + + // dimElemCounts is the multiples of elements that each array lies on. For + // example, a single dimension array of length 4 would have a dimElemCounts of + // [4]. A multi-dimensional array of lengths [3,5,2] would have a + // dimElemCounts of [30,10,2]. This is used to simplify when to render a '{' + // or '}'. + dimElemCounts := make([]int, len(src.Dimensions)) + dimElemCounts[len(src.Dimensions)-1] = int(src.Dimensions[len(src.Dimensions)-1].Length) + for i := len(src.Dimensions) - 2; i > -1; i-- { + dimElemCounts[i] = int(src.Dimensions[i].Length) * dimElemCounts[i+1] + } + + inElemBuf := make([]byte, 0, 32) + for i, elem := range src.Elements { + if i > 0 { + buf = append(buf, ',') + } + + for _, dec := range dimElemCounts { + if i%dec == 0 { + buf = append(buf, '{') + } + } + + elemBuf, err := elem.EncodeText(ci, inElemBuf) + if err != nil { + return nil, err + } + if elemBuf == nil { + buf = append(buf, `NULL`...) + } else { + buf = append(buf, QuoteArrayElementIfNeeded(string(elemBuf))...) + } + + for _, dec := range dimElemCounts { + if (i+1)%dec == 0 { + buf = append(buf, '}') + } + } + } + + return buf, nil +} + +func (src UUIDArray) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + arrayHeader := ArrayHeader{ + Dimensions: src.Dimensions, + } + + if dt, ok := ci.DataTypeForName("uuid"); ok { + arrayHeader.ElementOID = int32(dt.OID) + } else { + return nil, errors.Errorf("unable to find oid for type name %v", "uuid") + } + + for i := range src.Elements { + if src.Elements[i].Status == Null { + arrayHeader.ContainsNull = true + break + } + } + + buf = arrayHeader.EncodeBinary(ci, buf) + + for i := range src.Elements { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + elemBuf, err := src.Elements[i].EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if elemBuf != nil { + buf = elemBuf + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *UUIDArray) Scan(src interface{}) error { + if src == nil { + return dst.DecodeText(nil, nil) + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src UUIDArray) Value() (driver.Value, error) { + buf, err := src.EncodeText(nil, nil) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + + return string(buf), nil +} diff --git a/github.com/jackc/pgtype/varbit.go b/github.com/jackc/pgtype/varbit.go new file mode 100644 index 0000000000..7461bab37d --- /dev/null +++ b/github.com/jackc/pgtype/varbit.go @@ -0,0 +1,133 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type Varbit struct { + Bytes []byte + Len int32 // Number of bits + Status Status +} + +func (dst *Varbit) Set(src interface{}) error { + return errors.Errorf("cannot convert %v to Varbit", src) +} + +func (dst Varbit) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *Varbit) AssignTo(dst interface{}) error { + return errors.Errorf("cannot assign %v to %T", src, dst) +} + +func (dst *Varbit) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Varbit{Status: Null} + return nil + } + + bitLen := len(src) + byteLen := bitLen / 8 + if bitLen%8 > 0 { + byteLen++ + } + buf := make([]byte, byteLen) + + for i, b := range src { + if b == '1' { + byteIdx := i / 8 + bitIdx := uint(i % 8) + buf[byteIdx] = buf[byteIdx] | (128 >> bitIdx) + } + } + + *dst = Varbit{Bytes: buf, Len: int32(bitLen), Status: Present} + return nil +} + +func (dst *Varbit) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = Varbit{Status: Null} + return nil + } + + if len(src) < 4 { + return errors.Errorf("invalid length for varbit: %v", len(src)) + } + + bitLen := int32(binary.BigEndian.Uint32(src)) + rp := 4 + + *dst = Varbit{Bytes: src[rp:], Len: bitLen, Status: Present} + return nil +} + +func (src Varbit) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + for i := int32(0); i < src.Len; i++ { + byteIdx := i / 8 + bitMask := byte(128 >> byte(i%8)) + char := byte('0') + if src.Bytes[byteIdx]&bitMask > 0 { + char = '1' + } + buf = append(buf, char) + } + + return buf, nil +} + +func (src Varbit) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + buf = pgio.AppendInt32(buf, src.Len) + return append(buf, src.Bytes...), nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *Varbit) Scan(src interface{}) error { + if src == nil { + *dst = Varbit{Status: Null} + return nil + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Varbit) Value() (driver.Value, error) { + return EncodeValueText(src) +} diff --git a/github.com/jackc/pgtype/varchar.go b/github.com/jackc/pgtype/varchar.go new file mode 100644 index 0000000000..e4fa6869e8 --- /dev/null +++ b/github.com/jackc/pgtype/varchar.go @@ -0,0 +1,58 @@ +package pgtype + +import ( + "database/sql/driver" +) + +type Varchar Text + +// Set converts from src to dst. Note that as Varchar is not a general +// number type Set does not do automatic type conversion as other number +// types do. +func (dst *Varchar) Set(src interface{}) error { + return (*Text)(dst).Set(src) +} + +func (dst Varchar) Get() interface{} { + return (Text)(dst).Get() +} + +// AssignTo assigns from src to dst. Note that as Varchar is not a general number +// type AssignTo does not do automatic type conversion as other number types do. +func (src *Varchar) AssignTo(dst interface{}) error { + return (*Text)(src).AssignTo(dst) +} + +func (dst *Varchar) DecodeText(ci *ConnInfo, src []byte) error { + return (*Text)(dst).DecodeText(ci, src) +} + +func (dst *Varchar) DecodeBinary(ci *ConnInfo, src []byte) error { + return (*Text)(dst).DecodeBinary(ci, src) +} + +func (src Varchar) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + return (Text)(src).EncodeText(ci, buf) +} + +func (src Varchar) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + return (Text)(src).EncodeBinary(ci, buf) +} + +// Scan implements the database/sql Scanner interface. +func (dst *Varchar) Scan(src interface{}) error { + return (*Text)(dst).Scan(src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src Varchar) Value() (driver.Value, error) { + return (Text)(src).Value() +} + +func (src Varchar) MarshalJSON() ([]byte, error) { + return (Text)(src).MarshalJSON() +} + +func (dst *Varchar) UnmarshalJSON(b []byte) error { + return (*Text)(dst).UnmarshalJSON(b) +} diff --git a/github.com/jackc/pgtype/varchar_array.go b/github.com/jackc/pgtype/varchar_array.go new file mode 100644 index 0000000000..b13f29cef4 --- /dev/null +++ b/github.com/jackc/pgtype/varchar_array.go @@ -0,0 +1,320 @@ +package pgtype + +import ( + "database/sql/driver" + "encoding/binary" + + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +type VarcharArray struct { + Elements []Varchar + Dimensions []ArrayDimension + Status Status +} + +func (dst *VarcharArray) Set(src interface{}) error { + // untyped nil and typed nil interfaces are different + if src == nil { + *dst = VarcharArray{Status: Null} + return nil + } + + if value, ok := src.(interface{ Get() interface{} }); ok { + value2 := value.Get() + if value2 != value { + return dst.Set(value2) + } + } + + switch value := src.(type) { + + case []string: + if value == nil { + *dst = VarcharArray{Status: Null} + } else if len(value) == 0 { + *dst = VarcharArray{Status: Present} + } else { + elements := make([]Varchar, len(value)) + for i := range value { + if err := elements[i].Set(value[i]); err != nil { + return err + } + } + *dst = VarcharArray{ + Elements: elements, + Dimensions: []ArrayDimension{{Length: int32(len(elements)), LowerBound: 1}}, + Status: Present, + } + } + + case []Varchar: + if value == nil { + *dst = VarcharArray{Status: Null} + } else if len(value) == 0 { + *dst = VarcharArray{Status: Present} + } else { + *dst = VarcharArray{ + Elements: value, + Dimensions: []ArrayDimension{{Length: int32(len(value)), LowerBound: 1}}, + Status: Present, + } + } + default: + if originalSrc, ok := underlyingSliceType(src); ok { + return dst.Set(originalSrc) + } + return errors.Errorf("cannot convert %v to VarcharArray", value) + } + + return nil +} + +func (dst VarcharArray) Get() interface{} { + switch dst.Status { + case Present: + return dst + case Null: + return nil + default: + return dst.Status + } +} + +func (src *VarcharArray) AssignTo(dst interface{}) error { + switch src.Status { + case Present: + switch v := dst.(type) { + + case *[]string: + *v = make([]string, len(src.Elements)) + for i := range src.Elements { + if err := src.Elements[i].AssignTo(&((*v)[i])); err != nil { + return err + } + } + return nil + + default: + if nextDst, retry := GetAssignToDstType(dst); retry { + return src.AssignTo(nextDst) + } + return errors.Errorf("unable to assign to %T", dst) + } + case Null: + return NullAssignTo(dst) + } + + return errors.Errorf("cannot decode %#v into %T", src, dst) +} + +func (dst *VarcharArray) DecodeText(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = VarcharArray{Status: Null} + return nil + } + + uta, err := ParseUntypedTextArray(string(src)) + if err != nil { + return err + } + + var elements []Varchar + + if len(uta.Elements) > 0 { + elements = make([]Varchar, len(uta.Elements)) + + for i, s := range uta.Elements { + var elem Varchar + var elemSrc []byte + if s != "NULL" { + elemSrc = []byte(s) + } + err = elem.DecodeText(ci, elemSrc) + if err != nil { + return err + } + + elements[i] = elem + } + } + + *dst = VarcharArray{Elements: elements, Dimensions: uta.Dimensions, Status: Present} + + return nil +} + +func (dst *VarcharArray) DecodeBinary(ci *ConnInfo, src []byte) error { + if src == nil { + *dst = VarcharArray{Status: Null} + return nil + } + + var arrayHeader ArrayHeader + rp, err := arrayHeader.DecodeBinary(ci, src) + if err != nil { + return err + } + + if len(arrayHeader.Dimensions) == 0 { + *dst = VarcharArray{Dimensions: arrayHeader.Dimensions, Status: Present} + return nil + } + + elementCount := arrayHeader.Dimensions[0].Length + for _, d := range arrayHeader.Dimensions[1:] { + elementCount *= d.Length + } + + elements := make([]Varchar, elementCount) + + for i := range elements { + elemLen := int(int32(binary.BigEndian.Uint32(src[rp:]))) + rp += 4 + var elemSrc []byte + if elemLen >= 0 { + elemSrc = src[rp : rp+elemLen] + rp += elemLen + } + err = elements[i].DecodeBinary(ci, elemSrc) + if err != nil { + return err + } + } + + *dst = VarcharArray{Elements: elements, Dimensions: arrayHeader.Dimensions, Status: Present} + return nil +} + +func (src VarcharArray) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + if len(src.Dimensions) == 0 { + return append(buf, '{', '}'), nil + } + + buf = EncodeTextArrayDimensions(buf, src.Dimensions) + + // dimElemCounts is the multiples of elements that each array lies on. For + // example, a single dimension array of length 4 would have a dimElemCounts of + // [4]. A multi-dimensional array of lengths [3,5,2] would have a + // dimElemCounts of [30,10,2]. This is used to simplify when to render a '{' + // or '}'. + dimElemCounts := make([]int, len(src.Dimensions)) + dimElemCounts[len(src.Dimensions)-1] = int(src.Dimensions[len(src.Dimensions)-1].Length) + for i := len(src.Dimensions) - 2; i > -1; i-- { + dimElemCounts[i] = int(src.Dimensions[i].Length) * dimElemCounts[i+1] + } + + inElemBuf := make([]byte, 0, 32) + for i, elem := range src.Elements { + if i > 0 { + buf = append(buf, ',') + } + + for _, dec := range dimElemCounts { + if i%dec == 0 { + buf = append(buf, '{') + } + } + + elemBuf, err := elem.EncodeText(ci, inElemBuf) + if err != nil { + return nil, err + } + if elemBuf == nil { + buf = append(buf, `NULL`...) + } else { + buf = append(buf, QuoteArrayElementIfNeeded(string(elemBuf))...) + } + + for _, dec := range dimElemCounts { + if (i+1)%dec == 0 { + buf = append(buf, '}') + } + } + } + + return buf, nil +} + +func (src VarcharArray) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + switch src.Status { + case Null: + return nil, nil + case Undefined: + return nil, errUndefined + } + + arrayHeader := ArrayHeader{ + Dimensions: src.Dimensions, + } + + if dt, ok := ci.DataTypeForName("varchar"); ok { + arrayHeader.ElementOID = int32(dt.OID) + } else { + return nil, errors.Errorf("unable to find oid for type name %v", "varchar") + } + + for i := range src.Elements { + if src.Elements[i].Status == Null { + arrayHeader.ContainsNull = true + break + } + } + + buf = arrayHeader.EncodeBinary(ci, buf) + + for i := range src.Elements { + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + + elemBuf, err := src.Elements[i].EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if elemBuf != nil { + buf = elemBuf + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + } + + return buf, nil +} + +// Scan implements the database/sql Scanner interface. +func (dst *VarcharArray) Scan(src interface{}) error { + if src == nil { + return dst.DecodeText(nil, nil) + } + + switch src := src.(type) { + case string: + return dst.DecodeText(nil, []byte(src)) + case []byte: + srcCopy := make([]byte, len(src)) + copy(srcCopy, src) + return dst.DecodeText(nil, srcCopy) + } + + return errors.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src VarcharArray) Value() (driver.Value, error) { + buf, err := src.EncodeText(nil, nil) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + + return string(buf), nil +} diff --git a/github.com/jackc/pgtype/xid.go b/github.com/jackc/pgtype/xid.go new file mode 100644 index 0000000000..f6d6b22d5f --- /dev/null +++ b/github.com/jackc/pgtype/xid.go @@ -0,0 +1,64 @@ +package pgtype + +import ( + "database/sql/driver" +) + +// XID is PostgreSQL's Transaction ID type. +// +// In later versions of PostgreSQL, it is the type used for the backend_xid +// and backend_xmin columns of the pg_stat_activity system view. +// +// Also, when one does +// +// select xmin, xmax, * from some_table; +// +// it is the data type of the xmin and xmax hidden system columns. +// +// It is currently implemented as an unsigned four byte integer. +// Its definition can be found in src/include/postgres_ext.h as TransactionId +// in the PostgreSQL sources. +type XID pguint32 + +// Set converts from src to dst. Note that as XID is not a general +// number type Set does not do automatic type conversion as other number +// types do. +func (dst *XID) Set(src interface{}) error { + return (*pguint32)(dst).Set(src) +} + +func (dst XID) Get() interface{} { + return (pguint32)(dst).Get() +} + +// AssignTo assigns from src to dst. Note that as XID is not a general number +// type AssignTo does not do automatic type conversion as other number types do. +func (src *XID) AssignTo(dst interface{}) error { + return (*pguint32)(src).AssignTo(dst) +} + +func (dst *XID) DecodeText(ci *ConnInfo, src []byte) error { + return (*pguint32)(dst).DecodeText(ci, src) +} + +func (dst *XID) DecodeBinary(ci *ConnInfo, src []byte) error { + return (*pguint32)(dst).DecodeBinary(ci, src) +} + +func (src XID) EncodeText(ci *ConnInfo, buf []byte) ([]byte, error) { + return (pguint32)(src).EncodeText(ci, buf) +} + +func (src XID) EncodeBinary(ci *ConnInfo, buf []byte) ([]byte, error) { + return (pguint32)(src).EncodeBinary(ci, buf) +} + +// Scan implements the database/sql Scanner interface. +func (dst *XID) Scan(src interface{}) error { + return (*pguint32)(dst).Scan(src) +} + +// Value implements the database/sql/driver Valuer interface. +func (src XID) Value() (driver.Value, error) { + return (pguint32)(src).Value() +} diff --git a/github.com/jackc/pgx/v4/.gitignore b/github.com/jackc/pgx/v4/.gitignore new file mode 100644 index 0000000000..39175a965d --- /dev/null +++ b/github.com/jackc/pgx/v4/.gitignore @@ -0,0 +1,24 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe + +.envrc diff --git a/github.com/jackc/pgx/v4/.travis.yml b/github.com/jackc/pgx/v4/.travis.yml new file mode 100644 index 0000000000..1bc5a8ae41 --- /dev/null +++ b/github.com/jackc/pgx/v4/.travis.yml @@ -0,0 +1,33 @@ +language: go + +go: + - 1.x + - tip + +# Derived from https://github.com/lib/pq/blob/master/.travis.yml +before_install: + - ./travis/before_install.bash + +env: + global: + - GO111MODULE=on + - PGX_TEST_DATABASE=postgres://pgx_md5:secret@127.0.0.1/pgx_test + + matrix: + - CRATEVERSION=2.1 PGX_TEST_CRATEDB_CONN_STRING="host=127.0.0.1 port=6543 user=pgx database=pgx_test" + - PGVERSION=12 + - PGVERSION=11 + - PGVERSION=10 + - PGVERSION=9.6 + - PGVERSION=9.5 + - PGVERSION=9.4 + +before_script: + - ./travis/before_script.bash + +script: + - ./travis/script.bash + +matrix: + allow_failures: + - go: tip diff --git a/github.com/jackc/pgx/v4/CHANGELOG.md b/github.com/jackc/pgx/v4/CHANGELOG.md new file mode 100644 index 0000000000..39ab1466f8 --- /dev/null +++ b/github.com/jackc/pgx/v4/CHANGELOG.md @@ -0,0 +1,102 @@ +# 4.6.0 (March 30, 2020) + +* stdlib: Bail early if preloading rows.Next() results in rows.Err() (Bas van Beek) +* Sanitize time to microsecond accuracy (Andrew Nicoll) +* Update pgtype to v1.3.0 +* Update pgconn to v1.5.0 + * Update golang.org/x/crypto for security fix + * Implement "verify-ca" SSL mode + +# 4.5.0 (March 7, 2020) + +* Update to pgconn v1.4.0 + * Fixes QueryRow with empty SQL + * Adds PostgreSQL service file support +* Add Len() to *pgx.Batch (WGH) +* Better logging for individual batch items (Ben Bader) + +# 4.4.1 (February 14, 2020) + +* Update pgconn to v1.3.2 - better default read buffer size +* Fix race in CopyFrom + +# 4.4.0 (February 5, 2020) + +* Update puddle to v1.1.0 - fixes possible deadlock when acquire is cancelled +* Update pgconn to v1.3.1 - fixes CopyFrom deadlock when multiple NoticeResponse received during copy +* Update pgtype to v1.2.0 +* Add MaxConnIdleTime to pgxpool (Patrick Ellul) +* Add MinConns to pgxpool (Patrick Ellul) +* Fix: stdlib.ReleaseConn closes connections left in invalid state + +# 4.3.0 (January 23, 2020) + +* Fix Rows.Values panic when unable to decode +* Add Rows.Values support for unknown types +* Add DriverContext support for stdlib (Alex Gaynor) +* Update pgproto3 to v2.0.1 to never return an io.EOF as it would be misinterpreted by database/sql. Instead return io.UnexpectedEOF. + +# 4.2.1 (January 13, 2020) + +* Update pgconn to v1.2.1 (fixes context cancellation data race introduced in v1.2.0)) + +# 4.2.0 (January 11, 2020) + +* Update pgconn to v1.2.0. +* Update pgtype to v1.1.0. +* Return error instead of panic when wrong number of arguments passed to Exec. (malstoun) +* Fix large objects functionality when PreferSimpleProtocol = true. +* Restore GetDefaultDriver which existed in v3. (Johan Brandhorst) +* Add RegisterConnConfig to stdlib which replaces the removed RegisterDriverConfig from v3. + +# 4.1.2 (October 22, 2019) + +* Fix dbSavepoint.Begin recursive self call +* Upgrade pgtype to v1.0.2 - fix scan pointer to pointer + +# 4.1.1 (October 21, 2019) + +* Fix pgxpool Rows.CommandTag() infinite loop / typo + +# 4.1.0 (October 12, 2019) + +## Potentially Breaking Changes + +Technically, two changes are breaking changes, but in practice these are extremely unlikely to break existing code. + +* Conn.Begin and Conn.BeginTx return a Tx interface instead of the internal dbTx struct. This is necessary for the Conn.Begin method to signature as other methods that begin a transaction. +* Add Conn() to Tx interface. This is necessary to allow code using a Tx to access the *Conn (and pgconn.PgConn) on which the Tx is executing. + +## Fixes + +* Releasing a busy connection closes the connection instead of returning an unusable connection to the pool +* Do not mutate config.Config.OnNotification in connect + +# 4.0.1 (September 19, 2019) + +* Fix statement cache cleanup. +* Corrected daterange OID. +* Fix Tx when committing or rolling back multiple times in certain cases. +* Improve documentation. + +# 4.0.0 (September 14, 2019) + +v4 is a major release with many significant changes some of which are breaking changes. The most significant are +included below. + +* Simplified establishing a connection with a connection string. +* All potentially blocking operations now require a context.Context. The non-context aware functions have been removed. +* OIDs are hard-coded for known types. This saves the query on connection. +* Context cancellations while network activity is in progress is now always fatal. Previously, it was sometimes recoverable. This led to increased complexity in pgx itself and in application code. +* Go modules are required. +* Errors are now implemented in the Go 1.13 style. +* `Rows` and `Tx` are now interfaces. +* The connection pool as been decoupled from pgx and is now a separate, included package (github.com/jackc/pgx/v4/pgxpool). +* pgtype has been spun off to a separate package (github.com/jackc/pgtype). +* pgproto3 has been spun off to a separate package (github.com/jackc/pgproto3/v2). +* Logical replication support has been spun off to a separate package (github.com/jackc/pglogrepl). +* Lower level PostgreSQL functionality is now implemented in a separate package (github.com/jackc/pgconn). +* Tests are now configured with environment variables. +* Conn has an automatic statement cache by default. +* Batch interface has been simplified. +* QueryArgs has been removed. diff --git a/github.com/jackc/pgx/v4/LICENSE b/github.com/jackc/pgx/v4/LICENSE new file mode 100644 index 0000000000..7dee3daf8a --- /dev/null +++ b/github.com/jackc/pgx/v4/LICENSE @@ -0,0 +1,22 @@ +Copyright (c) 2013 Jack Christensen + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/github.com/jackc/pgx/v4/README.md b/github.com/jackc/pgx/v4/README.md new file mode 100644 index 0000000000..8ec6d00001 --- /dev/null +++ b/github.com/jackc/pgx/v4/README.md @@ -0,0 +1,186 @@ +[![](https://godoc.org/github.com/jackc/pgx?status.svg)](https://pkg.go.dev/github.com/jackc/pgx/v4) +[![Build Status](https://travis-ci.org/jackc/pgx.svg)](https://travis-ci.org/jackc/pgx) + +# pgx - PostgreSQL Driver and Toolkit + +pgx is a pure Go driver and toolkit for PostgreSQL. + +pgx aims to be low-level, fast, and performant, while also enabling PostgreSQL-specific features that the standard `database/sql` package does not allow for. + +The driver component of pgx can be used alongside the standard `database/sql` package. + +The toolkit component is a related set of packages that implement PostgreSQL functionality such as parsing the wire protocol +and type mapping between PostgreSQL and Go. These underlying packages can be used to implement alternative drivers, +proxies, load balancers, logical replication clients, etc. + +The current release of `pgx v4` requires Go modules. To use the previous version, checkout and vendor the `v3` branch. + +## Example Usage + +```go +package main + +import ( + "context" + "fmt" + "os" + + "github.com/jackc/pgx/v4" +) + +func main() { + conn, err := pgx.Connect(context.Background(), os.Getenv("DATABASE_URL")) + if err != nil { + fmt.Fprintf(os.Stderr, "Unable to connect to database: %v\n", err) + os.Exit(1) + } + defer conn.Close(context.Background()) + + var name string + var weight int64 + err = conn.QueryRow(context.Background(), "select name, weight from widgets where id=$1", 42).Scan(&name, &weight) + if err != nil { + fmt.Fprintf(os.Stderr, "QueryRow failed: %v\n", err) + os.Exit(1) + } + + fmt.Println(name, weight) +} +``` + +## Choosing Between the pgx and database/sql Interfaces + +It is recommended to use the pgx interface if: +1. The application only targets PostgreSQL. +2. No other libraries that require `database/sql` are in use. + +The pgx interface is faster and exposes more features. + +The `database/sql` interface only allows the underlying driver to return or receive the following types: `int64`, +`float64`, `bool`, `[]byte`, `string`, `time.Time`, or `nil`. Handling other types requires implementing the +`database/sql.Scanner` and the `database/sq/driver.Valuer` interfaces which require transmission of values in text format. The binary format can be substantially faster, which is what the pgx interface uses. + +## Features + +pgx supports many features beyond what is available through `database/sql`: + +* Support for approximately 70 different PostgreSQL types +* Automatic statement preparation and caching +* Batch queries +* Single-round trip query mode +* Full TLS connection control +* Binary format support for custom types (allows for much quicker encoding/decoding) +* Copy protocol support for faster bulk data loads +* Extendable logging support including built-in support for `log15adapter`, [`logrus`](https://github.com/sirupsen/logrus), [`zap`](https://github.com/uber-go/zap), and [`zerolog`](https://github.com/rs/zerolog) +* Connection pool with after-connect hook for arbitrary connection setup +* Listen / notify +* Conversion of PostgreSQL arrays to Go slice mappings for integers, floats, and strings +* Hstore support +* JSON and JSONB support +* Maps `inet` and `cidr` PostgreSQL types to `net.IPNet` and `net.IP` +* Large object support +* NULL mapping to Null* struct or pointer to pointer +* Supports `database/sql.Scanner` and `database/sql/driver.Valuer` interfaces for custom types +* Notice response handling +* Simulated nested transactions with savepoints + +## Performance + +There are three areas in particular where pgx can provide a significant performance advantage over the standard +`database/sql` interface and other drivers: + +1. PostgreSQL specific types - Types such as arrays can be parsed much quicker because pgx uses the binary format. +2. Automatic statement preparation and caching - pgx will prepare and cache statements by default. This can provide an + significant free improvement to code that does not explicitly use prepared statements. Under certain workloads, it can + perform nearly 3x the number of queries per second. +3. Batched queries - Multiple queries can be batched together to minimize network round trips. + +## Comparison with Alternatives + +* [pq](http://godoc.org/github.com/lib/pq) +* [go-pg](https://github.com/go-pg/pg) + +For prepared queries with small sets of simple data types, all drivers will have have similar performance. However, if prepared statements aren't being explicitly used, pgx can have a significant performance advantage due to automatic statement preparation. +pgx also can perform better when using PostgreSQL-specific data types or query batching. See +[go_db_bench](https://github.com/jackc/go_db_bench) for some database driver benchmarks. + +### Compatibility with `database/sql` + +pq is exclusively used with `database/sql`. go-pg does not use `database/sql` at all. pgx supports `database/sql` as well as +its own interface. + +### Level of access, ORM + +go-pg is a PostgreSQL client and ORM. It includes many features that traditionally sit above the database driver, such as ORM, struct mapping, soft deletes, schema migrations, and sharding support. + +pgx is "closer to the metal" and such abstractions are beyond the scope of the pgx project, which first and foremost, aims to be a performant driver and toolkit. + +## Testing + +pgx tests naturally require a PostgreSQL database. It will connect to the database specified in the `PGX_TEST_DATABASE` environment +variable. The `PGX_TEST_DATABASE` environment variable can either be a URL or DSN. In addition, the standard `PG*` environment +variables will be respected. Consider using [direnv](https://github.com/direnv/direnv) to simplify environment variable +handling. + +### Example Test Environment + +Connect to your PostgreSQL server and run: + +``` +create database pgx_test; +``` + +Connect to the newly-created database and run: + +``` +create domain uint64 as numeric(20,0); +``` + +Now, you can run the tests: + +``` +PGX_TEST_DATABASE="host=/var/run/postgresql database=pgx_test" go test ./... +``` + +In addition, there are tests specific for PgBouncer that will be executed if `PGX_TEST_PGBOUNCER_CONN_STRING` is set. + +## Version Policy + +pgx follows semantic versioning for the documented public API on stable releases. `v4` is the latest stable major version. + +## Related Libraries + +pgx is the head of a family of PostgreSQL libraries. Many of these can be used independently. Many can also be accessed +from pgx for lower-level control. + +### github.com/jackc/pgconn + +`pgconn` is a lower-level PostgreSQL database driver that operates at nearly the same level as the C library `libpq`. + +### github.com/jackc/pgx/v4/pgxpool + +`pgxpool` is a connection pool for pgx. pgx is entirely decoupled from its default pool implementation. This means that pgx can be used with a different pool. or without any pool at all. + +### github.com/jackc/pgx/v4/stdlib + +This is a `database/sql` compatibility layer for pgx. pgx can be used as a normal `database/sql` driver, but at any time, the native interface can be acquired for more performance or PostgreSQL specific functionality. + +### github.com/jackc/pgtype + +Over 70 PostgreSQL types are supported including `uuid`, `hstore`, `json`, `bytea`, `numeric`, `interval`, `inet`, and arrays. These types support `database/sql` interfaces and are usable outside of pgx. They are fully tested in pgx and pq. They also support a higher performance interface when used with the pgx driver. + +### github.com/jackc/pgproto3 + +pgproto3 provides standalone encoding and decoding of the PostgreSQL v3 wire protocol. This is useful for implementing very low level PostgreSQL tooling. + +### github.com/jackc/pglogrepl + +pglogrepl provides functionality to act as a client for PostgreSQL logical replication. + +### github.com/jackc/pgmock + +pgmock offers the ability to create a server that mocks the PostgreSQL wire protocol. This is used internally to test pgx by purposely inducing unusual errors. pgproto3 and pgmock together provide most of the foundational tooling required to implement a PostgreSQL proxy or MitM (such as for a custom connection pooler). + +### github.com/jackc/tern + +tern is a stand-alone SQL migration system. diff --git a/github.com/jackc/pgx/v4/batch.go b/github.com/jackc/pgx/v4/batch.go new file mode 100644 index 0000000000..9f787f9968 --- /dev/null +++ b/github.com/jackc/pgx/v4/batch.go @@ -0,0 +1,181 @@ +package pgx + +import ( + "context" + + "github.com/jackc/pgconn" + errors "golang.org/x/xerrors" +) + +type batchItem struct { + query string + arguments []interface{} +} + +// Batch queries are a way of bundling multiple queries together to avoid +// unnecessary network round trips. +type Batch struct { + items []*batchItem +} + +// Queue queues a query to batch b. query can be an SQL query or the name of a prepared statement. +func (b *Batch) Queue(query string, arguments ...interface{}) { + b.items = append(b.items, &batchItem{ + query: query, + arguments: arguments, + }) +} + +// Len returns number of queries that have been queued so far. +func (b *Batch) Len() int { + return len(b.items) +} + +type BatchResults interface { + // Exec reads the results from the next query in the batch as if the query has been sent with Conn.Exec. + Exec() (pgconn.CommandTag, error) + + // Query reads the results from the next query in the batch as if the query has been sent with Conn.Query. + Query() (Rows, error) + + // QueryRow reads the results from the next query in the batch as if the query has been sent with Conn.QueryRow. + QueryRow() Row + + // Close closes the batch operation. This must be called before the underlying connection can be used again. Any error + // that occurred during a batch operation may have made it impossible to resyncronize the connection with the server. + // In this case the underlying connection will have been closed. + Close() error +} + +type batchResults struct { + ctx context.Context + conn *Conn + mrr *pgconn.MultiResultReader + err error + b *Batch + ix int +} + +// Exec reads the results from the next query in the batch as if the query has been sent with Exec. +func (br *batchResults) Exec() (pgconn.CommandTag, error) { + if br.err != nil { + return nil, br.err + } + + query, arguments, _ := br.nextQueryAndArgs() + + if !br.mrr.NextResult() { + err := br.mrr.Close() + if err == nil { + err = errors.New("no result") + } + if br.conn.shouldLog(LogLevelError) { + br.conn.log(br.ctx, LogLevelError, "BatchResult.Exec", map[string]interface{} { + "sql": query, + "args": logQueryArgs(arguments), + "err": err, + }) + } + return nil, err + } + + commandTag, err := br.mrr.ResultReader().Close() + + if err != nil { + if br.conn.shouldLog(LogLevelError) { + br.conn.log(br.ctx, LogLevelError, "BatchResult.Exec", map[string]interface{}{ + "sql": query, + "args": logQueryArgs(arguments), + "err": err, + }) + } + } else if br.conn.shouldLog(LogLevelInfo) { + br.conn.log(br.ctx, LogLevelInfo, "BatchResult.Exec", map[string]interface{} { + "sql": query, + "args": logQueryArgs(arguments), + "commandTag": commandTag, + }) + } + + return commandTag, err +} + +// Query reads the results from the next query in the batch as if the query has been sent with Query. +func (br *batchResults) Query() (Rows, error) { + query, arguments, ok := br.nextQueryAndArgs() + if !ok { + query = "batch query" + } + + rows := br.conn.getRows(br.ctx, query, arguments) + + if br.err != nil { + rows.err = br.err + rows.closed = true + return rows, br.err + } + + if !br.mrr.NextResult() { + rows.err = br.mrr.Close() + if rows.err == nil { + rows.err = errors.New("no result") + } + rows.closed = true + + if br.conn.shouldLog(LogLevelError) { + br.conn.log(br.ctx, LogLevelError, "BatchResult.Query", map[string]interface{} { + "sql": query, + "args": logQueryArgs(arguments), + "err": rows.err, + }) + } + + return rows, rows.err + } + + rows.resultReader = br.mrr.ResultReader() + return rows, nil +} + +// QueryRow reads the results from the next query in the batch as if the query has been sent with QueryRow. +func (br *batchResults) QueryRow() Row { + rows, _ := br.Query() + return (*connRow)(rows.(*connRows)) + +} + +// Close closes the batch operation. Any error that occurred during a batch operation may have made it impossible to +// resyncronize the connection with the server. In this case the underlying connection will have been closed. +func (br *batchResults) Close() error { + if br.err != nil { + return br.err + } + + // log any queries that haven't yet been logged by Exec or Query + for { + query, args, ok := br.nextQueryAndArgs() + if !ok { + break + } + + if br.conn.shouldLog(LogLevelInfo) { + br.conn.log(br.ctx, LogLevelInfo, "BatchResult.Close", map[string]interface{} { + "sql": query, + "args": logQueryArgs(args), + }) + } + } + + return br.mrr.Close() +} + +func (br *batchResults) nextQueryAndArgs() (query string, args []interface{}, ok bool) { + if br.b != nil && br.ix < len(br.b.items) { + bi := br.b.items[br.ix] + query = bi.query + args = bi.arguments + ok = true + br.ix++ + } + return +} diff --git a/github.com/jackc/pgx/v4/conn.go b/github.com/jackc/pgx/v4/conn.go new file mode 100644 index 0000000000..0acd5c969c --- /dev/null +++ b/github.com/jackc/pgx/v4/conn.go @@ -0,0 +1,797 @@ +package pgx + +import ( + "context" + "strconv" + "strings" + "time" + + errors "golang.org/x/xerrors" + + "github.com/jackc/pgconn" + "github.com/jackc/pgconn/stmtcache" + "github.com/jackc/pgproto3/v2" + "github.com/jackc/pgtype" + "github.com/jackc/pgx/v4/internal/sanitize" +) + +const ( + connStatusUninitialized = iota + connStatusClosed + connStatusIdle + connStatusBusy +) + +// ConnConfig contains all the options used to establish a connection. It must be created by ParseConfig and +// then it can be modified. A manually initialized ConnConfig will cause ConnectConfig to panic. +type ConnConfig struct { + pgconn.Config + Logger Logger + LogLevel LogLevel + + // BuildStatementCache creates the stmtcache.Cache implementation for connections created with this config. Set + // to nil to disable automatic prepared statements. + BuildStatementCache BuildStatementCacheFunc + + // PreferSimpleProtocol disables implicit prepared statement usage. By default pgx automatically uses the extended + // protocol. This can improve performance due to being able to use the binary format. It also does not rely on client + // side parameter sanitization. However, it does incur two round-trips per query (unless using a prepared statement) + // and may be incompatible proxies such as PGBouncer. Setting PreferSimpleProtocol causes the simple protocol to be + // used by default. The same functionality can be controlled on a per query basis by setting + // QueryExOptions.SimpleProtocol. + PreferSimpleProtocol bool + + createdByParseConfig bool // Used to enforce created by ParseConfig rule. +} + +// BuildStatementCacheFunc is a function that can be used to create a stmtcache.Cache implementation for connection. +type BuildStatementCacheFunc func(conn *pgconn.PgConn) stmtcache.Cache + +// Conn is a PostgreSQL connection handle. It is not safe for concurrent usage. Use a connection pool to manage access +// to multiple database connections from multiple goroutines. +type Conn struct { + pgConn *pgconn.PgConn + config *ConnConfig // config used when establishing this connection + preparedStatements map[string]*pgconn.StatementDescription + stmtcache stmtcache.Cache + logger Logger + logLevel LogLevel + + notifications []*pgconn.Notification + + doneChan chan struct{} + closedChan chan error + + connInfo *pgtype.ConnInfo + + wbuf []byte + preallocatedRows []connRows + eqb extendedQueryBuilder +} + +// Identifier a PostgreSQL identifier or name. Identifiers can be composed of +// multiple parts such as ["schema", "table"] or ["table", "column"]. +type Identifier []string + +// Sanitize returns a sanitized string safe for SQL interpolation. +func (ident Identifier) Sanitize() string { + parts := make([]string, len(ident)) + for i := range ident { + s := strings.Replace(ident[i], string([]byte{0}), "", -1) + parts[i] = `"` + strings.Replace(s, `"`, `""`, -1) + `"` + } + return strings.Join(parts, ".") +} + +// ErrNoRows occurs when rows are expected but none are returned. +var ErrNoRows = errors.New("no rows in result set") + +// ErrInvalidLogLevel occurs on attempt to set an invalid log level. +var ErrInvalidLogLevel = errors.New("invalid log level") + +// Connect establishes a connection with a PostgreSQL server with a connection string. See +// pgconn.Connect for details. +func Connect(ctx context.Context, connString string) (*Conn, error) { + connConfig, err := ParseConfig(connString) + if err != nil { + return nil, err + } + return connect(ctx, connConfig) +} + +// Connect establishes a connection with a PostgreSQL server with a configuration struct. connConfig must have been +// created by ParseConfig. +func ConnectConfig(ctx context.Context, connConfig *ConnConfig) (*Conn, error) { + return connect(ctx, connConfig) +} + +// ParseConfig creates a ConnConfig from a connection string. ParseConfig handles all options that pgconn.ParseConfig +// does. In addition, it accepts the following options: +// +// statement_cache_capacity +// The maximum size of the automatic statement cache. Set to 0 to disable automatic statement caching. Default: 512. +// +// statement_cache_mode +// Possible values: "prepare" and "describe". "prepare" will create prepared statements on the PostgreSQL server. +// "describe" will use the anonymous prepared statement to describe a statement without creating a statement on the +// server. "describe" is primarily useful when the environment does not allow prepared statements such as when +// running a connection pooler like PgBouncer. Default: "prepare" +func ParseConfig(connString string) (*ConnConfig, error) { + config, err := pgconn.ParseConfig(connString) + if err != nil { + return nil, err + } + + var buildStatementCache BuildStatementCacheFunc + statementCacheCapacity := 512 + statementCacheMode := stmtcache.ModePrepare + if s, ok := config.RuntimeParams["statement_cache_capacity"]; ok { + delete(config.RuntimeParams, "statement_cache_capacity") + n, err := strconv.ParseInt(s, 10, 32) + if err != nil { + return nil, errors.Errorf("cannot parse statement_cache_capacity: %w", err) + } + statementCacheCapacity = int(n) + } + + if s, ok := config.RuntimeParams["statement_cache_mode"]; ok { + delete(config.RuntimeParams, "statement_cache_mode") + switch s { + case "prepare": + statementCacheMode = stmtcache.ModePrepare + case "describe": + statementCacheMode = stmtcache.ModeDescribe + default: + return nil, errors.Errorf("invalid statement_cache_mod: %s", s) + } + } + + if statementCacheCapacity > 0 { + buildStatementCache = func(conn *pgconn.PgConn) stmtcache.Cache { + return stmtcache.New(conn, statementCacheMode, statementCacheCapacity) + } + } + + connConfig := &ConnConfig{ + Config: *config, + createdByParseConfig: true, + LogLevel: LogLevelInfo, + BuildStatementCache: buildStatementCache, + } + + return connConfig, nil +} + +func connect(ctx context.Context, config *ConnConfig) (c *Conn, err error) { + // Default values are set in ParseConfig. Enforce initial creation by ParseConfig rather than setting defaults from + // zero values. + if !config.createdByParseConfig { + panic("config must be created by ParseConfig") + } + + // This isn't really a deep copy. But it is enough to avoid the config.Config.OnNotification mutation from affecting + // other connections with the same config. See https://github.com/jackc/pgx/issues/618. + { + configCopy := *config + config = &configCopy + } + + c = &Conn{ + config: config, + connInfo: pgtype.NewConnInfo(), + logLevel: config.LogLevel, + logger: config.Logger, + } + + // Only install pgx notification system if no other callback handler is present. + if config.Config.OnNotification == nil { + config.Config.OnNotification = c.bufferNotifications + } else { + if c.shouldLog(LogLevelDebug) { + c.log(ctx, LogLevelDebug, "pgx notification handler disabled by application supplied OnNotification", map[string]interface{}{"host": config.Config.Host}) + } + } + + if c.shouldLog(LogLevelInfo) { + c.log(ctx, LogLevelInfo, "Dialing PostgreSQL server", map[string]interface{}{"host": config.Config.Host}) + } + c.pgConn, err = pgconn.ConnectConfig(ctx, &config.Config) + if err != nil { + if c.shouldLog(LogLevelError) { + c.log(ctx, LogLevelError, "connect failed", map[string]interface{}{"err": err}) + } + return nil, err + } + + c.preparedStatements = make(map[string]*pgconn.StatementDescription) + c.doneChan = make(chan struct{}) + c.closedChan = make(chan error) + c.wbuf = make([]byte, 0, 1024) + + if c.config.BuildStatementCache != nil { + c.stmtcache = c.config.BuildStatementCache(c.pgConn) + } + + // Replication connections can't execute the queries to + // populate the c.PgTypes and c.pgsqlAfInet + if _, ok := config.Config.RuntimeParams["replication"]; ok { + return c, nil + } + + return c, nil +} + +// Close closes a connection. It is safe to call Close on a already closed +// connection. +func (c *Conn) Close(ctx context.Context) error { + if c.IsClosed() { + return nil + } + + err := c.pgConn.Close(ctx) + if c.shouldLog(LogLevelInfo) { + c.log(ctx, LogLevelInfo, "closed connection", nil) + } + return err +} + +// Prepare creates a prepared statement with name and sql. sql can contain placeholders +// for bound parameters. These placeholders are referenced positional as $1, $2, etc. +// +// Prepare is idempotent; i.e. it is safe to call Prepare multiple times with the same +// name and sql arguments. This allows a code path to Prepare and Query/Exec without +// concern for if the statement has already been prepared. +func (c *Conn) Prepare(ctx context.Context, name, sql string) (sd *pgconn.StatementDescription, err error) { + if name != "" { + var ok bool + if sd, ok = c.preparedStatements[name]; ok && sd.SQL == sql { + return sd, nil + } + } + + if c.shouldLog(LogLevelError) { + defer func() { + if err != nil { + c.log(ctx, LogLevelError, "Prepare failed", map[string]interface{}{"err": err, "name": name, "sql": sql}) + } + }() + } + + sd, err = c.pgConn.Prepare(ctx, name, sql, nil) + if err != nil { + return nil, err + } + + if name != "" { + c.preparedStatements[name] = sd + } + + return sd, nil +} + +// Deallocate released a prepared statement +func (c *Conn) Deallocate(ctx context.Context, name string) error { + delete(c.preparedStatements, name) + _, err := c.pgConn.Exec(ctx, "deallocate "+quoteIdentifier(name)).ReadAll() + return err +} + +func (c *Conn) bufferNotifications(_ *pgconn.PgConn, n *pgconn.Notification) { + c.notifications = append(c.notifications, n) +} + +// WaitForNotification waits for a PostgreSQL notification. It wraps the underlying pgconn notification system in a +// slightly more convenient form. +func (c *Conn) WaitForNotification(ctx context.Context) (*pgconn.Notification, error) { + var n *pgconn.Notification + + // Return already received notification immediately + if len(c.notifications) > 0 { + n = c.notifications[0] + c.notifications = c.notifications[1:] + return n, nil + } + + err := c.pgConn.WaitForNotification(ctx) + if len(c.notifications) > 0 { + n = c.notifications[0] + c.notifications = c.notifications[1:] + } + return n, err +} + +func (c *Conn) IsClosed() bool { + return c.pgConn.IsClosed() +} + +// Processes messages that are not exclusive to one context such as +// authentication or query response. The response to these messages is the same +// regardless of when they occur. It also ignores messages that are only +// meaningful in a given context. These messages can occur due to a context +// deadline interrupting message processing. For example, an interrupted query +// may have left DataRow messages on the wire. +func (c *Conn) processContextFreeMsg(msg pgproto3.BackendMessage) (err error) { + switch msg := msg.(type) { + case *pgproto3.ErrorResponse: + return c.rxErrorResponse(msg) + } + + return nil +} + +func (c *Conn) rxErrorResponse(msg *pgproto3.ErrorResponse) *pgconn.PgError { + err := &pgconn.PgError{ + Severity: msg.Severity, + Code: msg.Code, + Message: msg.Message, + Detail: msg.Detail, + Hint: msg.Hint, + Position: msg.Position, + InternalPosition: msg.InternalPosition, + InternalQuery: msg.InternalQuery, + Where: msg.Where, + SchemaName: msg.SchemaName, + TableName: msg.TableName, + ColumnName: msg.ColumnName, + DataTypeName: msg.DataTypeName, + ConstraintName: msg.ConstraintName, + File: msg.File, + Line: msg.Line, + Routine: msg.Routine, + } + + if err.Severity == "FATAL" { + c.die(err) + } + + return err +} + +func (c *Conn) die(err error) { + if c.IsClosed() { + return + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // force immediate hard cancel + c.pgConn.Close(ctx) +} + +func (c *Conn) shouldLog(lvl LogLevel) bool { + return c.logger != nil && c.logLevel >= lvl +} + +func (c *Conn) log(ctx context.Context, lvl LogLevel, msg string, data map[string]interface{}) { + if data == nil { + data = map[string]interface{}{} + } + if c.pgConn != nil && c.pgConn.PID() != 0 { + data["pid"] = c.pgConn.PID() + } + + c.logger.Log(ctx, lvl, msg, data) +} + +func quoteIdentifier(s string) string { + return `"` + strings.Replace(s, `"`, `""`, -1) + `"` +} + +func (c *Conn) Ping(ctx context.Context) error { + _, err := c.Exec(ctx, ";") + return err +} + +func connInfoFromRows(rows Rows, err error) (map[string]uint32, error) { + if err != nil { + return nil, err + } + defer rows.Close() + + nameOIDs := make(map[string]uint32, 256) + for rows.Next() { + var oid uint32 + var name pgtype.Text + if err = rows.Scan(&oid, &name); err != nil { + return nil, err + } + + nameOIDs[name.String] = oid + } + + if err = rows.Err(); err != nil { + return nil, err + } + + return nameOIDs, err +} + +// PgConn returns the underlying *pgconn.PgConn. This is an escape hatch method that allows lower level access to the +// PostgreSQL connection than pgx exposes. +// +// It is strongly recommended that the connection be idle (no in-progress queries) before the underlying *pgconn.PgConn +// is used and the connection must be returned to the same state before any *pgx.Conn methods are again used. +func (c *Conn) PgConn() *pgconn.PgConn { return c.pgConn } + +// StatementCache returns the statement cache used for this connection. +func (c *Conn) StatementCache() stmtcache.Cache { return c.stmtcache } + +// ConnInfo returns the connection info used for this connection. +func (c *Conn) ConnInfo() *pgtype.ConnInfo { return c.connInfo } + +// Exec executes sql. sql can be either a prepared statement name or an SQL string. arguments should be referenced +// positionally from the sql string as $1, $2, etc. +func (c *Conn) Exec(ctx context.Context, sql string, arguments ...interface{}) (pgconn.CommandTag, error) { + startTime := time.Now() + + commandTag, err := c.exec(ctx, sql, arguments...) + if err != nil { + if c.shouldLog(LogLevelError) { + c.log(ctx, LogLevelError, "Exec", map[string]interface{}{"sql": sql, "args": logQueryArgs(arguments), "err": err}) + } + return commandTag, err + } + + if c.shouldLog(LogLevelInfo) { + endTime := time.Now() + c.log(ctx, LogLevelInfo, "Exec", map[string]interface{}{"sql": sql, "args": logQueryArgs(arguments), "time": endTime.Sub(startTime), "commandTag": commandTag}) + } + + return commandTag, err +} + +func (c *Conn) exec(ctx context.Context, sql string, arguments ...interface{}) (commandTag pgconn.CommandTag, err error) { + simpleProtocol := c.config.PreferSimpleProtocol + +optionLoop: + for len(arguments) > 0 { + switch arg := arguments[0].(type) { + case QuerySimpleProtocol: + simpleProtocol = bool(arg) + arguments = arguments[1:] + default: + break optionLoop + } + } + + if simpleProtocol { + return c.execSimpleProtocol(ctx, sql, arguments) + } + + if sd, ok := c.preparedStatements[sql]; ok { + return c.execPrepared(ctx, sd, arguments) + } + + if len(arguments) == 0 { + return c.execSimpleProtocol(ctx, sql, arguments) + } + + if c.stmtcache != nil { + sd, err := c.stmtcache.Get(ctx, sql) + if err != nil { + return nil, err + } + + if c.stmtcache.Mode() == stmtcache.ModeDescribe { + return c.execParams(ctx, sd, arguments) + } + return c.execPrepared(ctx, sd, arguments) + } + + sd, err := c.Prepare(ctx, "", sql) + if err != nil { + return nil, err + } + return c.execPrepared(ctx, sd, arguments) +} + +func (c *Conn) execSimpleProtocol(ctx context.Context, sql string, arguments []interface{}) (commandTag pgconn.CommandTag, err error) { + if len(arguments) > 0 { + sql, err = c.sanitizeForSimpleQuery(sql, arguments...) + if err != nil { + return nil, err + } + } + + mrr := c.pgConn.Exec(ctx, sql) + for mrr.NextResult() { + commandTag, err = mrr.ResultReader().Close() + } + err = mrr.Close() + return commandTag, err +} + +func (c *Conn) execParamsAndPreparedPrefix(sd *pgconn.StatementDescription, arguments []interface{}) error { + if len(sd.ParamOIDs) != len(arguments) { + return errors.Errorf("expected %d arguments, got %d", len(sd.ParamOIDs), len(arguments)) + } + + c.eqb.Reset() + + args, err := convertDriverValuers(arguments) + if err != nil { + return err + } + + for i := range args { + err = c.eqb.AppendParam(c.connInfo, sd.ParamOIDs[i], args[i]) + if err != nil { + return err + } + } + + for i := range sd.Fields { + c.eqb.AppendResultFormat(c.ConnInfo().ResultFormatCodeForOID(sd.Fields[i].DataTypeOID)) + } + + return nil +} + +func (c *Conn) execParams(ctx context.Context, sd *pgconn.StatementDescription, arguments []interface{}) (pgconn.CommandTag, error) { + err := c.execParamsAndPreparedPrefix(sd, arguments) + if err != nil { + return nil, err + } + + result := c.pgConn.ExecParams(ctx, sd.SQL, c.eqb.paramValues, sd.ParamOIDs, c.eqb.paramFormats, c.eqb.resultFormats).Read() + return result.CommandTag, result.Err +} + +func (c *Conn) execPrepared(ctx context.Context, sd *pgconn.StatementDescription, arguments []interface{}) (pgconn.CommandTag, error) { + err := c.execParamsAndPreparedPrefix(sd, arguments) + if err != nil { + return nil, err + } + + result := c.pgConn.ExecPrepared(ctx, sd.Name, c.eqb.paramValues, c.eqb.paramFormats, c.eqb.resultFormats).Read() + return result.CommandTag, result.Err +} + +func (c *Conn) getRows(ctx context.Context, sql string, args []interface{}) *connRows { + if len(c.preallocatedRows) == 0 { + c.preallocatedRows = make([]connRows, 64) + } + + r := &c.preallocatedRows[len(c.preallocatedRows)-1] + c.preallocatedRows = c.preallocatedRows[0 : len(c.preallocatedRows)-1] + + r.ctx = ctx + r.logger = c + r.connInfo = c.connInfo + r.startTime = time.Now() + r.sql = sql + r.args = args + + return r +} + +// QuerySimpleProtocol controls whether the simple or extended protocol is used to send the query. +type QuerySimpleProtocol bool + +// QueryResultFormats controls the result format (text=0, binary=1) of a query by result column position. +type QueryResultFormats []int16 + +// QueryResultFormatsByOID controls the result format (text=0, binary=1) of a query by the result column OID. +type QueryResultFormatsByOID map[uint32]int16 + +// Query executes sql with args. If there is an error the returned Rows will be returned in an error state. So it is +// allowed to ignore the error returned from Query and handle it in Rows. +// +// For extra control over how the query is executed, the types QuerySimpleProtocol, QueryResultFormats, and +// QueryResultFormatsByOID may be used as the first args to control exactly how the query is executed. This is rarely +// needed. See the documentation for those types for details. +func (c *Conn) Query(ctx context.Context, sql string, args ...interface{}) (Rows, error) { + var resultFormats QueryResultFormats + var resultFormatsByOID QueryResultFormatsByOID + simpleProtocol := c.config.PreferSimpleProtocol + +optionLoop: + for len(args) > 0 { + switch arg := args[0].(type) { + case QueryResultFormats: + resultFormats = arg + args = args[1:] + case QueryResultFormatsByOID: + resultFormatsByOID = arg + args = args[1:] + case QuerySimpleProtocol: + simpleProtocol = bool(arg) + args = args[1:] + default: + break optionLoop + } + } + + rows := c.getRows(ctx, sql, args) + + var err error + if simpleProtocol { + sql, err = c.sanitizeForSimpleQuery(sql, args...) + if err != nil { + rows.fatal(err) + return rows, err + } + + mrr := c.pgConn.Exec(ctx, sql) + if mrr.NextResult() { + rows.resultReader = mrr.ResultReader() + rows.multiResultReader = mrr + } else { + err = mrr.Close() + rows.fatal(err) + return rows, err + } + + return rows, nil + } + + c.eqb.Reset() + + sd, ok := c.preparedStatements[sql] + if !ok { + if c.stmtcache != nil { + sd, err = c.stmtcache.Get(ctx, sql) + if err != nil { + rows.fatal(err) + return rows, rows.err + } + } else { + sd, err = c.pgConn.Prepare(ctx, "", sql, nil) + if err != nil { + rows.fatal(err) + return rows, rows.err + } + } + } + if len(sd.ParamOIDs) != len(args) { + rows.fatal(errors.Errorf("expected %d arguments, got %d", len(sd.ParamOIDs), len(args))) + return rows, rows.err + } + + rows.sql = sd.SQL + + args, err = convertDriverValuers(args) + if err != nil { + rows.fatal(err) + return rows, rows.err + } + + for i := range args { + err = c.eqb.AppendParam(c.connInfo, sd.ParamOIDs[i], args[i]) + if err != nil { + rows.fatal(err) + return rows, rows.err + } + } + + if resultFormatsByOID != nil { + resultFormats = make([]int16, len(sd.Fields)) + for i := range resultFormats { + resultFormats[i] = resultFormatsByOID[uint32(sd.Fields[i].DataTypeOID)] + } + } + + if resultFormats == nil { + for i := range sd.Fields { + c.eqb.AppendResultFormat(c.ConnInfo().ResultFormatCodeForOID(sd.Fields[i].DataTypeOID)) + } + + resultFormats = c.eqb.resultFormats + } + + if c.stmtcache != nil && c.stmtcache.Mode() == stmtcache.ModeDescribe { + rows.resultReader = c.pgConn.ExecParams(ctx, sql, c.eqb.paramValues, sd.ParamOIDs, c.eqb.paramFormats, resultFormats) + } else { + rows.resultReader = c.pgConn.ExecPrepared(ctx, sd.Name, c.eqb.paramValues, c.eqb.paramFormats, resultFormats) + } + + return rows, rows.err +} + +// QueryRow is a convenience wrapper over Query. Any error that occurs while +// querying is deferred until calling Scan on the returned Row. That Row will +// error with ErrNoRows if no rows are returned. +func (c *Conn) QueryRow(ctx context.Context, sql string, args ...interface{}) Row { + rows, _ := c.Query(ctx, sql, args...) + return (*connRow)(rows.(*connRows)) +} + +// SendBatch sends all queued queries to the server at once. All queries are run in an implicit transaction unless +// explicit transaction control statements are executed. The returned BatchResults must be closed before the connection +// is used again. +func (c *Conn) SendBatch(ctx context.Context, b *Batch) BatchResults { + distinctUnpreparedQueries := map[string]struct{}{} + + for _, bi := range b.items { + if _, ok := c.preparedStatements[bi.query]; ok { + continue + } + distinctUnpreparedQueries[bi.query] = struct{}{} + } + + var stmtCache stmtcache.Cache + if c.stmtcache != nil && c.stmtcache.Cap() >= len(distinctUnpreparedQueries) { + stmtCache = c.stmtcache + } else { + stmtCache = stmtcache.New(c.pgConn, stmtcache.ModeDescribe, len(distinctUnpreparedQueries)) + } + + for sql, _ := range distinctUnpreparedQueries { + _, err := stmtCache.Get(ctx, sql) + if err != nil { + return &batchResults{ctx: ctx, conn: c, err: err} + } + } + + batch := &pgconn.Batch{} + + for _, bi := range b.items { + c.eqb.Reset() + + sd := c.preparedStatements[bi.query] + if sd == nil { + var err error + sd, err = stmtCache.Get(ctx, bi.query) + if err != nil { + // the stmtCache was prefilled from distinctUnpreparedQueries above so we are guaranteed no errors + panic("BUG: unexpected error from stmtCache") + } + } + + if len(sd.ParamOIDs) != len(bi.arguments) { + return &batchResults{ctx: ctx, conn: c, err: errors.Errorf("mismatched param and argument count")} + } + + args, err := convertDriverValuers(bi.arguments) + if err != nil { + return &batchResults{ctx: ctx, conn: c, err: err} + } + + for i := range args { + err = c.eqb.AppendParam(c.connInfo, sd.ParamOIDs[i], args[i]) + if err != nil { + return &batchResults{ctx: ctx, conn: c, err: err} + } + } + + for i := range sd.Fields { + c.eqb.AppendResultFormat(c.ConnInfo().ResultFormatCodeForOID(sd.Fields[i].DataTypeOID)) + } + + if sd.Name == "" { + batch.ExecParams(bi.query, c.eqb.paramValues, sd.ParamOIDs, c.eqb.paramFormats, c.eqb.resultFormats) + } else { + batch.ExecPrepared(sd.Name, c.eqb.paramValues, c.eqb.paramFormats, c.eqb.resultFormats) + } + } + + mrr := c.pgConn.ExecBatch(ctx, batch) + + return &batchResults{ + ctx: ctx, + conn: c, + mrr: mrr, + b: b, + ix: 0, + } +} + +func (c *Conn) sanitizeForSimpleQuery(sql string, args ...interface{}) (string, error) { + if c.pgConn.ParameterStatus("standard_conforming_strings") != "on" { + return "", errors.New("simple protocol queries must be run with standard_conforming_strings=on") + } + + if c.pgConn.ParameterStatus("client_encoding") != "UTF8" { + return "", errors.New("simple protocol queries must be run with client_encoding=UTF8") + } + + var err error + valueArgs := make([]interface{}, len(args)) + for i, a := range args { + valueArgs[i], err = convertSimpleArgument(c.connInfo, a) + if err != nil { + return "", err + } + } + + return sanitize.SanitizeSQL(sql, valueArgs...) +} diff --git a/github.com/jackc/pgx/v4/copy_from.go b/github.com/jackc/pgx/v4/copy_from.go new file mode 100644 index 0000000000..afa80a1d14 --- /dev/null +++ b/github.com/jackc/pgx/v4/copy_from.go @@ -0,0 +1,169 @@ +package pgx + +import ( + "bytes" + "context" + "fmt" + "io" + + "github.com/jackc/pgconn" + "github.com/jackc/pgio" + errors "golang.org/x/xerrors" +) + +// CopyFromRows returns a CopyFromSource interface over the provided rows slice +// making it usable by *Conn.CopyFrom. +func CopyFromRows(rows [][]interface{}) CopyFromSource { + return ©FromRows{rows: rows, idx: -1} +} + +type copyFromRows struct { + rows [][]interface{} + idx int +} + +func (ctr *copyFromRows) Next() bool { + ctr.idx++ + return ctr.idx < len(ctr.rows) +} + +func (ctr *copyFromRows) Values() ([]interface{}, error) { + return ctr.rows[ctr.idx], nil +} + +func (ctr *copyFromRows) Err() error { + return nil +} + +// CopyFromSource is the interface used by *Conn.CopyFrom as the source for copy data. +type CopyFromSource interface { + // Next returns true if there is another row and makes the next row data + // available to Values(). When there are no more rows available or an error + // has occurred it returns false. + Next() bool + + // Values returns the values for the current row. + Values() ([]interface{}, error) + + // Err returns any error that has been encountered by the CopyFromSource. If + // this is not nil *Conn.CopyFrom will abort the copy. + Err() error +} + +type copyFrom struct { + conn *Conn + tableName Identifier + columnNames []string + rowSrc CopyFromSource + readerErrChan chan error +} + +func (ct *copyFrom) run(ctx context.Context) (int64, error) { + quotedTableName := ct.tableName.Sanitize() + cbuf := &bytes.Buffer{} + for i, cn := range ct.columnNames { + if i != 0 { + cbuf.WriteString(", ") + } + cbuf.WriteString(quoteIdentifier(cn)) + } + quotedColumnNames := cbuf.String() + + sd, err := ct.conn.Prepare(ctx, "", fmt.Sprintf("select %s from %s", quotedColumnNames, quotedTableName)) + if err != nil { + return 0, err + } + + r, w := io.Pipe() + doneChan := make(chan struct{}) + + go func() { + defer close(doneChan) + + // Purposely NOT using defer w.Close(). See https://github.com/golang/go/issues/24283. + buf := ct.conn.wbuf + + buf = append(buf, "PGCOPY\n\377\r\n\000"...) + buf = pgio.AppendInt32(buf, 0) + buf = pgio.AppendInt32(buf, 0) + + moreRows := true + for moreRows { + var err error + moreRows, buf, err = ct.buildCopyBuf(buf, sd) + if err != nil { + w.CloseWithError(err) + return + } + + if ct.rowSrc.Err() != nil { + w.CloseWithError(ct.rowSrc.Err()) + return + } + + if len(buf) > 0 { + _, err = w.Write(buf) + if err != nil { + w.Close() + return + } + } + + buf = buf[:0] + } + + w.Close() + }() + + commandTag, err := ct.conn.pgConn.CopyFrom(ctx, r, fmt.Sprintf("copy %s ( %s ) from stdin binary;", quotedTableName, quotedColumnNames)) + + r.Close() + <-doneChan + + return commandTag.RowsAffected(), err +} + +func (ct *copyFrom) buildCopyBuf(buf []byte, sd *pgconn.StatementDescription) (bool, []byte, error) { + + for ct.rowSrc.Next() { + values, err := ct.rowSrc.Values() + if err != nil { + return false, nil, err + } + if len(values) != len(ct.columnNames) { + return false, nil, errors.Errorf("expected %d values, got %d values", len(ct.columnNames), len(values)) + } + + buf = pgio.AppendInt16(buf, int16(len(ct.columnNames))) + for i, val := range values { + buf, err = encodePreparedStatementArgument(ct.conn.connInfo, buf, sd.Fields[i].DataTypeOID, val) + if err != nil { + return false, nil, err + } + } + + if len(buf) > 65536 { + return true, buf, nil + } + } + + return false, buf, nil +} + +// CopyFrom uses the PostgreSQL copy protocol to perform bulk data insertion. +// It returns the number of rows copied and an error. +// +// CopyFrom requires all values use the binary format. Almost all types +// implemented by pgx use the binary format by default. Types implementing +// Encoder can only be used if they encode to the binary format. +func (c *Conn) CopyFrom(ctx context.Context, tableName Identifier, columnNames []string, rowSrc CopyFromSource) (int64, error) { + ct := ©From{ + conn: c, + tableName: tableName, + columnNames: columnNames, + rowSrc: rowSrc, + readerErrChan: make(chan error), + } + + return ct.run(ctx) +} diff --git a/github.com/jackc/pgx/v4/doc.go b/github.com/jackc/pgx/v4/doc.go new file mode 100644 index 0000000000..53627999e9 --- /dev/null +++ b/github.com/jackc/pgx/v4/doc.go @@ -0,0 +1,250 @@ +// Package pgx is a PostgreSQL database driver. +/* +pgx provides lower level access to PostgreSQL than the standard database/sql. It remains as similar to the database/sql +interface as possible while providing better speed and access to PostgreSQL specific features. Import +github.com/jackc/pgx/v4/stdlib to use pgx as a database/sql compatible driver. + +Establishing a Connection + +The primary way of establishing a connection is with `pgx.Connect`. + + conn, err := pgx.Connect(context.Background(), os.Getenv("DATABASE_URL")) + +The database connection string can be in URL or DSN format. Both PostgreSQL settings and pgx settings can be specified +here. In addition, a config struct can be created by `ParseConfig` and modified before establishing the connection with +`ConnectConfig`. + + config, err := pgx.ParseConfig(os.Getenv("DATABASE_URL")) + if err != nil { + // ... + } + config.Logger = log15adapter.NewLogger(log.New("module", "pgx")) + + conn, err := pgx.ConnectConfig(context.Background(), config) + +Query Interface + +pgx implements Query and Scan in the familiar database/sql style. + + var sum int32 + + // Send the query to the server. The returned rows MUST be closed + // before conn can be used again. + rows, err := conn.Query(context.Background(), "select generate_series(1,$1)", 10) + if err != nil { + return err + } + + // rows.Close is called by rows.Next when all rows are read + // or an error occurs in Next or Scan. So it may optionally be + // omitted if nothing in the rows.Next loop can panic. It is + // safe to close rows multiple times. + defer rows.Close() + + // Iterate through the result set + for rows.Next() { + var n int32 + err = rows.Scan(&n) + if err != nil { + return err + } + sum += n + } + + // Any errors encountered by rows.Next or rows.Scan will be returned here + if rows.Err() != nil { + return err + } + + // No errors found - do something with sum + +pgx also implements QueryRow in the same style as database/sql. + + var name string + var weight int64 + err := conn.QueryRow(context.Background(), "select name, weight from widgets where id=$1", 42).Scan(&name, &weight) + if err != nil { + return err + } + +Use Exec to execute a query that does not return a result set. + + commandTag, err := conn.Exec(context.Background(), "delete from widgets where id=$1", 42) + if err != nil { + return err + } + if commandTag.RowsAffected() != 1 { + return errors.New("No row found to delete") + } + +Connection Pool + +See sub-package pgxpool for a connection pool. + +Base Type Mapping + +pgx maps between all common base types directly between Go and PostgreSQL. In particular: + + Go PostgreSQL + ----------------------- + string varchar + text + + // Integers are automatically be converted to any other integer type if + // it can be done without overflow or underflow. + int8 + int16 smallint + int32 int + int64 bigint + int + uint8 + uint16 + uint32 + uint64 + uint + + // Floats are strict and do not automatically convert like integers. + float32 float4 + float64 float8 + + time.Time date + timestamp + timestamptz + + []byte bytea + + +Null Mapping + +pgx can map nulls in two ways. The first is package pgtype provides types that have a data field and a status field. +They work in a similar fashion to database/sql. The second is to use a pointer to a pointer. + + var foo pgtype.Varchar + var bar *string + err := conn.QueryRow("select foo, bar from widgets where id=$1", 42).Scan(&foo, &bar) + if err != nil { + return err + } + +Array Mapping + +pgx maps between int16, int32, int64, float32, float64, and string Go slices and the equivalent PostgreSQL array type. +Go slices of native types do not support nulls, so if a PostgreSQL array that contains a null is read into a native Go +slice an error will occur. The pgtype package includes many more array types for PostgreSQL types that do not directly +map to native Go types. + +JSON and JSONB Mapping + +pgx includes built-in support to marshal and unmarshal between Go types and the PostgreSQL JSON and JSONB. + +Inet and CIDR Mapping + +pgx encodes from net.IPNet to and from inet and cidr PostgreSQL types. In addition, as a convenience pgx will encode +from a net.IP; it will assume a /32 netmask for IPv4 and a /128 for IPv6. + +Custom Type Support + +pgx includes support for the common data types like integers, floats, strings, dates, and times that have direct +mappings between Go and SQL. In addition, pgx uses the github.com/jackc/pgx/pgtype library to support more types. See +documention for that library for instructions on how to implement custom types. + +See example_custom_type_test.go for an example of a custom type for the PostgreSQL point type. + +pgx also includes support for custom types implementing the database/sql.Scanner and database/sql/driver.Valuer +interfaces. + +If pgx does cannot natively encode a type and that type is a renamed type (e.g. type MyTime time.Time) pgx will attempt +to encode the underlying type. While this is usually desired behavior it can produce surprising behavior if one the +underlying type and the renamed type each implement database/sql interfaces and the other implements pgx interfaces. It +is recommended that this situation be avoided by implementing pgx interfaces on the renamed type. + +Raw Bytes Mapping + +[]byte passed as arguments to Query, QueryRow, and Exec are passed unmodified to PostgreSQL. + +Transactions + +Transactions are started by calling Begin. + + tx, err := conn.Begin(context.Background()) + if err != nil { + return err + } + // Rollback is safe to call even if the tx is already closed, so if + // the tx commits successfully, this is a no-op + defer tx.Rollback(context.Background()) + + _, err = tx.Exec(context.Background(), "insert into foo(id) values (1)") + if err != nil { + return err + } + + err = tx.Commit(context.Background()) + if err != nil { + return err + } + +The Tx returned from Begin also implements the Begin method. This can be used to implement pseudo nested transactions. +These are internally implemented with savepoints. + +Use BeginTx to control the transaction mode. + +Prepared Statements + +Prepared statements can be manually created with the Prepare method. However, this is rarely necessary because pgx +includes an automatic statement cache by default. Queries run through the normal Query, QueryRow, and Exec functions are +automatically prepared on first execution and the prepared statement is reused on subsequent executions. See ParseConfig +for information on how to customize or disable the statement cache. + +Copy Protocol + +Use CopyFrom to efficiently insert multiple rows at a time using the PostgreSQL copy protocol. CopyFrom accepts a +CopyFromSource interface. If the data is already in a [][]interface{} use CopyFromRows to wrap it in a CopyFromSource +interface. Or implement CopyFromSource to avoid buffering the entire data set in memory. + + rows := [][]interface{}{ + {"John", "Smith", int32(36)}, + {"Jane", "Doe", int32(29)}, + } + + copyCount, err := conn.CopyFrom( + context.Background(), + pgx.Identifier{"people"}, + []string{"first_name", "last_name", "age"}, + pgx.CopyFromRows(rows), + ) + +CopyFrom can be faster than an insert with as few as 5 rows. + +Listen and Notify + +pgx can listen to the PostgreSQL notification system with the `Conn.WaitForNotification` method. It blocks until a +context is received or the context is canceled. + + _, err := conn.Exec(context.Background(), "listen channelname") + if err != nil { + return nil + } + + if notification, err := conn.WaitForNotification(context.Background()); err != nil { + // do something with notification + } + + +Logging + +pgx defines a simple logger interface. Connections optionally accept a logger that satisfies this interface. Set +LogLevel to control logging verbosity. Adapters for github.com/inconshreveable/log15, github.com/sirupsen/logrus, +go.uber.org/zap, github.com/rs/zerolog, and the testing log are provided in the log directory. + +Lower Level PostgreSQL Functionality + +pgx is implemented on top of github.com/jackc/pgconn a lower level PostgreSQL driver. The Conn.PgConn() method can be +used to access this lower layer. + +PgBouncer + +pgx is compatible with PgBouncer in two modes. One is when the connection has a statement cache in "describe" mode. The +other is when the connection is using the simple protocol. This can be set with the PreferSimpleProtocol config option. +*/ +package pgx diff --git a/github.com/jackc/pgx/v4/extended_query_builder.go b/github.com/jackc/pgx/v4/extended_query_builder.go new file mode 100644 index 0000000000..1d024911e3 --- /dev/null +++ b/github.com/jackc/pgx/v4/extended_query_builder.go @@ -0,0 +1,141 @@ +package pgx + +import ( + "database/sql/driver" + "fmt" + "reflect" + + "github.com/jackc/pgtype" +) + +type extendedQueryBuilder struct { + paramValues [][]byte + paramValueBytes []byte + paramFormats []int16 + resultFormats []int16 + + resetCount int +} + +func (eqb *extendedQueryBuilder) AppendParam(ci *pgtype.ConnInfo, oid uint32, arg interface{}) error { + f := chooseParameterFormatCode(ci, oid, arg) + eqb.paramFormats = append(eqb.paramFormats, f) + + v, err := eqb.encodeExtendedParamValue(ci, oid, arg) + if err != nil { + return err + } + eqb.paramValues = append(eqb.paramValues, v) + + return nil +} + +func (eqb *extendedQueryBuilder) AppendResultFormat(f int16) { + eqb.resultFormats = append(eqb.resultFormats, f) +} + +func (eqb *extendedQueryBuilder) Reset() { + eqb.paramValues = eqb.paramValues[0:0] + eqb.paramValueBytes = eqb.paramValueBytes[0:0] + eqb.paramFormats = eqb.paramFormats[0:0] + eqb.resultFormats = eqb.resultFormats[0:0] + + eqb.resetCount += 1 + + // Every so often shrink our reserved memory if it is abnormally high + if eqb.resetCount%128 == 0 { + if cap(eqb.paramValues) > 64 { + eqb.paramValues = make([][]byte, 0, cap(eqb.paramValues)/2) + } + + if cap(eqb.paramValueBytes) > 256 { + eqb.paramValueBytes = make([]byte, 0, cap(eqb.paramValueBytes)/2) + } + + if cap(eqb.paramFormats) > 64 { + eqb.paramFormats = make([]int16, 0, cap(eqb.paramFormats)/2) + } + if cap(eqb.resultFormats) > 64 { + eqb.resultFormats = make([]int16, 0, cap(eqb.resultFormats)/2) + } + } + +} + +func (eqb *extendedQueryBuilder) encodeExtendedParamValue(ci *pgtype.ConnInfo, oid uint32, arg interface{}) ([]byte, error) { + if arg == nil { + return nil, nil + } + + refVal := reflect.ValueOf(arg) + argIsPtr := refVal.Kind() == reflect.Ptr + + if argIsPtr && refVal.IsNil() { + return nil, nil + } + + if eqb.paramValueBytes == nil { + eqb.paramValueBytes = make([]byte, 0, 128) + } + + var err error + var buf []byte + pos := len(eqb.paramValueBytes) + + switch arg := arg.(type) { + case pgtype.BinaryEncoder: + buf, err = arg.EncodeBinary(ci, eqb.paramValueBytes) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + eqb.paramValueBytes = buf + return eqb.paramValueBytes[pos:], nil + case pgtype.TextEncoder: + buf, err = arg.EncodeText(ci, eqb.paramValueBytes) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + eqb.paramValueBytes = buf + return eqb.paramValueBytes[pos:], nil + case string: + return []byte(arg), nil + } + + if argIsPtr { + // We have already checked that arg is not pointing to nil, + // so it is safe to dereference here. + arg = refVal.Elem().Interface() + return eqb.encodeExtendedParamValue(ci, oid, arg) + } + + if dt, ok := ci.DataTypeForOID(oid); ok { + value := dt.Value + err := value.Set(arg) + if err != nil { + { + if arg, ok := arg.(driver.Valuer); ok { + v, err := callValuerValue(arg) + if err != nil { + return nil, err + } + return eqb.encodeExtendedParamValue(ci, oid, v) + } + } + + return nil, err + } + + return eqb.encodeExtendedParamValue(ci, oid, value) + } + + if strippedArg, ok := stripNamedType(&refVal); ok { + return eqb.encodeExtendedParamValue(ci, oid, strippedArg) + } + return nil, SerializationError(fmt.Sprintf("Cannot encode %T into oid %v - %T must implement Encoder or be converted to a string", arg, oid, arg)) +} diff --git a/github.com/jackc/pgx/v4/go.mod b/github.com/jackc/pgx/v4/go.mod new file mode 100644 index 0000000000..698cb61498 --- /dev/null +++ b/github.com/jackc/pgx/v4/go.mod @@ -0,0 +1,23 @@ +module github.com/jackc/pgx/v4 + +go 1.12 + +require ( + github.com/cockroachdb/apd v1.1.0 + github.com/gofrs/uuid v3.2.0+incompatible + github.com/jackc/pgconn v1.5.0 + github.com/jackc/pgio v1.0.0 + github.com/jackc/pgproto3/v2 v2.0.1 + github.com/jackc/pgtype v1.3.0 + github.com/jackc/puddle v1.1.0 + github.com/mattn/go-colorable v0.1.2 // indirect + github.com/mattn/go-isatty v0.0.9 // indirect + github.com/rs/zerolog v1.15.0 + github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 + github.com/sirupsen/logrus v1.4.2 + github.com/stretchr/testify v1.5.1 + go.uber.org/zap v1.10.0 + golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 // indirect + golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 + gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec +) diff --git a/github.com/jackc/pgx/v4/go.sum b/github.com/jackc/pgx/v4/go.sum new file mode 100644 index 0000000000..a82ba1127e --- /dev/null +++ b/github.com/jackc/pgx/v4/go.sum @@ -0,0 +1,172 @@ +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0 h1:DUwgMQuuPnS0rhMXenUtZpqZqrR/30NWY+qQvTpSvEs= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3 h1:ZFYpB74Kq8xE9gmfxCmXD6QxZ27ja+j3HwGFc+YurhQ= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb h1:d6GP9szHvXVopAOAnZ7WhRnF3Xdxrylmm/9jnfmW4Ag= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.5.0 h1:oFSOilzIZkyg787M1fEmyMfOUUvwj0daqYMfaWwNL4o= +github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2 h1:JVX6jT/XfzNqIjye4717ITLaNwV9mWbJx0dLCpcRzdA= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db h1:UpaKn/gYxzH6/zWyRQH1S260zvKqwJJ4h8+Kf09ooh0= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711 h1:vZp4bYotXUkFx7JUSm7U8KV/7Q0AOdrQxxBBj0ZmZsg= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29 h1:f2HwOeI1NIJyNFVVeh1gUISyt57iw/fmI/IXJfH3ATE= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.1 h1:Rdjp4NFjwHnEslx2b66FfCI2S0LhO4itac3hXz6WX9M= +github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8 h1:Q3tB+ExeflWUW7AFcAhXqk40s9mnNYLk1nOkKNZ5GnU= +github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0 h1:mX93v750WifMD1htCt7vqeolcnpaG1gz8URVGjSzcUM= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90 h1:aN5Vlwa2Q3QvxHDtZNi1x+GYkQyketBadMjtiug7AbM= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59 h1:xOamcCJ9MFJTxR5bvw3ZXmiP8evQMohdt2VJ57C0W8Q= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.3.0 h1:l8JvKrby3RI7Kg3bYEeU9TA4vqC38QDpFCfcrC7KuN0= +github.com/jackc/pgtype v1.3.0/go.mod h1:b0JqxHvPmljG+HQ5IsvQ0yqeSi4nGcDTVjFoiLDb0Ik= +github.com/jackc/pgx v3.6.2+incompatible h1:2zP5OD7kiyR3xzRYMhOcXVvkDZsImVXfj+yIyTQf3/o= +github.com/jackc/pgx v3.6.2+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b h1:cIcUpcEP55F/QuZWEtXyqHoWk+IV4TBiLjtBkeq/Q1c= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9 h1:KLBBPU++1T3DHtm1B1QaIHy80Vhu0wNMErIFCNgAL8Y= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.0 h1:musOWczZC/rSbqut475Vfcczg7jJsdUQf0D6oKPLgNU= +github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0 h1:/5u4a+KGJptBRqGzPvYQL9p0d/tPR4S31+Tnzj9lEO4= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0 h1:hSNcYHyxDWycfePW7pUI8swuFkcSMPKh3E63Pokg1Hk= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0 h1:uPRuwkWF4J6fGsJ2R0Gn2jB1EQiav9k3S6CSdygQJXY= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a h1:Igim7XhdOpBnWPuYJ70XcNpq8q3BCACtVgNfoJxOV7g= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 h1:7KByu05hhLed2MO29w7p1XfZvZ13m8mub3shuVftRs0= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 h1:3zb4D3T4G8jdExgVU/95+vQXfpEPiMdCaZgmGVxjNHM= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e h1:nFYrTHrdrAOpShe27kaFHjsqYSEQ0KWqdWLu3xuZJts= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373 h1:PPwnA7z1Pjf7XYaBP9GL1VAMZmcIWyFz7QCMSIIa3Bg= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522 h1:bhOzK9QyoD0ogCnFro1m2mz41+Ib0oOhfJnBp5MR4K4= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec h1:RlWgLqCMMIYYEVcAR5MDsuHlVkaIPDAF+5Dehzg8L5A= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/github.com/jackc/pgx/v4/go_stdlib.go b/github.com/jackc/pgx/v4/go_stdlib.go new file mode 100644 index 0000000000..9372f9efab --- /dev/null +++ b/github.com/jackc/pgx/v4/go_stdlib.go @@ -0,0 +1,61 @@ +package pgx + +import ( + "database/sql/driver" + "reflect" +) + +// This file contains code copied from the Go standard library due to the +// required function not being public. + +// Copyright (c) 2009 The Go Authors. All rights reserved. + +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: + +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. + +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// From database/sql/convert.go + +var valuerReflectType = reflect.TypeOf((*driver.Valuer)(nil)).Elem() + +// callValuerValue returns vr.Value(), with one exception: +// If vr.Value is an auto-generated method on a pointer type and the +// pointer is nil, it would panic at runtime in the panicwrap +// method. Treat it like nil instead. +// Issue 8415. +// +// This is so people can implement driver.Value on value types and +// still use nil pointers to those types to mean nil/NULL, just like +// string/*string. +// +// This function is mirrored in the database/sql/driver package. +func callValuerValue(vr driver.Valuer) (v driver.Value, err error) { + if rv := reflect.ValueOf(vr); rv.Kind() == reflect.Ptr && + rv.IsNil() && + rv.Type().Elem().Implements(valuerReflectType) { + return nil, nil + } + return vr.Value() +} diff --git a/github.com/jackc/pgx/v4/internal/sanitize/sanitize.go b/github.com/jackc/pgx/v4/internal/sanitize/sanitize.go new file mode 100644 index 0000000000..0e7ddf1326 --- /dev/null +++ b/github.com/jackc/pgx/v4/internal/sanitize/sanitize.go @@ -0,0 +1,237 @@ +package sanitize + +import ( + "bytes" + "encoding/hex" + "strconv" + "strings" + "time" + "unicode/utf8" + + errors "golang.org/x/xerrors" +) + +// Part is either a string or an int. A string is raw SQL. An int is a +// argument placeholder. +type Part interface{} + +type Query struct { + Parts []Part +} + +func (q *Query) Sanitize(args ...interface{}) (string, error) { + argUse := make([]bool, len(args)) + buf := &bytes.Buffer{} + + for _, part := range q.Parts { + var str string + switch part := part.(type) { + case string: + str = part + case int: + argIdx := part - 1 + if argIdx >= len(args) { + return "", errors.Errorf("insufficient arguments") + } + arg := args[argIdx] + switch arg := arg.(type) { + case nil: + str = "null" + case int64: + str = strconv.FormatInt(arg, 10) + case float64: + str = strconv.FormatFloat(arg, 'f', -1, 64) + case bool: + str = strconv.FormatBool(arg) + case []byte: + str = QuoteBytes(arg) + case string: + str = QuoteString(arg) + case time.Time: + str = arg.Truncate(time.Microsecond).Format("'2006-01-02 15:04:05.999999999Z07:00:00'") + default: + return "", errors.Errorf("invalid arg type: %T", arg) + } + argUse[argIdx] = true + default: + return "", errors.Errorf("invalid Part type: %T", part) + } + buf.WriteString(str) + } + + for i, used := range argUse { + if !used { + return "", errors.Errorf("unused argument: %d", i) + } + } + return buf.String(), nil +} + +func NewQuery(sql string) (*Query, error) { + l := &sqlLexer{ + src: sql, + stateFn: rawState, + } + + for l.stateFn != nil { + l.stateFn = l.stateFn(l) + } + + query := &Query{Parts: l.parts} + + return query, nil +} + +func QuoteString(str string) string { + return "'" + strings.Replace(str, "'", "''", -1) + "'" +} + +func QuoteBytes(buf []byte) string { + return `'\x` + hex.EncodeToString(buf) + "'" +} + +type sqlLexer struct { + src string + start int + pos int + stateFn stateFn + parts []Part +} + +type stateFn func(*sqlLexer) stateFn + +func rawState(l *sqlLexer) stateFn { + for { + r, width := utf8.DecodeRuneInString(l.src[l.pos:]) + l.pos += width + + switch r { + case 'e', 'E': + nextRune, width := utf8.DecodeRuneInString(l.src[l.pos:]) + if nextRune == '\'' { + l.pos += width + return escapeStringState + } + case '\'': + return singleQuoteState + case '"': + return doubleQuoteState + case '$': + nextRune, _ := utf8.DecodeRuneInString(l.src[l.pos:]) + if '0' <= nextRune && nextRune <= '9' { + if l.pos-l.start > 0 { + l.parts = append(l.parts, l.src[l.start:l.pos-width]) + } + l.start = l.pos + return placeholderState + } + case utf8.RuneError: + if l.pos-l.start > 0 { + l.parts = append(l.parts, l.src[l.start:l.pos]) + l.start = l.pos + } + return nil + } + } +} + +func singleQuoteState(l *sqlLexer) stateFn { + for { + r, width := utf8.DecodeRuneInString(l.src[l.pos:]) + l.pos += width + + switch r { + case '\'': + nextRune, width := utf8.DecodeRuneInString(l.src[l.pos:]) + if nextRune != '\'' { + return rawState + } + l.pos += width + case utf8.RuneError: + if l.pos-l.start > 0 { + l.parts = append(l.parts, l.src[l.start:l.pos]) + l.start = l.pos + } + return nil + } + } +} + +func doubleQuoteState(l *sqlLexer) stateFn { + for { + r, width := utf8.DecodeRuneInString(l.src[l.pos:]) + l.pos += width + + switch r { + case '"': + nextRune, width := utf8.DecodeRuneInString(l.src[l.pos:]) + if nextRune != '"' { + return rawState + } + l.pos += width + case utf8.RuneError: + if l.pos-l.start > 0 { + l.parts = append(l.parts, l.src[l.start:l.pos]) + l.start = l.pos + } + return nil + } + } +} + +// placeholderState consumes a placeholder value. The $ must have already has +// already been consumed. The first rune must be a digit. +func placeholderState(l *sqlLexer) stateFn { + num := 0 + + for { + r, width := utf8.DecodeRuneInString(l.src[l.pos:]) + l.pos += width + + if '0' <= r && r <= '9' { + num *= 10 + num += int(r - '0') + } else { + l.parts = append(l.parts, num) + l.pos -= width + l.start = l.pos + return rawState + } + } +} + +func escapeStringState(l *sqlLexer) stateFn { + for { + r, width := utf8.DecodeRuneInString(l.src[l.pos:]) + l.pos += width + + switch r { + case '\\': + _, width = utf8.DecodeRuneInString(l.src[l.pos:]) + l.pos += width + case '\'': + nextRune, width := utf8.DecodeRuneInString(l.src[l.pos:]) + if nextRune != '\'' { + return rawState + } + l.pos += width + case utf8.RuneError: + if l.pos-l.start > 0 { + l.parts = append(l.parts, l.src[l.start:l.pos]) + l.start = l.pos + } + return nil + } + } +} + +// SanitizeSQL replaces placeholder values with args. It quotes and escapes args +// as necessary. This function is only safe when standard_conforming_strings is +// on. +func SanitizeSQL(sql string, args ...interface{}) (string, error) { + query, err := NewQuery(sql) + if err != nil { + return "", err + } + return query.Sanitize(args...) +} diff --git a/github.com/jackc/pgx/v4/large_objects.go b/github.com/jackc/pgx/v4/large_objects.go new file mode 100644 index 0000000000..4fed79018e --- /dev/null +++ b/github.com/jackc/pgx/v4/large_objects.go @@ -0,0 +1,122 @@ +package pgx + +import ( + "context" + "io" + + errors "golang.org/x/xerrors" +) + +// LargeObjects is a structure used to access the large objects API. It is only valid within the transaction where it +// was created. +// +// For more details see: http://www.postgresql.org/docs/current/static/largeobjects.html +type LargeObjects struct { + tx Tx +} + +type LargeObjectMode int32 + +const ( + LargeObjectModeWrite LargeObjectMode = 0x20000 + LargeObjectModeRead LargeObjectMode = 0x40000 +) + +// Create creates a new large object. If oid is zero, the server assigns an unused OID. +func (o *LargeObjects) Create(ctx context.Context, oid uint32) (uint32, error) { + err := o.tx.QueryRow(ctx, "select lo_create($1)", oid).Scan(&oid) + return oid, err +} + +// Open opens an existing large object with the given mode. ctx will also be used for all operations on the opened large +// object. +func (o *LargeObjects) Open(ctx context.Context, oid uint32, mode LargeObjectMode) (*LargeObject, error) { + var fd int32 + err := o.tx.QueryRow(ctx, "select lo_open($1, $2)", oid, mode).Scan(&fd) + if err != nil { + return nil, err + } + return &LargeObject{fd: fd, tx: o.tx, ctx: ctx}, nil +} + +// Unlink removes a large object from the database. +func (o *LargeObjects) Unlink(ctx context.Context, oid uint32) error { + var result int32 + err := o.tx.QueryRow(ctx, "select lo_unlink($1)", oid).Scan(&result) + if err != nil { + return err + } + + if result != 1 { + return errors.New("failed to remove large object") + } + + return nil +} + +// A LargeObject is a large object stored on the server. It is only valid within the transaction that it was initialized +// in. It uses the context it was initialized with for all operations. It implements these interfaces: +// +// io.Writer +// io.Reader +// io.Seeker +// io.Closer +type LargeObject struct { + ctx context.Context + tx Tx + fd int32 +} + +// Write writes p to the large object and returns the number of bytes written and an error if not all of p was written. +func (o *LargeObject) Write(p []byte) (int, error) { + var n int + err := o.tx.QueryRow(o.ctx, "select lowrite($1, $2)", o.fd, p).Scan(&n) + if err != nil { + return n, err + } + + if n < 0 { + return 0, errors.New("failed to write to large object") + } + + return n, nil +} + +// Read reads up to len(p) bytes into p returning the number of bytes read. +func (o *LargeObject) Read(p []byte) (int, error) { + var res []byte + err := o.tx.QueryRow(o.ctx, "select loread($1, $2)", o.fd, len(p)).Scan(&res) + copy(p, res) + if err != nil { + return len(res), err + } + + if len(res) < len(p) { + err = io.EOF + } + return len(res), err +} + +// Seek moves the current location pointer to the new location specified by offset. +func (o *LargeObject) Seek(offset int64, whence int) (n int64, err error) { + err = o.tx.QueryRow(o.ctx, "select lo_lseek64($1, $2, $3)", o.fd, offset, whence).Scan(&n) + return n, err +} + +// Tell returns the current read or write location of the large object descriptor. +func (o *LargeObject) Tell() (n int64, err error) { + err = o.tx.QueryRow(o.ctx, "select lo_tell64($1)", o.fd).Scan(&n) + return n, err +} + +// Trunctes the large object to size. +func (o *LargeObject) Truncate(size int64) (err error) { + _, err = o.tx.Exec(o.ctx, "select lo_truncate64($1, $2)", o.fd, size) + return err +} + +// Close closees the large object descriptor. +func (o *LargeObject) Close() error { + _, err := o.tx.Exec(o.ctx, "select lo_close($1)", o.fd) + return err +} diff --git a/github.com/jackc/pgx/v4/logger.go b/github.com/jackc/pgx/v4/logger.go new file mode 100644 index 0000000000..f69a152f84 --- /dev/null +++ b/github.com/jackc/pgx/v4/logger.go @@ -0,0 +1,99 @@ +package pgx + +import ( + "context" + "encoding/hex" + "fmt" + + errors "golang.org/x/xerrors" +) + +// The values for log levels are chosen such that the zero value means that no +// log level was specified. +const ( + LogLevelTrace = 6 + LogLevelDebug = 5 + LogLevelInfo = 4 + LogLevelWarn = 3 + LogLevelError = 2 + LogLevelNone = 1 +) + +// LogLevel represents the pgx logging level. See LogLevel* constants for +// possible values. +type LogLevel int + +func (ll LogLevel) String() string { + switch ll { + case LogLevelTrace: + return "trace" + case LogLevelDebug: + return "debug" + case LogLevelInfo: + return "info" + case LogLevelWarn: + return "warn" + case LogLevelError: + return "error" + case LogLevelNone: + return "none" + default: + return fmt.Sprintf("invalid level %d", ll) + } +} + +// Logger is the interface used to get logging from pgx internals. +type Logger interface { + // Log a message at the given level with data key/value pairs. data may be nil. + Log(ctx context.Context, level LogLevel, msg string, data map[string]interface{}) +} + +// LogLevelFromString converts log level string to constant +// +// Valid levels: +// trace +// debug +// info +// warn +// error +// none +func LogLevelFromString(s string) (LogLevel, error) { + switch s { + case "trace": + return LogLevelTrace, nil + case "debug": + return LogLevelDebug, nil + case "info": + return LogLevelInfo, nil + case "warn": + return LogLevelWarn, nil + case "error": + return LogLevelError, nil + case "none": + return LogLevelNone, nil + default: + return 0, errors.New("invalid log level") + } +} + +func logQueryArgs(args []interface{}) []interface{} { + logArgs := make([]interface{}, 0, len(args)) + + for _, a := range args { + switch v := a.(type) { + case []byte: + if len(v) < 64 { + a = hex.EncodeToString(v) + } else { + a = fmt.Sprintf("%x (truncated %d bytes)", v[:64], len(v)-64) + } + case string: + if len(v) > 64 { + a = fmt.Sprintf("%s (truncated %d bytes)", v[:64], len(v)-64) + } + } + logArgs = append(logArgs, a) + } + + return logArgs +} diff --git a/github.com/jackc/pgx/v4/messages.go b/github.com/jackc/pgx/v4/messages.go new file mode 100644 index 0000000000..8e00f65f50 --- /dev/null +++ b/github.com/jackc/pgx/v4/messages.go @@ -0,0 +1,42 @@ +package pgx + +import ( + "database/sql/driver" + + "github.com/jackc/pgio" + "github.com/jackc/pgtype" +) + +const ( + copyData = 'd' + copyFail = 'f' + copyDone = 'c' +) + +func convertDriverValuers(args []interface{}) ([]interface{}, error) { + for i, arg := range args { + switch arg := arg.(type) { + case pgtype.BinaryEncoder: + case pgtype.TextEncoder: + case driver.Valuer: + v, err := callValuerValue(arg) + if err != nil { + return nil, err + } + args[i] = v + } + } + return args, nil +} + +// appendQuery appends a PostgreSQL wire protocol query message to buf and returns it. +func appendQuery(buf []byte, query string) []byte { + buf = append(buf, 'Q') + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + buf = append(buf, query...) + buf = append(buf, 0) + pgio.SetInt32(buf[sp:], int32(len(buf[sp:]))) + + return buf +} diff --git a/github.com/jackc/pgx/v4/rows.go b/github.com/jackc/pgx/v4/rows.go new file mode 100644 index 0000000000..c16a64bd63 --- /dev/null +++ b/github.com/jackc/pgx/v4/rows.go @@ -0,0 +1,308 @@ +package pgx + +import ( + "context" + "fmt" + "reflect" + "time" + + errors "golang.org/x/xerrors" + + "github.com/jackc/pgconn" + "github.com/jackc/pgproto3/v2" + "github.com/jackc/pgtype" +) + +// Rows is the result set returned from *Conn.Query. Rows must be closed before +// the *Conn can be used again. Rows are closed by explicitly calling Close(), +// calling Next() until it returns false, or when a fatal error occurs. +// +// Rows is an interface instead of a struct to allow tests to mock Query. However, +// adding a method to an interface is technically a breaking change. Because of this +// the Rows interface is partially excluded from semantic version requirements. +// Methods will not be removed or changed, but new methods may be added. +type Rows interface { + // Close closes the rows, making the connection ready for use again. It is safe + // to call Close after rows is already closed. + Close() + + // Err returns any error that occurred while reading. + Err() error + + // CommandTag returns the command tag from this query. It is only available after Rows is closed. + CommandTag() pgconn.CommandTag + + FieldDescriptions() []pgproto3.FieldDescription + + // Next prepares the next row for reading. It returns true if there is another + // row and false if no more rows are available. It automatically closes rows + // when all rows are read. + Next() bool + + // Scan reads the values from the current row into dest values positionally. + // dest can include pointers to core types, values implementing the Scanner + // interface, []byte, and nil. []byte will skip the decoding process and directly + // copy the raw bytes received from PostgreSQL. nil will skip the value entirely. + Scan(dest ...interface{}) error + + // Values returns the decoded row values. + Values() ([]interface{}, error) + + // RawValues returns the unparsed bytes of the row values. The returned [][]byte is only valid until the next Next + // call or the Rows is closed. However, the underlying byte data is safe to retain a reference to and mutate. + RawValues() [][]byte +} + +// Row is a convenience wrapper over Rows that is returned by QueryRow. +// +// Row is an interface instead of a struct to allow tests to mock QueryRow. However, +// adding a method to an interface is technically a breaking change. Because of this +// the Row interface is partially excluded from semantic version requirements. +// Methods will not be removed or changed, but new methods may be added. +type Row interface { + // Scan works the same as Rows. with the following exceptions. If no + // rows were found it returns ErrNoRows. If multiple rows are returned it + // ignores all but the first. + Scan(dest ...interface{}) error +} + +// connRow implements the Row interface for Conn.QueryRow. +type connRow connRows + +func (r *connRow) Scan(dest ...interface{}) (err error) { + rows := (*connRows)(r) + + if rows.Err() != nil { + return rows.Err() + } + + if !rows.Next() { + if rows.Err() == nil { + return ErrNoRows + } + return rows.Err() + } + + rows.Scan(dest...) + rows.Close() + return rows.Err() +} + +type rowLog interface { + shouldLog(lvl LogLevel) bool + log(ctx context.Context, lvl LogLevel, msg string, data map[string]interface{}) +} + +// connRows implements the Rows interface for Conn.Query. +type connRows struct { + ctx context.Context + logger rowLog + connInfo *pgtype.ConnInfo + values [][]byte + rowCount int + err error + commandTag pgconn.CommandTag + startTime time.Time + sql string + args []interface{} + closed bool + + resultReader *pgconn.ResultReader + multiResultReader *pgconn.MultiResultReader +} + +func (rows *connRows) FieldDescriptions() []pgproto3.FieldDescription { + return rows.resultReader.FieldDescriptions() +} + +func (rows *connRows) Close() { + if rows.closed { + return + } + + rows.closed = true + + if rows.resultReader != nil { + var closeErr error + rows.commandTag, closeErr = rows.resultReader.Close() + if rows.err == nil { + rows.err = closeErr + } + } + + if rows.multiResultReader != nil { + closeErr := rows.multiResultReader.Close() + if rows.err == nil { + rows.err = closeErr + } + } + + if rows.logger != nil { + if rows.err == nil { + if rows.logger.shouldLog(LogLevelInfo) { + endTime := time.Now() + rows.logger.log(rows.ctx, LogLevelInfo, "Query", map[string]interface{}{"sql": rows.sql, "args": logQueryArgs(rows.args), "time": endTime.Sub(rows.startTime), "rowCount": rows.rowCount}) + } + } else if rows.logger.shouldLog(LogLevelError) { + rows.logger.log(rows.ctx, LogLevelError, "Query", map[string]interface{}{"sql": rows.sql, "args": logQueryArgs(rows.args)}) + } + } +} + +func (rows *connRows) CommandTag() pgconn.CommandTag { + return rows.commandTag +} + +func (rows *connRows) Err() error { + return rows.err +} + +// fatal signals an error occurred after the query was sent to the server. It +// closes the rows automatically. +func (rows *connRows) fatal(err error) { + if rows.err != nil { + return + } + + rows.err = err + rows.Close() +} + +func (rows *connRows) Next() bool { + if rows.closed { + return false + } + + if rows.resultReader.NextRow() { + rows.rowCount++ + rows.values = rows.resultReader.Values() + return true + } else { + rows.Close() + return false + } +} + +func (rows *connRows) Scan(dest ...interface{}) error { + err := ScanRow(rows.connInfo, rows.FieldDescriptions(), rows.values, dest...) + if err != nil { + rows.fatal(err) + return err + } + + return nil +} + +func (rows *connRows) Values() ([]interface{}, error) { + if rows.closed { + return nil, errors.New("rows is closed") + } + + values := make([]interface{}, 0, len(rows.FieldDescriptions())) + + for i := range rows.FieldDescriptions() { + buf := rows.values[i] + fd := &rows.FieldDescriptions()[i] + + if buf == nil { + values = append(values, nil) + continue + } + + if dt, ok := rows.connInfo.DataTypeForOID(fd.DataTypeOID); ok { + value := reflect.New(reflect.ValueOf(dt.Value).Elem().Type()).Interface().(pgtype.Value) + + switch fd.Format { + case TextFormatCode: + decoder, ok := value.(pgtype.TextDecoder) + if !ok { + decoder = &pgtype.GenericText{} + } + err := decoder.DecodeText(rows.connInfo, buf) + if err != nil { + rows.fatal(err) + } + values = append(values, decoder.(pgtype.Value).Get()) + case BinaryFormatCode: + decoder, ok := value.(pgtype.BinaryDecoder) + if !ok { + decoder = &pgtype.GenericBinary{} + } + err := decoder.DecodeBinary(rows.connInfo, buf) + if err != nil { + rows.fatal(err) + } + values = append(values, value.Get()) + default: + rows.fatal(errors.New("Unknown format code")) + } + } else { + switch fd.Format { + case TextFormatCode: + decoder := &pgtype.GenericText{} + err := decoder.DecodeText(rows.connInfo, buf) + if err != nil { + rows.fatal(err) + } + values = append(values, decoder.Get()) + case BinaryFormatCode: + decoder := &pgtype.GenericBinary{} + err := decoder.DecodeBinary(rows.connInfo, buf) + if err != nil { + rows.fatal(err) + } + values = append(values, decoder.Get()) + default: + rows.fatal(errors.New("Unknown format code")) + } + } + + if rows.Err() != nil { + return nil, rows.Err() + } + } + + return values, rows.Err() +} + +func (rows *connRows) RawValues() [][]byte { + return rows.values +} + +type scanArgError struct { + col int + err error +} + +func (e scanArgError) Error() string { + return fmt.Sprintf("can't scan into dest[%d]: %v", e.col, e.err) +} + +// ScanRow decodes raw row data into dest. This is a low level function used internally to to implement the Rows +// interface Scan method. It can be used to scan rows read from the lower level pgconn interface. +// +// connInfo - OID to Go type mapping. +// fieldDescriptions - OID and format of values +// values - the raw data as returned from the PostgreSQL server +// dest - the destination that values will be decoded into +func ScanRow(connInfo *pgtype.ConnInfo, fieldDescriptions []pgproto3.FieldDescription, values [][]byte, dest ...interface{}) error { + if len(fieldDescriptions) != len(values) { + return errors.Errorf("number of field descriptions must equal number of values, got %d and %d", len(fieldDescriptions), len(values)) + } + if len(fieldDescriptions) != len(dest) { + return errors.Errorf("number of field descriptions must equal number of destinations, got %d and %d", len(fieldDescriptions), len(dest)) + } + + for i, d := range dest { + if d == nil { + continue + } + + err := connInfo.Scan(fieldDescriptions[i].DataTypeOID, fieldDescriptions[i].Format, values[i], d) + if err != nil { + return scanArgError{col: i, err: err} + } + } + + return nil +} diff --git a/github.com/jackc/pgx/v4/tx.go b/github.com/jackc/pgx/v4/tx.go new file mode 100644 index 0000000000..65b1d862c4 --- /dev/null +++ b/github.com/jackc/pgx/v4/tx.go @@ -0,0 +1,350 @@ +package pgx + +import ( + "bytes" + "context" + "fmt" + "strconv" + + "github.com/jackc/pgconn" + errors "golang.org/x/xerrors" +) + +type TxIsoLevel string + +// Transaction isolation levels +const ( + Serializable = TxIsoLevel("serializable") + RepeatableRead = TxIsoLevel("repeatable read") + ReadCommitted = TxIsoLevel("read committed") + ReadUncommitted = TxIsoLevel("read uncommitted") +) + +type TxAccessMode string + +// Transaction access modes +const ( + ReadWrite = TxAccessMode("read write") + ReadOnly = TxAccessMode("read only") +) + +type TxDeferrableMode string + +// Transaction deferrable modes +const ( + Deferrable = TxDeferrableMode("deferrable") + NotDeferrable = TxDeferrableMode("not deferrable") +) + +type TxOptions struct { + IsoLevel TxIsoLevel + AccessMode TxAccessMode + DeferrableMode TxDeferrableMode +} + +func (txOptions TxOptions) beginSQL() string { + buf := &bytes.Buffer{} + buf.WriteString("begin") + if txOptions.IsoLevel != "" { + fmt.Fprintf(buf, " isolation level %s", txOptions.IsoLevel) + } + if txOptions.AccessMode != "" { + fmt.Fprintf(buf, " %s", txOptions.AccessMode) + } + if txOptions.DeferrableMode != "" { + fmt.Fprintf(buf, " %s", txOptions.DeferrableMode) + } + + return buf.String() +} + +var ErrTxClosed = errors.New("tx is closed") + +// ErrTxCommitRollback occurs when an error has occurred in a transaction and +// Commit() is called. PostgreSQL accepts COMMIT on aborted transactions, but +// it is treated as ROLLBACK. +var ErrTxCommitRollback = errors.New("commit unexpectedly resulted in rollback") + +// Begin starts a transaction. Unlike database/sql, the context only affects the begin command. i.e. there is no +// auto-rollback on context cancellation. +func (c *Conn) Begin(ctx context.Context) (Tx, error) { + return c.BeginTx(ctx, TxOptions{}) +} + +// BeginTx starts a transaction with txOptions determining the transaction mode. Unlike database/sql, the context only +// affects the begin command. i.e. there is no auto-rollback on context cancellation. +func (c *Conn) BeginTx(ctx context.Context, txOptions TxOptions) (Tx, error) { + _, err := c.Exec(ctx, txOptions.beginSQL()) + if err != nil { + // begin should never fail unless there is an underlying connection issue or + // a context timeout. In either case, the connection is possibly broken. + c.die(errors.New("failed to begin transaction")) + return nil, err + } + + return &dbTx{conn: c}, nil +} + +// Tx represents a database transaction. +// +// Tx is an interface instead of a struct to enable connection pools to be implemented without relying on internal pgx +// state, to support pseudo-nested transactions with savepoints, and to allow tests to mock transactions. However, +// adding a method to an interface is technically a breaking change. If new methods are added to Conn it may be +// desirable to add them to Tx as well. Because of this the Tx interface is partially excluded from semantic version +// requirements. Methods will not be removed or changed, but new methods may be added. +type Tx interface { + // Begin starts a pseudo nested transaction. + Begin(ctx context.Context) (Tx, error) + + // Commit commits the transaction if this is a real transaction or releases the savepoint if this is a pseudo nested + // transaction. Commit will return ErrTxClosed if the Tx is already closed, but is otherwise safe to call multiple + // times. If the commit fails with a rollback status (e.g. a deferred constraint was violated) then + // ErrTxCommitRollback will be returned. Any other failure of a real transaction will result in the connection being + // closed. + Commit(ctx context.Context) error + + // Rollback rolls back the transaction if this is a real transaction or rolls back to the savepoint if this is a + // pseudo nested transaction. Rollback will return ErrTxClosed if the Tx is already closed, but is otherwise safe to + // call multiple times. Hence, a defer tx.Rollback() is safe even if tx.Commit() will be called first in a non-error + // condition. Any other failure of a real transaction will result in the connection being closed. + Rollback(ctx context.Context) error + + CopyFrom(ctx context.Context, tableName Identifier, columnNames []string, rowSrc CopyFromSource) (int64, error) + SendBatch(ctx context.Context, b *Batch) BatchResults + LargeObjects() LargeObjects + + Prepare(ctx context.Context, name, sql string) (*pgconn.StatementDescription, error) + + Exec(ctx context.Context, sql string, arguments ...interface{}) (commandTag pgconn.CommandTag, err error) + Query(ctx context.Context, sql string, args ...interface{}) (Rows, error) + QueryRow(ctx context.Context, sql string, args ...interface{}) Row + + // Conn returns the underlying *Conn that on which this transaction is executing. + Conn() *Conn +} + +// dbTx represents a database transaction. +// +// All dbTx methods return ErrTxClosed if Commit or Rollback has already been +// called on the dbTx. +type dbTx struct { + conn *Conn + err error + savepointNum int64 + closed bool +} + +// Begin starts a pseudo nested transaction implemented with a savepoint. +func (tx *dbTx) Begin(ctx context.Context) (Tx, error) { + if tx.closed { + return nil, ErrTxClosed + } + + tx.savepointNum += 1 + _, err := tx.conn.Exec(ctx, "savepoint sp_"+strconv.FormatInt(tx.savepointNum, 10)) + if err != nil { + return nil, err + } + + return &dbSavepoint{tx: tx, savepointNum: tx.savepointNum}, nil +} + +// Commit commits the transaction. +func (tx *dbTx) Commit(ctx context.Context) error { + if tx.closed { + return ErrTxClosed + } + + commandTag, err := tx.conn.Exec(ctx, "commit") + tx.closed = true + if err != nil { + // A commit failure leaves the connection in an undefined state so kill the connection (though any error that could + // cause this to fail should have already killed the connection) + tx.conn.die(errors.Errorf("commit failed: %w", err)) + return err + } + if string(commandTag) == "ROLLBACK" { + return ErrTxCommitRollback + } + + return nil +} + +// Rollback rolls back the transaction. Rollback will return ErrTxClosed if the +// Tx is already closed, but is otherwise safe to call multiple times. Hence, a +// defer tx.Rollback() is safe even if tx.Commit() will be called first in a +// non-error condition. +func (tx *dbTx) Rollback(ctx context.Context) error { + if tx.closed { + return ErrTxClosed + } + + _, err := tx.conn.Exec(ctx, "rollback") + tx.closed = true + if err != nil { + // A rollback failure leaves the connection in an undefined state + tx.conn.die(errors.Errorf("rollback failed: %w", err)) + return err + } + + return nil +} + +// Exec delegates to the underlying *Conn +func (tx *dbTx) Exec(ctx context.Context, sql string, arguments ...interface{}) (commandTag pgconn.CommandTag, err error) { + return tx.conn.Exec(ctx, sql, arguments...) +} + +// Prepare delegates to the underlying *Conn +func (tx *dbTx) Prepare(ctx context.Context, name, sql string) (*pgconn.StatementDescription, error) { + if tx.closed { + return nil, ErrTxClosed + } + + return tx.conn.Prepare(ctx, name, sql) +} + +// Query delegates to the underlying *Conn +func (tx *dbTx) Query(ctx context.Context, sql string, args ...interface{}) (Rows, error) { + if tx.closed { + // Because checking for errors can be deferred to the *Rows, build one with the error + err := ErrTxClosed + return &connRows{closed: true, err: err}, err + } + + return tx.conn.Query(ctx, sql, args...) +} + +// QueryRow delegates to the underlying *Conn +func (tx *dbTx) QueryRow(ctx context.Context, sql string, args ...interface{}) Row { + rows, _ := tx.Query(ctx, sql, args...) + return (*connRow)(rows.(*connRows)) +} + +// CopyFrom delegates to the underlying *Conn +func (tx *dbTx) CopyFrom(ctx context.Context, tableName Identifier, columnNames []string, rowSrc CopyFromSource) (int64, error) { + if tx.closed { + return 0, ErrTxClosed + } + + return tx.conn.CopyFrom(ctx, tableName, columnNames, rowSrc) +} + +// SendBatch delegates to the underlying *Conn +func (tx *dbTx) SendBatch(ctx context.Context, b *Batch) BatchResults { + if tx.closed { + return &batchResults{err: ErrTxClosed} + } + + return tx.conn.SendBatch(ctx, b) +} + +// LargeObjects returns a LargeObjects instance for the transaction. +func (tx *dbTx) LargeObjects() LargeObjects { + return LargeObjects{tx: tx} +} + +func (tx *dbTx) Conn() *Conn { + return tx.conn +} + +// dbSavepoint represents a nested transaction implemented by a savepoint. +type dbSavepoint struct { + tx Tx + savepointNum int64 + closed bool +} + +// Begin starts a pseudo nested transaction implemented with a savepoint. +func (sp *dbSavepoint) Begin(ctx context.Context) (Tx, error) { + if sp.closed { + return nil, ErrTxClosed + } + + return sp.tx.Begin(ctx) +} + +// Commit releases the savepoint essentially committing the pseudo nested transaction. +func (sp *dbSavepoint) Commit(ctx context.Context) error { + if sp.closed { + return ErrTxClosed + } + + _, err := sp.Exec(ctx, "release savepoint sp_"+strconv.FormatInt(sp.savepointNum, 10)) + sp.closed = true + return err +} + +// Rollback rolls back to the savepoint essentially rolling back the pseudo nested transaction. Rollback will return +// ErrTxClosed if the dbSavepoint is already closed, but is otherwise safe to call multiple times. Hence, a defer sp.Rollback() +// is safe even if sp.Commit() will be called first in a non-error condition. +func (sp *dbSavepoint) Rollback(ctx context.Context) error { + if sp.closed { + return ErrTxClosed + } + + _, err := sp.Exec(ctx, "rollback to savepoint sp_"+strconv.FormatInt(sp.savepointNum, 10)) + sp.closed = true + return err +} + +// Exec delegates to the underlying Tx +func (sp *dbSavepoint) Exec(ctx context.Context, sql string, arguments ...interface{}) (commandTag pgconn.CommandTag, err error) { + if sp.closed { + return nil, ErrTxClosed + } + + return sp.tx.Exec(ctx, sql, arguments...) +} + +// Prepare delegates to the underlying Tx +func (sp *dbSavepoint) Prepare(ctx context.Context, name, sql string) (*pgconn.StatementDescription, error) { + if sp.closed { + return nil, ErrTxClosed + } + + return sp.tx.Prepare(ctx, name, sql) +} + +// Query delegates to the underlying Tx +func (sp *dbSavepoint) Query(ctx context.Context, sql string, args ...interface{}) (Rows, error) { + if sp.closed { + // Because checking for errors can be deferred to the *Rows, build one with the error + err := ErrTxClosed + return &connRows{closed: true, err: err}, err + } + + return sp.tx.Query(ctx, sql, args...) +} + +// QueryRow delegates to the underlying Tx +func (sp *dbSavepoint) QueryRow(ctx context.Context, sql string, args ...interface{}) Row { + rows, _ := sp.Query(ctx, sql, args...) + return (*connRow)(rows.(*connRows)) +} + +// CopyFrom delegates to the underlying *Conn +func (sp *dbSavepoint) CopyFrom(ctx context.Context, tableName Identifier, columnNames []string, rowSrc CopyFromSource) (int64, error) { + if sp.closed { + return 0, ErrTxClosed + } + + return sp.tx.CopyFrom(ctx, tableName, columnNames, rowSrc) +} + +// SendBatch delegates to the underlying *Conn +func (sp *dbSavepoint) SendBatch(ctx context.Context, b *Batch) BatchResults { + if sp.closed { + return &batchResults{err: ErrTxClosed} + } + + return sp.tx.SendBatch(ctx, b) +} + +func (sp *dbSavepoint) LargeObjects() LargeObjects { + return LargeObjects{tx: sp} +} + +func (sp *dbSavepoint) Conn() *Conn { + return sp.tx.Conn() +} diff --git a/github.com/jackc/pgx/v4/values.go b/github.com/jackc/pgx/v4/values.go new file mode 100644 index 0000000000..da9bb70a87 --- /dev/null +++ b/github.com/jackc/pgx/v4/values.go @@ -0,0 +1,261 @@ +package pgx + +import ( + "database/sql/driver" + "fmt" + "math" + "reflect" + "time" + + "github.com/jackc/pgio" + "github.com/jackc/pgtype" + errors "golang.org/x/xerrors" +) + +// PostgreSQL format codes +const ( + TextFormatCode = 0 + BinaryFormatCode = 1 +) + +// SerializationError occurs on failure to encode or decode a value +type SerializationError string + +func (e SerializationError) Error() string { + return string(e) +} + +func convertSimpleArgument(ci *pgtype.ConnInfo, arg interface{}) (interface{}, error) { + if arg == nil { + return nil, nil + } + + switch arg := arg.(type) { + + // https://github.com/jackc/pgx/issues/409 Changed JSON and JSONB to surface + // []byte to database/sql instead of string. But that caused problems with the + // simple protocol because the driver.Valuer case got taken before the + // pgtype.TextEncoder case. And driver.Valuer needed to be first in the usual + // case because of https://github.com/jackc/pgx/issues/339. So instead we + // special case JSON and JSONB. + case *pgtype.JSON: + buf, err := arg.EncodeText(ci, nil) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + return string(buf), nil + case *pgtype.JSONB: + buf, err := arg.EncodeText(ci, nil) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + return string(buf), nil + + case driver.Valuer: + return callValuerValue(arg) + case pgtype.TextEncoder: + buf, err := arg.EncodeText(ci, nil) + if err != nil { + return nil, err + } + if buf == nil { + return nil, nil + } + return string(buf), nil + case int64: + return arg, nil + case float64: + return arg, nil + case bool: + return arg, nil + case time.Time: + return arg, nil + case string: + return arg, nil + case []byte: + return arg, nil + case int8: + return int64(arg), nil + case int16: + return int64(arg), nil + case int32: + return int64(arg), nil + case int: + return int64(arg), nil + case uint8: + return int64(arg), nil + case uint16: + return int64(arg), nil + case uint32: + return int64(arg), nil + case uint64: + if arg > math.MaxInt64 { + return nil, errors.Errorf("arg too big for int64: %v", arg) + } + return int64(arg), nil + case uint: + if uint64(arg) > math.MaxInt64 { + return nil, errors.Errorf("arg too big for int64: %v", arg) + } + return int64(arg), nil + case float32: + return float64(arg), nil + } + + refVal := reflect.ValueOf(arg) + + if refVal.Kind() == reflect.Ptr { + if refVal.IsNil() { + return nil, nil + } + arg = refVal.Elem().Interface() + return convertSimpleArgument(ci, arg) + } + + if strippedArg, ok := stripNamedType(&refVal); ok { + return convertSimpleArgument(ci, strippedArg) + } + return nil, SerializationError(fmt.Sprintf("Cannot encode %T in simple protocol - %T must implement driver.Valuer, pgtype.TextEncoder, or be a native type", arg, arg)) +} + +func encodePreparedStatementArgument(ci *pgtype.ConnInfo, buf []byte, oid uint32, arg interface{}) ([]byte, error) { + if arg == nil { + return pgio.AppendInt32(buf, -1), nil + } + + switch arg := arg.(type) { + case pgtype.BinaryEncoder: + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + argBuf, err := arg.EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if argBuf != nil { + buf = argBuf + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + return buf, nil + case pgtype.TextEncoder: + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + argBuf, err := arg.EncodeText(ci, buf) + if err != nil { + return nil, err + } + if argBuf != nil { + buf = argBuf + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + return buf, nil + case string: + buf = pgio.AppendInt32(buf, int32(len(arg))) + buf = append(buf, arg...) + return buf, nil + } + + refVal := reflect.ValueOf(arg) + + if refVal.Kind() == reflect.Ptr { + if refVal.IsNil() { + return pgio.AppendInt32(buf, -1), nil + } + arg = refVal.Elem().Interface() + return encodePreparedStatementArgument(ci, buf, oid, arg) + } + + if dt, ok := ci.DataTypeForOID(oid); ok { + value := dt.Value + err := value.Set(arg) + if err != nil { + { + if arg, ok := arg.(driver.Valuer); ok { + v, err := callValuerValue(arg) + if err != nil { + return nil, err + } + return encodePreparedStatementArgument(ci, buf, oid, v) + } + } + + return nil, err + } + + sp := len(buf) + buf = pgio.AppendInt32(buf, -1) + argBuf, err := value.(pgtype.BinaryEncoder).EncodeBinary(ci, buf) + if err != nil { + return nil, err + } + if argBuf != nil { + buf = argBuf + pgio.SetInt32(buf[sp:], int32(len(buf[sp:])-4)) + } + return buf, nil + } + + if strippedArg, ok := stripNamedType(&refVal); ok { + return encodePreparedStatementArgument(ci, buf, oid, strippedArg) + } + return nil, SerializationError(fmt.Sprintf("Cannot encode %T into oid %v - %T must implement Encoder or be converted to a string", arg, oid, arg)) +} + +// chooseParameterFormatCode determines the correct format code for an +// argument to a prepared statement. It defaults to TextFormatCode if no +// determination can be made. +func chooseParameterFormatCode(ci *pgtype.ConnInfo, oid uint32, arg interface{}) int16 { + switch arg.(type) { + case pgtype.BinaryEncoder: + return BinaryFormatCode + case string, *string, pgtype.TextEncoder: + return TextFormatCode + } + + return ci.ParamFormatCodeForOID(oid) +} + +func stripNamedType(val *reflect.Value) (interface{}, bool) { + switch val.Kind() { + case reflect.Int: + convVal := int(val.Int()) + return convVal, reflect.TypeOf(convVal) != val.Type() + case reflect.Int8: + convVal := int8(val.Int()) + return convVal, reflect.TypeOf(convVal) != val.Type() + case reflect.Int16: + convVal := int16(val.Int()) + return convVal, reflect.TypeOf(convVal) != val.Type() + case reflect.Int32: + convVal := int32(val.Int()) + return convVal, reflect.TypeOf(convVal) != val.Type() + case reflect.Int64: + convVal := int64(val.Int()) + return convVal, reflect.TypeOf(convVal) != val.Type() + case reflect.Uint: + convVal := uint(val.Uint()) + return convVal, reflect.TypeOf(convVal) != val.Type() + case reflect.Uint8: + convVal := uint8(val.Uint()) + return convVal, reflect.TypeOf(convVal) != val.Type() + case reflect.Uint16: + convVal := uint16(val.Uint()) + return convVal, reflect.TypeOf(convVal) != val.Type() + case reflect.Uint32: + convVal := uint32(val.Uint()) + return convVal, reflect.TypeOf(convVal) != val.Type() + case reflect.Uint64: + convVal := uint64(val.Uint()) + return convVal, reflect.TypeOf(convVal) != val.Type() + case reflect.String: + convVal := val.String() + return convVal, reflect.TypeOf(convVal) != val.Type() + } + + return nil, false +} diff --git a/github.com/mattn/go-isatty/.travis.yml b/github.com/mattn/go-isatty/.travis.yml index 5597e026dd..604314dd44 100644 --- a/github.com/mattn/go-isatty/.travis.yml +++ b/github.com/mattn/go-isatty/.travis.yml @@ -1,13 +1,14 @@ language: go +sudo: false go: + - 1.13.x - tip -os: - - linux - - osx - before_install: - - go get github.com/mattn/goveralls - - go get golang.org/x/tools/cmd/cover + - go get -t -v ./... + script: - - $HOME/gopath/bin/goveralls -repotoken 3gHdORO5k5ziZcWMBxnd9LrMZaJs8m9x5 + - ./go.test.sh + +after_success: + - bash <(curl -s https://codecov.io/bash) diff --git a/github.com/mattn/go-isatty/README.md b/github.com/mattn/go-isatty/README.md index 1e69004bb0..38418353e3 100644 --- a/github.com/mattn/go-isatty/README.md +++ b/github.com/mattn/go-isatty/README.md @@ -1,7 +1,7 @@ # go-isatty [![Godoc Reference](https://godoc.org/github.com/mattn/go-isatty?status.svg)](http://godoc.org/github.com/mattn/go-isatty) -[![Build Status](https://travis-ci.org/mattn/go-isatty.svg?branch=master)](https://travis-ci.org/mattn/go-isatty) +[![Codecov](https://codecov.io/gh/mattn/go-isatty/branch/master/graph/badge.svg)](https://codecov.io/gh/mattn/go-isatty) [![Coverage Status](https://coveralls.io/repos/github/mattn/go-isatty/badge.svg?branch=master)](https://coveralls.io/github/mattn/go-isatty?branch=master) [![Go Report Card](https://goreportcard.com/badge/mattn/go-isatty)](https://goreportcard.com/report/mattn/go-isatty) diff --git a/github.com/mattn/go-isatty/go.mod b/github.com/mattn/go-isatty/go.mod index 3b9b9abfb9..605c4c2210 100644 --- a/github.com/mattn/go-isatty/go.mod +++ b/github.com/mattn/go-isatty/go.mod @@ -1,3 +1,5 @@ module github.com/mattn/go-isatty -require golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a +go 1.12 + +require golang.org/x/sys v0.0.0-20200116001909-b77594299b42 diff --git a/github.com/mattn/go-isatty/go.sum b/github.com/mattn/go-isatty/go.sum index b1bd14d21d..912e29cbc1 100644 --- a/github.com/mattn/go-isatty/go.sum +++ b/github.com/mattn/go-isatty/go.sum @@ -1,2 +1,2 @@ -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/github.com/mattn/go-isatty/go.test.sh b/github.com/mattn/go-isatty/go.test.sh new file mode 100644 index 0000000000..012162b077 --- /dev/null +++ b/github.com/mattn/go-isatty/go.test.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -e +echo "" > coverage.txt + +for d in $(go list ./... | grep -v vendor); do + go test -race -coverprofile=profile.out -covermode=atomic "$d" + if [ -f profile.out ]; then + cat profile.out >> coverage.txt + rm profile.out + fi +done diff --git a/github.com/mattn/go-isatty/isatty_android.go b/github.com/mattn/go-isatty/isatty_android.go deleted file mode 100644 index d3567cb5bf..0000000000 --- a/github.com/mattn/go-isatty/isatty_android.go +++ /dev/null @@ -1,23 +0,0 @@ -// +build android - -package isatty - -import ( - "syscall" - "unsafe" -) - -const ioctlReadTermios = syscall.TCGETS - -// IsTerminal return true if the file descriptor is terminal. -func IsTerminal(fd uintptr) bool { - var termios syscall.Termios - _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, fd, ioctlReadTermios, uintptr(unsafe.Pointer(&termios)), 0, 0, 0) - return err == 0 -} - -// IsCygwinTerminal return true if the file descriptor is a cygwin or msys2 -// terminal. This is also always false on this environment. -func IsCygwinTerminal(fd uintptr) bool { - return false -} diff --git a/github.com/mattn/go-isatty/isatty_bsd.go b/github.com/mattn/go-isatty/isatty_bsd.go index 07e93039db..711f288085 100644 --- a/github.com/mattn/go-isatty/isatty_bsd.go +++ b/github.com/mattn/go-isatty/isatty_bsd.go @@ -3,18 +3,12 @@ package isatty -import ( - "syscall" - "unsafe" -) - -const ioctlReadTermios = syscall.TIOCGETA +import "golang.org/x/sys/unix" // IsTerminal return true if the file descriptor is terminal. func IsTerminal(fd uintptr) bool { - var termios syscall.Termios - _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, fd, ioctlReadTermios, uintptr(unsafe.Pointer(&termios)), 0, 0, 0) - return err == 0 + _, err := unix.IoctlGetTermios(int(fd), unix.TIOCGETA) + return err == nil } // IsCygwinTerminal return true if the file descriptor is a cygwin or msys2 diff --git a/github.com/mattn/go-isatty/isatty_plan9.go b/github.com/mattn/go-isatty/isatty_plan9.go new file mode 100644 index 0000000000..c5b6e0c084 --- /dev/null +++ b/github.com/mattn/go-isatty/isatty_plan9.go @@ -0,0 +1,22 @@ +// +build plan9 + +package isatty + +import ( + "syscall" +) + +// IsTerminal returns true if the given file descriptor is a terminal. +func IsTerminal(fd uintptr) bool { + path, err := syscall.Fd2path(int(fd)) + if err != nil { + return false + } + return path == "/dev/cons" || path == "/mnt/term/dev/cons" +} + +// IsCygwinTerminal return true if the file descriptor is a cygwin or msys2 +// terminal. This is also always false on this environment. +func IsCygwinTerminal(fd uintptr) bool { + return false +} diff --git a/github.com/mattn/go-isatty/isatty_tcgets.go b/github.com/mattn/go-isatty/isatty_tcgets.go index 453b025d0d..31a1ca973c 100644 --- a/github.com/mattn/go-isatty/isatty_tcgets.go +++ b/github.com/mattn/go-isatty/isatty_tcgets.go @@ -1,6 +1,5 @@ // +build linux aix // +build !appengine -// +build !android package isatty diff --git a/github.com/mattn/go-isatty/isatty_windows.go b/github.com/mattn/go-isatty/isatty_windows.go index af51cbcaa4..1fa8691540 100644 --- a/github.com/mattn/go-isatty/isatty_windows.go +++ b/github.com/mattn/go-isatty/isatty_windows.go @@ -4,6 +4,7 @@ package isatty import ( + "errors" "strings" "syscall" "unicode/utf16" @@ -11,15 +12,18 @@ import ( ) const ( - fileNameInfo uintptr = 2 - fileTypePipe = 3 + objectNameInfo uintptr = 1 + fileNameInfo = 2 + fileTypePipe = 3 ) var ( kernel32 = syscall.NewLazyDLL("kernel32.dll") + ntdll = syscall.NewLazyDLL("ntdll.dll") procGetConsoleMode = kernel32.NewProc("GetConsoleMode") procGetFileInformationByHandleEx = kernel32.NewProc("GetFileInformationByHandleEx") procGetFileType = kernel32.NewProc("GetFileType") + procNtQueryObject = ntdll.NewProc("NtQueryObject") ) func init() { @@ -45,7 +49,10 @@ func isCygwinPipeName(name string) bool { return false } - if token[0] != `\msys` && token[0] != `\cygwin` { + if token[0] != `\msys` && + token[0] != `\cygwin` && + token[0] != `\Device\NamedPipe\msys` && + token[0] != `\Device\NamedPipe\cygwin` { return false } @@ -68,11 +75,35 @@ func isCygwinPipeName(name string) bool { return true } +// getFileNameByHandle use the undocomented ntdll NtQueryObject to get file full name from file handler +// since GetFileInformationByHandleEx is not avilable under windows Vista and still some old fashion +// guys are using Windows XP, this is a workaround for those guys, it will also work on system from +// Windows vista to 10 +// see https://stackoverflow.com/a/18792477 for details +func getFileNameByHandle(fd uintptr) (string, error) { + if procNtQueryObject == nil { + return "", errors.New("ntdll.dll: NtQueryObject not supported") + } + + var buf [4 + syscall.MAX_PATH]uint16 + var result int + r, _, e := syscall.Syscall6(procNtQueryObject.Addr(), 5, + fd, objectNameInfo, uintptr(unsafe.Pointer(&buf)), uintptr(2*len(buf)), uintptr(unsafe.Pointer(&result)), 0) + if r != 0 { + return "", e + } + return string(utf16.Decode(buf[4 : 4+buf[0]/2])), nil +} + // IsCygwinTerminal() return true if the file descriptor is a cygwin or msys2 // terminal. func IsCygwinTerminal(fd uintptr) bool { if procGetFileInformationByHandleEx == nil { - return false + name, err := getFileNameByHandle(fd) + if err != nil { + return false + } + return isCygwinPipeName(name) } // Cygwin/msys's pty is a pipe. diff --git a/github.com/mattn/go-isatty/renovate.json b/github.com/mattn/go-isatty/renovate.json new file mode 100644 index 0000000000..5ae9d96b74 --- /dev/null +++ b/github.com/mattn/go-isatty/renovate.json @@ -0,0 +1,8 @@ +{ + "extends": [ + "config:base" + ], + "postUpdateOptions": [ + "gomodTidy" + ] +} diff --git a/golang.org/x/crypto/chacha20/chacha_generic.go b/golang.org/x/crypto/chacha20/chacha_generic.go index 7c498e90d9..a2ecf5c325 100644 --- a/golang.org/x/crypto/chacha20/chacha_generic.go +++ b/golang.org/x/crypto/chacha20/chacha_generic.go @@ -42,10 +42,14 @@ type Cipher struct { // The last len bytes of buf are leftover key stream bytes from the previous // XORKeyStream invocation. The size of buf depends on how many blocks are - // computed at a time. + // computed at a time by xorKeyStreamBlocks. buf [bufSize]byte len int + // overflow is set when the counter overflowed, no more blocks can be + // generated, and the next XORKeyStream call should panic. + overflow bool + // The counter-independent results of the first round are cached after they // are computed the first time. precompDone bool @@ -89,6 +93,7 @@ func newUnauthenticatedCipher(c *Cipher, key, nonce []byte) (*Cipher, error) { return nil, errors.New("chacha20: wrong nonce size") } + key, nonce = key[:KeySize], nonce[:NonceSize] // bounds check elimination hint c.key = [8]uint32{ binary.LittleEndian.Uint32(key[0:4]), binary.LittleEndian.Uint32(key[4:8]), @@ -139,15 +144,18 @@ func quarterRound(a, b, c, d uint32) (uint32, uint32, uint32, uint32) { // SetCounter sets the Cipher counter. The next invocation of XORKeyStream will // behave as if (64 * counter) bytes had been encrypted so far. // -// To prevent accidental counter reuse, SetCounter panics if counter is -// less than the current value. +// To prevent accidental counter reuse, SetCounter panics if counter is less +// than the current value. +// +// Note that the execution time of XORKeyStream is not independent of the +// counter value. func (s *Cipher) SetCounter(counter uint32) { // Internally, s may buffer multiple blocks, which complicates this // implementation slightly. When checking whether the counter has rolled // back, we must use both s.counter and s.len to determine how many blocks // we have already output. outputCounter := s.counter - uint32(s.len)/blockSize - if counter < outputCounter { + if s.overflow || counter < outputCounter { panic("chacha20: SetCounter attempted to rollback counter") } @@ -196,34 +204,52 @@ func (s *Cipher) XORKeyStream(dst, src []byte) { dst[i] = src[i] ^ b } s.len -= len(keyStream) - src = src[len(keyStream):] - dst = dst[len(keyStream):] + dst, src = dst[len(keyStream):], src[len(keyStream):] + } + if len(src) == 0 { + return } - const blocksPerBuf = bufSize / blockSize - numBufs := (uint64(len(src)) + bufSize - 1) / bufSize - if uint64(s.counter)+numBufs*blocksPerBuf >= 1<<32 { + // If we'd need to let the counter overflow and keep generating output, + // panic immediately. If instead we'd only reach the last block, remember + // not to generate any more output after the buffer is drained. + numBlocks := (uint64(len(src)) + blockSize - 1) / blockSize + if s.overflow || uint64(s.counter)+numBlocks > 1<<32 { panic("chacha20: counter overflow") + } else if uint64(s.counter)+numBlocks == 1<<32 { + s.overflow = true } // xorKeyStreamBlocks implementations expect input lengths that are a // multiple of bufSize. Platform-specific ones process multiple blocks at a // time, so have bufSizes that are a multiple of blockSize. - rem := len(src) % bufSize - full := len(src) - rem - + full := len(src) - len(src)%bufSize if full > 0 { s.xorKeyStreamBlocks(dst[:full], src[:full]) } + dst, src = dst[full:], src[full:] + + // If using a multi-block xorKeyStreamBlocks would overflow, use the generic + // one that does one block at a time. + const blocksPerBuf = bufSize / blockSize + if uint64(s.counter)+blocksPerBuf > 1<<32 { + s.buf = [bufSize]byte{} + numBlocks := (len(src) + blockSize - 1) / blockSize + buf := s.buf[bufSize-numBlocks*blockSize:] + copy(buf, src) + s.xorKeyStreamBlocksGeneric(buf, buf) + s.len = len(buf) - copy(dst, buf) + return + } // If we have a partial (multi-)block, pad it for xorKeyStreamBlocks, and // keep the leftover keystream for the next XORKeyStream invocation. - if rem > 0 { + if len(src) > 0 { s.buf = [bufSize]byte{} - copy(s.buf[:], src[full:]) + copy(s.buf[:], src) s.xorKeyStreamBlocks(s.buf[:], s.buf[:]) - s.len = bufSize - copy(dst[full:], s.buf[:]) + s.len = bufSize - copy(dst, s.buf[:]) } } @@ -260,7 +286,9 @@ func (s *Cipher) xorKeyStreamBlocksGeneric(dst, src []byte) { s.precompDone = true } - for i := 0; i < len(src); i += blockSize { + // A condition of len(src) > 0 would be sufficient, but this also + // acts as a bounds check elimination hint. + for len(src) >= 64 && len(dst) >= 64 { // The remainder of the first column round. fcr0, fcr4, fcr8, fcr12 := quarterRound(c0, c4, c8, s.counter) @@ -285,49 +313,28 @@ func (s *Cipher) xorKeyStreamBlocksGeneric(dst, src []byte) { x3, x4, x9, x14 = quarterRound(x3, x4, x9, x14) } - // Finally, add back the initial state to generate the key stream. - x0 += c0 - x1 += c1 - x2 += c2 - x3 += c3 - x4 += c4 - x5 += c5 - x6 += c6 - x7 += c7 - x8 += c8 - x9 += c9 - x10 += c10 - x11 += c11 - x12 += s.counter - x13 += c13 - x14 += c14 - x15 += c15 + // Add back the initial state to generate the key stream, then + // XOR the key stream with the source and write out the result. + addXor(dst[0:4], src[0:4], x0, c0) + addXor(dst[4:8], src[4:8], x1, c1) + addXor(dst[8:12], src[8:12], x2, c2) + addXor(dst[12:16], src[12:16], x3, c3) + addXor(dst[16:20], src[16:20], x4, c4) + addXor(dst[20:24], src[20:24], x5, c5) + addXor(dst[24:28], src[24:28], x6, c6) + addXor(dst[28:32], src[28:32], x7, c7) + addXor(dst[32:36], src[32:36], x8, c8) + addXor(dst[36:40], src[36:40], x9, c9) + addXor(dst[40:44], src[40:44], x10, c10) + addXor(dst[44:48], src[44:48], x11, c11) + addXor(dst[48:52], src[48:52], x12, s.counter) + addXor(dst[52:56], src[52:56], x13, c13) + addXor(dst[56:60], src[56:60], x14, c14) + addXor(dst[60:64], src[60:64], x15, c15) s.counter += 1 - if s.counter == 0 { - panic("chacha20: internal error: counter overflow") - } - in, out := src[i:], dst[i:] - in, out = in[:blockSize], out[:blockSize] // bounds check elimination hint - - // XOR the key stream with the source and write out the result. - xor(out[0:], in[0:], x0) - xor(out[4:], in[4:], x1) - xor(out[8:], in[8:], x2) - xor(out[12:], in[12:], x3) - xor(out[16:], in[16:], x4) - xor(out[20:], in[20:], x5) - xor(out[24:], in[24:], x6) - xor(out[28:], in[28:], x7) - xor(out[32:], in[32:], x8) - xor(out[36:], in[36:], x9) - xor(out[40:], in[40:], x10) - xor(out[44:], in[44:], x11) - xor(out[48:], in[48:], x12) - xor(out[52:], in[52:], x13) - xor(out[56:], in[56:], x14) - xor(out[60:], in[60:], x15) + src, dst = src[blockSize:], dst[blockSize:] } } diff --git a/golang.org/x/crypto/chacha20/xor.go b/golang.org/x/crypto/chacha20/xor.go index 0110c9865a..c2d04851e0 100644 --- a/golang.org/x/crypto/chacha20/xor.go +++ b/golang.org/x/crypto/chacha20/xor.go @@ -13,10 +13,10 @@ const unaligned = runtime.GOARCH == "386" || runtime.GOARCH == "ppc64le" || runtime.GOARCH == "s390x" -// xor reads a little endian uint32 from src, XORs it with u and +// addXor reads a little endian uint32 from src, XORs it with (a + b) and // places the result in little endian byte order in dst. -func xor(dst, src []byte, u uint32) { - _, _ = src[3], dst[3] // eliminate bounds checks +func addXor(dst, src []byte, a, b uint32) { + _, _ = src[3], dst[3] // bounds check elimination hint if unaligned { // The compiler should optimize this code into // 32-bit unaligned little endian loads and stores. @@ -27,15 +27,16 @@ func xor(dst, src []byte, u uint32) { v |= uint32(src[1]) << 8 v |= uint32(src[2]) << 16 v |= uint32(src[3]) << 24 - v ^= u + v ^= a + b dst[0] = byte(v) dst[1] = byte(v >> 8) dst[2] = byte(v >> 16) dst[3] = byte(v >> 24) } else { - dst[0] = src[0] ^ byte(u) - dst[1] = src[1] ^ byte(u>>8) - dst[2] = src[2] ^ byte(u>>16) - dst[3] = src[3] ^ byte(u>>24) + a += b + dst[0] = src[0] ^ byte(a) + dst[1] = src[1] ^ byte(a>>8) + dst[2] = src[2] ^ byte(a>>16) + dst[3] = src[3] ^ byte(a>>24) } } diff --git a/golang.org/x/crypto/poly1305/mac_noasm.go b/golang.org/x/crypto/poly1305/mac_noasm.go index b0c2cd0561..d118f30ed5 100644 --- a/golang.org/x/crypto/poly1305/mac_noasm.go +++ b/golang.org/x/crypto/poly1305/mac_noasm.go @@ -2,10 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build !amd64,!ppc64le gccgo purego +// +build !amd64,!ppc64le,!s390x gccgo purego package poly1305 type mac struct{ macGeneric } - -func newMAC(key *[32]byte) mac { return mac{newMACGeneric(key)} } diff --git a/golang.org/x/crypto/poly1305/poly1305.go b/golang.org/x/crypto/poly1305/poly1305.go index 066159b797..9d7a6af09f 100644 --- a/golang.org/x/crypto/poly1305/poly1305.go +++ b/golang.org/x/crypto/poly1305/poly1305.go @@ -26,7 +26,9 @@ const TagSize = 16 // 16-byte result into out. Authenticating two different messages with the same // key allows an attacker to forge messages at will. func Sum(out *[16]byte, m []byte, key *[32]byte) { - sum(out, m, key) + h := New(key) + h.Write(m) + h.Sum(out[:0]) } // Verify returns true if mac is a valid authenticator for m with the given key. @@ -46,10 +48,9 @@ func Verify(mac *[16]byte, m []byte, key *[32]byte) bool { // two different messages with the same key allows an attacker // to forge messages at will. func New(key *[32]byte) *MAC { - return &MAC{ - mac: newMAC(key), - finalized: false, - } + m := &MAC{} + initialize(key, &m.macState) + return m } // MAC is an io.Writer computing an authentication tag @@ -58,7 +59,7 @@ func New(key *[32]byte) *MAC { // MAC cannot be used like common hash.Hash implementations, // because using a poly1305 key twice breaks its security. // Therefore writing data to a running MAC after calling -// Sum causes it to panic. +// Sum or Verify causes it to panic. type MAC struct { mac // platform-dependent implementation @@ -71,10 +72,10 @@ func (h *MAC) Size() int { return TagSize } // Write adds more data to the running message authentication code. // It never returns an error. // -// It must not be called after the first call of Sum. +// It must not be called after the first call of Sum or Verify. func (h *MAC) Write(p []byte) (n int, err error) { if h.finalized { - panic("poly1305: write to MAC after Sum") + panic("poly1305: write to MAC after Sum or Verify") } return h.mac.Write(p) } @@ -87,3 +88,12 @@ func (h *MAC) Sum(b []byte) []byte { h.finalized = true return append(b, mac[:]...) } + +// Verify returns whether the authenticator of all data written to +// the message authentication code matches the expected value. +func (h *MAC) Verify(expected []byte) bool { + var mac [TagSize]byte + h.mac.Sum(&mac) + h.finalized = true + return subtle.ConstantTimeCompare(expected, mac[:]) == 1 +} diff --git a/golang.org/x/crypto/poly1305/sum_amd64.go b/golang.org/x/crypto/poly1305/sum_amd64.go index 35b9e38c90..99e5a1d50e 100644 --- a/golang.org/x/crypto/poly1305/sum_amd64.go +++ b/golang.org/x/crypto/poly1305/sum_amd64.go @@ -9,17 +9,6 @@ package poly1305 //go:noescape func update(state *macState, msg []byte) -func sum(out *[16]byte, m []byte, key *[32]byte) { - h := newMAC(key) - h.Write(m) - h.Sum(out) -} - -func newMAC(key *[32]byte) (h mac) { - initialize(key, &h.r, &h.s) - return -} - // mac is a wrapper for macGeneric that redirects calls that would have gone to // updateGeneric to update. // diff --git a/golang.org/x/crypto/poly1305/sum_generic.go b/golang.org/x/crypto/poly1305/sum_generic.go index 1187eab78f..c942a65904 100644 --- a/golang.org/x/crypto/poly1305/sum_generic.go +++ b/golang.org/x/crypto/poly1305/sum_generic.go @@ -31,16 +31,18 @@ func sumGeneric(out *[TagSize]byte, msg []byte, key *[32]byte) { h.Sum(out) } -func newMACGeneric(key *[32]byte) (h macGeneric) { - initialize(key, &h.r, &h.s) - return +func newMACGeneric(key *[32]byte) macGeneric { + m := macGeneric{} + initialize(key, &m.macState) + return m } // macState holds numbers in saturated 64-bit little-endian limbs. That is, // the value of [x0, x1, x2] is x[0] + x[1] * 2â¶â´ + x[2] * 2¹²â¸. type macState struct { // h is the main accumulator. It is to be interpreted modulo 2¹³Ⱐ- 5, but - // can grow larger during and after rounds. + // can grow larger during and after rounds. It must, however, remain below + // 2 * (2¹³Ⱐ- 5). h [3]uint64 // r and s are the private key components. r [2]uint64 @@ -97,11 +99,12 @@ const ( rMask1 = 0x0FFFFFFC0FFFFFFC ) -func initialize(key *[32]byte, r, s *[2]uint64) { - r[0] = binary.LittleEndian.Uint64(key[0:8]) & rMask0 - r[1] = binary.LittleEndian.Uint64(key[8:16]) & rMask1 - s[0] = binary.LittleEndian.Uint64(key[16:24]) - s[1] = binary.LittleEndian.Uint64(key[24:32]) +// initialize loads the 256-bit key into the two 128-bit secret values r and s. +func initialize(key *[32]byte, m *macState) { + m.r[0] = binary.LittleEndian.Uint64(key[0:8]) & rMask0 + m.r[1] = binary.LittleEndian.Uint64(key[8:16]) & rMask1 + m.s[0] = binary.LittleEndian.Uint64(key[16:24]) + m.s[1] = binary.LittleEndian.Uint64(key[24:32]) } // uint128 holds a 128-bit number as two 64-bit limbs, for use with the diff --git a/golang.org/x/crypto/poly1305/sum_noasm.go b/golang.org/x/crypto/poly1305/sum_noasm.go deleted file mode 100644 index 2e3ae34c7d..0000000000 --- a/golang.org/x/crypto/poly1305/sum_noasm.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build s390x,!go1.11 !amd64,!s390x,!ppc64le gccgo purego - -package poly1305 - -func sum(out *[TagSize]byte, msg []byte, key *[32]byte) { - h := newMAC(key) - h.Write(msg) - h.Sum(out) -} diff --git a/golang.org/x/crypto/poly1305/sum_ppc64le.go b/golang.org/x/crypto/poly1305/sum_ppc64le.go index 92597bb8c2..2e7a120b19 100644 --- a/golang.org/x/crypto/poly1305/sum_ppc64le.go +++ b/golang.org/x/crypto/poly1305/sum_ppc64le.go @@ -9,17 +9,6 @@ package poly1305 //go:noescape func update(state *macState, msg []byte) -func sum(out *[16]byte, m []byte, key *[32]byte) { - h := newMAC(key) - h.Write(m) - h.Sum(out) -} - -func newMAC(key *[32]byte) (h mac) { - initialize(key, &h.r, &h.s) - return -} - // mac is a wrapper for macGeneric that redirects calls that would have gone to // updateGeneric to update. // diff --git a/golang.org/x/crypto/poly1305/sum_s390x.go b/golang.org/x/crypto/poly1305/sum_s390x.go index 5f91ff84a9..958fedc079 100644 --- a/golang.org/x/crypto/poly1305/sum_s390x.go +++ b/golang.org/x/crypto/poly1305/sum_s390x.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build go1.11,!gccgo,!purego +// +build !gccgo,!purego package poly1305 @@ -10,30 +10,66 @@ import ( "golang.org/x/sys/cpu" ) -// poly1305vx is an assembly implementation of Poly1305 that uses vector +// updateVX is an assembly implementation of Poly1305 that uses vector // instructions. It must only be called if the vector facility (vx) is // available. //go:noescape -func poly1305vx(out *[16]byte, m *byte, mlen uint64, key *[32]byte) +func updateVX(state *macState, msg []byte) -// poly1305vmsl is an assembly implementation of Poly1305 that uses vector -// instructions, including VMSL. It must only be called if the vector facility (vx) is -// available and if VMSL is supported. -//go:noescape -func poly1305vmsl(out *[16]byte, m *byte, mlen uint64, key *[32]byte) +// mac is a replacement for macGeneric that uses a larger buffer and redirects +// calls that would have gone to updateGeneric to updateVX if the vector +// facility is installed. +// +// A larger buffer is required for good performance because the vector +// implementation has a higher fixed cost per call than the generic +// implementation. +type mac struct { + macState + + buffer [16 * TagSize]byte // size must be a multiple of block size (16) + offset int +} -func sum(out *[16]byte, m []byte, key *[32]byte) { - if cpu.S390X.HasVX { - var mPtr *byte - if len(m) > 0 { - mPtr = &m[0] +func (h *mac) Write(p []byte) (int, error) { + nn := len(p) + if h.offset > 0 { + n := copy(h.buffer[h.offset:], p) + if h.offset+n < len(h.buffer) { + h.offset += n + return nn, nil } - if cpu.S390X.HasVXE && len(m) > 256 { - poly1305vmsl(out, mPtr, uint64(len(m)), key) + p = p[n:] + h.offset = 0 + if cpu.S390X.HasVX { + updateVX(&h.macState, h.buffer[:]) } else { - poly1305vx(out, mPtr, uint64(len(m)), key) + updateGeneric(&h.macState, h.buffer[:]) } - } else { - sumGeneric(out, m, key) } + + tail := len(p) % len(h.buffer) // number of bytes to copy into buffer + body := len(p) - tail // number of bytes to process now + if body > 0 { + if cpu.S390X.HasVX { + updateVX(&h.macState, p[:body]) + } else { + updateGeneric(&h.macState, p[:body]) + } + } + h.offset = copy(h.buffer[:], p[body:]) // copy tail bytes - can be 0 + return nn, nil +} + +func (h *mac) Sum(out *[TagSize]byte) { + state := h.macState + remainder := h.buffer[:h.offset] + + // Use the generic implementation if we have 2 or fewer blocks left + // to sum. The vector implementation has a higher startup time. + if cpu.S390X.HasVX && len(remainder) > 2*TagSize { + updateVX(&state, remainder) + } else if len(remainder) > 0 { + updateGeneric(&state, remainder) + } + finalize(out, &state.h, &state.s) } diff --git a/golang.org/x/crypto/poly1305/sum_s390x.s b/golang.org/x/crypto/poly1305/sum_s390x.s index 806d1694b0..0fa9ee6e0b 100644 --- a/golang.org/x/crypto/poly1305/sum_s390x.s +++ b/golang.org/x/crypto/poly1305/sum_s390x.s @@ -2,115 +2,187 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build go1.11,!gccgo,!purego +// +build !gccgo,!purego #include "textflag.h" -// Implementation of Poly1305 using the vector facility (vx). - -// constants -#define MOD26 V0 -#define EX0 V1 -#define EX1 V2 -#define EX2 V3 - -// temporaries -#define T_0 V4 -#define T_1 V5 -#define T_2 V6 -#define T_3 V7 -#define T_4 V8 - -// key (r) -#define R_0 V9 -#define R_1 V10 -#define R_2 V11 -#define R_3 V12 -#define R_4 V13 -#define R5_1 V14 -#define R5_2 V15 -#define R5_3 V16 -#define R5_4 V17 -#define RSAVE_0 R5 -#define RSAVE_1 R6 -#define RSAVE_2 R7 -#define RSAVE_3 R8 -#define RSAVE_4 R9 -#define R5SAVE_1 V28 -#define R5SAVE_2 V29 -#define R5SAVE_3 V30 -#define R5SAVE_4 V31 - -// message block -#define F_0 V18 -#define F_1 V19 -#define F_2 V20 -#define F_3 V21 -#define F_4 V22 - -// accumulator -#define H_0 V23 -#define H_1 V24 -#define H_2 V25 -#define H_3 V26 -#define H_4 V27 - -GLOBL ·keyMask<>(SB), RODATA, $16 -DATA ·keyMask<>+0(SB)/8, $0xffffff0ffcffff0f -DATA ·keyMask<>+8(SB)/8, $0xfcffff0ffcffff0f - -GLOBL ·bswapMask<>(SB), RODATA, $16 -DATA ·bswapMask<>+0(SB)/8, $0x0f0e0d0c0b0a0908 -DATA ·bswapMask<>+8(SB)/8, $0x0706050403020100 - -GLOBL ·constants<>(SB), RODATA, $64 -// MOD26 -DATA ·constants<>+0(SB)/8, $0x3ffffff -DATA ·constants<>+8(SB)/8, $0x3ffffff +// This implementation of Poly1305 uses the vector facility (vx) +// to process up to 2 blocks (32 bytes) per iteration using an +// algorithm based on the one described in: +// +// NEON crypto, Daniel J. Bernstein & Peter Schwabe +// https://cryptojedi.org/papers/neoncrypto-20120320.pdf +// +// This algorithm uses 5 26-bit limbs to represent a 130-bit +// value. These limbs are, for the most part, zero extended and +// placed into 64-bit vector register elements. Each vector +// register is 128-bits wide and so holds 2 of these elements. +// Using 26-bit limbs allows us plenty of headroom to accomodate +// accumulations before and after multiplication without +// overflowing either 32-bits (before multiplication) or 64-bits +// (after multiplication). +// +// In order to parallelise the operations required to calculate +// the sum we use two separate accumulators and then sum those +// in an extra final step. For compatibility with the generic +// implementation we perform this summation at the end of every +// updateVX call. +// +// To use two accumulators we must multiply the message blocks +// by r² rather than r. Only the final message block should be +// multiplied by r. +// +// Example: +// +// We want to calculate the sum (h) for a 64 byte message (m): +// +// h = m[0:16]râ´ + m[16:32]r³ + m[32:48]r² + m[48:64]r +// +// To do this we split the calculation into the even indices +// and odd indices of the message. These form our SIMD 'lanes': +// +// h = m[ 0:16]râ´ + m[32:48]r² + <- lane 0 +// m[16:32]r³ + m[48:64]r <- lane 1 +// +// To calculate this iteratively we refactor so that both lanes +// are written in terms of r² and r: +// +// h = (m[ 0:16]r² + m[32:48])r² + <- lane 0 +// (m[16:32]r² + m[48:64])r <- lane 1 +// ^ ^ +// | coefficients for second iteration +// coefficients for first iteration +// +// So in this case we would have two iterations. In the first +// both lanes are multiplied by r². In the second only the +// first lane is multiplied by r² and the second lane is +// instead multiplied by r. This gives use the odd and even +// powers of r that we need from the original equation. +// +// Notation: +// +// h - accumulator +// r - key +// m - message +// +// [a, b] - SIMD register holding two 64-bit values +// [a, b, c, d] - SIMD register holding four 32-bit values +// xáµ¢[n] - limb n of variable x with bit width i +// +// Limbs are expressed in little endian order, so for 26-bit +// limbs x₂₆[4] will be the most significant limb and x₂₆[0] +// will be the least significant limb. + +// masking constants +#define MOD24 V0 // [0x0000000000ffffff, 0x0000000000ffffff] - mask low 24-bits +#define MOD26 V1 // [0x0000000003ffffff, 0x0000000003ffffff] - mask low 26-bits + +// expansion constants (see EXPAND macro) +#define EX0 V2 +#define EX1 V3 +#define EX2 V4 + +// key (r², r or 1 depending on context) +#define R_0 V5 +#define R_1 V6 +#define R_2 V7 +#define R_3 V8 +#define R_4 V9 + +// precalculated coefficients (5r², 5r or 0 depending on context) +#define R5_1 V10 +#define R5_2 V11 +#define R5_3 V12 +#define R5_4 V13 + +// message block (m) +#define M_0 V14 +#define M_1 V15 +#define M_2 V16 +#define M_3 V17 +#define M_4 V18 + +// accumulator (h) +#define H_0 V19 +#define H_1 V20 +#define H_2 V21 +#define H_3 V22 +#define H_4 V23 + +// temporary registers (for short-lived values) +#define T_0 V24 +#define T_1 V25 +#define T_2 V26 +#define T_3 V27 +#define T_4 V28 + +GLOBL ·constants<>(SB), RODATA, $0x30 // EX0 -DATA ·constants<>+16(SB)/8, $0x0006050403020100 -DATA ·constants<>+24(SB)/8, $0x1016151413121110 +DATA ·constants<>+0x00(SB)/8, $0x0006050403020100 +DATA ·constants<>+0x08(SB)/8, $0x1016151413121110 // EX1 -DATA ·constants<>+32(SB)/8, $0x060c0b0a09080706 -DATA ·constants<>+40(SB)/8, $0x161c1b1a19181716 +DATA ·constants<>+0x10(SB)/8, $0x060c0b0a09080706 +DATA ·constants<>+0x18(SB)/8, $0x161c1b1a19181716 // EX2 -DATA ·constants<>+48(SB)/8, $0x0d0d0d0d0d0f0e0d -DATA ·constants<>+56(SB)/8, $0x1d1d1d1d1d1f1e1d - -// h = (f*g) % (2**130-5) [partial reduction] +DATA ·constants<>+0x20(SB)/8, $0x0d0d0d0d0d0f0e0d +DATA ·constants<>+0x28(SB)/8, $0x1d1d1d1d1d1f1e1d + +// MULTIPLY multiplies each lane of f and g, partially reduced +// modulo 2¹³Ⱐ- 5. The result, h, consists of partial products +// in each lane that need to be reduced further to produce the +// final result. +// +// hâ‚₃₀ = (fâ‚₃₀gâ‚₃₀) % 2¹³Ⱐ+ (5fâ‚₃₀gâ‚₃₀) / 2¹³Ⱐ+// +// Note that the multiplication by 5 of the high bits is +// achieved by precalculating the multiplication of four of the +// g coefficients by 5. These are g51-g54. #define MULTIPLY(f0, f1, f2, f3, f4, g0, g1, g2, g3, g4, g51, g52, g53, g54, h0, h1, h2, h3, h4) \ VMLOF f0, g0, h0 \ - VMLOF f0, g1, h1 \ - VMLOF f0, g2, h2 \ VMLOF f0, g3, h3 \ + VMLOF f0, g1, h1 \ VMLOF f0, g4, h4 \ + VMLOF f0, g2, h2 \ VMLOF f1, g54, T_0 \ - VMLOF f1, g0, T_1 \ - VMLOF f1, g1, T_2 \ VMLOF f1, g2, T_3 \ + VMLOF f1, g0, T_1 \ VMLOF f1, g3, T_4 \ + VMLOF f1, g1, T_2 \ VMALOF f2, g53, h0, h0 \ - VMALOF f2, g54, h1, h1 \ - VMALOF f2, g0, h2, h2 \ VMALOF f2, g1, h3, h3 \ + VMALOF f2, g54, h1, h1 \ VMALOF f2, g2, h4, h4 \ + VMALOF f2, g0, h2, h2 \ VMALOF f3, g52, T_0, T_0 \ - VMALOF f3, g53, T_1, T_1 \ - VMALOF f3, g54, T_2, T_2 \ VMALOF f3, g0, T_3, T_3 \ + VMALOF f3, g53, T_1, T_1 \ VMALOF f3, g1, T_4, T_4 \ + VMALOF f3, g54, T_2, T_2 \ VMALOF f4, g51, h0, h0 \ - VMALOF f4, g52, h1, h1 \ - VMALOF f4, g53, h2, h2 \ VMALOF f4, g54, h3, h3 \ + VMALOF f4, g52, h1, h1 \ VMALOF f4, g0, h4, h4 \ + VMALOF f4, g53, h2, h2 \ VAG T_0, h0, h0 \ - VAG T_1, h1, h1 \ - VAG T_2, h2, h2 \ VAG T_3, h3, h3 \ - VAG T_4, h4, h4 - -// carry h0->h1 h3->h4, h1->h2 h4->h0, h0->h1 h2->h3, h3->h4 + VAG T_1, h1, h1 \ + VAG T_4, h4, h4 \ + VAG T_2, h2, h2 + +// REDUCE performs the following carry operations in four +// stages, as specified in Bernstein & Schwabe: +// +// 1: h₂₆[0]->h₂₆[1] h₂₆[3]->h₂₆[4] +// 2: h₂₆[1]->h₂₆[2] h₂₆[4]->h₂₆[0] +// 3: h₂₆[0]->h₂₆[1] h₂₆[2]->h₂₆[3] +// 4: h₂₆[3]->h₂₆[4] +// +// The result is that all of the limbs are limited to 26-bits +// except for h₂₆[1] and h₂₆[4] which are limited to 27-bits. +// +// Note that although each limb is aligned at 26-bit intervals +// they may contain values that exceed 2²ⶠ- 1, hence the need +// to carry the excess bits in each limb. #define REDUCE(h0, h1, h2, h3, h4) \ VESRLG $26, h0, T_0 \ VESRLG $26, h3, T_1 \ @@ -136,144 +208,155 @@ DATA ·constants<>+56(SB)/8, $0x1d1d1d1d1d1f1e1d VN MOD26, h3, h3 \ VAG T_2, h4, h4 -// expand in0 into d[0] and in1 into d[1] +// EXPAND splits the 128-bit little-endian values in0 and in1 +// into 26-bit big-endian limbs and places the results into +// the first and second lane of d₂₆[0:4] respectively. +// +// The EX0, EX1 and EX2 constants are arrays of byte indices +// for permutation. The permutation both reverses the bytes +// in the input and ensures the bytes are copied into the +// destination limb ready to be shifted into their final +// position. #define EXPAND(in0, in1, d0, d1, d2, d3, d4) \ - VGBM $0x0707, d1 \ // d1=tmp - VPERM in0, in1, EX2, d4 \ VPERM in0, in1, EX0, d0 \ VPERM in0, in1, EX1, d2 \ - VN d1, d4, d4 \ + VPERM in0, in1, EX2, d4 \ VESRLG $26, d0, d1 \ VESRLG $30, d2, d3 \ VESRLG $4, d2, d2 \ - VN MOD26, d0, d0 \ - VN MOD26, d1, d1 \ - VN MOD26, d2, d2 \ - VN MOD26, d3, d3 - -// pack h4:h0 into h1:h0 (no carry) -#define PACK(h0, h1, h2, h3, h4) \ - VESLG $26, h1, h1 \ - VESLG $26, h3, h3 \ - VO h0, h1, h0 \ - VO h2, h3, h2 \ - VESLG $4, h2, h2 \ - VLEIB $7, $48, h1 \ - VSLB h1, h2, h2 \ - VO h0, h2, h0 \ - VLEIB $7, $104, h1 \ - VSLB h1, h4, h3 \ - VO h3, h0, h0 \ - VLEIB $7, $24, h1 \ - VSRLB h1, h4, h1 - -// if h > 2**130-5 then h -= 2**130-5 -#define MOD(h0, h1, t0, t1, t2) \ - VZERO t0 \ - VLEIG $1, $5, t0 \ - VACCQ h0, t0, t1 \ - VAQ h0, t0, t0 \ - VONE t2 \ - VLEIG $1, $-4, t2 \ - VAQ t2, t1, t1 \ - VACCQ h1, t1, t1 \ - VONE t2 \ - VAQ t2, t1, t1 \ - VN h0, t1, t2 \ - VNC t0, t1, t1 \ - VO t1, t2, h0 - -// func poly1305vx(out *[16]byte, m *byte, mlen uint64, key *[32]key) -TEXT ·poly1305vx(SB), $0-32 - // This code processes up to 2 blocks (32 bytes) per iteration - // using the algorithm described in: - // NEON crypto, Daniel J. Bernstein & Peter Schwabe - // https://cryptojedi.org/papers/neoncrypto-20120320.pdf - LMG out+0(FP), R1, R4 // R1=out, R2=m, R3=mlen, R4=key - - // load MOD26, EX0, EX1 and EX2 + VN MOD26, d0, d0 \ // [in0₂₆[0], in1₂₆[0]] + VN MOD26, d3, d3 \ // [in0₂₆[3], in1₂₆[3]] + VN MOD26, d1, d1 \ // [in0₂₆[1], in1₂₆[1]] + VN MOD24, d4, d4 \ // [in0₂₆[4], in1₂₆[4]] + VN MOD26, d2, d2 // [in0₂₆[2], in1₂₆[2]] + +// func updateVX(state *macState, msg []byte) +TEXT ·updateVX(SB), NOSPLIT, $0 + MOVD state+0(FP), R1 + LMG msg+8(FP), R2, R3 // R2=msg_base, R3=msg_len + + // load EX0, EX1 and EX2 MOVD $·constants<>(SB), R5 - VLM (R5), MOD26, EX2 - - // setup r - VL (R4), T_0 - MOVD $·keyMask<>(SB), R6 - VL (R6), T_1 - VN T_0, T_1, T_0 - EXPAND(T_0, T_0, R_0, R_1, R_2, R_3, R_4) - - // setup r*5 - VLEIG $0, $5, T_0 - VLEIG $1, $5, T_0 - - // store r (for final block) - VMLOF T_0, R_1, R5SAVE_1 - VMLOF T_0, R_2, R5SAVE_2 - VMLOF T_0, R_3, R5SAVE_3 - VMLOF T_0, R_4, R5SAVE_4 - VLGVG $0, R_0, RSAVE_0 - VLGVG $0, R_1, RSAVE_1 - VLGVG $0, R_2, RSAVE_2 - VLGVG $0, R_3, RSAVE_3 - VLGVG $0, R_4, RSAVE_4 - - // skip r**2 calculation + VLM (R5), EX0, EX2 + + // generate masks + VGMG $(64-24), $63, MOD24 // [0x00ffffff, 0x00ffffff] + VGMG $(64-26), $63, MOD26 // [0x03ffffff, 0x03ffffff] + + // load h (accumulator) and r (key) from state + VZERO T_1 // [0, 0] + VL 0(R1), T_0 // [h₆₄[0], h₆₄[1]] + VLEG $0, 16(R1), T_1 // [h₆₄[2], 0] + VL 24(R1), T_2 // [r₆₄[0], r₆₄[1]] + VPDI $0, T_0, T_2, T_3 // [h₆₄[0], r₆₄[0]] + VPDI $5, T_0, T_2, T_4 // [h₆₄[1], r₆₄[1]] + + // unpack h and r into 26-bit limbs + // note: h₆₄[2] may have the low 3 bits set, so h₂₆[4] is a 27-bit value + VN MOD26, T_3, H_0 // [h₂₆[0], r₂₆[0]] + VZERO H_1 // [0, 0] + VZERO H_3 // [0, 0] + VGMG $(64-12-14), $(63-12), T_0 // [0x03fff000, 0x03fff000] - 26-bit mask with low 12 bits masked out + VESLG $24, T_1, T_1 // [h₆₄[2]<<24, 0] + VERIMG $-26&63, T_3, MOD26, H_1 // [h₂₆[1], r₂₆[1]] + VESRLG $+52&63, T_3, H_2 // [h₂₆[2], r₂₆[2]] - low 12 bits only + VERIMG $-14&63, T_4, MOD26, H_3 // [h₂₆[1], r₂₆[1]] + VESRLG $40, T_4, H_4 // [h₂₆[4], r₂₆[4]] - low 24 bits only + VERIMG $+12&63, T_4, T_0, H_2 // [h₂₆[2], r₂₆[2]] - complete + VO T_1, H_4, H_4 // [h₂₆[4], r₂₆[4]] - complete + + // replicate r across all 4 vector elements + VREPF $3, H_0, R_0 // [r₂₆[0], r₂₆[0], r₂₆[0], r₂₆[0]] + VREPF $3, H_1, R_1 // [r₂₆[1], r₂₆[1], r₂₆[1], r₂₆[1]] + VREPF $3, H_2, R_2 // [r₂₆[2], r₂₆[2], r₂₆[2], r₂₆[2]] + VREPF $3, H_3, R_3 // [r₂₆[3], r₂₆[3], r₂₆[3], r₂₆[3]] + VREPF $3, H_4, R_4 // [r₂₆[4], r₂₆[4], r₂₆[4], r₂₆[4]] + + // zero out lane 1 of h + VLEIG $1, $0, H_0 // [h₂₆[0], 0] + VLEIG $1, $0, H_1 // [h₂₆[1], 0] + VLEIG $1, $0, H_2 // [h₂₆[2], 0] + VLEIG $1, $0, H_3 // [h₂₆[3], 0] + VLEIG $1, $0, H_4 // [h₂₆[4], 0] + + // calculate 5r (ignore least significant limb) + VREPIF $5, T_0 + VMLF T_0, R_1, R5_1 // [5r₂₆[1], 5r₂₆[1], 5r₂₆[1], 5r₂₆[1]] + VMLF T_0, R_2, R5_2 // [5r₂₆[2], 5r₂₆[2], 5r₂₆[2], 5r₂₆[2]] + VMLF T_0, R_3, R5_3 // [5r₂₆[3], 5r₂₆[3], 5r₂₆[3], 5r₂₆[3]] + VMLF T_0, R_4, R5_4 // [5r₂₆[4], 5r₂₆[4], 5r₂₆[4], 5r₂₆[4]] + + // skip r² calculation if we are only calculating one block CMPBLE R3, $16, skip - // calculate r**2 - MULTIPLY(R_0, R_1, R_2, R_3, R_4, R_0, R_1, R_2, R_3, R_4, R5SAVE_1, R5SAVE_2, R5SAVE_3, R5SAVE_4, H_0, H_1, H_2, H_3, H_4) - REDUCE(H_0, H_1, H_2, H_3, H_4) - VLEIG $0, $5, T_0 - VLEIG $1, $5, T_0 - VMLOF T_0, H_1, R5_1 - VMLOF T_0, H_2, R5_2 - VMLOF T_0, H_3, R5_3 - VMLOF T_0, H_4, R5_4 - VLR H_0, R_0 - VLR H_1, R_1 - VLR H_2, R_2 - VLR H_3, R_3 - VLR H_4, R_4 - - // initialize h - VZERO H_0 - VZERO H_1 - VZERO H_2 - VZERO H_3 - VZERO H_4 + // calculate r² + MULTIPLY(R_0, R_1, R_2, R_3, R_4, R_0, R_1, R_2, R_3, R_4, R5_1, R5_2, R5_3, R5_4, M_0, M_1, M_2, M_3, M_4) + REDUCE(M_0, M_1, M_2, M_3, M_4) + VGBM $0x0f0f, T_0 + VERIMG $0, M_0, T_0, R_0 // [r₂₆[0], r²₂₆[0], r₂₆[0], r²₂₆[0]] + VERIMG $0, M_1, T_0, R_1 // [r₂₆[1], r²₂₆[1], r₂₆[1], r²₂₆[1]] + VERIMG $0, M_2, T_0, R_2 // [r₂₆[2], r²₂₆[2], r₂₆[2], r²₂₆[2]] + VERIMG $0, M_3, T_0, R_3 // [r₂₆[3], r²₂₆[3], r₂₆[3], r²₂₆[3]] + VERIMG $0, M_4, T_0, R_4 // [r₂₆[4], r²₂₆[4], r₂₆[4], r²₂₆[4]] + + // calculate 5r² (ignore least significant limb) + VREPIF $5, T_0 + VMLF T_0, R_1, R5_1 // [5r₂₆[1], 5r²₂₆[1], 5r₂₆[1], 5r²₂₆[1]] + VMLF T_0, R_2, R5_2 // [5r₂₆[2], 5r²₂₆[2], 5r₂₆[2], 5r²₂₆[2]] + VMLF T_0, R_3, R5_3 // [5r₂₆[3], 5r²₂₆[3], 5r₂₆[3], 5r²₂₆[3]] + VMLF T_0, R_4, R5_4 // [5r₂₆[4], 5r²₂₆[4], 5r₂₆[4], 5r²₂₆[4]] loop: - CMPBLE R3, $32, b2 - VLM (R2), T_0, T_1 - SUB $32, R3 - MOVD $32(R2), R2 - EXPAND(T_0, T_1, F_0, F_1, F_2, F_3, F_4) - VLEIB $4, $1, F_4 - VLEIB $12, $1, F_4 + CMPBLE R3, $32, b2 // 2 or fewer blocks remaining, need to change key coefficients + + // load next 2 blocks from message + VLM (R2), T_0, T_1 + + // update message slice + SUB $32, R3 + MOVD $32(R2), R2 + + // unpack message blocks into 26-bit big-endian limbs + EXPAND(T_0, T_1, M_0, M_1, M_2, M_3, M_4) + + // add 2¹²⸠to each message block value + VLEIB $4, $1, M_4 + VLEIB $12, $1, M_4 multiply: - VAG H_0, F_0, F_0 - VAG H_1, F_1, F_1 - VAG H_2, F_2, F_2 - VAG H_3, F_3, F_3 - VAG H_4, F_4, F_4 - MULTIPLY(F_0, F_1, F_2, F_3, F_4, R_0, R_1, R_2, R_3, R_4, R5_1, R5_2, R5_3, R5_4, H_0, H_1, H_2, H_3, H_4) + // accumulate the incoming message + VAG H_0, M_0, M_0 + VAG H_3, M_3, M_3 + VAG H_1, M_1, M_1 + VAG H_4, M_4, M_4 + VAG H_2, M_2, M_2 + + // multiply the accumulator by the key coefficient + MULTIPLY(M_0, M_1, M_2, M_3, M_4, R_0, R_1, R_2, R_3, R_4, R5_1, R5_2, R5_3, R5_4, H_0, H_1, H_2, H_3, H_4) + + // carry and partially reduce the partial products REDUCE(H_0, H_1, H_2, H_3, H_4) + CMPBNE R3, $0, loop finish: - // sum vectors + // sum lane 0 and lane 1 and put the result in lane 1 VZERO T_0 VSUMQG H_0, T_0, H_0 - VSUMQG H_1, T_0, H_1 - VSUMQG H_2, T_0, H_2 VSUMQG H_3, T_0, H_3 + VSUMQG H_1, T_0, H_1 VSUMQG H_4, T_0, H_4 + VSUMQG H_2, T_0, H_2 - // h may be >= 2*(2**130-5) so we need to reduce it again + // reduce again after summation + // TODO(mundaym): there might be a more efficient way to do this + // now that we only have 1 active lane. For example, we could + // simultaneously pack the values as we reduce them. REDUCE(H_0, H_1, H_2, H_3, H_4) - // carry h1->h4 + // carry h[1] through to h[4] so that only h[4] can exceed 2²ⶠ- 1 + // TODO(mundaym): in testing this final carry was unnecessary. + // Needs a proof before it can be removed though. VESRLG $26, H_1, T_1 VN MOD26, H_1, H_1 VAQ T_1, H_2, H_2 @@ -284,95 +367,137 @@ finish: VN MOD26, H_3, H_3 VAQ T_3, H_4, H_4 - // h is now < 2*(2**130-5) - // pack h into h1 (hi) and h0 (lo) - PACK(H_0, H_1, H_2, H_3, H_4) - - // if h > 2**130-5 then h -= 2**130-5 - MOD(H_0, H_1, T_0, T_1, T_2) - - // h += s - MOVD $·bswapMask<>(SB), R5 - VL (R5), T_1 - VL 16(R4), T_0 - VPERM T_0, T_0, T_1, T_0 // reverse bytes (to big) - VAQ T_0, H_0, H_0 - VPERM H_0, H_0, T_1, H_0 // reverse bytes (to little) - VST H_0, (R1) - + // h is now < 2(2¹³Ⱐ- 5) + // Pack each lane in h₂₆[0:4] into hâ‚₂₈[0:1]. + VESLG $26, H_1, H_1 + VESLG $26, H_3, H_3 + VO H_0, H_1, H_0 + VO H_2, H_3, H_2 + VESLG $4, H_2, H_2 + VLEIB $7, $48, H_1 + VSLB H_1, H_2, H_2 + VO H_0, H_2, H_0 + VLEIB $7, $104, H_1 + VSLB H_1, H_4, H_3 + VO H_3, H_0, H_0 + VLEIB $7, $24, H_1 + VSRLB H_1, H_4, H_1 + + // update state + VSTEG $1, H_0, 0(R1) + VSTEG $0, H_0, 8(R1) + VSTEG $1, H_1, 16(R1) RET -b2: +b2: // 2 or fewer blocks remaining CMPBLE R3, $16, b1 - // 2 blocks remaining - SUB $17, R3 - VL (R2), T_0 - VLL R3, 16(R2), T_1 - ADD $1, R3 + // Load the 2 remaining blocks (17-32 bytes remaining). + MOVD $-17(R3), R0 // index of final byte to load modulo 16 + VL (R2), T_0 // load full 16 byte block + VLL R0, 16(R2), T_1 // load final (possibly partial) block and pad with zeros to 16 bytes + + // The Poly1305 algorithm requires that a 1 bit be appended to + // each message block. If the final block is less than 16 bytes + // long then it is easiest to insert the 1 before the message + // block is split into 26-bit limbs. If, on the other hand, the + // final message block is 16 bytes long then we append the 1 bit + // after expansion as normal. MOVBZ $1, R0 - CMPBEQ R3, $16, 2(PC) - VLVGB R3, R0, T_1 - EXPAND(T_0, T_1, F_0, F_1, F_2, F_3, F_4) + MOVD $-16(R3), R3 // index of byte in last block to insert 1 at (could be 16) + CMPBEQ R3, $16, 2(PC) // skip the insertion if the final block is 16 bytes long + VLVGB R3, R0, T_1 // insert 1 into the byte at index R3 + + // Split both blocks into 26-bit limbs in the appropriate lanes. + EXPAND(T_0, T_1, M_0, M_1, M_2, M_3, M_4) + + // Append a 1 byte to the end of the second to last block. + VLEIB $4, $1, M_4 + + // Append a 1 byte to the end of the last block only if it is a + // full 16 byte block. CMPBNE R3, $16, 2(PC) - VLEIB $12, $1, F_4 - VLEIB $4, $1, F_4 - - // setup [r²,r] - VLVGG $1, RSAVE_0, R_0 - VLVGG $1, RSAVE_1, R_1 - VLVGG $1, RSAVE_2, R_2 - VLVGG $1, RSAVE_3, R_3 - VLVGG $1, RSAVE_4, R_4 - VPDI $0, R5_1, R5SAVE_1, R5_1 - VPDI $0, R5_2, R5SAVE_2, R5_2 - VPDI $0, R5_3, R5SAVE_3, R5_3 - VPDI $0, R5_4, R5SAVE_4, R5_4 + VLEIB $12, $1, M_4 + + // Finally, set up the coefficients for the final multiplication. + // We have previously saved r and 5r in the 32-bit even indexes + // of the R_[0-4] and R5_[1-4] coefficient registers. + // + // We want lane 0 to be multiplied by r² so that can be kept the + // same. We want lane 1 to be multiplied by r so we need to move + // the saved r value into the 32-bit odd index in lane 1 by + // rotating the 64-bit lane by 32. + VGBM $0x00ff, T_0 // [0, 0xffffffffffffffff] - mask lane 1 only + VERIMG $32, R_0, T_0, R_0 // [_, r²₂₆[0], _, r₂₆[0]] + VERIMG $32, R_1, T_0, R_1 // [_, r²₂₆[1], _, r₂₆[1]] + VERIMG $32, R_2, T_0, R_2 // [_, r²₂₆[2], _, r₂₆[2]] + VERIMG $32, R_3, T_0, R_3 // [_, r²₂₆[3], _, r₂₆[3]] + VERIMG $32, R_4, T_0, R_4 // [_, r²₂₆[4], _, r₂₆[4]] + VERIMG $32, R5_1, T_0, R5_1 // [_, 5r²₂₆[1], _, 5r₂₆[1]] + VERIMG $32, R5_2, T_0, R5_2 // [_, 5r²₂₆[2], _, 5r₂₆[2]] + VERIMG $32, R5_3, T_0, R5_3 // [_, 5r²₂₆[3], _, 5r₂₆[3]] + VERIMG $32, R5_4, T_0, R5_4 // [_, 5r²₂₆[4], _, 5r₂₆[4]] MOVD $0, R3 BR multiply skip: - VZERO H_0 - VZERO H_1 - VZERO H_2 - VZERO H_3 - VZERO H_4 - CMPBEQ R3, $0, finish -b1: - // 1 block remaining - SUB $1, R3 - VLL R3, (R2), T_0 - ADD $1, R3 +b1: // 1 block remaining + + // Load the final block (1-16 bytes). This will be placed into + // lane 0. + MOVD $-1(R3), R0 + VLL R0, (R2), T_0 // pad to 16 bytes with zeros + + // The Poly1305 algorithm requires that a 1 bit be appended to + // each message block. If the final block is less than 16 bytes + // long then it is easiest to insert the 1 before the message + // block is split into 26-bit limbs. If, on the other hand, the + // final message block is 16 bytes long then we append the 1 bit + // after expansion as normal. MOVBZ $1, R0 CMPBEQ R3, $16, 2(PC) VLVGB R3, R0, T_0 - VZERO T_1 - EXPAND(T_0, T_1, F_0, F_1, F_2, F_3, F_4) + + // Set the message block in lane 1 to the value 0 so that it + // can be accumulated without affecting the final result. + VZERO T_1 + + // Split the final message block into 26-bit limbs in lane 0. + // Lane 1 will be contain 0. + EXPAND(T_0, T_1, M_0, M_1, M_2, M_3, M_4) + + // Append a 1 byte to the end of the last block only if it is a + // full 16 byte block. CMPBNE R3, $16, 2(PC) - VLEIB $4, $1, F_4 - VLEIG $1, $1, R_0 - VZERO R_1 - VZERO R_2 - VZERO R_3 - VZERO R_4 - VZERO R5_1 - VZERO R5_2 - VZERO R5_3 - VZERO R5_4 - - // setup [r, 1] - VLVGG $0, RSAVE_0, R_0 - VLVGG $0, RSAVE_1, R_1 - VLVGG $0, RSAVE_2, R_2 - VLVGG $0, RSAVE_3, R_3 - VLVGG $0, RSAVE_4, R_4 - VPDI $0, R5SAVE_1, R5_1, R5_1 - VPDI $0, R5SAVE_2, R5_2, R5_2 - VPDI $0, R5SAVE_3, R5_3, R5_3 - VPDI $0, R5SAVE_4, R5_4, R5_4 + VLEIB $4, $1, M_4 + + // We have previously saved r and 5r in the 32-bit even indexes + // of the R_[0-4] and R5_[1-4] coefficient registers. + // + // We want lane 0 to be multiplied by r so we need to move the + // saved r value into the 32-bit odd index in lane 0. We want + // lane 1 to be set to the value 1. This makes multiplication + // a no-op. We do this by setting lane 1 in every register to 0 + // and then just setting the 32-bit index 3 in R_0 to 1. + VZERO T_0 + MOVD $0, R0 + MOVD $0x10111213, R12 + VLVGP R12, R0, T_1 // [_, 0x10111213, _, 0x00000000] + VPERM T_0, R_0, T_1, R_0 // [_, r₂₆[0], _, 0] + VPERM T_0, R_1, T_1, R_1 // [_, r₂₆[1], _, 0] + VPERM T_0, R_2, T_1, R_2 // [_, r₂₆[2], _, 0] + VPERM T_0, R_3, T_1, R_3 // [_, r₂₆[3], _, 0] + VPERM T_0, R_4, T_1, R_4 // [_, r₂₆[4], _, 0] + VPERM T_0, R5_1, T_1, R5_1 // [_, 5r₂₆[1], _, 0] + VPERM T_0, R5_2, T_1, R5_2 // [_, 5r₂₆[2], _, 0] + VPERM T_0, R5_3, T_1, R5_3 // [_, 5r₂₆[3], _, 0] + VPERM T_0, R5_4, T_1, R5_4 // [_, 5r₂₆[4], _, 0] + + // Set the value of lane 1 to be 1. + VLEIF $3, $1, R_0 // [_, r₂₆[0], _, 1] MOVD $0, R3 BR multiply diff --git a/golang.org/x/crypto/poly1305/sum_vmsl_s390x.s b/golang.org/x/crypto/poly1305/sum_vmsl_s390x.s deleted file mode 100644 index b439af9369..0000000000 --- a/golang.org/x/crypto/poly1305/sum_vmsl_s390x.s +++ /dev/null @@ -1,909 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build go1.11,!gccgo,!purego - -#include "textflag.h" - -// Implementation of Poly1305 using the vector facility (vx) and the VMSL instruction. - -// constants -#define EX0 V1 -#define EX1 V2 -#define EX2 V3 - -// temporaries -#define T_0 V4 -#define T_1 V5 -#define T_2 V6 -#define T_3 V7 -#define T_4 V8 -#define T_5 V9 -#define T_6 V10 -#define T_7 V11 -#define T_8 V12 -#define T_9 V13 -#define T_10 V14 - -// r**2 & r**4 -#define R_0 V15 -#define R_1 V16 -#define R_2 V17 -#define R5_1 V18 -#define R5_2 V19 -// key (r) -#define RSAVE_0 R7 -#define RSAVE_1 R8 -#define RSAVE_2 R9 -#define R5SAVE_1 R10 -#define R5SAVE_2 R11 - -// message block -#define M0 V20 -#define M1 V21 -#define M2 V22 -#define M3 V23 -#define M4 V24 -#define M5 V25 - -// accumulator -#define H0_0 V26 -#define H1_0 V27 -#define H2_0 V28 -#define H0_1 V29 -#define H1_1 V30 -#define H2_1 V31 - -GLOBL ·keyMask<>(SB), RODATA, $16 -DATA ·keyMask<>+0(SB)/8, $0xffffff0ffcffff0f -DATA ·keyMask<>+8(SB)/8, $0xfcffff0ffcffff0f - -GLOBL ·bswapMask<>(SB), RODATA, $16 -DATA ·bswapMask<>+0(SB)/8, $0x0f0e0d0c0b0a0908 -DATA ·bswapMask<>+8(SB)/8, $0x0706050403020100 - -GLOBL ·constants<>(SB), RODATA, $48 -// EX0 -DATA ·constants<>+0(SB)/8, $0x18191a1b1c1d1e1f -DATA ·constants<>+8(SB)/8, $0x0000050403020100 -// EX1 -DATA ·constants<>+16(SB)/8, $0x18191a1b1c1d1e1f -DATA ·constants<>+24(SB)/8, $0x00000a0908070605 -// EX2 -DATA ·constants<>+32(SB)/8, $0x18191a1b1c1d1e1f -DATA ·constants<>+40(SB)/8, $0x0000000f0e0d0c0b - -GLOBL ·c<>(SB), RODATA, $48 -// EX0 -DATA ·c<>+0(SB)/8, $0x0000050403020100 -DATA ·c<>+8(SB)/8, $0x0000151413121110 -// EX1 -DATA ·c<>+16(SB)/8, $0x00000a0908070605 -DATA ·c<>+24(SB)/8, $0x00001a1918171615 -// EX2 -DATA ·c<>+32(SB)/8, $0x0000000f0e0d0c0b -DATA ·c<>+40(SB)/8, $0x0000001f1e1d1c1b - -GLOBL ·reduce<>(SB), RODATA, $32 -// 44 bit -DATA ·reduce<>+0(SB)/8, $0x0 -DATA ·reduce<>+8(SB)/8, $0xfffffffffff -// 42 bit -DATA ·reduce<>+16(SB)/8, $0x0 -DATA ·reduce<>+24(SB)/8, $0x3ffffffffff - -// h = (f*g) % (2**130-5) [partial reduction] -// uses T_0...T_9 temporary registers -// input: m02_0, m02_1, m02_2, m13_0, m13_1, m13_2, r_0, r_1, r_2, r5_1, r5_2, m4_0, m4_1, m4_2, m5_0, m5_1, m5_2 -// temp: t0, t1, t2, t3, t4, t5, t6, t7, t8, t9 -// output: m02_0, m02_1, m02_2, m13_0, m13_1, m13_2 -#define MULTIPLY(m02_0, m02_1, m02_2, m13_0, m13_1, m13_2, r_0, r_1, r_2, r5_1, r5_2, m4_0, m4_1, m4_2, m5_0, m5_1, m5_2, t0, t1, t2, t3, t4, t5, t6, t7, t8, t9) \ - \ // Eliminate the dependency for the last 2 VMSLs - VMSLG m02_0, r_2, m4_2, m4_2 \ - VMSLG m13_0, r_2, m5_2, m5_2 \ // 8 VMSLs pipelined - VMSLG m02_0, r_0, m4_0, m4_0 \ - VMSLG m02_1, r5_2, V0, T_0 \ - VMSLG m02_0, r_1, m4_1, m4_1 \ - VMSLG m02_1, r_0, V0, T_1 \ - VMSLG m02_1, r_1, V0, T_2 \ - VMSLG m02_2, r5_1, V0, T_3 \ - VMSLG m02_2, r5_2, V0, T_4 \ - VMSLG m13_0, r_0, m5_0, m5_0 \ - VMSLG m13_1, r5_2, V0, T_5 \ - VMSLG m13_0, r_1, m5_1, m5_1 \ - VMSLG m13_1, r_0, V0, T_6 \ - VMSLG m13_1, r_1, V0, T_7 \ - VMSLG m13_2, r5_1, V0, T_8 \ - VMSLG m13_2, r5_2, V0, T_9 \ - VMSLG m02_2, r_0, m4_2, m4_2 \ - VMSLG m13_2, r_0, m5_2, m5_2 \ - VAQ m4_0, T_0, m02_0 \ - VAQ m4_1, T_1, m02_1 \ - VAQ m5_0, T_5, m13_0 \ - VAQ m5_1, T_6, m13_1 \ - VAQ m02_0, T_3, m02_0 \ - VAQ m02_1, T_4, m02_1 \ - VAQ m13_0, T_8, m13_0 \ - VAQ m13_1, T_9, m13_1 \ - VAQ m4_2, T_2, m02_2 \ - VAQ m5_2, T_7, m13_2 \ - -// SQUARE uses three limbs of r and r_2*5 to output square of r -// uses T_1, T_5 and T_7 temporary registers -// input: r_0, r_1, r_2, r5_2 -// temp: TEMP0, TEMP1, TEMP2 -// output: p0, p1, p2 -#define SQUARE(r_0, r_1, r_2, r5_2, p0, p1, p2, TEMP0, TEMP1, TEMP2) \ - VMSLG r_0, r_0, p0, p0 \ - VMSLG r_1, r5_2, V0, TEMP0 \ - VMSLG r_2, r5_2, p1, p1 \ - VMSLG r_0, r_1, V0, TEMP1 \ - VMSLG r_1, r_1, p2, p2 \ - VMSLG r_0, r_2, V0, TEMP2 \ - VAQ TEMP0, p0, p0 \ - VAQ TEMP1, p1, p1 \ - VAQ TEMP2, p2, p2 \ - VAQ TEMP0, p0, p0 \ - VAQ TEMP1, p1, p1 \ - VAQ TEMP2, p2, p2 \ - -// carry h0->h1->h2->h0 || h3->h4->h5->h3 -// uses T_2, T_4, T_5, T_7, T_8, T_9 -// t6, t7, t8, t9, t10, t11 -// input: h0, h1, h2, h3, h4, h5 -// temp: t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11 -// output: h0, h1, h2, h3, h4, h5 -#define REDUCE(h0, h1, h2, h3, h4, h5, t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11) \ - VLM (R12), t6, t7 \ // 44 and 42 bit clear mask - VLEIB $7, $0x28, t10 \ // 5 byte shift mask - VREPIB $4, t8 \ // 4 bit shift mask - VREPIB $2, t11 \ // 2 bit shift mask - VSRLB t10, h0, t0 \ // h0 byte shift - VSRLB t10, h1, t1 \ // h1 byte shift - VSRLB t10, h2, t2 \ // h2 byte shift - VSRLB t10, h3, t3 \ // h3 byte shift - VSRLB t10, h4, t4 \ // h4 byte shift - VSRLB t10, h5, t5 \ // h5 byte shift - VSRL t8, t0, t0 \ // h0 bit shift - VSRL t8, t1, t1 \ // h2 bit shift - VSRL t11, t2, t2 \ // h2 bit shift - VSRL t8, t3, t3 \ // h3 bit shift - VSRL t8, t4, t4 \ // h4 bit shift - VESLG $2, t2, t9 \ // h2 carry x5 - VSRL t11, t5, t5 \ // h5 bit shift - VN t6, h0, h0 \ // h0 clear carry - VAQ t2, t9, t2 \ // h2 carry x5 - VESLG $2, t5, t9 \ // h5 carry x5 - VN t6, h1, h1 \ // h1 clear carry - VN t7, h2, h2 \ // h2 clear carry - VAQ t5, t9, t5 \ // h5 carry x5 - VN t6, h3, h3 \ // h3 clear carry - VN t6, h4, h4 \ // h4 clear carry - VN t7, h5, h5 \ // h5 clear carry - VAQ t0, h1, h1 \ // h0->h1 - VAQ t3, h4, h4 \ // h3->h4 - VAQ t1, h2, h2 \ // h1->h2 - VAQ t4, h5, h5 \ // h4->h5 - VAQ t2, h0, h0 \ // h2->h0 - VAQ t5, h3, h3 \ // h5->h3 - VREPG $1, t6, t6 \ // 44 and 42 bit masks across both halves - VREPG $1, t7, t7 \ - VSLDB $8, h0, h0, h0 \ // set up [h0/1/2, h3/4/5] - VSLDB $8, h1, h1, h1 \ - VSLDB $8, h2, h2, h2 \ - VO h0, h3, h3 \ - VO h1, h4, h4 \ - VO h2, h5, h5 \ - VESRLG $44, h3, t0 \ // 44 bit shift right - VESRLG $44, h4, t1 \ - VESRLG $42, h5, t2 \ - VN t6, h3, h3 \ // clear carry bits - VN t6, h4, h4 \ - VN t7, h5, h5 \ - VESLG $2, t2, t9 \ // multiply carry by 5 - VAQ t9, t2, t2 \ - VAQ t0, h4, h4 \ - VAQ t1, h5, h5 \ - VAQ t2, h3, h3 \ - -// carry h0->h1->h2->h0 -// input: h0, h1, h2 -// temp: t0, t1, t2, t3, t4, t5, t6, t7, t8 -// output: h0, h1, h2 -#define REDUCE2(h0, h1, h2, t0, t1, t2, t3, t4, t5, t6, t7, t8) \ - VLEIB $7, $0x28, t3 \ // 5 byte shift mask - VREPIB $4, t4 \ // 4 bit shift mask - VREPIB $2, t7 \ // 2 bit shift mask - VGBM $0x003F, t5 \ // mask to clear carry bits - VSRLB t3, h0, t0 \ - VSRLB t3, h1, t1 \ - VSRLB t3, h2, t2 \ - VESRLG $4, t5, t5 \ // 44 bit clear mask - VSRL t4, t0, t0 \ - VSRL t4, t1, t1 \ - VSRL t7, t2, t2 \ - VESRLG $2, t5, t6 \ // 42 bit clear mask - VESLG $2, t2, t8 \ - VAQ t8, t2, t2 \ - VN t5, h0, h0 \ - VN t5, h1, h1 \ - VN t6, h2, h2 \ - VAQ t0, h1, h1 \ - VAQ t1, h2, h2 \ - VAQ t2, h0, h0 \ - VSRLB t3, h0, t0 \ - VSRLB t3, h1, t1 \ - VSRLB t3, h2, t2 \ - VSRL t4, t0, t0 \ - VSRL t4, t1, t1 \ - VSRL t7, t2, t2 \ - VN t5, h0, h0 \ - VN t5, h1, h1 \ - VESLG $2, t2, t8 \ - VN t6, h2, h2 \ - VAQ t0, h1, h1 \ - VAQ t8, t2, t2 \ - VAQ t1, h2, h2 \ - VAQ t2, h0, h0 \ - -// expands two message blocks into the lower halfs of the d registers -// moves the contents of the d registers into upper halfs -// input: in1, in2, d0, d1, d2, d3, d4, d5 -// temp: TEMP0, TEMP1, TEMP2, TEMP3 -// output: d0, d1, d2, d3, d4, d5 -#define EXPACC(in1, in2, d0, d1, d2, d3, d4, d5, TEMP0, TEMP1, TEMP2, TEMP3) \ - VGBM $0xff3f, TEMP0 \ - VGBM $0xff1f, TEMP1 \ - VESLG $4, d1, TEMP2 \ - VESLG $4, d4, TEMP3 \ - VESRLG $4, TEMP0, TEMP0 \ - VPERM in1, d0, EX0, d0 \ - VPERM in2, d3, EX0, d3 \ - VPERM in1, d2, EX2, d2 \ - VPERM in2, d5, EX2, d5 \ - VPERM in1, TEMP2, EX1, d1 \ - VPERM in2, TEMP3, EX1, d4 \ - VN TEMP0, d0, d0 \ - VN TEMP0, d3, d3 \ - VESRLG $4, d1, d1 \ - VESRLG $4, d4, d4 \ - VN TEMP1, d2, d2 \ - VN TEMP1, d5, d5 \ - VN TEMP0, d1, d1 \ - VN TEMP0, d4, d4 \ - -// expands one message block into the lower halfs of the d registers -// moves the contents of the d registers into upper halfs -// input: in, d0, d1, d2 -// temp: TEMP0, TEMP1, TEMP2 -// output: d0, d1, d2 -#define EXPACC2(in, d0, d1, d2, TEMP0, TEMP1, TEMP2) \ - VGBM $0xff3f, TEMP0 \ - VESLG $4, d1, TEMP2 \ - VGBM $0xff1f, TEMP1 \ - VPERM in, d0, EX0, d0 \ - VESRLG $4, TEMP0, TEMP0 \ - VPERM in, d2, EX2, d2 \ - VPERM in, TEMP2, EX1, d1 \ - VN TEMP0, d0, d0 \ - VN TEMP1, d2, d2 \ - VESRLG $4, d1, d1 \ - VN TEMP0, d1, d1 \ - -// pack h2:h0 into h1:h0 (no carry) -// input: h0, h1, h2 -// output: h0, h1, h2 -#define PACK(h0, h1, h2) \ - VMRLG h1, h2, h2 \ // copy h1 to upper half h2 - VESLG $44, h1, h1 \ // shift limb 1 44 bits, leaving 20 - VO h0, h1, h0 \ // combine h0 with 20 bits from limb 1 - VESRLG $20, h2, h1 \ // put top 24 bits of limb 1 into h1 - VLEIG $1, $0, h1 \ // clear h2 stuff from lower half of h1 - VO h0, h1, h0 \ // h0 now has 88 bits (limb 0 and 1) - VLEIG $0, $0, h2 \ // clear upper half of h2 - VESRLG $40, h2, h1 \ // h1 now has upper two bits of result - VLEIB $7, $88, h1 \ // for byte shift (11 bytes) - VSLB h1, h2, h2 \ // shift h2 11 bytes to the left - VO h0, h2, h0 \ // combine h0 with 20 bits from limb 1 - VLEIG $0, $0, h1 \ // clear upper half of h1 - -// if h > 2**130-5 then h -= 2**130-5 -// input: h0, h1 -// temp: t0, t1, t2 -// output: h0 -#define MOD(h0, h1, t0, t1, t2) \ - VZERO t0 \ - VLEIG $1, $5, t0 \ - VACCQ h0, t0, t1 \ - VAQ h0, t0, t0 \ - VONE t2 \ - VLEIG $1, $-4, t2 \ - VAQ t2, t1, t1 \ - VACCQ h1, t1, t1 \ - VONE t2 \ - VAQ t2, t1, t1 \ - VN h0, t1, t2 \ - VNC t0, t1, t1 \ - VO t1, t2, h0 \ - -// func poly1305vmsl(out *[16]byte, m *byte, mlen uint64, key *[32]key) -TEXT ·poly1305vmsl(SB), $0-32 - // This code processes 6 + up to 4 blocks (32 bytes) per iteration - // using the algorithm described in: - // NEON crypto, Daniel J. Bernstein & Peter Schwabe - // https://cryptojedi.org/papers/neoncrypto-20120320.pdf - // And as moddified for VMSL as described in - // Accelerating Poly1305 Cryptographic Message Authentication on the z14 - // O'Farrell et al, CASCON 2017, p48-55 - // https://ibm.ent.box.com/s/jf9gedj0e9d2vjctfyh186shaztavnht - - LMG out+0(FP), R1, R4 // R1=out, R2=m, R3=mlen, R4=key - VZERO V0 // c - - // load EX0, EX1 and EX2 - MOVD $·constants<>(SB), R5 - VLM (R5), EX0, EX2 // c - - // setup r - VL (R4), T_0 - MOVD $·keyMask<>(SB), R6 - VL (R6), T_1 - VN T_0, T_1, T_0 - VZERO T_2 // limbs for r - VZERO T_3 - VZERO T_4 - EXPACC2(T_0, T_2, T_3, T_4, T_1, T_5, T_7) - - // T_2, T_3, T_4: [0, r] - - // setup r*20 - VLEIG $0, $0, T_0 - VLEIG $1, $20, T_0 // T_0: [0, 20] - VZERO T_5 - VZERO T_6 - VMSLG T_0, T_3, T_5, T_5 - VMSLG T_0, T_4, T_6, T_6 - - // store r for final block in GR - VLGVG $1, T_2, RSAVE_0 // c - VLGVG $1, T_3, RSAVE_1 // c - VLGVG $1, T_4, RSAVE_2 // c - VLGVG $1, T_5, R5SAVE_1 // c - VLGVG $1, T_6, R5SAVE_2 // c - - // initialize h - VZERO H0_0 - VZERO H1_0 - VZERO H2_0 - VZERO H0_1 - VZERO H1_1 - VZERO H2_1 - - // initialize pointer for reduce constants - MOVD $·reduce<>(SB), R12 - - // calculate r**2 and 20*(r**2) - VZERO R_0 - VZERO R_1 - VZERO R_2 - SQUARE(T_2, T_3, T_4, T_6, R_0, R_1, R_2, T_1, T_5, T_7) - REDUCE2(R_0, R_1, R_2, M0, M1, M2, M3, M4, R5_1, R5_2, M5, T_1) - VZERO R5_1 - VZERO R5_2 - VMSLG T_0, R_1, R5_1, R5_1 - VMSLG T_0, R_2, R5_2, R5_2 - - // skip r**4 calculation if 3 blocks or less - CMPBLE R3, $48, b4 - - // calculate r**4 and 20*(r**4) - VZERO T_8 - VZERO T_9 - VZERO T_10 - SQUARE(R_0, R_1, R_2, R5_2, T_8, T_9, T_10, T_1, T_5, T_7) - REDUCE2(T_8, T_9, T_10, M0, M1, M2, M3, M4, T_2, T_3, M5, T_1) - VZERO T_2 - VZERO T_3 - VMSLG T_0, T_9, T_2, T_2 - VMSLG T_0, T_10, T_3, T_3 - - // put r**2 to the right and r**4 to the left of R_0, R_1, R_2 - VSLDB $8, T_8, T_8, T_8 - VSLDB $8, T_9, T_9, T_9 - VSLDB $8, T_10, T_10, T_10 - VSLDB $8, T_2, T_2, T_2 - VSLDB $8, T_3, T_3, T_3 - - VO T_8, R_0, R_0 - VO T_9, R_1, R_1 - VO T_10, R_2, R_2 - VO T_2, R5_1, R5_1 - VO T_3, R5_2, R5_2 - - CMPBLE R3, $80, load // less than or equal to 5 blocks in message - - // 6(or 5+1) blocks - SUB $81, R3 - VLM (R2), M0, M4 - VLL R3, 80(R2), M5 - ADD $1, R3 - MOVBZ $1, R0 - CMPBGE R3, $16, 2(PC) - VLVGB R3, R0, M5 - MOVD $96(R2), R2 - EXPACC(M0, M1, H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_0, T_1, T_2, T_3) - EXPACC(M2, M3, H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_0, T_1, T_2, T_3) - VLEIB $2, $1, H2_0 - VLEIB $2, $1, H2_1 - VLEIB $10, $1, H2_0 - VLEIB $10, $1, H2_1 - - VZERO M0 - VZERO M1 - VZERO M2 - VZERO M3 - VZERO T_4 - VZERO T_10 - EXPACC(M4, M5, M0, M1, M2, M3, T_4, T_10, T_0, T_1, T_2, T_3) - VLR T_4, M4 - VLEIB $10, $1, M2 - CMPBLT R3, $16, 2(PC) - VLEIB $10, $1, T_10 - MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, T_10, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9) - REDUCE(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_10, M0, M1, M2, M3, M4, T_4, T_5, T_2, T_7, T_8, T_9) - VMRHG V0, H0_1, H0_0 - VMRHG V0, H1_1, H1_0 - VMRHG V0, H2_1, H2_0 - VMRLG V0, H0_1, H0_1 - VMRLG V0, H1_1, H1_1 - VMRLG V0, H2_1, H2_1 - - SUB $16, R3 - CMPBLE R3, $0, square - -load: - // load EX0, EX1 and EX2 - MOVD $·c<>(SB), R5 - VLM (R5), EX0, EX2 - -loop: - CMPBLE R3, $64, add // b4 // last 4 or less blocks left - - // next 4 full blocks - VLM (R2), M2, M5 - SUB $64, R3 - MOVD $64(R2), R2 - REDUCE(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_10, M0, M1, T_0, T_1, T_3, T_4, T_5, T_2, T_7, T_8, T_9) - - // expacc in-lined to create [m2, m3] limbs - VGBM $0x3f3f, T_0 // 44 bit clear mask - VGBM $0x1f1f, T_1 // 40 bit clear mask - VPERM M2, M3, EX0, T_3 - VESRLG $4, T_0, T_0 // 44 bit clear mask ready - VPERM M2, M3, EX1, T_4 - VPERM M2, M3, EX2, T_5 - VN T_0, T_3, T_3 - VESRLG $4, T_4, T_4 - VN T_1, T_5, T_5 - VN T_0, T_4, T_4 - VMRHG H0_1, T_3, H0_0 - VMRHG H1_1, T_4, H1_0 - VMRHG H2_1, T_5, H2_0 - VMRLG H0_1, T_3, H0_1 - VMRLG H1_1, T_4, H1_1 - VMRLG H2_1, T_5, H2_1 - VLEIB $10, $1, H2_0 - VLEIB $10, $1, H2_1 - VPERM M4, M5, EX0, T_3 - VPERM M4, M5, EX1, T_4 - VPERM M4, M5, EX2, T_5 - VN T_0, T_3, T_3 - VESRLG $4, T_4, T_4 - VN T_1, T_5, T_5 - VN T_0, T_4, T_4 - VMRHG V0, T_3, M0 - VMRHG V0, T_4, M1 - VMRHG V0, T_5, M2 - VMRLG V0, T_3, M3 - VMRLG V0, T_4, M4 - VMRLG V0, T_5, M5 - VLEIB $10, $1, M2 - VLEIB $10, $1, M5 - - MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, M5, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9) - CMPBNE R3, $0, loop - REDUCE(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_10, M0, M1, M3, M4, M5, T_4, T_5, T_2, T_7, T_8, T_9) - VMRHG V0, H0_1, H0_0 - VMRHG V0, H1_1, H1_0 - VMRHG V0, H2_1, H2_0 - VMRLG V0, H0_1, H0_1 - VMRLG V0, H1_1, H1_1 - VMRLG V0, H2_1, H2_1 - - // load EX0, EX1, EX2 - MOVD $·constants<>(SB), R5 - VLM (R5), EX0, EX2 - - // sum vectors - VAQ H0_0, H0_1, H0_0 - VAQ H1_0, H1_1, H1_0 - VAQ H2_0, H2_1, H2_0 - - // h may be >= 2*(2**130-5) so we need to reduce it again - // M0...M4 are used as temps here - REDUCE2(H0_0, H1_0, H2_0, M0, M1, M2, M3, M4, T_9, T_10, H0_1, M5) - -next: // carry h1->h2 - VLEIB $7, $0x28, T_1 - VREPIB $4, T_2 - VGBM $0x003F, T_3 - VESRLG $4, T_3 - - // byte shift - VSRLB T_1, H1_0, T_4 - - // bit shift - VSRL T_2, T_4, T_4 - - // clear h1 carry bits - VN T_3, H1_0, H1_0 - - // add carry - VAQ T_4, H2_0, H2_0 - - // h is now < 2*(2**130-5) - // pack h into h1 (hi) and h0 (lo) - PACK(H0_0, H1_0, H2_0) - - // if h > 2**130-5 then h -= 2**130-5 - MOD(H0_0, H1_0, T_0, T_1, T_2) - - // h += s - MOVD $·bswapMask<>(SB), R5 - VL (R5), T_1 - VL 16(R4), T_0 - VPERM T_0, T_0, T_1, T_0 // reverse bytes (to big) - VAQ T_0, H0_0, H0_0 - VPERM H0_0, H0_0, T_1, H0_0 // reverse bytes (to little) - VST H0_0, (R1) - RET - -add: - // load EX0, EX1, EX2 - MOVD $·constants<>(SB), R5 - VLM (R5), EX0, EX2 - - REDUCE(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_10, M0, M1, M3, M4, M5, T_4, T_5, T_2, T_7, T_8, T_9) - VMRHG V0, H0_1, H0_0 - VMRHG V0, H1_1, H1_0 - VMRHG V0, H2_1, H2_0 - VMRLG V0, H0_1, H0_1 - VMRLG V0, H1_1, H1_1 - VMRLG V0, H2_1, H2_1 - CMPBLE R3, $64, b4 - -b4: - CMPBLE R3, $48, b3 // 3 blocks or less - - // 4(3+1) blocks remaining - SUB $49, R3 - VLM (R2), M0, M2 - VLL R3, 48(R2), M3 - ADD $1, R3 - MOVBZ $1, R0 - CMPBEQ R3, $16, 2(PC) - VLVGB R3, R0, M3 - MOVD $64(R2), R2 - EXPACC(M0, M1, H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_0, T_1, T_2, T_3) - VLEIB $10, $1, H2_0 - VLEIB $10, $1, H2_1 - VZERO M0 - VZERO M1 - VZERO M4 - VZERO M5 - VZERO T_4 - VZERO T_10 - EXPACC(M2, M3, M0, M1, M4, M5, T_4, T_10, T_0, T_1, T_2, T_3) - VLR T_4, M2 - VLEIB $10, $1, M4 - CMPBNE R3, $16, 2(PC) - VLEIB $10, $1, T_10 - MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M4, M5, M2, T_10, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9) - REDUCE(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_10, M0, M1, M3, M4, M5, T_4, T_5, T_2, T_7, T_8, T_9) - VMRHG V0, H0_1, H0_0 - VMRHG V0, H1_1, H1_0 - VMRHG V0, H2_1, H2_0 - VMRLG V0, H0_1, H0_1 - VMRLG V0, H1_1, H1_1 - VMRLG V0, H2_1, H2_1 - SUB $16, R3 - CMPBLE R3, $0, square // this condition must always hold true! - -b3: - CMPBLE R3, $32, b2 - - // 3 blocks remaining - - // setup [r²,r] - VSLDB $8, R_0, R_0, R_0 - VSLDB $8, R_1, R_1, R_1 - VSLDB $8, R_2, R_2, R_2 - VSLDB $8, R5_1, R5_1, R5_1 - VSLDB $8, R5_2, R5_2, R5_2 - - VLVGG $1, RSAVE_0, R_0 - VLVGG $1, RSAVE_1, R_1 - VLVGG $1, RSAVE_2, R_2 - VLVGG $1, R5SAVE_1, R5_1 - VLVGG $1, R5SAVE_2, R5_2 - - // setup [h0, h1] - VSLDB $8, H0_0, H0_0, H0_0 - VSLDB $8, H1_0, H1_0, H1_0 - VSLDB $8, H2_0, H2_0, H2_0 - VO H0_1, H0_0, H0_0 - VO H1_1, H1_0, H1_0 - VO H2_1, H2_0, H2_0 - VZERO H0_1 - VZERO H1_1 - VZERO H2_1 - - VZERO M0 - VZERO M1 - VZERO M2 - VZERO M3 - VZERO M4 - VZERO M5 - - // H*[r**2, r] - MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, M5, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9) - REDUCE2(H0_0, H1_0, H2_0, M0, M1, M2, M3, M4, H0_1, H1_1, T_10, M5) - - SUB $33, R3 - VLM (R2), M0, M1 - VLL R3, 32(R2), M2 - ADD $1, R3 - MOVBZ $1, R0 - CMPBEQ R3, $16, 2(PC) - VLVGB R3, R0, M2 - - // H += m0 - VZERO T_1 - VZERO T_2 - VZERO T_3 - EXPACC2(M0, T_1, T_2, T_3, T_4, T_5, T_6) - VLEIB $10, $1, T_3 - VAG H0_0, T_1, H0_0 - VAG H1_0, T_2, H1_0 - VAG H2_0, T_3, H2_0 - - VZERO M0 - VZERO M3 - VZERO M4 - VZERO M5 - VZERO T_10 - - // (H+m0)*r - MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M3, M4, M5, V0, T_10, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9) - REDUCE2(H0_0, H1_0, H2_0, M0, M3, M4, M5, T_10, H0_1, H1_1, H2_1, T_9) - - // H += m1 - VZERO V0 - VZERO T_1 - VZERO T_2 - VZERO T_3 - EXPACC2(M1, T_1, T_2, T_3, T_4, T_5, T_6) - VLEIB $10, $1, T_3 - VAQ H0_0, T_1, H0_0 - VAQ H1_0, T_2, H1_0 - VAQ H2_0, T_3, H2_0 - REDUCE2(H0_0, H1_0, H2_0, M0, M3, M4, M5, T_9, H0_1, H1_1, H2_1, T_10) - - // [H, m2] * [r**2, r] - EXPACC2(M2, H0_0, H1_0, H2_0, T_1, T_2, T_3) - CMPBNE R3, $16, 2(PC) - VLEIB $10, $1, H2_0 - VZERO M0 - VZERO M1 - VZERO M2 - VZERO M3 - VZERO M4 - VZERO M5 - MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, M5, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9) - REDUCE2(H0_0, H1_0, H2_0, M0, M1, M2, M3, M4, H0_1, H1_1, M5, T_10) - SUB $16, R3 - CMPBLE R3, $0, next // this condition must always hold true! - -b2: - CMPBLE R3, $16, b1 - - // 2 blocks remaining - - // setup [r²,r] - VSLDB $8, R_0, R_0, R_0 - VSLDB $8, R_1, R_1, R_1 - VSLDB $8, R_2, R_2, R_2 - VSLDB $8, R5_1, R5_1, R5_1 - VSLDB $8, R5_2, R5_2, R5_2 - - VLVGG $1, RSAVE_0, R_0 - VLVGG $1, RSAVE_1, R_1 - VLVGG $1, RSAVE_2, R_2 - VLVGG $1, R5SAVE_1, R5_1 - VLVGG $1, R5SAVE_2, R5_2 - - // setup [h0, h1] - VSLDB $8, H0_0, H0_0, H0_0 - VSLDB $8, H1_0, H1_0, H1_0 - VSLDB $8, H2_0, H2_0, H2_0 - VO H0_1, H0_0, H0_0 - VO H1_1, H1_0, H1_0 - VO H2_1, H2_0, H2_0 - VZERO H0_1 - VZERO H1_1 - VZERO H2_1 - - VZERO M0 - VZERO M1 - VZERO M2 - VZERO M3 - VZERO M4 - VZERO M5 - - // H*[r**2, r] - MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, M5, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9) - REDUCE(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, T_10, M0, M1, M2, M3, M4, T_4, T_5, T_2, T_7, T_8, T_9) - VMRHG V0, H0_1, H0_0 - VMRHG V0, H1_1, H1_0 - VMRHG V0, H2_1, H2_0 - VMRLG V0, H0_1, H0_1 - VMRLG V0, H1_1, H1_1 - VMRLG V0, H2_1, H2_1 - - // move h to the left and 0s at the right - VSLDB $8, H0_0, H0_0, H0_0 - VSLDB $8, H1_0, H1_0, H1_0 - VSLDB $8, H2_0, H2_0, H2_0 - - // get message blocks and append 1 to start - SUB $17, R3 - VL (R2), M0 - VLL R3, 16(R2), M1 - ADD $1, R3 - MOVBZ $1, R0 - CMPBEQ R3, $16, 2(PC) - VLVGB R3, R0, M1 - VZERO T_6 - VZERO T_7 - VZERO T_8 - EXPACC2(M0, T_6, T_7, T_8, T_1, T_2, T_3) - EXPACC2(M1, T_6, T_7, T_8, T_1, T_2, T_3) - VLEIB $2, $1, T_8 - CMPBNE R3, $16, 2(PC) - VLEIB $10, $1, T_8 - - // add [m0, m1] to h - VAG H0_0, T_6, H0_0 - VAG H1_0, T_7, H1_0 - VAG H2_0, T_8, H2_0 - - VZERO M2 - VZERO M3 - VZERO M4 - VZERO M5 - VZERO T_10 - VZERO M0 - - // at this point R_0 .. R5_2 look like [r**2, r] - MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M2, M3, M4, M5, T_10, M0, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9) - REDUCE2(H0_0, H1_0, H2_0, M2, M3, M4, M5, T_9, H0_1, H1_1, H2_1, T_10) - SUB $16, R3, R3 - CMPBLE R3, $0, next - -b1: - CMPBLE R3, $0, next - - // 1 block remaining - - // setup [r²,r] - VSLDB $8, R_0, R_0, R_0 - VSLDB $8, R_1, R_1, R_1 - VSLDB $8, R_2, R_2, R_2 - VSLDB $8, R5_1, R5_1, R5_1 - VSLDB $8, R5_2, R5_2, R5_2 - - VLVGG $1, RSAVE_0, R_0 - VLVGG $1, RSAVE_1, R_1 - VLVGG $1, RSAVE_2, R_2 - VLVGG $1, R5SAVE_1, R5_1 - VLVGG $1, R5SAVE_2, R5_2 - - // setup [h0, h1] - VSLDB $8, H0_0, H0_0, H0_0 - VSLDB $8, H1_0, H1_0, H1_0 - VSLDB $8, H2_0, H2_0, H2_0 - VO H0_1, H0_0, H0_0 - VO H1_1, H1_0, H1_0 - VO H2_1, H2_0, H2_0 - VZERO H0_1 - VZERO H1_1 - VZERO H2_1 - - VZERO M0 - VZERO M1 - VZERO M2 - VZERO M3 - VZERO M4 - VZERO M5 - - // H*[r**2, r] - MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, M5, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9) - REDUCE2(H0_0, H1_0, H2_0, M0, M1, M2, M3, M4, T_9, T_10, H0_1, M5) - - // set up [0, m0] limbs - SUB $1, R3 - VLL R3, (R2), M0 - ADD $1, R3 - MOVBZ $1, R0 - CMPBEQ R3, $16, 2(PC) - VLVGB R3, R0, M0 - VZERO T_1 - VZERO T_2 - VZERO T_3 - EXPACC2(M0, T_1, T_2, T_3, T_4, T_5, T_6)// limbs: [0, m] - CMPBNE R3, $16, 2(PC) - VLEIB $10, $1, T_3 - - // h+m0 - VAQ H0_0, T_1, H0_0 - VAQ H1_0, T_2, H1_0 - VAQ H2_0, T_3, H2_0 - - VZERO M0 - VZERO M1 - VZERO M2 - VZERO M3 - VZERO M4 - VZERO M5 - MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, M5, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9) - REDUCE2(H0_0, H1_0, H2_0, M0, M1, M2, M3, M4, T_9, T_10, H0_1, M5) - - BR next - -square: - // setup [r²,r] - VSLDB $8, R_0, R_0, R_0 - VSLDB $8, R_1, R_1, R_1 - VSLDB $8, R_2, R_2, R_2 - VSLDB $8, R5_1, R5_1, R5_1 - VSLDB $8, R5_2, R5_2, R5_2 - - VLVGG $1, RSAVE_0, R_0 - VLVGG $1, RSAVE_1, R_1 - VLVGG $1, RSAVE_2, R_2 - VLVGG $1, R5SAVE_1, R5_1 - VLVGG $1, R5SAVE_2, R5_2 - - // setup [h0, h1] - VSLDB $8, H0_0, H0_0, H0_0 - VSLDB $8, H1_0, H1_0, H1_0 - VSLDB $8, H2_0, H2_0, H2_0 - VO H0_1, H0_0, H0_0 - VO H1_1, H1_0, H1_0 - VO H2_1, H2_0, H2_0 - VZERO H0_1 - VZERO H1_1 - VZERO H2_1 - - VZERO M0 - VZERO M1 - VZERO M2 - VZERO M3 - VZERO M4 - VZERO M5 - - // (h0*r**2) + (h1*r) - MULTIPLY(H0_0, H1_0, H2_0, H0_1, H1_1, H2_1, R_0, R_1, R_2, R5_1, R5_2, M0, M1, M2, M3, M4, M5, T_0, T_1, T_2, T_3, T_4, T_5, T_6, T_7, T_8, T_9) - REDUCE2(H0_0, H1_0, H2_0, M0, M1, M2, M3, M4, T_9, T_10, H0_1, M5) - BR next diff --git a/golang.org/x/crypto/ssh/agent/client.go b/golang.org/x/crypto/ssh/agent/client.go index 51f740500e..b909471cc0 100644 --- a/golang.org/x/crypto/ssh/agent/client.go +++ b/golang.org/x/crypto/ssh/agent/client.go @@ -102,8 +102,9 @@ type ConstraintExtension struct { // AddedKey describes an SSH key to be added to an Agent. type AddedKey struct { - // PrivateKey must be a *rsa.PrivateKey, *dsa.PrivateKey or - // *ecdsa.PrivateKey, which will be inserted into the agent. + // PrivateKey must be a *rsa.PrivateKey, *dsa.PrivateKey, + // ed25519.PrivateKey or *ecdsa.PrivateKey, which will be inserted into the + // agent. PrivateKey interface{} // Certificate, if not nil, is communicated to the agent and will be // stored with the key. @@ -566,6 +567,17 @@ func (c *client) insertKey(s interface{}, comment string, constraints []byte) er Comments: comment, Constraints: constraints, }) + case ed25519.PrivateKey: + req = ssh.Marshal(ed25519KeyMsg{ + Type: ssh.KeyAlgoED25519, + Pub: []byte(k)[32:], + Priv: []byte(k), + Comments: comment, + Constraints: constraints, + }) + // This function originally supported only *ed25519.PrivateKey, however the + // general idiom is to pass ed25519.PrivateKey by value, not by pointer. + // We still support the pointer variant for backwards compatibility. case *ed25519.PrivateKey: req = ssh.Marshal(ed25519KeyMsg{ Type: ssh.KeyAlgoED25519, @@ -683,6 +695,18 @@ func (c *client) insertCert(s interface{}, cert *ssh.Certificate, comment string Comments: comment, Constraints: constraints, }) + case ed25519.PrivateKey: + req = ssh.Marshal(ed25519CertMsg{ + Type: cert.Type(), + CertBytes: cert.Marshal(), + Pub: []byte(k)[32:], + Priv: []byte(k), + Comments: comment, + Constraints: constraints, + }) + // This function originally supported only *ed25519.PrivateKey, however the + // general idiom is to pass ed25519.PrivateKey by value, not by pointer. + // We still support the pointer variant for backwards compatibility. case *ed25519.PrivateKey: req = ssh.Marshal(ed25519CertMsg{ Type: cert.Type(), diff --git a/golang.org/x/crypto/ssh/certs.go b/golang.org/x/crypto/ssh/certs.go index 0f89aec1c7..916c840b69 100644 --- a/golang.org/x/crypto/ssh/certs.go +++ b/golang.org/x/crypto/ssh/certs.go @@ -414,8 +414,8 @@ func (c *CertChecker) CheckCert(principal string, cert *Certificate) error { return nil } -// SignCert sets c.SignatureKey to the authority's public key and stores a -// Signature, by authority, in the certificate. +// SignCert signs the certificate with an authority, setting the Nonce, +// SignatureKey, and Signature fields. func (c *Certificate) SignCert(rand io.Reader, authority Signer) error { c.Nonce = make([]byte, 32) if _, err := io.ReadFull(rand, c.Nonce); err != nil { diff --git a/golang.org/x/crypto/ssh/cipher.go b/golang.org/x/crypto/ssh/cipher.go index b0204ee59f..8bd6b3daff 100644 --- a/golang.org/x/crypto/ssh/cipher.go +++ b/golang.org/x/crypto/ssh/cipher.go @@ -119,7 +119,7 @@ var cipherModes = map[string]*cipherMode{ chacha20Poly1305ID: {64, 0, newChaCha20Cipher}, // CBC mode is insecure and so is not included in the default config. - // (See http://www.isg.rhul.ac.uk/~kp/SandPfinal.pdf). If absolutely + // (See https://www.ieee-security.org/TC/SP2013/papers/4977a526.pdf). If absolutely // needed, it's possible to specify a custom Config to enable it. // You should expect that an active attacker can recover plaintext if // you do. diff --git a/golang.org/x/crypto/ssh/client_auth.go b/golang.org/x/crypto/ssh/client_auth.go index 0590070e22..f3265655ee 100644 --- a/golang.org/x/crypto/ssh/client_auth.go +++ b/golang.org/x/crypto/ssh/client_auth.go @@ -36,7 +36,7 @@ func (c *connection) clientAuthenticate(config *ClientConfig) error { // during the authentication phase the client first attempts the "none" method // then any untried methods suggested by the server. - tried := make(map[string]bool) + var tried []string var lastMethods []string sessionID := c.transport.getSessionID() @@ -49,7 +49,9 @@ func (c *connection) clientAuthenticate(config *ClientConfig) error { // success return nil } else if ok == authFailure { - tried[auth.method()] = true + if m := auth.method(); !contains(tried, m) { + tried = append(tried, m) + } } if methods == nil { methods = lastMethods @@ -61,7 +63,7 @@ func (c *connection) clientAuthenticate(config *ClientConfig) error { findNext: for _, a := range config.Auth { candidateMethod := a.method() - if tried[candidateMethod] { + if contains(tried, candidateMethod) { continue } for _, meth := range methods { @@ -72,16 +74,16 @@ func (c *connection) clientAuthenticate(config *ClientConfig) error { } } } - return fmt.Errorf("ssh: unable to authenticate, attempted methods %v, no supported methods remain", keys(tried)) + return fmt.Errorf("ssh: unable to authenticate, attempted methods %v, no supported methods remain", tried) } -func keys(m map[string]bool) []string { - s := make([]string, 0, len(m)) - - for key := range m { - s = append(s, key) +func contains(list []string, e string) bool { + for _, s := range list { + if s == e { + return true + } } - return s + return false } // An AuthMethod represents an instance of an RFC 4252 authentication method. diff --git a/golang.org/x/crypto/ssh/mux.go b/golang.org/x/crypto/ssh/mux.go index f19016270e..9654c01869 100644 --- a/golang.org/x/crypto/ssh/mux.go +++ b/golang.org/x/crypto/ssh/mux.go @@ -240,7 +240,7 @@ func (m *mux) onePacket() error { id := binary.BigEndian.Uint32(packet[1:]) ch := m.chanList.getChan(id) if ch == nil { - return fmt.Errorf("ssh: invalid channel %d", id) + return m.handleUnknownChannelPacket(id, packet) } return ch.handlePacket(packet) @@ -328,3 +328,24 @@ func (m *mux) openChannel(chanType string, extra []byte) (*channel, error) { return nil, fmt.Errorf("ssh: unexpected packet in response to channel open: %T", msg) } } + +func (m *mux) handleUnknownChannelPacket(id uint32, packet []byte) error { + msg, err := decode(packet) + if err != nil { + return err + } + + switch msg := msg.(type) { + // RFC 4254 section 5.4 says unrecognized channel requests should + // receive a failure response. + case *channelRequestMsg: + if msg.WantReply { + return m.sendMessage(channelRequestFailureMsg{ + PeersID: msg.PeersID, + }) + } + return nil + default: + return fmt.Errorf("ssh: invalid channel %d", id) + } +} diff --git a/golang.org/x/crypto/ssh/terminal/terminal.go b/golang.org/x/crypto/ssh/terminal/terminal.go index d1b4fca3a9..2ffb97bfb8 100644 --- a/golang.org/x/crypto/ssh/terminal/terminal.go +++ b/golang.org/x/crypto/ssh/terminal/terminal.go @@ -113,6 +113,7 @@ func NewTerminal(c io.ReadWriter, prompt string) *Terminal { } const ( + keyCtrlC = 3 keyCtrlD = 4 keyCtrlU = 21 keyEnter = '\r' @@ -151,8 +152,12 @@ func bytesToKey(b []byte, pasteActive bool) (rune, []byte) { switch b[0] { case 1: // ^A return keyHome, b[1:] + case 2: // ^B + return keyLeft, b[1:] case 5: // ^E return keyEnd, b[1:] + case 6: // ^F + return keyRight, b[1:] case 8: // ^H return keyBackspace, b[1:] case 11: // ^K @@ -738,6 +743,9 @@ func (t *Terminal) readLine() (line string, err error) { return "", io.EOF } } + if key == keyCtrlC { + return "", io.EOF + } if key == keyPasteStart { t.pasteActive = true if len(t.line) == 0 { diff --git a/golang.org/x/text/cases/tables11.0.0.go b/golang.org/x/text/cases/tables11.0.0.go index 21f06fdb8c..84d2701835 100644 --- a/golang.org/x/text/cases/tables11.0.0.go +++ b/golang.org/x/text/cases/tables11.0.0.go @@ -1,6 +1,6 @@ // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. -// +build go1.13 +// +build go1.13,!go1.14 package cases diff --git a/golang.org/x/text/cases/tables12.0.0.go b/golang.org/x/text/cases/tables12.0.0.go new file mode 100644 index 0000000000..875a21b4b7 --- /dev/null +++ b/golang.org/x/text/cases/tables12.0.0.go @@ -0,0 +1,2359 @@ +// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. + +// +build go1.14 + +package cases + +// UnicodeVersion is the Unicode version from which the tables in this package are derived. +const UnicodeVersion = "12.0.0" + +var xorData string = "" + // Size: 192 bytes + "\x00\x06\x07\x00\x01?\x00\x0f\x03\x00\x0f\x12\x00\x0f\x1f\x00\x0f\x1d" + + "\x00\x01\x13\x00\x0f\x16\x00\x0f\x0b\x00\x0f3\x00\x0f7\x00\x01#\x00\x0f?" + + "\x00\x0e'\x00\x0f/\x00\x0e>\x00\x0f*\x00\x0c&\x00\x0c*\x00\x0c;\x00\x0c9" + + "\x00\x0c%\x00\x01\x08\x00\x03\x0d\x00\x03\x09\x00\x02\x06\x00\x02\x02" + + "\x00\x02\x0c\x00\x01\x00\x00\x01\x03\x00\x01\x01\x00\x01 \x00\x01\x0c" + + "\x00\x01\x10\x00\x03\x10\x00\x036 \x00\x037 \x00\x0b#\x10\x00\x0b 0\x00" + + "\x0b!\x10\x00\x0b!0\x001\x00\x00\x0b(\x04\x00\x03\x04\x1e\x00\x0b)\x08" + + "\x00\x03\x0a\x00\x02:\x00\x02>\x00\x02,\x00\x02\x00\x00\x02\x10\x00\x01<" + + "\x00\x01&\x00\x01*\x00\x01.\x00\x010\x003 \x00\x01\x18\x00\x01(\x00\x01" + + "\x1e\x00\x01\x22" + +var exceptions string = "" + // Size: 2450 bytes + "\x00\x12\x12μΜΜ\x12\x12ssSSSs\x13\x18i̇i̇\x10\x09II\x13\x1bʼnʼNʼN\x11" + + "\x09sSS\x12\x12dždžDž\x12\x12dždžDŽ\x10\x12Ç„Ç…\x12\x12ljljLj\x12\x12ljljLJ\x10\x12LJLj" + + "\x12\x12njnjNj\x12\x12njnjNJ\x10\x12ÇŠÇ‹\x13\x1bjÌŒJÌŒJÌŒ\x12\x12dzdzDz\x12\x12dzdzDZ\x10" + + "\x12DZDz\x13\x18ⱥⱥ\x13\x18ⱦⱦ\x10\x1bⱾⱾ\x10\x1bⱿⱿ\x10\x1bⱯⱯ\x10\x1bâ±­â±­\x10" + + "\x1bâ±°â±°\x10\x1bêž«êž«\x10\x1bꞬꞬ\x10\x1bêžêž\x10\x1bꞪꞪ\x10\x1bꞮꞮ\x10\x1bⱢⱢ\x10" + + "\x1bêž­êž­\x10\x1bⱮⱮ\x10\x1bⱤⱤ\x10\x1bꟅꟅ\x10\x1bꞱꞱ\x10\x1bꞲꞲ\x10\x1bêž°êž°2\x12ι" + + "ΙΙ\x166ϊÌΪÌΪÌ\x166ϋÌΫÌΫÌ\x12\x12σΣΣ\x12\x12βΒΒ\x12\x12θΘΘ\x12\x12" + + "φΦΦ\x12\x12πΠΠ\x12\x12κΚΚ\x12\x12ÏΡΡ\x12\x12εΕΕ\x14$Õ¥Ö‚ÔµÕ’ÔµÖ‚\x10\x1bá²áƒ" + + "\x10\x1bᲑბ\x10\x1bᲒგ\x10\x1bᲓდ\x10\x1bᲔე\x10\x1bᲕვ\x10\x1bᲖზ\x10\x1bᲗთ" + + "\x10\x1bᲘი\x10\x1bᲙკ\x10\x1bᲚლ\x10\x1bᲛმ\x10\x1bᲜნ\x10\x1bá²áƒ\x10\x1bᲞპ" + + "\x10\x1bᲟჟ\x10\x1bᲠრ\x10\x1bᲡს\x10\x1bᲢტ\x10\x1bᲣუ\x10\x1bᲤფ\x10\x1bᲥქ" + + "\x10\x1bᲦღ\x10\x1bᲧყ\x10\x1bᲨშ\x10\x1bᲩჩ\x10\x1bᲪც\x10\x1bᲫძ\x10\x1bᲬწ" + + "\x10\x1bᲭჭ\x10\x1bᲮხ\x10\x1bᲯჯ\x10\x1bᲰჰ\x10\x1bᲱჱ\x10\x1bᲲჲ\x10\x1bᲳჳ" + + "\x10\x1bᲴჴ\x10\x1bᲵჵ\x10\x1bᲶჶ\x10\x1bᲷჷ\x10\x1bᲸჸ\x10\x1bᲹჹ\x10\x1bᲺჺ" + + "\x10\x1bᲽჽ\x10\x1bᲾჾ\x10\x1bᲿჿ\x12\x12вВВ\x12\x12дДД\x12\x12оОО\x12\x12Ñ" + + "СС\x12\x12тТТ\x12\x12тТТ\x12\x12ъЪЪ\x12\x12ѣѢѢ\x13\x1bꙋꙊꙊ\x13\x1bẖH̱H̱" + + "\x13\x1bẗT̈T̈\x13\x1bwÌŠWÌŠWÌŠ\x13\x1byÌŠYÌŠYÌŠ\x13\x1baʾAʾAʾ\x13\x1bṡṠṠ\x12" + + "\x10ssß\x14$ὐΥ̓Υ̓\x166ὒΥ̓̀Υ̓̀\x166Ï…Ì“ÌΥ̓ÌΥ̓Ì\x166ὖΥ̓͂Υ̓͂\x15+ἀιἈΙᾈ" + + "\x15+á¼Î¹á¼‰Î™á¾‰\x15+ἂιἊΙᾊ\x15+ἃιἋΙᾋ\x15+ἄιἌΙᾌ\x15+ἅιá¼Î™á¾\x15+ἆιἎΙᾎ\x15+ἇιá¼Î™á¾" + + "\x15\x1dἀιᾀἈΙ\x15\x1dá¼Î¹á¾á¼‰Î™\x15\x1dἂιᾂἊΙ\x15\x1dἃιᾃἋΙ\x15\x1dἄιᾄἌΙ\x15" + + "\x1dἅιᾅá¼Î™\x15\x1dἆιᾆἎΙ\x15\x1dἇιᾇá¼Î™\x15+ἠιἨΙᾘ\x15+ἡιἩΙᾙ\x15+ἢιἪΙᾚ\x15+ἣι" + + "ἫΙᾛ\x15+ἤιἬΙᾜ\x15+ἥιἭΙá¾\x15+ἦιἮΙᾞ\x15+ἧιἯΙᾟ\x15\x1dἠιá¾á¼¨Î™\x15\x1dἡιᾑἩΙ" + + "\x15\x1dἢιᾒἪΙ\x15\x1dἣιᾓἫΙ\x15\x1dἤιᾔἬΙ\x15\x1dἥιᾕἭΙ\x15\x1dἦιᾖἮΙ\x15" + + "\x1dἧιᾗἯΙ\x15+ὠιὨΙᾨ\x15+ὡιὩΙᾩ\x15+ὢιὪΙᾪ\x15+ὣιὫΙᾫ\x15+ὤιὬΙᾬ\x15+ὥιὭΙᾭ" + + "\x15+ὦιὮΙᾮ\x15+ὧιὯΙᾯ\x15\x1dὠιᾠὨΙ\x15\x1dὡιᾡὩΙ\x15\x1dὢιᾢὪΙ\x15\x1dὣιᾣὫΙ" + + "\x15\x1dὤιᾤὬΙ\x15\x1dὥιᾥὭΙ\x15\x1dὦιᾦὮΙ\x15\x1dὧιᾧὯΙ\x15-ὰιᾺΙᾺͅ\x14#αιΑΙ" + + "á¾¼\x14$άιΆΙΆͅ\x14$ᾶΑ͂Α͂\x166ᾶιΑ͂Ιᾼ͂\x14\x1cαιᾳΑΙ\x12\x12ιΙΙ\x15-ὴιῊΙ" + + "á¿ŠÍ…\x14#ηιΗΙῌ\x14$ήιΉΙΉͅ\x14$ῆΗ͂Η͂\x166ῆιΗ͂Ιῌ͂\x14\x1cηιῃΗΙ\x166ῒΙ" + + "̈̀Ϊ̀\x166ϊÌΪÌΪÌ\x14$ῖΙ͂Ι͂\x166ῗΪ͂Ϊ͂\x166ῢΫ̀Ϋ̀\x166ϋÌΫÌΫ" + + "Ì\x14$Ï̓Ρ̓Ρ̓\x14$ῦΥ͂Υ͂\x166ῧΫ͂Ϋ͂\x15-ὼιῺΙῺͅ\x14#ωιΩΙῼ\x14$ώιÎΙÎÍ…" + + "\x14$ῶΩ͂Ω͂\x166ῶιΩ͂Ιῼ͂\x14\x1cωιῳΩΙ\x12\x10ωω\x11\x08kk\x12\x10åå\x12" + + "\x10É«É«\x12\x10ɽɽ\x10\x12ȺȺ\x10\x12ȾȾ\x12\x10É‘É‘\x12\x10ɱɱ\x12\x10ÉÉ\x12" + + "\x10É’É’\x12\x10È¿È¿\x12\x10ɀɀ\x12\x10ɥɥ\x12\x10ɦɦ\x12\x10ɜɜ\x12\x10É¡É¡\x12" + + "\x10ɬɬ\x12\x10ɪɪ\x12\x10ʞʞ\x12\x10ʇʇ\x12\x10ÊÊ\x12\x10Ê‚Ê‚\x12\x12ffFFFf" + + "\x12\x12fiFIFi\x12\x12flFLFl\x13\x1bffiFFIFfi\x13\x1bfflFFLFfl\x12\x12st" + + "STSt\x12\x12stSTSt\x14$Õ´Õ¶Õ„Õ†Õ„Õ¶\x14$Õ´Õ¥Õ„ÔµÕ„Õ¥\x14$Õ´Õ«Õ„Ô»Õ„Õ«\x14$Õ¾Õ¶ÕŽÕ†ÕŽÕ¶\x14$Õ´Õ­Õ„Ô½Õ„" + + "Õ­" + +// lookup returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *caseTrie) lookup(s []byte) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return caseValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := caseIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := caseIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = caseIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := caseIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = caseIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = caseIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *caseTrie) lookupUnsafe(s []byte) uint16 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return caseValues[c0] + } + i := caseIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = caseIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = caseIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// lookupString returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *caseTrie) lookupString(s string) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return caseValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := caseIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := caseIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = caseIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := caseIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = caseIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = caseIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *caseTrie) lookupStringUnsafe(s string) uint16 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return caseValues[c0] + } + i := caseIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = caseIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = caseIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// caseTrie. Total size: 12396 bytes (12.11 KiB). Checksum: c0656238384c3da1. +type caseTrie struct{} + +func newCaseTrie(i int) *caseTrie { + return &caseTrie{} +} + +// lookupValue determines the type of block n and looks up the value for b. +func (t *caseTrie) lookupValue(n uint32, b byte) uint16 { + switch { + case n < 20: + return uint16(caseValues[n<<6+uint32(b)]) + default: + n -= 20 + return uint16(sparse.lookup(n, b)) + } +} + +// caseValues: 22 blocks, 1408 entries, 2816 bytes +// The third block is the zero block. +var caseValues = [1408]uint16{ + // Block 0x0, offset 0x0 + 0x27: 0x0054, + 0x2e: 0x0054, + 0x30: 0x0010, 0x31: 0x0010, 0x32: 0x0010, 0x33: 0x0010, 0x34: 0x0010, 0x35: 0x0010, + 0x36: 0x0010, 0x37: 0x0010, 0x38: 0x0010, 0x39: 0x0010, 0x3a: 0x0054, + // Block 0x1, offset 0x40 + 0x41: 0x2013, 0x42: 0x2013, 0x43: 0x2013, 0x44: 0x2013, 0x45: 0x2013, + 0x46: 0x2013, 0x47: 0x2013, 0x48: 0x2013, 0x49: 0x2013, 0x4a: 0x2013, 0x4b: 0x2013, + 0x4c: 0x2013, 0x4d: 0x2013, 0x4e: 0x2013, 0x4f: 0x2013, 0x50: 0x2013, 0x51: 0x2013, + 0x52: 0x2013, 0x53: 0x2013, 0x54: 0x2013, 0x55: 0x2013, 0x56: 0x2013, 0x57: 0x2013, + 0x58: 0x2013, 0x59: 0x2013, 0x5a: 0x2013, + 0x5e: 0x0004, 0x5f: 0x0010, 0x60: 0x0004, 0x61: 0x2012, 0x62: 0x2012, 0x63: 0x2012, + 0x64: 0x2012, 0x65: 0x2012, 0x66: 0x2012, 0x67: 0x2012, 0x68: 0x2012, 0x69: 0x2012, + 0x6a: 0x2012, 0x6b: 0x2012, 0x6c: 0x2012, 0x6d: 0x2012, 0x6e: 0x2012, 0x6f: 0x2012, + 0x70: 0x2012, 0x71: 0x2012, 0x72: 0x2012, 0x73: 0x2012, 0x74: 0x2012, 0x75: 0x2012, + 0x76: 0x2012, 0x77: 0x2012, 0x78: 0x2012, 0x79: 0x2012, 0x7a: 0x2012, + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xc0: 0x0852, 0xc1: 0x0b53, 0xc2: 0x0113, 0xc3: 0x0112, 0xc4: 0x0113, 0xc5: 0x0112, + 0xc6: 0x0b53, 0xc7: 0x0f13, 0xc8: 0x0f12, 0xc9: 0x0e53, 0xca: 0x1153, 0xcb: 0x0713, + 0xcc: 0x0712, 0xcd: 0x0012, 0xce: 0x1453, 0xcf: 0x1753, 0xd0: 0x1a53, 0xd1: 0x0313, + 0xd2: 0x0312, 0xd3: 0x1d53, 0xd4: 0x2053, 0xd5: 0x2352, 0xd6: 0x2653, 0xd7: 0x2653, + 0xd8: 0x0113, 0xd9: 0x0112, 0xda: 0x2952, 0xdb: 0x0012, 0xdc: 0x1d53, 0xdd: 0x2c53, + 0xde: 0x2f52, 0xdf: 0x3253, 0xe0: 0x0113, 0xe1: 0x0112, 0xe2: 0x0113, 0xe3: 0x0112, + 0xe4: 0x0113, 0xe5: 0x0112, 0xe6: 0x3553, 0xe7: 0x0f13, 0xe8: 0x0f12, 0xe9: 0x3853, + 0xea: 0x0012, 0xeb: 0x0012, 0xec: 0x0113, 0xed: 0x0112, 0xee: 0x3553, 0xef: 0x1f13, + 0xf0: 0x1f12, 0xf1: 0x3b53, 0xf2: 0x3e53, 0xf3: 0x0713, 0xf4: 0x0712, 0xf5: 0x0313, + 0xf6: 0x0312, 0xf7: 0x4153, 0xf8: 0x0113, 0xf9: 0x0112, 0xfa: 0x0012, 0xfb: 0x0010, + 0xfc: 0x0113, 0xfd: 0x0112, 0xfe: 0x0012, 0xff: 0x4452, + // Block 0x4, offset 0x100 + 0x100: 0x0010, 0x101: 0x0010, 0x102: 0x0010, 0x103: 0x0010, 0x104: 0x02db, 0x105: 0x0359, + 0x106: 0x03da, 0x107: 0x043b, 0x108: 0x04b9, 0x109: 0x053a, 0x10a: 0x059b, 0x10b: 0x0619, + 0x10c: 0x069a, 0x10d: 0x0313, 0x10e: 0x0312, 0x10f: 0x1f13, 0x110: 0x1f12, 0x111: 0x0313, + 0x112: 0x0312, 0x113: 0x0713, 0x114: 0x0712, 0x115: 0x0313, 0x116: 0x0312, 0x117: 0x0f13, + 0x118: 0x0f12, 0x119: 0x0313, 0x11a: 0x0312, 0x11b: 0x0713, 0x11c: 0x0712, 0x11d: 0x1452, + 0x11e: 0x0113, 0x11f: 0x0112, 0x120: 0x0113, 0x121: 0x0112, 0x122: 0x0113, 0x123: 0x0112, + 0x124: 0x0113, 0x125: 0x0112, 0x126: 0x0113, 0x127: 0x0112, 0x128: 0x0113, 0x129: 0x0112, + 0x12a: 0x0113, 0x12b: 0x0112, 0x12c: 0x0113, 0x12d: 0x0112, 0x12e: 0x0113, 0x12f: 0x0112, + 0x130: 0x06fa, 0x131: 0x07ab, 0x132: 0x0829, 0x133: 0x08aa, 0x134: 0x0113, 0x135: 0x0112, + 0x136: 0x2353, 0x137: 0x4453, 0x138: 0x0113, 0x139: 0x0112, 0x13a: 0x0113, 0x13b: 0x0112, + 0x13c: 0x0113, 0x13d: 0x0112, 0x13e: 0x0113, 0x13f: 0x0112, + // Block 0x5, offset 0x140 + 0x140: 0x0a8a, 0x141: 0x0313, 0x142: 0x0312, 0x143: 0x0853, 0x144: 0x4753, 0x145: 0x4a53, + 0x146: 0x0113, 0x147: 0x0112, 0x148: 0x0113, 0x149: 0x0112, 0x14a: 0x0113, 0x14b: 0x0112, + 0x14c: 0x0113, 0x14d: 0x0112, 0x14e: 0x0113, 0x14f: 0x0112, 0x150: 0x0b0a, 0x151: 0x0b8a, + 0x152: 0x0c0a, 0x153: 0x0b52, 0x154: 0x0b52, 0x155: 0x0012, 0x156: 0x0e52, 0x157: 0x1152, + 0x158: 0x0012, 0x159: 0x1752, 0x15a: 0x0012, 0x15b: 0x1a52, 0x15c: 0x0c8a, 0x15d: 0x0012, + 0x15e: 0x0012, 0x15f: 0x0012, 0x160: 0x1d52, 0x161: 0x0d0a, 0x162: 0x0012, 0x163: 0x2052, + 0x164: 0x0012, 0x165: 0x0d8a, 0x166: 0x0e0a, 0x167: 0x0012, 0x168: 0x2652, 0x169: 0x2652, + 0x16a: 0x0e8a, 0x16b: 0x0f0a, 0x16c: 0x0f8a, 0x16d: 0x0012, 0x16e: 0x0012, 0x16f: 0x1d52, + 0x170: 0x0012, 0x171: 0x100a, 0x172: 0x2c52, 0x173: 0x0012, 0x174: 0x0012, 0x175: 0x3252, + 0x176: 0x0012, 0x177: 0x0012, 0x178: 0x0012, 0x179: 0x0012, 0x17a: 0x0012, 0x17b: 0x0012, + 0x17c: 0x0012, 0x17d: 0x108a, 0x17e: 0x0012, 0x17f: 0x0012, + // Block 0x6, offset 0x180 + 0x180: 0x3552, 0x181: 0x0012, 0x182: 0x110a, 0x183: 0x3852, 0x184: 0x0012, 0x185: 0x0012, + 0x186: 0x0012, 0x187: 0x118a, 0x188: 0x3552, 0x189: 0x4752, 0x18a: 0x3b52, 0x18b: 0x3e52, + 0x18c: 0x4a52, 0x18d: 0x0012, 0x18e: 0x0012, 0x18f: 0x0012, 0x190: 0x0012, 0x191: 0x0012, + 0x192: 0x4152, 0x193: 0x0012, 0x194: 0x0010, 0x195: 0x0012, 0x196: 0x0012, 0x197: 0x0012, + 0x198: 0x0012, 0x199: 0x0012, 0x19a: 0x0012, 0x19b: 0x0012, 0x19c: 0x0012, 0x19d: 0x120a, + 0x19e: 0x128a, 0x19f: 0x0012, 0x1a0: 0x0012, 0x1a1: 0x0012, 0x1a2: 0x0012, 0x1a3: 0x0012, + 0x1a4: 0x0012, 0x1a5: 0x0012, 0x1a6: 0x0012, 0x1a7: 0x0012, 0x1a8: 0x0012, 0x1a9: 0x0012, + 0x1aa: 0x0012, 0x1ab: 0x0012, 0x1ac: 0x0012, 0x1ad: 0x0012, 0x1ae: 0x0012, 0x1af: 0x0012, + 0x1b0: 0x0015, 0x1b1: 0x0015, 0x1b2: 0x0015, 0x1b3: 0x0015, 0x1b4: 0x0015, 0x1b5: 0x0015, + 0x1b6: 0x0015, 0x1b7: 0x0015, 0x1b8: 0x0015, 0x1b9: 0x0014, 0x1ba: 0x0014, 0x1bb: 0x0014, + 0x1bc: 0x0014, 0x1bd: 0x0014, 0x1be: 0x0014, 0x1bf: 0x0014, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x0024, 0x1c1: 0x0024, 0x1c2: 0x0024, 0x1c3: 0x0024, 0x1c4: 0x0024, 0x1c5: 0x130d, + 0x1c6: 0x0024, 0x1c7: 0x0034, 0x1c8: 0x0034, 0x1c9: 0x0034, 0x1ca: 0x0024, 0x1cb: 0x0024, + 0x1cc: 0x0024, 0x1cd: 0x0034, 0x1ce: 0x0034, 0x1cf: 0x0014, 0x1d0: 0x0024, 0x1d1: 0x0024, + 0x1d2: 0x0024, 0x1d3: 0x0034, 0x1d4: 0x0034, 0x1d5: 0x0034, 0x1d6: 0x0034, 0x1d7: 0x0024, + 0x1d8: 0x0034, 0x1d9: 0x0034, 0x1da: 0x0034, 0x1db: 0x0024, 0x1dc: 0x0034, 0x1dd: 0x0034, + 0x1de: 0x0034, 0x1df: 0x0034, 0x1e0: 0x0034, 0x1e1: 0x0034, 0x1e2: 0x0034, 0x1e3: 0x0024, + 0x1e4: 0x0024, 0x1e5: 0x0024, 0x1e6: 0x0024, 0x1e7: 0x0024, 0x1e8: 0x0024, 0x1e9: 0x0024, + 0x1ea: 0x0024, 0x1eb: 0x0024, 0x1ec: 0x0024, 0x1ed: 0x0024, 0x1ee: 0x0024, 0x1ef: 0x0024, + 0x1f0: 0x0113, 0x1f1: 0x0112, 0x1f2: 0x0113, 0x1f3: 0x0112, 0x1f4: 0x0014, 0x1f5: 0x0004, + 0x1f6: 0x0113, 0x1f7: 0x0112, 0x1fa: 0x0015, 0x1fb: 0x4d52, + 0x1fc: 0x5052, 0x1fd: 0x5052, 0x1ff: 0x5353, + // Block 0x8, offset 0x200 + 0x204: 0x0004, 0x205: 0x0004, + 0x206: 0x2a13, 0x207: 0x0054, 0x208: 0x2513, 0x209: 0x2713, 0x20a: 0x2513, + 0x20c: 0x5653, 0x20e: 0x5953, 0x20f: 0x5c53, 0x210: 0x138a, 0x211: 0x2013, + 0x212: 0x2013, 0x213: 0x2013, 0x214: 0x2013, 0x215: 0x2013, 0x216: 0x2013, 0x217: 0x2013, + 0x218: 0x2013, 0x219: 0x2013, 0x21a: 0x2013, 0x21b: 0x2013, 0x21c: 0x2013, 0x21d: 0x2013, + 0x21e: 0x2013, 0x21f: 0x2013, 0x220: 0x5f53, 0x221: 0x5f53, 0x223: 0x5f53, + 0x224: 0x5f53, 0x225: 0x5f53, 0x226: 0x5f53, 0x227: 0x5f53, 0x228: 0x5f53, 0x229: 0x5f53, + 0x22a: 0x5f53, 0x22b: 0x5f53, 0x22c: 0x2a12, 0x22d: 0x2512, 0x22e: 0x2712, 0x22f: 0x2512, + 0x230: 0x14ca, 0x231: 0x2012, 0x232: 0x2012, 0x233: 0x2012, 0x234: 0x2012, 0x235: 0x2012, + 0x236: 0x2012, 0x237: 0x2012, 0x238: 0x2012, 0x239: 0x2012, 0x23a: 0x2012, 0x23b: 0x2012, + 0x23c: 0x2012, 0x23d: 0x2012, 0x23e: 0x2012, 0x23f: 0x2012, + // Block 0x9, offset 0x240 + 0x240: 0x5f52, 0x241: 0x5f52, 0x242: 0x160a, 0x243: 0x5f52, 0x244: 0x5f52, 0x245: 0x5f52, + 0x246: 0x5f52, 0x247: 0x5f52, 0x248: 0x5f52, 0x249: 0x5f52, 0x24a: 0x5f52, 0x24b: 0x5f52, + 0x24c: 0x5652, 0x24d: 0x5952, 0x24e: 0x5c52, 0x24f: 0x1813, 0x250: 0x168a, 0x251: 0x170a, + 0x252: 0x0013, 0x253: 0x0013, 0x254: 0x0013, 0x255: 0x178a, 0x256: 0x180a, 0x257: 0x1812, + 0x258: 0x0113, 0x259: 0x0112, 0x25a: 0x0113, 0x25b: 0x0112, 0x25c: 0x0113, 0x25d: 0x0112, + 0x25e: 0x0113, 0x25f: 0x0112, 0x260: 0x0113, 0x261: 0x0112, 0x262: 0x0113, 0x263: 0x0112, + 0x264: 0x0113, 0x265: 0x0112, 0x266: 0x0113, 0x267: 0x0112, 0x268: 0x0113, 0x269: 0x0112, + 0x26a: 0x0113, 0x26b: 0x0112, 0x26c: 0x0113, 0x26d: 0x0112, 0x26e: 0x0113, 0x26f: 0x0112, + 0x270: 0x188a, 0x271: 0x190a, 0x272: 0x0b12, 0x273: 0x5352, 0x274: 0x6253, 0x275: 0x198a, + 0x277: 0x0f13, 0x278: 0x0f12, 0x279: 0x0b13, 0x27a: 0x0113, 0x27b: 0x0112, + 0x27c: 0x0012, 0x27d: 0x4d53, 0x27e: 0x5053, 0x27f: 0x5053, + // Block 0xa, offset 0x280 + 0x280: 0x6852, 0x281: 0x6852, 0x282: 0x6852, 0x283: 0x6852, 0x284: 0x6852, 0x285: 0x6852, + 0x286: 0x6852, 0x287: 0x1a0a, 0x288: 0x0012, + 0x291: 0x0034, + 0x292: 0x0024, 0x293: 0x0024, 0x294: 0x0024, 0x295: 0x0024, 0x296: 0x0034, 0x297: 0x0024, + 0x298: 0x0024, 0x299: 0x0024, 0x29a: 0x0034, 0x29b: 0x0034, 0x29c: 0x0024, 0x29d: 0x0024, + 0x29e: 0x0024, 0x29f: 0x0024, 0x2a0: 0x0024, 0x2a1: 0x0024, 0x2a2: 0x0034, 0x2a3: 0x0034, + 0x2a4: 0x0034, 0x2a5: 0x0034, 0x2a6: 0x0034, 0x2a7: 0x0034, 0x2a8: 0x0024, 0x2a9: 0x0024, + 0x2aa: 0x0034, 0x2ab: 0x0024, 0x2ac: 0x0024, 0x2ad: 0x0034, 0x2ae: 0x0034, 0x2af: 0x0024, + 0x2b0: 0x0034, 0x2b1: 0x0034, 0x2b2: 0x0034, 0x2b3: 0x0034, 0x2b4: 0x0034, 0x2b5: 0x0034, + 0x2b6: 0x0034, 0x2b7: 0x0034, 0x2b8: 0x0034, 0x2b9: 0x0034, 0x2ba: 0x0034, 0x2bb: 0x0034, + 0x2bc: 0x0034, 0x2bd: 0x0034, 0x2bf: 0x0034, + // Block 0xb, offset 0x2c0 + 0x2c0: 0x7053, 0x2c1: 0x7053, 0x2c2: 0x7053, 0x2c3: 0x7053, 0x2c4: 0x7053, 0x2c5: 0x7053, + 0x2c7: 0x7053, + 0x2cd: 0x7053, 0x2d0: 0x1aea, 0x2d1: 0x1b6a, + 0x2d2: 0x1bea, 0x2d3: 0x1c6a, 0x2d4: 0x1cea, 0x2d5: 0x1d6a, 0x2d6: 0x1dea, 0x2d7: 0x1e6a, + 0x2d8: 0x1eea, 0x2d9: 0x1f6a, 0x2da: 0x1fea, 0x2db: 0x206a, 0x2dc: 0x20ea, 0x2dd: 0x216a, + 0x2de: 0x21ea, 0x2df: 0x226a, 0x2e0: 0x22ea, 0x2e1: 0x236a, 0x2e2: 0x23ea, 0x2e3: 0x246a, + 0x2e4: 0x24ea, 0x2e5: 0x256a, 0x2e6: 0x25ea, 0x2e7: 0x266a, 0x2e8: 0x26ea, 0x2e9: 0x276a, + 0x2ea: 0x27ea, 0x2eb: 0x286a, 0x2ec: 0x28ea, 0x2ed: 0x296a, 0x2ee: 0x29ea, 0x2ef: 0x2a6a, + 0x2f0: 0x2aea, 0x2f1: 0x2b6a, 0x2f2: 0x2bea, 0x2f3: 0x2c6a, 0x2f4: 0x2cea, 0x2f5: 0x2d6a, + 0x2f6: 0x2dea, 0x2f7: 0x2e6a, 0x2f8: 0x2eea, 0x2f9: 0x2f6a, 0x2fa: 0x2fea, + 0x2fc: 0x0014, 0x2fd: 0x306a, 0x2fe: 0x30ea, 0x2ff: 0x316a, + // Block 0xc, offset 0x300 + 0x300: 0x0812, 0x301: 0x0812, 0x302: 0x0812, 0x303: 0x0812, 0x304: 0x0812, 0x305: 0x0812, + 0x308: 0x0813, 0x309: 0x0813, 0x30a: 0x0813, 0x30b: 0x0813, + 0x30c: 0x0813, 0x30d: 0x0813, 0x310: 0x3b1a, 0x311: 0x0812, + 0x312: 0x3bfa, 0x313: 0x0812, 0x314: 0x3d3a, 0x315: 0x0812, 0x316: 0x3e7a, 0x317: 0x0812, + 0x319: 0x0813, 0x31b: 0x0813, 0x31d: 0x0813, + 0x31f: 0x0813, 0x320: 0x0812, 0x321: 0x0812, 0x322: 0x0812, 0x323: 0x0812, + 0x324: 0x0812, 0x325: 0x0812, 0x326: 0x0812, 0x327: 0x0812, 0x328: 0x0813, 0x329: 0x0813, + 0x32a: 0x0813, 0x32b: 0x0813, 0x32c: 0x0813, 0x32d: 0x0813, 0x32e: 0x0813, 0x32f: 0x0813, + 0x330: 0x9252, 0x331: 0x9252, 0x332: 0x9552, 0x333: 0x9552, 0x334: 0x9852, 0x335: 0x9852, + 0x336: 0x9b52, 0x337: 0x9b52, 0x338: 0x9e52, 0x339: 0x9e52, 0x33a: 0xa152, 0x33b: 0xa152, + 0x33c: 0x4d52, 0x33d: 0x4d52, + // Block 0xd, offset 0x340 + 0x340: 0x3fba, 0x341: 0x40aa, 0x342: 0x419a, 0x343: 0x428a, 0x344: 0x437a, 0x345: 0x446a, + 0x346: 0x455a, 0x347: 0x464a, 0x348: 0x4739, 0x349: 0x4829, 0x34a: 0x4919, 0x34b: 0x4a09, + 0x34c: 0x4af9, 0x34d: 0x4be9, 0x34e: 0x4cd9, 0x34f: 0x4dc9, 0x350: 0x4eba, 0x351: 0x4faa, + 0x352: 0x509a, 0x353: 0x518a, 0x354: 0x527a, 0x355: 0x536a, 0x356: 0x545a, 0x357: 0x554a, + 0x358: 0x5639, 0x359: 0x5729, 0x35a: 0x5819, 0x35b: 0x5909, 0x35c: 0x59f9, 0x35d: 0x5ae9, + 0x35e: 0x5bd9, 0x35f: 0x5cc9, 0x360: 0x5dba, 0x361: 0x5eaa, 0x362: 0x5f9a, 0x363: 0x608a, + 0x364: 0x617a, 0x365: 0x626a, 0x366: 0x635a, 0x367: 0x644a, 0x368: 0x6539, 0x369: 0x6629, + 0x36a: 0x6719, 0x36b: 0x6809, 0x36c: 0x68f9, 0x36d: 0x69e9, 0x36e: 0x6ad9, 0x36f: 0x6bc9, + 0x370: 0x0812, 0x371: 0x0812, 0x372: 0x6cba, 0x373: 0x6dca, 0x374: 0x6e9a, + 0x376: 0x6f7a, 0x377: 0x705a, 0x378: 0x0813, 0x379: 0x0813, 0x37a: 0x9253, 0x37b: 0x9253, + 0x37c: 0x7199, 0x37d: 0x0004, 0x37e: 0x726a, 0x37f: 0x0004, + // Block 0xe, offset 0x380 + 0x380: 0x0004, 0x381: 0x0004, 0x382: 0x72ea, 0x383: 0x73fa, 0x384: 0x74ca, + 0x386: 0x75aa, 0x387: 0x768a, 0x388: 0x9553, 0x389: 0x9553, 0x38a: 0x9853, 0x38b: 0x9853, + 0x38c: 0x77c9, 0x38d: 0x0004, 0x38e: 0x0004, 0x38f: 0x0004, 0x390: 0x0812, 0x391: 0x0812, + 0x392: 0x789a, 0x393: 0x79da, 0x396: 0x7b1a, 0x397: 0x7bfa, + 0x398: 0x0813, 0x399: 0x0813, 0x39a: 0x9b53, 0x39b: 0x9b53, 0x39d: 0x0004, + 0x39e: 0x0004, 0x39f: 0x0004, 0x3a0: 0x0812, 0x3a1: 0x0812, 0x3a2: 0x7d3a, 0x3a3: 0x7e7a, + 0x3a4: 0x7fba, 0x3a5: 0x0912, 0x3a6: 0x809a, 0x3a7: 0x817a, 0x3a8: 0x0813, 0x3a9: 0x0813, + 0x3aa: 0xa153, 0x3ab: 0xa153, 0x3ac: 0x0913, 0x3ad: 0x0004, 0x3ae: 0x0004, 0x3af: 0x0004, + 0x3b2: 0x82ba, 0x3b3: 0x83ca, 0x3b4: 0x849a, + 0x3b6: 0x857a, 0x3b7: 0x865a, 0x3b8: 0x9e53, 0x3b9: 0x9e53, 0x3ba: 0x4d53, 0x3bb: 0x4d53, + 0x3bc: 0x8799, 0x3bd: 0x0004, 0x3be: 0x0004, + // Block 0xf, offset 0x3c0 + 0x3c2: 0x0013, + 0x3c7: 0x0013, 0x3ca: 0x0012, 0x3cb: 0x0013, + 0x3cc: 0x0013, 0x3cd: 0x0013, 0x3ce: 0x0012, 0x3cf: 0x0012, 0x3d0: 0x0013, 0x3d1: 0x0013, + 0x3d2: 0x0013, 0x3d3: 0x0012, 0x3d5: 0x0013, + 0x3d9: 0x0013, 0x3da: 0x0013, 0x3db: 0x0013, 0x3dc: 0x0013, 0x3dd: 0x0013, + 0x3e4: 0x0013, 0x3e6: 0x886b, 0x3e8: 0x0013, + 0x3ea: 0x88cb, 0x3eb: 0x890b, 0x3ec: 0x0013, 0x3ed: 0x0013, 0x3ef: 0x0012, + 0x3f0: 0x0013, 0x3f1: 0x0013, 0x3f2: 0xa453, 0x3f3: 0x0013, 0x3f4: 0x0012, 0x3f5: 0x0010, + 0x3f6: 0x0010, 0x3f7: 0x0010, 0x3f8: 0x0010, 0x3f9: 0x0012, + 0x3fc: 0x0012, 0x3fd: 0x0012, 0x3fe: 0x0013, 0x3ff: 0x0013, + // Block 0x10, offset 0x400 + 0x400: 0x1a13, 0x401: 0x1a13, 0x402: 0x1e13, 0x403: 0x1e13, 0x404: 0x1a13, 0x405: 0x1a13, + 0x406: 0x2613, 0x407: 0x2613, 0x408: 0x2a13, 0x409: 0x2a13, 0x40a: 0x2e13, 0x40b: 0x2e13, + 0x40c: 0x2a13, 0x40d: 0x2a13, 0x40e: 0x2613, 0x40f: 0x2613, 0x410: 0xa752, 0x411: 0xa752, + 0x412: 0xaa52, 0x413: 0xaa52, 0x414: 0xad52, 0x415: 0xad52, 0x416: 0xaa52, 0x417: 0xaa52, + 0x418: 0xa752, 0x419: 0xa752, 0x41a: 0x1a12, 0x41b: 0x1a12, 0x41c: 0x1e12, 0x41d: 0x1e12, + 0x41e: 0x1a12, 0x41f: 0x1a12, 0x420: 0x2612, 0x421: 0x2612, 0x422: 0x2a12, 0x423: 0x2a12, + 0x424: 0x2e12, 0x425: 0x2e12, 0x426: 0x2a12, 0x427: 0x2a12, 0x428: 0x2612, 0x429: 0x2612, + // Block 0x11, offset 0x440 + 0x440: 0x6552, 0x441: 0x6552, 0x442: 0x6552, 0x443: 0x6552, 0x444: 0x6552, 0x445: 0x6552, + 0x446: 0x6552, 0x447: 0x6552, 0x448: 0x6552, 0x449: 0x6552, 0x44a: 0x6552, 0x44b: 0x6552, + 0x44c: 0x6552, 0x44d: 0x6552, 0x44e: 0x6552, 0x44f: 0x6552, 0x450: 0xb052, 0x451: 0xb052, + 0x452: 0xb052, 0x453: 0xb052, 0x454: 0xb052, 0x455: 0xb052, 0x456: 0xb052, 0x457: 0xb052, + 0x458: 0xb052, 0x459: 0xb052, 0x45a: 0xb052, 0x45b: 0xb052, 0x45c: 0xb052, 0x45d: 0xb052, + 0x45e: 0xb052, 0x460: 0x0113, 0x461: 0x0112, 0x462: 0x896b, 0x463: 0x8b53, + 0x464: 0x89cb, 0x465: 0x8a2a, 0x466: 0x8a8a, 0x467: 0x0f13, 0x468: 0x0f12, 0x469: 0x0313, + 0x46a: 0x0312, 0x46b: 0x0713, 0x46c: 0x0712, 0x46d: 0x8aeb, 0x46e: 0x8b4b, 0x46f: 0x8bab, + 0x470: 0x8c0b, 0x471: 0x0012, 0x472: 0x0113, 0x473: 0x0112, 0x474: 0x0012, 0x475: 0x0313, + 0x476: 0x0312, 0x477: 0x0012, 0x478: 0x0012, 0x479: 0x0012, 0x47a: 0x0012, 0x47b: 0x0012, + 0x47c: 0x0015, 0x47d: 0x0015, 0x47e: 0x8c6b, 0x47f: 0x8ccb, + // Block 0x12, offset 0x480 + 0x480: 0x0113, 0x481: 0x0112, 0x482: 0x0113, 0x483: 0x0112, 0x484: 0x0113, 0x485: 0x0112, + 0x486: 0x0113, 0x487: 0x0112, 0x488: 0x0014, 0x489: 0x0014, 0x48a: 0x0014, 0x48b: 0x0713, + 0x48c: 0x0712, 0x48d: 0x8d2b, 0x48e: 0x0012, 0x48f: 0x0010, 0x490: 0x0113, 0x491: 0x0112, + 0x492: 0x0113, 0x493: 0x0112, 0x494: 0x6552, 0x495: 0x0012, 0x496: 0x0113, 0x497: 0x0112, + 0x498: 0x0113, 0x499: 0x0112, 0x49a: 0x0113, 0x49b: 0x0112, 0x49c: 0x0113, 0x49d: 0x0112, + 0x49e: 0x0113, 0x49f: 0x0112, 0x4a0: 0x0113, 0x4a1: 0x0112, 0x4a2: 0x0113, 0x4a3: 0x0112, + 0x4a4: 0x0113, 0x4a5: 0x0112, 0x4a6: 0x0113, 0x4a7: 0x0112, 0x4a8: 0x0113, 0x4a9: 0x0112, + 0x4aa: 0x8d8b, 0x4ab: 0x8deb, 0x4ac: 0x8e4b, 0x4ad: 0x8eab, 0x4ae: 0x8f0b, 0x4af: 0x0012, + 0x4b0: 0x8f6b, 0x4b1: 0x8fcb, 0x4b2: 0x902b, 0x4b3: 0xb353, 0x4b4: 0x0113, 0x4b5: 0x0112, + 0x4b6: 0x0113, 0x4b7: 0x0112, 0x4b8: 0x0113, 0x4b9: 0x0112, 0x4ba: 0x0113, 0x4bb: 0x0112, + 0x4bc: 0x0113, 0x4bd: 0x0112, 0x4be: 0x0113, 0x4bf: 0x0112, + // Block 0x13, offset 0x4c0 + 0x4c0: 0x90ea, 0x4c1: 0x916a, 0x4c2: 0x91ea, 0x4c3: 0x926a, 0x4c4: 0x931a, 0x4c5: 0x93ca, + 0x4c6: 0x944a, + 0x4d3: 0x94ca, 0x4d4: 0x95aa, 0x4d5: 0x968a, 0x4d6: 0x976a, 0x4d7: 0x984a, + 0x4dd: 0x0010, + 0x4de: 0x0034, 0x4df: 0x0010, 0x4e0: 0x0010, 0x4e1: 0x0010, 0x4e2: 0x0010, 0x4e3: 0x0010, + 0x4e4: 0x0010, 0x4e5: 0x0010, 0x4e6: 0x0010, 0x4e7: 0x0010, 0x4e8: 0x0010, + 0x4ea: 0x0010, 0x4eb: 0x0010, 0x4ec: 0x0010, 0x4ed: 0x0010, 0x4ee: 0x0010, 0x4ef: 0x0010, + 0x4f0: 0x0010, 0x4f1: 0x0010, 0x4f2: 0x0010, 0x4f3: 0x0010, 0x4f4: 0x0010, 0x4f5: 0x0010, + 0x4f6: 0x0010, 0x4f8: 0x0010, 0x4f9: 0x0010, 0x4fa: 0x0010, 0x4fb: 0x0010, + 0x4fc: 0x0010, 0x4fe: 0x0010, + // Block 0x14, offset 0x500 + 0x500: 0x2213, 0x501: 0x2213, 0x502: 0x2613, 0x503: 0x2613, 0x504: 0x2213, 0x505: 0x2213, + 0x506: 0x2e13, 0x507: 0x2e13, 0x508: 0x2213, 0x509: 0x2213, 0x50a: 0x2613, 0x50b: 0x2613, + 0x50c: 0x2213, 0x50d: 0x2213, 0x50e: 0x3e13, 0x50f: 0x3e13, 0x510: 0x2213, 0x511: 0x2213, + 0x512: 0x2613, 0x513: 0x2613, 0x514: 0x2213, 0x515: 0x2213, 0x516: 0x2e13, 0x517: 0x2e13, + 0x518: 0x2213, 0x519: 0x2213, 0x51a: 0x2613, 0x51b: 0x2613, 0x51c: 0x2213, 0x51d: 0x2213, + 0x51e: 0xbc53, 0x51f: 0xbc53, 0x520: 0xbf53, 0x521: 0xbf53, 0x522: 0x2212, 0x523: 0x2212, + 0x524: 0x2612, 0x525: 0x2612, 0x526: 0x2212, 0x527: 0x2212, 0x528: 0x2e12, 0x529: 0x2e12, + 0x52a: 0x2212, 0x52b: 0x2212, 0x52c: 0x2612, 0x52d: 0x2612, 0x52e: 0x2212, 0x52f: 0x2212, + 0x530: 0x3e12, 0x531: 0x3e12, 0x532: 0x2212, 0x533: 0x2212, 0x534: 0x2612, 0x535: 0x2612, + 0x536: 0x2212, 0x537: 0x2212, 0x538: 0x2e12, 0x539: 0x2e12, 0x53a: 0x2212, 0x53b: 0x2212, + 0x53c: 0x2612, 0x53d: 0x2612, 0x53e: 0x2212, 0x53f: 0x2212, + // Block 0x15, offset 0x540 + 0x542: 0x0010, + 0x547: 0x0010, 0x549: 0x0010, 0x54b: 0x0010, + 0x54d: 0x0010, 0x54e: 0x0010, 0x54f: 0x0010, 0x551: 0x0010, + 0x552: 0x0010, 0x554: 0x0010, 0x557: 0x0010, + 0x559: 0x0010, 0x55b: 0x0010, 0x55d: 0x0010, + 0x55f: 0x0010, 0x561: 0x0010, 0x562: 0x0010, + 0x564: 0x0010, 0x567: 0x0010, 0x568: 0x0010, 0x569: 0x0010, + 0x56a: 0x0010, 0x56c: 0x0010, 0x56d: 0x0010, 0x56e: 0x0010, 0x56f: 0x0010, + 0x570: 0x0010, 0x571: 0x0010, 0x572: 0x0010, 0x574: 0x0010, 0x575: 0x0010, + 0x576: 0x0010, 0x577: 0x0010, 0x579: 0x0010, 0x57a: 0x0010, 0x57b: 0x0010, + 0x57c: 0x0010, 0x57e: 0x0010, +} + +// caseIndex: 25 blocks, 1600 entries, 3200 bytes +// Block 0 is the zero block. +var caseIndex = [1600]uint16{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xc2: 0x14, 0xc3: 0x15, 0xc4: 0x16, 0xc5: 0x17, 0xc6: 0x01, 0xc7: 0x02, + 0xc8: 0x18, 0xc9: 0x03, 0xca: 0x04, 0xcb: 0x19, 0xcc: 0x1a, 0xcd: 0x05, 0xce: 0x06, 0xcf: 0x07, + 0xd0: 0x1b, 0xd1: 0x1c, 0xd2: 0x1d, 0xd3: 0x1e, 0xd4: 0x1f, 0xd5: 0x20, 0xd6: 0x08, 0xd7: 0x21, + 0xd8: 0x22, 0xd9: 0x23, 0xda: 0x24, 0xdb: 0x25, 0xdc: 0x26, 0xdd: 0x27, 0xde: 0x28, 0xdf: 0x29, + 0xe0: 0x02, 0xe1: 0x03, 0xe2: 0x04, 0xe3: 0x05, + 0xea: 0x06, 0xeb: 0x07, 0xec: 0x07, 0xed: 0x08, 0xef: 0x09, + 0xf0: 0x14, 0xf3: 0x16, + // Block 0x4, offset 0x100 + 0x120: 0x2a, 0x121: 0x2b, 0x122: 0x2c, 0x123: 0x2d, 0x124: 0x2e, 0x125: 0x2f, 0x126: 0x30, 0x127: 0x31, + 0x128: 0x32, 0x129: 0x33, 0x12a: 0x34, 0x12b: 0x35, 0x12c: 0x36, 0x12d: 0x37, 0x12e: 0x38, 0x12f: 0x39, + 0x130: 0x3a, 0x131: 0x3b, 0x132: 0x3c, 0x133: 0x3d, 0x134: 0x3e, 0x135: 0x3f, 0x136: 0x40, 0x137: 0x41, + 0x138: 0x42, 0x139: 0x43, 0x13a: 0x44, 0x13b: 0x45, 0x13c: 0x46, 0x13d: 0x47, 0x13e: 0x48, 0x13f: 0x49, + // Block 0x5, offset 0x140 + 0x140: 0x4a, 0x141: 0x4b, 0x142: 0x4c, 0x143: 0x09, 0x144: 0x24, 0x145: 0x24, 0x146: 0x24, 0x147: 0x24, + 0x148: 0x24, 0x149: 0x4d, 0x14a: 0x4e, 0x14b: 0x4f, 0x14c: 0x50, 0x14d: 0x51, 0x14e: 0x52, 0x14f: 0x53, + 0x150: 0x54, 0x151: 0x24, 0x152: 0x24, 0x153: 0x24, 0x154: 0x24, 0x155: 0x24, 0x156: 0x24, 0x157: 0x24, + 0x158: 0x24, 0x159: 0x55, 0x15a: 0x56, 0x15b: 0x57, 0x15c: 0x58, 0x15d: 0x59, 0x15e: 0x5a, 0x15f: 0x5b, + 0x160: 0x5c, 0x161: 0x5d, 0x162: 0x5e, 0x163: 0x5f, 0x164: 0x60, 0x165: 0x61, 0x167: 0x62, + 0x168: 0x63, 0x169: 0x64, 0x16a: 0x65, 0x16c: 0x66, 0x16d: 0x67, 0x16e: 0x68, 0x16f: 0x69, + 0x170: 0x6a, 0x171: 0x6b, 0x172: 0x6c, 0x173: 0x6d, 0x174: 0x6e, 0x175: 0x6f, 0x176: 0x70, 0x177: 0x71, + 0x178: 0x72, 0x179: 0x72, 0x17a: 0x73, 0x17b: 0x72, 0x17c: 0x74, 0x17d: 0x0a, 0x17e: 0x0b, 0x17f: 0x0c, + // Block 0x6, offset 0x180 + 0x180: 0x75, 0x181: 0x76, 0x182: 0x77, 0x183: 0x78, 0x184: 0x0d, 0x185: 0x79, 0x186: 0x7a, + 0x192: 0x7b, 0x193: 0x0e, + 0x1b0: 0x7c, 0x1b1: 0x0f, 0x1b2: 0x72, 0x1b3: 0x7d, 0x1b4: 0x7e, 0x1b5: 0x7f, 0x1b6: 0x80, 0x1b7: 0x81, + 0x1b8: 0x82, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x83, 0x1c2: 0x84, 0x1c3: 0x85, 0x1c4: 0x86, 0x1c5: 0x24, 0x1c6: 0x87, + // Block 0x8, offset 0x200 + 0x200: 0x88, 0x201: 0x24, 0x202: 0x24, 0x203: 0x24, 0x204: 0x24, 0x205: 0x24, 0x206: 0x24, 0x207: 0x24, + 0x208: 0x24, 0x209: 0x24, 0x20a: 0x24, 0x20b: 0x24, 0x20c: 0x24, 0x20d: 0x24, 0x20e: 0x24, 0x20f: 0x24, + 0x210: 0x24, 0x211: 0x24, 0x212: 0x89, 0x213: 0x8a, 0x214: 0x24, 0x215: 0x24, 0x216: 0x24, 0x217: 0x24, + 0x218: 0x8b, 0x219: 0x8c, 0x21a: 0x8d, 0x21b: 0x8e, 0x21c: 0x8f, 0x21d: 0x90, 0x21e: 0x10, 0x21f: 0x91, + 0x220: 0x92, 0x221: 0x93, 0x222: 0x24, 0x223: 0x94, 0x224: 0x95, 0x225: 0x96, 0x226: 0x97, 0x227: 0x98, + 0x228: 0x99, 0x229: 0x9a, 0x22a: 0x9b, 0x22b: 0x9c, 0x22c: 0x9d, 0x22d: 0x9e, 0x22e: 0x9f, 0x22f: 0xa0, + 0x230: 0x24, 0x231: 0x24, 0x232: 0x24, 0x233: 0x24, 0x234: 0x24, 0x235: 0x24, 0x236: 0x24, 0x237: 0x24, + 0x238: 0x24, 0x239: 0x24, 0x23a: 0x24, 0x23b: 0x24, 0x23c: 0x24, 0x23d: 0x24, 0x23e: 0x24, 0x23f: 0x24, + // Block 0x9, offset 0x240 + 0x240: 0x24, 0x241: 0x24, 0x242: 0x24, 0x243: 0x24, 0x244: 0x24, 0x245: 0x24, 0x246: 0x24, 0x247: 0x24, + 0x248: 0x24, 0x249: 0x24, 0x24a: 0x24, 0x24b: 0x24, 0x24c: 0x24, 0x24d: 0x24, 0x24e: 0x24, 0x24f: 0x24, + 0x250: 0x24, 0x251: 0x24, 0x252: 0x24, 0x253: 0x24, 0x254: 0x24, 0x255: 0x24, 0x256: 0x24, 0x257: 0x24, + 0x258: 0x24, 0x259: 0x24, 0x25a: 0x24, 0x25b: 0x24, 0x25c: 0x24, 0x25d: 0x24, 0x25e: 0x24, 0x25f: 0x24, + 0x260: 0x24, 0x261: 0x24, 0x262: 0x24, 0x263: 0x24, 0x264: 0x24, 0x265: 0x24, 0x266: 0x24, 0x267: 0x24, + 0x268: 0x24, 0x269: 0x24, 0x26a: 0x24, 0x26b: 0x24, 0x26c: 0x24, 0x26d: 0x24, 0x26e: 0x24, 0x26f: 0x24, + 0x270: 0x24, 0x271: 0x24, 0x272: 0x24, 0x273: 0x24, 0x274: 0x24, 0x275: 0x24, 0x276: 0x24, 0x277: 0x24, + 0x278: 0x24, 0x279: 0x24, 0x27a: 0x24, 0x27b: 0x24, 0x27c: 0x24, 0x27d: 0x24, 0x27e: 0x24, 0x27f: 0x24, + // Block 0xa, offset 0x280 + 0x280: 0x24, 0x281: 0x24, 0x282: 0x24, 0x283: 0x24, 0x284: 0x24, 0x285: 0x24, 0x286: 0x24, 0x287: 0x24, + 0x288: 0x24, 0x289: 0x24, 0x28a: 0x24, 0x28b: 0x24, 0x28c: 0x24, 0x28d: 0x24, 0x28e: 0x24, 0x28f: 0x24, + 0x290: 0x24, 0x291: 0x24, 0x292: 0x24, 0x293: 0x24, 0x294: 0x24, 0x295: 0x24, 0x296: 0x24, 0x297: 0x24, + 0x298: 0x24, 0x299: 0x24, 0x29a: 0x24, 0x29b: 0x24, 0x29c: 0x24, 0x29d: 0x24, 0x29e: 0xa1, 0x29f: 0xa2, + // Block 0xb, offset 0x2c0 + 0x2ec: 0x11, 0x2ed: 0xa3, 0x2ee: 0xa4, 0x2ef: 0xa5, + 0x2f0: 0x24, 0x2f1: 0x24, 0x2f2: 0x24, 0x2f3: 0x24, 0x2f4: 0xa6, 0x2f5: 0xa7, 0x2f6: 0xa8, 0x2f7: 0xa9, + 0x2f8: 0xaa, 0x2f9: 0xab, 0x2fa: 0x24, 0x2fb: 0xac, 0x2fc: 0xad, 0x2fd: 0xae, 0x2fe: 0xaf, 0x2ff: 0xb0, + // Block 0xc, offset 0x300 + 0x300: 0xb1, 0x301: 0xb2, 0x302: 0x24, 0x303: 0xb3, 0x305: 0xb4, 0x307: 0xb5, + 0x30a: 0xb6, 0x30b: 0xb7, 0x30c: 0xb8, 0x30d: 0xb9, 0x30e: 0xba, 0x30f: 0xbb, + 0x310: 0xbc, 0x311: 0xbd, 0x312: 0xbe, 0x313: 0xbf, 0x314: 0xc0, 0x315: 0xc1, + 0x318: 0x24, 0x319: 0x24, 0x31a: 0x24, 0x31b: 0x24, 0x31c: 0xc2, 0x31d: 0xc3, + 0x320: 0xc4, 0x321: 0xc5, 0x322: 0xc6, 0x323: 0xc7, 0x324: 0xc8, 0x326: 0xc9, + 0x328: 0xca, 0x329: 0xcb, 0x32a: 0xcc, 0x32b: 0xcd, 0x32c: 0x5f, 0x32d: 0xce, 0x32e: 0xcf, + 0x330: 0x24, 0x331: 0xd0, 0x332: 0xd1, 0x333: 0xd2, 0x334: 0xd3, + 0x33c: 0xd4, 0x33d: 0xd5, 0x33f: 0xd6, + // Block 0xd, offset 0x340 + 0x340: 0xd7, 0x341: 0xd8, 0x342: 0xd9, 0x343: 0xda, 0x344: 0xdb, 0x345: 0xdc, 0x346: 0xdd, 0x347: 0xde, + 0x348: 0xdf, 0x34a: 0xe0, 0x34b: 0xe1, 0x34c: 0xe2, 0x34d: 0xe3, + 0x350: 0xe4, 0x351: 0xe5, 0x352: 0xe6, 0x353: 0xe7, 0x356: 0xe8, 0x357: 0xe9, + 0x358: 0xea, 0x359: 0xeb, 0x35a: 0xec, 0x35b: 0xed, 0x35c: 0xee, + 0x360: 0xef, 0x362: 0xf0, 0x363: 0xf1, 0x366: 0xf2, 0x367: 0xf3, + 0x368: 0xf4, 0x369: 0xf5, 0x36a: 0xf6, 0x36b: 0xf7, + 0x370: 0xf8, 0x371: 0xf9, 0x372: 0xfa, 0x374: 0xfb, 0x375: 0xfc, 0x376: 0xfd, + 0x37b: 0xfe, + // Block 0xe, offset 0x380 + 0x380: 0x24, 0x381: 0x24, 0x382: 0x24, 0x383: 0x24, 0x384: 0x24, 0x385: 0x24, 0x386: 0x24, 0x387: 0x24, + 0x388: 0x24, 0x389: 0x24, 0x38a: 0x24, 0x38b: 0x24, 0x38c: 0x24, 0x38d: 0x24, 0x38e: 0xff, + 0x390: 0x24, 0x391: 0x100, 0x392: 0x24, 0x393: 0x24, 0x394: 0x24, 0x395: 0x101, + // Block 0xf, offset 0x3c0 + 0x3c0: 0x24, 0x3c1: 0x24, 0x3c2: 0x24, 0x3c3: 0x24, 0x3c4: 0x24, 0x3c5: 0x24, 0x3c6: 0x24, 0x3c7: 0x24, + 0x3c8: 0x24, 0x3c9: 0x24, 0x3ca: 0x24, 0x3cb: 0x24, 0x3cc: 0x24, 0x3cd: 0x24, 0x3ce: 0x24, 0x3cf: 0x24, + 0x3d0: 0x102, + // Block 0x10, offset 0x400 + 0x410: 0x24, 0x411: 0x24, 0x412: 0x24, 0x413: 0x24, 0x414: 0x24, 0x415: 0x24, 0x416: 0x24, 0x417: 0x24, + 0x418: 0x24, 0x419: 0x103, + // Block 0x11, offset 0x440 + 0x460: 0x24, 0x461: 0x24, 0x462: 0x24, 0x463: 0x24, 0x464: 0x24, 0x465: 0x24, 0x466: 0x24, 0x467: 0x24, + 0x468: 0xf7, 0x469: 0x104, 0x46b: 0x105, 0x46c: 0x106, 0x46d: 0x107, 0x46e: 0x108, + 0x479: 0x109, 0x47c: 0x24, 0x47d: 0x10a, 0x47e: 0x10b, 0x47f: 0x10c, + // Block 0x12, offset 0x480 + 0x4b0: 0x24, 0x4b1: 0x10d, 0x4b2: 0x10e, + // Block 0x13, offset 0x4c0 + 0x4c5: 0x10f, 0x4c6: 0x110, + 0x4c9: 0x111, + 0x4d0: 0x112, 0x4d1: 0x113, 0x4d2: 0x114, 0x4d3: 0x115, 0x4d4: 0x116, 0x4d5: 0x117, 0x4d6: 0x118, 0x4d7: 0x119, + 0x4d8: 0x11a, 0x4d9: 0x11b, 0x4da: 0x11c, 0x4db: 0x11d, 0x4dc: 0x11e, 0x4dd: 0x11f, 0x4de: 0x120, 0x4df: 0x121, + 0x4e8: 0x122, 0x4e9: 0x123, 0x4ea: 0x124, + // Block 0x14, offset 0x500 + 0x500: 0x125, 0x504: 0x126, 0x505: 0x127, + 0x50b: 0x128, + 0x520: 0x24, 0x521: 0x24, 0x522: 0x24, 0x523: 0x129, 0x524: 0x12, 0x525: 0x12a, + 0x538: 0x12b, 0x539: 0x13, 0x53a: 0x12c, + // Block 0x15, offset 0x540 + 0x544: 0x12d, 0x545: 0x12e, 0x546: 0x12f, + 0x54f: 0x130, + // Block 0x16, offset 0x580 + 0x590: 0x0a, 0x591: 0x0b, 0x592: 0x0c, 0x593: 0x0d, 0x594: 0x0e, 0x596: 0x0f, + 0x59b: 0x10, 0x59d: 0x11, 0x59e: 0x12, 0x59f: 0x13, + // Block 0x17, offset 0x5c0 + 0x5c0: 0x131, 0x5c1: 0x132, 0x5c4: 0x132, 0x5c5: 0x132, 0x5c6: 0x132, 0x5c7: 0x133, + // Block 0x18, offset 0x600 + 0x620: 0x15, +} + +// sparseOffsets: 289 entries, 578 bytes +var sparseOffsets = []uint16{0x0, 0x9, 0xf, 0x18, 0x24, 0x2e, 0x35, 0x38, 0x3c, 0x3f, 0x43, 0x4d, 0x4f, 0x57, 0x5e, 0x63, 0x71, 0x72, 0x80, 0x8f, 0x99, 0x9c, 0xa3, 0xab, 0xae, 0xb0, 0xbf, 0xc5, 0xd3, 0xde, 0xeb, 0xf6, 0x102, 0x10c, 0x118, 0x123, 0x12f, 0x13b, 0x143, 0x14c, 0x156, 0x161, 0x16d, 0x174, 0x17f, 0x184, 0x18c, 0x18f, 0x194, 0x198, 0x19c, 0x1a3, 0x1ac, 0x1b4, 0x1b5, 0x1be, 0x1c5, 0x1cd, 0x1d3, 0x1d8, 0x1dc, 0x1df, 0x1e1, 0x1e4, 0x1e9, 0x1ea, 0x1ec, 0x1ee, 0x1f0, 0x1f7, 0x1fc, 0x200, 0x209, 0x20c, 0x20f, 0x215, 0x216, 0x221, 0x222, 0x223, 0x228, 0x235, 0x23d, 0x245, 0x24e, 0x257, 0x260, 0x265, 0x268, 0x273, 0x281, 0x283, 0x28a, 0x28e, 0x29a, 0x29b, 0x2a6, 0x2ae, 0x2b6, 0x2bc, 0x2bd, 0x2cb, 0x2d0, 0x2d3, 0x2d8, 0x2dc, 0x2e2, 0x2e7, 0x2ea, 0x2ef, 0x2f4, 0x2f5, 0x2fb, 0x2fd, 0x2fe, 0x300, 0x302, 0x305, 0x306, 0x308, 0x30b, 0x311, 0x315, 0x317, 0x31c, 0x323, 0x32b, 0x334, 0x335, 0x33e, 0x342, 0x347, 0x34f, 0x355, 0x35b, 0x365, 0x36a, 0x373, 0x379, 0x380, 0x384, 0x38c, 0x38e, 0x390, 0x393, 0x395, 0x397, 0x398, 0x399, 0x39b, 0x39d, 0x3a3, 0x3a8, 0x3aa, 0x3b1, 0x3b4, 0x3b6, 0x3bc, 0x3c1, 0x3c3, 0x3c4, 0x3c5, 0x3c6, 0x3c8, 0x3ca, 0x3cc, 0x3cf, 0x3d1, 0x3d4, 0x3dc, 0x3df, 0x3e3, 0x3eb, 0x3ed, 0x3ee, 0x3ef, 0x3f1, 0x3f7, 0x3f9, 0x3fa, 0x3fc, 0x3fe, 0x400, 0x40d, 0x40e, 0x40f, 0x413, 0x415, 0x416, 0x417, 0x418, 0x419, 0x41c, 0x41f, 0x425, 0x426, 0x42a, 0x42e, 0x434, 0x437, 0x43e, 0x442, 0x446, 0x44d, 0x456, 0x45c, 0x462, 0x46c, 0x476, 0x478, 0x481, 0x487, 0x48d, 0x493, 0x496, 0x49c, 0x49f, 0x4a8, 0x4a9, 0x4b0, 0x4b4, 0x4b5, 0x4b8, 0x4ba, 0x4c1, 0x4c9, 0x4cf, 0x4d5, 0x4d6, 0x4dc, 0x4df, 0x4e7, 0x4ee, 0x4f8, 0x500, 0x503, 0x504, 0x505, 0x506, 0x508, 0x509, 0x50b, 0x50d, 0x50f, 0x513, 0x514, 0x516, 0x519, 0x51b, 0x51d, 0x51f, 0x524, 0x529, 0x52d, 0x52e, 0x531, 0x535, 0x540, 0x544, 0x54c, 0x551, 0x555, 0x558, 0x55c, 0x55f, 0x562, 0x567, 0x56b, 0x56f, 0x573, 0x577, 0x579, 0x57b, 0x57e, 0x583, 0x586, 0x588, 0x58b, 0x58d, 0x593, 0x59c, 0x5a1, 0x5a2, 0x5a5, 0x5a6, 0x5a7, 0x5a9, 0x5aa, 0x5ab} + +// sparseValues: 1451 entries, 5804 bytes +var sparseValues = [1451]valueRange{ + // Block 0x0, offset 0x0 + {value: 0x0004, lo: 0xa8, hi: 0xa8}, + {value: 0x0012, lo: 0xaa, hi: 0xaa}, + {value: 0x0014, lo: 0xad, hi: 0xad}, + {value: 0x0004, lo: 0xaf, hi: 0xaf}, + {value: 0x0004, lo: 0xb4, hi: 0xb4}, + {value: 0x001a, lo: 0xb5, hi: 0xb5}, + {value: 0x0054, lo: 0xb7, hi: 0xb7}, + {value: 0x0004, lo: 0xb8, hi: 0xb8}, + {value: 0x0012, lo: 0xba, hi: 0xba}, + // Block 0x1, offset 0x9 + {value: 0x2013, lo: 0x80, hi: 0x96}, + {value: 0x2013, lo: 0x98, hi: 0x9e}, + {value: 0x009a, lo: 0x9f, hi: 0x9f}, + {value: 0x2012, lo: 0xa0, hi: 0xb6}, + {value: 0x2012, lo: 0xb8, hi: 0xbe}, + {value: 0x0252, lo: 0xbf, hi: 0xbf}, + // Block 0x2, offset 0xf + {value: 0x0117, lo: 0x80, hi: 0xaf}, + {value: 0x011b, lo: 0xb0, hi: 0xb0}, + {value: 0x019a, lo: 0xb1, hi: 0xb1}, + {value: 0x0117, lo: 0xb2, hi: 0xb7}, + {value: 0x0012, lo: 0xb8, hi: 0xb8}, + {value: 0x0316, lo: 0xb9, hi: 0xba}, + {value: 0x0716, lo: 0xbb, hi: 0xbc}, + {value: 0x0316, lo: 0xbd, hi: 0xbe}, + {value: 0x0553, lo: 0xbf, hi: 0xbf}, + // Block 0x3, offset 0x18 + {value: 0x0552, lo: 0x80, hi: 0x80}, + {value: 0x0316, lo: 0x81, hi: 0x82}, + {value: 0x0716, lo: 0x83, hi: 0x84}, + {value: 0x0316, lo: 0x85, hi: 0x86}, + {value: 0x0f16, lo: 0x87, hi: 0x88}, + {value: 0x01da, lo: 0x89, hi: 0x89}, + {value: 0x0117, lo: 0x8a, hi: 0xb7}, + {value: 0x0253, lo: 0xb8, hi: 0xb8}, + {value: 0x0316, lo: 0xb9, hi: 0xba}, + {value: 0x0716, lo: 0xbb, hi: 0xbc}, + {value: 0x0316, lo: 0xbd, hi: 0xbe}, + {value: 0x028a, lo: 0xbf, hi: 0xbf}, + // Block 0x4, offset 0x24 + {value: 0x0117, lo: 0x80, hi: 0x9f}, + {value: 0x2f53, lo: 0xa0, hi: 0xa0}, + {value: 0x0012, lo: 0xa1, hi: 0xa1}, + {value: 0x0117, lo: 0xa2, hi: 0xb3}, + {value: 0x0012, lo: 0xb4, hi: 0xb9}, + {value: 0x090b, lo: 0xba, hi: 0xba}, + {value: 0x0716, lo: 0xbb, hi: 0xbc}, + {value: 0x2953, lo: 0xbd, hi: 0xbd}, + {value: 0x098b, lo: 0xbe, hi: 0xbe}, + {value: 0x0a0a, lo: 0xbf, hi: 0xbf}, + // Block 0x5, offset 0x2e + {value: 0x0015, lo: 0x80, hi: 0x81}, + {value: 0x0014, lo: 0x82, hi: 0x97}, + {value: 0x0004, lo: 0x98, hi: 0x9d}, + {value: 0x0014, lo: 0x9e, hi: 0x9f}, + {value: 0x0015, lo: 0xa0, hi: 0xa4}, + {value: 0x0004, lo: 0xa5, hi: 0xab}, + {value: 0x0014, lo: 0xac, hi: 0xbf}, + // Block 0x6, offset 0x35 + {value: 0x0024, lo: 0x80, hi: 0x94}, + {value: 0x0034, lo: 0x95, hi: 0xbc}, + {value: 0x0024, lo: 0xbd, hi: 0xbf}, + // Block 0x7, offset 0x38 + {value: 0x6553, lo: 0x80, hi: 0x8f}, + {value: 0x2013, lo: 0x90, hi: 0x9f}, + {value: 0x5f53, lo: 0xa0, hi: 0xaf}, + {value: 0x2012, lo: 0xb0, hi: 0xbf}, + // Block 0x8, offset 0x3c + {value: 0x5f52, lo: 0x80, hi: 0x8f}, + {value: 0x6552, lo: 0x90, hi: 0x9f}, + {value: 0x0117, lo: 0xa0, hi: 0xbf}, + // Block 0x9, offset 0x3f + {value: 0x0117, lo: 0x80, hi: 0x81}, + {value: 0x0024, lo: 0x83, hi: 0x87}, + {value: 0x0014, lo: 0x88, hi: 0x89}, + {value: 0x0117, lo: 0x8a, hi: 0xbf}, + // Block 0xa, offset 0x43 + {value: 0x0f13, lo: 0x80, hi: 0x80}, + {value: 0x0316, lo: 0x81, hi: 0x82}, + {value: 0x0716, lo: 0x83, hi: 0x84}, + {value: 0x0316, lo: 0x85, hi: 0x86}, + {value: 0x0f16, lo: 0x87, hi: 0x88}, + {value: 0x0316, lo: 0x89, hi: 0x8a}, + {value: 0x0716, lo: 0x8b, hi: 0x8c}, + {value: 0x0316, lo: 0x8d, hi: 0x8e}, + {value: 0x0f12, lo: 0x8f, hi: 0x8f}, + {value: 0x0117, lo: 0x90, hi: 0xbf}, + // Block 0xb, offset 0x4d + {value: 0x0117, lo: 0x80, hi: 0xaf}, + {value: 0x6553, lo: 0xb1, hi: 0xbf}, + // Block 0xc, offset 0x4f + {value: 0x3013, lo: 0x80, hi: 0x8f}, + {value: 0x6853, lo: 0x90, hi: 0x96}, + {value: 0x0014, lo: 0x99, hi: 0x99}, + {value: 0x0010, lo: 0x9b, hi: 0x9c}, + {value: 0x0010, lo: 0x9e, hi: 0x9e}, + {value: 0x0012, lo: 0xa0, hi: 0xa0}, + {value: 0x6552, lo: 0xa1, hi: 0xaf}, + {value: 0x3012, lo: 0xb0, hi: 0xbf}, + // Block 0xd, offset 0x57 + {value: 0x0034, lo: 0x81, hi: 0x82}, + {value: 0x0024, lo: 0x84, hi: 0x84}, + {value: 0x0034, lo: 0x85, hi: 0x85}, + {value: 0x0034, lo: 0x87, hi: 0x87}, + {value: 0x0010, lo: 0x90, hi: 0xaa}, + {value: 0x0010, lo: 0xaf, hi: 0xb3}, + {value: 0x0054, lo: 0xb4, hi: 0xb4}, + // Block 0xe, offset 0x5e + {value: 0x0014, lo: 0x80, hi: 0x85}, + {value: 0x0024, lo: 0x90, hi: 0x97}, + {value: 0x0034, lo: 0x98, hi: 0x9a}, + {value: 0x0014, lo: 0x9c, hi: 0x9c}, + {value: 0x0010, lo: 0xa0, hi: 0xbf}, + // Block 0xf, offset 0x63 + {value: 0x0014, lo: 0x80, hi: 0x80}, + {value: 0x0010, lo: 0x81, hi: 0x8a}, + {value: 0x0034, lo: 0x8b, hi: 0x92}, + {value: 0x0024, lo: 0x93, hi: 0x94}, + {value: 0x0034, lo: 0x95, hi: 0x96}, + {value: 0x0024, lo: 0x97, hi: 0x9b}, + {value: 0x0034, lo: 0x9c, hi: 0x9c}, + {value: 0x0024, lo: 0x9d, hi: 0x9e}, + {value: 0x0034, lo: 0x9f, hi: 0x9f}, + {value: 0x0010, lo: 0xa0, hi: 0xa9}, + {value: 0x0010, lo: 0xab, hi: 0xab}, + {value: 0x0010, lo: 0xae, hi: 0xaf}, + {value: 0x0034, lo: 0xb0, hi: 0xb0}, + {value: 0x0010, lo: 0xb1, hi: 0xbf}, + // Block 0x10, offset 0x71 + {value: 0x0010, lo: 0x80, hi: 0xbf}, + // Block 0x11, offset 0x72 + {value: 0x0010, lo: 0x80, hi: 0x93}, + {value: 0x0010, lo: 0x95, hi: 0x95}, + {value: 0x0024, lo: 0x96, hi: 0x9c}, + {value: 0x0014, lo: 0x9d, hi: 0x9d}, + {value: 0x0024, lo: 0x9f, hi: 0xa2}, + {value: 0x0034, lo: 0xa3, hi: 0xa3}, + {value: 0x0024, lo: 0xa4, hi: 0xa4}, + {value: 0x0014, lo: 0xa5, hi: 0xa6}, + {value: 0x0024, lo: 0xa7, hi: 0xa8}, + {value: 0x0034, lo: 0xaa, hi: 0xaa}, + {value: 0x0024, lo: 0xab, hi: 0xac}, + {value: 0x0034, lo: 0xad, hi: 0xad}, + {value: 0x0010, lo: 0xae, hi: 0xbc}, + {value: 0x0010, lo: 0xbf, hi: 0xbf}, + // Block 0x12, offset 0x80 + {value: 0x0014, lo: 0x8f, hi: 0x8f}, + {value: 0x0010, lo: 0x90, hi: 0x90}, + {value: 0x0034, lo: 0x91, hi: 0x91}, + {value: 0x0010, lo: 0x92, hi: 0xaf}, + {value: 0x0024, lo: 0xb0, hi: 0xb0}, + {value: 0x0034, lo: 0xb1, hi: 0xb1}, + {value: 0x0024, lo: 0xb2, hi: 0xb3}, + {value: 0x0034, lo: 0xb4, hi: 0xb4}, + {value: 0x0024, lo: 0xb5, hi: 0xb6}, + {value: 0x0034, lo: 0xb7, hi: 0xb9}, + {value: 0x0024, lo: 0xba, hi: 0xba}, + {value: 0x0034, lo: 0xbb, hi: 0xbc}, + {value: 0x0024, lo: 0xbd, hi: 0xbd}, + {value: 0x0034, lo: 0xbe, hi: 0xbe}, + {value: 0x0024, lo: 0xbf, hi: 0xbf}, + // Block 0x13, offset 0x8f + {value: 0x0024, lo: 0x80, hi: 0x81}, + {value: 0x0034, lo: 0x82, hi: 0x82}, + {value: 0x0024, lo: 0x83, hi: 0x83}, + {value: 0x0034, lo: 0x84, hi: 0x84}, + {value: 0x0024, lo: 0x85, hi: 0x85}, + {value: 0x0034, lo: 0x86, hi: 0x86}, + {value: 0x0024, lo: 0x87, hi: 0x87}, + {value: 0x0034, lo: 0x88, hi: 0x88}, + {value: 0x0024, lo: 0x89, hi: 0x8a}, + {value: 0x0010, lo: 0x8d, hi: 0xbf}, + // Block 0x14, offset 0x99 + {value: 0x0010, lo: 0x80, hi: 0xa5}, + {value: 0x0014, lo: 0xa6, hi: 0xb0}, + {value: 0x0010, lo: 0xb1, hi: 0xb1}, + // Block 0x15, offset 0x9c + {value: 0x0010, lo: 0x80, hi: 0xaa}, + {value: 0x0024, lo: 0xab, hi: 0xb1}, + {value: 0x0034, lo: 0xb2, hi: 0xb2}, + {value: 0x0024, lo: 0xb3, hi: 0xb3}, + {value: 0x0014, lo: 0xb4, hi: 0xb5}, + {value: 0x0014, lo: 0xba, hi: 0xba}, + {value: 0x0034, lo: 0xbd, hi: 0xbd}, + // Block 0x16, offset 0xa3 + {value: 0x0010, lo: 0x80, hi: 0x95}, + {value: 0x0024, lo: 0x96, hi: 0x99}, + {value: 0x0014, lo: 0x9a, hi: 0x9a}, + {value: 0x0024, lo: 0x9b, hi: 0xa3}, + {value: 0x0014, lo: 0xa4, hi: 0xa4}, + {value: 0x0024, lo: 0xa5, hi: 0xa7}, + {value: 0x0014, lo: 0xa8, hi: 0xa8}, + {value: 0x0024, lo: 0xa9, hi: 0xad}, + // Block 0x17, offset 0xab + {value: 0x0010, lo: 0x80, hi: 0x98}, + {value: 0x0034, lo: 0x99, hi: 0x9b}, + {value: 0x0010, lo: 0xa0, hi: 0xaa}, + // Block 0x18, offset 0xae + {value: 0x0010, lo: 0xa0, hi: 0xb4}, + {value: 0x0010, lo: 0xb6, hi: 0xbd}, + // Block 0x19, offset 0xb0 + {value: 0x0034, lo: 0x93, hi: 0x93}, + {value: 0x0024, lo: 0x94, hi: 0xa1}, + {value: 0x0014, lo: 0xa2, hi: 0xa2}, + {value: 0x0034, lo: 0xa3, hi: 0xa3}, + {value: 0x0024, lo: 0xa4, hi: 0xa5}, + {value: 0x0034, lo: 0xa6, hi: 0xa6}, + {value: 0x0024, lo: 0xa7, hi: 0xa8}, + {value: 0x0034, lo: 0xa9, hi: 0xa9}, + {value: 0x0024, lo: 0xaa, hi: 0xac}, + {value: 0x0034, lo: 0xad, hi: 0xb2}, + {value: 0x0024, lo: 0xb3, hi: 0xb5}, + {value: 0x0034, lo: 0xb6, hi: 0xb6}, + {value: 0x0024, lo: 0xb7, hi: 0xb8}, + {value: 0x0034, lo: 0xb9, hi: 0xba}, + {value: 0x0024, lo: 0xbb, hi: 0xbf}, + // Block 0x1a, offset 0xbf + {value: 0x0014, lo: 0x80, hi: 0x82}, + {value: 0x0010, lo: 0x83, hi: 0xb9}, + {value: 0x0014, lo: 0xba, hi: 0xba}, + {value: 0x0010, lo: 0xbb, hi: 0xbb}, + {value: 0x0034, lo: 0xbc, hi: 0xbc}, + {value: 0x0010, lo: 0xbd, hi: 0xbf}, + // Block 0x1b, offset 0xc5 + {value: 0x0010, lo: 0x80, hi: 0x80}, + {value: 0x0014, lo: 0x81, hi: 0x88}, + {value: 0x0010, lo: 0x89, hi: 0x8c}, + {value: 0x0034, lo: 0x8d, hi: 0x8d}, + {value: 0x0010, lo: 0x8e, hi: 0x90}, + {value: 0x0024, lo: 0x91, hi: 0x91}, + {value: 0x0034, lo: 0x92, hi: 0x92}, + {value: 0x0024, lo: 0x93, hi: 0x94}, + {value: 0x0014, lo: 0x95, hi: 0x97}, + {value: 0x0010, lo: 0x98, hi: 0xa1}, + {value: 0x0014, lo: 0xa2, hi: 0xa3}, + {value: 0x0010, lo: 0xa6, hi: 0xaf}, + {value: 0x0014, lo: 0xb1, hi: 0xb1}, + {value: 0x0010, lo: 0xb2, hi: 0xbf}, + // Block 0x1c, offset 0xd3 + {value: 0x0010, lo: 0x80, hi: 0x80}, + {value: 0x0014, lo: 0x81, hi: 0x81}, + {value: 0x0010, lo: 0x82, hi: 0x83}, + {value: 0x0010, lo: 0x85, hi: 0x8c}, + {value: 0x0010, lo: 0x8f, hi: 0x90}, + {value: 0x0010, lo: 0x93, hi: 0xa8}, + {value: 0x0010, lo: 0xaa, hi: 0xb0}, + {value: 0x0010, lo: 0xb2, hi: 0xb2}, + {value: 0x0010, lo: 0xb6, hi: 0xb9}, + {value: 0x0034, lo: 0xbc, hi: 0xbc}, + {value: 0x0010, lo: 0xbd, hi: 0xbf}, + // Block 0x1d, offset 0xde + {value: 0x0010, lo: 0x80, hi: 0x80}, + {value: 0x0014, lo: 0x81, hi: 0x84}, + {value: 0x0010, lo: 0x87, hi: 0x88}, + {value: 0x0010, lo: 0x8b, hi: 0x8c}, + {value: 0x0034, lo: 0x8d, hi: 0x8d}, + {value: 0x0010, lo: 0x8e, hi: 0x8e}, + {value: 0x0010, lo: 0x97, hi: 0x97}, + {value: 0x0010, lo: 0x9c, hi: 0x9d}, + {value: 0x0010, lo: 0x9f, hi: 0xa1}, + {value: 0x0014, lo: 0xa2, hi: 0xa3}, + {value: 0x0010, lo: 0xa6, hi: 0xb1}, + {value: 0x0010, lo: 0xbc, hi: 0xbc}, + {value: 0x0024, lo: 0xbe, hi: 0xbe}, + // Block 0x1e, offset 0xeb + {value: 0x0014, lo: 0x81, hi: 0x82}, + {value: 0x0010, lo: 0x83, hi: 0x83}, + {value: 0x0010, lo: 0x85, hi: 0x8a}, + {value: 0x0010, lo: 0x8f, hi: 0x90}, + {value: 0x0010, lo: 0x93, hi: 0xa8}, + {value: 0x0010, lo: 0xaa, hi: 0xb0}, + {value: 0x0010, lo: 0xb2, hi: 0xb3}, + {value: 0x0010, lo: 0xb5, hi: 0xb6}, + {value: 0x0010, lo: 0xb8, hi: 0xb9}, + {value: 0x0034, lo: 0xbc, hi: 0xbc}, + {value: 0x0010, lo: 0xbe, hi: 0xbf}, + // Block 0x1f, offset 0xf6 + {value: 0x0010, lo: 0x80, hi: 0x80}, + {value: 0x0014, lo: 0x81, hi: 0x82}, + {value: 0x0014, lo: 0x87, hi: 0x88}, + {value: 0x0014, lo: 0x8b, hi: 0x8c}, + {value: 0x0034, lo: 0x8d, hi: 0x8d}, + {value: 0x0014, lo: 0x91, hi: 0x91}, + {value: 0x0010, lo: 0x99, hi: 0x9c}, + {value: 0x0010, lo: 0x9e, hi: 0x9e}, + {value: 0x0010, lo: 0xa6, hi: 0xaf}, + {value: 0x0014, lo: 0xb0, hi: 0xb1}, + {value: 0x0010, lo: 0xb2, hi: 0xb4}, + {value: 0x0014, lo: 0xb5, hi: 0xb5}, + // Block 0x20, offset 0x102 + {value: 0x0014, lo: 0x81, hi: 0x82}, + {value: 0x0010, lo: 0x83, hi: 0x83}, + {value: 0x0010, lo: 0x85, hi: 0x8d}, + {value: 0x0010, lo: 0x8f, hi: 0x91}, + {value: 0x0010, lo: 0x93, hi: 0xa8}, + {value: 0x0010, lo: 0xaa, hi: 0xb0}, + {value: 0x0010, lo: 0xb2, hi: 0xb3}, + {value: 0x0010, lo: 0xb5, hi: 0xb9}, + {value: 0x0034, lo: 0xbc, hi: 0xbc}, + {value: 0x0010, lo: 0xbd, hi: 0xbf}, + // Block 0x21, offset 0x10c + {value: 0x0010, lo: 0x80, hi: 0x80}, + {value: 0x0014, lo: 0x81, hi: 0x85}, + {value: 0x0014, lo: 0x87, hi: 0x88}, + {value: 0x0010, lo: 0x89, hi: 0x89}, + {value: 0x0010, lo: 0x8b, hi: 0x8c}, + {value: 0x0034, lo: 0x8d, hi: 0x8d}, + {value: 0x0010, lo: 0x90, hi: 0x90}, + {value: 0x0010, lo: 0xa0, hi: 0xa1}, + {value: 0x0014, lo: 0xa2, hi: 0xa3}, + {value: 0x0010, lo: 0xa6, hi: 0xaf}, + {value: 0x0010, lo: 0xb9, hi: 0xb9}, + {value: 0x0014, lo: 0xba, hi: 0xbf}, + // Block 0x22, offset 0x118 + {value: 0x0014, lo: 0x81, hi: 0x81}, + {value: 0x0010, lo: 0x82, hi: 0x83}, + {value: 0x0010, lo: 0x85, hi: 0x8c}, + {value: 0x0010, lo: 0x8f, hi: 0x90}, + {value: 0x0010, lo: 0x93, hi: 0xa8}, + {value: 0x0010, lo: 0xaa, hi: 0xb0}, + {value: 0x0010, lo: 0xb2, hi: 0xb3}, + {value: 0x0010, lo: 0xb5, hi: 0xb9}, + {value: 0x0034, lo: 0xbc, hi: 0xbc}, + {value: 0x0010, lo: 0xbd, hi: 0xbe}, + {value: 0x0014, lo: 0xbf, hi: 0xbf}, + // Block 0x23, offset 0x123 + {value: 0x0010, lo: 0x80, hi: 0x80}, + {value: 0x0014, lo: 0x81, hi: 0x84}, + {value: 0x0010, lo: 0x87, hi: 0x88}, + {value: 0x0010, lo: 0x8b, hi: 0x8c}, + {value: 0x0034, lo: 0x8d, hi: 0x8d}, + {value: 0x0014, lo: 0x96, hi: 0x96}, + {value: 0x0010, lo: 0x97, hi: 0x97}, + {value: 0x0010, lo: 0x9c, hi: 0x9d}, + {value: 0x0010, lo: 0x9f, hi: 0xa1}, + {value: 0x0014, lo: 0xa2, hi: 0xa3}, + {value: 0x0010, lo: 0xa6, hi: 0xaf}, + {value: 0x0010, lo: 0xb1, hi: 0xb1}, + // Block 0x24, offset 0x12f + {value: 0x0014, lo: 0x82, hi: 0x82}, + {value: 0x0010, lo: 0x83, hi: 0x83}, + {value: 0x0010, lo: 0x85, hi: 0x8a}, + {value: 0x0010, lo: 0x8e, hi: 0x90}, + {value: 0x0010, lo: 0x92, hi: 0x95}, + {value: 0x0010, lo: 0x99, hi: 0x9a}, + {value: 0x0010, lo: 0x9c, hi: 0x9c}, + {value: 0x0010, lo: 0x9e, hi: 0x9f}, + {value: 0x0010, lo: 0xa3, hi: 0xa4}, + {value: 0x0010, lo: 0xa8, hi: 0xaa}, + {value: 0x0010, lo: 0xae, hi: 0xb9}, + {value: 0x0010, lo: 0xbe, hi: 0xbf}, + // Block 0x25, offset 0x13b + {value: 0x0014, lo: 0x80, hi: 0x80}, + {value: 0x0010, lo: 0x81, hi: 0x82}, + {value: 0x0010, lo: 0x86, hi: 0x88}, + {value: 0x0010, lo: 0x8a, hi: 0x8c}, + {value: 0x0034, lo: 0x8d, hi: 0x8d}, + {value: 0x0010, lo: 0x90, hi: 0x90}, + {value: 0x0010, lo: 0x97, hi: 0x97}, + {value: 0x0010, lo: 0xa6, hi: 0xaf}, + // Block 0x26, offset 0x143 + {value: 0x0014, lo: 0x80, hi: 0x80}, + {value: 0x0010, lo: 0x81, hi: 0x83}, + {value: 0x0014, lo: 0x84, hi: 0x84}, + {value: 0x0010, lo: 0x85, hi: 0x8c}, + {value: 0x0010, lo: 0x8e, hi: 0x90}, + {value: 0x0010, lo: 0x92, hi: 0xa8}, + {value: 0x0010, lo: 0xaa, hi: 0xb9}, + {value: 0x0010, lo: 0xbd, hi: 0xbd}, + {value: 0x0014, lo: 0xbe, hi: 0xbf}, + // Block 0x27, offset 0x14c + {value: 0x0014, lo: 0x80, hi: 0x80}, + {value: 0x0010, lo: 0x81, hi: 0x84}, + {value: 0x0014, lo: 0x86, hi: 0x88}, + {value: 0x0014, lo: 0x8a, hi: 0x8c}, + {value: 0x0034, lo: 0x8d, hi: 0x8d}, + {value: 0x0034, lo: 0x95, hi: 0x96}, + {value: 0x0010, lo: 0x98, hi: 0x9a}, + {value: 0x0010, lo: 0xa0, hi: 0xa1}, + {value: 0x0014, lo: 0xa2, hi: 0xa3}, + {value: 0x0010, lo: 0xa6, hi: 0xaf}, + // Block 0x28, offset 0x156 + {value: 0x0010, lo: 0x80, hi: 0x80}, + {value: 0x0014, lo: 0x81, hi: 0x81}, + {value: 0x0010, lo: 0x82, hi: 0x83}, + {value: 0x0010, lo: 0x85, hi: 0x8c}, + {value: 0x0010, lo: 0x8e, hi: 0x90}, + {value: 0x0010, lo: 0x92, hi: 0xa8}, + {value: 0x0010, lo: 0xaa, hi: 0xb3}, + {value: 0x0010, lo: 0xb5, hi: 0xb9}, + {value: 0x0034, lo: 0xbc, hi: 0xbc}, + {value: 0x0010, lo: 0xbd, hi: 0xbe}, + {value: 0x0014, lo: 0xbf, hi: 0xbf}, + // Block 0x29, offset 0x161 + {value: 0x0010, lo: 0x80, hi: 0x84}, + {value: 0x0014, lo: 0x86, hi: 0x86}, + {value: 0x0010, lo: 0x87, hi: 0x88}, + {value: 0x0010, lo: 0x8a, hi: 0x8b}, + {value: 0x0014, lo: 0x8c, hi: 0x8c}, + {value: 0x0034, lo: 0x8d, hi: 0x8d}, + {value: 0x0010, lo: 0x95, hi: 0x96}, + {value: 0x0010, lo: 0x9e, hi: 0x9e}, + {value: 0x0010, lo: 0xa0, hi: 0xa1}, + {value: 0x0014, lo: 0xa2, hi: 0xa3}, + {value: 0x0010, lo: 0xa6, hi: 0xaf}, + {value: 0x0010, lo: 0xb1, hi: 0xb2}, + // Block 0x2a, offset 0x16d + {value: 0x0014, lo: 0x80, hi: 0x81}, + {value: 0x0010, lo: 0x82, hi: 0x83}, + {value: 0x0010, lo: 0x85, hi: 0x8c}, + {value: 0x0010, lo: 0x8e, hi: 0x90}, + {value: 0x0010, lo: 0x92, hi: 0xba}, + {value: 0x0034, lo: 0xbb, hi: 0xbc}, + {value: 0x0010, lo: 0xbd, hi: 0xbf}, + // Block 0x2b, offset 0x174 + {value: 0x0010, lo: 0x80, hi: 0x80}, + {value: 0x0014, lo: 0x81, hi: 0x84}, + {value: 0x0010, lo: 0x86, hi: 0x88}, + {value: 0x0010, lo: 0x8a, hi: 0x8c}, + {value: 0x0034, lo: 0x8d, hi: 0x8d}, + {value: 0x0010, lo: 0x8e, hi: 0x8e}, + {value: 0x0010, lo: 0x94, hi: 0x97}, + {value: 0x0010, lo: 0x9f, hi: 0xa1}, + {value: 0x0014, lo: 0xa2, hi: 0xa3}, + {value: 0x0010, lo: 0xa6, hi: 0xaf}, + {value: 0x0010, lo: 0xba, hi: 0xbf}, + // Block 0x2c, offset 0x17f + {value: 0x0010, lo: 0x82, hi: 0x83}, + {value: 0x0010, lo: 0x85, hi: 0x96}, + {value: 0x0010, lo: 0x9a, hi: 0xb1}, + {value: 0x0010, lo: 0xb3, hi: 0xbb}, + {value: 0x0010, lo: 0xbd, hi: 0xbd}, + // Block 0x2d, offset 0x184 + {value: 0x0010, lo: 0x80, hi: 0x86}, + {value: 0x0034, lo: 0x8a, hi: 0x8a}, + {value: 0x0010, lo: 0x8f, hi: 0x91}, + {value: 0x0014, lo: 0x92, hi: 0x94}, + {value: 0x0014, lo: 0x96, hi: 0x96}, + {value: 0x0010, lo: 0x98, hi: 0x9f}, + {value: 0x0010, lo: 0xa6, hi: 0xaf}, + {value: 0x0010, lo: 0xb2, hi: 0xb3}, + // Block 0x2e, offset 0x18c + {value: 0x0014, lo: 0xb1, hi: 0xb1}, + {value: 0x0014, lo: 0xb4, hi: 0xb7}, + {value: 0x0034, lo: 0xb8, hi: 0xba}, + // Block 0x2f, offset 0x18f + {value: 0x0004, lo: 0x86, hi: 0x86}, + {value: 0x0014, lo: 0x87, hi: 0x87}, + {value: 0x0034, lo: 0x88, hi: 0x8b}, + {value: 0x0014, lo: 0x8c, hi: 0x8e}, + {value: 0x0010, lo: 0x90, hi: 0x99}, + // Block 0x30, offset 0x194 + {value: 0x0014, lo: 0xb1, hi: 0xb1}, + {value: 0x0014, lo: 0xb4, hi: 0xb7}, + {value: 0x0034, lo: 0xb8, hi: 0xba}, + {value: 0x0014, lo: 0xbb, hi: 0xbc}, + // Block 0x31, offset 0x198 + {value: 0x0004, lo: 0x86, hi: 0x86}, + {value: 0x0034, lo: 0x88, hi: 0x8b}, + {value: 0x0014, lo: 0x8c, hi: 0x8d}, + {value: 0x0010, lo: 0x90, hi: 0x99}, + // Block 0x32, offset 0x19c + {value: 0x0010, lo: 0x80, hi: 0x80}, + {value: 0x0034, lo: 0x98, hi: 0x99}, + {value: 0x0010, lo: 0xa0, hi: 0xa9}, + {value: 0x0034, lo: 0xb5, hi: 0xb5}, + {value: 0x0034, lo: 0xb7, hi: 0xb7}, + {value: 0x0034, lo: 0xb9, hi: 0xb9}, + {value: 0x0010, lo: 0xbe, hi: 0xbf}, + // Block 0x33, offset 0x1a3 + {value: 0x0010, lo: 0x80, hi: 0x87}, + {value: 0x0010, lo: 0x89, hi: 0xac}, + {value: 0x0034, lo: 0xb1, hi: 0xb2}, + {value: 0x0014, lo: 0xb3, hi: 0xb3}, + {value: 0x0034, lo: 0xb4, hi: 0xb4}, + {value: 0x0014, lo: 0xb5, hi: 0xb9}, + {value: 0x0034, lo: 0xba, hi: 0xbd}, + {value: 0x0014, lo: 0xbe, hi: 0xbe}, + {value: 0x0010, lo: 0xbf, hi: 0xbf}, + // Block 0x34, offset 0x1ac + {value: 0x0034, lo: 0x80, hi: 0x80}, + {value: 0x0014, lo: 0x81, hi: 0x81}, + {value: 0x0024, lo: 0x82, hi: 0x83}, + {value: 0x0034, lo: 0x84, hi: 0x84}, + {value: 0x0024, lo: 0x86, hi: 0x87}, + {value: 0x0010, lo: 0x88, hi: 0x8c}, + {value: 0x0014, lo: 0x8d, hi: 0x97}, + {value: 0x0014, lo: 0x99, hi: 0xbc}, + // Block 0x35, offset 0x1b4 + {value: 0x0034, lo: 0x86, hi: 0x86}, + // Block 0x36, offset 0x1b5 + {value: 0x0010, lo: 0xab, hi: 0xac}, + {value: 0x0014, lo: 0xad, hi: 0xb0}, + {value: 0x0010, lo: 0xb1, hi: 0xb1}, + {value: 0x0014, lo: 0xb2, hi: 0xb6}, + {value: 0x0034, lo: 0xb7, hi: 0xb7}, + {value: 0x0010, lo: 0xb8, hi: 0xb8}, + {value: 0x0034, lo: 0xb9, hi: 0xba}, + {value: 0x0010, lo: 0xbb, hi: 0xbc}, + {value: 0x0014, lo: 0xbd, hi: 0xbe}, + // Block 0x37, offset 0x1be + {value: 0x0010, lo: 0x80, hi: 0x89}, + {value: 0x0010, lo: 0x96, hi: 0x97}, + {value: 0x0014, lo: 0x98, hi: 0x99}, + {value: 0x0014, lo: 0x9e, hi: 0xa0}, + {value: 0x0010, lo: 0xa2, hi: 0xa4}, + {value: 0x0010, lo: 0xa7, hi: 0xad}, + {value: 0x0014, lo: 0xb1, hi: 0xb4}, + // Block 0x38, offset 0x1c5 + {value: 0x0014, lo: 0x82, hi: 0x82}, + {value: 0x0010, lo: 0x83, hi: 0x84}, + {value: 0x0014, lo: 0x85, hi: 0x86}, + {value: 0x0010, lo: 0x87, hi: 0x8c}, + {value: 0x0034, lo: 0x8d, hi: 0x8d}, + {value: 0x0010, lo: 0x8f, hi: 0x9c}, + {value: 0x0014, lo: 0x9d, hi: 0x9d}, + {value: 0x6c53, lo: 0xa0, hi: 0xbf}, + // Block 0x39, offset 0x1cd + {value: 0x0010, lo: 0x80, hi: 0x88}, + {value: 0x0010, lo: 0x8a, hi: 0x8d}, + {value: 0x0010, lo: 0x90, hi: 0x96}, + {value: 0x0010, lo: 0x98, hi: 0x98}, + {value: 0x0010, lo: 0x9a, hi: 0x9d}, + {value: 0x0010, lo: 0xa0, hi: 0xbf}, + // Block 0x3a, offset 0x1d3 + {value: 0x0010, lo: 0x80, hi: 0x88}, + {value: 0x0010, lo: 0x8a, hi: 0x8d}, + {value: 0x0010, lo: 0x90, hi: 0xb0}, + {value: 0x0010, lo: 0xb2, hi: 0xb5}, + {value: 0x0010, lo: 0xb8, hi: 0xbe}, + // Block 0x3b, offset 0x1d8 + {value: 0x0010, lo: 0x80, hi: 0x80}, + {value: 0x0010, lo: 0x82, hi: 0x85}, + {value: 0x0010, lo: 0x88, hi: 0x96}, + {value: 0x0010, lo: 0x98, hi: 0xbf}, + // Block 0x3c, offset 0x1dc + {value: 0x0010, lo: 0x80, hi: 0x90}, + {value: 0x0010, lo: 0x92, hi: 0x95}, + {value: 0x0010, lo: 0x98, hi: 0xbf}, + // Block 0x3d, offset 0x1df + {value: 0x0010, lo: 0x80, hi: 0x9a}, + {value: 0x0024, lo: 0x9d, hi: 0x9f}, + // Block 0x3e, offset 0x1e1 + {value: 0x0010, lo: 0x80, hi: 0x8f}, + {value: 0x7453, lo: 0xa0, hi: 0xaf}, + {value: 0x7853, lo: 0xb0, hi: 0xbf}, + // Block 0x3f, offset 0x1e4 + {value: 0x7c53, lo: 0x80, hi: 0x8f}, + {value: 0x8053, lo: 0x90, hi: 0x9f}, + {value: 0x7c53, lo: 0xa0, hi: 0xaf}, + {value: 0x0813, lo: 0xb0, hi: 0xb5}, + {value: 0x0892, lo: 0xb8, hi: 0xbd}, + // Block 0x40, offset 0x1e9 + {value: 0x0010, lo: 0x81, hi: 0xbf}, + // Block 0x41, offset 0x1ea + {value: 0x0010, lo: 0x80, hi: 0xac}, + {value: 0x0010, lo: 0xaf, hi: 0xbf}, + // Block 0x42, offset 0x1ec + {value: 0x0010, lo: 0x81, hi: 0x9a}, + {value: 0x0010, lo: 0xa0, hi: 0xbf}, + // Block 0x43, offset 0x1ee + {value: 0x0010, lo: 0x80, hi: 0xaa}, + {value: 0x0010, lo: 0xae, hi: 0xb8}, + // Block 0x44, offset 0x1f0 + {value: 0x0010, lo: 0x80, hi: 0x8c}, + {value: 0x0010, lo: 0x8e, hi: 0x91}, + {value: 0x0014, lo: 0x92, hi: 0x93}, + {value: 0x0034, lo: 0x94, hi: 0x94}, + {value: 0x0010, lo: 0xa0, hi: 0xb1}, + {value: 0x0014, lo: 0xb2, hi: 0xb3}, + {value: 0x0034, lo: 0xb4, hi: 0xb4}, + // Block 0x45, offset 0x1f7 + {value: 0x0010, lo: 0x80, hi: 0x91}, + {value: 0x0014, lo: 0x92, hi: 0x93}, + {value: 0x0010, lo: 0xa0, hi: 0xac}, + {value: 0x0010, lo: 0xae, hi: 0xb0}, + {value: 0x0014, lo: 0xb2, hi: 0xb3}, + // Block 0x46, offset 0x1fc + {value: 0x0014, lo: 0xb4, hi: 0xb5}, + {value: 0x0010, lo: 0xb6, hi: 0xb6}, + {value: 0x0014, lo: 0xb7, hi: 0xbd}, + {value: 0x0010, lo: 0xbe, hi: 0xbf}, + // Block 0x47, offset 0x200 + {value: 0x0010, lo: 0x80, hi: 0x85}, + {value: 0x0014, lo: 0x86, hi: 0x86}, + {value: 0x0010, lo: 0x87, hi: 0x88}, + {value: 0x0014, lo: 0x89, hi: 0x91}, + {value: 0x0034, lo: 0x92, hi: 0x92}, + {value: 0x0014, lo: 0x93, hi: 0x93}, + {value: 0x0004, lo: 0x97, hi: 0x97}, + {value: 0x0024, lo: 0x9d, hi: 0x9d}, + {value: 0x0010, lo: 0xa0, hi: 0xa9}, + // Block 0x48, offset 0x209 + {value: 0x0014, lo: 0x8b, hi: 0x8e}, + {value: 0x0010, lo: 0x90, hi: 0x99}, + {value: 0x0010, lo: 0xa0, hi: 0xbf}, + // Block 0x49, offset 0x20c + {value: 0x0010, lo: 0x80, hi: 0x82}, + {value: 0x0014, lo: 0x83, hi: 0x83}, + {value: 0x0010, lo: 0x84, hi: 0xb8}, + // Block 0x4a, offset 0x20f + {value: 0x0010, lo: 0x80, hi: 0x84}, + {value: 0x0014, lo: 0x85, hi: 0x86}, + {value: 0x0010, lo: 0x87, hi: 0xa8}, + {value: 0x0034, lo: 0xa9, hi: 0xa9}, + {value: 0x0010, lo: 0xaa, hi: 0xaa}, + {value: 0x0010, lo: 0xb0, hi: 0xbf}, + // Block 0x4b, offset 0x215 + {value: 0x0010, lo: 0x80, hi: 0xb5}, + // Block 0x4c, offset 0x216 + {value: 0x0010, lo: 0x80, hi: 0x9e}, + {value: 0x0014, lo: 0xa0, hi: 0xa2}, + {value: 0x0010, lo: 0xa3, hi: 0xa6}, + {value: 0x0014, lo: 0xa7, hi: 0xa8}, + {value: 0x0010, lo: 0xa9, hi: 0xab}, + {value: 0x0010, lo: 0xb0, hi: 0xb1}, + {value: 0x0014, lo: 0xb2, hi: 0xb2}, + {value: 0x0010, lo: 0xb3, hi: 0xb8}, + {value: 0x0034, lo: 0xb9, hi: 0xb9}, + {value: 0x0024, lo: 0xba, hi: 0xba}, + {value: 0x0034, lo: 0xbb, hi: 0xbb}, + // Block 0x4d, offset 0x221 + {value: 0x0010, lo: 0x86, hi: 0x8f}, + // Block 0x4e, offset 0x222 + {value: 0x0010, lo: 0x90, hi: 0x99}, + // Block 0x4f, offset 0x223 + {value: 0x0010, lo: 0x80, hi: 0x96}, + {value: 0x0024, lo: 0x97, hi: 0x97}, + {value: 0x0034, lo: 0x98, hi: 0x98}, + {value: 0x0010, lo: 0x99, hi: 0x9a}, + {value: 0x0014, lo: 0x9b, hi: 0x9b}, + // Block 0x50, offset 0x228 + {value: 0x0010, lo: 0x95, hi: 0x95}, + {value: 0x0014, lo: 0x96, hi: 0x96}, + {value: 0x0010, lo: 0x97, hi: 0x97}, + {value: 0x0014, lo: 0x98, hi: 0x9e}, + {value: 0x0034, lo: 0xa0, hi: 0xa0}, + {value: 0x0010, lo: 0xa1, hi: 0xa1}, + {value: 0x0014, lo: 0xa2, hi: 0xa2}, + {value: 0x0010, lo: 0xa3, hi: 0xa4}, + {value: 0x0014, lo: 0xa5, hi: 0xac}, + {value: 0x0010, lo: 0xad, hi: 0xb2}, + {value: 0x0014, lo: 0xb3, hi: 0xb4}, + {value: 0x0024, lo: 0xb5, hi: 0xbc}, + {value: 0x0034, lo: 0xbf, hi: 0xbf}, + // Block 0x51, offset 0x235 + {value: 0x0010, lo: 0x80, hi: 0x89}, + {value: 0x0010, lo: 0x90, hi: 0x99}, + {value: 0x0004, lo: 0xa7, hi: 0xa7}, + {value: 0x0024, lo: 0xb0, hi: 0xb4}, + {value: 0x0034, lo: 0xb5, hi: 0xba}, + {value: 0x0024, lo: 0xbb, hi: 0xbc}, + {value: 0x0034, lo: 0xbd, hi: 0xbd}, + {value: 0x0014, lo: 0xbe, hi: 0xbe}, + // Block 0x52, offset 0x23d + {value: 0x0014, lo: 0x80, hi: 0x83}, + {value: 0x0010, lo: 0x84, hi: 0xb3}, + {value: 0x0034, lo: 0xb4, hi: 0xb4}, + {value: 0x0010, lo: 0xb5, hi: 0xb5}, + {value: 0x0014, lo: 0xb6, hi: 0xba}, + {value: 0x0010, lo: 0xbb, hi: 0xbb}, + {value: 0x0014, lo: 0xbc, hi: 0xbc}, + {value: 0x0010, lo: 0xbd, hi: 0xbf}, + // Block 0x53, offset 0x245 + {value: 0x0010, lo: 0x80, hi: 0x81}, + {value: 0x0014, lo: 0x82, hi: 0x82}, + {value: 0x0010, lo: 0x83, hi: 0x83}, + {value: 0x0030, lo: 0x84, hi: 0x84}, + {value: 0x0010, lo: 0x85, hi: 0x8b}, + {value: 0x0010, lo: 0x90, hi: 0x99}, + {value: 0x0024, lo: 0xab, hi: 0xab}, + {value: 0x0034, lo: 0xac, hi: 0xac}, + {value: 0x0024, lo: 0xad, hi: 0xb3}, + // Block 0x54, offset 0x24e + {value: 0x0014, lo: 0x80, hi: 0x81}, + {value: 0x0010, lo: 0x82, hi: 0xa1}, + {value: 0x0014, lo: 0xa2, hi: 0xa5}, + {value: 0x0010, lo: 0xa6, hi: 0xa7}, + {value: 0x0014, lo: 0xa8, hi: 0xa9}, + {value: 0x0030, lo: 0xaa, hi: 0xaa}, + {value: 0x0034, lo: 0xab, hi: 0xab}, + {value: 0x0014, lo: 0xac, hi: 0xad}, + {value: 0x0010, lo: 0xae, hi: 0xbf}, + // Block 0x55, offset 0x257 + {value: 0x0010, lo: 0x80, hi: 0xa5}, + {value: 0x0034, lo: 0xa6, hi: 0xa6}, + {value: 0x0010, lo: 0xa7, hi: 0xa7}, + {value: 0x0014, lo: 0xa8, hi: 0xa9}, + {value: 0x0010, lo: 0xaa, hi: 0xac}, + {value: 0x0014, lo: 0xad, hi: 0xad}, + {value: 0x0010, lo: 0xae, hi: 0xae}, + {value: 0x0014, lo: 0xaf, hi: 0xb1}, + {value: 0x0030, lo: 0xb2, hi: 0xb3}, + // Block 0x56, offset 0x260 + {value: 0x0010, lo: 0x80, hi: 0xab}, + {value: 0x0014, lo: 0xac, hi: 0xb3}, + {value: 0x0010, lo: 0xb4, hi: 0xb5}, + {value: 0x0014, lo: 0xb6, hi: 0xb6}, + {value: 0x0034, lo: 0xb7, hi: 0xb7}, + // Block 0x57, offset 0x265 + {value: 0x0010, lo: 0x80, hi: 0x89}, + {value: 0x0010, lo: 0x8d, hi: 0xb7}, + {value: 0x0014, lo: 0xb8, hi: 0xbd}, + // Block 0x58, offset 0x268 + {value: 0x31ea, lo: 0x80, hi: 0x80}, + {value: 0x326a, lo: 0x81, hi: 0x81}, + {value: 0x32ea, lo: 0x82, hi: 0x82}, + {value: 0x336a, lo: 0x83, hi: 0x83}, + {value: 0x33ea, lo: 0x84, hi: 0x84}, + {value: 0x346a, lo: 0x85, hi: 0x85}, + {value: 0x34ea, lo: 0x86, hi: 0x86}, + {value: 0x356a, lo: 0x87, hi: 0x87}, + {value: 0x35ea, lo: 0x88, hi: 0x88}, + {value: 0x8353, lo: 0x90, hi: 0xba}, + {value: 0x8353, lo: 0xbd, hi: 0xbf}, + // Block 0x59, offset 0x273 + {value: 0x0024, lo: 0x90, hi: 0x92}, + {value: 0x0034, lo: 0x94, hi: 0x99}, + {value: 0x0024, lo: 0x9a, hi: 0x9b}, + {value: 0x0034, lo: 0x9c, hi: 0x9f}, + {value: 0x0024, lo: 0xa0, hi: 0xa0}, + {value: 0x0010, lo: 0xa1, hi: 0xa1}, + {value: 0x0034, lo: 0xa2, hi: 0xa8}, + {value: 0x0010, lo: 0xa9, hi: 0xac}, + {value: 0x0034, lo: 0xad, hi: 0xad}, + {value: 0x0010, lo: 0xae, hi: 0xb3}, + {value: 0x0024, lo: 0xb4, hi: 0xb4}, + {value: 0x0010, lo: 0xb5, hi: 0xb7}, + {value: 0x0024, lo: 0xb8, hi: 0xb9}, + {value: 0x0010, lo: 0xba, hi: 0xba}, + // Block 0x5a, offset 0x281 + {value: 0x0012, lo: 0x80, hi: 0xab}, + {value: 0x0015, lo: 0xac, hi: 0xbf}, + // Block 0x5b, offset 0x283 + {value: 0x0015, lo: 0x80, hi: 0xaa}, + {value: 0x0012, lo: 0xab, hi: 0xb7}, + {value: 0x0015, lo: 0xb8, hi: 0xb8}, + {value: 0x8752, lo: 0xb9, hi: 0xb9}, + {value: 0x0012, lo: 0xba, hi: 0xbc}, + {value: 0x8b52, lo: 0xbd, hi: 0xbd}, + {value: 0x0012, lo: 0xbe, hi: 0xbf}, + // Block 0x5c, offset 0x28a + {value: 0x0012, lo: 0x80, hi: 0x8d}, + {value: 0x8f52, lo: 0x8e, hi: 0x8e}, + {value: 0x0012, lo: 0x8f, hi: 0x9a}, + {value: 0x0015, lo: 0x9b, hi: 0xbf}, + // Block 0x5d, offset 0x28e + {value: 0x0024, lo: 0x80, hi: 0x81}, + {value: 0x0034, lo: 0x82, hi: 0x82}, + {value: 0x0024, lo: 0x83, hi: 0x89}, + {value: 0x0034, lo: 0x8a, hi: 0x8a}, + {value: 0x0024, lo: 0x8b, hi: 0x8c}, + {value: 0x0034, lo: 0x8d, hi: 0x90}, + {value: 0x0024, lo: 0x91, hi: 0xb5}, + {value: 0x0034, lo: 0xb6, hi: 0xb9}, + {value: 0x0024, lo: 0xbb, hi: 0xbb}, + {value: 0x0034, lo: 0xbc, hi: 0xbd}, + {value: 0x0024, lo: 0xbe, hi: 0xbe}, + {value: 0x0034, lo: 0xbf, hi: 0xbf}, + // Block 0x5e, offset 0x29a + {value: 0x0117, lo: 0x80, hi: 0xbf}, + // Block 0x5f, offset 0x29b + {value: 0x0117, lo: 0x80, hi: 0x95}, + {value: 0x369a, lo: 0x96, hi: 0x96}, + {value: 0x374a, lo: 0x97, hi: 0x97}, + {value: 0x37fa, lo: 0x98, hi: 0x98}, + {value: 0x38aa, lo: 0x99, hi: 0x99}, + {value: 0x395a, lo: 0x9a, hi: 0x9a}, + {value: 0x3a0a, lo: 0x9b, hi: 0x9b}, + {value: 0x0012, lo: 0x9c, hi: 0x9d}, + {value: 0x3abb, lo: 0x9e, hi: 0x9e}, + {value: 0x0012, lo: 0x9f, hi: 0x9f}, + {value: 0x0117, lo: 0xa0, hi: 0xbf}, + // Block 0x60, offset 0x2a6 + {value: 0x0812, lo: 0x80, hi: 0x87}, + {value: 0x0813, lo: 0x88, hi: 0x8f}, + {value: 0x0812, lo: 0x90, hi: 0x95}, + {value: 0x0813, lo: 0x98, hi: 0x9d}, + {value: 0x0812, lo: 0xa0, hi: 0xa7}, + {value: 0x0813, lo: 0xa8, hi: 0xaf}, + {value: 0x0812, lo: 0xb0, hi: 0xb7}, + {value: 0x0813, lo: 0xb8, hi: 0xbf}, + // Block 0x61, offset 0x2ae + {value: 0x0004, lo: 0x8b, hi: 0x8b}, + {value: 0x0014, lo: 0x8c, hi: 0x8f}, + {value: 0x0054, lo: 0x98, hi: 0x99}, + {value: 0x0054, lo: 0xa4, hi: 0xa4}, + {value: 0x0054, lo: 0xa7, hi: 0xa7}, + {value: 0x0014, lo: 0xaa, hi: 0xae}, + {value: 0x0010, lo: 0xaf, hi: 0xaf}, + {value: 0x0010, lo: 0xbf, hi: 0xbf}, + // Block 0x62, offset 0x2b6 + {value: 0x0010, lo: 0x80, hi: 0x80}, + {value: 0x0010, lo: 0x94, hi: 0x94}, + {value: 0x0014, lo: 0xa0, hi: 0xa4}, + {value: 0x0014, lo: 0xa6, hi: 0xaf}, + {value: 0x0015, lo: 0xb1, hi: 0xb1}, + {value: 0x0015, lo: 0xbf, hi: 0xbf}, + // Block 0x63, offset 0x2bc + {value: 0x0015, lo: 0x90, hi: 0x9c}, + // Block 0x64, offset 0x2bd + {value: 0x0024, lo: 0x90, hi: 0x91}, + {value: 0x0034, lo: 0x92, hi: 0x93}, + {value: 0x0024, lo: 0x94, hi: 0x97}, + {value: 0x0034, lo: 0x98, hi: 0x9a}, + {value: 0x0024, lo: 0x9b, hi: 0x9c}, + {value: 0x0014, lo: 0x9d, hi: 0xa0}, + {value: 0x0024, lo: 0xa1, hi: 0xa1}, + {value: 0x0014, lo: 0xa2, hi: 0xa4}, + {value: 0x0034, lo: 0xa5, hi: 0xa6}, + {value: 0x0024, lo: 0xa7, hi: 0xa7}, + {value: 0x0034, lo: 0xa8, hi: 0xa8}, + {value: 0x0024, lo: 0xa9, hi: 0xa9}, + {value: 0x0034, lo: 0xaa, hi: 0xaf}, + {value: 0x0024, lo: 0xb0, hi: 0xb0}, + // Block 0x65, offset 0x2cb + {value: 0x0016, lo: 0x85, hi: 0x86}, + {value: 0x0012, lo: 0x87, hi: 0x89}, + {value: 0xa452, lo: 0x8e, hi: 0x8e}, + {value: 0x1013, lo: 0xa0, hi: 0xaf}, + {value: 0x1012, lo: 0xb0, hi: 0xbf}, + // Block 0x66, offset 0x2d0 + {value: 0x0010, lo: 0x80, hi: 0x82}, + {value: 0x0716, lo: 0x83, hi: 0x84}, + {value: 0x0010, lo: 0x85, hi: 0x88}, + // Block 0x67, offset 0x2d3 + {value: 0xa753, lo: 0xb6, hi: 0xb7}, + {value: 0xaa53, lo: 0xb8, hi: 0xb9}, + {value: 0xad53, lo: 0xba, hi: 0xbb}, + {value: 0xaa53, lo: 0xbc, hi: 0xbd}, + {value: 0xa753, lo: 0xbe, hi: 0xbf}, + // Block 0x68, offset 0x2d8 + {value: 0x3013, lo: 0x80, hi: 0x8f}, + {value: 0x6553, lo: 0x90, hi: 0x9f}, + {value: 0xb053, lo: 0xa0, hi: 0xae}, + {value: 0x3012, lo: 0xb0, hi: 0xbf}, + // Block 0x69, offset 0x2dc + {value: 0x0117, lo: 0x80, hi: 0xa3}, + {value: 0x0012, lo: 0xa4, hi: 0xa4}, + {value: 0x0716, lo: 0xab, hi: 0xac}, + {value: 0x0316, lo: 0xad, hi: 0xae}, + {value: 0x0024, lo: 0xaf, hi: 0xb1}, + {value: 0x0117, lo: 0xb2, hi: 0xb3}, + // Block 0x6a, offset 0x2e2 + {value: 0x6c52, lo: 0x80, hi: 0x9f}, + {value: 0x7052, lo: 0xa0, hi: 0xa5}, + {value: 0x7052, lo: 0xa7, hi: 0xa7}, + {value: 0x7052, lo: 0xad, hi: 0xad}, + {value: 0x0010, lo: 0xb0, hi: 0xbf}, + // Block 0x6b, offset 0x2e7 + {value: 0x0010, lo: 0x80, hi: 0xa7}, + {value: 0x0014, lo: 0xaf, hi: 0xaf}, + {value: 0x0034, lo: 0xbf, hi: 0xbf}, + // Block 0x6c, offset 0x2ea + {value: 0x0010, lo: 0x80, hi: 0x96}, + {value: 0x0010, lo: 0xa0, hi: 0xa6}, + {value: 0x0010, lo: 0xa8, hi: 0xae}, + {value: 0x0010, lo: 0xb0, hi: 0xb6}, + {value: 0x0010, lo: 0xb8, hi: 0xbe}, + // Block 0x6d, offset 0x2ef + {value: 0x0010, lo: 0x80, hi: 0x86}, + {value: 0x0010, lo: 0x88, hi: 0x8e}, + {value: 0x0010, lo: 0x90, hi: 0x96}, + {value: 0x0010, lo: 0x98, hi: 0x9e}, + {value: 0x0024, lo: 0xa0, hi: 0xbf}, + // Block 0x6e, offset 0x2f4 + {value: 0x0014, lo: 0xaf, hi: 0xaf}, + // Block 0x6f, offset 0x2f5 + {value: 0x0014, lo: 0x85, hi: 0x85}, + {value: 0x0034, lo: 0xaa, hi: 0xad}, + {value: 0x0030, lo: 0xae, hi: 0xaf}, + {value: 0x0004, lo: 0xb1, hi: 0xb5}, + {value: 0x0014, lo: 0xbb, hi: 0xbb}, + {value: 0x0010, lo: 0xbc, hi: 0xbc}, + // Block 0x70, offset 0x2fb + {value: 0x0034, lo: 0x99, hi: 0x9a}, + {value: 0x0004, lo: 0x9b, hi: 0x9e}, + // Block 0x71, offset 0x2fd + {value: 0x0004, lo: 0xbc, hi: 0xbe}, + // Block 0x72, offset 0x2fe + {value: 0x0010, lo: 0x85, hi: 0xaf}, + {value: 0x0010, lo: 0xb1, hi: 0xbf}, + // Block 0x73, offset 0x300 + {value: 0x0010, lo: 0x80, hi: 0x8e}, + {value: 0x0010, lo: 0xa0, hi: 0xba}, + // Block 0x74, offset 0x302 + {value: 0x0010, lo: 0x80, hi: 0x94}, + {value: 0x0014, lo: 0x95, hi: 0x95}, + {value: 0x0010, lo: 0x96, hi: 0xbf}, + // Block 0x75, offset 0x305 + {value: 0x0010, lo: 0x80, hi: 0x8c}, + // Block 0x76, offset 0x306 + {value: 0x0010, lo: 0x90, hi: 0xb7}, + {value: 0x0014, lo: 0xb8, hi: 0xbd}, + // Block 0x77, offset 0x308 + {value: 0x0010, lo: 0x80, hi: 0x8b}, + {value: 0x0014, lo: 0x8c, hi: 0x8c}, + {value: 0x0010, lo: 0x90, hi: 0xab}, + // Block 0x78, offset 0x30b + {value: 0x0117, lo: 0x80, hi: 0xad}, + {value: 0x0010, lo: 0xae, hi: 0xae}, + {value: 0x0024, lo: 0xaf, hi: 0xaf}, + {value: 0x0014, lo: 0xb0, hi: 0xb2}, + {value: 0x0024, lo: 0xb4, hi: 0xbd}, + {value: 0x0014, lo: 0xbf, hi: 0xbf}, + // Block 0x79, offset 0x311 + {value: 0x0117, lo: 0x80, hi: 0x9b}, + {value: 0x0015, lo: 0x9c, hi: 0x9d}, + {value: 0x0024, lo: 0x9e, hi: 0x9f}, + {value: 0x0010, lo: 0xa0, hi: 0xbf}, + // Block 0x7a, offset 0x315 + {value: 0x0010, lo: 0x80, hi: 0xaf}, + {value: 0x0024, lo: 0xb0, hi: 0xb1}, + // Block 0x7b, offset 0x317 + {value: 0x0004, lo: 0x80, hi: 0x96}, + {value: 0x0014, lo: 0x97, hi: 0xa1}, + {value: 0x0117, lo: 0xa2, hi: 0xaf}, + {value: 0x0012, lo: 0xb0, hi: 0xb1}, + {value: 0x0117, lo: 0xb2, hi: 0xbf}, + // Block 0x7c, offset 0x31c + {value: 0x0117, lo: 0x80, hi: 0xaf}, + {value: 0x0015, lo: 0xb0, hi: 0xb0}, + {value: 0x0012, lo: 0xb1, hi: 0xb8}, + {value: 0x0316, lo: 0xb9, hi: 0xba}, + {value: 0x0716, lo: 0xbb, hi: 0xbc}, + {value: 0x8753, lo: 0xbd, hi: 0xbd}, + {value: 0x0117, lo: 0xbe, hi: 0xbf}, + // Block 0x7d, offset 0x323 + {value: 0x0117, lo: 0x82, hi: 0x83}, + {value: 0x6553, lo: 0x84, hi: 0x84}, + {value: 0x908b, lo: 0x85, hi: 0x85}, + {value: 0x8f53, lo: 0x86, hi: 0x86}, + {value: 0x0010, lo: 0xb7, hi: 0xb7}, + {value: 0x0015, lo: 0xb8, hi: 0xb9}, + {value: 0x0012, lo: 0xba, hi: 0xba}, + {value: 0x0010, lo: 0xbb, hi: 0xbf}, + // Block 0x7e, offset 0x32b + {value: 0x0010, lo: 0x80, hi: 0x81}, + {value: 0x0014, lo: 0x82, hi: 0x82}, + {value: 0x0010, lo: 0x83, hi: 0x85}, + {value: 0x0034, lo: 0x86, hi: 0x86}, + {value: 0x0010, lo: 0x87, hi: 0x8a}, + {value: 0x0014, lo: 0x8b, hi: 0x8b}, + {value: 0x0010, lo: 0x8c, hi: 0xa4}, + {value: 0x0014, lo: 0xa5, hi: 0xa6}, + {value: 0x0010, lo: 0xa7, hi: 0xa7}, + // Block 0x7f, offset 0x334 + {value: 0x0010, lo: 0x80, hi: 0xb3}, + // Block 0x80, offset 0x335 + {value: 0x0010, lo: 0x80, hi: 0x83}, + {value: 0x0034, lo: 0x84, hi: 0x84}, + {value: 0x0014, lo: 0x85, hi: 0x85}, + {value: 0x0010, lo: 0x90, hi: 0x99}, + {value: 0x0024, lo: 0xa0, hi: 0xb1}, + {value: 0x0010, lo: 0xb2, hi: 0xb7}, + {value: 0x0010, lo: 0xbb, hi: 0xbb}, + {value: 0x0010, lo: 0xbd, hi: 0xbe}, + {value: 0x0014, lo: 0xbf, hi: 0xbf}, + // Block 0x81, offset 0x33e + {value: 0x0010, lo: 0x80, hi: 0xa5}, + {value: 0x0014, lo: 0xa6, hi: 0xaa}, + {value: 0x0034, lo: 0xab, hi: 0xad}, + {value: 0x0010, lo: 0xb0, hi: 0xbf}, + // Block 0x82, offset 0x342 + {value: 0x0010, lo: 0x80, hi: 0x86}, + {value: 0x0014, lo: 0x87, hi: 0x91}, + {value: 0x0010, lo: 0x92, hi: 0x92}, + {value: 0x0030, lo: 0x93, hi: 0x93}, + {value: 0x0010, lo: 0xa0, hi: 0xbc}, + // Block 0x83, offset 0x347 + {value: 0x0014, lo: 0x80, hi: 0x82}, + {value: 0x0010, lo: 0x83, hi: 0xb2}, + {value: 0x0034, lo: 0xb3, hi: 0xb3}, + {value: 0x0010, lo: 0xb4, hi: 0xb5}, + {value: 0x0014, lo: 0xb6, hi: 0xb9}, + {value: 0x0010, lo: 0xba, hi: 0xbb}, + {value: 0x0014, lo: 0xbc, hi: 0xbd}, + {value: 0x0010, lo: 0xbe, hi: 0xbf}, + // Block 0x84, offset 0x34f + {value: 0x0030, lo: 0x80, hi: 0x80}, + {value: 0x0014, lo: 0x8f, hi: 0x8f}, + {value: 0x0010, lo: 0x90, hi: 0x99}, + {value: 0x0014, lo: 0xa5, hi: 0xa5}, + {value: 0x0004, lo: 0xa6, hi: 0xa6}, + {value: 0x0010, lo: 0xb0, hi: 0xb9}, + // Block 0x85, offset 0x355 + {value: 0x0010, lo: 0x80, hi: 0xa8}, + {value: 0x0014, lo: 0xa9, hi: 0xae}, + {value: 0x0010, lo: 0xaf, hi: 0xb0}, + {value: 0x0014, lo: 0xb1, hi: 0xb2}, + {value: 0x0010, lo: 0xb3, hi: 0xb4}, + {value: 0x0014, lo: 0xb5, hi: 0xb6}, + // Block 0x86, offset 0x35b + {value: 0x0010, lo: 0x80, hi: 0x82}, + {value: 0x0014, lo: 0x83, hi: 0x83}, + {value: 0x0010, lo: 0x84, hi: 0x8b}, + {value: 0x0014, lo: 0x8c, hi: 0x8c}, + {value: 0x0010, lo: 0x8d, hi: 0x8d}, + {value: 0x0010, lo: 0x90, hi: 0x99}, + {value: 0x0004, lo: 0xb0, hi: 0xb0}, + {value: 0x0010, lo: 0xbb, hi: 0xbb}, + {value: 0x0014, lo: 0xbc, hi: 0xbc}, + {value: 0x0010, lo: 0xbd, hi: 0xbd}, + // Block 0x87, offset 0x365 + {value: 0x0024, lo: 0xb0, hi: 0xb0}, + {value: 0x0024, lo: 0xb2, hi: 0xb3}, + {value: 0x0034, lo: 0xb4, hi: 0xb4}, + {value: 0x0024, lo: 0xb7, hi: 0xb8}, + {value: 0x0024, lo: 0xbe, hi: 0xbf}, + // Block 0x88, offset 0x36a + {value: 0x0024, lo: 0x81, hi: 0x81}, + {value: 0x0004, lo: 0x9d, hi: 0x9d}, + {value: 0x0010, lo: 0xa0, hi: 0xab}, + {value: 0x0014, lo: 0xac, hi: 0xad}, + {value: 0x0010, lo: 0xae, hi: 0xaf}, + {value: 0x0010, lo: 0xb2, hi: 0xb2}, + {value: 0x0014, lo: 0xb3, hi: 0xb4}, + {value: 0x0010, lo: 0xb5, hi: 0xb5}, + {value: 0x0034, lo: 0xb6, hi: 0xb6}, + // Block 0x89, offset 0x373 + {value: 0x0010, lo: 0x81, hi: 0x86}, + {value: 0x0010, lo: 0x89, hi: 0x8e}, + {value: 0x0010, lo: 0x91, hi: 0x96}, + {value: 0x0010, lo: 0xa0, hi: 0xa6}, + {value: 0x0010, lo: 0xa8, hi: 0xae}, + {value: 0x0012, lo: 0xb0, hi: 0xbf}, + // Block 0x8a, offset 0x379 + {value: 0x0012, lo: 0x80, hi: 0x92}, + {value: 0xb352, lo: 0x93, hi: 0x93}, + {value: 0x0012, lo: 0x94, hi: 0x9a}, + {value: 0x0014, lo: 0x9b, hi: 0x9b}, + {value: 0x0015, lo: 0x9c, hi: 0x9f}, + {value: 0x0012, lo: 0xa0, hi: 0xa7}, + {value: 0x74d2, lo: 0xb0, hi: 0xbf}, + // Block 0x8b, offset 0x380 + {value: 0x78d2, lo: 0x80, hi: 0x8f}, + {value: 0x7cd2, lo: 0x90, hi: 0x9f}, + {value: 0x80d2, lo: 0xa0, hi: 0xaf}, + {value: 0x7cd2, lo: 0xb0, hi: 0xbf}, + // Block 0x8c, offset 0x384 + {value: 0x0010, lo: 0x80, hi: 0xa4}, + {value: 0x0014, lo: 0xa5, hi: 0xa5}, + {value: 0x0010, lo: 0xa6, hi: 0xa7}, + {value: 0x0014, lo: 0xa8, hi: 0xa8}, + {value: 0x0010, lo: 0xa9, hi: 0xaa}, + {value: 0x0010, lo: 0xac, hi: 0xac}, + {value: 0x0034, lo: 0xad, hi: 0xad}, + {value: 0x0010, lo: 0xb0, hi: 0xb9}, + // Block 0x8d, offset 0x38c + {value: 0x0010, lo: 0x80, hi: 0xa3}, + {value: 0x0010, lo: 0xb0, hi: 0xbf}, + // Block 0x8e, offset 0x38e + {value: 0x0010, lo: 0x80, hi: 0x86}, + {value: 0x0010, lo: 0x8b, hi: 0xbb}, + // Block 0x8f, offset 0x390 + {value: 0x0010, lo: 0x80, hi: 0x81}, + {value: 0x0010, lo: 0x83, hi: 0x84}, + {value: 0x0010, lo: 0x86, hi: 0xbf}, + // Block 0x90, offset 0x393 + {value: 0x0010, lo: 0x80, hi: 0xb1}, + {value: 0x0004, lo: 0xb2, hi: 0xbf}, + // Block 0x91, offset 0x395 + {value: 0x0004, lo: 0x80, hi: 0x81}, + {value: 0x0010, lo: 0x93, hi: 0xbf}, + // Block 0x92, offset 0x397 + {value: 0x0010, lo: 0x80, hi: 0xbd}, + // Block 0x93, offset 0x398 + {value: 0x0010, lo: 0x90, hi: 0xbf}, + // Block 0x94, offset 0x399 + {value: 0x0010, lo: 0x80, hi: 0x8f}, + {value: 0x0010, lo: 0x92, hi: 0xbf}, + // Block 0x95, offset 0x39b + {value: 0x0010, lo: 0x80, hi: 0x87}, + {value: 0x0010, lo: 0xb0, hi: 0xbb}, + // Block 0x96, offset 0x39d + {value: 0x0014, lo: 0x80, hi: 0x8f}, + {value: 0x0054, lo: 0x93, hi: 0x93}, + {value: 0x0024, lo: 0xa0, hi: 0xa6}, + {value: 0x0034, lo: 0xa7, hi: 0xad}, + {value: 0x0024, lo: 0xae, hi: 0xaf}, + {value: 0x0010, lo: 0xb3, hi: 0xb4}, + // Block 0x97, offset 0x3a3 + {value: 0x0010, lo: 0x8d, hi: 0x8f}, + {value: 0x0054, lo: 0x92, hi: 0x92}, + {value: 0x0054, lo: 0x95, hi: 0x95}, + {value: 0x0010, lo: 0xb0, hi: 0xb4}, + {value: 0x0010, lo: 0xb6, hi: 0xbf}, + // Block 0x98, offset 0x3a8 + {value: 0x0010, lo: 0x80, hi: 0xbc}, + {value: 0x0014, lo: 0xbf, hi: 0xbf}, + // Block 0x99, offset 0x3aa + {value: 0x0054, lo: 0x87, hi: 0x87}, + {value: 0x0054, lo: 0x8e, hi: 0x8e}, + {value: 0x0010, lo: 0x90, hi: 0x99}, + {value: 0x0054, lo: 0x9a, hi: 0x9a}, + {value: 0x5f53, lo: 0xa1, hi: 0xba}, + {value: 0x0004, lo: 0xbe, hi: 0xbe}, + {value: 0x0010, lo: 0xbf, hi: 0xbf}, + // Block 0x9a, offset 0x3b1 + {value: 0x0004, lo: 0x80, hi: 0x80}, + {value: 0x5f52, lo: 0x81, hi: 0x9a}, + {value: 0x0004, lo: 0xb0, hi: 0xb0}, + // Block 0x9b, offset 0x3b4 + {value: 0x0014, lo: 0x9e, hi: 0x9f}, + {value: 0x0010, lo: 0xa0, hi: 0xbe}, + // Block 0x9c, offset 0x3b6 + {value: 0x0010, lo: 0x82, hi: 0x87}, + {value: 0x0010, lo: 0x8a, hi: 0x8f}, + {value: 0x0010, lo: 0x92, hi: 0x97}, + {value: 0x0010, lo: 0x9a, hi: 0x9c}, + {value: 0x0004, lo: 0xa3, hi: 0xa3}, + {value: 0x0014, lo: 0xb9, hi: 0xbb}, + // Block 0x9d, offset 0x3bc + {value: 0x0010, lo: 0x80, hi: 0x8b}, + {value: 0x0010, lo: 0x8d, hi: 0xa6}, + {value: 0x0010, lo: 0xa8, hi: 0xba}, + {value: 0x0010, lo: 0xbc, hi: 0xbd}, + {value: 0x0010, lo: 0xbf, hi: 0xbf}, + // Block 0x9e, offset 0x3c1 + {value: 0x0010, lo: 0x80, hi: 0x8d}, + {value: 0x0010, lo: 0x90, hi: 0x9d}, + // Block 0x9f, offset 0x3c3 + {value: 0x0010, lo: 0x80, hi: 0xba}, + // Block 0xa0, offset 0x3c4 + {value: 0x0010, lo: 0x80, hi: 0xb4}, + // Block 0xa1, offset 0x3c5 + {value: 0x0034, lo: 0xbd, hi: 0xbd}, + // Block 0xa2, offset 0x3c6 + {value: 0x0010, lo: 0x80, hi: 0x9c}, + {value: 0x0010, lo: 0xa0, hi: 0xbf}, + // Block 0xa3, offset 0x3c8 + {value: 0x0010, lo: 0x80, hi: 0x90}, + {value: 0x0034, lo: 0xa0, hi: 0xa0}, + // Block 0xa4, offset 0x3ca + {value: 0x0010, lo: 0x80, hi: 0x9f}, + {value: 0x0010, lo: 0xad, hi: 0xbf}, + // Block 0xa5, offset 0x3cc + {value: 0x0010, lo: 0x80, hi: 0x8a}, + {value: 0x0010, lo: 0x90, hi: 0xb5}, + {value: 0x0024, lo: 0xb6, hi: 0xba}, + // Block 0xa6, offset 0x3cf + {value: 0x0010, lo: 0x80, hi: 0x9d}, + {value: 0x0010, lo: 0xa0, hi: 0xbf}, + // Block 0xa7, offset 0x3d1 + {value: 0x0010, lo: 0x80, hi: 0x83}, + {value: 0x0010, lo: 0x88, hi: 0x8f}, + {value: 0x0010, lo: 0x91, hi: 0x95}, + // Block 0xa8, offset 0x3d4 + {value: 0x2813, lo: 0x80, hi: 0x87}, + {value: 0x3813, lo: 0x88, hi: 0x8f}, + {value: 0x2813, lo: 0x90, hi: 0x97}, + {value: 0xb653, lo: 0x98, hi: 0x9f}, + {value: 0xb953, lo: 0xa0, hi: 0xa7}, + {value: 0x2812, lo: 0xa8, hi: 0xaf}, + {value: 0x3812, lo: 0xb0, hi: 0xb7}, + {value: 0x2812, lo: 0xb8, hi: 0xbf}, + // Block 0xa9, offset 0x3dc + {value: 0xb652, lo: 0x80, hi: 0x87}, + {value: 0xb952, lo: 0x88, hi: 0x8f}, + {value: 0x0010, lo: 0x90, hi: 0xbf}, + // Block 0xaa, offset 0x3df + {value: 0x0010, lo: 0x80, hi: 0x9d}, + {value: 0x0010, lo: 0xa0, hi: 0xa9}, + {value: 0xb953, lo: 0xb0, hi: 0xb7}, + {value: 0xb653, lo: 0xb8, hi: 0xbf}, + // Block 0xab, offset 0x3e3 + {value: 0x2813, lo: 0x80, hi: 0x87}, + {value: 0x3813, lo: 0x88, hi: 0x8f}, + {value: 0x2813, lo: 0x90, hi: 0x93}, + {value: 0xb952, lo: 0x98, hi: 0x9f}, + {value: 0xb652, lo: 0xa0, hi: 0xa7}, + {value: 0x2812, lo: 0xa8, hi: 0xaf}, + {value: 0x3812, lo: 0xb0, hi: 0xb7}, + {value: 0x2812, lo: 0xb8, hi: 0xbb}, + // Block 0xac, offset 0x3eb + {value: 0x0010, lo: 0x80, hi: 0xa7}, + {value: 0x0010, lo: 0xb0, hi: 0xbf}, + // Block 0xad, offset 0x3ed + {value: 0x0010, lo: 0x80, hi: 0xa3}, + // Block 0xae, offset 0x3ee + {value: 0x0010, lo: 0x80, hi: 0xb6}, + // Block 0xaf, offset 0x3ef + {value: 0x0010, lo: 0x80, hi: 0x95}, + {value: 0x0010, lo: 0xa0, hi: 0xa7}, + // Block 0xb0, offset 0x3f1 + {value: 0x0010, lo: 0x80, hi: 0x85}, + {value: 0x0010, lo: 0x88, hi: 0x88}, + {value: 0x0010, lo: 0x8a, hi: 0xb5}, + {value: 0x0010, lo: 0xb7, hi: 0xb8}, + {value: 0x0010, lo: 0xbc, hi: 0xbc}, + {value: 0x0010, lo: 0xbf, hi: 0xbf}, + // Block 0xb1, offset 0x3f7 + {value: 0x0010, lo: 0x80, hi: 0x95}, + {value: 0x0010, lo: 0xa0, hi: 0xb6}, + // Block 0xb2, offset 0x3f9 + {value: 0x0010, lo: 0x80, hi: 0x9e}, + // Block 0xb3, offset 0x3fa + {value: 0x0010, lo: 0xa0, hi: 0xb2}, + {value: 0x0010, lo: 0xb4, hi: 0xb5}, + // Block 0xb4, offset 0x3fc + {value: 0x0010, lo: 0x80, hi: 0x95}, + {value: 0x0010, lo: 0xa0, hi: 0xb9}, + // Block 0xb5, offset 0x3fe + {value: 0x0010, lo: 0x80, hi: 0xb7}, + {value: 0x0010, lo: 0xbe, hi: 0xbf}, + // Block 0xb6, offset 0x400 + {value: 0x0010, lo: 0x80, hi: 0x80}, + {value: 0x0014, lo: 0x81, hi: 0x83}, + {value: 0x0014, lo: 0x85, hi: 0x86}, + {value: 0x0014, lo: 0x8c, hi: 0x8c}, + {value: 0x0034, lo: 0x8d, hi: 0x8d}, + {value: 0x0014, lo: 0x8e, hi: 0x8e}, + {value: 0x0024, lo: 0x8f, hi: 0x8f}, + {value: 0x0010, lo: 0x90, hi: 0x93}, + {value: 0x0010, lo: 0x95, hi: 0x97}, + {value: 0x0010, lo: 0x99, hi: 0xb5}, + {value: 0x0024, lo: 0xb8, hi: 0xb8}, + {value: 0x0034, lo: 0xb9, hi: 0xba}, + {value: 0x0034, lo: 0xbf, hi: 0xbf}, + // Block 0xb7, offset 0x40d + {value: 0x0010, lo: 0xa0, hi: 0xbc}, + // Block 0xb8, offset 0x40e + {value: 0x0010, lo: 0x80, hi: 0x9c}, + // Block 0xb9, offset 0x40f + {value: 0x0010, lo: 0x80, hi: 0x87}, + {value: 0x0010, lo: 0x89, hi: 0xa4}, + {value: 0x0024, lo: 0xa5, hi: 0xa5}, + {value: 0x0034, lo: 0xa6, hi: 0xa6}, + // Block 0xba, offset 0x413 + {value: 0x0010, lo: 0x80, hi: 0x95}, + {value: 0x0010, lo: 0xa0, hi: 0xb2}, + // Block 0xbb, offset 0x415 + {value: 0x0010, lo: 0x80, hi: 0x91}, + // Block 0xbc, offset 0x416 + {value: 0x0010, lo: 0x80, hi: 0x88}, + // Block 0xbd, offset 0x417 + {value: 0x5653, lo: 0x80, hi: 0xb2}, + // Block 0xbe, offset 0x418 + {value: 0x5652, lo: 0x80, hi: 0xb2}, + // Block 0xbf, offset 0x419 + {value: 0x0010, lo: 0x80, hi: 0xa3}, + {value: 0x0024, lo: 0xa4, hi: 0xa7}, + {value: 0x0010, lo: 0xb0, hi: 0xb9}, + // Block 0xc0, offset 0x41c + {value: 0x0010, lo: 0x80, hi: 0x9c}, + {value: 0x0010, lo: 0xa7, hi: 0xa7}, + {value: 0x0010, lo: 0xb0, hi: 0xbf}, + // Block 0xc1, offset 0x41f + {value: 0x0010, lo: 0x80, hi: 0x85}, + {value: 0x0034, lo: 0x86, hi: 0x87}, + {value: 0x0024, lo: 0x88, hi: 0x8a}, + {value: 0x0034, lo: 0x8b, hi: 0x8b}, + {value: 0x0024, lo: 0x8c, hi: 0x8c}, + {value: 0x0034, lo: 0x8d, hi: 0x90}, + // Block 0xc2, offset 0x425 + {value: 0x0010, lo: 0xa0, hi: 0xb6}, + // Block 0xc3, offset 0x426 + {value: 0x0010, lo: 0x80, hi: 0x80}, + {value: 0x0014, lo: 0x81, hi: 0x81}, + {value: 0x0010, lo: 0x82, hi: 0xb7}, + {value: 0x0014, lo: 0xb8, hi: 0xbf}, + // Block 0xc4, offset 0x42a + {value: 0x0014, lo: 0x80, hi: 0x85}, + {value: 0x0034, lo: 0x86, hi: 0x86}, + {value: 0x0010, lo: 0xa6, hi: 0xaf}, + {value: 0x0034, lo: 0xbf, hi: 0xbf}, + // Block 0xc5, offset 0x42e + {value: 0x0014, lo: 0x80, hi: 0x81}, + {value: 0x0010, lo: 0x82, hi: 0xb2}, + {value: 0x0014, lo: 0xb3, hi: 0xb6}, + {value: 0x0010, lo: 0xb7, hi: 0xb8}, + {value: 0x0034, lo: 0xb9, hi: 0xba}, + {value: 0x0014, lo: 0xbd, hi: 0xbd}, + // Block 0xc6, offset 0x434 + {value: 0x0014, lo: 0x8d, hi: 0x8d}, + {value: 0x0010, lo: 0x90, hi: 0xa8}, + {value: 0x0010, lo: 0xb0, hi: 0xb9}, + // Block 0xc7, offset 0x437 + {value: 0x0024, lo: 0x80, hi: 0x82}, + {value: 0x0010, lo: 0x83, hi: 0xa6}, + {value: 0x0014, lo: 0xa7, hi: 0xab}, + {value: 0x0010, lo: 0xac, hi: 0xac}, + {value: 0x0014, lo: 0xad, hi: 0xb2}, + {value: 0x0034, lo: 0xb3, hi: 0xb4}, + {value: 0x0010, lo: 0xb6, hi: 0xbf}, + // Block 0xc8, offset 0x43e + {value: 0x0010, lo: 0x84, hi: 0x86}, + {value: 0x0010, lo: 0x90, hi: 0xb2}, + {value: 0x0034, lo: 0xb3, hi: 0xb3}, + {value: 0x0010, lo: 0xb6, hi: 0xb6}, + // Block 0xc9, offset 0x442 + {value: 0x0014, lo: 0x80, hi: 0x81}, + {value: 0x0010, lo: 0x82, hi: 0xb5}, + {value: 0x0014, lo: 0xb6, hi: 0xbe}, + {value: 0x0010, lo: 0xbf, hi: 0xbf}, + // Block 0xca, offset 0x446 + {value: 0x0030, lo: 0x80, hi: 0x80}, + {value: 0x0010, lo: 0x81, hi: 0x84}, + {value: 0x0014, lo: 0x89, hi: 0x89}, + {value: 0x0034, lo: 0x8a, hi: 0x8a}, + {value: 0x0014, lo: 0x8b, hi: 0x8c}, + {value: 0x0010, lo: 0x90, hi: 0x9a}, + {value: 0x0010, lo: 0x9c, hi: 0x9c}, + // Block 0xcb, offset 0x44d + {value: 0x0010, lo: 0x80, hi: 0x91}, + {value: 0x0010, lo: 0x93, hi: 0xae}, + {value: 0x0014, lo: 0xaf, hi: 0xb1}, + {value: 0x0010, lo: 0xb2, hi: 0xb3}, + {value: 0x0014, lo: 0xb4, hi: 0xb4}, + {value: 0x0030, lo: 0xb5, hi: 0xb5}, + {value: 0x0034, lo: 0xb6, hi: 0xb6}, + {value: 0x0014, lo: 0xb7, hi: 0xb7}, + {value: 0x0014, lo: 0xbe, hi: 0xbe}, + // Block 0xcc, offset 0x456 + {value: 0x0010, lo: 0x80, hi: 0x86}, + {value: 0x0010, lo: 0x88, hi: 0x88}, + {value: 0x0010, lo: 0x8a, hi: 0x8d}, + {value: 0x0010, lo: 0x8f, hi: 0x9d}, + {value: 0x0010, lo: 0x9f, hi: 0xa8}, + {value: 0x0010, lo: 0xb0, hi: 0xbf}, + // Block 0xcd, offset 0x45c + {value: 0x0010, lo: 0x80, hi: 0x9e}, + {value: 0x0014, lo: 0x9f, hi: 0x9f}, + {value: 0x0010, lo: 0xa0, hi: 0xa2}, + {value: 0x0014, lo: 0xa3, hi: 0xa8}, + {value: 0x0034, lo: 0xa9, hi: 0xaa}, + {value: 0x0010, lo: 0xb0, hi: 0xb9}, + // Block 0xce, offset 0x462 + {value: 0x0014, lo: 0x80, hi: 0x81}, + {value: 0x0010, lo: 0x82, hi: 0x83}, + {value: 0x0010, lo: 0x85, hi: 0x8c}, + {value: 0x0010, lo: 0x8f, hi: 0x90}, + {value: 0x0010, lo: 0x93, hi: 0xa8}, + {value: 0x0010, lo: 0xaa, hi: 0xb0}, + {value: 0x0010, lo: 0xb2, hi: 0xb3}, + {value: 0x0010, lo: 0xb5, hi: 0xb9}, + {value: 0x0034, lo: 0xbb, hi: 0xbc}, + {value: 0x0010, lo: 0xbd, hi: 0xbf}, + // Block 0xcf, offset 0x46c + {value: 0x0014, lo: 0x80, hi: 0x80}, + {value: 0x0010, lo: 0x81, hi: 0x84}, + {value: 0x0010, lo: 0x87, hi: 0x88}, + {value: 0x0010, lo: 0x8b, hi: 0x8c}, + {value: 0x0030, lo: 0x8d, hi: 0x8d}, + {value: 0x0010, lo: 0x90, hi: 0x90}, + {value: 0x0010, lo: 0x97, hi: 0x97}, + {value: 0x0010, lo: 0x9d, hi: 0xa3}, + {value: 0x0024, lo: 0xa6, hi: 0xac}, + {value: 0x0024, lo: 0xb0, hi: 0xb4}, + // Block 0xd0, offset 0x476 + {value: 0x0010, lo: 0x80, hi: 0xb7}, + {value: 0x0014, lo: 0xb8, hi: 0xbf}, + // Block 0xd1, offset 0x478 + {value: 0x0010, lo: 0x80, hi: 0x81}, + {value: 0x0034, lo: 0x82, hi: 0x82}, + {value: 0x0014, lo: 0x83, hi: 0x84}, + {value: 0x0010, lo: 0x85, hi: 0x85}, + {value: 0x0034, lo: 0x86, hi: 0x86}, + {value: 0x0010, lo: 0x87, hi: 0x8a}, + {value: 0x0010, lo: 0x90, hi: 0x99}, + {value: 0x0024, lo: 0x9e, hi: 0x9e}, + {value: 0x0010, lo: 0x9f, hi: 0x9f}, + // Block 0xd2, offset 0x481 + {value: 0x0010, lo: 0x80, hi: 0xb2}, + {value: 0x0014, lo: 0xb3, hi: 0xb8}, + {value: 0x0010, lo: 0xb9, hi: 0xb9}, + {value: 0x0014, lo: 0xba, hi: 0xba}, + {value: 0x0010, lo: 0xbb, hi: 0xbe}, + {value: 0x0014, lo: 0xbf, hi: 0xbf}, + // Block 0xd3, offset 0x487 + {value: 0x0014, lo: 0x80, hi: 0x80}, + {value: 0x0010, lo: 0x81, hi: 0x81}, + {value: 0x0034, lo: 0x82, hi: 0x83}, + {value: 0x0010, lo: 0x84, hi: 0x85}, + {value: 0x0010, lo: 0x87, hi: 0x87}, + {value: 0x0010, lo: 0x90, hi: 0x99}, + // Block 0xd4, offset 0x48d + {value: 0x0010, lo: 0x80, hi: 0xb1}, + {value: 0x0014, lo: 0xb2, hi: 0xb5}, + {value: 0x0010, lo: 0xb8, hi: 0xbb}, + {value: 0x0014, lo: 0xbc, hi: 0xbd}, + {value: 0x0010, lo: 0xbe, hi: 0xbe}, + {value: 0x0034, lo: 0xbf, hi: 0xbf}, + // Block 0xd5, offset 0x493 + {value: 0x0034, lo: 0x80, hi: 0x80}, + {value: 0x0010, lo: 0x98, hi: 0x9b}, + {value: 0x0014, lo: 0x9c, hi: 0x9d}, + // Block 0xd6, offset 0x496 + {value: 0x0010, lo: 0x80, hi: 0xb2}, + {value: 0x0014, lo: 0xb3, hi: 0xba}, + {value: 0x0010, lo: 0xbb, hi: 0xbc}, + {value: 0x0014, lo: 0xbd, hi: 0xbd}, + {value: 0x0010, lo: 0xbe, hi: 0xbe}, + {value: 0x0034, lo: 0xbf, hi: 0xbf}, + // Block 0xd7, offset 0x49c + {value: 0x0014, lo: 0x80, hi: 0x80}, + {value: 0x0010, lo: 0x84, hi: 0x84}, + {value: 0x0010, lo: 0x90, hi: 0x99}, + // Block 0xd8, offset 0x49f + {value: 0x0010, lo: 0x80, hi: 0xaa}, + {value: 0x0014, lo: 0xab, hi: 0xab}, + {value: 0x0010, lo: 0xac, hi: 0xac}, + {value: 0x0014, lo: 0xad, hi: 0xad}, + {value: 0x0010, lo: 0xae, hi: 0xaf}, + {value: 0x0014, lo: 0xb0, hi: 0xb5}, + {value: 0x0030, lo: 0xb6, hi: 0xb6}, + {value: 0x0034, lo: 0xb7, hi: 0xb7}, + {value: 0x0010, lo: 0xb8, hi: 0xb8}, + // Block 0xd9, offset 0x4a8 + {value: 0x0010, lo: 0x80, hi: 0x89}, + // Block 0xda, offset 0x4a9 + {value: 0x0014, lo: 0x9d, hi: 0x9f}, + {value: 0x0010, lo: 0xa0, hi: 0xa1}, + {value: 0x0014, lo: 0xa2, hi: 0xa5}, + {value: 0x0010, lo: 0xa6, hi: 0xa6}, + {value: 0x0014, lo: 0xa7, hi: 0xaa}, + {value: 0x0034, lo: 0xab, hi: 0xab}, + {value: 0x0010, lo: 0xb0, hi: 0xb9}, + // Block 0xdb, offset 0x4b0 + {value: 0x0010, lo: 0x80, hi: 0xae}, + {value: 0x0014, lo: 0xaf, hi: 0xb7}, + {value: 0x0010, lo: 0xb8, hi: 0xb8}, + {value: 0x0034, lo: 0xb9, hi: 0xba}, + // Block 0xdc, offset 0x4b4 + {value: 0x5f53, lo: 0xa0, hi: 0xbf}, + // Block 0xdd, offset 0x4b5 + {value: 0x5f52, lo: 0x80, hi: 0x9f}, + {value: 0x0010, lo: 0xa0, hi: 0xa9}, + {value: 0x0010, lo: 0xbf, hi: 0xbf}, + // Block 0xde, offset 0x4b8 + {value: 0x0010, lo: 0xa0, hi: 0xa7}, + {value: 0x0010, lo: 0xaa, hi: 0xbf}, + // Block 0xdf, offset 0x4ba + {value: 0x0010, lo: 0x80, hi: 0x93}, + {value: 0x0014, lo: 0x94, hi: 0x97}, + {value: 0x0014, lo: 0x9a, hi: 0x9b}, + {value: 0x0010, lo: 0x9c, hi: 0x9f}, + {value: 0x0034, lo: 0xa0, hi: 0xa0}, + {value: 0x0010, lo: 0xa1, hi: 0xa1}, + {value: 0x0010, lo: 0xa3, hi: 0xa4}, + // Block 0xe0, offset 0x4c1 + {value: 0x0010, lo: 0x80, hi: 0x80}, + {value: 0x0014, lo: 0x81, hi: 0x8a}, + {value: 0x0010, lo: 0x8b, hi: 0xb2}, + {value: 0x0014, lo: 0xb3, hi: 0xb3}, + {value: 0x0034, lo: 0xb4, hi: 0xb4}, + {value: 0x0014, lo: 0xb5, hi: 0xb8}, + {value: 0x0010, lo: 0xb9, hi: 0xba}, + {value: 0x0014, lo: 0xbb, hi: 0xbe}, + // Block 0xe1, offset 0x4c9 + {value: 0x0034, lo: 0x87, hi: 0x87}, + {value: 0x0010, lo: 0x90, hi: 0x90}, + {value: 0x0014, lo: 0x91, hi: 0x96}, + {value: 0x0010, lo: 0x97, hi: 0x98}, + {value: 0x0014, lo: 0x99, hi: 0x9b}, + {value: 0x0010, lo: 0x9c, hi: 0xbf}, + // Block 0xe2, offset 0x4cf + {value: 0x0010, lo: 0x80, hi: 0x89}, + {value: 0x0014, lo: 0x8a, hi: 0x96}, + {value: 0x0010, lo: 0x97, hi: 0x97}, + {value: 0x0014, lo: 0x98, hi: 0x98}, + {value: 0x0034, lo: 0x99, hi: 0x99}, + {value: 0x0010, lo: 0x9d, hi: 0x9d}, + // Block 0xe3, offset 0x4d5 + {value: 0x0010, lo: 0x80, hi: 0xb8}, + // Block 0xe4, offset 0x4d6 + {value: 0x0010, lo: 0x80, hi: 0x88}, + {value: 0x0010, lo: 0x8a, hi: 0xaf}, + {value: 0x0014, lo: 0xb0, hi: 0xb6}, + {value: 0x0014, lo: 0xb8, hi: 0xbd}, + {value: 0x0010, lo: 0xbe, hi: 0xbe}, + {value: 0x0034, lo: 0xbf, hi: 0xbf}, + // Block 0xe5, offset 0x4dc + {value: 0x0010, lo: 0x80, hi: 0x80}, + {value: 0x0010, lo: 0x90, hi: 0x99}, + {value: 0x0010, lo: 0xb2, hi: 0xbf}, + // Block 0xe6, offset 0x4df + {value: 0x0010, lo: 0x80, hi: 0x8f}, + {value: 0x0014, lo: 0x92, hi: 0xa7}, + {value: 0x0010, lo: 0xa9, hi: 0xa9}, + {value: 0x0014, lo: 0xaa, hi: 0xb0}, + {value: 0x0010, lo: 0xb1, hi: 0xb1}, + {value: 0x0014, lo: 0xb2, hi: 0xb3}, + {value: 0x0010, lo: 0xb4, hi: 0xb4}, + {value: 0x0014, lo: 0xb5, hi: 0xb6}, + // Block 0xe7, offset 0x4e7 + {value: 0x0010, lo: 0x80, hi: 0x86}, + {value: 0x0010, lo: 0x88, hi: 0x89}, + {value: 0x0010, lo: 0x8b, hi: 0xb0}, + {value: 0x0014, lo: 0xb1, hi: 0xb6}, + {value: 0x0014, lo: 0xba, hi: 0xba}, + {value: 0x0014, lo: 0xbc, hi: 0xbd}, + {value: 0x0014, lo: 0xbf, hi: 0xbf}, + // Block 0xe8, offset 0x4ee + {value: 0x0014, lo: 0x80, hi: 0x81}, + {value: 0x0034, lo: 0x82, hi: 0x82}, + {value: 0x0014, lo: 0x83, hi: 0x83}, + {value: 0x0034, lo: 0x84, hi: 0x85}, + {value: 0x0010, lo: 0x86, hi: 0x86}, + {value: 0x0014, lo: 0x87, hi: 0x87}, + {value: 0x0010, lo: 0x90, hi: 0x99}, + {value: 0x0010, lo: 0xa0, hi: 0xa5}, + {value: 0x0010, lo: 0xa7, hi: 0xa8}, + {value: 0x0010, lo: 0xaa, hi: 0xbf}, + // Block 0xe9, offset 0x4f8 + {value: 0x0010, lo: 0x80, hi: 0x8e}, + {value: 0x0014, lo: 0x90, hi: 0x91}, + {value: 0x0010, lo: 0x93, hi: 0x94}, + {value: 0x0014, lo: 0x95, hi: 0x95}, + {value: 0x0010, lo: 0x96, hi: 0x96}, + {value: 0x0034, lo: 0x97, hi: 0x97}, + {value: 0x0010, lo: 0x98, hi: 0x98}, + {value: 0x0010, lo: 0xa0, hi: 0xa9}, + // Block 0xea, offset 0x500 + {value: 0x0010, lo: 0xa0, hi: 0xb2}, + {value: 0x0014, lo: 0xb3, hi: 0xb4}, + {value: 0x0010, lo: 0xb5, hi: 0xb6}, + // Block 0xeb, offset 0x503 + {value: 0x0010, lo: 0x80, hi: 0x99}, + // Block 0xec, offset 0x504 + {value: 0x0010, lo: 0x80, hi: 0xae}, + // Block 0xed, offset 0x505 + {value: 0x0010, lo: 0x80, hi: 0x83}, + // Block 0xee, offset 0x506 + {value: 0x0010, lo: 0x80, hi: 0xae}, + {value: 0x0014, lo: 0xb0, hi: 0xb8}, + // Block 0xef, offset 0x508 + {value: 0x0010, lo: 0x80, hi: 0x86}, + // Block 0xf0, offset 0x509 + {value: 0x0010, lo: 0x80, hi: 0x9e}, + {value: 0x0010, lo: 0xa0, hi: 0xa9}, + // Block 0xf1, offset 0x50b + {value: 0x0010, lo: 0x90, hi: 0xad}, + {value: 0x0034, lo: 0xb0, hi: 0xb4}, + // Block 0xf2, offset 0x50d + {value: 0x0010, lo: 0x80, hi: 0xaf}, + {value: 0x0024, lo: 0xb0, hi: 0xb6}, + // Block 0xf3, offset 0x50f + {value: 0x0014, lo: 0x80, hi: 0x83}, + {value: 0x0010, lo: 0x90, hi: 0x99}, + {value: 0x0010, lo: 0xa3, hi: 0xb7}, + {value: 0x0010, lo: 0xbd, hi: 0xbf}, + // Block 0xf4, offset 0x513 + {value: 0x0010, lo: 0x80, hi: 0x8f}, + // Block 0xf5, offset 0x514 + {value: 0x2013, lo: 0x80, hi: 0x9f}, + {value: 0x2012, lo: 0xa0, hi: 0xbf}, + // Block 0xf6, offset 0x516 + {value: 0x0010, lo: 0x80, hi: 0x8a}, + {value: 0x0014, lo: 0x8f, hi: 0x8f}, + {value: 0x0010, lo: 0x90, hi: 0xbf}, + // Block 0xf7, offset 0x519 + {value: 0x0010, lo: 0x80, hi: 0x87}, + {value: 0x0014, lo: 0x8f, hi: 0x9f}, + // Block 0xf8, offset 0x51b + {value: 0x0014, lo: 0xa0, hi: 0xa1}, + {value: 0x0014, lo: 0xa3, hi: 0xa3}, + // Block 0xf9, offset 0x51d + {value: 0x0010, lo: 0x80, hi: 0xaa}, + {value: 0x0010, lo: 0xb0, hi: 0xbc}, + // Block 0xfa, offset 0x51f + {value: 0x0010, lo: 0x80, hi: 0x88}, + {value: 0x0010, lo: 0x90, hi: 0x99}, + {value: 0x0014, lo: 0x9d, hi: 0x9d}, + {value: 0x0034, lo: 0x9e, hi: 0x9e}, + {value: 0x0014, lo: 0xa0, hi: 0xa3}, + // Block 0xfb, offset 0x524 + {value: 0x0030, lo: 0xa5, hi: 0xa6}, + {value: 0x0034, lo: 0xa7, hi: 0xa9}, + {value: 0x0030, lo: 0xad, hi: 0xb2}, + {value: 0x0014, lo: 0xb3, hi: 0xba}, + {value: 0x0034, lo: 0xbb, hi: 0xbf}, + // Block 0xfc, offset 0x529 + {value: 0x0034, lo: 0x80, hi: 0x82}, + {value: 0x0024, lo: 0x85, hi: 0x89}, + {value: 0x0034, lo: 0x8a, hi: 0x8b}, + {value: 0x0024, lo: 0xaa, hi: 0xad}, + // Block 0xfd, offset 0x52d + {value: 0x0024, lo: 0x82, hi: 0x84}, + // Block 0xfe, offset 0x52e + {value: 0x0013, lo: 0x80, hi: 0x99}, + {value: 0x0012, lo: 0x9a, hi: 0xb3}, + {value: 0x0013, lo: 0xb4, hi: 0xbf}, + // Block 0xff, offset 0x531 + {value: 0x0013, lo: 0x80, hi: 0x8d}, + {value: 0x0012, lo: 0x8e, hi: 0x94}, + {value: 0x0012, lo: 0x96, hi: 0xa7}, + {value: 0x0013, lo: 0xa8, hi: 0xbf}, + // Block 0x100, offset 0x535 + {value: 0x0013, lo: 0x80, hi: 0x81}, + {value: 0x0012, lo: 0x82, hi: 0x9b}, + {value: 0x0013, lo: 0x9c, hi: 0x9c}, + {value: 0x0013, lo: 0x9e, hi: 0x9f}, + {value: 0x0013, lo: 0xa2, hi: 0xa2}, + {value: 0x0013, lo: 0xa5, hi: 0xa6}, + {value: 0x0013, lo: 0xa9, hi: 0xac}, + {value: 0x0013, lo: 0xae, hi: 0xb5}, + {value: 0x0012, lo: 0xb6, hi: 0xb9}, + {value: 0x0012, lo: 0xbb, hi: 0xbb}, + {value: 0x0012, lo: 0xbd, hi: 0xbf}, + // Block 0x101, offset 0x540 + {value: 0x0012, lo: 0x80, hi: 0x83}, + {value: 0x0012, lo: 0x85, hi: 0x8f}, + {value: 0x0013, lo: 0x90, hi: 0xa9}, + {value: 0x0012, lo: 0xaa, hi: 0xbf}, + // Block 0x102, offset 0x544 + {value: 0x0012, lo: 0x80, hi: 0x83}, + {value: 0x0013, lo: 0x84, hi: 0x85}, + {value: 0x0013, lo: 0x87, hi: 0x8a}, + {value: 0x0013, lo: 0x8d, hi: 0x94}, + {value: 0x0013, lo: 0x96, hi: 0x9c}, + {value: 0x0012, lo: 0x9e, hi: 0xb7}, + {value: 0x0013, lo: 0xb8, hi: 0xb9}, + {value: 0x0013, lo: 0xbb, hi: 0xbe}, + // Block 0x103, offset 0x54c + {value: 0x0013, lo: 0x80, hi: 0x84}, + {value: 0x0013, lo: 0x86, hi: 0x86}, + {value: 0x0013, lo: 0x8a, hi: 0x90}, + {value: 0x0012, lo: 0x92, hi: 0xab}, + {value: 0x0013, lo: 0xac, hi: 0xbf}, + // Block 0x104, offset 0x551 + {value: 0x0013, lo: 0x80, hi: 0x85}, + {value: 0x0012, lo: 0x86, hi: 0x9f}, + {value: 0x0013, lo: 0xa0, hi: 0xb9}, + {value: 0x0012, lo: 0xba, hi: 0xbf}, + // Block 0x105, offset 0x555 + {value: 0x0012, lo: 0x80, hi: 0x93}, + {value: 0x0013, lo: 0x94, hi: 0xad}, + {value: 0x0012, lo: 0xae, hi: 0xbf}, + // Block 0x106, offset 0x558 + {value: 0x0012, lo: 0x80, hi: 0x87}, + {value: 0x0013, lo: 0x88, hi: 0xa1}, + {value: 0x0012, lo: 0xa2, hi: 0xbb}, + {value: 0x0013, lo: 0xbc, hi: 0xbf}, + // Block 0x107, offset 0x55c + {value: 0x0013, lo: 0x80, hi: 0x95}, + {value: 0x0012, lo: 0x96, hi: 0xaf}, + {value: 0x0013, lo: 0xb0, hi: 0xbf}, + // Block 0x108, offset 0x55f + {value: 0x0013, lo: 0x80, hi: 0x89}, + {value: 0x0012, lo: 0x8a, hi: 0xa5}, + {value: 0x0013, lo: 0xa8, hi: 0xbf}, + // Block 0x109, offset 0x562 + {value: 0x0013, lo: 0x80, hi: 0x80}, + {value: 0x0012, lo: 0x82, hi: 0x9a}, + {value: 0x0012, lo: 0x9c, hi: 0xa1}, + {value: 0x0013, lo: 0xa2, hi: 0xba}, + {value: 0x0012, lo: 0xbc, hi: 0xbf}, + // Block 0x10a, offset 0x567 + {value: 0x0012, lo: 0x80, hi: 0x94}, + {value: 0x0012, lo: 0x96, hi: 0x9b}, + {value: 0x0013, lo: 0x9c, hi: 0xb4}, + {value: 0x0012, lo: 0xb6, hi: 0xbf}, + // Block 0x10b, offset 0x56b + {value: 0x0012, lo: 0x80, hi: 0x8e}, + {value: 0x0012, lo: 0x90, hi: 0x95}, + {value: 0x0013, lo: 0x96, hi: 0xae}, + {value: 0x0012, lo: 0xb0, hi: 0xbf}, + // Block 0x10c, offset 0x56f + {value: 0x0012, lo: 0x80, hi: 0x88}, + {value: 0x0012, lo: 0x8a, hi: 0x8f}, + {value: 0x0013, lo: 0x90, hi: 0xa8}, + {value: 0x0012, lo: 0xaa, hi: 0xbf}, + // Block 0x10d, offset 0x573 + {value: 0x0012, lo: 0x80, hi: 0x82}, + {value: 0x0012, lo: 0x84, hi: 0x89}, + {value: 0x0017, lo: 0x8a, hi: 0x8b}, + {value: 0x0010, lo: 0x8e, hi: 0xbf}, + // Block 0x10e, offset 0x577 + {value: 0x0014, lo: 0x80, hi: 0xb6}, + {value: 0x0014, lo: 0xbb, hi: 0xbf}, + // Block 0x10f, offset 0x579 + {value: 0x0014, lo: 0x80, hi: 0xac}, + {value: 0x0014, lo: 0xb5, hi: 0xb5}, + // Block 0x110, offset 0x57b + {value: 0x0014, lo: 0x84, hi: 0x84}, + {value: 0x0014, lo: 0x9b, hi: 0x9f}, + {value: 0x0014, lo: 0xa1, hi: 0xaf}, + // Block 0x111, offset 0x57e + {value: 0x0024, lo: 0x80, hi: 0x86}, + {value: 0x0024, lo: 0x88, hi: 0x98}, + {value: 0x0024, lo: 0x9b, hi: 0xa1}, + {value: 0x0024, lo: 0xa3, hi: 0xa4}, + {value: 0x0024, lo: 0xa6, hi: 0xaa}, + // Block 0x112, offset 0x583 + {value: 0x0010, lo: 0x80, hi: 0xac}, + {value: 0x0024, lo: 0xb0, hi: 0xb6}, + {value: 0x0014, lo: 0xb7, hi: 0xbd}, + // Block 0x113, offset 0x586 + {value: 0x0010, lo: 0x80, hi: 0x89}, + {value: 0x0010, lo: 0x8e, hi: 0x8e}, + // Block 0x114, offset 0x588 + {value: 0x0010, lo: 0x80, hi: 0xab}, + {value: 0x0024, lo: 0xac, hi: 0xaf}, + {value: 0x0010, lo: 0xb0, hi: 0xb9}, + // Block 0x115, offset 0x58b + {value: 0x0010, lo: 0x80, hi: 0x84}, + {value: 0x0034, lo: 0x90, hi: 0x96}, + // Block 0x116, offset 0x58d + {value: 0xbc52, lo: 0x80, hi: 0x81}, + {value: 0xbf52, lo: 0x82, hi: 0x83}, + {value: 0x0024, lo: 0x84, hi: 0x89}, + {value: 0x0034, lo: 0x8a, hi: 0x8a}, + {value: 0x0014, lo: 0x8b, hi: 0x8b}, + {value: 0x0010, lo: 0x90, hi: 0x99}, + // Block 0x117, offset 0x593 + {value: 0x0010, lo: 0x80, hi: 0x83}, + {value: 0x0010, lo: 0x85, hi: 0x9f}, + {value: 0x0010, lo: 0xa1, hi: 0xa2}, + {value: 0x0010, lo: 0xa4, hi: 0xa4}, + {value: 0x0010, lo: 0xa7, hi: 0xa7}, + {value: 0x0010, lo: 0xa9, hi: 0xb2}, + {value: 0x0010, lo: 0xb4, hi: 0xb7}, + {value: 0x0010, lo: 0xb9, hi: 0xb9}, + {value: 0x0010, lo: 0xbb, hi: 0xbb}, + // Block 0x118, offset 0x59c + {value: 0x0010, lo: 0x80, hi: 0x89}, + {value: 0x0010, lo: 0x8b, hi: 0x9b}, + {value: 0x0010, lo: 0xa1, hi: 0xa3}, + {value: 0x0010, lo: 0xa5, hi: 0xa9}, + {value: 0x0010, lo: 0xab, hi: 0xbb}, + // Block 0x119, offset 0x5a1 + {value: 0x0013, lo: 0xb0, hi: 0xbf}, + // Block 0x11a, offset 0x5a2 + {value: 0x0013, lo: 0x80, hi: 0x89}, + {value: 0x0013, lo: 0x90, hi: 0xa9}, + {value: 0x0013, lo: 0xb0, hi: 0xbf}, + // Block 0x11b, offset 0x5a5 + {value: 0x0013, lo: 0x80, hi: 0x89}, + // Block 0x11c, offset 0x5a6 + {value: 0x0014, lo: 0xbb, hi: 0xbf}, + // Block 0x11d, offset 0x5a7 + {value: 0x0014, lo: 0x81, hi: 0x81}, + {value: 0x0014, lo: 0xa0, hi: 0xbf}, + // Block 0x11e, offset 0x5a9 + {value: 0x0014, lo: 0x80, hi: 0xbf}, + // Block 0x11f, offset 0x5aa + {value: 0x0014, lo: 0x80, hi: 0xaf}, +} + +// Total table size 15070 bytes (14KiB); checksum: 1EB13752 diff --git a/golang.org/x/text/secure/precis/tables11.0.0.go b/golang.org/x/text/secure/precis/tables11.0.0.go index e8371374b8..0db5a9e7b1 100644 --- a/golang.org/x/text/secure/precis/tables11.0.0.go +++ b/golang.org/x/text/secure/precis/tables11.0.0.go @@ -1,6 +1,6 @@ // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. -// +build go1.13 +// +build go1.13,!go1.14 package precis diff --git a/golang.org/x/text/secure/precis/tables12.0.0.go b/golang.org/x/text/secure/precis/tables12.0.0.go new file mode 100644 index 0000000000..014e6bc2fa --- /dev/null +++ b/golang.org/x/text/secure/precis/tables12.0.0.go @@ -0,0 +1,4118 @@ +// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. + +// +build go1.14 + +package precis + +// UnicodeVersion is the Unicode version from which the tables in this package are derived. +const UnicodeVersion = "12.0.0" + +// lookup returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *derivedPropertiesTrie) lookup(s []byte) (v uint8, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return derivedPropertiesValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := derivedPropertiesIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := derivedPropertiesIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = derivedPropertiesIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := derivedPropertiesIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = derivedPropertiesIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = derivedPropertiesIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *derivedPropertiesTrie) lookupUnsafe(s []byte) uint8 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return derivedPropertiesValues[c0] + } + i := derivedPropertiesIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = derivedPropertiesIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = derivedPropertiesIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// lookupString returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *derivedPropertiesTrie) lookupString(s string) (v uint8, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return derivedPropertiesValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := derivedPropertiesIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := derivedPropertiesIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = derivedPropertiesIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := derivedPropertiesIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = derivedPropertiesIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = derivedPropertiesIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *derivedPropertiesTrie) lookupStringUnsafe(s string) uint8 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return derivedPropertiesValues[c0] + } + i := derivedPropertiesIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = derivedPropertiesIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = derivedPropertiesIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// derivedPropertiesTrie. Total size: 27264 bytes (26.62 KiB). Checksum: 529104c7ecd852f6. +type derivedPropertiesTrie struct{} + +func newDerivedPropertiesTrie(i int) *derivedPropertiesTrie { + return &derivedPropertiesTrie{} +} + +// lookupValue determines the type of block n and looks up the value for b. +func (t *derivedPropertiesTrie) lookupValue(n uint32, b byte) uint8 { + switch { + default: + return uint8(derivedPropertiesValues[n<<6+uint32(b)]) + } +} + +// derivedPropertiesValues: 352 blocks, 22528 entries, 22528 bytes +// The third block is the zero block. +var derivedPropertiesValues = [22528]uint8{ + // Block 0x0, offset 0x0 + 0x00: 0x0040, 0x01: 0x0040, 0x02: 0x0040, 0x03: 0x0040, 0x04: 0x0040, 0x05: 0x0040, + 0x06: 0x0040, 0x07: 0x0040, 0x08: 0x0040, 0x09: 0x0040, 0x0a: 0x0040, 0x0b: 0x0040, + 0x0c: 0x0040, 0x0d: 0x0040, 0x0e: 0x0040, 0x0f: 0x0040, 0x10: 0x0040, 0x11: 0x0040, + 0x12: 0x0040, 0x13: 0x0040, 0x14: 0x0040, 0x15: 0x0040, 0x16: 0x0040, 0x17: 0x0040, + 0x18: 0x0040, 0x19: 0x0040, 0x1a: 0x0040, 0x1b: 0x0040, 0x1c: 0x0040, 0x1d: 0x0040, + 0x1e: 0x0040, 0x1f: 0x0040, 0x20: 0x0080, 0x21: 0x00c0, 0x22: 0x00c0, 0x23: 0x00c0, + 0x24: 0x00c0, 0x25: 0x00c0, 0x26: 0x00c0, 0x27: 0x00c0, 0x28: 0x00c0, 0x29: 0x00c0, + 0x2a: 0x00c0, 0x2b: 0x00c0, 0x2c: 0x00c0, 0x2d: 0x00c0, 0x2e: 0x00c0, 0x2f: 0x00c0, + 0x30: 0x00c0, 0x31: 0x00c0, 0x32: 0x00c0, 0x33: 0x00c0, 0x34: 0x00c0, 0x35: 0x00c0, + 0x36: 0x00c0, 0x37: 0x00c0, 0x38: 0x00c0, 0x39: 0x00c0, 0x3a: 0x00c0, 0x3b: 0x00c0, + 0x3c: 0x00c0, 0x3d: 0x00c0, 0x3e: 0x00c0, 0x3f: 0x00c0, + // Block 0x1, offset 0x40 + 0x40: 0x00c0, 0x41: 0x00c0, 0x42: 0x00c0, 0x43: 0x00c0, 0x44: 0x00c0, 0x45: 0x00c0, + 0x46: 0x00c0, 0x47: 0x00c0, 0x48: 0x00c0, 0x49: 0x00c0, 0x4a: 0x00c0, 0x4b: 0x00c0, + 0x4c: 0x00c0, 0x4d: 0x00c0, 0x4e: 0x00c0, 0x4f: 0x00c0, 0x50: 0x00c0, 0x51: 0x00c0, + 0x52: 0x00c0, 0x53: 0x00c0, 0x54: 0x00c0, 0x55: 0x00c0, 0x56: 0x00c0, 0x57: 0x00c0, + 0x58: 0x00c0, 0x59: 0x00c0, 0x5a: 0x00c0, 0x5b: 0x00c0, 0x5c: 0x00c0, 0x5d: 0x00c0, + 0x5e: 0x00c0, 0x5f: 0x00c0, 0x60: 0x00c0, 0x61: 0x00c0, 0x62: 0x00c0, 0x63: 0x00c0, + 0x64: 0x00c0, 0x65: 0x00c0, 0x66: 0x00c0, 0x67: 0x00c0, 0x68: 0x00c0, 0x69: 0x00c0, + 0x6a: 0x00c0, 0x6b: 0x00c0, 0x6c: 0x00c7, 0x6d: 0x00c0, 0x6e: 0x00c0, 0x6f: 0x00c0, + 0x70: 0x00c0, 0x71: 0x00c0, 0x72: 0x00c0, 0x73: 0x00c0, 0x74: 0x00c0, 0x75: 0x00c0, + 0x76: 0x00c0, 0x77: 0x00c0, 0x78: 0x00c0, 0x79: 0x00c0, 0x7a: 0x00c0, 0x7b: 0x00c0, + 0x7c: 0x00c0, 0x7d: 0x00c0, 0x7e: 0x00c0, 0x7f: 0x0040, + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xc0: 0x0040, 0xc1: 0x0040, 0xc2: 0x0040, 0xc3: 0x0040, 0xc4: 0x0040, 0xc5: 0x0040, + 0xc6: 0x0040, 0xc7: 0x0040, 0xc8: 0x0040, 0xc9: 0x0040, 0xca: 0x0040, 0xcb: 0x0040, + 0xcc: 0x0040, 0xcd: 0x0040, 0xce: 0x0040, 0xcf: 0x0040, 0xd0: 0x0040, 0xd1: 0x0040, + 0xd2: 0x0040, 0xd3: 0x0040, 0xd4: 0x0040, 0xd5: 0x0040, 0xd6: 0x0040, 0xd7: 0x0040, + 0xd8: 0x0040, 0xd9: 0x0040, 0xda: 0x0040, 0xdb: 0x0040, 0xdc: 0x0040, 0xdd: 0x0040, + 0xde: 0x0040, 0xdf: 0x0040, 0xe0: 0x0080, 0xe1: 0x0080, 0xe2: 0x0080, 0xe3: 0x0080, + 0xe4: 0x0080, 0xe5: 0x0080, 0xe6: 0x0080, 0xe7: 0x0080, 0xe8: 0x0080, 0xe9: 0x0080, + 0xea: 0x0080, 0xeb: 0x0080, 0xec: 0x0080, 0xed: 0x0040, 0xee: 0x0080, 0xef: 0x0080, + 0xf0: 0x0080, 0xf1: 0x0080, 0xf2: 0x0080, 0xf3: 0x0080, 0xf4: 0x0080, 0xf5: 0x0080, + 0xf6: 0x0080, 0xf7: 0x004f, 0xf8: 0x0080, 0xf9: 0x0080, 0xfa: 0x0080, 0xfb: 0x0080, + 0xfc: 0x0080, 0xfd: 0x0080, 0xfe: 0x0080, 0xff: 0x0080, + // Block 0x4, offset 0x100 + 0x100: 0x00c0, 0x101: 0x00c0, 0x102: 0x00c0, 0x103: 0x00c0, 0x104: 0x00c0, 0x105: 0x00c0, + 0x106: 0x00c0, 0x107: 0x00c0, 0x108: 0x00c0, 0x109: 0x00c0, 0x10a: 0x00c0, 0x10b: 0x00c0, + 0x10c: 0x00c0, 0x10d: 0x00c0, 0x10e: 0x00c0, 0x10f: 0x00c0, 0x110: 0x00c0, 0x111: 0x00c0, + 0x112: 0x00c0, 0x113: 0x00c0, 0x114: 0x00c0, 0x115: 0x00c0, 0x116: 0x00c0, 0x117: 0x0080, + 0x118: 0x00c0, 0x119: 0x00c0, 0x11a: 0x00c0, 0x11b: 0x00c0, 0x11c: 0x00c0, 0x11d: 0x00c0, + 0x11e: 0x00c0, 0x11f: 0x00c0, 0x120: 0x00c0, 0x121: 0x00c0, 0x122: 0x00c0, 0x123: 0x00c0, + 0x124: 0x00c0, 0x125: 0x00c0, 0x126: 0x00c0, 0x127: 0x00c0, 0x128: 0x00c0, 0x129: 0x00c0, + 0x12a: 0x00c0, 0x12b: 0x00c0, 0x12c: 0x00c0, 0x12d: 0x00c0, 0x12e: 0x00c0, 0x12f: 0x00c0, + 0x130: 0x00c0, 0x131: 0x00c0, 0x132: 0x00c0, 0x133: 0x00c0, 0x134: 0x00c0, 0x135: 0x00c0, + 0x136: 0x00c0, 0x137: 0x0080, 0x138: 0x00c0, 0x139: 0x00c0, 0x13a: 0x00c0, 0x13b: 0x00c0, + 0x13c: 0x00c0, 0x13d: 0x00c0, 0x13e: 0x00c0, 0x13f: 0x00c0, + // Block 0x5, offset 0x140 + 0x140: 0x00c0, 0x141: 0x00c0, 0x142: 0x00c0, 0x143: 0x00c0, 0x144: 0x00c0, 0x145: 0x00c0, + 0x146: 0x00c0, 0x147: 0x00c0, 0x148: 0x00c0, 0x149: 0x00c0, 0x14a: 0x00c0, 0x14b: 0x00c0, + 0x14c: 0x00c0, 0x14d: 0x00c0, 0x14e: 0x00c0, 0x14f: 0x00c0, 0x150: 0x00c0, 0x151: 0x00c0, + 0x152: 0x00c0, 0x153: 0x00c0, 0x154: 0x00c0, 0x155: 0x00c0, 0x156: 0x00c0, 0x157: 0x00c0, + 0x158: 0x00c0, 0x159: 0x00c0, 0x15a: 0x00c0, 0x15b: 0x00c0, 0x15c: 0x00c0, 0x15d: 0x00c0, + 0x15e: 0x00c0, 0x15f: 0x00c0, 0x160: 0x00c0, 0x161: 0x00c0, 0x162: 0x00c0, 0x163: 0x00c0, + 0x164: 0x00c0, 0x165: 0x00c0, 0x166: 0x00c0, 0x167: 0x00c0, 0x168: 0x00c0, 0x169: 0x00c0, + 0x16a: 0x00c0, 0x16b: 0x00c0, 0x16c: 0x00c0, 0x16d: 0x00c0, 0x16e: 0x00c0, 0x16f: 0x00c0, + 0x170: 0x00c0, 0x171: 0x00c0, 0x172: 0x0080, 0x173: 0x0080, 0x174: 0x00c0, 0x175: 0x00c0, + 0x176: 0x00c0, 0x177: 0x00c0, 0x178: 0x00c0, 0x179: 0x00c0, 0x17a: 0x00c0, 0x17b: 0x00c0, + 0x17c: 0x00c0, 0x17d: 0x00c0, 0x17e: 0x00c0, 0x17f: 0x0080, + // Block 0x6, offset 0x180 + 0x180: 0x0080, 0x181: 0x00c0, 0x182: 0x00c0, 0x183: 0x00c0, 0x184: 0x00c0, 0x185: 0x00c0, + 0x186: 0x00c0, 0x187: 0x00c0, 0x188: 0x00c0, 0x189: 0x0080, 0x18a: 0x00c0, 0x18b: 0x00c0, + 0x18c: 0x00c0, 0x18d: 0x00c0, 0x18e: 0x00c0, 0x18f: 0x00c0, 0x190: 0x00c0, 0x191: 0x00c0, + 0x192: 0x00c0, 0x193: 0x00c0, 0x194: 0x00c0, 0x195: 0x00c0, 0x196: 0x00c0, 0x197: 0x00c0, + 0x198: 0x00c0, 0x199: 0x00c0, 0x19a: 0x00c0, 0x19b: 0x00c0, 0x19c: 0x00c0, 0x19d: 0x00c0, + 0x19e: 0x00c0, 0x19f: 0x00c0, 0x1a0: 0x00c0, 0x1a1: 0x00c0, 0x1a2: 0x00c0, 0x1a3: 0x00c0, + 0x1a4: 0x00c0, 0x1a5: 0x00c0, 0x1a6: 0x00c0, 0x1a7: 0x00c0, 0x1a8: 0x00c0, 0x1a9: 0x00c0, + 0x1aa: 0x00c0, 0x1ab: 0x00c0, 0x1ac: 0x00c0, 0x1ad: 0x00c0, 0x1ae: 0x00c0, 0x1af: 0x00c0, + 0x1b0: 0x00c0, 0x1b1: 0x00c0, 0x1b2: 0x00c0, 0x1b3: 0x00c0, 0x1b4: 0x00c0, 0x1b5: 0x00c0, + 0x1b6: 0x00c0, 0x1b7: 0x00c0, 0x1b8: 0x00c0, 0x1b9: 0x00c0, 0x1ba: 0x00c0, 0x1bb: 0x00c0, + 0x1bc: 0x00c0, 0x1bd: 0x00c0, 0x1be: 0x00c0, 0x1bf: 0x0080, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x00c0, 0x1c1: 0x00c0, 0x1c2: 0x00c0, 0x1c3: 0x00c0, 0x1c4: 0x00c0, 0x1c5: 0x00c0, + 0x1c6: 0x00c0, 0x1c7: 0x00c0, 0x1c8: 0x00c0, 0x1c9: 0x00c0, 0x1ca: 0x00c0, 0x1cb: 0x00c0, + 0x1cc: 0x00c0, 0x1cd: 0x00c0, 0x1ce: 0x00c0, 0x1cf: 0x00c0, 0x1d0: 0x00c0, 0x1d1: 0x00c0, + 0x1d2: 0x00c0, 0x1d3: 0x00c0, 0x1d4: 0x00c0, 0x1d5: 0x00c0, 0x1d6: 0x00c0, 0x1d7: 0x00c0, + 0x1d8: 0x00c0, 0x1d9: 0x00c0, 0x1da: 0x00c0, 0x1db: 0x00c0, 0x1dc: 0x00c0, 0x1dd: 0x00c0, + 0x1de: 0x00c0, 0x1df: 0x00c0, 0x1e0: 0x00c0, 0x1e1: 0x00c0, 0x1e2: 0x00c0, 0x1e3: 0x00c0, + 0x1e4: 0x00c0, 0x1e5: 0x00c0, 0x1e6: 0x00c0, 0x1e7: 0x00c0, 0x1e8: 0x00c0, 0x1e9: 0x00c0, + 0x1ea: 0x00c0, 0x1eb: 0x00c0, 0x1ec: 0x00c0, 0x1ed: 0x00c0, 0x1ee: 0x00c0, 0x1ef: 0x00c0, + 0x1f0: 0x00c0, 0x1f1: 0x00c0, 0x1f2: 0x00c0, 0x1f3: 0x00c0, 0x1f4: 0x00c0, 0x1f5: 0x00c0, + 0x1f6: 0x00c0, 0x1f7: 0x00c0, 0x1f8: 0x00c0, 0x1f9: 0x00c0, 0x1fa: 0x00c0, 0x1fb: 0x00c0, + 0x1fc: 0x00c0, 0x1fd: 0x00c0, 0x1fe: 0x00c0, 0x1ff: 0x00c0, + // Block 0x8, offset 0x200 + 0x200: 0x00c0, 0x201: 0x00c0, 0x202: 0x00c0, 0x203: 0x00c0, 0x204: 0x0080, 0x205: 0x0080, + 0x206: 0x0080, 0x207: 0x0080, 0x208: 0x0080, 0x209: 0x0080, 0x20a: 0x0080, 0x20b: 0x0080, + 0x20c: 0x0080, 0x20d: 0x00c0, 0x20e: 0x00c0, 0x20f: 0x00c0, 0x210: 0x00c0, 0x211: 0x00c0, + 0x212: 0x00c0, 0x213: 0x00c0, 0x214: 0x00c0, 0x215: 0x00c0, 0x216: 0x00c0, 0x217: 0x00c0, + 0x218: 0x00c0, 0x219: 0x00c0, 0x21a: 0x00c0, 0x21b: 0x00c0, 0x21c: 0x00c0, 0x21d: 0x00c0, + 0x21e: 0x00c0, 0x21f: 0x00c0, 0x220: 0x00c0, 0x221: 0x00c0, 0x222: 0x00c0, 0x223: 0x00c0, + 0x224: 0x00c0, 0x225: 0x00c0, 0x226: 0x00c0, 0x227: 0x00c0, 0x228: 0x00c0, 0x229: 0x00c0, + 0x22a: 0x00c0, 0x22b: 0x00c0, 0x22c: 0x00c0, 0x22d: 0x00c0, 0x22e: 0x00c0, 0x22f: 0x00c0, + 0x230: 0x00c0, 0x231: 0x0080, 0x232: 0x0080, 0x233: 0x0080, 0x234: 0x00c0, 0x235: 0x00c0, + 0x236: 0x00c0, 0x237: 0x00c0, 0x238: 0x00c0, 0x239: 0x00c0, 0x23a: 0x00c0, 0x23b: 0x00c0, + 0x23c: 0x00c0, 0x23d: 0x00c0, 0x23e: 0x00c0, 0x23f: 0x00c0, + // Block 0x9, offset 0x240 + 0x240: 0x00c0, 0x241: 0x00c0, 0x242: 0x00c0, 0x243: 0x00c0, 0x244: 0x00c0, 0x245: 0x00c0, + 0x246: 0x00c0, 0x247: 0x00c0, 0x248: 0x00c0, 0x249: 0x00c0, 0x24a: 0x00c0, 0x24b: 0x00c0, + 0x24c: 0x00c0, 0x24d: 0x00c0, 0x24e: 0x00c0, 0x24f: 0x00c0, 0x250: 0x00c0, 0x251: 0x00c0, + 0x252: 0x00c0, 0x253: 0x00c0, 0x254: 0x00c0, 0x255: 0x00c0, 0x256: 0x00c0, 0x257: 0x00c0, + 0x258: 0x00c0, 0x259: 0x00c0, 0x25a: 0x00c0, 0x25b: 0x00c0, 0x25c: 0x00c0, 0x25d: 0x00c0, + 0x25e: 0x00c0, 0x25f: 0x00c0, 0x260: 0x00c0, 0x261: 0x00c0, 0x262: 0x00c0, 0x263: 0x00c0, + 0x264: 0x00c0, 0x265: 0x00c0, 0x266: 0x00c0, 0x267: 0x00c0, 0x268: 0x00c0, 0x269: 0x00c0, + 0x26a: 0x00c0, 0x26b: 0x00c0, 0x26c: 0x00c0, 0x26d: 0x00c0, 0x26e: 0x00c0, 0x26f: 0x00c0, + 0x270: 0x0080, 0x271: 0x0080, 0x272: 0x0080, 0x273: 0x0080, 0x274: 0x0080, 0x275: 0x0080, + 0x276: 0x0080, 0x277: 0x0080, 0x278: 0x0080, 0x279: 0x00c0, 0x27a: 0x00c0, 0x27b: 0x00c0, + 0x27c: 0x00c0, 0x27d: 0x00c0, 0x27e: 0x00c0, 0x27f: 0x00c0, + // Block 0xa, offset 0x280 + 0x280: 0x00c0, 0x281: 0x00c0, 0x282: 0x0080, 0x283: 0x0080, 0x284: 0x0080, 0x285: 0x0080, + 0x286: 0x00c0, 0x287: 0x00c0, 0x288: 0x00c0, 0x289: 0x00c0, 0x28a: 0x00c0, 0x28b: 0x00c0, + 0x28c: 0x00c0, 0x28d: 0x00c0, 0x28e: 0x00c0, 0x28f: 0x00c0, 0x290: 0x00c0, 0x291: 0x00c0, + 0x292: 0x0080, 0x293: 0x0080, 0x294: 0x0080, 0x295: 0x0080, 0x296: 0x0080, 0x297: 0x0080, + 0x298: 0x0080, 0x299: 0x0080, 0x29a: 0x0080, 0x29b: 0x0080, 0x29c: 0x0080, 0x29d: 0x0080, + 0x29e: 0x0080, 0x29f: 0x0080, 0x2a0: 0x0080, 0x2a1: 0x0080, 0x2a2: 0x0080, 0x2a3: 0x0080, + 0x2a4: 0x0080, 0x2a5: 0x0080, 0x2a6: 0x0080, 0x2a7: 0x0080, 0x2a8: 0x0080, 0x2a9: 0x0080, + 0x2aa: 0x0080, 0x2ab: 0x0080, 0x2ac: 0x00c0, 0x2ad: 0x0080, 0x2ae: 0x00c0, 0x2af: 0x0080, + 0x2b0: 0x0080, 0x2b1: 0x0080, 0x2b2: 0x0080, 0x2b3: 0x0080, 0x2b4: 0x0080, 0x2b5: 0x0080, + 0x2b6: 0x0080, 0x2b7: 0x0080, 0x2b8: 0x0080, 0x2b9: 0x0080, 0x2ba: 0x0080, 0x2bb: 0x0080, + 0x2bc: 0x0080, 0x2bd: 0x0080, 0x2be: 0x0080, 0x2bf: 0x0080, + // Block 0xb, offset 0x2c0 + 0x2c0: 0x00c3, 0x2c1: 0x00c3, 0x2c2: 0x00c3, 0x2c3: 0x00c3, 0x2c4: 0x00c3, 0x2c5: 0x00c3, + 0x2c6: 0x00c3, 0x2c7: 0x00c3, 0x2c8: 0x00c3, 0x2c9: 0x00c3, 0x2ca: 0x00c3, 0x2cb: 0x00c3, + 0x2cc: 0x00c3, 0x2cd: 0x00c3, 0x2ce: 0x00c3, 0x2cf: 0x00c3, 0x2d0: 0x00c3, 0x2d1: 0x00c3, + 0x2d2: 0x00c3, 0x2d3: 0x00c3, 0x2d4: 0x00c3, 0x2d5: 0x00c3, 0x2d6: 0x00c3, 0x2d7: 0x00c3, + 0x2d8: 0x00c3, 0x2d9: 0x00c3, 0x2da: 0x00c3, 0x2db: 0x00c3, 0x2dc: 0x00c3, 0x2dd: 0x00c3, + 0x2de: 0x00c3, 0x2df: 0x00c3, 0x2e0: 0x00c3, 0x2e1: 0x00c3, 0x2e2: 0x00c3, 0x2e3: 0x00c3, + 0x2e4: 0x00c3, 0x2e5: 0x00c3, 0x2e6: 0x00c3, 0x2e7: 0x00c3, 0x2e8: 0x00c3, 0x2e9: 0x00c3, + 0x2ea: 0x00c3, 0x2eb: 0x00c3, 0x2ec: 0x00c3, 0x2ed: 0x00c3, 0x2ee: 0x00c3, 0x2ef: 0x00c3, + 0x2f0: 0x00c3, 0x2f1: 0x00c3, 0x2f2: 0x00c3, 0x2f3: 0x00c3, 0x2f4: 0x00c3, 0x2f5: 0x00c3, + 0x2f6: 0x00c3, 0x2f7: 0x00c3, 0x2f8: 0x00c3, 0x2f9: 0x00c3, 0x2fa: 0x00c3, 0x2fb: 0x00c3, + 0x2fc: 0x00c3, 0x2fd: 0x00c3, 0x2fe: 0x00c3, 0x2ff: 0x00c3, + // Block 0xc, offset 0x300 + 0x300: 0x0083, 0x301: 0x0083, 0x302: 0x00c3, 0x303: 0x0083, 0x304: 0x0083, 0x305: 0x00c3, + 0x306: 0x00c3, 0x307: 0x00c3, 0x308: 0x00c3, 0x309: 0x00c3, 0x30a: 0x00c3, 0x30b: 0x00c3, + 0x30c: 0x00c3, 0x30d: 0x00c3, 0x30e: 0x00c3, 0x30f: 0x0040, 0x310: 0x00c3, 0x311: 0x00c3, + 0x312: 0x00c3, 0x313: 0x00c3, 0x314: 0x00c3, 0x315: 0x00c3, 0x316: 0x00c3, 0x317: 0x00c3, + 0x318: 0x00c3, 0x319: 0x00c3, 0x31a: 0x00c3, 0x31b: 0x00c3, 0x31c: 0x00c3, 0x31d: 0x00c3, + 0x31e: 0x00c3, 0x31f: 0x00c3, 0x320: 0x00c3, 0x321: 0x00c3, 0x322: 0x00c3, 0x323: 0x00c3, + 0x324: 0x00c3, 0x325: 0x00c3, 0x326: 0x00c3, 0x327: 0x00c3, 0x328: 0x00c3, 0x329: 0x00c3, + 0x32a: 0x00c3, 0x32b: 0x00c3, 0x32c: 0x00c3, 0x32d: 0x00c3, 0x32e: 0x00c3, 0x32f: 0x00c3, + 0x330: 0x00c8, 0x331: 0x00c8, 0x332: 0x00c8, 0x333: 0x00c8, 0x334: 0x0080, 0x335: 0x0050, + 0x336: 0x00c8, 0x337: 0x00c8, 0x33a: 0x0088, 0x33b: 0x00c8, + 0x33c: 0x00c8, 0x33d: 0x00c8, 0x33e: 0x0080, 0x33f: 0x00c8, + // Block 0xd, offset 0x340 + 0x344: 0x0088, 0x345: 0x0080, + 0x346: 0x00c8, 0x347: 0x0080, 0x348: 0x00c8, 0x349: 0x00c8, 0x34a: 0x00c8, + 0x34c: 0x00c8, 0x34e: 0x00c8, 0x34f: 0x00c8, 0x350: 0x00c8, 0x351: 0x00c8, + 0x352: 0x00c8, 0x353: 0x00c8, 0x354: 0x00c8, 0x355: 0x00c8, 0x356: 0x00c8, 0x357: 0x00c8, + 0x358: 0x00c8, 0x359: 0x00c8, 0x35a: 0x00c8, 0x35b: 0x00c8, 0x35c: 0x00c8, 0x35d: 0x00c8, + 0x35e: 0x00c8, 0x35f: 0x00c8, 0x360: 0x00c8, 0x361: 0x00c8, 0x363: 0x00c8, + 0x364: 0x00c8, 0x365: 0x00c8, 0x366: 0x00c8, 0x367: 0x00c8, 0x368: 0x00c8, 0x369: 0x00c8, + 0x36a: 0x00c8, 0x36b: 0x00c8, 0x36c: 0x00c8, 0x36d: 0x00c8, 0x36e: 0x00c8, 0x36f: 0x00c8, + 0x370: 0x00c8, 0x371: 0x00c8, 0x372: 0x00c8, 0x373: 0x00c8, 0x374: 0x00c8, 0x375: 0x00c8, + 0x376: 0x00c8, 0x377: 0x00c8, 0x378: 0x00c8, 0x379: 0x00c8, 0x37a: 0x00c8, 0x37b: 0x00c8, + 0x37c: 0x00c8, 0x37d: 0x00c8, 0x37e: 0x00c8, 0x37f: 0x00c8, + // Block 0xe, offset 0x380 + 0x380: 0x00c8, 0x381: 0x00c8, 0x382: 0x00c8, 0x383: 0x00c8, 0x384: 0x00c8, 0x385: 0x00c8, + 0x386: 0x00c8, 0x387: 0x00c8, 0x388: 0x00c8, 0x389: 0x00c8, 0x38a: 0x00c8, 0x38b: 0x00c8, + 0x38c: 0x00c8, 0x38d: 0x00c8, 0x38e: 0x00c8, 0x38f: 0x00c8, 0x390: 0x0088, 0x391: 0x0088, + 0x392: 0x0088, 0x393: 0x0088, 0x394: 0x0088, 0x395: 0x0088, 0x396: 0x0088, 0x397: 0x00c8, + 0x398: 0x00c8, 0x399: 0x00c8, 0x39a: 0x00c8, 0x39b: 0x00c8, 0x39c: 0x00c8, 0x39d: 0x00c8, + 0x39e: 0x00c8, 0x39f: 0x00c8, 0x3a0: 0x00c8, 0x3a1: 0x00c8, 0x3a2: 0x00c0, 0x3a3: 0x00c0, + 0x3a4: 0x00c0, 0x3a5: 0x00c0, 0x3a6: 0x00c0, 0x3a7: 0x00c0, 0x3a8: 0x00c0, 0x3a9: 0x00c0, + 0x3aa: 0x00c0, 0x3ab: 0x00c0, 0x3ac: 0x00c0, 0x3ad: 0x00c0, 0x3ae: 0x00c0, 0x3af: 0x00c0, + 0x3b0: 0x0088, 0x3b1: 0x0088, 0x3b2: 0x0088, 0x3b3: 0x00c8, 0x3b4: 0x0088, 0x3b5: 0x0088, + 0x3b6: 0x0088, 0x3b7: 0x00c8, 0x3b8: 0x00c8, 0x3b9: 0x0088, 0x3ba: 0x00c8, 0x3bb: 0x00c8, + 0x3bc: 0x00c8, 0x3bd: 0x00c8, 0x3be: 0x00c8, 0x3bf: 0x00c8, + // Block 0xf, offset 0x3c0 + 0x3c0: 0x00c0, 0x3c1: 0x00c0, 0x3c2: 0x0080, 0x3c3: 0x00c3, 0x3c4: 0x00c3, 0x3c5: 0x00c3, + 0x3c6: 0x00c3, 0x3c7: 0x00c3, 0x3c8: 0x0083, 0x3c9: 0x0083, 0x3ca: 0x00c0, 0x3cb: 0x00c0, + 0x3cc: 0x00c0, 0x3cd: 0x00c0, 0x3ce: 0x00c0, 0x3cf: 0x00c0, 0x3d0: 0x00c0, 0x3d1: 0x00c0, + 0x3d2: 0x00c0, 0x3d3: 0x00c0, 0x3d4: 0x00c0, 0x3d5: 0x00c0, 0x3d6: 0x00c0, 0x3d7: 0x00c0, + 0x3d8: 0x00c0, 0x3d9: 0x00c0, 0x3da: 0x00c0, 0x3db: 0x00c0, 0x3dc: 0x00c0, 0x3dd: 0x00c0, + 0x3de: 0x00c0, 0x3df: 0x00c0, 0x3e0: 0x00c0, 0x3e1: 0x00c0, 0x3e2: 0x00c0, 0x3e3: 0x00c0, + 0x3e4: 0x00c0, 0x3e5: 0x00c0, 0x3e6: 0x00c0, 0x3e7: 0x00c0, 0x3e8: 0x00c0, 0x3e9: 0x00c0, + 0x3ea: 0x00c0, 0x3eb: 0x00c0, 0x3ec: 0x00c0, 0x3ed: 0x00c0, 0x3ee: 0x00c0, 0x3ef: 0x00c0, + 0x3f0: 0x00c0, 0x3f1: 0x00c0, 0x3f2: 0x00c0, 0x3f3: 0x00c0, 0x3f4: 0x00c0, 0x3f5: 0x00c0, + 0x3f6: 0x00c0, 0x3f7: 0x00c0, 0x3f8: 0x00c0, 0x3f9: 0x00c0, 0x3fa: 0x00c0, 0x3fb: 0x00c0, + 0x3fc: 0x00c0, 0x3fd: 0x00c0, 0x3fe: 0x00c0, 0x3ff: 0x00c0, + // Block 0x10, offset 0x400 + 0x400: 0x00c0, 0x401: 0x00c0, 0x402: 0x00c0, 0x403: 0x00c0, 0x404: 0x00c0, 0x405: 0x00c0, + 0x406: 0x00c0, 0x407: 0x00c0, 0x408: 0x00c0, 0x409: 0x00c0, 0x40a: 0x00c0, 0x40b: 0x00c0, + 0x40c: 0x00c0, 0x40d: 0x00c0, 0x40e: 0x00c0, 0x40f: 0x00c0, 0x410: 0x00c0, 0x411: 0x00c0, + 0x412: 0x00c0, 0x413: 0x00c0, 0x414: 0x00c0, 0x415: 0x00c0, 0x416: 0x00c0, 0x417: 0x00c0, + 0x418: 0x00c0, 0x419: 0x00c0, 0x41a: 0x00c0, 0x41b: 0x00c0, 0x41c: 0x00c0, 0x41d: 0x00c0, + 0x41e: 0x00c0, 0x41f: 0x00c0, 0x420: 0x00c0, 0x421: 0x00c0, 0x422: 0x00c0, 0x423: 0x00c0, + 0x424: 0x00c0, 0x425: 0x00c0, 0x426: 0x00c0, 0x427: 0x00c0, 0x428: 0x00c0, 0x429: 0x00c0, + 0x42a: 0x00c0, 0x42b: 0x00c0, 0x42c: 0x00c0, 0x42d: 0x00c0, 0x42e: 0x00c0, 0x42f: 0x00c0, + 0x431: 0x00c0, 0x432: 0x00c0, 0x433: 0x00c0, 0x434: 0x00c0, 0x435: 0x00c0, + 0x436: 0x00c0, 0x437: 0x00c0, 0x438: 0x00c0, 0x439: 0x00c0, 0x43a: 0x00c0, 0x43b: 0x00c0, + 0x43c: 0x00c0, 0x43d: 0x00c0, 0x43e: 0x00c0, 0x43f: 0x00c0, + // Block 0x11, offset 0x440 + 0x440: 0x00c0, 0x441: 0x00c0, 0x442: 0x00c0, 0x443: 0x00c0, 0x444: 0x00c0, 0x445: 0x00c0, + 0x446: 0x00c0, 0x447: 0x00c0, 0x448: 0x00c0, 0x449: 0x00c0, 0x44a: 0x00c0, 0x44b: 0x00c0, + 0x44c: 0x00c0, 0x44d: 0x00c0, 0x44e: 0x00c0, 0x44f: 0x00c0, 0x450: 0x00c0, 0x451: 0x00c0, + 0x452: 0x00c0, 0x453: 0x00c0, 0x454: 0x00c0, 0x455: 0x00c0, 0x456: 0x00c0, + 0x459: 0x00c0, 0x45a: 0x0080, 0x45b: 0x0080, 0x45c: 0x0080, 0x45d: 0x0080, + 0x45e: 0x0080, 0x45f: 0x0080, 0x460: 0x00c0, 0x461: 0x00c0, 0x462: 0x00c0, 0x463: 0x00c0, + 0x464: 0x00c0, 0x465: 0x00c0, 0x466: 0x00c0, 0x467: 0x00c0, 0x468: 0x00c0, 0x469: 0x00c0, + 0x46a: 0x00c0, 0x46b: 0x00c0, 0x46c: 0x00c0, 0x46d: 0x00c0, 0x46e: 0x00c0, 0x46f: 0x00c0, + 0x470: 0x00c0, 0x471: 0x00c0, 0x472: 0x00c0, 0x473: 0x00c0, 0x474: 0x00c0, 0x475: 0x00c0, + 0x476: 0x00c0, 0x477: 0x00c0, 0x478: 0x00c0, 0x479: 0x00c0, 0x47a: 0x00c0, 0x47b: 0x00c0, + 0x47c: 0x00c0, 0x47d: 0x00c0, 0x47e: 0x00c0, 0x47f: 0x00c0, + // Block 0x12, offset 0x480 + 0x480: 0x00c0, 0x481: 0x00c0, 0x482: 0x00c0, 0x483: 0x00c0, 0x484: 0x00c0, 0x485: 0x00c0, + 0x486: 0x00c0, 0x487: 0x0080, 0x488: 0x00c0, 0x489: 0x0080, 0x48a: 0x0080, + 0x48d: 0x0080, 0x48e: 0x0080, 0x48f: 0x0080, 0x491: 0x00cb, + 0x492: 0x00cb, 0x493: 0x00cb, 0x494: 0x00cb, 0x495: 0x00cb, 0x496: 0x00cb, 0x497: 0x00cb, + 0x498: 0x00cb, 0x499: 0x00cb, 0x49a: 0x00cb, 0x49b: 0x00cb, 0x49c: 0x00cb, 0x49d: 0x00cb, + 0x49e: 0x00cb, 0x49f: 0x00cb, 0x4a0: 0x00cb, 0x4a1: 0x00cb, 0x4a2: 0x00cb, 0x4a3: 0x00cb, + 0x4a4: 0x00cb, 0x4a5: 0x00cb, 0x4a6: 0x00cb, 0x4a7: 0x00cb, 0x4a8: 0x00cb, 0x4a9: 0x00cb, + 0x4aa: 0x00cb, 0x4ab: 0x00cb, 0x4ac: 0x00cb, 0x4ad: 0x00cb, 0x4ae: 0x00cb, 0x4af: 0x00cb, + 0x4b0: 0x00cb, 0x4b1: 0x00cb, 0x4b2: 0x00cb, 0x4b3: 0x00cb, 0x4b4: 0x00cb, 0x4b5: 0x00cb, + 0x4b6: 0x00cb, 0x4b7: 0x00cb, 0x4b8: 0x00cb, 0x4b9: 0x00cb, 0x4ba: 0x00cb, 0x4bb: 0x00cb, + 0x4bc: 0x00cb, 0x4bd: 0x00cb, 0x4be: 0x008a, 0x4bf: 0x00cb, + // Block 0x13, offset 0x4c0 + 0x4c0: 0x008a, 0x4c1: 0x00cb, 0x4c2: 0x00cb, 0x4c3: 0x008a, 0x4c4: 0x00cb, 0x4c5: 0x00cb, + 0x4c6: 0x008a, 0x4c7: 0x00cb, + 0x4d0: 0x00ca, 0x4d1: 0x00ca, + 0x4d2: 0x00ca, 0x4d3: 0x00ca, 0x4d4: 0x00ca, 0x4d5: 0x00ca, 0x4d6: 0x00ca, 0x4d7: 0x00ca, + 0x4d8: 0x00ca, 0x4d9: 0x00ca, 0x4da: 0x00ca, 0x4db: 0x00ca, 0x4dc: 0x00ca, 0x4dd: 0x00ca, + 0x4de: 0x00ca, 0x4df: 0x00ca, 0x4e0: 0x00ca, 0x4e1: 0x00ca, 0x4e2: 0x00ca, 0x4e3: 0x00ca, + 0x4e4: 0x00ca, 0x4e5: 0x00ca, 0x4e6: 0x00ca, 0x4e7: 0x00ca, 0x4e8: 0x00ca, 0x4e9: 0x00ca, + 0x4ea: 0x00ca, 0x4ef: 0x00ca, + 0x4f0: 0x00ca, 0x4f1: 0x00ca, 0x4f2: 0x00ca, 0x4f3: 0x0051, 0x4f4: 0x0051, + // Block 0x14, offset 0x500 + 0x500: 0x0040, 0x501: 0x0040, 0x502: 0x0040, 0x503: 0x0040, 0x504: 0x0040, 0x505: 0x0040, + 0x506: 0x0080, 0x507: 0x0080, 0x508: 0x0080, 0x509: 0x0080, 0x50a: 0x0080, 0x50b: 0x0080, + 0x50c: 0x0080, 0x50d: 0x0080, 0x50e: 0x0080, 0x50f: 0x0080, 0x510: 0x00c3, 0x511: 0x00c3, + 0x512: 0x00c3, 0x513: 0x00c3, 0x514: 0x00c3, 0x515: 0x00c3, 0x516: 0x00c3, 0x517: 0x00c3, + 0x518: 0x00c3, 0x519: 0x00c3, 0x51a: 0x00c3, 0x51b: 0x0080, 0x51c: 0x0040, + 0x51e: 0x0080, 0x51f: 0x0080, 0x520: 0x00c2, 0x521: 0x00c0, 0x522: 0x00c4, 0x523: 0x00c4, + 0x524: 0x00c4, 0x525: 0x00c4, 0x526: 0x00c2, 0x527: 0x00c4, 0x528: 0x00c2, 0x529: 0x00c4, + 0x52a: 0x00c2, 0x52b: 0x00c2, 0x52c: 0x00c2, 0x52d: 0x00c2, 0x52e: 0x00c2, 0x52f: 0x00c4, + 0x530: 0x00c4, 0x531: 0x00c4, 0x532: 0x00c4, 0x533: 0x00c2, 0x534: 0x00c2, 0x535: 0x00c2, + 0x536: 0x00c2, 0x537: 0x00c2, 0x538: 0x00c2, 0x539: 0x00c2, 0x53a: 0x00c2, 0x53b: 0x00c2, + 0x53c: 0x00c2, 0x53d: 0x00c2, 0x53e: 0x00c2, 0x53f: 0x00c2, + // Block 0x15, offset 0x540 + 0x540: 0x0040, 0x541: 0x00c2, 0x542: 0x00c2, 0x543: 0x00c2, 0x544: 0x00c2, 0x545: 0x00c2, + 0x546: 0x00c2, 0x547: 0x00c2, 0x548: 0x00c4, 0x549: 0x00c2, 0x54a: 0x00c2, 0x54b: 0x00c3, + 0x54c: 0x00c3, 0x54d: 0x00c3, 0x54e: 0x00c3, 0x54f: 0x00c3, 0x550: 0x00c3, 0x551: 0x00c3, + 0x552: 0x00c3, 0x553: 0x00c3, 0x554: 0x00c3, 0x555: 0x00c3, 0x556: 0x00c3, 0x557: 0x00c3, + 0x558: 0x00c3, 0x559: 0x00c3, 0x55a: 0x00c3, 0x55b: 0x00c3, 0x55c: 0x00c3, 0x55d: 0x00c3, + 0x55e: 0x00c3, 0x55f: 0x00c3, 0x560: 0x0053, 0x561: 0x0053, 0x562: 0x0053, 0x563: 0x0053, + 0x564: 0x0053, 0x565: 0x0053, 0x566: 0x0053, 0x567: 0x0053, 0x568: 0x0053, 0x569: 0x0053, + 0x56a: 0x0080, 0x56b: 0x0080, 0x56c: 0x0080, 0x56d: 0x0080, 0x56e: 0x00c2, 0x56f: 0x00c2, + 0x570: 0x00c3, 0x571: 0x00c4, 0x572: 0x00c4, 0x573: 0x00c4, 0x574: 0x00c0, 0x575: 0x0084, + 0x576: 0x0084, 0x577: 0x0084, 0x578: 0x0082, 0x579: 0x00c2, 0x57a: 0x00c2, 0x57b: 0x00c2, + 0x57c: 0x00c2, 0x57d: 0x00c2, 0x57e: 0x00c2, 0x57f: 0x00c2, + // Block 0x16, offset 0x580 + 0x580: 0x00c2, 0x581: 0x00c2, 0x582: 0x00c2, 0x583: 0x00c2, 0x584: 0x00c2, 0x585: 0x00c2, + 0x586: 0x00c2, 0x587: 0x00c2, 0x588: 0x00c4, 0x589: 0x00c4, 0x58a: 0x00c4, 0x58b: 0x00c4, + 0x58c: 0x00c4, 0x58d: 0x00c4, 0x58e: 0x00c4, 0x58f: 0x00c4, 0x590: 0x00c4, 0x591: 0x00c4, + 0x592: 0x00c4, 0x593: 0x00c4, 0x594: 0x00c4, 0x595: 0x00c4, 0x596: 0x00c4, 0x597: 0x00c4, + 0x598: 0x00c4, 0x599: 0x00c4, 0x59a: 0x00c2, 0x59b: 0x00c2, 0x59c: 0x00c2, 0x59d: 0x00c2, + 0x59e: 0x00c2, 0x59f: 0x00c2, 0x5a0: 0x00c2, 0x5a1: 0x00c2, 0x5a2: 0x00c2, 0x5a3: 0x00c2, + 0x5a4: 0x00c2, 0x5a5: 0x00c2, 0x5a6: 0x00c2, 0x5a7: 0x00c2, 0x5a8: 0x00c2, 0x5a9: 0x00c2, + 0x5aa: 0x00c2, 0x5ab: 0x00c2, 0x5ac: 0x00c2, 0x5ad: 0x00c2, 0x5ae: 0x00c2, 0x5af: 0x00c2, + 0x5b0: 0x00c2, 0x5b1: 0x00c2, 0x5b2: 0x00c2, 0x5b3: 0x00c2, 0x5b4: 0x00c2, 0x5b5: 0x00c2, + 0x5b6: 0x00c2, 0x5b7: 0x00c2, 0x5b8: 0x00c2, 0x5b9: 0x00c2, 0x5ba: 0x00c2, 0x5bb: 0x00c2, + 0x5bc: 0x00c2, 0x5bd: 0x00c2, 0x5be: 0x00c2, 0x5bf: 0x00c2, + // Block 0x17, offset 0x5c0 + 0x5c0: 0x00c4, 0x5c1: 0x00c2, 0x5c2: 0x00c2, 0x5c3: 0x00c4, 0x5c4: 0x00c4, 0x5c5: 0x00c4, + 0x5c6: 0x00c4, 0x5c7: 0x00c4, 0x5c8: 0x00c4, 0x5c9: 0x00c4, 0x5ca: 0x00c4, 0x5cb: 0x00c4, + 0x5cc: 0x00c2, 0x5cd: 0x00c4, 0x5ce: 0x00c2, 0x5cf: 0x00c4, 0x5d0: 0x00c2, 0x5d1: 0x00c2, + 0x5d2: 0x00c4, 0x5d3: 0x00c4, 0x5d4: 0x0080, 0x5d5: 0x00c4, 0x5d6: 0x00c3, 0x5d7: 0x00c3, + 0x5d8: 0x00c3, 0x5d9: 0x00c3, 0x5da: 0x00c3, 0x5db: 0x00c3, 0x5dc: 0x00c3, 0x5dd: 0x0040, + 0x5de: 0x0080, 0x5df: 0x00c3, 0x5e0: 0x00c3, 0x5e1: 0x00c3, 0x5e2: 0x00c3, 0x5e3: 0x00c3, + 0x5e4: 0x00c3, 0x5e5: 0x00c0, 0x5e6: 0x00c0, 0x5e7: 0x00c3, 0x5e8: 0x00c3, 0x5e9: 0x0080, + 0x5ea: 0x00c3, 0x5eb: 0x00c3, 0x5ec: 0x00c3, 0x5ed: 0x00c3, 0x5ee: 0x00c4, 0x5ef: 0x00c4, + 0x5f0: 0x0054, 0x5f1: 0x0054, 0x5f2: 0x0054, 0x5f3: 0x0054, 0x5f4: 0x0054, 0x5f5: 0x0054, + 0x5f6: 0x0054, 0x5f7: 0x0054, 0x5f8: 0x0054, 0x5f9: 0x0054, 0x5fa: 0x00c2, 0x5fb: 0x00c2, + 0x5fc: 0x00c2, 0x5fd: 0x00c0, 0x5fe: 0x00c0, 0x5ff: 0x00c2, + // Block 0x18, offset 0x600 + 0x600: 0x0080, 0x601: 0x0080, 0x602: 0x0080, 0x603: 0x0080, 0x604: 0x0080, 0x605: 0x0080, + 0x606: 0x0080, 0x607: 0x0080, 0x608: 0x0080, 0x609: 0x0080, 0x60a: 0x0080, 0x60b: 0x0080, + 0x60c: 0x0080, 0x60d: 0x0080, 0x60f: 0x0040, 0x610: 0x00c4, 0x611: 0x00c3, + 0x612: 0x00c2, 0x613: 0x00c2, 0x614: 0x00c2, 0x615: 0x00c4, 0x616: 0x00c4, 0x617: 0x00c4, + 0x618: 0x00c4, 0x619: 0x00c4, 0x61a: 0x00c2, 0x61b: 0x00c2, 0x61c: 0x00c2, 0x61d: 0x00c2, + 0x61e: 0x00c4, 0x61f: 0x00c2, 0x620: 0x00c2, 0x621: 0x00c2, 0x622: 0x00c2, 0x623: 0x00c2, + 0x624: 0x00c2, 0x625: 0x00c2, 0x626: 0x00c2, 0x627: 0x00c2, 0x628: 0x00c4, 0x629: 0x00c2, + 0x62a: 0x00c4, 0x62b: 0x00c2, 0x62c: 0x00c4, 0x62d: 0x00c2, 0x62e: 0x00c2, 0x62f: 0x00c4, + 0x630: 0x00c3, 0x631: 0x00c3, 0x632: 0x00c3, 0x633: 0x00c3, 0x634: 0x00c3, 0x635: 0x00c3, + 0x636: 0x00c3, 0x637: 0x00c3, 0x638: 0x00c3, 0x639: 0x00c3, 0x63a: 0x00c3, 0x63b: 0x00c3, + 0x63c: 0x00c3, 0x63d: 0x00c3, 0x63e: 0x00c3, 0x63f: 0x00c3, + // Block 0x19, offset 0x640 + 0x640: 0x00c3, 0x641: 0x00c3, 0x642: 0x00c3, 0x643: 0x00c3, 0x644: 0x00c3, 0x645: 0x00c3, + 0x646: 0x00c3, 0x647: 0x00c3, 0x648: 0x00c3, 0x649: 0x00c3, 0x64a: 0x00c3, + 0x64d: 0x00c4, 0x64e: 0x00c2, 0x64f: 0x00c2, 0x650: 0x00c2, 0x651: 0x00c2, + 0x652: 0x00c2, 0x653: 0x00c2, 0x654: 0x00c2, 0x655: 0x00c2, 0x656: 0x00c2, 0x657: 0x00c2, + 0x658: 0x00c2, 0x659: 0x00c4, 0x65a: 0x00c4, 0x65b: 0x00c4, 0x65c: 0x00c2, 0x65d: 0x00c2, + 0x65e: 0x00c2, 0x65f: 0x00c2, 0x660: 0x00c2, 0x661: 0x00c2, 0x662: 0x00c2, 0x663: 0x00c2, + 0x664: 0x00c2, 0x665: 0x00c2, 0x666: 0x00c2, 0x667: 0x00c2, 0x668: 0x00c2, 0x669: 0x00c2, + 0x66a: 0x00c2, 0x66b: 0x00c4, 0x66c: 0x00c4, 0x66d: 0x00c2, 0x66e: 0x00c2, 0x66f: 0x00c2, + 0x670: 0x00c2, 0x671: 0x00c4, 0x672: 0x00c2, 0x673: 0x00c4, 0x674: 0x00c4, 0x675: 0x00c2, + 0x676: 0x00c2, 0x677: 0x00c2, 0x678: 0x00c4, 0x679: 0x00c4, 0x67a: 0x00c2, 0x67b: 0x00c2, + 0x67c: 0x00c2, 0x67d: 0x00c2, 0x67e: 0x00c2, 0x67f: 0x00c2, + // Block 0x1a, offset 0x680 + 0x680: 0x00c0, 0x681: 0x00c0, 0x682: 0x00c0, 0x683: 0x00c0, 0x684: 0x00c0, 0x685: 0x00c0, + 0x686: 0x00c0, 0x687: 0x00c0, 0x688: 0x00c0, 0x689: 0x00c0, 0x68a: 0x00c0, 0x68b: 0x00c0, + 0x68c: 0x00c0, 0x68d: 0x00c0, 0x68e: 0x00c0, 0x68f: 0x00c0, 0x690: 0x00c0, 0x691: 0x00c0, + 0x692: 0x00c0, 0x693: 0x00c0, 0x694: 0x00c0, 0x695: 0x00c0, 0x696: 0x00c0, 0x697: 0x00c0, + 0x698: 0x00c0, 0x699: 0x00c0, 0x69a: 0x00c0, 0x69b: 0x00c0, 0x69c: 0x00c0, 0x69d: 0x00c0, + 0x69e: 0x00c0, 0x69f: 0x00c0, 0x6a0: 0x00c0, 0x6a1: 0x00c0, 0x6a2: 0x00c0, 0x6a3: 0x00c0, + 0x6a4: 0x00c0, 0x6a5: 0x00c0, 0x6a6: 0x00c3, 0x6a7: 0x00c3, 0x6a8: 0x00c3, 0x6a9: 0x00c3, + 0x6aa: 0x00c3, 0x6ab: 0x00c3, 0x6ac: 0x00c3, 0x6ad: 0x00c3, 0x6ae: 0x00c3, 0x6af: 0x00c3, + 0x6b0: 0x00c3, 0x6b1: 0x00c0, + // Block 0x1b, offset 0x6c0 + 0x6c0: 0x00c0, 0x6c1: 0x00c0, 0x6c2: 0x00c0, 0x6c3: 0x00c0, 0x6c4: 0x00c0, 0x6c5: 0x00c0, + 0x6c6: 0x00c0, 0x6c7: 0x00c0, 0x6c8: 0x00c0, 0x6c9: 0x00c0, 0x6ca: 0x00c2, 0x6cb: 0x00c2, + 0x6cc: 0x00c2, 0x6cd: 0x00c2, 0x6ce: 0x00c2, 0x6cf: 0x00c2, 0x6d0: 0x00c2, 0x6d1: 0x00c2, + 0x6d2: 0x00c2, 0x6d3: 0x00c2, 0x6d4: 0x00c2, 0x6d5: 0x00c2, 0x6d6: 0x00c2, 0x6d7: 0x00c2, + 0x6d8: 0x00c2, 0x6d9: 0x00c2, 0x6da: 0x00c2, 0x6db: 0x00c2, 0x6dc: 0x00c2, 0x6dd: 0x00c2, + 0x6de: 0x00c2, 0x6df: 0x00c2, 0x6e0: 0x00c2, 0x6e1: 0x00c2, 0x6e2: 0x00c2, 0x6e3: 0x00c2, + 0x6e4: 0x00c2, 0x6e5: 0x00c2, 0x6e6: 0x00c2, 0x6e7: 0x00c2, 0x6e8: 0x00c2, 0x6e9: 0x00c2, + 0x6ea: 0x00c2, 0x6eb: 0x00c3, 0x6ec: 0x00c3, 0x6ed: 0x00c3, 0x6ee: 0x00c3, 0x6ef: 0x00c3, + 0x6f0: 0x00c3, 0x6f1: 0x00c3, 0x6f2: 0x00c3, 0x6f3: 0x00c3, 0x6f4: 0x00c0, 0x6f5: 0x00c0, + 0x6f6: 0x0080, 0x6f7: 0x0080, 0x6f8: 0x0080, 0x6f9: 0x0080, 0x6fa: 0x0040, + 0x6fd: 0x00c3, 0x6fe: 0x0080, 0x6ff: 0x0080, + // Block 0x1c, offset 0x700 + 0x700: 0x00c0, 0x701: 0x00c0, 0x702: 0x00c0, 0x703: 0x00c0, 0x704: 0x00c0, 0x705: 0x00c0, + 0x706: 0x00c0, 0x707: 0x00c0, 0x708: 0x00c0, 0x709: 0x00c0, 0x70a: 0x00c0, 0x70b: 0x00c0, + 0x70c: 0x00c0, 0x70d: 0x00c0, 0x70e: 0x00c0, 0x70f: 0x00c0, 0x710: 0x00c0, 0x711: 0x00c0, + 0x712: 0x00c0, 0x713: 0x00c0, 0x714: 0x00c0, 0x715: 0x00c0, 0x716: 0x00c3, 0x717: 0x00c3, + 0x718: 0x00c3, 0x719: 0x00c3, 0x71a: 0x00c0, 0x71b: 0x00c3, 0x71c: 0x00c3, 0x71d: 0x00c3, + 0x71e: 0x00c3, 0x71f: 0x00c3, 0x720: 0x00c3, 0x721: 0x00c3, 0x722: 0x00c3, 0x723: 0x00c3, + 0x724: 0x00c0, 0x725: 0x00c3, 0x726: 0x00c3, 0x727: 0x00c3, 0x728: 0x00c0, 0x729: 0x00c3, + 0x72a: 0x00c3, 0x72b: 0x00c3, 0x72c: 0x00c3, 0x72d: 0x00c3, + 0x730: 0x0080, 0x731: 0x0080, 0x732: 0x0080, 0x733: 0x0080, 0x734: 0x0080, 0x735: 0x0080, + 0x736: 0x0080, 0x737: 0x0080, 0x738: 0x0080, 0x739: 0x0080, 0x73a: 0x0080, 0x73b: 0x0080, + 0x73c: 0x0080, 0x73d: 0x0080, 0x73e: 0x0080, + // Block 0x1d, offset 0x740 + 0x740: 0x00c4, 0x741: 0x00c2, 0x742: 0x00c2, 0x743: 0x00c2, 0x744: 0x00c2, 0x745: 0x00c2, + 0x746: 0x00c4, 0x747: 0x00c4, 0x748: 0x00c2, 0x749: 0x00c4, 0x74a: 0x00c2, 0x74b: 0x00c2, + 0x74c: 0x00c2, 0x74d: 0x00c2, 0x74e: 0x00c2, 0x74f: 0x00c2, 0x750: 0x00c2, 0x751: 0x00c2, + 0x752: 0x00c2, 0x753: 0x00c2, 0x754: 0x00c4, 0x755: 0x00c2, 0x756: 0x00c0, 0x757: 0x00c0, + 0x758: 0x00c0, 0x759: 0x00c3, 0x75a: 0x00c3, 0x75b: 0x00c3, + 0x75e: 0x0080, 0x760: 0x00c2, 0x761: 0x00c0, 0x762: 0x00c2, 0x763: 0x00c2, + 0x764: 0x00c2, 0x765: 0x00c2, 0x766: 0x00c0, 0x767: 0x00c4, 0x768: 0x00c2, 0x769: 0x00c4, + 0x76a: 0x00c4, + // Block 0x1e, offset 0x780 + 0x7a0: 0x00c2, 0x7a1: 0x00c2, 0x7a2: 0x00c2, 0x7a3: 0x00c2, + 0x7a4: 0x00c2, 0x7a5: 0x00c2, 0x7a6: 0x00c2, 0x7a7: 0x00c2, 0x7a8: 0x00c2, 0x7a9: 0x00c2, + 0x7aa: 0x00c4, 0x7ab: 0x00c4, 0x7ac: 0x00c4, 0x7ad: 0x00c0, 0x7ae: 0x00c4, 0x7af: 0x00c2, + 0x7b0: 0x00c2, 0x7b1: 0x00c4, 0x7b2: 0x00c4, 0x7b3: 0x00c2, 0x7b4: 0x00c2, + 0x7b6: 0x00c2, 0x7b7: 0x00c2, 0x7b8: 0x00c2, 0x7b9: 0x00c4, 0x7ba: 0x00c2, 0x7bb: 0x00c2, + 0x7bc: 0x00c2, 0x7bd: 0x00c2, + // Block 0x1f, offset 0x7c0 + 0x7d3: 0x00c3, 0x7d4: 0x00c3, 0x7d5: 0x00c3, 0x7d6: 0x00c3, 0x7d7: 0x00c3, + 0x7d8: 0x00c3, 0x7d9: 0x00c3, 0x7da: 0x00c3, 0x7db: 0x00c3, 0x7dc: 0x00c3, 0x7dd: 0x00c3, + 0x7de: 0x00c3, 0x7df: 0x00c3, 0x7e0: 0x00c3, 0x7e1: 0x00c3, 0x7e2: 0x0040, 0x7e3: 0x00c3, + 0x7e4: 0x00c3, 0x7e5: 0x00c3, 0x7e6: 0x00c3, 0x7e7: 0x00c3, 0x7e8: 0x00c3, 0x7e9: 0x00c3, + 0x7ea: 0x00c3, 0x7eb: 0x00c3, 0x7ec: 0x00c3, 0x7ed: 0x00c3, 0x7ee: 0x00c3, 0x7ef: 0x00c3, + 0x7f0: 0x00c3, 0x7f1: 0x00c3, 0x7f2: 0x00c3, 0x7f3: 0x00c3, 0x7f4: 0x00c3, 0x7f5: 0x00c3, + 0x7f6: 0x00c3, 0x7f7: 0x00c3, 0x7f8: 0x00c3, 0x7f9: 0x00c3, 0x7fa: 0x00c3, 0x7fb: 0x00c3, + 0x7fc: 0x00c3, 0x7fd: 0x00c3, 0x7fe: 0x00c3, 0x7ff: 0x00c3, + // Block 0x20, offset 0x800 + 0x800: 0x00c3, 0x801: 0x00c3, 0x802: 0x00c3, 0x803: 0x00c0, 0x804: 0x00c0, 0x805: 0x00c0, + 0x806: 0x00c0, 0x807: 0x00c0, 0x808: 0x00c0, 0x809: 0x00c0, 0x80a: 0x00c0, 0x80b: 0x00c0, + 0x80c: 0x00c0, 0x80d: 0x00c0, 0x80e: 0x00c0, 0x80f: 0x00c0, 0x810: 0x00c0, 0x811: 0x00c0, + 0x812: 0x00c0, 0x813: 0x00c0, 0x814: 0x00c0, 0x815: 0x00c0, 0x816: 0x00c0, 0x817: 0x00c0, + 0x818: 0x00c0, 0x819: 0x00c0, 0x81a: 0x00c0, 0x81b: 0x00c0, 0x81c: 0x00c0, 0x81d: 0x00c0, + 0x81e: 0x00c0, 0x81f: 0x00c0, 0x820: 0x00c0, 0x821: 0x00c0, 0x822: 0x00c0, 0x823: 0x00c0, + 0x824: 0x00c0, 0x825: 0x00c0, 0x826: 0x00c0, 0x827: 0x00c0, 0x828: 0x00c0, 0x829: 0x00c0, + 0x82a: 0x00c0, 0x82b: 0x00c0, 0x82c: 0x00c0, 0x82d: 0x00c0, 0x82e: 0x00c0, 0x82f: 0x00c0, + 0x830: 0x00c0, 0x831: 0x00c0, 0x832: 0x00c0, 0x833: 0x00c0, 0x834: 0x00c0, 0x835: 0x00c0, + 0x836: 0x00c0, 0x837: 0x00c0, 0x838: 0x00c0, 0x839: 0x00c0, 0x83a: 0x00c3, 0x83b: 0x00c0, + 0x83c: 0x00c3, 0x83d: 0x00c0, 0x83e: 0x00c0, 0x83f: 0x00c0, + // Block 0x21, offset 0x840 + 0x840: 0x00c0, 0x841: 0x00c3, 0x842: 0x00c3, 0x843: 0x00c3, 0x844: 0x00c3, 0x845: 0x00c3, + 0x846: 0x00c3, 0x847: 0x00c3, 0x848: 0x00c3, 0x849: 0x00c0, 0x84a: 0x00c0, 0x84b: 0x00c0, + 0x84c: 0x00c0, 0x84d: 0x00c6, 0x84e: 0x00c0, 0x84f: 0x00c0, 0x850: 0x00c0, 0x851: 0x00c3, + 0x852: 0x00c3, 0x853: 0x00c3, 0x854: 0x00c3, 0x855: 0x00c3, 0x856: 0x00c3, 0x857: 0x00c3, + 0x858: 0x0080, 0x859: 0x0080, 0x85a: 0x0080, 0x85b: 0x0080, 0x85c: 0x0080, 0x85d: 0x0080, + 0x85e: 0x0080, 0x85f: 0x0080, 0x860: 0x00c0, 0x861: 0x00c0, 0x862: 0x00c3, 0x863: 0x00c3, + 0x864: 0x0080, 0x865: 0x0080, 0x866: 0x00c0, 0x867: 0x00c0, 0x868: 0x00c0, 0x869: 0x00c0, + 0x86a: 0x00c0, 0x86b: 0x00c0, 0x86c: 0x00c0, 0x86d: 0x00c0, 0x86e: 0x00c0, 0x86f: 0x00c0, + 0x870: 0x0080, 0x871: 0x00c0, 0x872: 0x00c0, 0x873: 0x00c0, 0x874: 0x00c0, 0x875: 0x00c0, + 0x876: 0x00c0, 0x877: 0x00c0, 0x878: 0x00c0, 0x879: 0x00c0, 0x87a: 0x00c0, 0x87b: 0x00c0, + 0x87c: 0x00c0, 0x87d: 0x00c0, 0x87e: 0x00c0, 0x87f: 0x00c0, + // Block 0x22, offset 0x880 + 0x880: 0x00c0, 0x881: 0x00c3, 0x882: 0x00c0, 0x883: 0x00c0, 0x885: 0x00c0, + 0x886: 0x00c0, 0x887: 0x00c0, 0x888: 0x00c0, 0x889: 0x00c0, 0x88a: 0x00c0, 0x88b: 0x00c0, + 0x88c: 0x00c0, 0x88f: 0x00c0, 0x890: 0x00c0, + 0x893: 0x00c0, 0x894: 0x00c0, 0x895: 0x00c0, 0x896: 0x00c0, 0x897: 0x00c0, + 0x898: 0x00c0, 0x899: 0x00c0, 0x89a: 0x00c0, 0x89b: 0x00c0, 0x89c: 0x00c0, 0x89d: 0x00c0, + 0x89e: 0x00c0, 0x89f: 0x00c0, 0x8a0: 0x00c0, 0x8a1: 0x00c0, 0x8a2: 0x00c0, 0x8a3: 0x00c0, + 0x8a4: 0x00c0, 0x8a5: 0x00c0, 0x8a6: 0x00c0, 0x8a7: 0x00c0, 0x8a8: 0x00c0, + 0x8aa: 0x00c0, 0x8ab: 0x00c0, 0x8ac: 0x00c0, 0x8ad: 0x00c0, 0x8ae: 0x00c0, 0x8af: 0x00c0, + 0x8b0: 0x00c0, 0x8b2: 0x00c0, + 0x8b6: 0x00c0, 0x8b7: 0x00c0, 0x8b8: 0x00c0, 0x8b9: 0x00c0, + 0x8bc: 0x00c3, 0x8bd: 0x00c0, 0x8be: 0x00c0, 0x8bf: 0x00c0, + // Block 0x23, offset 0x8c0 + 0x8c0: 0x00c0, 0x8c1: 0x00c3, 0x8c2: 0x00c3, 0x8c3: 0x00c3, 0x8c4: 0x00c3, + 0x8c7: 0x00c0, 0x8c8: 0x00c0, 0x8cb: 0x00c0, + 0x8cc: 0x00c0, 0x8cd: 0x00c6, 0x8ce: 0x00c0, + 0x8d7: 0x00c0, + 0x8dc: 0x0080, 0x8dd: 0x0080, + 0x8df: 0x0080, 0x8e0: 0x00c0, 0x8e1: 0x00c0, 0x8e2: 0x00c3, 0x8e3: 0x00c3, + 0x8e6: 0x00c0, 0x8e7: 0x00c0, 0x8e8: 0x00c0, 0x8e9: 0x00c0, + 0x8ea: 0x00c0, 0x8eb: 0x00c0, 0x8ec: 0x00c0, 0x8ed: 0x00c0, 0x8ee: 0x00c0, 0x8ef: 0x00c0, + 0x8f0: 0x00c0, 0x8f1: 0x00c0, 0x8f2: 0x0080, 0x8f3: 0x0080, 0x8f4: 0x0080, 0x8f5: 0x0080, + 0x8f6: 0x0080, 0x8f7: 0x0080, 0x8f8: 0x0080, 0x8f9: 0x0080, 0x8fa: 0x0080, 0x8fb: 0x0080, + 0x8fc: 0x00c0, 0x8fd: 0x0080, 0x8fe: 0x00c3, + // Block 0x24, offset 0x900 + 0x901: 0x00c3, 0x902: 0x00c3, 0x903: 0x00c0, 0x905: 0x00c0, + 0x906: 0x00c0, 0x907: 0x00c0, 0x908: 0x00c0, 0x909: 0x00c0, 0x90a: 0x00c0, + 0x90f: 0x00c0, 0x910: 0x00c0, + 0x913: 0x00c0, 0x914: 0x00c0, 0x915: 0x00c0, 0x916: 0x00c0, 0x917: 0x00c0, + 0x918: 0x00c0, 0x919: 0x00c0, 0x91a: 0x00c0, 0x91b: 0x00c0, 0x91c: 0x00c0, 0x91d: 0x00c0, + 0x91e: 0x00c0, 0x91f: 0x00c0, 0x920: 0x00c0, 0x921: 0x00c0, 0x922: 0x00c0, 0x923: 0x00c0, + 0x924: 0x00c0, 0x925: 0x00c0, 0x926: 0x00c0, 0x927: 0x00c0, 0x928: 0x00c0, + 0x92a: 0x00c0, 0x92b: 0x00c0, 0x92c: 0x00c0, 0x92d: 0x00c0, 0x92e: 0x00c0, 0x92f: 0x00c0, + 0x930: 0x00c0, 0x932: 0x00c0, 0x933: 0x0080, 0x935: 0x00c0, + 0x936: 0x0080, 0x938: 0x00c0, 0x939: 0x00c0, + 0x93c: 0x00c3, 0x93e: 0x00c0, 0x93f: 0x00c0, + // Block 0x25, offset 0x940 + 0x940: 0x00c0, 0x941: 0x00c3, 0x942: 0x00c3, + 0x947: 0x00c3, 0x948: 0x00c3, 0x94b: 0x00c3, + 0x94c: 0x00c3, 0x94d: 0x00c6, 0x951: 0x00c3, + 0x959: 0x0080, 0x95a: 0x0080, 0x95b: 0x0080, 0x95c: 0x00c0, + 0x95e: 0x0080, + 0x966: 0x00c0, 0x967: 0x00c0, 0x968: 0x00c0, 0x969: 0x00c0, + 0x96a: 0x00c0, 0x96b: 0x00c0, 0x96c: 0x00c0, 0x96d: 0x00c0, 0x96e: 0x00c0, 0x96f: 0x00c0, + 0x970: 0x00c3, 0x971: 0x00c3, 0x972: 0x00c0, 0x973: 0x00c0, 0x974: 0x00c0, 0x975: 0x00c3, + 0x976: 0x0080, + // Block 0x26, offset 0x980 + 0x981: 0x00c3, 0x982: 0x00c3, 0x983: 0x00c0, 0x985: 0x00c0, + 0x986: 0x00c0, 0x987: 0x00c0, 0x988: 0x00c0, 0x989: 0x00c0, 0x98a: 0x00c0, 0x98b: 0x00c0, + 0x98c: 0x00c0, 0x98d: 0x00c0, 0x98f: 0x00c0, 0x990: 0x00c0, 0x991: 0x00c0, + 0x993: 0x00c0, 0x994: 0x00c0, 0x995: 0x00c0, 0x996: 0x00c0, 0x997: 0x00c0, + 0x998: 0x00c0, 0x999: 0x00c0, 0x99a: 0x00c0, 0x99b: 0x00c0, 0x99c: 0x00c0, 0x99d: 0x00c0, + 0x99e: 0x00c0, 0x99f: 0x00c0, 0x9a0: 0x00c0, 0x9a1: 0x00c0, 0x9a2: 0x00c0, 0x9a3: 0x00c0, + 0x9a4: 0x00c0, 0x9a5: 0x00c0, 0x9a6: 0x00c0, 0x9a7: 0x00c0, 0x9a8: 0x00c0, + 0x9aa: 0x00c0, 0x9ab: 0x00c0, 0x9ac: 0x00c0, 0x9ad: 0x00c0, 0x9ae: 0x00c0, 0x9af: 0x00c0, + 0x9b0: 0x00c0, 0x9b2: 0x00c0, 0x9b3: 0x00c0, 0x9b5: 0x00c0, + 0x9b6: 0x00c0, 0x9b7: 0x00c0, 0x9b8: 0x00c0, 0x9b9: 0x00c0, + 0x9bc: 0x00c3, 0x9bd: 0x00c0, 0x9be: 0x00c0, 0x9bf: 0x00c0, + // Block 0x27, offset 0x9c0 + 0x9c0: 0x00c0, 0x9c1: 0x00c3, 0x9c2: 0x00c3, 0x9c3: 0x00c3, 0x9c4: 0x00c3, 0x9c5: 0x00c3, + 0x9c7: 0x00c3, 0x9c8: 0x00c3, 0x9c9: 0x00c0, 0x9cb: 0x00c0, + 0x9cc: 0x00c0, 0x9cd: 0x00c6, 0x9d0: 0x00c0, + 0x9e0: 0x00c0, 0x9e1: 0x00c0, 0x9e2: 0x00c3, 0x9e3: 0x00c3, + 0x9e6: 0x00c0, 0x9e7: 0x00c0, 0x9e8: 0x00c0, 0x9e9: 0x00c0, + 0x9ea: 0x00c0, 0x9eb: 0x00c0, 0x9ec: 0x00c0, 0x9ed: 0x00c0, 0x9ee: 0x00c0, 0x9ef: 0x00c0, + 0x9f0: 0x0080, 0x9f1: 0x0080, + 0x9f9: 0x00c0, 0x9fa: 0x00c3, 0x9fb: 0x00c3, + 0x9fc: 0x00c3, 0x9fd: 0x00c3, 0x9fe: 0x00c3, 0x9ff: 0x00c3, + // Block 0x28, offset 0xa00 + 0xa01: 0x00c3, 0xa02: 0x00c0, 0xa03: 0x00c0, 0xa05: 0x00c0, + 0xa06: 0x00c0, 0xa07: 0x00c0, 0xa08: 0x00c0, 0xa09: 0x00c0, 0xa0a: 0x00c0, 0xa0b: 0x00c0, + 0xa0c: 0x00c0, 0xa0f: 0x00c0, 0xa10: 0x00c0, + 0xa13: 0x00c0, 0xa14: 0x00c0, 0xa15: 0x00c0, 0xa16: 0x00c0, 0xa17: 0x00c0, + 0xa18: 0x00c0, 0xa19: 0x00c0, 0xa1a: 0x00c0, 0xa1b: 0x00c0, 0xa1c: 0x00c0, 0xa1d: 0x00c0, + 0xa1e: 0x00c0, 0xa1f: 0x00c0, 0xa20: 0x00c0, 0xa21: 0x00c0, 0xa22: 0x00c0, 0xa23: 0x00c0, + 0xa24: 0x00c0, 0xa25: 0x00c0, 0xa26: 0x00c0, 0xa27: 0x00c0, 0xa28: 0x00c0, + 0xa2a: 0x00c0, 0xa2b: 0x00c0, 0xa2c: 0x00c0, 0xa2d: 0x00c0, 0xa2e: 0x00c0, 0xa2f: 0x00c0, + 0xa30: 0x00c0, 0xa32: 0x00c0, 0xa33: 0x00c0, 0xa35: 0x00c0, + 0xa36: 0x00c0, 0xa37: 0x00c0, 0xa38: 0x00c0, 0xa39: 0x00c0, + 0xa3c: 0x00c3, 0xa3d: 0x00c0, 0xa3e: 0x00c0, 0xa3f: 0x00c3, + // Block 0x29, offset 0xa40 + 0xa40: 0x00c0, 0xa41: 0x00c3, 0xa42: 0x00c3, 0xa43: 0x00c3, 0xa44: 0x00c3, + 0xa47: 0x00c0, 0xa48: 0x00c0, 0xa4b: 0x00c0, + 0xa4c: 0x00c0, 0xa4d: 0x00c6, + 0xa56: 0x00c3, 0xa57: 0x00c0, + 0xa5c: 0x0080, 0xa5d: 0x0080, + 0xa5f: 0x00c0, 0xa60: 0x00c0, 0xa61: 0x00c0, 0xa62: 0x00c3, 0xa63: 0x00c3, + 0xa66: 0x00c0, 0xa67: 0x00c0, 0xa68: 0x00c0, 0xa69: 0x00c0, + 0xa6a: 0x00c0, 0xa6b: 0x00c0, 0xa6c: 0x00c0, 0xa6d: 0x00c0, 0xa6e: 0x00c0, 0xa6f: 0x00c0, + 0xa70: 0x0080, 0xa71: 0x00c0, 0xa72: 0x0080, 0xa73: 0x0080, 0xa74: 0x0080, 0xa75: 0x0080, + 0xa76: 0x0080, 0xa77: 0x0080, + // Block 0x2a, offset 0xa80 + 0xa82: 0x00c3, 0xa83: 0x00c0, 0xa85: 0x00c0, + 0xa86: 0x00c0, 0xa87: 0x00c0, 0xa88: 0x00c0, 0xa89: 0x00c0, 0xa8a: 0x00c0, + 0xa8e: 0x00c0, 0xa8f: 0x00c0, 0xa90: 0x00c0, + 0xa92: 0x00c0, 0xa93: 0x00c0, 0xa94: 0x00c0, 0xa95: 0x00c0, + 0xa99: 0x00c0, 0xa9a: 0x00c0, 0xa9c: 0x00c0, + 0xa9e: 0x00c0, 0xa9f: 0x00c0, 0xaa3: 0x00c0, + 0xaa4: 0x00c0, 0xaa8: 0x00c0, 0xaa9: 0x00c0, + 0xaaa: 0x00c0, 0xaae: 0x00c0, 0xaaf: 0x00c0, + 0xab0: 0x00c0, 0xab1: 0x00c0, 0xab2: 0x00c0, 0xab3: 0x00c0, 0xab4: 0x00c0, 0xab5: 0x00c0, + 0xab6: 0x00c0, 0xab7: 0x00c0, 0xab8: 0x00c0, 0xab9: 0x00c0, + 0xabe: 0x00c0, 0xabf: 0x00c0, + // Block 0x2b, offset 0xac0 + 0xac0: 0x00c3, 0xac1: 0x00c0, 0xac2: 0x00c0, + 0xac6: 0x00c0, 0xac7: 0x00c0, 0xac8: 0x00c0, 0xaca: 0x00c0, 0xacb: 0x00c0, + 0xacc: 0x00c0, 0xacd: 0x00c6, 0xad0: 0x00c0, + 0xad7: 0x00c0, + 0xae6: 0x00c0, 0xae7: 0x00c0, 0xae8: 0x00c0, 0xae9: 0x00c0, + 0xaea: 0x00c0, 0xaeb: 0x00c0, 0xaec: 0x00c0, 0xaed: 0x00c0, 0xaee: 0x00c0, 0xaef: 0x00c0, + 0xaf0: 0x0080, 0xaf1: 0x0080, 0xaf2: 0x0080, 0xaf3: 0x0080, 0xaf4: 0x0080, 0xaf5: 0x0080, + 0xaf6: 0x0080, 0xaf7: 0x0080, 0xaf8: 0x0080, 0xaf9: 0x0080, 0xafa: 0x0080, + // Block 0x2c, offset 0xb00 + 0xb00: 0x00c3, 0xb01: 0x00c0, 0xb02: 0x00c0, 0xb03: 0x00c0, 0xb04: 0x00c3, 0xb05: 0x00c0, + 0xb06: 0x00c0, 0xb07: 0x00c0, 0xb08: 0x00c0, 0xb09: 0x00c0, 0xb0a: 0x00c0, 0xb0b: 0x00c0, + 0xb0c: 0x00c0, 0xb0e: 0x00c0, 0xb0f: 0x00c0, 0xb10: 0x00c0, + 0xb12: 0x00c0, 0xb13: 0x00c0, 0xb14: 0x00c0, 0xb15: 0x00c0, 0xb16: 0x00c0, 0xb17: 0x00c0, + 0xb18: 0x00c0, 0xb19: 0x00c0, 0xb1a: 0x00c0, 0xb1b: 0x00c0, 0xb1c: 0x00c0, 0xb1d: 0x00c0, + 0xb1e: 0x00c0, 0xb1f: 0x00c0, 0xb20: 0x00c0, 0xb21: 0x00c0, 0xb22: 0x00c0, 0xb23: 0x00c0, + 0xb24: 0x00c0, 0xb25: 0x00c0, 0xb26: 0x00c0, 0xb27: 0x00c0, 0xb28: 0x00c0, + 0xb2a: 0x00c0, 0xb2b: 0x00c0, 0xb2c: 0x00c0, 0xb2d: 0x00c0, 0xb2e: 0x00c0, 0xb2f: 0x00c0, + 0xb30: 0x00c0, 0xb31: 0x00c0, 0xb32: 0x00c0, 0xb33: 0x00c0, 0xb34: 0x00c0, 0xb35: 0x00c0, + 0xb36: 0x00c0, 0xb37: 0x00c0, 0xb38: 0x00c0, 0xb39: 0x00c0, + 0xb3d: 0x00c0, 0xb3e: 0x00c3, 0xb3f: 0x00c3, + // Block 0x2d, offset 0xb40 + 0xb40: 0x00c3, 0xb41: 0x00c0, 0xb42: 0x00c0, 0xb43: 0x00c0, 0xb44: 0x00c0, + 0xb46: 0x00c3, 0xb47: 0x00c3, 0xb48: 0x00c3, 0xb4a: 0x00c3, 0xb4b: 0x00c3, + 0xb4c: 0x00c3, 0xb4d: 0x00c6, + 0xb55: 0x00c3, 0xb56: 0x00c3, + 0xb58: 0x00c0, 0xb59: 0x00c0, 0xb5a: 0x00c0, + 0xb60: 0x00c0, 0xb61: 0x00c0, 0xb62: 0x00c3, 0xb63: 0x00c3, + 0xb66: 0x00c0, 0xb67: 0x00c0, 0xb68: 0x00c0, 0xb69: 0x00c0, + 0xb6a: 0x00c0, 0xb6b: 0x00c0, 0xb6c: 0x00c0, 0xb6d: 0x00c0, 0xb6e: 0x00c0, 0xb6f: 0x00c0, + 0xb77: 0x0080, 0xb78: 0x0080, 0xb79: 0x0080, 0xb7a: 0x0080, 0xb7b: 0x0080, + 0xb7c: 0x0080, 0xb7d: 0x0080, 0xb7e: 0x0080, 0xb7f: 0x0080, + // Block 0x2e, offset 0xb80 + 0xb80: 0x00c0, 0xb81: 0x00c3, 0xb82: 0x00c0, 0xb83: 0x00c0, 0xb84: 0x0080, 0xb85: 0x00c0, + 0xb86: 0x00c0, 0xb87: 0x00c0, 0xb88: 0x00c0, 0xb89: 0x00c0, 0xb8a: 0x00c0, 0xb8b: 0x00c0, + 0xb8c: 0x00c0, 0xb8e: 0x00c0, 0xb8f: 0x00c0, 0xb90: 0x00c0, + 0xb92: 0x00c0, 0xb93: 0x00c0, 0xb94: 0x00c0, 0xb95: 0x00c0, 0xb96: 0x00c0, 0xb97: 0x00c0, + 0xb98: 0x00c0, 0xb99: 0x00c0, 0xb9a: 0x00c0, 0xb9b: 0x00c0, 0xb9c: 0x00c0, 0xb9d: 0x00c0, + 0xb9e: 0x00c0, 0xb9f: 0x00c0, 0xba0: 0x00c0, 0xba1: 0x00c0, 0xba2: 0x00c0, 0xba3: 0x00c0, + 0xba4: 0x00c0, 0xba5: 0x00c0, 0xba6: 0x00c0, 0xba7: 0x00c0, 0xba8: 0x00c0, + 0xbaa: 0x00c0, 0xbab: 0x00c0, 0xbac: 0x00c0, 0xbad: 0x00c0, 0xbae: 0x00c0, 0xbaf: 0x00c0, + 0xbb0: 0x00c0, 0xbb1: 0x00c0, 0xbb2: 0x00c0, 0xbb3: 0x00c0, 0xbb5: 0x00c0, + 0xbb6: 0x00c0, 0xbb7: 0x00c0, 0xbb8: 0x00c0, 0xbb9: 0x00c0, + 0xbbc: 0x00c3, 0xbbd: 0x00c0, 0xbbe: 0x00c0, 0xbbf: 0x00c3, + // Block 0x2f, offset 0xbc0 + 0xbc0: 0x00c0, 0xbc1: 0x00c0, 0xbc2: 0x00c0, 0xbc3: 0x00c0, 0xbc4: 0x00c0, + 0xbc6: 0x00c3, 0xbc7: 0x00c0, 0xbc8: 0x00c0, 0xbca: 0x00c0, 0xbcb: 0x00c0, + 0xbcc: 0x00c3, 0xbcd: 0x00c6, + 0xbd5: 0x00c0, 0xbd6: 0x00c0, + 0xbde: 0x00c0, 0xbe0: 0x00c0, 0xbe1: 0x00c0, 0xbe2: 0x00c3, 0xbe3: 0x00c3, + 0xbe6: 0x00c0, 0xbe7: 0x00c0, 0xbe8: 0x00c0, 0xbe9: 0x00c0, + 0xbea: 0x00c0, 0xbeb: 0x00c0, 0xbec: 0x00c0, 0xbed: 0x00c0, 0xbee: 0x00c0, 0xbef: 0x00c0, + 0xbf1: 0x00c0, 0xbf2: 0x00c0, + // Block 0x30, offset 0xc00 + 0xc00: 0x00c3, 0xc01: 0x00c3, 0xc02: 0x00c0, 0xc03: 0x00c0, 0xc05: 0x00c0, + 0xc06: 0x00c0, 0xc07: 0x00c0, 0xc08: 0x00c0, 0xc09: 0x00c0, 0xc0a: 0x00c0, 0xc0b: 0x00c0, + 0xc0c: 0x00c0, 0xc0e: 0x00c0, 0xc0f: 0x00c0, 0xc10: 0x00c0, + 0xc12: 0x00c0, 0xc13: 0x00c0, 0xc14: 0x00c0, 0xc15: 0x00c0, 0xc16: 0x00c0, 0xc17: 0x00c0, + 0xc18: 0x00c0, 0xc19: 0x00c0, 0xc1a: 0x00c0, 0xc1b: 0x00c0, 0xc1c: 0x00c0, 0xc1d: 0x00c0, + 0xc1e: 0x00c0, 0xc1f: 0x00c0, 0xc20: 0x00c0, 0xc21: 0x00c0, 0xc22: 0x00c0, 0xc23: 0x00c0, + 0xc24: 0x00c0, 0xc25: 0x00c0, 0xc26: 0x00c0, 0xc27: 0x00c0, 0xc28: 0x00c0, 0xc29: 0x00c0, + 0xc2a: 0x00c0, 0xc2b: 0x00c0, 0xc2c: 0x00c0, 0xc2d: 0x00c0, 0xc2e: 0x00c0, 0xc2f: 0x00c0, + 0xc30: 0x00c0, 0xc31: 0x00c0, 0xc32: 0x00c0, 0xc33: 0x00c0, 0xc34: 0x00c0, 0xc35: 0x00c0, + 0xc36: 0x00c0, 0xc37: 0x00c0, 0xc38: 0x00c0, 0xc39: 0x00c0, 0xc3a: 0x00c0, 0xc3b: 0x00c6, + 0xc3c: 0x00c6, 0xc3d: 0x00c0, 0xc3e: 0x00c0, 0xc3f: 0x00c0, + // Block 0x31, offset 0xc40 + 0xc40: 0x00c0, 0xc41: 0x00c3, 0xc42: 0x00c3, 0xc43: 0x00c3, 0xc44: 0x00c3, + 0xc46: 0x00c0, 0xc47: 0x00c0, 0xc48: 0x00c0, 0xc4a: 0x00c0, 0xc4b: 0x00c0, + 0xc4c: 0x00c0, 0xc4d: 0x00c6, 0xc4e: 0x00c0, 0xc4f: 0x0080, + 0xc54: 0x00c0, 0xc55: 0x00c0, 0xc56: 0x00c0, 0xc57: 0x00c0, + 0xc58: 0x0080, 0xc59: 0x0080, 0xc5a: 0x0080, 0xc5b: 0x0080, 0xc5c: 0x0080, 0xc5d: 0x0080, + 0xc5e: 0x0080, 0xc5f: 0x00c0, 0xc60: 0x00c0, 0xc61: 0x00c0, 0xc62: 0x00c3, 0xc63: 0x00c3, + 0xc66: 0x00c0, 0xc67: 0x00c0, 0xc68: 0x00c0, 0xc69: 0x00c0, + 0xc6a: 0x00c0, 0xc6b: 0x00c0, 0xc6c: 0x00c0, 0xc6d: 0x00c0, 0xc6e: 0x00c0, 0xc6f: 0x00c0, + 0xc70: 0x0080, 0xc71: 0x0080, 0xc72: 0x0080, 0xc73: 0x0080, 0xc74: 0x0080, 0xc75: 0x0080, + 0xc76: 0x0080, 0xc77: 0x0080, 0xc78: 0x0080, 0xc79: 0x0080, 0xc7a: 0x00c0, 0xc7b: 0x00c0, + 0xc7c: 0x00c0, 0xc7d: 0x00c0, 0xc7e: 0x00c0, 0xc7f: 0x00c0, + // Block 0x32, offset 0xc80 + 0xc82: 0x00c0, 0xc83: 0x00c0, 0xc85: 0x00c0, + 0xc86: 0x00c0, 0xc87: 0x00c0, 0xc88: 0x00c0, 0xc89: 0x00c0, 0xc8a: 0x00c0, 0xc8b: 0x00c0, + 0xc8c: 0x00c0, 0xc8d: 0x00c0, 0xc8e: 0x00c0, 0xc8f: 0x00c0, 0xc90: 0x00c0, 0xc91: 0x00c0, + 0xc92: 0x00c0, 0xc93: 0x00c0, 0xc94: 0x00c0, 0xc95: 0x00c0, 0xc96: 0x00c0, + 0xc9a: 0x00c0, 0xc9b: 0x00c0, 0xc9c: 0x00c0, 0xc9d: 0x00c0, + 0xc9e: 0x00c0, 0xc9f: 0x00c0, 0xca0: 0x00c0, 0xca1: 0x00c0, 0xca2: 0x00c0, 0xca3: 0x00c0, + 0xca4: 0x00c0, 0xca5: 0x00c0, 0xca6: 0x00c0, 0xca7: 0x00c0, 0xca8: 0x00c0, 0xca9: 0x00c0, + 0xcaa: 0x00c0, 0xcab: 0x00c0, 0xcac: 0x00c0, 0xcad: 0x00c0, 0xcae: 0x00c0, 0xcaf: 0x00c0, + 0xcb0: 0x00c0, 0xcb1: 0x00c0, 0xcb3: 0x00c0, 0xcb4: 0x00c0, 0xcb5: 0x00c0, + 0xcb6: 0x00c0, 0xcb7: 0x00c0, 0xcb8: 0x00c0, 0xcb9: 0x00c0, 0xcba: 0x00c0, 0xcbb: 0x00c0, + 0xcbd: 0x00c0, + // Block 0x33, offset 0xcc0 + 0xcc0: 0x00c0, 0xcc1: 0x00c0, 0xcc2: 0x00c0, 0xcc3: 0x00c0, 0xcc4: 0x00c0, 0xcc5: 0x00c0, + 0xcc6: 0x00c0, 0xcca: 0x00c6, + 0xccf: 0x00c0, 0xcd0: 0x00c0, 0xcd1: 0x00c0, + 0xcd2: 0x00c3, 0xcd3: 0x00c3, 0xcd4: 0x00c3, 0xcd6: 0x00c3, + 0xcd8: 0x00c0, 0xcd9: 0x00c0, 0xcda: 0x00c0, 0xcdb: 0x00c0, 0xcdc: 0x00c0, 0xcdd: 0x00c0, + 0xcde: 0x00c0, 0xcdf: 0x00c0, + 0xce6: 0x00c0, 0xce7: 0x00c0, 0xce8: 0x00c0, 0xce9: 0x00c0, + 0xcea: 0x00c0, 0xceb: 0x00c0, 0xcec: 0x00c0, 0xced: 0x00c0, 0xcee: 0x00c0, 0xcef: 0x00c0, + 0xcf2: 0x00c0, 0xcf3: 0x00c0, 0xcf4: 0x0080, + // Block 0x34, offset 0xd00 + 0xd01: 0x00c0, 0xd02: 0x00c0, 0xd03: 0x00c0, 0xd04: 0x00c0, 0xd05: 0x00c0, + 0xd06: 0x00c0, 0xd07: 0x00c0, 0xd08: 0x00c0, 0xd09: 0x00c0, 0xd0a: 0x00c0, 0xd0b: 0x00c0, + 0xd0c: 0x00c0, 0xd0d: 0x00c0, 0xd0e: 0x00c0, 0xd0f: 0x00c0, 0xd10: 0x00c0, 0xd11: 0x00c0, + 0xd12: 0x00c0, 0xd13: 0x00c0, 0xd14: 0x00c0, 0xd15: 0x00c0, 0xd16: 0x00c0, 0xd17: 0x00c0, + 0xd18: 0x00c0, 0xd19: 0x00c0, 0xd1a: 0x00c0, 0xd1b: 0x00c0, 0xd1c: 0x00c0, 0xd1d: 0x00c0, + 0xd1e: 0x00c0, 0xd1f: 0x00c0, 0xd20: 0x00c0, 0xd21: 0x00c0, 0xd22: 0x00c0, 0xd23: 0x00c0, + 0xd24: 0x00c0, 0xd25: 0x00c0, 0xd26: 0x00c0, 0xd27: 0x00c0, 0xd28: 0x00c0, 0xd29: 0x00c0, + 0xd2a: 0x00c0, 0xd2b: 0x00c0, 0xd2c: 0x00c0, 0xd2d: 0x00c0, 0xd2e: 0x00c0, 0xd2f: 0x00c0, + 0xd30: 0x00c0, 0xd31: 0x00c3, 0xd32: 0x00c0, 0xd33: 0x0080, 0xd34: 0x00c3, 0xd35: 0x00c3, + 0xd36: 0x00c3, 0xd37: 0x00c3, 0xd38: 0x00c3, 0xd39: 0x00c3, 0xd3a: 0x00c6, + 0xd3f: 0x0080, + // Block 0x35, offset 0xd40 + 0xd40: 0x00c0, 0xd41: 0x00c0, 0xd42: 0x00c0, 0xd43: 0x00c0, 0xd44: 0x00c0, 0xd45: 0x00c0, + 0xd46: 0x00c0, 0xd47: 0x00c3, 0xd48: 0x00c3, 0xd49: 0x00c3, 0xd4a: 0x00c3, 0xd4b: 0x00c3, + 0xd4c: 0x00c3, 0xd4d: 0x00c3, 0xd4e: 0x00c3, 0xd4f: 0x0080, 0xd50: 0x00c0, 0xd51: 0x00c0, + 0xd52: 0x00c0, 0xd53: 0x00c0, 0xd54: 0x00c0, 0xd55: 0x00c0, 0xd56: 0x00c0, 0xd57: 0x00c0, + 0xd58: 0x00c0, 0xd59: 0x00c0, 0xd5a: 0x0080, 0xd5b: 0x0080, + // Block 0x36, offset 0xd80 + 0xd81: 0x00c0, 0xd82: 0x00c0, 0xd84: 0x00c0, + 0xd86: 0x00c0, 0xd87: 0x00c0, 0xd88: 0x00c0, 0xd89: 0x00c0, 0xd8a: 0x00c0, + 0xd8c: 0x00c0, 0xd8d: 0x00c0, 0xd8e: 0x00c0, 0xd8f: 0x00c0, 0xd90: 0x00c0, 0xd91: 0x00c0, + 0xd92: 0x00c0, 0xd93: 0x00c0, 0xd94: 0x00c0, 0xd95: 0x00c0, 0xd96: 0x00c0, 0xd97: 0x00c0, + 0xd98: 0x00c0, 0xd99: 0x00c0, 0xd9a: 0x00c0, 0xd9b: 0x00c0, 0xd9c: 0x00c0, 0xd9d: 0x00c0, + 0xd9e: 0x00c0, 0xd9f: 0x00c0, 0xda0: 0x00c0, 0xda1: 0x00c0, 0xda2: 0x00c0, 0xda3: 0x00c0, + 0xda5: 0x00c0, 0xda7: 0x00c0, 0xda8: 0x00c0, 0xda9: 0x00c0, + 0xdaa: 0x00c0, 0xdab: 0x00c0, 0xdac: 0x00c0, 0xdad: 0x00c0, 0xdae: 0x00c0, 0xdaf: 0x00c0, + 0xdb0: 0x00c0, 0xdb1: 0x00c3, 0xdb2: 0x00c0, 0xdb3: 0x0080, 0xdb4: 0x00c3, 0xdb5: 0x00c3, + 0xdb6: 0x00c3, 0xdb7: 0x00c3, 0xdb8: 0x00c3, 0xdb9: 0x00c3, 0xdba: 0x00c6, 0xdbb: 0x00c3, + 0xdbc: 0x00c3, 0xdbd: 0x00c0, + // Block 0x37, offset 0xdc0 + 0xdc0: 0x00c0, 0xdc1: 0x00c0, 0xdc2: 0x00c0, 0xdc3: 0x00c0, 0xdc4: 0x00c0, + 0xdc6: 0x00c0, 0xdc8: 0x00c3, 0xdc9: 0x00c3, 0xdca: 0x00c3, 0xdcb: 0x00c3, + 0xdcc: 0x00c3, 0xdcd: 0x00c3, 0xdd0: 0x00c0, 0xdd1: 0x00c0, + 0xdd2: 0x00c0, 0xdd3: 0x00c0, 0xdd4: 0x00c0, 0xdd5: 0x00c0, 0xdd6: 0x00c0, 0xdd7: 0x00c0, + 0xdd8: 0x00c0, 0xdd9: 0x00c0, 0xddc: 0x0080, 0xddd: 0x0080, + 0xdde: 0x00c0, 0xddf: 0x00c0, + // Block 0x38, offset 0xe00 + 0xe00: 0x00c0, 0xe01: 0x0080, 0xe02: 0x0080, 0xe03: 0x0080, 0xe04: 0x0080, 0xe05: 0x0080, + 0xe06: 0x0080, 0xe07: 0x0080, 0xe08: 0x0080, 0xe09: 0x0080, 0xe0a: 0x0080, 0xe0b: 0x00c0, + 0xe0c: 0x0080, 0xe0d: 0x0080, 0xe0e: 0x0080, 0xe0f: 0x0080, 0xe10: 0x0080, 0xe11: 0x0080, + 0xe12: 0x0080, 0xe13: 0x0080, 0xe14: 0x0080, 0xe15: 0x0080, 0xe16: 0x0080, 0xe17: 0x0080, + 0xe18: 0x00c3, 0xe19: 0x00c3, 0xe1a: 0x0080, 0xe1b: 0x0080, 0xe1c: 0x0080, 0xe1d: 0x0080, + 0xe1e: 0x0080, 0xe1f: 0x0080, 0xe20: 0x00c0, 0xe21: 0x00c0, 0xe22: 0x00c0, 0xe23: 0x00c0, + 0xe24: 0x00c0, 0xe25: 0x00c0, 0xe26: 0x00c0, 0xe27: 0x00c0, 0xe28: 0x00c0, 0xe29: 0x00c0, + 0xe2a: 0x0080, 0xe2b: 0x0080, 0xe2c: 0x0080, 0xe2d: 0x0080, 0xe2e: 0x0080, 0xe2f: 0x0080, + 0xe30: 0x0080, 0xe31: 0x0080, 0xe32: 0x0080, 0xe33: 0x0080, 0xe34: 0x0080, 0xe35: 0x00c3, + 0xe36: 0x0080, 0xe37: 0x00c3, 0xe38: 0x0080, 0xe39: 0x00c3, 0xe3a: 0x0080, 0xe3b: 0x0080, + 0xe3c: 0x0080, 0xe3d: 0x0080, 0xe3e: 0x00c0, 0xe3f: 0x00c0, + // Block 0x39, offset 0xe40 + 0xe40: 0x00c0, 0xe41: 0x00c0, 0xe42: 0x00c0, 0xe43: 0x0080, 0xe44: 0x00c0, 0xe45: 0x00c0, + 0xe46: 0x00c0, 0xe47: 0x00c0, 0xe49: 0x00c0, 0xe4a: 0x00c0, 0xe4b: 0x00c0, + 0xe4c: 0x00c0, 0xe4d: 0x0080, 0xe4e: 0x00c0, 0xe4f: 0x00c0, 0xe50: 0x00c0, 0xe51: 0x00c0, + 0xe52: 0x0080, 0xe53: 0x00c0, 0xe54: 0x00c0, 0xe55: 0x00c0, 0xe56: 0x00c0, 0xe57: 0x0080, + 0xe58: 0x00c0, 0xe59: 0x00c0, 0xe5a: 0x00c0, 0xe5b: 0x00c0, 0xe5c: 0x0080, 0xe5d: 0x00c0, + 0xe5e: 0x00c0, 0xe5f: 0x00c0, 0xe60: 0x00c0, 0xe61: 0x00c0, 0xe62: 0x00c0, 0xe63: 0x00c0, + 0xe64: 0x00c0, 0xe65: 0x00c0, 0xe66: 0x00c0, 0xe67: 0x00c0, 0xe68: 0x00c0, 0xe69: 0x0080, + 0xe6a: 0x00c0, 0xe6b: 0x00c0, 0xe6c: 0x00c0, + 0xe71: 0x00c3, 0xe72: 0x00c3, 0xe73: 0x0083, 0xe74: 0x00c3, 0xe75: 0x0083, + 0xe76: 0x0083, 0xe77: 0x0083, 0xe78: 0x0083, 0xe79: 0x0083, 0xe7a: 0x00c3, 0xe7b: 0x00c3, + 0xe7c: 0x00c3, 0xe7d: 0x00c3, 0xe7e: 0x00c3, 0xe7f: 0x00c0, + // Block 0x3a, offset 0xe80 + 0xe80: 0x00c3, 0xe81: 0x0083, 0xe82: 0x00c3, 0xe83: 0x00c3, 0xe84: 0x00c6, 0xe85: 0x0080, + 0xe86: 0x00c3, 0xe87: 0x00c3, 0xe88: 0x00c0, 0xe89: 0x00c0, 0xe8a: 0x00c0, 0xe8b: 0x00c0, + 0xe8c: 0x00c0, 0xe8d: 0x00c3, 0xe8e: 0x00c3, 0xe8f: 0x00c3, 0xe90: 0x00c3, 0xe91: 0x00c3, + 0xe92: 0x00c3, 0xe93: 0x0083, 0xe94: 0x00c3, 0xe95: 0x00c3, 0xe96: 0x00c3, 0xe97: 0x00c3, + 0xe99: 0x00c3, 0xe9a: 0x00c3, 0xe9b: 0x00c3, 0xe9c: 0x00c3, 0xe9d: 0x0083, + 0xe9e: 0x00c3, 0xe9f: 0x00c3, 0xea0: 0x00c3, 0xea1: 0x00c3, 0xea2: 0x0083, 0xea3: 0x00c3, + 0xea4: 0x00c3, 0xea5: 0x00c3, 0xea6: 0x00c3, 0xea7: 0x0083, 0xea8: 0x00c3, 0xea9: 0x00c3, + 0xeaa: 0x00c3, 0xeab: 0x00c3, 0xeac: 0x0083, 0xead: 0x00c3, 0xeae: 0x00c3, 0xeaf: 0x00c3, + 0xeb0: 0x00c3, 0xeb1: 0x00c3, 0xeb2: 0x00c3, 0xeb3: 0x00c3, 0xeb4: 0x00c3, 0xeb5: 0x00c3, + 0xeb6: 0x00c3, 0xeb7: 0x00c3, 0xeb8: 0x00c3, 0xeb9: 0x0083, 0xeba: 0x00c3, 0xebb: 0x00c3, + 0xebc: 0x00c3, 0xebe: 0x0080, 0xebf: 0x0080, + // Block 0x3b, offset 0xec0 + 0xec0: 0x0080, 0xec1: 0x0080, 0xec2: 0x0080, 0xec3: 0x0080, 0xec4: 0x0080, 0xec5: 0x0080, + 0xec6: 0x00c3, 0xec7: 0x0080, 0xec8: 0x0080, 0xec9: 0x0080, 0xeca: 0x0080, 0xecb: 0x0080, + 0xecc: 0x0080, 0xece: 0x0080, 0xecf: 0x0080, 0xed0: 0x0080, 0xed1: 0x0080, + 0xed2: 0x0080, 0xed3: 0x0080, 0xed4: 0x0080, 0xed5: 0x0080, 0xed6: 0x0080, 0xed7: 0x0080, + 0xed8: 0x0080, 0xed9: 0x0080, 0xeda: 0x0080, + // Block 0x3c, offset 0xf00 + 0xf00: 0x00c0, 0xf01: 0x00c0, 0xf02: 0x00c0, 0xf03: 0x00c0, 0xf04: 0x00c0, 0xf05: 0x00c0, + 0xf06: 0x00c0, 0xf07: 0x00c0, 0xf08: 0x00c0, 0xf09: 0x00c0, 0xf0a: 0x00c0, 0xf0b: 0x00c0, + 0xf0c: 0x00c0, 0xf0d: 0x00c0, 0xf0e: 0x00c0, 0xf0f: 0x00c0, 0xf10: 0x00c0, 0xf11: 0x00c0, + 0xf12: 0x00c0, 0xf13: 0x00c0, 0xf14: 0x00c0, 0xf15: 0x00c0, 0xf16: 0x00c0, 0xf17: 0x00c0, + 0xf18: 0x00c0, 0xf19: 0x00c0, 0xf1a: 0x00c0, 0xf1b: 0x00c0, 0xf1c: 0x00c0, 0xf1d: 0x00c0, + 0xf1e: 0x00c0, 0xf1f: 0x00c0, 0xf20: 0x00c0, 0xf21: 0x00c0, 0xf22: 0x00c0, 0xf23: 0x00c0, + 0xf24: 0x00c0, 0xf25: 0x00c0, 0xf26: 0x00c0, 0xf27: 0x00c0, 0xf28: 0x00c0, 0xf29: 0x00c0, + 0xf2a: 0x00c0, 0xf2b: 0x00c0, 0xf2c: 0x00c0, 0xf2d: 0x00c3, 0xf2e: 0x00c3, 0xf2f: 0x00c3, + 0xf30: 0x00c3, 0xf31: 0x00c0, 0xf32: 0x00c3, 0xf33: 0x00c3, 0xf34: 0x00c3, 0xf35: 0x00c3, + 0xf36: 0x00c3, 0xf37: 0x00c3, 0xf38: 0x00c0, 0xf39: 0x00c6, 0xf3a: 0x00c6, 0xf3b: 0x00c0, + 0xf3c: 0x00c0, 0xf3d: 0x00c3, 0xf3e: 0x00c3, 0xf3f: 0x00c0, + // Block 0x3d, offset 0xf40 + 0xf40: 0x00c0, 0xf41: 0x00c0, 0xf42: 0x00c0, 0xf43: 0x00c0, 0xf44: 0x00c0, 0xf45: 0x00c0, + 0xf46: 0x00c0, 0xf47: 0x00c0, 0xf48: 0x00c0, 0xf49: 0x00c0, 0xf4a: 0x0080, 0xf4b: 0x0080, + 0xf4c: 0x0080, 0xf4d: 0x0080, 0xf4e: 0x0080, 0xf4f: 0x0080, 0xf50: 0x00c0, 0xf51: 0x00c0, + 0xf52: 0x00c0, 0xf53: 0x00c0, 0xf54: 0x00c0, 0xf55: 0x00c0, 0xf56: 0x00c0, 0xf57: 0x00c0, + 0xf58: 0x00c3, 0xf59: 0x00c3, 0xf5a: 0x00c0, 0xf5b: 0x00c0, 0xf5c: 0x00c0, 0xf5d: 0x00c0, + 0xf5e: 0x00c3, 0xf5f: 0x00c3, 0xf60: 0x00c3, 0xf61: 0x00c0, 0xf62: 0x00c0, 0xf63: 0x00c0, + 0xf64: 0x00c0, 0xf65: 0x00c0, 0xf66: 0x00c0, 0xf67: 0x00c0, 0xf68: 0x00c0, 0xf69: 0x00c0, + 0xf6a: 0x00c0, 0xf6b: 0x00c0, 0xf6c: 0x00c0, 0xf6d: 0x00c0, 0xf6e: 0x00c0, 0xf6f: 0x00c0, + 0xf70: 0x00c0, 0xf71: 0x00c3, 0xf72: 0x00c3, 0xf73: 0x00c3, 0xf74: 0x00c3, 0xf75: 0x00c0, + 0xf76: 0x00c0, 0xf77: 0x00c0, 0xf78: 0x00c0, 0xf79: 0x00c0, 0xf7a: 0x00c0, 0xf7b: 0x00c0, + 0xf7c: 0x00c0, 0xf7d: 0x00c0, 0xf7e: 0x00c0, 0xf7f: 0x00c0, + // Block 0x3e, offset 0xf80 + 0xf80: 0x00c0, 0xf81: 0x00c0, 0xf82: 0x00c3, 0xf83: 0x00c0, 0xf84: 0x00c0, 0xf85: 0x00c3, + 0xf86: 0x00c3, 0xf87: 0x00c0, 0xf88: 0x00c0, 0xf89: 0x00c0, 0xf8a: 0x00c0, 0xf8b: 0x00c0, + 0xf8c: 0x00c0, 0xf8d: 0x00c3, 0xf8e: 0x00c0, 0xf8f: 0x00c0, 0xf90: 0x00c0, 0xf91: 0x00c0, + 0xf92: 0x00c0, 0xf93: 0x00c0, 0xf94: 0x00c0, 0xf95: 0x00c0, 0xf96: 0x00c0, 0xf97: 0x00c0, + 0xf98: 0x00c0, 0xf99: 0x00c0, 0xf9a: 0x00c0, 0xf9b: 0x00c0, 0xf9c: 0x00c0, 0xf9d: 0x00c3, + 0xf9e: 0x0080, 0xf9f: 0x0080, 0xfa0: 0x00c0, 0xfa1: 0x00c0, 0xfa2: 0x00c0, 0xfa3: 0x00c0, + 0xfa4: 0x00c0, 0xfa5: 0x00c0, 0xfa6: 0x00c0, 0xfa7: 0x00c0, 0xfa8: 0x00c0, 0xfa9: 0x00c0, + 0xfaa: 0x00c0, 0xfab: 0x00c0, 0xfac: 0x00c0, 0xfad: 0x00c0, 0xfae: 0x00c0, 0xfaf: 0x00c0, + 0xfb0: 0x00c0, 0xfb1: 0x00c0, 0xfb2: 0x00c0, 0xfb3: 0x00c0, 0xfb4: 0x00c0, 0xfb5: 0x00c0, + 0xfb6: 0x00c0, 0xfb7: 0x00c0, 0xfb8: 0x00c0, 0xfb9: 0x00c0, 0xfba: 0x00c0, 0xfbb: 0x00c0, + 0xfbc: 0x00c0, 0xfbd: 0x00c0, 0xfbe: 0x00c0, 0xfbf: 0x00c0, + // Block 0x3f, offset 0xfc0 + 0xfc0: 0x00c0, 0xfc1: 0x00c0, 0xfc2: 0x00c0, 0xfc3: 0x00c0, 0xfc4: 0x00c0, 0xfc5: 0x00c0, + 0xfc7: 0x00c0, + 0xfcd: 0x00c0, 0xfd0: 0x00c0, 0xfd1: 0x00c0, + 0xfd2: 0x00c0, 0xfd3: 0x00c0, 0xfd4: 0x00c0, 0xfd5: 0x00c0, 0xfd6: 0x00c0, 0xfd7: 0x00c0, + 0xfd8: 0x00c0, 0xfd9: 0x00c0, 0xfda: 0x00c0, 0xfdb: 0x00c0, 0xfdc: 0x00c0, 0xfdd: 0x00c0, + 0xfde: 0x00c0, 0xfdf: 0x00c0, 0xfe0: 0x00c0, 0xfe1: 0x00c0, 0xfe2: 0x00c0, 0xfe3: 0x00c0, + 0xfe4: 0x00c0, 0xfe5: 0x00c0, 0xfe6: 0x00c0, 0xfe7: 0x00c0, 0xfe8: 0x00c0, 0xfe9: 0x00c0, + 0xfea: 0x00c0, 0xfeb: 0x00c0, 0xfec: 0x00c0, 0xfed: 0x00c0, 0xfee: 0x00c0, 0xfef: 0x00c0, + 0xff0: 0x00c0, 0xff1: 0x00c0, 0xff2: 0x00c0, 0xff3: 0x00c0, 0xff4: 0x00c0, 0xff5: 0x00c0, + 0xff6: 0x00c0, 0xff7: 0x00c0, 0xff8: 0x00c0, 0xff9: 0x00c0, 0xffa: 0x00c0, 0xffb: 0x0080, + 0xffc: 0x0080, 0xffd: 0x00c0, 0xffe: 0x00c0, 0xfff: 0x00c0, + // Block 0x40, offset 0x1000 + 0x1000: 0x0040, 0x1001: 0x0040, 0x1002: 0x0040, 0x1003: 0x0040, 0x1004: 0x0040, 0x1005: 0x0040, + 0x1006: 0x0040, 0x1007: 0x0040, 0x1008: 0x0040, 0x1009: 0x0040, 0x100a: 0x0040, 0x100b: 0x0040, + 0x100c: 0x0040, 0x100d: 0x0040, 0x100e: 0x0040, 0x100f: 0x0040, 0x1010: 0x0040, 0x1011: 0x0040, + 0x1012: 0x0040, 0x1013: 0x0040, 0x1014: 0x0040, 0x1015: 0x0040, 0x1016: 0x0040, 0x1017: 0x0040, + 0x1018: 0x0040, 0x1019: 0x0040, 0x101a: 0x0040, 0x101b: 0x0040, 0x101c: 0x0040, 0x101d: 0x0040, + 0x101e: 0x0040, 0x101f: 0x0040, 0x1020: 0x0040, 0x1021: 0x0040, 0x1022: 0x0040, 0x1023: 0x0040, + 0x1024: 0x0040, 0x1025: 0x0040, 0x1026: 0x0040, 0x1027: 0x0040, 0x1028: 0x0040, 0x1029: 0x0040, + 0x102a: 0x0040, 0x102b: 0x0040, 0x102c: 0x0040, 0x102d: 0x0040, 0x102e: 0x0040, 0x102f: 0x0040, + 0x1030: 0x0040, 0x1031: 0x0040, 0x1032: 0x0040, 0x1033: 0x0040, 0x1034: 0x0040, 0x1035: 0x0040, + 0x1036: 0x0040, 0x1037: 0x0040, 0x1038: 0x0040, 0x1039: 0x0040, 0x103a: 0x0040, 0x103b: 0x0040, + 0x103c: 0x0040, 0x103d: 0x0040, 0x103e: 0x0040, 0x103f: 0x0040, + // Block 0x41, offset 0x1040 + 0x1040: 0x00c0, 0x1041: 0x00c0, 0x1042: 0x00c0, 0x1043: 0x00c0, 0x1044: 0x00c0, 0x1045: 0x00c0, + 0x1046: 0x00c0, 0x1047: 0x00c0, 0x1048: 0x00c0, 0x104a: 0x00c0, 0x104b: 0x00c0, + 0x104c: 0x00c0, 0x104d: 0x00c0, 0x1050: 0x00c0, 0x1051: 0x00c0, + 0x1052: 0x00c0, 0x1053: 0x00c0, 0x1054: 0x00c0, 0x1055: 0x00c0, 0x1056: 0x00c0, + 0x1058: 0x00c0, 0x105a: 0x00c0, 0x105b: 0x00c0, 0x105c: 0x00c0, 0x105d: 0x00c0, + 0x1060: 0x00c0, 0x1061: 0x00c0, 0x1062: 0x00c0, 0x1063: 0x00c0, + 0x1064: 0x00c0, 0x1065: 0x00c0, 0x1066: 0x00c0, 0x1067: 0x00c0, 0x1068: 0x00c0, 0x1069: 0x00c0, + 0x106a: 0x00c0, 0x106b: 0x00c0, 0x106c: 0x00c0, 0x106d: 0x00c0, 0x106e: 0x00c0, 0x106f: 0x00c0, + 0x1070: 0x00c0, 0x1071: 0x00c0, 0x1072: 0x00c0, 0x1073: 0x00c0, 0x1074: 0x00c0, 0x1075: 0x00c0, + 0x1076: 0x00c0, 0x1077: 0x00c0, 0x1078: 0x00c0, 0x1079: 0x00c0, 0x107a: 0x00c0, 0x107b: 0x00c0, + 0x107c: 0x00c0, 0x107d: 0x00c0, 0x107e: 0x00c0, 0x107f: 0x00c0, + // Block 0x42, offset 0x1080 + 0x1080: 0x00c0, 0x1081: 0x00c0, 0x1082: 0x00c0, 0x1083: 0x00c0, 0x1084: 0x00c0, 0x1085: 0x00c0, + 0x1086: 0x00c0, 0x1087: 0x00c0, 0x1088: 0x00c0, 0x108a: 0x00c0, 0x108b: 0x00c0, + 0x108c: 0x00c0, 0x108d: 0x00c0, 0x1090: 0x00c0, 0x1091: 0x00c0, + 0x1092: 0x00c0, 0x1093: 0x00c0, 0x1094: 0x00c0, 0x1095: 0x00c0, 0x1096: 0x00c0, 0x1097: 0x00c0, + 0x1098: 0x00c0, 0x1099: 0x00c0, 0x109a: 0x00c0, 0x109b: 0x00c0, 0x109c: 0x00c0, 0x109d: 0x00c0, + 0x109e: 0x00c0, 0x109f: 0x00c0, 0x10a0: 0x00c0, 0x10a1: 0x00c0, 0x10a2: 0x00c0, 0x10a3: 0x00c0, + 0x10a4: 0x00c0, 0x10a5: 0x00c0, 0x10a6: 0x00c0, 0x10a7: 0x00c0, 0x10a8: 0x00c0, 0x10a9: 0x00c0, + 0x10aa: 0x00c0, 0x10ab: 0x00c0, 0x10ac: 0x00c0, 0x10ad: 0x00c0, 0x10ae: 0x00c0, 0x10af: 0x00c0, + 0x10b0: 0x00c0, 0x10b2: 0x00c0, 0x10b3: 0x00c0, 0x10b4: 0x00c0, 0x10b5: 0x00c0, + 0x10b8: 0x00c0, 0x10b9: 0x00c0, 0x10ba: 0x00c0, 0x10bb: 0x00c0, + 0x10bc: 0x00c0, 0x10bd: 0x00c0, 0x10be: 0x00c0, + // Block 0x43, offset 0x10c0 + 0x10c0: 0x00c0, 0x10c2: 0x00c0, 0x10c3: 0x00c0, 0x10c4: 0x00c0, 0x10c5: 0x00c0, + 0x10c8: 0x00c0, 0x10c9: 0x00c0, 0x10ca: 0x00c0, 0x10cb: 0x00c0, + 0x10cc: 0x00c0, 0x10cd: 0x00c0, 0x10ce: 0x00c0, 0x10cf: 0x00c0, 0x10d0: 0x00c0, 0x10d1: 0x00c0, + 0x10d2: 0x00c0, 0x10d3: 0x00c0, 0x10d4: 0x00c0, 0x10d5: 0x00c0, 0x10d6: 0x00c0, + 0x10d8: 0x00c0, 0x10d9: 0x00c0, 0x10da: 0x00c0, 0x10db: 0x00c0, 0x10dc: 0x00c0, 0x10dd: 0x00c0, + 0x10de: 0x00c0, 0x10df: 0x00c0, 0x10e0: 0x00c0, 0x10e1: 0x00c0, 0x10e2: 0x00c0, 0x10e3: 0x00c0, + 0x10e4: 0x00c0, 0x10e5: 0x00c0, 0x10e6: 0x00c0, 0x10e7: 0x00c0, 0x10e8: 0x00c0, 0x10e9: 0x00c0, + 0x10ea: 0x00c0, 0x10eb: 0x00c0, 0x10ec: 0x00c0, 0x10ed: 0x00c0, 0x10ee: 0x00c0, 0x10ef: 0x00c0, + 0x10f0: 0x00c0, 0x10f1: 0x00c0, 0x10f2: 0x00c0, 0x10f3: 0x00c0, 0x10f4: 0x00c0, 0x10f5: 0x00c0, + 0x10f6: 0x00c0, 0x10f7: 0x00c0, 0x10f8: 0x00c0, 0x10f9: 0x00c0, 0x10fa: 0x00c0, 0x10fb: 0x00c0, + 0x10fc: 0x00c0, 0x10fd: 0x00c0, 0x10fe: 0x00c0, 0x10ff: 0x00c0, + // Block 0x44, offset 0x1100 + 0x1100: 0x00c0, 0x1101: 0x00c0, 0x1102: 0x00c0, 0x1103: 0x00c0, 0x1104: 0x00c0, 0x1105: 0x00c0, + 0x1106: 0x00c0, 0x1107: 0x00c0, 0x1108: 0x00c0, 0x1109: 0x00c0, 0x110a: 0x00c0, 0x110b: 0x00c0, + 0x110c: 0x00c0, 0x110d: 0x00c0, 0x110e: 0x00c0, 0x110f: 0x00c0, 0x1110: 0x00c0, + 0x1112: 0x00c0, 0x1113: 0x00c0, 0x1114: 0x00c0, 0x1115: 0x00c0, + 0x1118: 0x00c0, 0x1119: 0x00c0, 0x111a: 0x00c0, 0x111b: 0x00c0, 0x111c: 0x00c0, 0x111d: 0x00c0, + 0x111e: 0x00c0, 0x111f: 0x00c0, 0x1120: 0x00c0, 0x1121: 0x00c0, 0x1122: 0x00c0, 0x1123: 0x00c0, + 0x1124: 0x00c0, 0x1125: 0x00c0, 0x1126: 0x00c0, 0x1127: 0x00c0, 0x1128: 0x00c0, 0x1129: 0x00c0, + 0x112a: 0x00c0, 0x112b: 0x00c0, 0x112c: 0x00c0, 0x112d: 0x00c0, 0x112e: 0x00c0, 0x112f: 0x00c0, + 0x1130: 0x00c0, 0x1131: 0x00c0, 0x1132: 0x00c0, 0x1133: 0x00c0, 0x1134: 0x00c0, 0x1135: 0x00c0, + 0x1136: 0x00c0, 0x1137: 0x00c0, 0x1138: 0x00c0, 0x1139: 0x00c0, 0x113a: 0x00c0, 0x113b: 0x00c0, + 0x113c: 0x00c0, 0x113d: 0x00c0, 0x113e: 0x00c0, 0x113f: 0x00c0, + // Block 0x45, offset 0x1140 + 0x1140: 0x00c0, 0x1141: 0x00c0, 0x1142: 0x00c0, 0x1143: 0x00c0, 0x1144: 0x00c0, 0x1145: 0x00c0, + 0x1146: 0x00c0, 0x1147: 0x00c0, 0x1148: 0x00c0, 0x1149: 0x00c0, 0x114a: 0x00c0, 0x114b: 0x00c0, + 0x114c: 0x00c0, 0x114d: 0x00c0, 0x114e: 0x00c0, 0x114f: 0x00c0, 0x1150: 0x00c0, 0x1151: 0x00c0, + 0x1152: 0x00c0, 0x1153: 0x00c0, 0x1154: 0x00c0, 0x1155: 0x00c0, 0x1156: 0x00c0, 0x1157: 0x00c0, + 0x1158: 0x00c0, 0x1159: 0x00c0, 0x115a: 0x00c0, 0x115d: 0x00c3, + 0x115e: 0x00c3, 0x115f: 0x00c3, 0x1160: 0x0080, 0x1161: 0x0080, 0x1162: 0x0080, 0x1163: 0x0080, + 0x1164: 0x0080, 0x1165: 0x0080, 0x1166: 0x0080, 0x1167: 0x0080, 0x1168: 0x0080, 0x1169: 0x0080, + 0x116a: 0x0080, 0x116b: 0x0080, 0x116c: 0x0080, 0x116d: 0x0080, 0x116e: 0x0080, 0x116f: 0x0080, + 0x1170: 0x0080, 0x1171: 0x0080, 0x1172: 0x0080, 0x1173: 0x0080, 0x1174: 0x0080, 0x1175: 0x0080, + 0x1176: 0x0080, 0x1177: 0x0080, 0x1178: 0x0080, 0x1179: 0x0080, 0x117a: 0x0080, 0x117b: 0x0080, + 0x117c: 0x0080, + // Block 0x46, offset 0x1180 + 0x1180: 0x00c0, 0x1181: 0x00c0, 0x1182: 0x00c0, 0x1183: 0x00c0, 0x1184: 0x00c0, 0x1185: 0x00c0, + 0x1186: 0x00c0, 0x1187: 0x00c0, 0x1188: 0x00c0, 0x1189: 0x00c0, 0x118a: 0x00c0, 0x118b: 0x00c0, + 0x118c: 0x00c0, 0x118d: 0x00c0, 0x118e: 0x00c0, 0x118f: 0x00c0, 0x1190: 0x0080, 0x1191: 0x0080, + 0x1192: 0x0080, 0x1193: 0x0080, 0x1194: 0x0080, 0x1195: 0x0080, 0x1196: 0x0080, 0x1197: 0x0080, + 0x1198: 0x0080, 0x1199: 0x0080, + 0x11a0: 0x00c0, 0x11a1: 0x00c0, 0x11a2: 0x00c0, 0x11a3: 0x00c0, + 0x11a4: 0x00c0, 0x11a5: 0x00c0, 0x11a6: 0x00c0, 0x11a7: 0x00c0, 0x11a8: 0x00c0, 0x11a9: 0x00c0, + 0x11aa: 0x00c0, 0x11ab: 0x00c0, 0x11ac: 0x00c0, 0x11ad: 0x00c0, 0x11ae: 0x00c0, 0x11af: 0x00c0, + 0x11b0: 0x00c0, 0x11b1: 0x00c0, 0x11b2: 0x00c0, 0x11b3: 0x00c0, 0x11b4: 0x00c0, 0x11b5: 0x00c0, + 0x11b6: 0x00c0, 0x11b7: 0x00c0, 0x11b8: 0x00c0, 0x11b9: 0x00c0, 0x11ba: 0x00c0, 0x11bb: 0x00c0, + 0x11bc: 0x00c0, 0x11bd: 0x00c0, 0x11be: 0x00c0, 0x11bf: 0x00c0, + // Block 0x47, offset 0x11c0 + 0x11c0: 0x00c0, 0x11c1: 0x00c0, 0x11c2: 0x00c0, 0x11c3: 0x00c0, 0x11c4: 0x00c0, 0x11c5: 0x00c0, + 0x11c6: 0x00c0, 0x11c7: 0x00c0, 0x11c8: 0x00c0, 0x11c9: 0x00c0, 0x11ca: 0x00c0, 0x11cb: 0x00c0, + 0x11cc: 0x00c0, 0x11cd: 0x00c0, 0x11ce: 0x00c0, 0x11cf: 0x00c0, 0x11d0: 0x00c0, 0x11d1: 0x00c0, + 0x11d2: 0x00c0, 0x11d3: 0x00c0, 0x11d4: 0x00c0, 0x11d5: 0x00c0, 0x11d6: 0x00c0, 0x11d7: 0x00c0, + 0x11d8: 0x00c0, 0x11d9: 0x00c0, 0x11da: 0x00c0, 0x11db: 0x00c0, 0x11dc: 0x00c0, 0x11dd: 0x00c0, + 0x11de: 0x00c0, 0x11df: 0x00c0, 0x11e0: 0x00c0, 0x11e1: 0x00c0, 0x11e2: 0x00c0, 0x11e3: 0x00c0, + 0x11e4: 0x00c0, 0x11e5: 0x00c0, 0x11e6: 0x00c0, 0x11e7: 0x00c0, 0x11e8: 0x00c0, 0x11e9: 0x00c0, + 0x11ea: 0x00c0, 0x11eb: 0x00c0, 0x11ec: 0x00c0, 0x11ed: 0x00c0, 0x11ee: 0x00c0, 0x11ef: 0x00c0, + 0x11f0: 0x00c0, 0x11f1: 0x00c0, 0x11f2: 0x00c0, 0x11f3: 0x00c0, 0x11f4: 0x00c0, 0x11f5: 0x00c0, + 0x11f8: 0x00c0, 0x11f9: 0x00c0, 0x11fa: 0x00c0, 0x11fb: 0x00c0, + 0x11fc: 0x00c0, 0x11fd: 0x00c0, + // Block 0x48, offset 0x1200 + 0x1200: 0x0080, 0x1201: 0x00c0, 0x1202: 0x00c0, 0x1203: 0x00c0, 0x1204: 0x00c0, 0x1205: 0x00c0, + 0x1206: 0x00c0, 0x1207: 0x00c0, 0x1208: 0x00c0, 0x1209: 0x00c0, 0x120a: 0x00c0, 0x120b: 0x00c0, + 0x120c: 0x00c0, 0x120d: 0x00c0, 0x120e: 0x00c0, 0x120f: 0x00c0, 0x1210: 0x00c0, 0x1211: 0x00c0, + 0x1212: 0x00c0, 0x1213: 0x00c0, 0x1214: 0x00c0, 0x1215: 0x00c0, 0x1216: 0x00c0, 0x1217: 0x00c0, + 0x1218: 0x00c0, 0x1219: 0x00c0, 0x121a: 0x00c0, 0x121b: 0x00c0, 0x121c: 0x00c0, 0x121d: 0x00c0, + 0x121e: 0x00c0, 0x121f: 0x00c0, 0x1220: 0x00c0, 0x1221: 0x00c0, 0x1222: 0x00c0, 0x1223: 0x00c0, + 0x1224: 0x00c0, 0x1225: 0x00c0, 0x1226: 0x00c0, 0x1227: 0x00c0, 0x1228: 0x00c0, 0x1229: 0x00c0, + 0x122a: 0x00c0, 0x122b: 0x00c0, 0x122c: 0x00c0, 0x122d: 0x00c0, 0x122e: 0x00c0, 0x122f: 0x00c0, + 0x1230: 0x00c0, 0x1231: 0x00c0, 0x1232: 0x00c0, 0x1233: 0x00c0, 0x1234: 0x00c0, 0x1235: 0x00c0, + 0x1236: 0x00c0, 0x1237: 0x00c0, 0x1238: 0x00c0, 0x1239: 0x00c0, 0x123a: 0x00c0, 0x123b: 0x00c0, + 0x123c: 0x00c0, 0x123d: 0x00c0, 0x123e: 0x00c0, 0x123f: 0x00c0, + // Block 0x49, offset 0x1240 + 0x1240: 0x00c0, 0x1241: 0x00c0, 0x1242: 0x00c0, 0x1243: 0x00c0, 0x1244: 0x00c0, 0x1245: 0x00c0, + 0x1246: 0x00c0, 0x1247: 0x00c0, 0x1248: 0x00c0, 0x1249: 0x00c0, 0x124a: 0x00c0, 0x124b: 0x00c0, + 0x124c: 0x00c0, 0x124d: 0x00c0, 0x124e: 0x00c0, 0x124f: 0x00c0, 0x1250: 0x00c0, 0x1251: 0x00c0, + 0x1252: 0x00c0, 0x1253: 0x00c0, 0x1254: 0x00c0, 0x1255: 0x00c0, 0x1256: 0x00c0, 0x1257: 0x00c0, + 0x1258: 0x00c0, 0x1259: 0x00c0, 0x125a: 0x00c0, 0x125b: 0x00c0, 0x125c: 0x00c0, 0x125d: 0x00c0, + 0x125e: 0x00c0, 0x125f: 0x00c0, 0x1260: 0x00c0, 0x1261: 0x00c0, 0x1262: 0x00c0, 0x1263: 0x00c0, + 0x1264: 0x00c0, 0x1265: 0x00c0, 0x1266: 0x00c0, 0x1267: 0x00c0, 0x1268: 0x00c0, 0x1269: 0x00c0, + 0x126a: 0x00c0, 0x126b: 0x00c0, 0x126c: 0x00c0, 0x126d: 0x0080, 0x126e: 0x0080, 0x126f: 0x00c0, + 0x1270: 0x00c0, 0x1271: 0x00c0, 0x1272: 0x00c0, 0x1273: 0x00c0, 0x1274: 0x00c0, 0x1275: 0x00c0, + 0x1276: 0x00c0, 0x1277: 0x00c0, 0x1278: 0x00c0, 0x1279: 0x00c0, 0x127a: 0x00c0, 0x127b: 0x00c0, + 0x127c: 0x00c0, 0x127d: 0x00c0, 0x127e: 0x00c0, 0x127f: 0x00c0, + // Block 0x4a, offset 0x1280 + 0x1280: 0x0080, 0x1281: 0x00c0, 0x1282: 0x00c0, 0x1283: 0x00c0, 0x1284: 0x00c0, 0x1285: 0x00c0, + 0x1286: 0x00c0, 0x1287: 0x00c0, 0x1288: 0x00c0, 0x1289: 0x00c0, 0x128a: 0x00c0, 0x128b: 0x00c0, + 0x128c: 0x00c0, 0x128d: 0x00c0, 0x128e: 0x00c0, 0x128f: 0x00c0, 0x1290: 0x00c0, 0x1291: 0x00c0, + 0x1292: 0x00c0, 0x1293: 0x00c0, 0x1294: 0x00c0, 0x1295: 0x00c0, 0x1296: 0x00c0, 0x1297: 0x00c0, + 0x1298: 0x00c0, 0x1299: 0x00c0, 0x129a: 0x00c0, 0x129b: 0x0080, 0x129c: 0x0080, + 0x12a0: 0x00c0, 0x12a1: 0x00c0, 0x12a2: 0x00c0, 0x12a3: 0x00c0, + 0x12a4: 0x00c0, 0x12a5: 0x00c0, 0x12a6: 0x00c0, 0x12a7: 0x00c0, 0x12a8: 0x00c0, 0x12a9: 0x00c0, + 0x12aa: 0x00c0, 0x12ab: 0x00c0, 0x12ac: 0x00c0, 0x12ad: 0x00c0, 0x12ae: 0x00c0, 0x12af: 0x00c0, + 0x12b0: 0x00c0, 0x12b1: 0x00c0, 0x12b2: 0x00c0, 0x12b3: 0x00c0, 0x12b4: 0x00c0, 0x12b5: 0x00c0, + 0x12b6: 0x00c0, 0x12b7: 0x00c0, 0x12b8: 0x00c0, 0x12b9: 0x00c0, 0x12ba: 0x00c0, 0x12bb: 0x00c0, + 0x12bc: 0x00c0, 0x12bd: 0x00c0, 0x12be: 0x00c0, 0x12bf: 0x00c0, + // Block 0x4b, offset 0x12c0 + 0x12c0: 0x00c0, 0x12c1: 0x00c0, 0x12c2: 0x00c0, 0x12c3: 0x00c0, 0x12c4: 0x00c0, 0x12c5: 0x00c0, + 0x12c6: 0x00c0, 0x12c7: 0x00c0, 0x12c8: 0x00c0, 0x12c9: 0x00c0, 0x12ca: 0x00c0, 0x12cb: 0x00c0, + 0x12cc: 0x00c0, 0x12cd: 0x00c0, 0x12ce: 0x00c0, 0x12cf: 0x00c0, 0x12d0: 0x00c0, 0x12d1: 0x00c0, + 0x12d2: 0x00c0, 0x12d3: 0x00c0, 0x12d4: 0x00c0, 0x12d5: 0x00c0, 0x12d6: 0x00c0, 0x12d7: 0x00c0, + 0x12d8: 0x00c0, 0x12d9: 0x00c0, 0x12da: 0x00c0, 0x12db: 0x00c0, 0x12dc: 0x00c0, 0x12dd: 0x00c0, + 0x12de: 0x00c0, 0x12df: 0x00c0, 0x12e0: 0x00c0, 0x12e1: 0x00c0, 0x12e2: 0x00c0, 0x12e3: 0x00c0, + 0x12e4: 0x00c0, 0x12e5: 0x00c0, 0x12e6: 0x00c0, 0x12e7: 0x00c0, 0x12e8: 0x00c0, 0x12e9: 0x00c0, + 0x12ea: 0x00c0, 0x12eb: 0x0080, 0x12ec: 0x0080, 0x12ed: 0x0080, 0x12ee: 0x0080, 0x12ef: 0x0080, + 0x12f0: 0x0080, 0x12f1: 0x00c0, 0x12f2: 0x00c0, 0x12f3: 0x00c0, 0x12f4: 0x00c0, 0x12f5: 0x00c0, + 0x12f6: 0x00c0, 0x12f7: 0x00c0, 0x12f8: 0x00c0, + // Block 0x4c, offset 0x1300 + 0x1300: 0x00c0, 0x1301: 0x00c0, 0x1302: 0x00c0, 0x1303: 0x00c0, 0x1304: 0x00c0, 0x1305: 0x00c0, + 0x1306: 0x00c0, 0x1307: 0x00c0, 0x1308: 0x00c0, 0x1309: 0x00c0, 0x130a: 0x00c0, 0x130b: 0x00c0, + 0x130c: 0x00c0, 0x130e: 0x00c0, 0x130f: 0x00c0, 0x1310: 0x00c0, 0x1311: 0x00c0, + 0x1312: 0x00c3, 0x1313: 0x00c3, 0x1314: 0x00c6, + 0x1320: 0x00c0, 0x1321: 0x00c0, 0x1322: 0x00c0, 0x1323: 0x00c0, + 0x1324: 0x00c0, 0x1325: 0x00c0, 0x1326: 0x00c0, 0x1327: 0x00c0, 0x1328: 0x00c0, 0x1329: 0x00c0, + 0x132a: 0x00c0, 0x132b: 0x00c0, 0x132c: 0x00c0, 0x132d: 0x00c0, 0x132e: 0x00c0, 0x132f: 0x00c0, + 0x1330: 0x00c0, 0x1331: 0x00c0, 0x1332: 0x00c3, 0x1333: 0x00c3, 0x1334: 0x00c6, 0x1335: 0x0080, + 0x1336: 0x0080, + // Block 0x4d, offset 0x1340 + 0x1340: 0x00c0, 0x1341: 0x00c0, 0x1342: 0x00c0, 0x1343: 0x00c0, 0x1344: 0x00c0, 0x1345: 0x00c0, + 0x1346: 0x00c0, 0x1347: 0x00c0, 0x1348: 0x00c0, 0x1349: 0x00c0, 0x134a: 0x00c0, 0x134b: 0x00c0, + 0x134c: 0x00c0, 0x134d: 0x00c0, 0x134e: 0x00c0, 0x134f: 0x00c0, 0x1350: 0x00c0, 0x1351: 0x00c0, + 0x1352: 0x00c3, 0x1353: 0x00c3, + 0x1360: 0x00c0, 0x1361: 0x00c0, 0x1362: 0x00c0, 0x1363: 0x00c0, + 0x1364: 0x00c0, 0x1365: 0x00c0, 0x1366: 0x00c0, 0x1367: 0x00c0, 0x1368: 0x00c0, 0x1369: 0x00c0, + 0x136a: 0x00c0, 0x136b: 0x00c0, 0x136c: 0x00c0, 0x136e: 0x00c0, 0x136f: 0x00c0, + 0x1370: 0x00c0, 0x1372: 0x00c3, 0x1373: 0x00c3, + // Block 0x4e, offset 0x1380 + 0x1380: 0x00c0, 0x1381: 0x00c0, 0x1382: 0x00c0, 0x1383: 0x00c0, 0x1384: 0x00c0, 0x1385: 0x00c0, + 0x1386: 0x00c0, 0x1387: 0x00c0, 0x1388: 0x00c0, 0x1389: 0x00c0, 0x138a: 0x00c0, 0x138b: 0x00c0, + 0x138c: 0x00c0, 0x138d: 0x00c0, 0x138e: 0x00c0, 0x138f: 0x00c0, 0x1390: 0x00c0, 0x1391: 0x00c0, + 0x1392: 0x00c0, 0x1393: 0x00c0, 0x1394: 0x00c0, 0x1395: 0x00c0, 0x1396: 0x00c0, 0x1397: 0x00c0, + 0x1398: 0x00c0, 0x1399: 0x00c0, 0x139a: 0x00c0, 0x139b: 0x00c0, 0x139c: 0x00c0, 0x139d: 0x00c0, + 0x139e: 0x00c0, 0x139f: 0x00c0, 0x13a0: 0x00c0, 0x13a1: 0x00c0, 0x13a2: 0x00c0, 0x13a3: 0x00c0, + 0x13a4: 0x00c0, 0x13a5: 0x00c0, 0x13a6: 0x00c0, 0x13a7: 0x00c0, 0x13a8: 0x00c0, 0x13a9: 0x00c0, + 0x13aa: 0x00c0, 0x13ab: 0x00c0, 0x13ac: 0x00c0, 0x13ad: 0x00c0, 0x13ae: 0x00c0, 0x13af: 0x00c0, + 0x13b0: 0x00c0, 0x13b1: 0x00c0, 0x13b2: 0x00c0, 0x13b3: 0x00c0, 0x13b4: 0x0040, 0x13b5: 0x0040, + 0x13b6: 0x00c0, 0x13b7: 0x00c3, 0x13b8: 0x00c3, 0x13b9: 0x00c3, 0x13ba: 0x00c3, 0x13bb: 0x00c3, + 0x13bc: 0x00c3, 0x13bd: 0x00c3, 0x13be: 0x00c0, 0x13bf: 0x00c0, + // Block 0x4f, offset 0x13c0 + 0x13c0: 0x00c0, 0x13c1: 0x00c0, 0x13c2: 0x00c0, 0x13c3: 0x00c0, 0x13c4: 0x00c0, 0x13c5: 0x00c0, + 0x13c6: 0x00c3, 0x13c7: 0x00c0, 0x13c8: 0x00c0, 0x13c9: 0x00c3, 0x13ca: 0x00c3, 0x13cb: 0x00c3, + 0x13cc: 0x00c3, 0x13cd: 0x00c3, 0x13ce: 0x00c3, 0x13cf: 0x00c3, 0x13d0: 0x00c3, 0x13d1: 0x00c3, + 0x13d2: 0x00c6, 0x13d3: 0x00c3, 0x13d4: 0x0080, 0x13d5: 0x0080, 0x13d6: 0x0080, 0x13d7: 0x00c0, + 0x13d8: 0x0080, 0x13d9: 0x0080, 0x13da: 0x0080, 0x13db: 0x0080, 0x13dc: 0x00c0, 0x13dd: 0x00c3, + 0x13e0: 0x00c0, 0x13e1: 0x00c0, 0x13e2: 0x00c0, 0x13e3: 0x00c0, + 0x13e4: 0x00c0, 0x13e5: 0x00c0, 0x13e6: 0x00c0, 0x13e7: 0x00c0, 0x13e8: 0x00c0, 0x13e9: 0x00c0, + 0x13f0: 0x0080, 0x13f1: 0x0080, 0x13f2: 0x0080, 0x13f3: 0x0080, 0x13f4: 0x0080, 0x13f5: 0x0080, + 0x13f6: 0x0080, 0x13f7: 0x0080, 0x13f8: 0x0080, 0x13f9: 0x0080, + // Block 0x50, offset 0x1400 + 0x1400: 0x0080, 0x1401: 0x0080, 0x1402: 0x0080, 0x1403: 0x0080, 0x1404: 0x0080, 0x1405: 0x0080, + 0x1406: 0x0080, 0x1407: 0x0082, 0x1408: 0x0080, 0x1409: 0x0080, 0x140a: 0x0080, 0x140b: 0x0040, + 0x140c: 0x0040, 0x140d: 0x0040, 0x140e: 0x0040, 0x1410: 0x00c0, 0x1411: 0x00c0, + 0x1412: 0x00c0, 0x1413: 0x00c0, 0x1414: 0x00c0, 0x1415: 0x00c0, 0x1416: 0x00c0, 0x1417: 0x00c0, + 0x1418: 0x00c0, 0x1419: 0x00c0, + 0x1420: 0x00c2, 0x1421: 0x00c2, 0x1422: 0x00c2, 0x1423: 0x00c2, + 0x1424: 0x00c2, 0x1425: 0x00c2, 0x1426: 0x00c2, 0x1427: 0x00c2, 0x1428: 0x00c2, 0x1429: 0x00c2, + 0x142a: 0x00c2, 0x142b: 0x00c2, 0x142c: 0x00c2, 0x142d: 0x00c2, 0x142e: 0x00c2, 0x142f: 0x00c2, + 0x1430: 0x00c2, 0x1431: 0x00c2, 0x1432: 0x00c2, 0x1433: 0x00c2, 0x1434: 0x00c2, 0x1435: 0x00c2, + 0x1436: 0x00c2, 0x1437: 0x00c2, 0x1438: 0x00c2, 0x1439: 0x00c2, 0x143a: 0x00c2, 0x143b: 0x00c2, + 0x143c: 0x00c2, 0x143d: 0x00c2, 0x143e: 0x00c2, 0x143f: 0x00c2, + // Block 0x51, offset 0x1440 + 0x1440: 0x00c2, 0x1441: 0x00c2, 0x1442: 0x00c2, 0x1443: 0x00c2, 0x1444: 0x00c2, 0x1445: 0x00c2, + 0x1446: 0x00c2, 0x1447: 0x00c2, 0x1448: 0x00c2, 0x1449: 0x00c2, 0x144a: 0x00c2, 0x144b: 0x00c2, + 0x144c: 0x00c2, 0x144d: 0x00c2, 0x144e: 0x00c2, 0x144f: 0x00c2, 0x1450: 0x00c2, 0x1451: 0x00c2, + 0x1452: 0x00c2, 0x1453: 0x00c2, 0x1454: 0x00c2, 0x1455: 0x00c2, 0x1456: 0x00c2, 0x1457: 0x00c2, + 0x1458: 0x00c2, 0x1459: 0x00c2, 0x145a: 0x00c2, 0x145b: 0x00c2, 0x145c: 0x00c2, 0x145d: 0x00c2, + 0x145e: 0x00c2, 0x145f: 0x00c2, 0x1460: 0x00c2, 0x1461: 0x00c2, 0x1462: 0x00c2, 0x1463: 0x00c2, + 0x1464: 0x00c2, 0x1465: 0x00c2, 0x1466: 0x00c2, 0x1467: 0x00c2, 0x1468: 0x00c2, 0x1469: 0x00c2, + 0x146a: 0x00c2, 0x146b: 0x00c2, 0x146c: 0x00c2, 0x146d: 0x00c2, 0x146e: 0x00c2, 0x146f: 0x00c2, + 0x1470: 0x00c2, 0x1471: 0x00c2, 0x1472: 0x00c2, 0x1473: 0x00c2, 0x1474: 0x00c2, 0x1475: 0x00c2, + 0x1476: 0x00c2, 0x1477: 0x00c2, 0x1478: 0x00c2, + // Block 0x52, offset 0x1480 + 0x1480: 0x00c0, 0x1481: 0x00c0, 0x1482: 0x00c0, 0x1483: 0x00c0, 0x1484: 0x00c0, 0x1485: 0x00c3, + 0x1486: 0x00c3, 0x1487: 0x00c2, 0x1488: 0x00c2, 0x1489: 0x00c2, 0x148a: 0x00c2, 0x148b: 0x00c2, + 0x148c: 0x00c2, 0x148d: 0x00c2, 0x148e: 0x00c2, 0x148f: 0x00c2, 0x1490: 0x00c2, 0x1491: 0x00c2, + 0x1492: 0x00c2, 0x1493: 0x00c2, 0x1494: 0x00c2, 0x1495: 0x00c2, 0x1496: 0x00c2, 0x1497: 0x00c2, + 0x1498: 0x00c2, 0x1499: 0x00c2, 0x149a: 0x00c2, 0x149b: 0x00c2, 0x149c: 0x00c2, 0x149d: 0x00c2, + 0x149e: 0x00c2, 0x149f: 0x00c2, 0x14a0: 0x00c2, 0x14a1: 0x00c2, 0x14a2: 0x00c2, 0x14a3: 0x00c2, + 0x14a4: 0x00c2, 0x14a5: 0x00c2, 0x14a6: 0x00c2, 0x14a7: 0x00c2, 0x14a8: 0x00c2, 0x14a9: 0x00c3, + 0x14aa: 0x00c2, + 0x14b0: 0x00c0, 0x14b1: 0x00c0, 0x14b2: 0x00c0, 0x14b3: 0x00c0, 0x14b4: 0x00c0, 0x14b5: 0x00c0, + 0x14b6: 0x00c0, 0x14b7: 0x00c0, 0x14b8: 0x00c0, 0x14b9: 0x00c0, 0x14ba: 0x00c0, 0x14bb: 0x00c0, + 0x14bc: 0x00c0, 0x14bd: 0x00c0, 0x14be: 0x00c0, 0x14bf: 0x00c0, + // Block 0x53, offset 0x14c0 + 0x14c0: 0x00c0, 0x14c1: 0x00c0, 0x14c2: 0x00c0, 0x14c3: 0x00c0, 0x14c4: 0x00c0, 0x14c5: 0x00c0, + 0x14c6: 0x00c0, 0x14c7: 0x00c0, 0x14c8: 0x00c0, 0x14c9: 0x00c0, 0x14ca: 0x00c0, 0x14cb: 0x00c0, + 0x14cc: 0x00c0, 0x14cd: 0x00c0, 0x14ce: 0x00c0, 0x14cf: 0x00c0, 0x14d0: 0x00c0, 0x14d1: 0x00c0, + 0x14d2: 0x00c0, 0x14d3: 0x00c0, 0x14d4: 0x00c0, 0x14d5: 0x00c0, 0x14d6: 0x00c0, 0x14d7: 0x00c0, + 0x14d8: 0x00c0, 0x14d9: 0x00c0, 0x14da: 0x00c0, 0x14db: 0x00c0, 0x14dc: 0x00c0, 0x14dd: 0x00c0, + 0x14de: 0x00c0, 0x14df: 0x00c0, 0x14e0: 0x00c0, 0x14e1: 0x00c0, 0x14e2: 0x00c0, 0x14e3: 0x00c0, + 0x14e4: 0x00c0, 0x14e5: 0x00c0, 0x14e6: 0x00c0, 0x14e7: 0x00c0, 0x14e8: 0x00c0, 0x14e9: 0x00c0, + 0x14ea: 0x00c0, 0x14eb: 0x00c0, 0x14ec: 0x00c0, 0x14ed: 0x00c0, 0x14ee: 0x00c0, 0x14ef: 0x00c0, + 0x14f0: 0x00c0, 0x14f1: 0x00c0, 0x14f2: 0x00c0, 0x14f3: 0x00c0, 0x14f4: 0x00c0, 0x14f5: 0x00c0, + // Block 0x54, offset 0x1500 + 0x1500: 0x00c0, 0x1501: 0x00c0, 0x1502: 0x00c0, 0x1503: 0x00c0, 0x1504: 0x00c0, 0x1505: 0x00c0, + 0x1506: 0x00c0, 0x1507: 0x00c0, 0x1508: 0x00c0, 0x1509: 0x00c0, 0x150a: 0x00c0, 0x150b: 0x00c0, + 0x150c: 0x00c0, 0x150d: 0x00c0, 0x150e: 0x00c0, 0x150f: 0x00c0, 0x1510: 0x00c0, 0x1511: 0x00c0, + 0x1512: 0x00c0, 0x1513: 0x00c0, 0x1514: 0x00c0, 0x1515: 0x00c0, 0x1516: 0x00c0, 0x1517: 0x00c0, + 0x1518: 0x00c0, 0x1519: 0x00c0, 0x151a: 0x00c0, 0x151b: 0x00c0, 0x151c: 0x00c0, 0x151d: 0x00c0, + 0x151e: 0x00c0, 0x1520: 0x00c3, 0x1521: 0x00c3, 0x1522: 0x00c3, 0x1523: 0x00c0, + 0x1524: 0x00c0, 0x1525: 0x00c0, 0x1526: 0x00c0, 0x1527: 0x00c3, 0x1528: 0x00c3, 0x1529: 0x00c0, + 0x152a: 0x00c0, 0x152b: 0x00c0, + 0x1530: 0x00c0, 0x1531: 0x00c0, 0x1532: 0x00c3, 0x1533: 0x00c0, 0x1534: 0x00c0, 0x1535: 0x00c0, + 0x1536: 0x00c0, 0x1537: 0x00c0, 0x1538: 0x00c0, 0x1539: 0x00c3, 0x153a: 0x00c3, 0x153b: 0x00c3, + // Block 0x55, offset 0x1540 + 0x1540: 0x0080, 0x1544: 0x0080, 0x1545: 0x0080, + 0x1546: 0x00c0, 0x1547: 0x00c0, 0x1548: 0x00c0, 0x1549: 0x00c0, 0x154a: 0x00c0, 0x154b: 0x00c0, + 0x154c: 0x00c0, 0x154d: 0x00c0, 0x154e: 0x00c0, 0x154f: 0x00c0, 0x1550: 0x00c0, 0x1551: 0x00c0, + 0x1552: 0x00c0, 0x1553: 0x00c0, 0x1554: 0x00c0, 0x1555: 0x00c0, 0x1556: 0x00c0, 0x1557: 0x00c0, + 0x1558: 0x00c0, 0x1559: 0x00c0, 0x155a: 0x00c0, 0x155b: 0x00c0, 0x155c: 0x00c0, 0x155d: 0x00c0, + 0x155e: 0x00c0, 0x155f: 0x00c0, 0x1560: 0x00c0, 0x1561: 0x00c0, 0x1562: 0x00c0, 0x1563: 0x00c0, + 0x1564: 0x00c0, 0x1565: 0x00c0, 0x1566: 0x00c0, 0x1567: 0x00c0, 0x1568: 0x00c0, 0x1569: 0x00c0, + 0x156a: 0x00c0, 0x156b: 0x00c0, 0x156c: 0x00c0, 0x156d: 0x00c0, + 0x1570: 0x00c0, 0x1571: 0x00c0, 0x1572: 0x00c0, 0x1573: 0x00c0, 0x1574: 0x00c0, + // Block 0x56, offset 0x1580 + 0x1580: 0x00c0, 0x1581: 0x00c0, 0x1582: 0x00c0, 0x1583: 0x00c0, 0x1584: 0x00c0, 0x1585: 0x00c0, + 0x1586: 0x00c0, 0x1587: 0x00c0, 0x1588: 0x00c0, 0x1589: 0x00c0, 0x158a: 0x00c0, 0x158b: 0x00c0, + 0x158c: 0x00c0, 0x158d: 0x00c0, 0x158e: 0x00c0, 0x158f: 0x00c0, 0x1590: 0x00c0, 0x1591: 0x00c0, + 0x1592: 0x00c0, 0x1593: 0x00c0, 0x1594: 0x00c0, 0x1595: 0x00c0, 0x1596: 0x00c0, 0x1597: 0x00c0, + 0x1598: 0x00c0, 0x1599: 0x00c0, 0x159a: 0x00c0, 0x159b: 0x00c0, 0x159c: 0x00c0, 0x159d: 0x00c0, + 0x159e: 0x00c0, 0x159f: 0x00c0, 0x15a0: 0x00c0, 0x15a1: 0x00c0, 0x15a2: 0x00c0, 0x15a3: 0x00c0, + 0x15a4: 0x00c0, 0x15a5: 0x00c0, 0x15a6: 0x00c0, 0x15a7: 0x00c0, 0x15a8: 0x00c0, 0x15a9: 0x00c0, + 0x15aa: 0x00c0, 0x15ab: 0x00c0, + 0x15b0: 0x00c0, 0x15b1: 0x00c0, 0x15b2: 0x00c0, 0x15b3: 0x00c0, 0x15b4: 0x00c0, 0x15b5: 0x00c0, + 0x15b6: 0x00c0, 0x15b7: 0x00c0, 0x15b8: 0x00c0, 0x15b9: 0x00c0, 0x15ba: 0x00c0, 0x15bb: 0x00c0, + 0x15bc: 0x00c0, 0x15bd: 0x00c0, 0x15be: 0x00c0, 0x15bf: 0x00c0, + // Block 0x57, offset 0x15c0 + 0x15c0: 0x00c0, 0x15c1: 0x00c0, 0x15c2: 0x00c0, 0x15c3: 0x00c0, 0x15c4: 0x00c0, 0x15c5: 0x00c0, + 0x15c6: 0x00c0, 0x15c7: 0x00c0, 0x15c8: 0x00c0, 0x15c9: 0x00c0, + 0x15d0: 0x00c0, 0x15d1: 0x00c0, + 0x15d2: 0x00c0, 0x15d3: 0x00c0, 0x15d4: 0x00c0, 0x15d5: 0x00c0, 0x15d6: 0x00c0, 0x15d7: 0x00c0, + 0x15d8: 0x00c0, 0x15d9: 0x00c0, 0x15da: 0x0080, + 0x15de: 0x0080, 0x15df: 0x0080, 0x15e0: 0x0080, 0x15e1: 0x0080, 0x15e2: 0x0080, 0x15e3: 0x0080, + 0x15e4: 0x0080, 0x15e5: 0x0080, 0x15e6: 0x0080, 0x15e7: 0x0080, 0x15e8: 0x0080, 0x15e9: 0x0080, + 0x15ea: 0x0080, 0x15eb: 0x0080, 0x15ec: 0x0080, 0x15ed: 0x0080, 0x15ee: 0x0080, 0x15ef: 0x0080, + 0x15f0: 0x0080, 0x15f1: 0x0080, 0x15f2: 0x0080, 0x15f3: 0x0080, 0x15f4: 0x0080, 0x15f5: 0x0080, + 0x15f6: 0x0080, 0x15f7: 0x0080, 0x15f8: 0x0080, 0x15f9: 0x0080, 0x15fa: 0x0080, 0x15fb: 0x0080, + 0x15fc: 0x0080, 0x15fd: 0x0080, 0x15fe: 0x0080, 0x15ff: 0x0080, + // Block 0x58, offset 0x1600 + 0x1600: 0x00c0, 0x1601: 0x00c0, 0x1602: 0x00c0, 0x1603: 0x00c0, 0x1604: 0x00c0, 0x1605: 0x00c0, + 0x1606: 0x00c0, 0x1607: 0x00c0, 0x1608: 0x00c0, 0x1609: 0x00c0, 0x160a: 0x00c0, 0x160b: 0x00c0, + 0x160c: 0x00c0, 0x160d: 0x00c0, 0x160e: 0x00c0, 0x160f: 0x00c0, 0x1610: 0x00c0, 0x1611: 0x00c0, + 0x1612: 0x00c0, 0x1613: 0x00c0, 0x1614: 0x00c0, 0x1615: 0x00c0, 0x1616: 0x00c0, 0x1617: 0x00c3, + 0x1618: 0x00c3, 0x1619: 0x00c0, 0x161a: 0x00c0, 0x161b: 0x00c3, + 0x161e: 0x0080, 0x161f: 0x0080, 0x1620: 0x00c0, 0x1621: 0x00c0, 0x1622: 0x00c0, 0x1623: 0x00c0, + 0x1624: 0x00c0, 0x1625: 0x00c0, 0x1626: 0x00c0, 0x1627: 0x00c0, 0x1628: 0x00c0, 0x1629: 0x00c0, + 0x162a: 0x00c0, 0x162b: 0x00c0, 0x162c: 0x00c0, 0x162d: 0x00c0, 0x162e: 0x00c0, 0x162f: 0x00c0, + 0x1630: 0x00c0, 0x1631: 0x00c0, 0x1632: 0x00c0, 0x1633: 0x00c0, 0x1634: 0x00c0, 0x1635: 0x00c0, + 0x1636: 0x00c0, 0x1637: 0x00c0, 0x1638: 0x00c0, 0x1639: 0x00c0, 0x163a: 0x00c0, 0x163b: 0x00c0, + 0x163c: 0x00c0, 0x163d: 0x00c0, 0x163e: 0x00c0, 0x163f: 0x00c0, + // Block 0x59, offset 0x1640 + 0x1640: 0x00c0, 0x1641: 0x00c0, 0x1642: 0x00c0, 0x1643: 0x00c0, 0x1644: 0x00c0, 0x1645: 0x00c0, + 0x1646: 0x00c0, 0x1647: 0x00c0, 0x1648: 0x00c0, 0x1649: 0x00c0, 0x164a: 0x00c0, 0x164b: 0x00c0, + 0x164c: 0x00c0, 0x164d: 0x00c0, 0x164e: 0x00c0, 0x164f: 0x00c0, 0x1650: 0x00c0, 0x1651: 0x00c0, + 0x1652: 0x00c0, 0x1653: 0x00c0, 0x1654: 0x00c0, 0x1655: 0x00c0, 0x1656: 0x00c3, 0x1657: 0x00c0, + 0x1658: 0x00c3, 0x1659: 0x00c3, 0x165a: 0x00c3, 0x165b: 0x00c3, 0x165c: 0x00c3, 0x165d: 0x00c3, + 0x165e: 0x00c3, 0x1660: 0x00c6, 0x1661: 0x00c0, 0x1662: 0x00c3, 0x1663: 0x00c0, + 0x1664: 0x00c0, 0x1665: 0x00c3, 0x1666: 0x00c3, 0x1667: 0x00c3, 0x1668: 0x00c3, 0x1669: 0x00c3, + 0x166a: 0x00c3, 0x166b: 0x00c3, 0x166c: 0x00c3, 0x166d: 0x00c0, 0x166e: 0x00c0, 0x166f: 0x00c0, + 0x1670: 0x00c0, 0x1671: 0x00c0, 0x1672: 0x00c0, 0x1673: 0x00c3, 0x1674: 0x00c3, 0x1675: 0x00c3, + 0x1676: 0x00c3, 0x1677: 0x00c3, 0x1678: 0x00c3, 0x1679: 0x00c3, 0x167a: 0x00c3, 0x167b: 0x00c3, + 0x167c: 0x00c3, 0x167f: 0x00c3, + // Block 0x5a, offset 0x1680 + 0x1680: 0x00c0, 0x1681: 0x00c0, 0x1682: 0x00c0, 0x1683: 0x00c0, 0x1684: 0x00c0, 0x1685: 0x00c0, + 0x1686: 0x00c0, 0x1687: 0x00c0, 0x1688: 0x00c0, 0x1689: 0x00c0, + 0x1690: 0x00c0, 0x1691: 0x00c0, + 0x1692: 0x00c0, 0x1693: 0x00c0, 0x1694: 0x00c0, 0x1695: 0x00c0, 0x1696: 0x00c0, 0x1697: 0x00c0, + 0x1698: 0x00c0, 0x1699: 0x00c0, + 0x16a0: 0x0080, 0x16a1: 0x0080, 0x16a2: 0x0080, 0x16a3: 0x0080, + 0x16a4: 0x0080, 0x16a5: 0x0080, 0x16a6: 0x0080, 0x16a7: 0x00c0, 0x16a8: 0x0080, 0x16a9: 0x0080, + 0x16aa: 0x0080, 0x16ab: 0x0080, 0x16ac: 0x0080, 0x16ad: 0x0080, + 0x16b0: 0x00c3, 0x16b1: 0x00c3, 0x16b2: 0x00c3, 0x16b3: 0x00c3, 0x16b4: 0x00c3, 0x16b5: 0x00c3, + 0x16b6: 0x00c3, 0x16b7: 0x00c3, 0x16b8: 0x00c3, 0x16b9: 0x00c3, 0x16ba: 0x00c3, 0x16bb: 0x00c3, + 0x16bc: 0x00c3, 0x16bd: 0x00c3, 0x16be: 0x0083, + // Block 0x5b, offset 0x16c0 + 0x16c0: 0x00c3, 0x16c1: 0x00c3, 0x16c2: 0x00c3, 0x16c3: 0x00c3, 0x16c4: 0x00c0, 0x16c5: 0x00c0, + 0x16c6: 0x00c0, 0x16c7: 0x00c0, 0x16c8: 0x00c0, 0x16c9: 0x00c0, 0x16ca: 0x00c0, 0x16cb: 0x00c0, + 0x16cc: 0x00c0, 0x16cd: 0x00c0, 0x16ce: 0x00c0, 0x16cf: 0x00c0, 0x16d0: 0x00c0, 0x16d1: 0x00c0, + 0x16d2: 0x00c0, 0x16d3: 0x00c0, 0x16d4: 0x00c0, 0x16d5: 0x00c0, 0x16d6: 0x00c0, 0x16d7: 0x00c0, + 0x16d8: 0x00c0, 0x16d9: 0x00c0, 0x16da: 0x00c0, 0x16db: 0x00c0, 0x16dc: 0x00c0, 0x16dd: 0x00c0, + 0x16de: 0x00c0, 0x16df: 0x00c0, 0x16e0: 0x00c0, 0x16e1: 0x00c0, 0x16e2: 0x00c0, 0x16e3: 0x00c0, + 0x16e4: 0x00c0, 0x16e5: 0x00c0, 0x16e6: 0x00c0, 0x16e7: 0x00c0, 0x16e8: 0x00c0, 0x16e9: 0x00c0, + 0x16ea: 0x00c0, 0x16eb: 0x00c0, 0x16ec: 0x00c0, 0x16ed: 0x00c0, 0x16ee: 0x00c0, 0x16ef: 0x00c0, + 0x16f0: 0x00c0, 0x16f1: 0x00c0, 0x16f2: 0x00c0, 0x16f3: 0x00c0, 0x16f4: 0x00c3, 0x16f5: 0x00c0, + 0x16f6: 0x00c3, 0x16f7: 0x00c3, 0x16f8: 0x00c3, 0x16f9: 0x00c3, 0x16fa: 0x00c3, 0x16fb: 0x00c0, + 0x16fc: 0x00c3, 0x16fd: 0x00c0, 0x16fe: 0x00c0, 0x16ff: 0x00c0, + // Block 0x5c, offset 0x1700 + 0x1700: 0x00c0, 0x1701: 0x00c0, 0x1702: 0x00c3, 0x1703: 0x00c0, 0x1704: 0x00c5, 0x1705: 0x00c0, + 0x1706: 0x00c0, 0x1707: 0x00c0, 0x1708: 0x00c0, 0x1709: 0x00c0, 0x170a: 0x00c0, 0x170b: 0x00c0, + 0x1710: 0x00c0, 0x1711: 0x00c0, + 0x1712: 0x00c0, 0x1713: 0x00c0, 0x1714: 0x00c0, 0x1715: 0x00c0, 0x1716: 0x00c0, 0x1717: 0x00c0, + 0x1718: 0x00c0, 0x1719: 0x00c0, 0x171a: 0x0080, 0x171b: 0x0080, 0x171c: 0x0080, 0x171d: 0x0080, + 0x171e: 0x0080, 0x171f: 0x0080, 0x1720: 0x0080, 0x1721: 0x0080, 0x1722: 0x0080, 0x1723: 0x0080, + 0x1724: 0x0080, 0x1725: 0x0080, 0x1726: 0x0080, 0x1727: 0x0080, 0x1728: 0x0080, 0x1729: 0x0080, + 0x172a: 0x0080, 0x172b: 0x00c3, 0x172c: 0x00c3, 0x172d: 0x00c3, 0x172e: 0x00c3, 0x172f: 0x00c3, + 0x1730: 0x00c3, 0x1731: 0x00c3, 0x1732: 0x00c3, 0x1733: 0x00c3, 0x1734: 0x0080, 0x1735: 0x0080, + 0x1736: 0x0080, 0x1737: 0x0080, 0x1738: 0x0080, 0x1739: 0x0080, 0x173a: 0x0080, 0x173b: 0x0080, + 0x173c: 0x0080, + // Block 0x5d, offset 0x1740 + 0x1740: 0x00c3, 0x1741: 0x00c3, 0x1742: 0x00c0, 0x1743: 0x00c0, 0x1744: 0x00c0, 0x1745: 0x00c0, + 0x1746: 0x00c0, 0x1747: 0x00c0, 0x1748: 0x00c0, 0x1749: 0x00c0, 0x174a: 0x00c0, 0x174b: 0x00c0, + 0x174c: 0x00c0, 0x174d: 0x00c0, 0x174e: 0x00c0, 0x174f: 0x00c0, 0x1750: 0x00c0, 0x1751: 0x00c0, + 0x1752: 0x00c0, 0x1753: 0x00c0, 0x1754: 0x00c0, 0x1755: 0x00c0, 0x1756: 0x00c0, 0x1757: 0x00c0, + 0x1758: 0x00c0, 0x1759: 0x00c0, 0x175a: 0x00c0, 0x175b: 0x00c0, 0x175c: 0x00c0, 0x175d: 0x00c0, + 0x175e: 0x00c0, 0x175f: 0x00c0, 0x1760: 0x00c0, 0x1761: 0x00c0, 0x1762: 0x00c3, 0x1763: 0x00c3, + 0x1764: 0x00c3, 0x1765: 0x00c3, 0x1766: 0x00c0, 0x1767: 0x00c0, 0x1768: 0x00c3, 0x1769: 0x00c3, + 0x176a: 0x00c5, 0x176b: 0x00c6, 0x176c: 0x00c3, 0x176d: 0x00c3, 0x176e: 0x00c0, 0x176f: 0x00c0, + 0x1770: 0x00c0, 0x1771: 0x00c0, 0x1772: 0x00c0, 0x1773: 0x00c0, 0x1774: 0x00c0, 0x1775: 0x00c0, + 0x1776: 0x00c0, 0x1777: 0x00c0, 0x1778: 0x00c0, 0x1779: 0x00c0, 0x177a: 0x00c0, 0x177b: 0x00c0, + 0x177c: 0x00c0, 0x177d: 0x00c0, 0x177e: 0x00c0, 0x177f: 0x00c0, + // Block 0x5e, offset 0x1780 + 0x1780: 0x00c0, 0x1781: 0x00c0, 0x1782: 0x00c0, 0x1783: 0x00c0, 0x1784: 0x00c0, 0x1785: 0x00c0, + 0x1786: 0x00c0, 0x1787: 0x00c0, 0x1788: 0x00c0, 0x1789: 0x00c0, 0x178a: 0x00c0, 0x178b: 0x00c0, + 0x178c: 0x00c0, 0x178d: 0x00c0, 0x178e: 0x00c0, 0x178f: 0x00c0, 0x1790: 0x00c0, 0x1791: 0x00c0, + 0x1792: 0x00c0, 0x1793: 0x00c0, 0x1794: 0x00c0, 0x1795: 0x00c0, 0x1796: 0x00c0, 0x1797: 0x00c0, + 0x1798: 0x00c0, 0x1799: 0x00c0, 0x179a: 0x00c0, 0x179b: 0x00c0, 0x179c: 0x00c0, 0x179d: 0x00c0, + 0x179e: 0x00c0, 0x179f: 0x00c0, 0x17a0: 0x00c0, 0x17a1: 0x00c0, 0x17a2: 0x00c0, 0x17a3: 0x00c0, + 0x17a4: 0x00c0, 0x17a5: 0x00c0, 0x17a6: 0x00c3, 0x17a7: 0x00c0, 0x17a8: 0x00c3, 0x17a9: 0x00c3, + 0x17aa: 0x00c0, 0x17ab: 0x00c0, 0x17ac: 0x00c0, 0x17ad: 0x00c3, 0x17ae: 0x00c0, 0x17af: 0x00c3, + 0x17b0: 0x00c3, 0x17b1: 0x00c3, 0x17b2: 0x00c5, 0x17b3: 0x00c5, + 0x17bc: 0x0080, 0x17bd: 0x0080, 0x17be: 0x0080, 0x17bf: 0x0080, + // Block 0x5f, offset 0x17c0 + 0x17c0: 0x00c0, 0x17c1: 0x00c0, 0x17c2: 0x00c0, 0x17c3: 0x00c0, 0x17c4: 0x00c0, 0x17c5: 0x00c0, + 0x17c6: 0x00c0, 0x17c7: 0x00c0, 0x17c8: 0x00c0, 0x17c9: 0x00c0, 0x17ca: 0x00c0, 0x17cb: 0x00c0, + 0x17cc: 0x00c0, 0x17cd: 0x00c0, 0x17ce: 0x00c0, 0x17cf: 0x00c0, 0x17d0: 0x00c0, 0x17d1: 0x00c0, + 0x17d2: 0x00c0, 0x17d3: 0x00c0, 0x17d4: 0x00c0, 0x17d5: 0x00c0, 0x17d6: 0x00c0, 0x17d7: 0x00c0, + 0x17d8: 0x00c0, 0x17d9: 0x00c0, 0x17da: 0x00c0, 0x17db: 0x00c0, 0x17dc: 0x00c0, 0x17dd: 0x00c0, + 0x17de: 0x00c0, 0x17df: 0x00c0, 0x17e0: 0x00c0, 0x17e1: 0x00c0, 0x17e2: 0x00c0, 0x17e3: 0x00c0, + 0x17e4: 0x00c0, 0x17e5: 0x00c0, 0x17e6: 0x00c0, 0x17e7: 0x00c0, 0x17e8: 0x00c0, 0x17e9: 0x00c0, + 0x17ea: 0x00c0, 0x17eb: 0x00c0, 0x17ec: 0x00c3, 0x17ed: 0x00c3, 0x17ee: 0x00c3, 0x17ef: 0x00c3, + 0x17f0: 0x00c3, 0x17f1: 0x00c3, 0x17f2: 0x00c3, 0x17f3: 0x00c3, 0x17f4: 0x00c0, 0x17f5: 0x00c0, + 0x17f6: 0x00c3, 0x17f7: 0x00c3, 0x17fb: 0x0080, + 0x17fc: 0x0080, 0x17fd: 0x0080, 0x17fe: 0x0080, 0x17ff: 0x0080, + // Block 0x60, offset 0x1800 + 0x1800: 0x00c0, 0x1801: 0x00c0, 0x1802: 0x00c0, 0x1803: 0x00c0, 0x1804: 0x00c0, 0x1805: 0x00c0, + 0x1806: 0x00c0, 0x1807: 0x00c0, 0x1808: 0x00c0, 0x1809: 0x00c0, + 0x180d: 0x00c0, 0x180e: 0x00c0, 0x180f: 0x00c0, 0x1810: 0x00c0, 0x1811: 0x00c0, + 0x1812: 0x00c0, 0x1813: 0x00c0, 0x1814: 0x00c0, 0x1815: 0x00c0, 0x1816: 0x00c0, 0x1817: 0x00c0, + 0x1818: 0x00c0, 0x1819: 0x00c0, 0x181a: 0x00c0, 0x181b: 0x00c0, 0x181c: 0x00c0, 0x181d: 0x00c0, + 0x181e: 0x00c0, 0x181f: 0x00c0, 0x1820: 0x00c0, 0x1821: 0x00c0, 0x1822: 0x00c0, 0x1823: 0x00c0, + 0x1824: 0x00c0, 0x1825: 0x00c0, 0x1826: 0x00c0, 0x1827: 0x00c0, 0x1828: 0x00c0, 0x1829: 0x00c0, + 0x182a: 0x00c0, 0x182b: 0x00c0, 0x182c: 0x00c0, 0x182d: 0x00c0, 0x182e: 0x00c0, 0x182f: 0x00c0, + 0x1830: 0x00c0, 0x1831: 0x00c0, 0x1832: 0x00c0, 0x1833: 0x00c0, 0x1834: 0x00c0, 0x1835: 0x00c0, + 0x1836: 0x00c0, 0x1837: 0x00c0, 0x1838: 0x00c0, 0x1839: 0x00c0, 0x183a: 0x00c0, 0x183b: 0x00c0, + 0x183c: 0x00c0, 0x183d: 0x00c0, 0x183e: 0x0080, 0x183f: 0x0080, + // Block 0x61, offset 0x1840 + 0x1840: 0x00c0, 0x1841: 0x00c0, 0x1842: 0x00c0, 0x1843: 0x00c0, 0x1844: 0x00c0, 0x1845: 0x00c0, + 0x1846: 0x00c0, 0x1847: 0x00c0, 0x1848: 0x00c0, + 0x1850: 0x00c0, 0x1851: 0x00c0, + 0x1852: 0x00c0, 0x1853: 0x00c0, 0x1854: 0x00c0, 0x1855: 0x00c0, 0x1856: 0x00c0, 0x1857: 0x00c0, + 0x1858: 0x00c0, 0x1859: 0x00c0, 0x185a: 0x00c0, 0x185b: 0x00c0, 0x185c: 0x00c0, 0x185d: 0x00c0, + 0x185e: 0x00c0, 0x185f: 0x00c0, 0x1860: 0x00c0, 0x1861: 0x00c0, 0x1862: 0x00c0, 0x1863: 0x00c0, + 0x1864: 0x00c0, 0x1865: 0x00c0, 0x1866: 0x00c0, 0x1867: 0x00c0, 0x1868: 0x00c0, 0x1869: 0x00c0, + 0x186a: 0x00c0, 0x186b: 0x00c0, 0x186c: 0x00c0, 0x186d: 0x00c0, 0x186e: 0x00c0, 0x186f: 0x00c0, + 0x1870: 0x00c0, 0x1871: 0x00c0, 0x1872: 0x00c0, 0x1873: 0x00c0, 0x1874: 0x00c0, 0x1875: 0x00c0, + 0x1876: 0x00c0, 0x1877: 0x00c0, 0x1878: 0x00c0, 0x1879: 0x00c0, 0x187a: 0x00c0, + 0x187d: 0x00c0, 0x187e: 0x00c0, 0x187f: 0x00c0, + // Block 0x62, offset 0x1880 + 0x1880: 0x0080, 0x1881: 0x0080, 0x1882: 0x0080, 0x1883: 0x0080, 0x1884: 0x0080, 0x1885: 0x0080, + 0x1886: 0x0080, 0x1887: 0x0080, + 0x1890: 0x00c3, 0x1891: 0x00c3, + 0x1892: 0x00c3, 0x1893: 0x0080, 0x1894: 0x00c3, 0x1895: 0x00c3, 0x1896: 0x00c3, 0x1897: 0x00c3, + 0x1898: 0x00c3, 0x1899: 0x00c3, 0x189a: 0x00c3, 0x189b: 0x00c3, 0x189c: 0x00c3, 0x189d: 0x00c3, + 0x189e: 0x00c3, 0x189f: 0x00c3, 0x18a0: 0x00c3, 0x18a1: 0x00c0, 0x18a2: 0x00c3, 0x18a3: 0x00c3, + 0x18a4: 0x00c3, 0x18a5: 0x00c3, 0x18a6: 0x00c3, 0x18a7: 0x00c3, 0x18a8: 0x00c3, 0x18a9: 0x00c0, + 0x18aa: 0x00c0, 0x18ab: 0x00c0, 0x18ac: 0x00c0, 0x18ad: 0x00c3, 0x18ae: 0x00c0, 0x18af: 0x00c0, + 0x18b0: 0x00c0, 0x18b1: 0x00c0, 0x18b2: 0x00c0, 0x18b3: 0x00c0, 0x18b4: 0x00c3, 0x18b5: 0x00c0, + 0x18b6: 0x00c0, 0x18b7: 0x00c0, 0x18b8: 0x00c3, 0x18b9: 0x00c3, 0x18ba: 0x00c0, + // Block 0x63, offset 0x18c0 + 0x18c0: 0x00c0, 0x18c1: 0x00c0, 0x18c2: 0x00c0, 0x18c3: 0x00c0, 0x18c4: 0x00c0, 0x18c5: 0x00c0, + 0x18c6: 0x00c0, 0x18c7: 0x00c0, 0x18c8: 0x00c0, 0x18c9: 0x00c0, 0x18ca: 0x00c0, 0x18cb: 0x00c0, + 0x18cc: 0x00c0, 0x18cd: 0x00c0, 0x18ce: 0x00c0, 0x18cf: 0x00c0, 0x18d0: 0x00c0, 0x18d1: 0x00c0, + 0x18d2: 0x00c0, 0x18d3: 0x00c0, 0x18d4: 0x00c0, 0x18d5: 0x00c0, 0x18d6: 0x00c0, 0x18d7: 0x00c0, + 0x18d8: 0x00c0, 0x18d9: 0x00c0, 0x18da: 0x00c0, 0x18db: 0x00c0, 0x18dc: 0x00c0, 0x18dd: 0x00c0, + 0x18de: 0x00c0, 0x18df: 0x00c0, 0x18e0: 0x00c0, 0x18e1: 0x00c0, 0x18e2: 0x00c0, 0x18e3: 0x00c0, + 0x18e4: 0x00c0, 0x18e5: 0x00c0, 0x18e6: 0x00c8, 0x18e7: 0x00c8, 0x18e8: 0x00c8, 0x18e9: 0x00c8, + 0x18ea: 0x00c8, 0x18eb: 0x00c0, 0x18ec: 0x0080, 0x18ed: 0x0080, 0x18ee: 0x0080, 0x18ef: 0x00c0, + 0x18f0: 0x0080, 0x18f1: 0x0080, 0x18f2: 0x0080, 0x18f3: 0x0080, 0x18f4: 0x0080, 0x18f5: 0x0080, + 0x18f6: 0x0080, 0x18f7: 0x0080, 0x18f8: 0x0080, 0x18f9: 0x0080, 0x18fa: 0x0080, 0x18fb: 0x00c0, + 0x18fc: 0x0080, 0x18fd: 0x0080, 0x18fe: 0x0080, 0x18ff: 0x0080, + // Block 0x64, offset 0x1900 + 0x1900: 0x0080, 0x1901: 0x0080, 0x1902: 0x0080, 0x1903: 0x0080, 0x1904: 0x0080, 0x1905: 0x0080, + 0x1906: 0x0080, 0x1907: 0x0080, 0x1908: 0x0080, 0x1909: 0x0080, 0x190a: 0x0080, 0x190b: 0x0080, + 0x190c: 0x0080, 0x190d: 0x0080, 0x190e: 0x00c0, 0x190f: 0x0080, 0x1910: 0x0080, 0x1911: 0x0080, + 0x1912: 0x0080, 0x1913: 0x0080, 0x1914: 0x0080, 0x1915: 0x0080, 0x1916: 0x0080, 0x1917: 0x0080, + 0x1918: 0x0080, 0x1919: 0x0080, 0x191a: 0x0080, 0x191b: 0x0080, 0x191c: 0x0080, 0x191d: 0x0088, + 0x191e: 0x0088, 0x191f: 0x0088, 0x1920: 0x0088, 0x1921: 0x0088, 0x1922: 0x0080, 0x1923: 0x0080, + 0x1924: 0x0080, 0x1925: 0x0080, 0x1926: 0x0088, 0x1927: 0x0088, 0x1928: 0x0088, 0x1929: 0x0088, + 0x192a: 0x0088, 0x192b: 0x00c0, 0x192c: 0x00c0, 0x192d: 0x00c0, 0x192e: 0x00c0, 0x192f: 0x00c0, + 0x1930: 0x00c0, 0x1931: 0x00c0, 0x1932: 0x00c0, 0x1933: 0x00c0, 0x1934: 0x00c0, 0x1935: 0x00c0, + 0x1936: 0x00c0, 0x1937: 0x00c0, 0x1938: 0x0080, 0x1939: 0x00c0, 0x193a: 0x00c0, 0x193b: 0x00c0, + 0x193c: 0x00c0, 0x193d: 0x00c0, 0x193e: 0x00c0, 0x193f: 0x00c0, + // Block 0x65, offset 0x1940 + 0x1940: 0x00c0, 0x1941: 0x00c0, 0x1942: 0x00c0, 0x1943: 0x00c0, 0x1944: 0x00c0, 0x1945: 0x00c0, + 0x1946: 0x00c0, 0x1947: 0x00c0, 0x1948: 0x00c0, 0x1949: 0x00c0, 0x194a: 0x00c0, 0x194b: 0x00c0, + 0x194c: 0x00c0, 0x194d: 0x00c0, 0x194e: 0x00c0, 0x194f: 0x00c0, 0x1950: 0x00c0, 0x1951: 0x00c0, + 0x1952: 0x00c0, 0x1953: 0x00c0, 0x1954: 0x00c0, 0x1955: 0x00c0, 0x1956: 0x00c0, 0x1957: 0x00c0, + 0x1958: 0x00c0, 0x1959: 0x00c0, 0x195a: 0x00c0, 0x195b: 0x0080, 0x195c: 0x0080, 0x195d: 0x0080, + 0x195e: 0x0080, 0x195f: 0x0080, 0x1960: 0x0080, 0x1961: 0x0080, 0x1962: 0x0080, 0x1963: 0x0080, + 0x1964: 0x0080, 0x1965: 0x0080, 0x1966: 0x0080, 0x1967: 0x0080, 0x1968: 0x0080, 0x1969: 0x0080, + 0x196a: 0x0080, 0x196b: 0x0080, 0x196c: 0x0080, 0x196d: 0x0080, 0x196e: 0x0080, 0x196f: 0x0080, + 0x1970: 0x0080, 0x1971: 0x0080, 0x1972: 0x0080, 0x1973: 0x0080, 0x1974: 0x0080, 0x1975: 0x0080, + 0x1976: 0x0080, 0x1977: 0x0080, 0x1978: 0x0080, 0x1979: 0x0080, 0x197a: 0x0080, 0x197b: 0x0080, + 0x197c: 0x0080, 0x197d: 0x0080, 0x197e: 0x0080, 0x197f: 0x0088, + // Block 0x66, offset 0x1980 + 0x1980: 0x00c3, 0x1981: 0x00c3, 0x1982: 0x00c3, 0x1983: 0x00c3, 0x1984: 0x00c3, 0x1985: 0x00c3, + 0x1986: 0x00c3, 0x1987: 0x00c3, 0x1988: 0x00c3, 0x1989: 0x00c3, 0x198a: 0x00c3, 0x198b: 0x00c3, + 0x198c: 0x00c3, 0x198d: 0x00c3, 0x198e: 0x00c3, 0x198f: 0x00c3, 0x1990: 0x00c3, 0x1991: 0x00c3, + 0x1992: 0x00c3, 0x1993: 0x00c3, 0x1994: 0x00c3, 0x1995: 0x00c3, 0x1996: 0x00c3, 0x1997: 0x00c3, + 0x1998: 0x00c3, 0x1999: 0x00c3, 0x199a: 0x00c3, 0x199b: 0x00c3, 0x199c: 0x00c3, 0x199d: 0x00c3, + 0x199e: 0x00c3, 0x199f: 0x00c3, 0x19a0: 0x00c3, 0x19a1: 0x00c3, 0x19a2: 0x00c3, 0x19a3: 0x00c3, + 0x19a4: 0x00c3, 0x19a5: 0x00c3, 0x19a6: 0x00c3, 0x19a7: 0x00c3, 0x19a8: 0x00c3, 0x19a9: 0x00c3, + 0x19aa: 0x00c3, 0x19ab: 0x00c3, 0x19ac: 0x00c3, 0x19ad: 0x00c3, 0x19ae: 0x00c3, 0x19af: 0x00c3, + 0x19b0: 0x00c3, 0x19b1: 0x00c3, 0x19b2: 0x00c3, 0x19b3: 0x00c3, 0x19b4: 0x00c3, 0x19b5: 0x00c3, + 0x19b6: 0x00c3, 0x19b7: 0x00c3, 0x19b8: 0x00c3, 0x19b9: 0x00c3, 0x19bb: 0x00c3, + 0x19bc: 0x00c3, 0x19bd: 0x00c3, 0x19be: 0x00c3, 0x19bf: 0x00c3, + // Block 0x67, offset 0x19c0 + 0x19c0: 0x00c0, 0x19c1: 0x00c0, 0x19c2: 0x00c0, 0x19c3: 0x00c0, 0x19c4: 0x00c0, 0x19c5: 0x00c0, + 0x19c6: 0x00c0, 0x19c7: 0x00c0, 0x19c8: 0x00c0, 0x19c9: 0x00c0, 0x19ca: 0x00c0, 0x19cb: 0x00c0, + 0x19cc: 0x00c0, 0x19cd: 0x00c0, 0x19ce: 0x00c0, 0x19cf: 0x00c0, 0x19d0: 0x00c0, 0x19d1: 0x00c0, + 0x19d2: 0x00c0, 0x19d3: 0x00c0, 0x19d4: 0x00c0, 0x19d5: 0x00c0, 0x19d6: 0x00c0, 0x19d7: 0x00c0, + 0x19d8: 0x00c0, 0x19d9: 0x00c0, 0x19da: 0x0080, 0x19db: 0x0080, 0x19dc: 0x00c0, 0x19dd: 0x00c0, + 0x19de: 0x00c0, 0x19df: 0x00c0, 0x19e0: 0x00c0, 0x19e1: 0x00c0, 0x19e2: 0x00c0, 0x19e3: 0x00c0, + 0x19e4: 0x00c0, 0x19e5: 0x00c0, 0x19e6: 0x00c0, 0x19e7: 0x00c0, 0x19e8: 0x00c0, 0x19e9: 0x00c0, + 0x19ea: 0x00c0, 0x19eb: 0x00c0, 0x19ec: 0x00c0, 0x19ed: 0x00c0, 0x19ee: 0x00c0, 0x19ef: 0x00c0, + 0x19f0: 0x00c0, 0x19f1: 0x00c0, 0x19f2: 0x00c0, 0x19f3: 0x00c0, 0x19f4: 0x00c0, 0x19f5: 0x00c0, + 0x19f6: 0x00c0, 0x19f7: 0x00c0, 0x19f8: 0x00c0, 0x19f9: 0x00c0, 0x19fa: 0x00c0, 0x19fb: 0x00c0, + 0x19fc: 0x00c0, 0x19fd: 0x00c0, 0x19fe: 0x00c0, 0x19ff: 0x00c0, + // Block 0x68, offset 0x1a00 + 0x1a00: 0x00c8, 0x1a01: 0x00c8, 0x1a02: 0x00c8, 0x1a03: 0x00c8, 0x1a04: 0x00c8, 0x1a05: 0x00c8, + 0x1a06: 0x00c8, 0x1a07: 0x00c8, 0x1a08: 0x00c8, 0x1a09: 0x00c8, 0x1a0a: 0x00c8, 0x1a0b: 0x00c8, + 0x1a0c: 0x00c8, 0x1a0d: 0x00c8, 0x1a0e: 0x00c8, 0x1a0f: 0x00c8, 0x1a10: 0x00c8, 0x1a11: 0x00c8, + 0x1a12: 0x00c8, 0x1a13: 0x00c8, 0x1a14: 0x00c8, 0x1a15: 0x00c8, + 0x1a18: 0x00c8, 0x1a19: 0x00c8, 0x1a1a: 0x00c8, 0x1a1b: 0x00c8, 0x1a1c: 0x00c8, 0x1a1d: 0x00c8, + 0x1a20: 0x00c8, 0x1a21: 0x00c8, 0x1a22: 0x00c8, 0x1a23: 0x00c8, + 0x1a24: 0x00c8, 0x1a25: 0x00c8, 0x1a26: 0x00c8, 0x1a27: 0x00c8, 0x1a28: 0x00c8, 0x1a29: 0x00c8, + 0x1a2a: 0x00c8, 0x1a2b: 0x00c8, 0x1a2c: 0x00c8, 0x1a2d: 0x00c8, 0x1a2e: 0x00c8, 0x1a2f: 0x00c8, + 0x1a30: 0x00c8, 0x1a31: 0x00c8, 0x1a32: 0x00c8, 0x1a33: 0x00c8, 0x1a34: 0x00c8, 0x1a35: 0x00c8, + 0x1a36: 0x00c8, 0x1a37: 0x00c8, 0x1a38: 0x00c8, 0x1a39: 0x00c8, 0x1a3a: 0x00c8, 0x1a3b: 0x00c8, + 0x1a3c: 0x00c8, 0x1a3d: 0x00c8, 0x1a3e: 0x00c8, 0x1a3f: 0x00c8, + // Block 0x69, offset 0x1a40 + 0x1a40: 0x00c8, 0x1a41: 0x00c8, 0x1a42: 0x00c8, 0x1a43: 0x00c8, 0x1a44: 0x00c8, 0x1a45: 0x00c8, + 0x1a48: 0x00c8, 0x1a49: 0x00c8, 0x1a4a: 0x00c8, 0x1a4b: 0x00c8, + 0x1a4c: 0x00c8, 0x1a4d: 0x00c8, 0x1a50: 0x00c8, 0x1a51: 0x00c8, + 0x1a52: 0x00c8, 0x1a53: 0x00c8, 0x1a54: 0x00c8, 0x1a55: 0x00c8, 0x1a56: 0x00c8, 0x1a57: 0x00c8, + 0x1a59: 0x00c8, 0x1a5b: 0x00c8, 0x1a5d: 0x00c8, + 0x1a5f: 0x00c8, 0x1a60: 0x00c8, 0x1a61: 0x00c8, 0x1a62: 0x00c8, 0x1a63: 0x00c8, + 0x1a64: 0x00c8, 0x1a65: 0x00c8, 0x1a66: 0x00c8, 0x1a67: 0x00c8, 0x1a68: 0x00c8, 0x1a69: 0x00c8, + 0x1a6a: 0x00c8, 0x1a6b: 0x00c8, 0x1a6c: 0x00c8, 0x1a6d: 0x00c8, 0x1a6e: 0x00c8, 0x1a6f: 0x00c8, + 0x1a70: 0x00c8, 0x1a71: 0x0088, 0x1a72: 0x00c8, 0x1a73: 0x0088, 0x1a74: 0x00c8, 0x1a75: 0x0088, + 0x1a76: 0x00c8, 0x1a77: 0x0088, 0x1a78: 0x00c8, 0x1a79: 0x0088, 0x1a7a: 0x00c8, 0x1a7b: 0x0088, + 0x1a7c: 0x00c8, 0x1a7d: 0x0088, + // Block 0x6a, offset 0x1a80 + 0x1a80: 0x00c8, 0x1a81: 0x00c8, 0x1a82: 0x00c8, 0x1a83: 0x00c8, 0x1a84: 0x00c8, 0x1a85: 0x00c8, + 0x1a86: 0x00c8, 0x1a87: 0x00c8, 0x1a88: 0x0088, 0x1a89: 0x0088, 0x1a8a: 0x0088, 0x1a8b: 0x0088, + 0x1a8c: 0x0088, 0x1a8d: 0x0088, 0x1a8e: 0x0088, 0x1a8f: 0x0088, 0x1a90: 0x00c8, 0x1a91: 0x00c8, + 0x1a92: 0x00c8, 0x1a93: 0x00c8, 0x1a94: 0x00c8, 0x1a95: 0x00c8, 0x1a96: 0x00c8, 0x1a97: 0x00c8, + 0x1a98: 0x0088, 0x1a99: 0x0088, 0x1a9a: 0x0088, 0x1a9b: 0x0088, 0x1a9c: 0x0088, 0x1a9d: 0x0088, + 0x1a9e: 0x0088, 0x1a9f: 0x0088, 0x1aa0: 0x00c8, 0x1aa1: 0x00c8, 0x1aa2: 0x00c8, 0x1aa3: 0x00c8, + 0x1aa4: 0x00c8, 0x1aa5: 0x00c8, 0x1aa6: 0x00c8, 0x1aa7: 0x00c8, 0x1aa8: 0x0088, 0x1aa9: 0x0088, + 0x1aaa: 0x0088, 0x1aab: 0x0088, 0x1aac: 0x0088, 0x1aad: 0x0088, 0x1aae: 0x0088, 0x1aaf: 0x0088, + 0x1ab0: 0x00c8, 0x1ab1: 0x00c8, 0x1ab2: 0x00c8, 0x1ab3: 0x00c8, 0x1ab4: 0x00c8, + 0x1ab6: 0x00c8, 0x1ab7: 0x00c8, 0x1ab8: 0x00c8, 0x1ab9: 0x00c8, 0x1aba: 0x00c8, 0x1abb: 0x0088, + 0x1abc: 0x0088, 0x1abd: 0x0088, 0x1abe: 0x0088, 0x1abf: 0x0088, + // Block 0x6b, offset 0x1ac0 + 0x1ac0: 0x0088, 0x1ac1: 0x0088, 0x1ac2: 0x00c8, 0x1ac3: 0x00c8, 0x1ac4: 0x00c8, + 0x1ac6: 0x00c8, 0x1ac7: 0x00c8, 0x1ac8: 0x00c8, 0x1ac9: 0x0088, 0x1aca: 0x00c8, 0x1acb: 0x0088, + 0x1acc: 0x0088, 0x1acd: 0x0088, 0x1ace: 0x0088, 0x1acf: 0x0088, 0x1ad0: 0x00c8, 0x1ad1: 0x00c8, + 0x1ad2: 0x00c8, 0x1ad3: 0x0088, 0x1ad6: 0x00c8, 0x1ad7: 0x00c8, + 0x1ad8: 0x00c8, 0x1ad9: 0x00c8, 0x1ada: 0x00c8, 0x1adb: 0x0088, 0x1add: 0x0088, + 0x1ade: 0x0088, 0x1adf: 0x0088, 0x1ae0: 0x00c8, 0x1ae1: 0x00c8, 0x1ae2: 0x00c8, 0x1ae3: 0x0088, + 0x1ae4: 0x00c8, 0x1ae5: 0x00c8, 0x1ae6: 0x00c8, 0x1ae7: 0x00c8, 0x1ae8: 0x00c8, 0x1ae9: 0x00c8, + 0x1aea: 0x00c8, 0x1aeb: 0x0088, 0x1aec: 0x00c8, 0x1aed: 0x0088, 0x1aee: 0x0088, 0x1aef: 0x0088, + 0x1af2: 0x00c8, 0x1af3: 0x00c8, 0x1af4: 0x00c8, + 0x1af6: 0x00c8, 0x1af7: 0x00c8, 0x1af8: 0x00c8, 0x1af9: 0x0088, 0x1afa: 0x00c8, 0x1afb: 0x0088, + 0x1afc: 0x0088, 0x1afd: 0x0088, 0x1afe: 0x0088, + // Block 0x6c, offset 0x1b00 + 0x1b00: 0x0080, 0x1b01: 0x0080, 0x1b02: 0x0080, 0x1b03: 0x0080, 0x1b04: 0x0080, 0x1b05: 0x0080, + 0x1b06: 0x0080, 0x1b07: 0x0080, 0x1b08: 0x0080, 0x1b09: 0x0080, 0x1b0a: 0x0080, 0x1b0b: 0x0040, + 0x1b0c: 0x004d, 0x1b0d: 0x004e, 0x1b0e: 0x0040, 0x1b0f: 0x0040, 0x1b10: 0x0080, 0x1b11: 0x0080, + 0x1b12: 0x0080, 0x1b13: 0x0080, 0x1b14: 0x0080, 0x1b15: 0x0080, 0x1b16: 0x0080, 0x1b17: 0x0080, + 0x1b18: 0x0080, 0x1b19: 0x0080, 0x1b1a: 0x0080, 0x1b1b: 0x0080, 0x1b1c: 0x0080, 0x1b1d: 0x0080, + 0x1b1e: 0x0080, 0x1b1f: 0x0080, 0x1b20: 0x0080, 0x1b21: 0x0080, 0x1b22: 0x0080, 0x1b23: 0x0080, + 0x1b24: 0x0080, 0x1b25: 0x0080, 0x1b26: 0x0080, 0x1b27: 0x0080, 0x1b28: 0x0040, 0x1b29: 0x0040, + 0x1b2a: 0x0040, 0x1b2b: 0x0040, 0x1b2c: 0x0040, 0x1b2d: 0x0040, 0x1b2e: 0x0040, 0x1b2f: 0x0080, + 0x1b30: 0x0080, 0x1b31: 0x0080, 0x1b32: 0x0080, 0x1b33: 0x0080, 0x1b34: 0x0080, 0x1b35: 0x0080, + 0x1b36: 0x0080, 0x1b37: 0x0080, 0x1b38: 0x0080, 0x1b39: 0x0080, 0x1b3a: 0x0080, 0x1b3b: 0x0080, + 0x1b3c: 0x0080, 0x1b3d: 0x0080, 0x1b3e: 0x0080, 0x1b3f: 0x0080, + // Block 0x6d, offset 0x1b40 + 0x1b40: 0x0080, 0x1b41: 0x0080, 0x1b42: 0x0080, 0x1b43: 0x0080, 0x1b44: 0x0080, 0x1b45: 0x0080, + 0x1b46: 0x0080, 0x1b47: 0x0080, 0x1b48: 0x0080, 0x1b49: 0x0080, 0x1b4a: 0x0080, 0x1b4b: 0x0080, + 0x1b4c: 0x0080, 0x1b4d: 0x0080, 0x1b4e: 0x0080, 0x1b4f: 0x0080, 0x1b50: 0x0080, 0x1b51: 0x0080, + 0x1b52: 0x0080, 0x1b53: 0x0080, 0x1b54: 0x0080, 0x1b55: 0x0080, 0x1b56: 0x0080, 0x1b57: 0x0080, + 0x1b58: 0x0080, 0x1b59: 0x0080, 0x1b5a: 0x0080, 0x1b5b: 0x0080, 0x1b5c: 0x0080, 0x1b5d: 0x0080, + 0x1b5e: 0x0080, 0x1b5f: 0x0080, 0x1b60: 0x0040, 0x1b61: 0x0040, 0x1b62: 0x0040, 0x1b63: 0x0040, + 0x1b64: 0x0040, 0x1b66: 0x0040, 0x1b67: 0x0040, 0x1b68: 0x0040, 0x1b69: 0x0040, + 0x1b6a: 0x0040, 0x1b6b: 0x0040, 0x1b6c: 0x0040, 0x1b6d: 0x0040, 0x1b6e: 0x0040, 0x1b6f: 0x0040, + 0x1b70: 0x0080, 0x1b71: 0x0080, 0x1b74: 0x0080, 0x1b75: 0x0080, + 0x1b76: 0x0080, 0x1b77: 0x0080, 0x1b78: 0x0080, 0x1b79: 0x0080, 0x1b7a: 0x0080, 0x1b7b: 0x0080, + 0x1b7c: 0x0080, 0x1b7d: 0x0080, 0x1b7e: 0x0080, 0x1b7f: 0x0080, + // Block 0x6e, offset 0x1b80 + 0x1b80: 0x0080, 0x1b81: 0x0080, 0x1b82: 0x0080, 0x1b83: 0x0080, 0x1b84: 0x0080, 0x1b85: 0x0080, + 0x1b86: 0x0080, 0x1b87: 0x0080, 0x1b88: 0x0080, 0x1b89: 0x0080, 0x1b8a: 0x0080, 0x1b8b: 0x0080, + 0x1b8c: 0x0080, 0x1b8d: 0x0080, 0x1b8e: 0x0080, 0x1b90: 0x0080, 0x1b91: 0x0080, + 0x1b92: 0x0080, 0x1b93: 0x0080, 0x1b94: 0x0080, 0x1b95: 0x0080, 0x1b96: 0x0080, 0x1b97: 0x0080, + 0x1b98: 0x0080, 0x1b99: 0x0080, 0x1b9a: 0x0080, 0x1b9b: 0x0080, 0x1b9c: 0x0080, + 0x1ba0: 0x0080, 0x1ba1: 0x0080, 0x1ba2: 0x0080, 0x1ba3: 0x0080, + 0x1ba4: 0x0080, 0x1ba5: 0x0080, 0x1ba6: 0x0080, 0x1ba7: 0x0080, 0x1ba8: 0x0080, 0x1ba9: 0x0080, + 0x1baa: 0x0080, 0x1bab: 0x0080, 0x1bac: 0x0080, 0x1bad: 0x0080, 0x1bae: 0x0080, 0x1baf: 0x0080, + 0x1bb0: 0x0080, 0x1bb1: 0x0080, 0x1bb2: 0x0080, 0x1bb3: 0x0080, 0x1bb4: 0x0080, 0x1bb5: 0x0080, + 0x1bb6: 0x0080, 0x1bb7: 0x0080, 0x1bb8: 0x0080, 0x1bb9: 0x0080, 0x1bba: 0x0080, 0x1bbb: 0x0080, + 0x1bbc: 0x0080, 0x1bbd: 0x0080, 0x1bbe: 0x0080, 0x1bbf: 0x0080, + // Block 0x6f, offset 0x1bc0 + 0x1bd0: 0x00c3, 0x1bd1: 0x00c3, + 0x1bd2: 0x00c3, 0x1bd3: 0x00c3, 0x1bd4: 0x00c3, 0x1bd5: 0x00c3, 0x1bd6: 0x00c3, 0x1bd7: 0x00c3, + 0x1bd8: 0x00c3, 0x1bd9: 0x00c3, 0x1bda: 0x00c3, 0x1bdb: 0x00c3, 0x1bdc: 0x00c3, 0x1bdd: 0x0083, + 0x1bde: 0x0083, 0x1bdf: 0x0083, 0x1be0: 0x0083, 0x1be1: 0x00c3, 0x1be2: 0x0083, 0x1be3: 0x0083, + 0x1be4: 0x0083, 0x1be5: 0x00c3, 0x1be6: 0x00c3, 0x1be7: 0x00c3, 0x1be8: 0x00c3, 0x1be9: 0x00c3, + 0x1bea: 0x00c3, 0x1beb: 0x00c3, 0x1bec: 0x00c3, 0x1bed: 0x00c3, 0x1bee: 0x00c3, 0x1bef: 0x00c3, + 0x1bf0: 0x00c3, + // Block 0x70, offset 0x1c00 + 0x1c00: 0x0080, 0x1c01: 0x0080, 0x1c02: 0x0080, 0x1c03: 0x0080, 0x1c04: 0x0080, 0x1c05: 0x0080, + 0x1c06: 0x0080, 0x1c07: 0x0080, 0x1c08: 0x0080, 0x1c09: 0x0080, 0x1c0a: 0x0080, 0x1c0b: 0x0080, + 0x1c0c: 0x0080, 0x1c0d: 0x0080, 0x1c0e: 0x0080, 0x1c0f: 0x0080, 0x1c10: 0x0080, 0x1c11: 0x0080, + 0x1c12: 0x0080, 0x1c13: 0x0080, 0x1c14: 0x0080, 0x1c15: 0x0080, 0x1c16: 0x0080, 0x1c17: 0x0080, + 0x1c18: 0x0080, 0x1c19: 0x0080, 0x1c1a: 0x0080, 0x1c1b: 0x0080, 0x1c1c: 0x0080, 0x1c1d: 0x0080, + 0x1c1e: 0x0080, 0x1c1f: 0x0080, 0x1c20: 0x0080, 0x1c21: 0x0080, 0x1c22: 0x0080, 0x1c23: 0x0080, + 0x1c24: 0x0080, 0x1c25: 0x0080, 0x1c26: 0x0088, 0x1c27: 0x0080, 0x1c28: 0x0080, 0x1c29: 0x0080, + 0x1c2a: 0x0080, 0x1c2b: 0x0080, 0x1c2c: 0x0080, 0x1c2d: 0x0080, 0x1c2e: 0x0080, 0x1c2f: 0x0080, + 0x1c30: 0x0080, 0x1c31: 0x0080, 0x1c32: 0x00c0, 0x1c33: 0x0080, 0x1c34: 0x0080, 0x1c35: 0x0080, + 0x1c36: 0x0080, 0x1c37: 0x0080, 0x1c38: 0x0080, 0x1c39: 0x0080, 0x1c3a: 0x0080, 0x1c3b: 0x0080, + 0x1c3c: 0x0080, 0x1c3d: 0x0080, 0x1c3e: 0x0080, 0x1c3f: 0x0080, + // Block 0x71, offset 0x1c40 + 0x1c40: 0x0080, 0x1c41: 0x0080, 0x1c42: 0x0080, 0x1c43: 0x0080, 0x1c44: 0x0080, 0x1c45: 0x0080, + 0x1c46: 0x0080, 0x1c47: 0x0080, 0x1c48: 0x0080, 0x1c49: 0x0080, 0x1c4a: 0x0080, 0x1c4b: 0x0080, + 0x1c4c: 0x0080, 0x1c4d: 0x0080, 0x1c4e: 0x00c0, 0x1c4f: 0x0080, 0x1c50: 0x0080, 0x1c51: 0x0080, + 0x1c52: 0x0080, 0x1c53: 0x0080, 0x1c54: 0x0080, 0x1c55: 0x0080, 0x1c56: 0x0080, 0x1c57: 0x0080, + 0x1c58: 0x0080, 0x1c59: 0x0080, 0x1c5a: 0x0080, 0x1c5b: 0x0080, 0x1c5c: 0x0080, 0x1c5d: 0x0080, + 0x1c5e: 0x0080, 0x1c5f: 0x0080, 0x1c60: 0x0080, 0x1c61: 0x0080, 0x1c62: 0x0080, 0x1c63: 0x0080, + 0x1c64: 0x0080, 0x1c65: 0x0080, 0x1c66: 0x0080, 0x1c67: 0x0080, 0x1c68: 0x0080, 0x1c69: 0x0080, + 0x1c6a: 0x0080, 0x1c6b: 0x0080, 0x1c6c: 0x0080, 0x1c6d: 0x0080, 0x1c6e: 0x0080, 0x1c6f: 0x0080, + 0x1c70: 0x0080, 0x1c71: 0x0080, 0x1c72: 0x0080, 0x1c73: 0x0080, 0x1c74: 0x0080, 0x1c75: 0x0080, + 0x1c76: 0x0080, 0x1c77: 0x0080, 0x1c78: 0x0080, 0x1c79: 0x0080, 0x1c7a: 0x0080, 0x1c7b: 0x0080, + 0x1c7c: 0x0080, 0x1c7d: 0x0080, 0x1c7e: 0x0080, 0x1c7f: 0x0080, + // Block 0x72, offset 0x1c80 + 0x1c80: 0x0080, 0x1c81: 0x0080, 0x1c82: 0x0080, 0x1c83: 0x00c0, 0x1c84: 0x00c0, 0x1c85: 0x0080, + 0x1c86: 0x0080, 0x1c87: 0x0080, 0x1c88: 0x0080, 0x1c89: 0x0080, 0x1c8a: 0x0080, 0x1c8b: 0x0080, + 0x1c90: 0x0080, 0x1c91: 0x0080, + 0x1c92: 0x0080, 0x1c93: 0x0080, 0x1c94: 0x0080, 0x1c95: 0x0080, 0x1c96: 0x0080, 0x1c97: 0x0080, + 0x1c98: 0x0080, 0x1c99: 0x0080, 0x1c9a: 0x0080, 0x1c9b: 0x0080, 0x1c9c: 0x0080, 0x1c9d: 0x0080, + 0x1c9e: 0x0080, 0x1c9f: 0x0080, 0x1ca0: 0x0080, 0x1ca1: 0x0080, 0x1ca2: 0x0080, 0x1ca3: 0x0080, + 0x1ca4: 0x0080, 0x1ca5: 0x0080, 0x1ca6: 0x0080, 0x1ca7: 0x0080, 0x1ca8: 0x0080, 0x1ca9: 0x0080, + 0x1caa: 0x0080, 0x1cab: 0x0080, 0x1cac: 0x0080, 0x1cad: 0x0080, 0x1cae: 0x0080, 0x1caf: 0x0080, + 0x1cb0: 0x0080, 0x1cb1: 0x0080, 0x1cb2: 0x0080, 0x1cb3: 0x0080, 0x1cb4: 0x0080, 0x1cb5: 0x0080, + 0x1cb6: 0x0080, 0x1cb7: 0x0080, 0x1cb8: 0x0080, 0x1cb9: 0x0080, 0x1cba: 0x0080, 0x1cbb: 0x0080, + 0x1cbc: 0x0080, 0x1cbd: 0x0080, 0x1cbe: 0x0080, 0x1cbf: 0x0080, + // Block 0x73, offset 0x1cc0 + 0x1cc0: 0x0080, 0x1cc1: 0x0080, 0x1cc2: 0x0080, 0x1cc3: 0x0080, 0x1cc4: 0x0080, 0x1cc5: 0x0080, + 0x1cc6: 0x0080, 0x1cc7: 0x0080, 0x1cc8: 0x0080, 0x1cc9: 0x0080, 0x1cca: 0x0080, 0x1ccb: 0x0080, + 0x1ccc: 0x0080, 0x1ccd: 0x0080, 0x1cce: 0x0080, 0x1ccf: 0x0080, 0x1cd0: 0x0080, 0x1cd1: 0x0080, + 0x1cd2: 0x0080, 0x1cd3: 0x0080, 0x1cd4: 0x0080, 0x1cd5: 0x0080, 0x1cd6: 0x0080, 0x1cd7: 0x0080, + 0x1cd8: 0x0080, 0x1cd9: 0x0080, 0x1cda: 0x0080, 0x1cdb: 0x0080, 0x1cdc: 0x0080, 0x1cdd: 0x0080, + 0x1cde: 0x0080, 0x1cdf: 0x0080, 0x1ce0: 0x0080, 0x1ce1: 0x0080, 0x1ce2: 0x0080, 0x1ce3: 0x0080, + 0x1ce4: 0x0080, 0x1ce5: 0x0080, 0x1ce6: 0x0080, 0x1ce7: 0x0080, 0x1ce8: 0x0080, 0x1ce9: 0x0080, + 0x1cea: 0x0080, 0x1ceb: 0x0080, 0x1cec: 0x0080, 0x1ced: 0x0080, 0x1cee: 0x0080, 0x1cef: 0x0080, + 0x1cf0: 0x0080, 0x1cf1: 0x0080, 0x1cf2: 0x0080, 0x1cf3: 0x0080, 0x1cf4: 0x0080, 0x1cf5: 0x0080, + 0x1cf6: 0x0080, 0x1cf7: 0x0080, 0x1cf8: 0x0080, 0x1cf9: 0x0080, 0x1cfa: 0x0080, 0x1cfb: 0x0080, + 0x1cfc: 0x0080, 0x1cfd: 0x0080, 0x1cfe: 0x0080, 0x1cff: 0x0080, + // Block 0x74, offset 0x1d00 + 0x1d00: 0x0080, 0x1d01: 0x0080, 0x1d02: 0x0080, 0x1d03: 0x0080, 0x1d04: 0x0080, 0x1d05: 0x0080, + 0x1d06: 0x0080, 0x1d07: 0x0080, 0x1d08: 0x0080, 0x1d09: 0x0080, 0x1d0a: 0x0080, 0x1d0b: 0x0080, + 0x1d0c: 0x0080, 0x1d0d: 0x0080, 0x1d0e: 0x0080, 0x1d0f: 0x0080, 0x1d10: 0x0080, 0x1d11: 0x0080, + 0x1d12: 0x0080, 0x1d13: 0x0080, 0x1d14: 0x0080, 0x1d15: 0x0080, 0x1d16: 0x0080, 0x1d17: 0x0080, + 0x1d18: 0x0080, 0x1d19: 0x0080, 0x1d1a: 0x0080, 0x1d1b: 0x0080, 0x1d1c: 0x0080, 0x1d1d: 0x0080, + 0x1d1e: 0x0080, 0x1d1f: 0x0080, 0x1d20: 0x0080, 0x1d21: 0x0080, 0x1d22: 0x0080, 0x1d23: 0x0080, + 0x1d24: 0x0080, 0x1d25: 0x0080, 0x1d26: 0x0080, + // Block 0x75, offset 0x1d40 + 0x1d40: 0x0080, 0x1d41: 0x0080, 0x1d42: 0x0080, 0x1d43: 0x0080, 0x1d44: 0x0080, 0x1d45: 0x0080, + 0x1d46: 0x0080, 0x1d47: 0x0080, 0x1d48: 0x0080, 0x1d49: 0x0080, 0x1d4a: 0x0080, + 0x1d60: 0x0080, 0x1d61: 0x0080, 0x1d62: 0x0080, 0x1d63: 0x0080, + 0x1d64: 0x0080, 0x1d65: 0x0080, 0x1d66: 0x0080, 0x1d67: 0x0080, 0x1d68: 0x0080, 0x1d69: 0x0080, + 0x1d6a: 0x0080, 0x1d6b: 0x0080, 0x1d6c: 0x0080, 0x1d6d: 0x0080, 0x1d6e: 0x0080, 0x1d6f: 0x0080, + 0x1d70: 0x0080, 0x1d71: 0x0080, 0x1d72: 0x0080, 0x1d73: 0x0080, 0x1d74: 0x0080, 0x1d75: 0x0080, + 0x1d76: 0x0080, 0x1d77: 0x0080, 0x1d78: 0x0080, 0x1d79: 0x0080, 0x1d7a: 0x0080, 0x1d7b: 0x0080, + 0x1d7c: 0x0080, 0x1d7d: 0x0080, 0x1d7e: 0x0080, 0x1d7f: 0x0080, + // Block 0x76, offset 0x1d80 + 0x1d80: 0x0080, 0x1d81: 0x0080, 0x1d82: 0x0080, 0x1d83: 0x0080, 0x1d84: 0x0080, 0x1d85: 0x0080, + 0x1d86: 0x0080, 0x1d87: 0x0080, 0x1d88: 0x0080, 0x1d89: 0x0080, 0x1d8a: 0x0080, 0x1d8b: 0x0080, + 0x1d8c: 0x0080, 0x1d8d: 0x0080, 0x1d8e: 0x0080, 0x1d8f: 0x0080, 0x1d90: 0x0080, 0x1d91: 0x0080, + 0x1d92: 0x0080, 0x1d93: 0x0080, 0x1d94: 0x0080, 0x1d95: 0x0080, 0x1d96: 0x0080, 0x1d97: 0x0080, + 0x1d98: 0x0080, 0x1d99: 0x0080, 0x1d9a: 0x0080, 0x1d9b: 0x0080, 0x1d9c: 0x0080, 0x1d9d: 0x0080, + 0x1d9e: 0x0080, 0x1d9f: 0x0080, 0x1da0: 0x0080, 0x1da1: 0x0080, 0x1da2: 0x0080, 0x1da3: 0x0080, + 0x1da4: 0x0080, 0x1da5: 0x0080, 0x1da6: 0x0080, 0x1da7: 0x0080, 0x1da8: 0x0080, 0x1da9: 0x0080, + 0x1daa: 0x0080, 0x1dab: 0x0080, 0x1dac: 0x0080, 0x1dad: 0x0080, 0x1dae: 0x0080, 0x1daf: 0x0080, + 0x1db0: 0x0080, 0x1db1: 0x0080, 0x1db2: 0x0080, 0x1db3: 0x0080, + 0x1db6: 0x0080, 0x1db7: 0x0080, 0x1db8: 0x0080, 0x1db9: 0x0080, 0x1dba: 0x0080, 0x1dbb: 0x0080, + 0x1dbc: 0x0080, 0x1dbd: 0x0080, 0x1dbe: 0x0080, 0x1dbf: 0x0080, + // Block 0x77, offset 0x1dc0 + 0x1dc0: 0x0080, 0x1dc1: 0x0080, 0x1dc2: 0x0080, 0x1dc3: 0x0080, 0x1dc4: 0x0080, 0x1dc5: 0x0080, + 0x1dc6: 0x0080, 0x1dc7: 0x0080, 0x1dc8: 0x0080, 0x1dc9: 0x0080, 0x1dca: 0x0080, 0x1dcb: 0x0080, + 0x1dcc: 0x0080, 0x1dcd: 0x0080, 0x1dce: 0x0080, 0x1dcf: 0x0080, 0x1dd0: 0x0080, 0x1dd1: 0x0080, + 0x1dd2: 0x0080, 0x1dd3: 0x0080, 0x1dd4: 0x0080, 0x1dd5: 0x0080, + 0x1dd8: 0x0080, 0x1dd9: 0x0080, 0x1dda: 0x0080, 0x1ddb: 0x0080, 0x1ddc: 0x0080, 0x1ddd: 0x0080, + 0x1dde: 0x0080, 0x1ddf: 0x0080, 0x1de0: 0x0080, 0x1de1: 0x0080, 0x1de2: 0x0080, 0x1de3: 0x0080, + 0x1de4: 0x0080, 0x1de5: 0x0080, 0x1de6: 0x0080, 0x1de7: 0x0080, 0x1de8: 0x0080, 0x1de9: 0x0080, + 0x1dea: 0x0080, 0x1deb: 0x0080, 0x1dec: 0x0080, 0x1ded: 0x0080, 0x1dee: 0x0080, 0x1def: 0x0080, + 0x1df0: 0x0080, 0x1df1: 0x0080, 0x1df2: 0x0080, 0x1df3: 0x0080, 0x1df4: 0x0080, 0x1df5: 0x0080, + 0x1df6: 0x0080, 0x1df7: 0x0080, 0x1df8: 0x0080, 0x1df9: 0x0080, 0x1dfa: 0x0080, 0x1dfb: 0x0080, + 0x1dfc: 0x0080, 0x1dfd: 0x0080, 0x1dfe: 0x0080, 0x1dff: 0x0080, + // Block 0x78, offset 0x1e00 + 0x1e00: 0x00c0, 0x1e01: 0x00c0, 0x1e02: 0x00c0, 0x1e03: 0x00c0, 0x1e04: 0x00c0, 0x1e05: 0x00c0, + 0x1e06: 0x00c0, 0x1e07: 0x00c0, 0x1e08: 0x00c0, 0x1e09: 0x00c0, 0x1e0a: 0x00c0, 0x1e0b: 0x00c0, + 0x1e0c: 0x00c0, 0x1e0d: 0x00c0, 0x1e0e: 0x00c0, 0x1e0f: 0x00c0, 0x1e10: 0x00c0, 0x1e11: 0x00c0, + 0x1e12: 0x00c0, 0x1e13: 0x00c0, 0x1e14: 0x00c0, 0x1e15: 0x00c0, 0x1e16: 0x00c0, 0x1e17: 0x00c0, + 0x1e18: 0x00c0, 0x1e19: 0x00c0, 0x1e1a: 0x00c0, 0x1e1b: 0x00c0, 0x1e1c: 0x00c0, 0x1e1d: 0x00c0, + 0x1e1e: 0x00c0, 0x1e1f: 0x00c0, 0x1e20: 0x00c0, 0x1e21: 0x00c0, 0x1e22: 0x00c0, 0x1e23: 0x00c0, + 0x1e24: 0x00c0, 0x1e25: 0x00c0, 0x1e26: 0x00c0, 0x1e27: 0x00c0, 0x1e28: 0x00c0, 0x1e29: 0x00c0, + 0x1e2a: 0x00c0, 0x1e2b: 0x00c0, 0x1e2c: 0x00c0, 0x1e2d: 0x00c0, 0x1e2e: 0x00c0, + 0x1e30: 0x00c0, 0x1e31: 0x00c0, 0x1e32: 0x00c0, 0x1e33: 0x00c0, 0x1e34: 0x00c0, 0x1e35: 0x00c0, + 0x1e36: 0x00c0, 0x1e37: 0x00c0, 0x1e38: 0x00c0, 0x1e39: 0x00c0, 0x1e3a: 0x00c0, 0x1e3b: 0x00c0, + 0x1e3c: 0x00c0, 0x1e3d: 0x00c0, 0x1e3e: 0x00c0, 0x1e3f: 0x00c0, + // Block 0x79, offset 0x1e40 + 0x1e40: 0x00c0, 0x1e41: 0x00c0, 0x1e42: 0x00c0, 0x1e43: 0x00c0, 0x1e44: 0x00c0, 0x1e45: 0x00c0, + 0x1e46: 0x00c0, 0x1e47: 0x00c0, 0x1e48: 0x00c0, 0x1e49: 0x00c0, 0x1e4a: 0x00c0, 0x1e4b: 0x00c0, + 0x1e4c: 0x00c0, 0x1e4d: 0x00c0, 0x1e4e: 0x00c0, 0x1e4f: 0x00c0, 0x1e50: 0x00c0, 0x1e51: 0x00c0, + 0x1e52: 0x00c0, 0x1e53: 0x00c0, 0x1e54: 0x00c0, 0x1e55: 0x00c0, 0x1e56: 0x00c0, 0x1e57: 0x00c0, + 0x1e58: 0x00c0, 0x1e59: 0x00c0, 0x1e5a: 0x00c0, 0x1e5b: 0x00c0, 0x1e5c: 0x00c0, 0x1e5d: 0x00c0, + 0x1e5e: 0x00c0, 0x1e60: 0x00c0, 0x1e61: 0x00c0, 0x1e62: 0x00c0, 0x1e63: 0x00c0, + 0x1e64: 0x00c0, 0x1e65: 0x00c0, 0x1e66: 0x00c0, 0x1e67: 0x00c0, 0x1e68: 0x00c0, 0x1e69: 0x00c0, + 0x1e6a: 0x00c0, 0x1e6b: 0x00c0, 0x1e6c: 0x00c0, 0x1e6d: 0x00c0, 0x1e6e: 0x00c0, 0x1e6f: 0x00c0, + 0x1e70: 0x00c0, 0x1e71: 0x00c0, 0x1e72: 0x00c0, 0x1e73: 0x00c0, 0x1e74: 0x00c0, 0x1e75: 0x00c0, + 0x1e76: 0x00c0, 0x1e77: 0x00c0, 0x1e78: 0x00c0, 0x1e79: 0x00c0, 0x1e7a: 0x00c0, 0x1e7b: 0x00c0, + 0x1e7c: 0x0080, 0x1e7d: 0x0080, 0x1e7e: 0x00c0, 0x1e7f: 0x00c0, + // Block 0x7a, offset 0x1e80 + 0x1e80: 0x00c0, 0x1e81: 0x00c0, 0x1e82: 0x00c0, 0x1e83: 0x00c0, 0x1e84: 0x00c0, 0x1e85: 0x00c0, + 0x1e86: 0x00c0, 0x1e87: 0x00c0, 0x1e88: 0x00c0, 0x1e89: 0x00c0, 0x1e8a: 0x00c0, 0x1e8b: 0x00c0, + 0x1e8c: 0x00c0, 0x1e8d: 0x00c0, 0x1e8e: 0x00c0, 0x1e8f: 0x00c0, 0x1e90: 0x00c0, 0x1e91: 0x00c0, + 0x1e92: 0x00c0, 0x1e93: 0x00c0, 0x1e94: 0x00c0, 0x1e95: 0x00c0, 0x1e96: 0x00c0, 0x1e97: 0x00c0, + 0x1e98: 0x00c0, 0x1e99: 0x00c0, 0x1e9a: 0x00c0, 0x1e9b: 0x00c0, 0x1e9c: 0x00c0, 0x1e9d: 0x00c0, + 0x1e9e: 0x00c0, 0x1e9f: 0x00c0, 0x1ea0: 0x00c0, 0x1ea1: 0x00c0, 0x1ea2: 0x00c0, 0x1ea3: 0x00c0, + 0x1ea4: 0x00c0, 0x1ea5: 0x0080, 0x1ea6: 0x0080, 0x1ea7: 0x0080, 0x1ea8: 0x0080, 0x1ea9: 0x0080, + 0x1eaa: 0x0080, 0x1eab: 0x00c0, 0x1eac: 0x00c0, 0x1ead: 0x00c0, 0x1eae: 0x00c0, 0x1eaf: 0x00c3, + 0x1eb0: 0x00c3, 0x1eb1: 0x00c3, 0x1eb2: 0x00c0, 0x1eb3: 0x00c0, + 0x1eb9: 0x0080, 0x1eba: 0x0080, 0x1ebb: 0x0080, + 0x1ebc: 0x0080, 0x1ebd: 0x0080, 0x1ebe: 0x0080, 0x1ebf: 0x0080, + // Block 0x7b, offset 0x1ec0 + 0x1ec0: 0x00c0, 0x1ec1: 0x00c0, 0x1ec2: 0x00c0, 0x1ec3: 0x00c0, 0x1ec4: 0x00c0, 0x1ec5: 0x00c0, + 0x1ec6: 0x00c0, 0x1ec7: 0x00c0, 0x1ec8: 0x00c0, 0x1ec9: 0x00c0, 0x1eca: 0x00c0, 0x1ecb: 0x00c0, + 0x1ecc: 0x00c0, 0x1ecd: 0x00c0, 0x1ece: 0x00c0, 0x1ecf: 0x00c0, 0x1ed0: 0x00c0, 0x1ed1: 0x00c0, + 0x1ed2: 0x00c0, 0x1ed3: 0x00c0, 0x1ed4: 0x00c0, 0x1ed5: 0x00c0, 0x1ed6: 0x00c0, 0x1ed7: 0x00c0, + 0x1ed8: 0x00c0, 0x1ed9: 0x00c0, 0x1eda: 0x00c0, 0x1edb: 0x00c0, 0x1edc: 0x00c0, 0x1edd: 0x00c0, + 0x1ede: 0x00c0, 0x1edf: 0x00c0, 0x1ee0: 0x00c0, 0x1ee1: 0x00c0, 0x1ee2: 0x00c0, 0x1ee3: 0x00c0, + 0x1ee4: 0x00c0, 0x1ee5: 0x00c0, 0x1ee7: 0x00c0, + 0x1eed: 0x00c0, + 0x1ef0: 0x00c0, 0x1ef1: 0x00c0, 0x1ef2: 0x00c0, 0x1ef3: 0x00c0, 0x1ef4: 0x00c0, 0x1ef5: 0x00c0, + 0x1ef6: 0x00c0, 0x1ef7: 0x00c0, 0x1ef8: 0x00c0, 0x1ef9: 0x00c0, 0x1efa: 0x00c0, 0x1efb: 0x00c0, + 0x1efc: 0x00c0, 0x1efd: 0x00c0, 0x1efe: 0x00c0, 0x1eff: 0x00c0, + // Block 0x7c, offset 0x1f00 + 0x1f00: 0x00c0, 0x1f01: 0x00c0, 0x1f02: 0x00c0, 0x1f03: 0x00c0, 0x1f04: 0x00c0, 0x1f05: 0x00c0, + 0x1f06: 0x00c0, 0x1f07: 0x00c0, 0x1f08: 0x00c0, 0x1f09: 0x00c0, 0x1f0a: 0x00c0, 0x1f0b: 0x00c0, + 0x1f0c: 0x00c0, 0x1f0d: 0x00c0, 0x1f0e: 0x00c0, 0x1f0f: 0x00c0, 0x1f10: 0x00c0, 0x1f11: 0x00c0, + 0x1f12: 0x00c0, 0x1f13: 0x00c0, 0x1f14: 0x00c0, 0x1f15: 0x00c0, 0x1f16: 0x00c0, 0x1f17: 0x00c0, + 0x1f18: 0x00c0, 0x1f19: 0x00c0, 0x1f1a: 0x00c0, 0x1f1b: 0x00c0, 0x1f1c: 0x00c0, 0x1f1d: 0x00c0, + 0x1f1e: 0x00c0, 0x1f1f: 0x00c0, 0x1f20: 0x00c0, 0x1f21: 0x00c0, 0x1f22: 0x00c0, 0x1f23: 0x00c0, + 0x1f24: 0x00c0, 0x1f25: 0x00c0, 0x1f26: 0x00c0, 0x1f27: 0x00c0, + 0x1f2f: 0x0080, + 0x1f30: 0x0080, + 0x1f3f: 0x00c6, + // Block 0x7d, offset 0x1f40 + 0x1f40: 0x00c0, 0x1f41: 0x00c0, 0x1f42: 0x00c0, 0x1f43: 0x00c0, 0x1f44: 0x00c0, 0x1f45: 0x00c0, + 0x1f46: 0x00c0, 0x1f47: 0x00c0, 0x1f48: 0x00c0, 0x1f49: 0x00c0, 0x1f4a: 0x00c0, 0x1f4b: 0x00c0, + 0x1f4c: 0x00c0, 0x1f4d: 0x00c0, 0x1f4e: 0x00c0, 0x1f4f: 0x00c0, 0x1f50: 0x00c0, 0x1f51: 0x00c0, + 0x1f52: 0x00c0, 0x1f53: 0x00c0, 0x1f54: 0x00c0, 0x1f55: 0x00c0, 0x1f56: 0x00c0, + 0x1f60: 0x00c0, 0x1f61: 0x00c0, 0x1f62: 0x00c0, 0x1f63: 0x00c0, + 0x1f64: 0x00c0, 0x1f65: 0x00c0, 0x1f66: 0x00c0, 0x1f68: 0x00c0, 0x1f69: 0x00c0, + 0x1f6a: 0x00c0, 0x1f6b: 0x00c0, 0x1f6c: 0x00c0, 0x1f6d: 0x00c0, 0x1f6e: 0x00c0, + 0x1f70: 0x00c0, 0x1f71: 0x00c0, 0x1f72: 0x00c0, 0x1f73: 0x00c0, 0x1f74: 0x00c0, 0x1f75: 0x00c0, + 0x1f76: 0x00c0, 0x1f78: 0x00c0, 0x1f79: 0x00c0, 0x1f7a: 0x00c0, 0x1f7b: 0x00c0, + 0x1f7c: 0x00c0, 0x1f7d: 0x00c0, 0x1f7e: 0x00c0, + // Block 0x7e, offset 0x1f80 + 0x1f80: 0x00c0, 0x1f81: 0x00c0, 0x1f82: 0x00c0, 0x1f83: 0x00c0, 0x1f84: 0x00c0, 0x1f85: 0x00c0, + 0x1f86: 0x00c0, 0x1f88: 0x00c0, 0x1f89: 0x00c0, 0x1f8a: 0x00c0, 0x1f8b: 0x00c0, + 0x1f8c: 0x00c0, 0x1f8d: 0x00c0, 0x1f8e: 0x00c0, 0x1f90: 0x00c0, 0x1f91: 0x00c0, + 0x1f92: 0x00c0, 0x1f93: 0x00c0, 0x1f94: 0x00c0, 0x1f95: 0x00c0, 0x1f96: 0x00c0, + 0x1f98: 0x00c0, 0x1f99: 0x00c0, 0x1f9a: 0x00c0, 0x1f9b: 0x00c0, 0x1f9c: 0x00c0, 0x1f9d: 0x00c0, + 0x1f9e: 0x00c0, 0x1fa0: 0x00c3, 0x1fa1: 0x00c3, 0x1fa2: 0x00c3, 0x1fa3: 0x00c3, + 0x1fa4: 0x00c3, 0x1fa5: 0x00c3, 0x1fa6: 0x00c3, 0x1fa7: 0x00c3, 0x1fa8: 0x00c3, 0x1fa9: 0x00c3, + 0x1faa: 0x00c3, 0x1fab: 0x00c3, 0x1fac: 0x00c3, 0x1fad: 0x00c3, 0x1fae: 0x00c3, 0x1faf: 0x00c3, + 0x1fb0: 0x00c3, 0x1fb1: 0x00c3, 0x1fb2: 0x00c3, 0x1fb3: 0x00c3, 0x1fb4: 0x00c3, 0x1fb5: 0x00c3, + 0x1fb6: 0x00c3, 0x1fb7: 0x00c3, 0x1fb8: 0x00c3, 0x1fb9: 0x00c3, 0x1fba: 0x00c3, 0x1fbb: 0x00c3, + 0x1fbc: 0x00c3, 0x1fbd: 0x00c3, 0x1fbe: 0x00c3, 0x1fbf: 0x00c3, + // Block 0x7f, offset 0x1fc0 + 0x1fc0: 0x0080, 0x1fc1: 0x0080, 0x1fc2: 0x0080, 0x1fc3: 0x0080, 0x1fc4: 0x0080, 0x1fc5: 0x0080, + 0x1fc6: 0x0080, 0x1fc7: 0x0080, 0x1fc8: 0x0080, 0x1fc9: 0x0080, 0x1fca: 0x0080, 0x1fcb: 0x0080, + 0x1fcc: 0x0080, 0x1fcd: 0x0080, 0x1fce: 0x0080, 0x1fcf: 0x0080, 0x1fd0: 0x0080, 0x1fd1: 0x0080, + 0x1fd2: 0x0080, 0x1fd3: 0x0080, 0x1fd4: 0x0080, 0x1fd5: 0x0080, 0x1fd6: 0x0080, 0x1fd7: 0x0080, + 0x1fd8: 0x0080, 0x1fd9: 0x0080, 0x1fda: 0x0080, 0x1fdb: 0x0080, 0x1fdc: 0x0080, 0x1fdd: 0x0080, + 0x1fde: 0x0080, 0x1fdf: 0x0080, 0x1fe0: 0x0080, 0x1fe1: 0x0080, 0x1fe2: 0x0080, 0x1fe3: 0x0080, + 0x1fe4: 0x0080, 0x1fe5: 0x0080, 0x1fe6: 0x0080, 0x1fe7: 0x0080, 0x1fe8: 0x0080, 0x1fe9: 0x0080, + 0x1fea: 0x0080, 0x1feb: 0x0080, 0x1fec: 0x0080, 0x1fed: 0x0080, 0x1fee: 0x0080, 0x1fef: 0x00c0, + 0x1ff0: 0x0080, 0x1ff1: 0x0080, 0x1ff2: 0x0080, 0x1ff3: 0x0080, 0x1ff4: 0x0080, 0x1ff5: 0x0080, + 0x1ff6: 0x0080, 0x1ff7: 0x0080, 0x1ff8: 0x0080, 0x1ff9: 0x0080, 0x1ffa: 0x0080, 0x1ffb: 0x0080, + 0x1ffc: 0x0080, 0x1ffd: 0x0080, 0x1ffe: 0x0080, 0x1fff: 0x0080, + // Block 0x80, offset 0x2000 + 0x2000: 0x0080, 0x2001: 0x0080, 0x2002: 0x0080, 0x2003: 0x0080, 0x2004: 0x0080, 0x2005: 0x0080, + 0x2006: 0x0080, 0x2007: 0x0080, 0x2008: 0x0080, 0x2009: 0x0080, 0x200a: 0x0080, 0x200b: 0x0080, + 0x200c: 0x0080, 0x200d: 0x0080, 0x200e: 0x0080, 0x200f: 0x0080, + // Block 0x81, offset 0x2040 + 0x2040: 0x008c, 0x2041: 0x008c, 0x2042: 0x008c, 0x2043: 0x008c, 0x2044: 0x008c, 0x2045: 0x008c, + 0x2046: 0x008c, 0x2047: 0x008c, 0x2048: 0x008c, 0x2049: 0x008c, 0x204a: 0x008c, 0x204b: 0x008c, + 0x204c: 0x008c, 0x204d: 0x008c, 0x204e: 0x008c, 0x204f: 0x008c, 0x2050: 0x008c, 0x2051: 0x008c, + 0x2052: 0x008c, 0x2053: 0x008c, 0x2054: 0x008c, 0x2055: 0x008c, 0x2056: 0x008c, 0x2057: 0x008c, + 0x2058: 0x008c, 0x2059: 0x008c, 0x205b: 0x008c, 0x205c: 0x008c, 0x205d: 0x008c, + 0x205e: 0x008c, 0x205f: 0x008c, 0x2060: 0x008c, 0x2061: 0x008c, 0x2062: 0x008c, 0x2063: 0x008c, + 0x2064: 0x008c, 0x2065: 0x008c, 0x2066: 0x008c, 0x2067: 0x008c, 0x2068: 0x008c, 0x2069: 0x008c, + 0x206a: 0x008c, 0x206b: 0x008c, 0x206c: 0x008c, 0x206d: 0x008c, 0x206e: 0x008c, 0x206f: 0x008c, + 0x2070: 0x008c, 0x2071: 0x008c, 0x2072: 0x008c, 0x2073: 0x008c, 0x2074: 0x008c, 0x2075: 0x008c, + 0x2076: 0x008c, 0x2077: 0x008c, 0x2078: 0x008c, 0x2079: 0x008c, 0x207a: 0x008c, 0x207b: 0x008c, + 0x207c: 0x008c, 0x207d: 0x008c, 0x207e: 0x008c, 0x207f: 0x008c, + // Block 0x82, offset 0x2080 + 0x2080: 0x008c, 0x2081: 0x008c, 0x2082: 0x008c, 0x2083: 0x008c, 0x2084: 0x008c, 0x2085: 0x008c, + 0x2086: 0x008c, 0x2087: 0x008c, 0x2088: 0x008c, 0x2089: 0x008c, 0x208a: 0x008c, 0x208b: 0x008c, + 0x208c: 0x008c, 0x208d: 0x008c, 0x208e: 0x008c, 0x208f: 0x008c, 0x2090: 0x008c, 0x2091: 0x008c, + 0x2092: 0x008c, 0x2093: 0x008c, 0x2094: 0x008c, 0x2095: 0x008c, 0x2096: 0x008c, 0x2097: 0x008c, + 0x2098: 0x008c, 0x2099: 0x008c, 0x209a: 0x008c, 0x209b: 0x008c, 0x209c: 0x008c, 0x209d: 0x008c, + 0x209e: 0x008c, 0x209f: 0x008c, 0x20a0: 0x008c, 0x20a1: 0x008c, 0x20a2: 0x008c, 0x20a3: 0x008c, + 0x20a4: 0x008c, 0x20a5: 0x008c, 0x20a6: 0x008c, 0x20a7: 0x008c, 0x20a8: 0x008c, 0x20a9: 0x008c, + 0x20aa: 0x008c, 0x20ab: 0x008c, 0x20ac: 0x008c, 0x20ad: 0x008c, 0x20ae: 0x008c, 0x20af: 0x008c, + 0x20b0: 0x008c, 0x20b1: 0x008c, 0x20b2: 0x008c, 0x20b3: 0x008c, + // Block 0x83, offset 0x20c0 + 0x20c0: 0x008c, 0x20c1: 0x008c, 0x20c2: 0x008c, 0x20c3: 0x008c, 0x20c4: 0x008c, 0x20c5: 0x008c, + 0x20c6: 0x008c, 0x20c7: 0x008c, 0x20c8: 0x008c, 0x20c9: 0x008c, 0x20ca: 0x008c, 0x20cb: 0x008c, + 0x20cc: 0x008c, 0x20cd: 0x008c, 0x20ce: 0x008c, 0x20cf: 0x008c, 0x20d0: 0x008c, 0x20d1: 0x008c, + 0x20d2: 0x008c, 0x20d3: 0x008c, 0x20d4: 0x008c, 0x20d5: 0x008c, 0x20d6: 0x008c, 0x20d7: 0x008c, + 0x20d8: 0x008c, 0x20d9: 0x008c, 0x20da: 0x008c, 0x20db: 0x008c, 0x20dc: 0x008c, 0x20dd: 0x008c, + 0x20de: 0x008c, 0x20df: 0x008c, 0x20e0: 0x008c, 0x20e1: 0x008c, 0x20e2: 0x008c, 0x20e3: 0x008c, + 0x20e4: 0x008c, 0x20e5: 0x008c, 0x20e6: 0x008c, 0x20e7: 0x008c, 0x20e8: 0x008c, 0x20e9: 0x008c, + 0x20ea: 0x008c, 0x20eb: 0x008c, 0x20ec: 0x008c, 0x20ed: 0x008c, 0x20ee: 0x008c, 0x20ef: 0x008c, + 0x20f0: 0x008c, 0x20f1: 0x008c, 0x20f2: 0x008c, 0x20f3: 0x008c, 0x20f4: 0x008c, 0x20f5: 0x008c, + 0x20f6: 0x008c, 0x20f7: 0x008c, 0x20f8: 0x008c, 0x20f9: 0x008c, 0x20fa: 0x008c, 0x20fb: 0x008c, + 0x20fc: 0x008c, 0x20fd: 0x008c, 0x20fe: 0x008c, 0x20ff: 0x008c, + // Block 0x84, offset 0x2100 + 0x2100: 0x008c, 0x2101: 0x008c, 0x2102: 0x008c, 0x2103: 0x008c, 0x2104: 0x008c, 0x2105: 0x008c, + 0x2106: 0x008c, 0x2107: 0x008c, 0x2108: 0x008c, 0x2109: 0x008c, 0x210a: 0x008c, 0x210b: 0x008c, + 0x210c: 0x008c, 0x210d: 0x008c, 0x210e: 0x008c, 0x210f: 0x008c, 0x2110: 0x008c, 0x2111: 0x008c, + 0x2112: 0x008c, 0x2113: 0x008c, 0x2114: 0x008c, 0x2115: 0x008c, + 0x2130: 0x0080, 0x2131: 0x0080, 0x2132: 0x0080, 0x2133: 0x0080, 0x2134: 0x0080, 0x2135: 0x0080, + 0x2136: 0x0080, 0x2137: 0x0080, 0x2138: 0x0080, 0x2139: 0x0080, 0x213a: 0x0080, 0x213b: 0x0080, + // Block 0x85, offset 0x2140 + 0x2140: 0x0080, 0x2141: 0x0080, 0x2142: 0x0080, 0x2143: 0x0080, 0x2144: 0x0080, 0x2145: 0x00cc, + 0x2146: 0x00c0, 0x2147: 0x00cc, 0x2148: 0x0080, 0x2149: 0x0080, 0x214a: 0x0080, 0x214b: 0x0080, + 0x214c: 0x0080, 0x214d: 0x0080, 0x214e: 0x0080, 0x214f: 0x0080, 0x2150: 0x0080, 0x2151: 0x0080, + 0x2152: 0x0080, 0x2153: 0x0080, 0x2154: 0x0080, 0x2155: 0x0080, 0x2156: 0x0080, 0x2157: 0x0080, + 0x2158: 0x0080, 0x2159: 0x0080, 0x215a: 0x0080, 0x215b: 0x0080, 0x215c: 0x0080, 0x215d: 0x0080, + 0x215e: 0x0080, 0x215f: 0x0080, 0x2160: 0x0080, 0x2161: 0x008c, 0x2162: 0x008c, 0x2163: 0x008c, + 0x2164: 0x008c, 0x2165: 0x008c, 0x2166: 0x008c, 0x2167: 0x008c, 0x2168: 0x008c, 0x2169: 0x008c, + 0x216a: 0x00c3, 0x216b: 0x00c3, 0x216c: 0x00c3, 0x216d: 0x00c3, 0x216e: 0x0040, 0x216f: 0x0040, + 0x2170: 0x0080, 0x2171: 0x0040, 0x2172: 0x0040, 0x2173: 0x0040, 0x2174: 0x0040, 0x2175: 0x0040, + 0x2176: 0x0080, 0x2177: 0x0080, 0x2178: 0x008c, 0x2179: 0x008c, 0x217a: 0x008c, 0x217b: 0x0040, + 0x217c: 0x00c0, 0x217d: 0x0080, 0x217e: 0x0080, 0x217f: 0x0080, + // Block 0x86, offset 0x2180 + 0x2181: 0x00cc, 0x2182: 0x00cc, 0x2183: 0x00cc, 0x2184: 0x00cc, 0x2185: 0x00cc, + 0x2186: 0x00cc, 0x2187: 0x00cc, 0x2188: 0x00cc, 0x2189: 0x00cc, 0x218a: 0x00cc, 0x218b: 0x00cc, + 0x218c: 0x00cc, 0x218d: 0x00cc, 0x218e: 0x00cc, 0x218f: 0x00cc, 0x2190: 0x00cc, 0x2191: 0x00cc, + 0x2192: 0x00cc, 0x2193: 0x00cc, 0x2194: 0x00cc, 0x2195: 0x00cc, 0x2196: 0x00cc, 0x2197: 0x00cc, + 0x2198: 0x00cc, 0x2199: 0x00cc, 0x219a: 0x00cc, 0x219b: 0x00cc, 0x219c: 0x00cc, 0x219d: 0x00cc, + 0x219e: 0x00cc, 0x219f: 0x00cc, 0x21a0: 0x00cc, 0x21a1: 0x00cc, 0x21a2: 0x00cc, 0x21a3: 0x00cc, + 0x21a4: 0x00cc, 0x21a5: 0x00cc, 0x21a6: 0x00cc, 0x21a7: 0x00cc, 0x21a8: 0x00cc, 0x21a9: 0x00cc, + 0x21aa: 0x00cc, 0x21ab: 0x00cc, 0x21ac: 0x00cc, 0x21ad: 0x00cc, 0x21ae: 0x00cc, 0x21af: 0x00cc, + 0x21b0: 0x00cc, 0x21b1: 0x00cc, 0x21b2: 0x00cc, 0x21b3: 0x00cc, 0x21b4: 0x00cc, 0x21b5: 0x00cc, + 0x21b6: 0x00cc, 0x21b7: 0x00cc, 0x21b8: 0x00cc, 0x21b9: 0x00cc, 0x21ba: 0x00cc, 0x21bb: 0x00cc, + 0x21bc: 0x00cc, 0x21bd: 0x00cc, 0x21be: 0x00cc, 0x21bf: 0x00cc, + // Block 0x87, offset 0x21c0 + 0x21c0: 0x00cc, 0x21c1: 0x00cc, 0x21c2: 0x00cc, 0x21c3: 0x00cc, 0x21c4: 0x00cc, 0x21c5: 0x00cc, + 0x21c6: 0x00cc, 0x21c7: 0x00cc, 0x21c8: 0x00cc, 0x21c9: 0x00cc, 0x21ca: 0x00cc, 0x21cb: 0x00cc, + 0x21cc: 0x00cc, 0x21cd: 0x00cc, 0x21ce: 0x00cc, 0x21cf: 0x00cc, 0x21d0: 0x00cc, 0x21d1: 0x00cc, + 0x21d2: 0x00cc, 0x21d3: 0x00cc, 0x21d4: 0x00cc, 0x21d5: 0x00cc, 0x21d6: 0x00cc, + 0x21d9: 0x00c3, 0x21da: 0x00c3, 0x21db: 0x0080, 0x21dc: 0x0080, 0x21dd: 0x00cc, + 0x21de: 0x00cc, 0x21df: 0x008c, 0x21e0: 0x0080, 0x21e1: 0x00cc, 0x21e2: 0x00cc, 0x21e3: 0x00cc, + 0x21e4: 0x00cc, 0x21e5: 0x00cc, 0x21e6: 0x00cc, 0x21e7: 0x00cc, 0x21e8: 0x00cc, 0x21e9: 0x00cc, + 0x21ea: 0x00cc, 0x21eb: 0x00cc, 0x21ec: 0x00cc, 0x21ed: 0x00cc, 0x21ee: 0x00cc, 0x21ef: 0x00cc, + 0x21f0: 0x00cc, 0x21f1: 0x00cc, 0x21f2: 0x00cc, 0x21f3: 0x00cc, 0x21f4: 0x00cc, 0x21f5: 0x00cc, + 0x21f6: 0x00cc, 0x21f7: 0x00cc, 0x21f8: 0x00cc, 0x21f9: 0x00cc, 0x21fa: 0x00cc, 0x21fb: 0x00cc, + 0x21fc: 0x00cc, 0x21fd: 0x00cc, 0x21fe: 0x00cc, 0x21ff: 0x00cc, + // Block 0x88, offset 0x2200 + 0x2200: 0x00cc, 0x2201: 0x00cc, 0x2202: 0x00cc, 0x2203: 0x00cc, 0x2204: 0x00cc, 0x2205: 0x00cc, + 0x2206: 0x00cc, 0x2207: 0x00cc, 0x2208: 0x00cc, 0x2209: 0x00cc, 0x220a: 0x00cc, 0x220b: 0x00cc, + 0x220c: 0x00cc, 0x220d: 0x00cc, 0x220e: 0x00cc, 0x220f: 0x00cc, 0x2210: 0x00cc, 0x2211: 0x00cc, + 0x2212: 0x00cc, 0x2213: 0x00cc, 0x2214: 0x00cc, 0x2215: 0x00cc, 0x2216: 0x00cc, 0x2217: 0x00cc, + 0x2218: 0x00cc, 0x2219: 0x00cc, 0x221a: 0x00cc, 0x221b: 0x00cc, 0x221c: 0x00cc, 0x221d: 0x00cc, + 0x221e: 0x00cc, 0x221f: 0x00cc, 0x2220: 0x00cc, 0x2221: 0x00cc, 0x2222: 0x00cc, 0x2223: 0x00cc, + 0x2224: 0x00cc, 0x2225: 0x00cc, 0x2226: 0x00cc, 0x2227: 0x00cc, 0x2228: 0x00cc, 0x2229: 0x00cc, + 0x222a: 0x00cc, 0x222b: 0x00cc, 0x222c: 0x00cc, 0x222d: 0x00cc, 0x222e: 0x00cc, 0x222f: 0x00cc, + 0x2230: 0x00cc, 0x2231: 0x00cc, 0x2232: 0x00cc, 0x2233: 0x00cc, 0x2234: 0x00cc, 0x2235: 0x00cc, + 0x2236: 0x00cc, 0x2237: 0x00cc, 0x2238: 0x00cc, 0x2239: 0x00cc, 0x223a: 0x00cc, 0x223b: 0x00d2, + 0x223c: 0x00c0, 0x223d: 0x00cc, 0x223e: 0x00cc, 0x223f: 0x008c, + // Block 0x89, offset 0x2240 + 0x2245: 0x00c0, + 0x2246: 0x00c0, 0x2247: 0x00c0, 0x2248: 0x00c0, 0x2249: 0x00c0, 0x224a: 0x00c0, 0x224b: 0x00c0, + 0x224c: 0x00c0, 0x224d: 0x00c0, 0x224e: 0x00c0, 0x224f: 0x00c0, 0x2250: 0x00c0, 0x2251: 0x00c0, + 0x2252: 0x00c0, 0x2253: 0x00c0, 0x2254: 0x00c0, 0x2255: 0x00c0, 0x2256: 0x00c0, 0x2257: 0x00c0, + 0x2258: 0x00c0, 0x2259: 0x00c0, 0x225a: 0x00c0, 0x225b: 0x00c0, 0x225c: 0x00c0, 0x225d: 0x00c0, + 0x225e: 0x00c0, 0x225f: 0x00c0, 0x2260: 0x00c0, 0x2261: 0x00c0, 0x2262: 0x00c0, 0x2263: 0x00c0, + 0x2264: 0x00c0, 0x2265: 0x00c0, 0x2266: 0x00c0, 0x2267: 0x00c0, 0x2268: 0x00c0, 0x2269: 0x00c0, + 0x226a: 0x00c0, 0x226b: 0x00c0, 0x226c: 0x00c0, 0x226d: 0x00c0, 0x226e: 0x00c0, 0x226f: 0x00c0, + 0x2271: 0x0080, 0x2272: 0x0080, 0x2273: 0x0080, 0x2274: 0x0080, 0x2275: 0x0080, + 0x2276: 0x0080, 0x2277: 0x0080, 0x2278: 0x0080, 0x2279: 0x0080, 0x227a: 0x0080, 0x227b: 0x0080, + 0x227c: 0x0080, 0x227d: 0x0080, 0x227e: 0x0080, 0x227f: 0x0080, + // Block 0x8a, offset 0x2280 + 0x2280: 0x0080, 0x2281: 0x0080, 0x2282: 0x0080, 0x2283: 0x0080, 0x2284: 0x0080, 0x2285: 0x0080, + 0x2286: 0x0080, 0x2287: 0x0080, 0x2288: 0x0080, 0x2289: 0x0080, 0x228a: 0x0080, 0x228b: 0x0080, + 0x228c: 0x0080, 0x228d: 0x0080, 0x228e: 0x0080, 0x228f: 0x0080, 0x2290: 0x0080, 0x2291: 0x0080, + 0x2292: 0x0080, 0x2293: 0x0080, 0x2294: 0x0080, 0x2295: 0x0080, 0x2296: 0x0080, 0x2297: 0x0080, + 0x2298: 0x0080, 0x2299: 0x0080, 0x229a: 0x0080, 0x229b: 0x0080, 0x229c: 0x0080, 0x229d: 0x0080, + 0x229e: 0x0080, 0x229f: 0x0080, 0x22a0: 0x0080, 0x22a1: 0x0080, 0x22a2: 0x0080, 0x22a3: 0x0080, + 0x22a4: 0x0040, 0x22a5: 0x0080, 0x22a6: 0x0080, 0x22a7: 0x0080, 0x22a8: 0x0080, 0x22a9: 0x0080, + 0x22aa: 0x0080, 0x22ab: 0x0080, 0x22ac: 0x0080, 0x22ad: 0x0080, 0x22ae: 0x0080, 0x22af: 0x0080, + 0x22b0: 0x0080, 0x22b1: 0x0080, 0x22b2: 0x0080, 0x22b3: 0x0080, 0x22b4: 0x0080, 0x22b5: 0x0080, + 0x22b6: 0x0080, 0x22b7: 0x0080, 0x22b8: 0x0080, 0x22b9: 0x0080, 0x22ba: 0x0080, 0x22bb: 0x0080, + 0x22bc: 0x0080, 0x22bd: 0x0080, 0x22be: 0x0080, 0x22bf: 0x0080, + // Block 0x8b, offset 0x22c0 + 0x22c0: 0x0080, 0x22c1: 0x0080, 0x22c2: 0x0080, 0x22c3: 0x0080, 0x22c4: 0x0080, 0x22c5: 0x0080, + 0x22c6: 0x0080, 0x22c7: 0x0080, 0x22c8: 0x0080, 0x22c9: 0x0080, 0x22ca: 0x0080, 0x22cb: 0x0080, + 0x22cc: 0x0080, 0x22cd: 0x0080, 0x22ce: 0x0080, 0x22d0: 0x0080, 0x22d1: 0x0080, + 0x22d2: 0x0080, 0x22d3: 0x0080, 0x22d4: 0x0080, 0x22d5: 0x0080, 0x22d6: 0x0080, 0x22d7: 0x0080, + 0x22d8: 0x0080, 0x22d9: 0x0080, 0x22da: 0x0080, 0x22db: 0x0080, 0x22dc: 0x0080, 0x22dd: 0x0080, + 0x22de: 0x0080, 0x22df: 0x0080, 0x22e0: 0x00c0, 0x22e1: 0x00c0, 0x22e2: 0x00c0, 0x22e3: 0x00c0, + 0x22e4: 0x00c0, 0x22e5: 0x00c0, 0x22e6: 0x00c0, 0x22e7: 0x00c0, 0x22e8: 0x00c0, 0x22e9: 0x00c0, + 0x22ea: 0x00c0, 0x22eb: 0x00c0, 0x22ec: 0x00c0, 0x22ed: 0x00c0, 0x22ee: 0x00c0, 0x22ef: 0x00c0, + 0x22f0: 0x00c0, 0x22f1: 0x00c0, 0x22f2: 0x00c0, 0x22f3: 0x00c0, 0x22f4: 0x00c0, 0x22f5: 0x00c0, + 0x22f6: 0x00c0, 0x22f7: 0x00c0, 0x22f8: 0x00c0, 0x22f9: 0x00c0, 0x22fa: 0x00c0, + // Block 0x8c, offset 0x2300 + 0x2300: 0x0080, 0x2301: 0x0080, 0x2302: 0x0080, 0x2303: 0x0080, 0x2304: 0x0080, 0x2305: 0x0080, + 0x2306: 0x0080, 0x2307: 0x0080, 0x2308: 0x0080, 0x2309: 0x0080, 0x230a: 0x0080, 0x230b: 0x0080, + 0x230c: 0x0080, 0x230d: 0x0080, 0x230e: 0x0080, 0x230f: 0x0080, 0x2310: 0x0080, 0x2311: 0x0080, + 0x2312: 0x0080, 0x2313: 0x0080, 0x2314: 0x0080, 0x2315: 0x0080, 0x2316: 0x0080, 0x2317: 0x0080, + 0x2318: 0x0080, 0x2319: 0x0080, 0x231a: 0x0080, 0x231b: 0x0080, 0x231c: 0x0080, 0x231d: 0x0080, + 0x231e: 0x0080, 0x231f: 0x0080, 0x2320: 0x0080, 0x2321: 0x0080, 0x2322: 0x0080, 0x2323: 0x0080, + 0x2330: 0x00cc, 0x2331: 0x00cc, 0x2332: 0x00cc, 0x2333: 0x00cc, 0x2334: 0x00cc, 0x2335: 0x00cc, + 0x2336: 0x00cc, 0x2337: 0x00cc, 0x2338: 0x00cc, 0x2339: 0x00cc, 0x233a: 0x00cc, 0x233b: 0x00cc, + 0x233c: 0x00cc, 0x233d: 0x00cc, 0x233e: 0x00cc, 0x233f: 0x00cc, + // Block 0x8d, offset 0x2340 + 0x2340: 0x0080, 0x2341: 0x0080, 0x2342: 0x0080, 0x2343: 0x0080, 0x2344: 0x0080, 0x2345: 0x0080, + 0x2346: 0x0080, 0x2347: 0x0080, 0x2348: 0x0080, 0x2349: 0x0080, 0x234a: 0x0080, 0x234b: 0x0080, + 0x234c: 0x0080, 0x234d: 0x0080, 0x234e: 0x0080, 0x234f: 0x0080, 0x2350: 0x0080, 0x2351: 0x0080, + 0x2352: 0x0080, 0x2353: 0x0080, 0x2354: 0x0080, 0x2355: 0x0080, 0x2356: 0x0080, 0x2357: 0x0080, + 0x2358: 0x0080, 0x2359: 0x0080, 0x235a: 0x0080, 0x235b: 0x0080, 0x235c: 0x0080, 0x235d: 0x0080, + 0x235e: 0x0080, 0x2360: 0x0080, 0x2361: 0x0080, 0x2362: 0x0080, 0x2363: 0x0080, + 0x2364: 0x0080, 0x2365: 0x0080, 0x2366: 0x0080, 0x2367: 0x0080, 0x2368: 0x0080, 0x2369: 0x0080, + 0x236a: 0x0080, 0x236b: 0x0080, 0x236c: 0x0080, 0x236d: 0x0080, 0x236e: 0x0080, 0x236f: 0x0080, + 0x2370: 0x0080, 0x2371: 0x0080, 0x2372: 0x0080, 0x2373: 0x0080, 0x2374: 0x0080, 0x2375: 0x0080, + 0x2376: 0x0080, 0x2377: 0x0080, 0x2378: 0x0080, 0x2379: 0x0080, 0x237a: 0x0080, 0x237b: 0x0080, + 0x237c: 0x0080, 0x237d: 0x0080, 0x237e: 0x0080, 0x237f: 0x0080, + // Block 0x8e, offset 0x2380 + 0x2380: 0x0080, 0x2381: 0x0080, 0x2382: 0x0080, 0x2383: 0x0080, 0x2384: 0x0080, 0x2385: 0x0080, + 0x2386: 0x0080, 0x2387: 0x0080, 0x2388: 0x0080, 0x2389: 0x0080, 0x238a: 0x0080, 0x238b: 0x0080, + 0x238c: 0x0080, 0x238d: 0x0080, 0x238e: 0x0080, 0x238f: 0x0080, 0x2390: 0x008c, 0x2391: 0x008c, + 0x2392: 0x008c, 0x2393: 0x008c, 0x2394: 0x008c, 0x2395: 0x008c, 0x2396: 0x008c, 0x2397: 0x008c, + 0x2398: 0x008c, 0x2399: 0x008c, 0x239a: 0x008c, 0x239b: 0x008c, 0x239c: 0x008c, 0x239d: 0x008c, + 0x239e: 0x008c, 0x239f: 0x008c, 0x23a0: 0x008c, 0x23a1: 0x008c, 0x23a2: 0x008c, 0x23a3: 0x008c, + 0x23a4: 0x008c, 0x23a5: 0x008c, 0x23a6: 0x008c, 0x23a7: 0x008c, 0x23a8: 0x008c, 0x23a9: 0x008c, + 0x23aa: 0x008c, 0x23ab: 0x008c, 0x23ac: 0x008c, 0x23ad: 0x008c, 0x23ae: 0x008c, 0x23af: 0x008c, + 0x23b0: 0x008c, 0x23b1: 0x008c, 0x23b2: 0x008c, 0x23b3: 0x008c, 0x23b4: 0x008c, 0x23b5: 0x008c, + 0x23b6: 0x008c, 0x23b7: 0x008c, 0x23b8: 0x008c, 0x23b9: 0x008c, 0x23ba: 0x008c, 0x23bb: 0x008c, + 0x23bc: 0x008c, 0x23bd: 0x008c, 0x23be: 0x008c, + // Block 0x8f, offset 0x23c0 + 0x23c0: 0x008c, 0x23c1: 0x008c, 0x23c2: 0x008c, 0x23c3: 0x008c, 0x23c4: 0x008c, 0x23c5: 0x008c, + 0x23c6: 0x008c, 0x23c7: 0x008c, 0x23c8: 0x008c, 0x23c9: 0x008c, 0x23ca: 0x008c, 0x23cb: 0x008c, + 0x23cc: 0x008c, 0x23cd: 0x008c, 0x23ce: 0x008c, 0x23cf: 0x008c, 0x23d0: 0x008c, 0x23d1: 0x008c, + 0x23d2: 0x008c, 0x23d3: 0x008c, 0x23d4: 0x008c, 0x23d5: 0x008c, 0x23d6: 0x008c, 0x23d7: 0x008c, + 0x23d8: 0x0080, 0x23d9: 0x0080, 0x23da: 0x0080, 0x23db: 0x0080, 0x23dc: 0x0080, 0x23dd: 0x0080, + 0x23de: 0x0080, 0x23df: 0x0080, 0x23e0: 0x0080, 0x23e1: 0x0080, 0x23e2: 0x0080, 0x23e3: 0x0080, + 0x23e4: 0x0080, 0x23e5: 0x0080, 0x23e6: 0x0080, 0x23e7: 0x0080, 0x23e8: 0x0080, 0x23e9: 0x0080, + 0x23ea: 0x0080, 0x23eb: 0x0080, 0x23ec: 0x0080, 0x23ed: 0x0080, 0x23ee: 0x0080, 0x23ef: 0x0080, + 0x23f0: 0x0080, 0x23f1: 0x0080, 0x23f2: 0x0080, 0x23f3: 0x0080, 0x23f4: 0x0080, 0x23f5: 0x0080, + 0x23f6: 0x0080, 0x23f7: 0x0080, 0x23f8: 0x0080, 0x23f9: 0x0080, 0x23fa: 0x0080, 0x23fb: 0x0080, + 0x23fc: 0x0080, 0x23fd: 0x0080, 0x23fe: 0x0080, 0x23ff: 0x0080, + // Block 0x90, offset 0x2400 + 0x2400: 0x00cc, 0x2401: 0x00cc, 0x2402: 0x00cc, 0x2403: 0x00cc, 0x2404: 0x00cc, 0x2405: 0x00cc, + 0x2406: 0x00cc, 0x2407: 0x00cc, 0x2408: 0x00cc, 0x2409: 0x00cc, 0x240a: 0x00cc, 0x240b: 0x00cc, + 0x240c: 0x00cc, 0x240d: 0x00cc, 0x240e: 0x00cc, 0x240f: 0x00cc, 0x2410: 0x00cc, 0x2411: 0x00cc, + 0x2412: 0x00cc, 0x2413: 0x00cc, 0x2414: 0x00cc, 0x2415: 0x00cc, 0x2416: 0x00cc, 0x2417: 0x00cc, + 0x2418: 0x00cc, 0x2419: 0x00cc, 0x241a: 0x00cc, 0x241b: 0x00cc, 0x241c: 0x00cc, 0x241d: 0x00cc, + 0x241e: 0x00cc, 0x241f: 0x00cc, 0x2420: 0x00cc, 0x2421: 0x00cc, 0x2422: 0x00cc, 0x2423: 0x00cc, + 0x2424: 0x00cc, 0x2425: 0x00cc, 0x2426: 0x00cc, 0x2427: 0x00cc, 0x2428: 0x00cc, 0x2429: 0x00cc, + 0x242a: 0x00cc, 0x242b: 0x00cc, 0x242c: 0x00cc, 0x242d: 0x00cc, 0x242e: 0x00cc, 0x242f: 0x00cc, + 0x2430: 0x00cc, 0x2431: 0x00cc, 0x2432: 0x00cc, 0x2433: 0x00cc, 0x2434: 0x00cc, 0x2435: 0x00cc, + 0x2436: 0x00cc, 0x2437: 0x00cc, 0x2438: 0x00cc, 0x2439: 0x00cc, 0x243a: 0x00cc, 0x243b: 0x00cc, + 0x243c: 0x00cc, 0x243d: 0x00cc, 0x243e: 0x00cc, 0x243f: 0x00cc, + // Block 0x91, offset 0x2440 + 0x2440: 0x00cc, 0x2441: 0x00cc, 0x2442: 0x00cc, 0x2443: 0x00cc, 0x2444: 0x00cc, 0x2445: 0x00cc, + 0x2446: 0x00cc, 0x2447: 0x00cc, 0x2448: 0x00cc, 0x2449: 0x00cc, 0x244a: 0x00cc, 0x244b: 0x00cc, + 0x244c: 0x00cc, 0x244d: 0x00cc, 0x244e: 0x00cc, 0x244f: 0x00cc, 0x2450: 0x00cc, 0x2451: 0x00cc, + 0x2452: 0x00cc, 0x2453: 0x00cc, 0x2454: 0x00cc, 0x2455: 0x00cc, 0x2456: 0x00cc, 0x2457: 0x00cc, + 0x2458: 0x00cc, 0x2459: 0x00cc, 0x245a: 0x00cc, 0x245b: 0x00cc, 0x245c: 0x00cc, 0x245d: 0x00cc, + 0x245e: 0x00cc, 0x245f: 0x00cc, 0x2460: 0x00cc, 0x2461: 0x00cc, 0x2462: 0x00cc, 0x2463: 0x00cc, + 0x2464: 0x00cc, 0x2465: 0x00cc, 0x2466: 0x00cc, 0x2467: 0x00cc, 0x2468: 0x00cc, 0x2469: 0x00cc, + 0x246a: 0x00cc, 0x246b: 0x00cc, 0x246c: 0x00cc, 0x246d: 0x00cc, 0x246e: 0x00cc, 0x246f: 0x00cc, + 0x2470: 0x00cc, 0x2471: 0x00cc, 0x2472: 0x00cc, 0x2473: 0x00cc, 0x2474: 0x00cc, 0x2475: 0x00cc, + // Block 0x92, offset 0x2480 + 0x2480: 0x00cc, 0x2481: 0x00cc, 0x2482: 0x00cc, 0x2483: 0x00cc, 0x2484: 0x00cc, 0x2485: 0x00cc, + 0x2486: 0x00cc, 0x2487: 0x00cc, 0x2488: 0x00cc, 0x2489: 0x00cc, 0x248a: 0x00cc, 0x248b: 0x00cc, + 0x248c: 0x00cc, 0x248d: 0x00cc, 0x248e: 0x00cc, 0x248f: 0x00cc, 0x2490: 0x00cc, 0x2491: 0x00cc, + 0x2492: 0x00cc, 0x2493: 0x00cc, 0x2494: 0x00cc, 0x2495: 0x00cc, 0x2496: 0x00cc, 0x2497: 0x00cc, + 0x2498: 0x00cc, 0x2499: 0x00cc, 0x249a: 0x00cc, 0x249b: 0x00cc, 0x249c: 0x00cc, 0x249d: 0x00cc, + 0x249e: 0x00cc, 0x249f: 0x00cc, 0x24a0: 0x00cc, 0x24a1: 0x00cc, 0x24a2: 0x00cc, 0x24a3: 0x00cc, + 0x24a4: 0x00cc, 0x24a5: 0x00cc, 0x24a6: 0x00cc, 0x24a7: 0x00cc, 0x24a8: 0x00cc, 0x24a9: 0x00cc, + 0x24aa: 0x00cc, 0x24ab: 0x00cc, 0x24ac: 0x00cc, 0x24ad: 0x00cc, 0x24ae: 0x00cc, 0x24af: 0x00cc, + // Block 0x93, offset 0x24c0 + 0x24c0: 0x00c0, 0x24c1: 0x00c0, 0x24c2: 0x00c0, 0x24c3: 0x00c0, 0x24c4: 0x00c0, 0x24c5: 0x00c0, + 0x24c6: 0x00c0, 0x24c7: 0x00c0, 0x24c8: 0x00c0, 0x24c9: 0x00c0, 0x24ca: 0x00c0, 0x24cb: 0x00c0, + 0x24cc: 0x00c0, 0x24d0: 0x0080, 0x24d1: 0x0080, + 0x24d2: 0x0080, 0x24d3: 0x0080, 0x24d4: 0x0080, 0x24d5: 0x0080, 0x24d6: 0x0080, 0x24d7: 0x0080, + 0x24d8: 0x0080, 0x24d9: 0x0080, 0x24da: 0x0080, 0x24db: 0x0080, 0x24dc: 0x0080, 0x24dd: 0x0080, + 0x24de: 0x0080, 0x24df: 0x0080, 0x24e0: 0x0080, 0x24e1: 0x0080, 0x24e2: 0x0080, 0x24e3: 0x0080, + 0x24e4: 0x0080, 0x24e5: 0x0080, 0x24e6: 0x0080, 0x24e7: 0x0080, 0x24e8: 0x0080, 0x24e9: 0x0080, + 0x24ea: 0x0080, 0x24eb: 0x0080, 0x24ec: 0x0080, 0x24ed: 0x0080, 0x24ee: 0x0080, 0x24ef: 0x0080, + 0x24f0: 0x0080, 0x24f1: 0x0080, 0x24f2: 0x0080, 0x24f3: 0x0080, 0x24f4: 0x0080, 0x24f5: 0x0080, + 0x24f6: 0x0080, 0x24f7: 0x0080, 0x24f8: 0x0080, 0x24f9: 0x0080, 0x24fa: 0x0080, 0x24fb: 0x0080, + 0x24fc: 0x0080, 0x24fd: 0x0080, 0x24fe: 0x0080, 0x24ff: 0x0080, + // Block 0x94, offset 0x2500 + 0x2500: 0x0080, 0x2501: 0x0080, 0x2502: 0x0080, 0x2503: 0x0080, 0x2504: 0x0080, 0x2505: 0x0080, + 0x2506: 0x0080, + 0x2510: 0x00c0, 0x2511: 0x00c0, + 0x2512: 0x00c0, 0x2513: 0x00c0, 0x2514: 0x00c0, 0x2515: 0x00c0, 0x2516: 0x00c0, 0x2517: 0x00c0, + 0x2518: 0x00c0, 0x2519: 0x00c0, 0x251a: 0x00c0, 0x251b: 0x00c0, 0x251c: 0x00c0, 0x251d: 0x00c0, + 0x251e: 0x00c0, 0x251f: 0x00c0, 0x2520: 0x00c0, 0x2521: 0x00c0, 0x2522: 0x00c0, 0x2523: 0x00c0, + 0x2524: 0x00c0, 0x2525: 0x00c0, 0x2526: 0x00c0, 0x2527: 0x00c0, 0x2528: 0x00c0, 0x2529: 0x00c0, + 0x252a: 0x00c0, 0x252b: 0x00c0, 0x252c: 0x00c0, 0x252d: 0x00c0, 0x252e: 0x00c0, 0x252f: 0x00c0, + 0x2530: 0x00c0, 0x2531: 0x00c0, 0x2532: 0x00c0, 0x2533: 0x00c0, 0x2534: 0x00c0, 0x2535: 0x00c0, + 0x2536: 0x00c0, 0x2537: 0x00c0, 0x2538: 0x00c0, 0x2539: 0x00c0, 0x253a: 0x00c0, 0x253b: 0x00c0, + 0x253c: 0x00c0, 0x253d: 0x00c0, 0x253e: 0x0080, 0x253f: 0x0080, + // Block 0x95, offset 0x2540 + 0x2540: 0x00c0, 0x2541: 0x00c0, 0x2542: 0x00c0, 0x2543: 0x00c0, 0x2544: 0x00c0, 0x2545: 0x00c0, + 0x2546: 0x00c0, 0x2547: 0x00c0, 0x2548: 0x00c0, 0x2549: 0x00c0, 0x254a: 0x00c0, 0x254b: 0x00c0, + 0x254c: 0x00c0, 0x254d: 0x0080, 0x254e: 0x0080, 0x254f: 0x0080, 0x2550: 0x00c0, 0x2551: 0x00c0, + 0x2552: 0x00c0, 0x2553: 0x00c0, 0x2554: 0x00c0, 0x2555: 0x00c0, 0x2556: 0x00c0, 0x2557: 0x00c0, + 0x2558: 0x00c0, 0x2559: 0x00c0, 0x255a: 0x00c0, 0x255b: 0x00c0, 0x255c: 0x00c0, 0x255d: 0x00c0, + 0x255e: 0x00c0, 0x255f: 0x00c0, 0x2560: 0x00c0, 0x2561: 0x00c0, 0x2562: 0x00c0, 0x2563: 0x00c0, + 0x2564: 0x00c0, 0x2565: 0x00c0, 0x2566: 0x00c0, 0x2567: 0x00c0, 0x2568: 0x00c0, 0x2569: 0x00c0, + 0x256a: 0x00c0, 0x256b: 0x00c0, + // Block 0x96, offset 0x2580 + 0x2580: 0x00c0, 0x2581: 0x00c0, 0x2582: 0x00c0, 0x2583: 0x00c0, 0x2584: 0x00c0, 0x2585: 0x00c0, + 0x2586: 0x00c0, 0x2587: 0x00c0, 0x2588: 0x00c0, 0x2589: 0x00c0, 0x258a: 0x00c0, 0x258b: 0x00c0, + 0x258c: 0x00c0, 0x258d: 0x00c0, 0x258e: 0x00c0, 0x258f: 0x00c0, 0x2590: 0x00c0, 0x2591: 0x00c0, + 0x2592: 0x00c0, 0x2593: 0x00c0, 0x2594: 0x00c0, 0x2595: 0x00c0, 0x2596: 0x00c0, 0x2597: 0x00c0, + 0x2598: 0x00c0, 0x2599: 0x00c0, 0x259a: 0x00c0, 0x259b: 0x00c0, 0x259c: 0x00c0, 0x259d: 0x00c0, + 0x259e: 0x00c0, 0x259f: 0x00c0, 0x25a0: 0x00c0, 0x25a1: 0x00c0, 0x25a2: 0x00c0, 0x25a3: 0x00c0, + 0x25a4: 0x00c0, 0x25a5: 0x00c0, 0x25a6: 0x00c0, 0x25a7: 0x00c0, 0x25a8: 0x00c0, 0x25a9: 0x00c0, + 0x25aa: 0x00c0, 0x25ab: 0x00c0, 0x25ac: 0x00c0, 0x25ad: 0x00c0, 0x25ae: 0x00c0, 0x25af: 0x00c3, + 0x25b0: 0x0083, 0x25b1: 0x0083, 0x25b2: 0x0083, 0x25b3: 0x0080, 0x25b4: 0x00c3, 0x25b5: 0x00c3, + 0x25b6: 0x00c3, 0x25b7: 0x00c3, 0x25b8: 0x00c3, 0x25b9: 0x00c3, 0x25ba: 0x00c3, 0x25bb: 0x00c3, + 0x25bc: 0x00c3, 0x25bd: 0x00c3, 0x25be: 0x0080, 0x25bf: 0x00c0, + // Block 0x97, offset 0x25c0 + 0x25c0: 0x00c0, 0x25c1: 0x00c0, 0x25c2: 0x00c0, 0x25c3: 0x00c0, 0x25c4: 0x00c0, 0x25c5: 0x00c0, + 0x25c6: 0x00c0, 0x25c7: 0x00c0, 0x25c8: 0x00c0, 0x25c9: 0x00c0, 0x25ca: 0x00c0, 0x25cb: 0x00c0, + 0x25cc: 0x00c0, 0x25cd: 0x00c0, 0x25ce: 0x00c0, 0x25cf: 0x00c0, 0x25d0: 0x00c0, 0x25d1: 0x00c0, + 0x25d2: 0x00c0, 0x25d3: 0x00c0, 0x25d4: 0x00c0, 0x25d5: 0x00c0, 0x25d6: 0x00c0, 0x25d7: 0x00c0, + 0x25d8: 0x00c0, 0x25d9: 0x00c0, 0x25da: 0x00c0, 0x25db: 0x00c0, 0x25dc: 0x0080, 0x25dd: 0x0080, + 0x25de: 0x00c3, 0x25df: 0x00c3, 0x25e0: 0x00c0, 0x25e1: 0x00c0, 0x25e2: 0x00c0, 0x25e3: 0x00c0, + 0x25e4: 0x00c0, 0x25e5: 0x00c0, 0x25e6: 0x00c0, 0x25e7: 0x00c0, 0x25e8: 0x00c0, 0x25e9: 0x00c0, + 0x25ea: 0x00c0, 0x25eb: 0x00c0, 0x25ec: 0x00c0, 0x25ed: 0x00c0, 0x25ee: 0x00c0, 0x25ef: 0x00c0, + 0x25f0: 0x00c0, 0x25f1: 0x00c0, 0x25f2: 0x00c0, 0x25f3: 0x00c0, 0x25f4: 0x00c0, 0x25f5: 0x00c0, + 0x25f6: 0x00c0, 0x25f7: 0x00c0, 0x25f8: 0x00c0, 0x25f9: 0x00c0, 0x25fa: 0x00c0, 0x25fb: 0x00c0, + 0x25fc: 0x00c0, 0x25fd: 0x00c0, 0x25fe: 0x00c0, 0x25ff: 0x00c0, + // Block 0x98, offset 0x2600 + 0x2600: 0x00c0, 0x2601: 0x00c0, 0x2602: 0x00c0, 0x2603: 0x00c0, 0x2604: 0x00c0, 0x2605: 0x00c0, + 0x2606: 0x00c0, 0x2607: 0x00c0, 0x2608: 0x00c0, 0x2609: 0x00c0, 0x260a: 0x00c0, 0x260b: 0x00c0, + 0x260c: 0x00c0, 0x260d: 0x00c0, 0x260e: 0x00c0, 0x260f: 0x00c0, 0x2610: 0x00c0, 0x2611: 0x00c0, + 0x2612: 0x00c0, 0x2613: 0x00c0, 0x2614: 0x00c0, 0x2615: 0x00c0, 0x2616: 0x00c0, 0x2617: 0x00c0, + 0x2618: 0x00c0, 0x2619: 0x00c0, 0x261a: 0x00c0, 0x261b: 0x00c0, 0x261c: 0x00c0, 0x261d: 0x00c0, + 0x261e: 0x00c0, 0x261f: 0x00c0, 0x2620: 0x00c0, 0x2621: 0x00c0, 0x2622: 0x00c0, 0x2623: 0x00c0, + 0x2624: 0x00c0, 0x2625: 0x00c0, 0x2626: 0x0080, 0x2627: 0x0080, 0x2628: 0x0080, 0x2629: 0x0080, + 0x262a: 0x0080, 0x262b: 0x0080, 0x262c: 0x0080, 0x262d: 0x0080, 0x262e: 0x0080, 0x262f: 0x0080, + 0x2630: 0x00c3, 0x2631: 0x00c3, 0x2632: 0x0080, 0x2633: 0x0080, 0x2634: 0x0080, 0x2635: 0x0080, + 0x2636: 0x0080, 0x2637: 0x0080, + // Block 0x99, offset 0x2640 + 0x2640: 0x0080, 0x2641: 0x0080, 0x2642: 0x0080, 0x2643: 0x0080, 0x2644: 0x0080, 0x2645: 0x0080, + 0x2646: 0x0080, 0x2647: 0x0080, 0x2648: 0x0080, 0x2649: 0x0080, 0x264a: 0x0080, 0x264b: 0x0080, + 0x264c: 0x0080, 0x264d: 0x0080, 0x264e: 0x0080, 0x264f: 0x0080, 0x2650: 0x0080, 0x2651: 0x0080, + 0x2652: 0x0080, 0x2653: 0x0080, 0x2654: 0x0080, 0x2655: 0x0080, 0x2656: 0x0080, 0x2657: 0x00c0, + 0x2658: 0x00c0, 0x2659: 0x00c0, 0x265a: 0x00c0, 0x265b: 0x00c0, 0x265c: 0x00c0, 0x265d: 0x00c0, + 0x265e: 0x00c0, 0x265f: 0x00c0, 0x2660: 0x0080, 0x2661: 0x0080, 0x2662: 0x00c0, 0x2663: 0x00c0, + 0x2664: 0x00c0, 0x2665: 0x00c0, 0x2666: 0x00c0, 0x2667: 0x00c0, 0x2668: 0x00c0, 0x2669: 0x00c0, + 0x266a: 0x00c0, 0x266b: 0x00c0, 0x266c: 0x00c0, 0x266d: 0x00c0, 0x266e: 0x00c0, 0x266f: 0x00c0, + 0x2670: 0x00c0, 0x2671: 0x00c0, 0x2672: 0x00c0, 0x2673: 0x00c0, 0x2674: 0x00c0, 0x2675: 0x00c0, + 0x2676: 0x00c0, 0x2677: 0x00c0, 0x2678: 0x00c0, 0x2679: 0x00c0, 0x267a: 0x00c0, 0x267b: 0x00c0, + 0x267c: 0x00c0, 0x267d: 0x00c0, 0x267e: 0x00c0, 0x267f: 0x00c0, + // Block 0x9a, offset 0x2680 + 0x2680: 0x00c0, 0x2681: 0x00c0, 0x2682: 0x00c0, 0x2683: 0x00c0, 0x2684: 0x00c0, 0x2685: 0x00c0, + 0x2686: 0x00c0, 0x2687: 0x00c0, 0x2688: 0x00c0, 0x2689: 0x00c0, 0x268a: 0x00c0, 0x268b: 0x00c0, + 0x268c: 0x00c0, 0x268d: 0x00c0, 0x268e: 0x00c0, 0x268f: 0x00c0, 0x2690: 0x00c0, 0x2691: 0x00c0, + 0x2692: 0x00c0, 0x2693: 0x00c0, 0x2694: 0x00c0, 0x2695: 0x00c0, 0x2696: 0x00c0, 0x2697: 0x00c0, + 0x2698: 0x00c0, 0x2699: 0x00c0, 0x269a: 0x00c0, 0x269b: 0x00c0, 0x269c: 0x00c0, 0x269d: 0x00c0, + 0x269e: 0x00c0, 0x269f: 0x00c0, 0x26a0: 0x00c0, 0x26a1: 0x00c0, 0x26a2: 0x00c0, 0x26a3: 0x00c0, + 0x26a4: 0x00c0, 0x26a5: 0x00c0, 0x26a6: 0x00c0, 0x26a7: 0x00c0, 0x26a8: 0x00c0, 0x26a9: 0x00c0, + 0x26aa: 0x00c0, 0x26ab: 0x00c0, 0x26ac: 0x00c0, 0x26ad: 0x00c0, 0x26ae: 0x00c0, 0x26af: 0x00c0, + 0x26b0: 0x0080, 0x26b1: 0x00c0, 0x26b2: 0x00c0, 0x26b3: 0x00c0, 0x26b4: 0x00c0, 0x26b5: 0x00c0, + 0x26b6: 0x00c0, 0x26b7: 0x00c0, 0x26b8: 0x00c0, 0x26b9: 0x00c0, 0x26ba: 0x00c0, 0x26bb: 0x00c0, + 0x26bc: 0x00c0, 0x26bd: 0x00c0, 0x26be: 0x00c0, 0x26bf: 0x00c0, + // Block 0x9b, offset 0x26c0 + 0x26c0: 0x00c0, 0x26c1: 0x00c0, 0x26c2: 0x00c0, 0x26c3: 0x00c0, 0x26c4: 0x00c0, 0x26c5: 0x00c0, + 0x26c6: 0x00c0, 0x26c7: 0x00c0, 0x26c8: 0x00c0, 0x26c9: 0x0080, 0x26ca: 0x0080, 0x26cb: 0x00c0, + 0x26cc: 0x00c0, 0x26cd: 0x00c0, 0x26ce: 0x00c0, 0x26cf: 0x00c0, 0x26d0: 0x00c0, 0x26d1: 0x00c0, + 0x26d2: 0x00c0, 0x26d3: 0x00c0, 0x26d4: 0x00c0, 0x26d5: 0x00c0, 0x26d6: 0x00c0, 0x26d7: 0x00c0, + 0x26d8: 0x00c0, 0x26d9: 0x00c0, 0x26da: 0x00c0, 0x26db: 0x00c0, 0x26dc: 0x00c0, 0x26dd: 0x00c0, + 0x26de: 0x00c0, 0x26df: 0x00c0, 0x26e0: 0x00c0, 0x26e1: 0x00c0, 0x26e2: 0x00c0, 0x26e3: 0x00c0, + 0x26e4: 0x00c0, 0x26e5: 0x00c0, 0x26e6: 0x00c0, 0x26e7: 0x00c0, 0x26e8: 0x00c0, 0x26e9: 0x00c0, + 0x26ea: 0x00c0, 0x26eb: 0x00c0, 0x26ec: 0x00c0, 0x26ed: 0x00c0, 0x26ee: 0x00c0, 0x26ef: 0x00c0, + 0x26f0: 0x00c0, 0x26f1: 0x00c0, 0x26f2: 0x00c0, 0x26f3: 0x00c0, 0x26f4: 0x00c0, 0x26f5: 0x00c0, + 0x26f6: 0x00c0, 0x26f7: 0x00c0, 0x26f8: 0x00c0, 0x26f9: 0x00c0, 0x26fa: 0x00c0, 0x26fb: 0x00c0, + 0x26fc: 0x00c0, 0x26fd: 0x00c0, 0x26fe: 0x00c0, 0x26ff: 0x00c0, + // Block 0x9c, offset 0x2700 + 0x2702: 0x00c0, 0x2703: 0x00c0, 0x2704: 0x00c0, 0x2705: 0x00c0, + 0x2706: 0x00c0, + 0x2737: 0x00c0, 0x2738: 0x0080, 0x2739: 0x0080, 0x273a: 0x00c0, 0x273b: 0x00c0, + 0x273c: 0x00c0, 0x273d: 0x00c0, 0x273e: 0x00c0, 0x273f: 0x00c0, + // Block 0x9d, offset 0x2740 + 0x2740: 0x00c0, 0x2741: 0x00c0, 0x2742: 0x00c3, 0x2743: 0x00c0, 0x2744: 0x00c0, 0x2745: 0x00c0, + 0x2746: 0x00c6, 0x2747: 0x00c0, 0x2748: 0x00c0, 0x2749: 0x00c0, 0x274a: 0x00c0, 0x274b: 0x00c3, + 0x274c: 0x00c0, 0x274d: 0x00c0, 0x274e: 0x00c0, 0x274f: 0x00c0, 0x2750: 0x00c0, 0x2751: 0x00c0, + 0x2752: 0x00c0, 0x2753: 0x00c0, 0x2754: 0x00c0, 0x2755: 0x00c0, 0x2756: 0x00c0, 0x2757: 0x00c0, + 0x2758: 0x00c0, 0x2759: 0x00c0, 0x275a: 0x00c0, 0x275b: 0x00c0, 0x275c: 0x00c0, 0x275d: 0x00c0, + 0x275e: 0x00c0, 0x275f: 0x00c0, 0x2760: 0x00c0, 0x2761: 0x00c0, 0x2762: 0x00c0, 0x2763: 0x00c0, + 0x2764: 0x00c0, 0x2765: 0x00c3, 0x2766: 0x00c3, 0x2767: 0x00c0, 0x2768: 0x0080, 0x2769: 0x0080, + 0x276a: 0x0080, 0x276b: 0x0080, + 0x2770: 0x0080, 0x2771: 0x0080, 0x2772: 0x0080, 0x2773: 0x0080, 0x2774: 0x0080, 0x2775: 0x0080, + 0x2776: 0x0080, 0x2777: 0x0080, 0x2778: 0x0080, 0x2779: 0x0080, + // Block 0x9e, offset 0x2780 + 0x2780: 0x00c2, 0x2781: 0x00c2, 0x2782: 0x00c2, 0x2783: 0x00c2, 0x2784: 0x00c2, 0x2785: 0x00c2, + 0x2786: 0x00c2, 0x2787: 0x00c2, 0x2788: 0x00c2, 0x2789: 0x00c2, 0x278a: 0x00c2, 0x278b: 0x00c2, + 0x278c: 0x00c2, 0x278d: 0x00c2, 0x278e: 0x00c2, 0x278f: 0x00c2, 0x2790: 0x00c2, 0x2791: 0x00c2, + 0x2792: 0x00c2, 0x2793: 0x00c2, 0x2794: 0x00c2, 0x2795: 0x00c2, 0x2796: 0x00c2, 0x2797: 0x00c2, + 0x2798: 0x00c2, 0x2799: 0x00c2, 0x279a: 0x00c2, 0x279b: 0x00c2, 0x279c: 0x00c2, 0x279d: 0x00c2, + 0x279e: 0x00c2, 0x279f: 0x00c2, 0x27a0: 0x00c2, 0x27a1: 0x00c2, 0x27a2: 0x00c2, 0x27a3: 0x00c2, + 0x27a4: 0x00c2, 0x27a5: 0x00c2, 0x27a6: 0x00c2, 0x27a7: 0x00c2, 0x27a8: 0x00c2, 0x27a9: 0x00c2, + 0x27aa: 0x00c2, 0x27ab: 0x00c2, 0x27ac: 0x00c2, 0x27ad: 0x00c2, 0x27ae: 0x00c2, 0x27af: 0x00c2, + 0x27b0: 0x00c2, 0x27b1: 0x00c2, 0x27b2: 0x00c1, 0x27b3: 0x00c0, 0x27b4: 0x0080, 0x27b5: 0x0080, + 0x27b6: 0x0080, 0x27b7: 0x0080, + // Block 0x9f, offset 0x27c0 + 0x27c0: 0x00c0, 0x27c1: 0x00c0, 0x27c2: 0x00c0, 0x27c3: 0x00c0, 0x27c4: 0x00c6, 0x27c5: 0x00c3, + 0x27ce: 0x0080, 0x27cf: 0x0080, 0x27d0: 0x00c0, 0x27d1: 0x00c0, + 0x27d2: 0x00c0, 0x27d3: 0x00c0, 0x27d4: 0x00c0, 0x27d5: 0x00c0, 0x27d6: 0x00c0, 0x27d7: 0x00c0, + 0x27d8: 0x00c0, 0x27d9: 0x00c0, + 0x27e0: 0x00c3, 0x27e1: 0x00c3, 0x27e2: 0x00c3, 0x27e3: 0x00c3, + 0x27e4: 0x00c3, 0x27e5: 0x00c3, 0x27e6: 0x00c3, 0x27e7: 0x00c3, 0x27e8: 0x00c3, 0x27e9: 0x00c3, + 0x27ea: 0x00c3, 0x27eb: 0x00c3, 0x27ec: 0x00c3, 0x27ed: 0x00c3, 0x27ee: 0x00c3, 0x27ef: 0x00c3, + 0x27f0: 0x00c3, 0x27f1: 0x00c3, 0x27f2: 0x00c0, 0x27f3: 0x00c0, 0x27f4: 0x00c0, 0x27f5: 0x00c0, + 0x27f6: 0x00c0, 0x27f7: 0x00c0, 0x27f8: 0x0080, 0x27f9: 0x0080, 0x27fa: 0x0080, 0x27fb: 0x00c0, + 0x27fc: 0x0080, 0x27fd: 0x00c0, 0x27fe: 0x00c0, 0x27ff: 0x00c3, + // Block 0xa0, offset 0x2800 + 0x2800: 0x00c0, 0x2801: 0x00c0, 0x2802: 0x00c0, 0x2803: 0x00c0, 0x2804: 0x00c0, 0x2805: 0x00c0, + 0x2806: 0x00c0, 0x2807: 0x00c0, 0x2808: 0x00c0, 0x2809: 0x00c0, 0x280a: 0x00c0, 0x280b: 0x00c0, + 0x280c: 0x00c0, 0x280d: 0x00c0, 0x280e: 0x00c0, 0x280f: 0x00c0, 0x2810: 0x00c0, 0x2811: 0x00c0, + 0x2812: 0x00c0, 0x2813: 0x00c0, 0x2814: 0x00c0, 0x2815: 0x00c0, 0x2816: 0x00c0, 0x2817: 0x00c0, + 0x2818: 0x00c0, 0x2819: 0x00c0, 0x281a: 0x00c0, 0x281b: 0x00c0, 0x281c: 0x00c0, 0x281d: 0x00c0, + 0x281e: 0x00c0, 0x281f: 0x00c0, 0x2820: 0x00c0, 0x2821: 0x00c0, 0x2822: 0x00c0, 0x2823: 0x00c0, + 0x2824: 0x00c0, 0x2825: 0x00c0, 0x2826: 0x00c3, 0x2827: 0x00c3, 0x2828: 0x00c3, 0x2829: 0x00c3, + 0x282a: 0x00c3, 0x282b: 0x00c3, 0x282c: 0x00c3, 0x282d: 0x00c3, 0x282e: 0x0080, 0x282f: 0x0080, + 0x2830: 0x00c0, 0x2831: 0x00c0, 0x2832: 0x00c0, 0x2833: 0x00c0, 0x2834: 0x00c0, 0x2835: 0x00c0, + 0x2836: 0x00c0, 0x2837: 0x00c0, 0x2838: 0x00c0, 0x2839: 0x00c0, 0x283a: 0x00c0, 0x283b: 0x00c0, + 0x283c: 0x00c0, 0x283d: 0x00c0, 0x283e: 0x00c0, 0x283f: 0x00c0, + // Block 0xa1, offset 0x2840 + 0x2840: 0x00c0, 0x2841: 0x00c0, 0x2842: 0x00c0, 0x2843: 0x00c0, 0x2844: 0x00c0, 0x2845: 0x00c0, + 0x2846: 0x00c0, 0x2847: 0x00c3, 0x2848: 0x00c3, 0x2849: 0x00c3, 0x284a: 0x00c3, 0x284b: 0x00c3, + 0x284c: 0x00c3, 0x284d: 0x00c3, 0x284e: 0x00c3, 0x284f: 0x00c3, 0x2850: 0x00c3, 0x2851: 0x00c3, + 0x2852: 0x00c0, 0x2853: 0x00c5, + 0x285f: 0x0080, 0x2860: 0x0040, 0x2861: 0x0040, 0x2862: 0x0040, 0x2863: 0x0040, + 0x2864: 0x0040, 0x2865: 0x0040, 0x2866: 0x0040, 0x2867: 0x0040, 0x2868: 0x0040, 0x2869: 0x0040, + 0x286a: 0x0040, 0x286b: 0x0040, 0x286c: 0x0040, 0x286d: 0x0040, 0x286e: 0x0040, 0x286f: 0x0040, + 0x2870: 0x0040, 0x2871: 0x0040, 0x2872: 0x0040, 0x2873: 0x0040, 0x2874: 0x0040, 0x2875: 0x0040, + 0x2876: 0x0040, 0x2877: 0x0040, 0x2878: 0x0040, 0x2879: 0x0040, 0x287a: 0x0040, 0x287b: 0x0040, + 0x287c: 0x0040, + // Block 0xa2, offset 0x2880 + 0x2880: 0x00c3, 0x2881: 0x00c3, 0x2882: 0x00c3, 0x2883: 0x00c0, 0x2884: 0x00c0, 0x2885: 0x00c0, + 0x2886: 0x00c0, 0x2887: 0x00c0, 0x2888: 0x00c0, 0x2889: 0x00c0, 0x288a: 0x00c0, 0x288b: 0x00c0, + 0x288c: 0x00c0, 0x288d: 0x00c0, 0x288e: 0x00c0, 0x288f: 0x00c0, 0x2890: 0x00c0, 0x2891: 0x00c0, + 0x2892: 0x00c0, 0x2893: 0x00c0, 0x2894: 0x00c0, 0x2895: 0x00c0, 0x2896: 0x00c0, 0x2897: 0x00c0, + 0x2898: 0x00c0, 0x2899: 0x00c0, 0x289a: 0x00c0, 0x289b: 0x00c0, 0x289c: 0x00c0, 0x289d: 0x00c0, + 0x289e: 0x00c0, 0x289f: 0x00c0, 0x28a0: 0x00c0, 0x28a1: 0x00c0, 0x28a2: 0x00c0, 0x28a3: 0x00c0, + 0x28a4: 0x00c0, 0x28a5: 0x00c0, 0x28a6: 0x00c0, 0x28a7: 0x00c0, 0x28a8: 0x00c0, 0x28a9: 0x00c0, + 0x28aa: 0x00c0, 0x28ab: 0x00c0, 0x28ac: 0x00c0, 0x28ad: 0x00c0, 0x28ae: 0x00c0, 0x28af: 0x00c0, + 0x28b0: 0x00c0, 0x28b1: 0x00c0, 0x28b2: 0x00c0, 0x28b3: 0x00c3, 0x28b4: 0x00c0, 0x28b5: 0x00c0, + 0x28b6: 0x00c3, 0x28b7: 0x00c3, 0x28b8: 0x00c3, 0x28b9: 0x00c3, 0x28ba: 0x00c0, 0x28bb: 0x00c0, + 0x28bc: 0x00c3, 0x28bd: 0x00c3, 0x28be: 0x00c0, 0x28bf: 0x00c0, + // Block 0xa3, offset 0x28c0 + 0x28c0: 0x00c5, 0x28c1: 0x0080, 0x28c2: 0x0080, 0x28c3: 0x0080, 0x28c4: 0x0080, 0x28c5: 0x0080, + 0x28c6: 0x0080, 0x28c7: 0x0080, 0x28c8: 0x0080, 0x28c9: 0x0080, 0x28ca: 0x0080, 0x28cb: 0x0080, + 0x28cc: 0x0080, 0x28cd: 0x0080, 0x28cf: 0x00c0, 0x28d0: 0x00c0, 0x28d1: 0x00c0, + 0x28d2: 0x00c0, 0x28d3: 0x00c0, 0x28d4: 0x00c0, 0x28d5: 0x00c0, 0x28d6: 0x00c0, 0x28d7: 0x00c0, + 0x28d8: 0x00c0, 0x28d9: 0x00c0, + 0x28de: 0x0080, 0x28df: 0x0080, 0x28e0: 0x00c0, 0x28e1: 0x00c0, 0x28e2: 0x00c0, 0x28e3: 0x00c0, + 0x28e4: 0x00c0, 0x28e5: 0x00c3, 0x28e6: 0x00c0, 0x28e7: 0x00c0, 0x28e8: 0x00c0, 0x28e9: 0x00c0, + 0x28ea: 0x00c0, 0x28eb: 0x00c0, 0x28ec: 0x00c0, 0x28ed: 0x00c0, 0x28ee: 0x00c0, 0x28ef: 0x00c0, + 0x28f0: 0x00c0, 0x28f1: 0x00c0, 0x28f2: 0x00c0, 0x28f3: 0x00c0, 0x28f4: 0x00c0, 0x28f5: 0x00c0, + 0x28f6: 0x00c0, 0x28f7: 0x00c0, 0x28f8: 0x00c0, 0x28f9: 0x00c0, 0x28fa: 0x00c0, 0x28fb: 0x00c0, + 0x28fc: 0x00c0, 0x28fd: 0x00c0, 0x28fe: 0x00c0, + // Block 0xa4, offset 0x2900 + 0x2900: 0x00c0, 0x2901: 0x00c0, 0x2902: 0x00c0, 0x2903: 0x00c0, 0x2904: 0x00c0, 0x2905: 0x00c0, + 0x2906: 0x00c0, 0x2907: 0x00c0, 0x2908: 0x00c0, 0x2909: 0x00c0, 0x290a: 0x00c0, 0x290b: 0x00c0, + 0x290c: 0x00c0, 0x290d: 0x00c0, 0x290e: 0x00c0, 0x290f: 0x00c0, 0x2910: 0x00c0, 0x2911: 0x00c0, + 0x2912: 0x00c0, 0x2913: 0x00c0, 0x2914: 0x00c0, 0x2915: 0x00c0, 0x2916: 0x00c0, 0x2917: 0x00c0, + 0x2918: 0x00c0, 0x2919: 0x00c0, 0x291a: 0x00c0, 0x291b: 0x00c0, 0x291c: 0x00c0, 0x291d: 0x00c0, + 0x291e: 0x00c0, 0x291f: 0x00c0, 0x2920: 0x00c0, 0x2921: 0x00c0, 0x2922: 0x00c0, 0x2923: 0x00c0, + 0x2924: 0x00c0, 0x2925: 0x00c0, 0x2926: 0x00c0, 0x2927: 0x00c0, 0x2928: 0x00c0, 0x2929: 0x00c3, + 0x292a: 0x00c3, 0x292b: 0x00c3, 0x292c: 0x00c3, 0x292d: 0x00c3, 0x292e: 0x00c3, 0x292f: 0x00c0, + 0x2930: 0x00c0, 0x2931: 0x00c3, 0x2932: 0x00c3, 0x2933: 0x00c0, 0x2934: 0x00c0, 0x2935: 0x00c3, + 0x2936: 0x00c3, + // Block 0xa5, offset 0x2940 + 0x2940: 0x00c0, 0x2941: 0x00c0, 0x2942: 0x00c0, 0x2943: 0x00c3, 0x2944: 0x00c0, 0x2945: 0x00c0, + 0x2946: 0x00c0, 0x2947: 0x00c0, 0x2948: 0x00c0, 0x2949: 0x00c0, 0x294a: 0x00c0, 0x294b: 0x00c0, + 0x294c: 0x00c3, 0x294d: 0x00c0, 0x2950: 0x00c0, 0x2951: 0x00c0, + 0x2952: 0x00c0, 0x2953: 0x00c0, 0x2954: 0x00c0, 0x2955: 0x00c0, 0x2956: 0x00c0, 0x2957: 0x00c0, + 0x2958: 0x00c0, 0x2959: 0x00c0, 0x295c: 0x0080, 0x295d: 0x0080, + 0x295e: 0x0080, 0x295f: 0x0080, 0x2960: 0x00c0, 0x2961: 0x00c0, 0x2962: 0x00c0, 0x2963: 0x00c0, + 0x2964: 0x00c0, 0x2965: 0x00c0, 0x2966: 0x00c0, 0x2967: 0x00c0, 0x2968: 0x00c0, 0x2969: 0x00c0, + 0x296a: 0x00c0, 0x296b: 0x00c0, 0x296c: 0x00c0, 0x296d: 0x00c0, 0x296e: 0x00c0, 0x296f: 0x00c0, + 0x2970: 0x00c0, 0x2971: 0x00c0, 0x2972: 0x00c0, 0x2973: 0x00c0, 0x2974: 0x00c0, 0x2975: 0x00c0, + 0x2976: 0x00c0, 0x2977: 0x0080, 0x2978: 0x0080, 0x2979: 0x0080, 0x297a: 0x00c0, 0x297b: 0x00c0, + 0x297c: 0x00c3, 0x297d: 0x00c0, 0x297e: 0x00c0, 0x297f: 0x00c0, + // Block 0xa6, offset 0x2980 + 0x2980: 0x00c0, 0x2981: 0x00c0, 0x2982: 0x00c0, 0x2983: 0x00c0, 0x2984: 0x00c0, 0x2985: 0x00c0, + 0x2986: 0x00c0, 0x2987: 0x00c0, 0x2988: 0x00c0, 0x2989: 0x00c0, 0x298a: 0x00c0, 0x298b: 0x00c0, + 0x298c: 0x00c0, 0x298d: 0x00c0, 0x298e: 0x00c0, 0x298f: 0x00c0, 0x2990: 0x00c0, 0x2991: 0x00c0, + 0x2992: 0x00c0, 0x2993: 0x00c0, 0x2994: 0x00c0, 0x2995: 0x00c0, 0x2996: 0x00c0, 0x2997: 0x00c0, + 0x2998: 0x00c0, 0x2999: 0x00c0, 0x299a: 0x00c0, 0x299b: 0x00c0, 0x299c: 0x00c0, 0x299d: 0x00c0, + 0x299e: 0x00c0, 0x299f: 0x00c0, 0x29a0: 0x00c0, 0x29a1: 0x00c0, 0x29a2: 0x00c0, 0x29a3: 0x00c0, + 0x29a4: 0x00c0, 0x29a5: 0x00c0, 0x29a6: 0x00c0, 0x29a7: 0x00c0, 0x29a8: 0x00c0, 0x29a9: 0x00c0, + 0x29aa: 0x00c0, 0x29ab: 0x00c0, 0x29ac: 0x00c0, 0x29ad: 0x00c0, 0x29ae: 0x00c0, 0x29af: 0x00c0, + 0x29b0: 0x00c3, 0x29b1: 0x00c0, 0x29b2: 0x00c3, 0x29b3: 0x00c3, 0x29b4: 0x00c3, 0x29b5: 0x00c0, + 0x29b6: 0x00c0, 0x29b7: 0x00c3, 0x29b8: 0x00c3, 0x29b9: 0x00c0, 0x29ba: 0x00c0, 0x29bb: 0x00c0, + 0x29bc: 0x00c0, 0x29bd: 0x00c0, 0x29be: 0x00c3, 0x29bf: 0x00c3, + // Block 0xa7, offset 0x29c0 + 0x29c0: 0x00c0, 0x29c1: 0x00c3, 0x29c2: 0x00c0, + 0x29db: 0x00c0, 0x29dc: 0x00c0, 0x29dd: 0x00c0, + 0x29de: 0x0080, 0x29df: 0x0080, 0x29e0: 0x00c0, 0x29e1: 0x00c0, 0x29e2: 0x00c0, 0x29e3: 0x00c0, + 0x29e4: 0x00c0, 0x29e5: 0x00c0, 0x29e6: 0x00c0, 0x29e7: 0x00c0, 0x29e8: 0x00c0, 0x29e9: 0x00c0, + 0x29ea: 0x00c0, 0x29eb: 0x00c0, 0x29ec: 0x00c3, 0x29ed: 0x00c3, 0x29ee: 0x00c0, 0x29ef: 0x00c0, + 0x29f0: 0x0080, 0x29f1: 0x0080, 0x29f2: 0x00c0, 0x29f3: 0x00c0, 0x29f4: 0x00c0, 0x29f5: 0x00c0, + 0x29f6: 0x00c6, + // Block 0xa8, offset 0x2a00 + 0x2a01: 0x00c0, 0x2a02: 0x00c0, 0x2a03: 0x00c0, 0x2a04: 0x00c0, 0x2a05: 0x00c0, + 0x2a06: 0x00c0, 0x2a09: 0x00c0, 0x2a0a: 0x00c0, 0x2a0b: 0x00c0, + 0x2a0c: 0x00c0, 0x2a0d: 0x00c0, 0x2a0e: 0x00c0, 0x2a11: 0x00c0, + 0x2a12: 0x00c0, 0x2a13: 0x00c0, 0x2a14: 0x00c0, 0x2a15: 0x00c0, 0x2a16: 0x00c0, + 0x2a20: 0x00c0, 0x2a21: 0x00c0, 0x2a22: 0x00c0, 0x2a23: 0x00c0, + 0x2a24: 0x00c0, 0x2a25: 0x00c0, 0x2a26: 0x00c0, 0x2a28: 0x00c0, 0x2a29: 0x00c0, + 0x2a2a: 0x00c0, 0x2a2b: 0x00c0, 0x2a2c: 0x00c0, 0x2a2d: 0x00c0, 0x2a2e: 0x00c0, + 0x2a30: 0x00c0, 0x2a31: 0x00c0, 0x2a32: 0x00c0, 0x2a33: 0x00c0, 0x2a34: 0x00c0, 0x2a35: 0x00c0, + 0x2a36: 0x00c0, 0x2a37: 0x00c0, 0x2a38: 0x00c0, 0x2a39: 0x00c0, 0x2a3a: 0x00c0, 0x2a3b: 0x00c0, + 0x2a3c: 0x00c0, 0x2a3d: 0x00c0, 0x2a3e: 0x00c0, 0x2a3f: 0x00c0, + // Block 0xa9, offset 0x2a40 + 0x2a40: 0x00c0, 0x2a41: 0x00c0, 0x2a42: 0x00c0, 0x2a43: 0x00c0, 0x2a44: 0x00c0, 0x2a45: 0x00c0, + 0x2a46: 0x00c0, 0x2a47: 0x00c0, 0x2a48: 0x00c0, 0x2a49: 0x00c0, 0x2a4a: 0x00c0, 0x2a4b: 0x00c0, + 0x2a4c: 0x00c0, 0x2a4d: 0x00c0, 0x2a4e: 0x00c0, 0x2a4f: 0x00c0, 0x2a50: 0x00c0, 0x2a51: 0x00c0, + 0x2a52: 0x00c0, 0x2a53: 0x00c0, 0x2a54: 0x00c0, 0x2a55: 0x00c0, 0x2a56: 0x00c0, 0x2a57: 0x00c0, + 0x2a58: 0x00c0, 0x2a59: 0x00c0, 0x2a5a: 0x00c0, 0x2a5b: 0x0080, 0x2a5c: 0x0080, 0x2a5d: 0x0080, + 0x2a5e: 0x0080, 0x2a5f: 0x0080, 0x2a60: 0x00c0, 0x2a61: 0x00c0, 0x2a62: 0x00c0, 0x2a63: 0x00c0, + 0x2a64: 0x00c0, 0x2a65: 0x00c8, 0x2a66: 0x00c0, 0x2a67: 0x00c0, + 0x2a70: 0x00c0, 0x2a71: 0x00c0, 0x2a72: 0x00c0, 0x2a73: 0x00c0, 0x2a74: 0x00c0, 0x2a75: 0x00c0, + 0x2a76: 0x00c0, 0x2a77: 0x00c0, 0x2a78: 0x00c0, 0x2a79: 0x00c0, 0x2a7a: 0x00c0, 0x2a7b: 0x00c0, + 0x2a7c: 0x00c0, 0x2a7d: 0x00c0, 0x2a7e: 0x00c0, 0x2a7f: 0x00c0, + // Block 0xaa, offset 0x2a80 + 0x2a80: 0x00c0, 0x2a81: 0x00c0, 0x2a82: 0x00c0, 0x2a83: 0x00c0, 0x2a84: 0x00c0, 0x2a85: 0x00c0, + 0x2a86: 0x00c0, 0x2a87: 0x00c0, 0x2a88: 0x00c0, 0x2a89: 0x00c0, 0x2a8a: 0x00c0, 0x2a8b: 0x00c0, + 0x2a8c: 0x00c0, 0x2a8d: 0x00c0, 0x2a8e: 0x00c0, 0x2a8f: 0x00c0, 0x2a90: 0x00c0, 0x2a91: 0x00c0, + 0x2a92: 0x00c0, 0x2a93: 0x00c0, 0x2a94: 0x00c0, 0x2a95: 0x00c0, 0x2a96: 0x00c0, 0x2a97: 0x00c0, + 0x2a98: 0x00c0, 0x2a99: 0x00c0, 0x2a9a: 0x00c0, 0x2a9b: 0x00c0, 0x2a9c: 0x00c0, 0x2a9d: 0x00c0, + 0x2a9e: 0x00c0, 0x2a9f: 0x00c0, 0x2aa0: 0x00c0, 0x2aa1: 0x00c0, 0x2aa2: 0x00c0, 0x2aa3: 0x00c0, + 0x2aa4: 0x00c0, 0x2aa5: 0x00c3, 0x2aa6: 0x00c0, 0x2aa7: 0x00c0, 0x2aa8: 0x00c3, 0x2aa9: 0x00c0, + 0x2aaa: 0x00c0, 0x2aab: 0x0080, 0x2aac: 0x00c0, 0x2aad: 0x00c6, + 0x2ab0: 0x00c0, 0x2ab1: 0x00c0, 0x2ab2: 0x00c0, 0x2ab3: 0x00c0, 0x2ab4: 0x00c0, 0x2ab5: 0x00c0, + 0x2ab6: 0x00c0, 0x2ab7: 0x00c0, 0x2ab8: 0x00c0, 0x2ab9: 0x00c0, + // Block 0xab, offset 0x2ac0 + 0x2ac0: 0x00c0, 0x2ac1: 0x00c0, 0x2ac2: 0x00c0, 0x2ac3: 0x00c0, 0x2ac4: 0x00c0, 0x2ac5: 0x00c0, + 0x2ac6: 0x00c0, 0x2ac7: 0x00c0, 0x2ac8: 0x00c0, 0x2ac9: 0x00c0, 0x2aca: 0x00c0, 0x2acb: 0x00c0, + 0x2acc: 0x00c0, 0x2acd: 0x00c0, 0x2ace: 0x00c0, 0x2acf: 0x00c0, 0x2ad0: 0x00c0, 0x2ad1: 0x00c0, + 0x2ad2: 0x00c0, 0x2ad3: 0x00c0, 0x2ad4: 0x00c0, 0x2ad5: 0x00c0, 0x2ad6: 0x00c0, 0x2ad7: 0x00c0, + 0x2ad8: 0x00c0, 0x2ad9: 0x00c0, 0x2ada: 0x00c0, 0x2adb: 0x00c0, 0x2adc: 0x00c0, 0x2add: 0x00c0, + 0x2ade: 0x00c0, 0x2adf: 0x00c0, 0x2ae0: 0x00c0, 0x2ae1: 0x00c0, 0x2ae2: 0x00c0, 0x2ae3: 0x00c0, + 0x2af0: 0x0040, 0x2af1: 0x0040, 0x2af2: 0x0040, 0x2af3: 0x0040, 0x2af4: 0x0040, 0x2af5: 0x0040, + 0x2af6: 0x0040, 0x2af7: 0x0040, 0x2af8: 0x0040, 0x2af9: 0x0040, 0x2afa: 0x0040, 0x2afb: 0x0040, + 0x2afc: 0x0040, 0x2afd: 0x0040, 0x2afe: 0x0040, 0x2aff: 0x0040, + // Block 0xac, offset 0x2b00 + 0x2b00: 0x0040, 0x2b01: 0x0040, 0x2b02: 0x0040, 0x2b03: 0x0040, 0x2b04: 0x0040, 0x2b05: 0x0040, + 0x2b06: 0x0040, 0x2b0b: 0x0040, + 0x2b0c: 0x0040, 0x2b0d: 0x0040, 0x2b0e: 0x0040, 0x2b0f: 0x0040, 0x2b10: 0x0040, 0x2b11: 0x0040, + 0x2b12: 0x0040, 0x2b13: 0x0040, 0x2b14: 0x0040, 0x2b15: 0x0040, 0x2b16: 0x0040, 0x2b17: 0x0040, + 0x2b18: 0x0040, 0x2b19: 0x0040, 0x2b1a: 0x0040, 0x2b1b: 0x0040, 0x2b1c: 0x0040, 0x2b1d: 0x0040, + 0x2b1e: 0x0040, 0x2b1f: 0x0040, 0x2b20: 0x0040, 0x2b21: 0x0040, 0x2b22: 0x0040, 0x2b23: 0x0040, + 0x2b24: 0x0040, 0x2b25: 0x0040, 0x2b26: 0x0040, 0x2b27: 0x0040, 0x2b28: 0x0040, 0x2b29: 0x0040, + 0x2b2a: 0x0040, 0x2b2b: 0x0040, 0x2b2c: 0x0040, 0x2b2d: 0x0040, 0x2b2e: 0x0040, 0x2b2f: 0x0040, + 0x2b30: 0x0040, 0x2b31: 0x0040, 0x2b32: 0x0040, 0x2b33: 0x0040, 0x2b34: 0x0040, 0x2b35: 0x0040, + 0x2b36: 0x0040, 0x2b37: 0x0040, 0x2b38: 0x0040, 0x2b39: 0x0040, 0x2b3a: 0x0040, 0x2b3b: 0x0040, + // Block 0xad, offset 0x2b40 + 0x2b40: 0x008c, 0x2b41: 0x008c, 0x2b42: 0x008c, 0x2b43: 0x008c, 0x2b44: 0x008c, 0x2b45: 0x008c, + 0x2b46: 0x008c, 0x2b47: 0x008c, 0x2b48: 0x008c, 0x2b49: 0x008c, 0x2b4a: 0x008c, 0x2b4b: 0x008c, + 0x2b4c: 0x008c, 0x2b4d: 0x008c, 0x2b4e: 0x00cc, 0x2b4f: 0x00cc, 0x2b50: 0x008c, 0x2b51: 0x00cc, + 0x2b52: 0x008c, 0x2b53: 0x00cc, 0x2b54: 0x00cc, 0x2b55: 0x008c, 0x2b56: 0x008c, 0x2b57: 0x008c, + 0x2b58: 0x008c, 0x2b59: 0x008c, 0x2b5a: 0x008c, 0x2b5b: 0x008c, 0x2b5c: 0x008c, 0x2b5d: 0x008c, + 0x2b5e: 0x008c, 0x2b5f: 0x00cc, 0x2b60: 0x008c, 0x2b61: 0x00cc, 0x2b62: 0x008c, 0x2b63: 0x00cc, + 0x2b64: 0x00cc, 0x2b65: 0x008c, 0x2b66: 0x008c, 0x2b67: 0x00cc, 0x2b68: 0x00cc, 0x2b69: 0x00cc, + 0x2b6a: 0x008c, 0x2b6b: 0x008c, 0x2b6c: 0x008c, 0x2b6d: 0x008c, 0x2b6e: 0x008c, 0x2b6f: 0x008c, + 0x2b70: 0x008c, 0x2b71: 0x008c, 0x2b72: 0x008c, 0x2b73: 0x008c, 0x2b74: 0x008c, 0x2b75: 0x008c, + 0x2b76: 0x008c, 0x2b77: 0x008c, 0x2b78: 0x008c, 0x2b79: 0x008c, 0x2b7a: 0x008c, 0x2b7b: 0x008c, + 0x2b7c: 0x008c, 0x2b7d: 0x008c, 0x2b7e: 0x008c, 0x2b7f: 0x008c, + // Block 0xae, offset 0x2b80 + 0x2b80: 0x008c, 0x2b81: 0x008c, 0x2b82: 0x008c, 0x2b83: 0x008c, 0x2b84: 0x008c, 0x2b85: 0x008c, + 0x2b86: 0x008c, 0x2b87: 0x008c, 0x2b88: 0x008c, 0x2b89: 0x008c, 0x2b8a: 0x008c, 0x2b8b: 0x008c, + 0x2b8c: 0x008c, 0x2b8d: 0x008c, 0x2b8e: 0x008c, 0x2b8f: 0x008c, 0x2b90: 0x008c, 0x2b91: 0x008c, + 0x2b92: 0x008c, 0x2b93: 0x008c, 0x2b94: 0x008c, 0x2b95: 0x008c, 0x2b96: 0x008c, 0x2b97: 0x008c, + 0x2b98: 0x008c, 0x2b99: 0x008c, 0x2b9a: 0x008c, 0x2b9b: 0x008c, 0x2b9c: 0x008c, 0x2b9d: 0x008c, + 0x2b9e: 0x008c, 0x2b9f: 0x008c, 0x2ba0: 0x008c, 0x2ba1: 0x008c, 0x2ba2: 0x008c, 0x2ba3: 0x008c, + 0x2ba4: 0x008c, 0x2ba5: 0x008c, 0x2ba6: 0x008c, 0x2ba7: 0x008c, 0x2ba8: 0x008c, 0x2ba9: 0x008c, + 0x2baa: 0x008c, 0x2bab: 0x008c, 0x2bac: 0x008c, 0x2bad: 0x008c, + 0x2bb0: 0x008c, 0x2bb1: 0x008c, 0x2bb2: 0x008c, 0x2bb3: 0x008c, 0x2bb4: 0x008c, 0x2bb5: 0x008c, + 0x2bb6: 0x008c, 0x2bb7: 0x008c, 0x2bb8: 0x008c, 0x2bb9: 0x008c, 0x2bba: 0x008c, 0x2bbb: 0x008c, + 0x2bbc: 0x008c, 0x2bbd: 0x008c, 0x2bbe: 0x008c, 0x2bbf: 0x008c, + // Block 0xaf, offset 0x2bc0 + 0x2bc0: 0x008c, 0x2bc1: 0x008c, 0x2bc2: 0x008c, 0x2bc3: 0x008c, 0x2bc4: 0x008c, 0x2bc5: 0x008c, + 0x2bc6: 0x008c, 0x2bc7: 0x008c, 0x2bc8: 0x008c, 0x2bc9: 0x008c, 0x2bca: 0x008c, 0x2bcb: 0x008c, + 0x2bcc: 0x008c, 0x2bcd: 0x008c, 0x2bce: 0x008c, 0x2bcf: 0x008c, 0x2bd0: 0x008c, 0x2bd1: 0x008c, + 0x2bd2: 0x008c, 0x2bd3: 0x008c, 0x2bd4: 0x008c, 0x2bd5: 0x008c, 0x2bd6: 0x008c, 0x2bd7: 0x008c, + 0x2bd8: 0x008c, 0x2bd9: 0x008c, + // Block 0xb0, offset 0x2c00 + 0x2c00: 0x0080, 0x2c01: 0x0080, 0x2c02: 0x0080, 0x2c03: 0x0080, 0x2c04: 0x0080, 0x2c05: 0x0080, + 0x2c06: 0x0080, + 0x2c13: 0x0080, 0x2c14: 0x0080, 0x2c15: 0x0080, 0x2c16: 0x0080, 0x2c17: 0x0080, + 0x2c1d: 0x008a, + 0x2c1e: 0x00cb, 0x2c1f: 0x008a, 0x2c20: 0x008a, 0x2c21: 0x008a, 0x2c22: 0x008a, 0x2c23: 0x008a, + 0x2c24: 0x008a, 0x2c25: 0x008a, 0x2c26: 0x008a, 0x2c27: 0x008a, 0x2c28: 0x008a, 0x2c29: 0x008a, + 0x2c2a: 0x008a, 0x2c2b: 0x008a, 0x2c2c: 0x008a, 0x2c2d: 0x008a, 0x2c2e: 0x008a, 0x2c2f: 0x008a, + 0x2c30: 0x008a, 0x2c31: 0x008a, 0x2c32: 0x008a, 0x2c33: 0x008a, 0x2c34: 0x008a, 0x2c35: 0x008a, + 0x2c36: 0x008a, 0x2c38: 0x008a, 0x2c39: 0x008a, 0x2c3a: 0x008a, 0x2c3b: 0x008a, + 0x2c3c: 0x008a, 0x2c3e: 0x008a, + // Block 0xb1, offset 0x2c40 + 0x2c40: 0x008a, 0x2c41: 0x008a, 0x2c43: 0x008a, 0x2c44: 0x008a, + 0x2c46: 0x008a, 0x2c47: 0x008a, 0x2c48: 0x008a, 0x2c49: 0x008a, 0x2c4a: 0x008a, 0x2c4b: 0x008a, + 0x2c4c: 0x008a, 0x2c4d: 0x008a, 0x2c4e: 0x008a, 0x2c4f: 0x008a, 0x2c50: 0x0080, 0x2c51: 0x0080, + 0x2c52: 0x0080, 0x2c53: 0x0080, 0x2c54: 0x0080, 0x2c55: 0x0080, 0x2c56: 0x0080, 0x2c57: 0x0080, + 0x2c58: 0x0080, 0x2c59: 0x0080, 0x2c5a: 0x0080, 0x2c5b: 0x0080, 0x2c5c: 0x0080, 0x2c5d: 0x0080, + 0x2c5e: 0x0080, 0x2c5f: 0x0080, 0x2c60: 0x0080, 0x2c61: 0x0080, 0x2c62: 0x0080, 0x2c63: 0x0080, + 0x2c64: 0x0080, 0x2c65: 0x0080, 0x2c66: 0x0080, 0x2c67: 0x0080, 0x2c68: 0x0080, 0x2c69: 0x0080, + 0x2c6a: 0x0080, 0x2c6b: 0x0080, 0x2c6c: 0x0080, 0x2c6d: 0x0080, 0x2c6e: 0x0080, 0x2c6f: 0x0080, + 0x2c70: 0x0080, 0x2c71: 0x0080, 0x2c72: 0x0080, 0x2c73: 0x0080, 0x2c74: 0x0080, 0x2c75: 0x0080, + 0x2c76: 0x0080, 0x2c77: 0x0080, 0x2c78: 0x0080, 0x2c79: 0x0080, 0x2c7a: 0x0080, 0x2c7b: 0x0080, + 0x2c7c: 0x0080, 0x2c7d: 0x0080, 0x2c7e: 0x0080, 0x2c7f: 0x0080, + // Block 0xb2, offset 0x2c80 + 0x2c80: 0x0080, 0x2c81: 0x0080, + 0x2c93: 0x0080, 0x2c94: 0x0080, 0x2c95: 0x0080, 0x2c96: 0x0080, 0x2c97: 0x0080, + 0x2c98: 0x0080, 0x2c99: 0x0080, 0x2c9a: 0x0080, 0x2c9b: 0x0080, 0x2c9c: 0x0080, 0x2c9d: 0x0080, + 0x2c9e: 0x0080, 0x2c9f: 0x0080, 0x2ca0: 0x0080, 0x2ca1: 0x0080, 0x2ca2: 0x0080, 0x2ca3: 0x0080, + 0x2ca4: 0x0080, 0x2ca5: 0x0080, 0x2ca6: 0x0080, 0x2ca7: 0x0080, 0x2ca8: 0x0080, 0x2ca9: 0x0080, + 0x2caa: 0x0080, 0x2cab: 0x0080, 0x2cac: 0x0080, 0x2cad: 0x0080, 0x2cae: 0x0080, 0x2caf: 0x0080, + 0x2cb0: 0x0080, 0x2cb1: 0x0080, 0x2cb2: 0x0080, 0x2cb3: 0x0080, 0x2cb4: 0x0080, 0x2cb5: 0x0080, + 0x2cb6: 0x0080, 0x2cb7: 0x0080, 0x2cb8: 0x0080, 0x2cb9: 0x0080, 0x2cba: 0x0080, 0x2cbb: 0x0080, + 0x2cbc: 0x0080, 0x2cbd: 0x0080, 0x2cbe: 0x0080, 0x2cbf: 0x0080, + // Block 0xb3, offset 0x2cc0 + 0x2cd0: 0x0080, 0x2cd1: 0x0080, + 0x2cd2: 0x0080, 0x2cd3: 0x0080, 0x2cd4: 0x0080, 0x2cd5: 0x0080, 0x2cd6: 0x0080, 0x2cd7: 0x0080, + 0x2cd8: 0x0080, 0x2cd9: 0x0080, 0x2cda: 0x0080, 0x2cdb: 0x0080, 0x2cdc: 0x0080, 0x2cdd: 0x0080, + 0x2cde: 0x0080, 0x2cdf: 0x0080, 0x2ce0: 0x0080, 0x2ce1: 0x0080, 0x2ce2: 0x0080, 0x2ce3: 0x0080, + 0x2ce4: 0x0080, 0x2ce5: 0x0080, 0x2ce6: 0x0080, 0x2ce7: 0x0080, 0x2ce8: 0x0080, 0x2ce9: 0x0080, + 0x2cea: 0x0080, 0x2ceb: 0x0080, 0x2cec: 0x0080, 0x2ced: 0x0080, 0x2cee: 0x0080, 0x2cef: 0x0080, + 0x2cf0: 0x0080, 0x2cf1: 0x0080, 0x2cf2: 0x0080, 0x2cf3: 0x0080, 0x2cf4: 0x0080, 0x2cf5: 0x0080, + 0x2cf6: 0x0080, 0x2cf7: 0x0080, 0x2cf8: 0x0080, 0x2cf9: 0x0080, 0x2cfa: 0x0080, 0x2cfb: 0x0080, + 0x2cfc: 0x0080, 0x2cfd: 0x0080, 0x2cfe: 0x0080, 0x2cff: 0x0080, + // Block 0xb4, offset 0x2d00 + 0x2d00: 0x0080, 0x2d01: 0x0080, 0x2d02: 0x0080, 0x2d03: 0x0080, 0x2d04: 0x0080, 0x2d05: 0x0080, + 0x2d06: 0x0080, 0x2d07: 0x0080, 0x2d08: 0x0080, 0x2d09: 0x0080, 0x2d0a: 0x0080, 0x2d0b: 0x0080, + 0x2d0c: 0x0080, 0x2d0d: 0x0080, 0x2d0e: 0x0080, 0x2d0f: 0x0080, + 0x2d12: 0x0080, 0x2d13: 0x0080, 0x2d14: 0x0080, 0x2d15: 0x0080, 0x2d16: 0x0080, 0x2d17: 0x0080, + 0x2d18: 0x0080, 0x2d19: 0x0080, 0x2d1a: 0x0080, 0x2d1b: 0x0080, 0x2d1c: 0x0080, 0x2d1d: 0x0080, + 0x2d1e: 0x0080, 0x2d1f: 0x0080, 0x2d20: 0x0080, 0x2d21: 0x0080, 0x2d22: 0x0080, 0x2d23: 0x0080, + 0x2d24: 0x0080, 0x2d25: 0x0080, 0x2d26: 0x0080, 0x2d27: 0x0080, 0x2d28: 0x0080, 0x2d29: 0x0080, + 0x2d2a: 0x0080, 0x2d2b: 0x0080, 0x2d2c: 0x0080, 0x2d2d: 0x0080, 0x2d2e: 0x0080, 0x2d2f: 0x0080, + 0x2d30: 0x0080, 0x2d31: 0x0080, 0x2d32: 0x0080, 0x2d33: 0x0080, 0x2d34: 0x0080, 0x2d35: 0x0080, + 0x2d36: 0x0080, 0x2d37: 0x0080, 0x2d38: 0x0080, 0x2d39: 0x0080, 0x2d3a: 0x0080, 0x2d3b: 0x0080, + 0x2d3c: 0x0080, 0x2d3d: 0x0080, 0x2d3e: 0x0080, 0x2d3f: 0x0080, + // Block 0xb5, offset 0x2d40 + 0x2d40: 0x0080, 0x2d41: 0x0080, 0x2d42: 0x0080, 0x2d43: 0x0080, 0x2d44: 0x0080, 0x2d45: 0x0080, + 0x2d46: 0x0080, 0x2d47: 0x0080, + 0x2d70: 0x0080, 0x2d71: 0x0080, 0x2d72: 0x0080, 0x2d73: 0x0080, 0x2d74: 0x0080, 0x2d75: 0x0080, + 0x2d76: 0x0080, 0x2d77: 0x0080, 0x2d78: 0x0080, 0x2d79: 0x0080, 0x2d7a: 0x0080, 0x2d7b: 0x0080, + 0x2d7c: 0x0080, 0x2d7d: 0x0080, + // Block 0xb6, offset 0x2d80 + 0x2d80: 0x0040, 0x2d81: 0x0040, 0x2d82: 0x0040, 0x2d83: 0x0040, 0x2d84: 0x0040, 0x2d85: 0x0040, + 0x2d86: 0x0040, 0x2d87: 0x0040, 0x2d88: 0x0040, 0x2d89: 0x0040, 0x2d8a: 0x0040, 0x2d8b: 0x0040, + 0x2d8c: 0x0040, 0x2d8d: 0x0040, 0x2d8e: 0x0040, 0x2d8f: 0x0040, 0x2d90: 0x0080, 0x2d91: 0x0080, + 0x2d92: 0x0080, 0x2d93: 0x0080, 0x2d94: 0x0080, 0x2d95: 0x0080, 0x2d96: 0x0080, 0x2d97: 0x0080, + 0x2d98: 0x0080, 0x2d99: 0x0080, + 0x2da0: 0x00c3, 0x2da1: 0x00c3, 0x2da2: 0x00c3, 0x2da3: 0x00c3, + 0x2da4: 0x00c3, 0x2da5: 0x00c3, 0x2da6: 0x00c3, 0x2da7: 0x00c3, 0x2da8: 0x00c3, 0x2da9: 0x00c3, + 0x2daa: 0x00c3, 0x2dab: 0x00c3, 0x2dac: 0x00c3, 0x2dad: 0x00c3, 0x2dae: 0x00c3, 0x2daf: 0x00c3, + 0x2db0: 0x0080, 0x2db1: 0x0080, 0x2db2: 0x0080, 0x2db3: 0x0080, 0x2db4: 0x0080, 0x2db5: 0x0080, + 0x2db6: 0x0080, 0x2db7: 0x0080, 0x2db8: 0x0080, 0x2db9: 0x0080, 0x2dba: 0x0080, 0x2dbb: 0x0080, + 0x2dbc: 0x0080, 0x2dbd: 0x0080, 0x2dbe: 0x0080, 0x2dbf: 0x0080, + // Block 0xb7, offset 0x2dc0 + 0x2dc0: 0x0080, 0x2dc1: 0x0080, 0x2dc2: 0x0080, 0x2dc3: 0x0080, 0x2dc4: 0x0080, 0x2dc5: 0x0080, + 0x2dc6: 0x0080, 0x2dc7: 0x0080, 0x2dc8: 0x0080, 0x2dc9: 0x0080, 0x2dca: 0x0080, 0x2dcb: 0x0080, + 0x2dcc: 0x0080, 0x2dcd: 0x0080, 0x2dce: 0x0080, 0x2dcf: 0x0080, 0x2dd0: 0x0080, 0x2dd1: 0x0080, + 0x2dd2: 0x0080, 0x2dd4: 0x0080, 0x2dd5: 0x0080, 0x2dd6: 0x0080, 0x2dd7: 0x0080, + 0x2dd8: 0x0080, 0x2dd9: 0x0080, 0x2dda: 0x0080, 0x2ddb: 0x0080, 0x2ddc: 0x0080, 0x2ddd: 0x0080, + 0x2dde: 0x0080, 0x2ddf: 0x0080, 0x2de0: 0x0080, 0x2de1: 0x0080, 0x2de2: 0x0080, 0x2de3: 0x0080, + 0x2de4: 0x0080, 0x2de5: 0x0080, 0x2de6: 0x0080, 0x2de8: 0x0080, 0x2de9: 0x0080, + 0x2dea: 0x0080, 0x2deb: 0x0080, + 0x2df0: 0x0080, 0x2df1: 0x0080, 0x2df2: 0x0080, 0x2df3: 0x00c0, 0x2df4: 0x0080, + 0x2df6: 0x0080, 0x2df7: 0x0080, 0x2df8: 0x0080, 0x2df9: 0x0080, 0x2dfa: 0x0080, 0x2dfb: 0x0080, + 0x2dfc: 0x0080, 0x2dfd: 0x0080, 0x2dfe: 0x0080, 0x2dff: 0x0080, + // Block 0xb8, offset 0x2e00 + 0x2e00: 0x0080, 0x2e01: 0x0080, 0x2e02: 0x0080, 0x2e03: 0x0080, 0x2e04: 0x0080, 0x2e05: 0x0080, + 0x2e06: 0x0080, 0x2e07: 0x0080, 0x2e08: 0x0080, 0x2e09: 0x0080, 0x2e0a: 0x0080, 0x2e0b: 0x0080, + 0x2e0c: 0x0080, 0x2e0d: 0x0080, 0x2e0e: 0x0080, 0x2e0f: 0x0080, 0x2e10: 0x0080, 0x2e11: 0x0080, + 0x2e12: 0x0080, 0x2e13: 0x0080, 0x2e14: 0x0080, 0x2e15: 0x0080, 0x2e16: 0x0080, 0x2e17: 0x0080, + 0x2e18: 0x0080, 0x2e19: 0x0080, 0x2e1a: 0x0080, 0x2e1b: 0x0080, 0x2e1c: 0x0080, 0x2e1d: 0x0080, + 0x2e1e: 0x0080, 0x2e1f: 0x0080, 0x2e20: 0x0080, 0x2e21: 0x0080, 0x2e22: 0x0080, 0x2e23: 0x0080, + 0x2e24: 0x0080, 0x2e25: 0x0080, 0x2e26: 0x0080, 0x2e27: 0x0080, 0x2e28: 0x0080, 0x2e29: 0x0080, + 0x2e2a: 0x0080, 0x2e2b: 0x0080, 0x2e2c: 0x0080, 0x2e2d: 0x0080, 0x2e2e: 0x0080, 0x2e2f: 0x0080, + 0x2e30: 0x0080, 0x2e31: 0x0080, 0x2e32: 0x0080, 0x2e33: 0x0080, 0x2e34: 0x0080, 0x2e35: 0x0080, + 0x2e36: 0x0080, 0x2e37: 0x0080, 0x2e38: 0x0080, 0x2e39: 0x0080, 0x2e3a: 0x0080, 0x2e3b: 0x0080, + 0x2e3c: 0x0080, 0x2e3f: 0x0040, + // Block 0xb9, offset 0x2e40 + 0x2e41: 0x0080, 0x2e42: 0x0080, 0x2e43: 0x0080, 0x2e44: 0x0080, 0x2e45: 0x0080, + 0x2e46: 0x0080, 0x2e47: 0x0080, 0x2e48: 0x0080, 0x2e49: 0x0080, 0x2e4a: 0x0080, 0x2e4b: 0x0080, + 0x2e4c: 0x0080, 0x2e4d: 0x0080, 0x2e4e: 0x0080, 0x2e4f: 0x0080, 0x2e50: 0x0080, 0x2e51: 0x0080, + 0x2e52: 0x0080, 0x2e53: 0x0080, 0x2e54: 0x0080, 0x2e55: 0x0080, 0x2e56: 0x0080, 0x2e57: 0x0080, + 0x2e58: 0x0080, 0x2e59: 0x0080, 0x2e5a: 0x0080, 0x2e5b: 0x0080, 0x2e5c: 0x0080, 0x2e5d: 0x0080, + 0x2e5e: 0x0080, 0x2e5f: 0x0080, 0x2e60: 0x0080, 0x2e61: 0x0080, 0x2e62: 0x0080, 0x2e63: 0x0080, + 0x2e64: 0x0080, 0x2e65: 0x0080, 0x2e66: 0x0080, 0x2e67: 0x0080, 0x2e68: 0x0080, 0x2e69: 0x0080, + 0x2e6a: 0x0080, 0x2e6b: 0x0080, 0x2e6c: 0x0080, 0x2e6d: 0x0080, 0x2e6e: 0x0080, 0x2e6f: 0x0080, + 0x2e70: 0x0080, 0x2e71: 0x0080, 0x2e72: 0x0080, 0x2e73: 0x0080, 0x2e74: 0x0080, 0x2e75: 0x0080, + 0x2e76: 0x0080, 0x2e77: 0x0080, 0x2e78: 0x0080, 0x2e79: 0x0080, 0x2e7a: 0x0080, 0x2e7b: 0x0080, + 0x2e7c: 0x0080, 0x2e7d: 0x0080, 0x2e7e: 0x0080, 0x2e7f: 0x0080, + // Block 0xba, offset 0x2e80 + 0x2e80: 0x0080, 0x2e81: 0x0080, 0x2e82: 0x0080, 0x2e83: 0x0080, 0x2e84: 0x0080, 0x2e85: 0x0080, + 0x2e86: 0x0080, 0x2e87: 0x0080, 0x2e88: 0x0080, 0x2e89: 0x0080, 0x2e8a: 0x0080, 0x2e8b: 0x0080, + 0x2e8c: 0x0080, 0x2e8d: 0x0080, 0x2e8e: 0x0080, 0x2e8f: 0x0080, 0x2e90: 0x0080, 0x2e91: 0x0080, + 0x2e92: 0x0080, 0x2e93: 0x0080, 0x2e94: 0x0080, 0x2e95: 0x0080, 0x2e96: 0x0080, 0x2e97: 0x0080, + 0x2e98: 0x0080, 0x2e99: 0x0080, 0x2e9a: 0x0080, 0x2e9b: 0x0080, 0x2e9c: 0x0080, 0x2e9d: 0x0080, + 0x2e9e: 0x0080, 0x2e9f: 0x0080, 0x2ea0: 0x0080, 0x2ea1: 0x0080, 0x2ea2: 0x0080, 0x2ea3: 0x0080, + 0x2ea4: 0x0080, 0x2ea5: 0x0080, 0x2ea6: 0x008c, 0x2ea7: 0x008c, 0x2ea8: 0x008c, 0x2ea9: 0x008c, + 0x2eaa: 0x008c, 0x2eab: 0x008c, 0x2eac: 0x008c, 0x2ead: 0x008c, 0x2eae: 0x008c, 0x2eaf: 0x008c, + 0x2eb0: 0x0080, 0x2eb1: 0x008c, 0x2eb2: 0x008c, 0x2eb3: 0x008c, 0x2eb4: 0x008c, 0x2eb5: 0x008c, + 0x2eb6: 0x008c, 0x2eb7: 0x008c, 0x2eb8: 0x008c, 0x2eb9: 0x008c, 0x2eba: 0x008c, 0x2ebb: 0x008c, + 0x2ebc: 0x008c, 0x2ebd: 0x008c, 0x2ebe: 0x008c, 0x2ebf: 0x008c, + // Block 0xbb, offset 0x2ec0 + 0x2ec0: 0x008c, 0x2ec1: 0x008c, 0x2ec2: 0x008c, 0x2ec3: 0x008c, 0x2ec4: 0x008c, 0x2ec5: 0x008c, + 0x2ec6: 0x008c, 0x2ec7: 0x008c, 0x2ec8: 0x008c, 0x2ec9: 0x008c, 0x2eca: 0x008c, 0x2ecb: 0x008c, + 0x2ecc: 0x008c, 0x2ecd: 0x008c, 0x2ece: 0x008c, 0x2ecf: 0x008c, 0x2ed0: 0x008c, 0x2ed1: 0x008c, + 0x2ed2: 0x008c, 0x2ed3: 0x008c, 0x2ed4: 0x008c, 0x2ed5: 0x008c, 0x2ed6: 0x008c, 0x2ed7: 0x008c, + 0x2ed8: 0x008c, 0x2ed9: 0x008c, 0x2eda: 0x008c, 0x2edb: 0x008c, 0x2edc: 0x008c, 0x2edd: 0x008c, + 0x2ede: 0x0080, 0x2edf: 0x0080, 0x2ee0: 0x0040, 0x2ee1: 0x0080, 0x2ee2: 0x0080, 0x2ee3: 0x0080, + 0x2ee4: 0x0080, 0x2ee5: 0x0080, 0x2ee6: 0x0080, 0x2ee7: 0x0080, 0x2ee8: 0x0080, 0x2ee9: 0x0080, + 0x2eea: 0x0080, 0x2eeb: 0x0080, 0x2eec: 0x0080, 0x2eed: 0x0080, 0x2eee: 0x0080, 0x2eef: 0x0080, + 0x2ef0: 0x0080, 0x2ef1: 0x0080, 0x2ef2: 0x0080, 0x2ef3: 0x0080, 0x2ef4: 0x0080, 0x2ef5: 0x0080, + 0x2ef6: 0x0080, 0x2ef7: 0x0080, 0x2ef8: 0x0080, 0x2ef9: 0x0080, 0x2efa: 0x0080, 0x2efb: 0x0080, + 0x2efc: 0x0080, 0x2efd: 0x0080, 0x2efe: 0x0080, + // Block 0xbc, offset 0x2f00 + 0x2f02: 0x0080, 0x2f03: 0x0080, 0x2f04: 0x0080, 0x2f05: 0x0080, + 0x2f06: 0x0080, 0x2f07: 0x0080, 0x2f0a: 0x0080, 0x2f0b: 0x0080, + 0x2f0c: 0x0080, 0x2f0d: 0x0080, 0x2f0e: 0x0080, 0x2f0f: 0x0080, + 0x2f12: 0x0080, 0x2f13: 0x0080, 0x2f14: 0x0080, 0x2f15: 0x0080, 0x2f16: 0x0080, 0x2f17: 0x0080, + 0x2f1a: 0x0080, 0x2f1b: 0x0080, 0x2f1c: 0x0080, + 0x2f20: 0x0080, 0x2f21: 0x0080, 0x2f22: 0x0080, 0x2f23: 0x0080, + 0x2f24: 0x0080, 0x2f25: 0x0080, 0x2f26: 0x0080, 0x2f28: 0x0080, 0x2f29: 0x0080, + 0x2f2a: 0x0080, 0x2f2b: 0x0080, 0x2f2c: 0x0080, 0x2f2d: 0x0080, 0x2f2e: 0x0080, + 0x2f39: 0x0040, 0x2f3a: 0x0040, 0x2f3b: 0x0040, + 0x2f3c: 0x0080, 0x2f3d: 0x0080, + // Block 0xbd, offset 0x2f40 + 0x2f40: 0x00c0, 0x2f41: 0x00c0, 0x2f42: 0x00c0, 0x2f43: 0x00c0, 0x2f44: 0x00c0, 0x2f45: 0x00c0, + 0x2f46: 0x00c0, 0x2f47: 0x00c0, 0x2f48: 0x00c0, 0x2f49: 0x00c0, 0x2f4a: 0x00c0, 0x2f4b: 0x00c0, + 0x2f4d: 0x00c0, 0x2f4e: 0x00c0, 0x2f4f: 0x00c0, 0x2f50: 0x00c0, 0x2f51: 0x00c0, + 0x2f52: 0x00c0, 0x2f53: 0x00c0, 0x2f54: 0x00c0, 0x2f55: 0x00c0, 0x2f56: 0x00c0, 0x2f57: 0x00c0, + 0x2f58: 0x00c0, 0x2f59: 0x00c0, 0x2f5a: 0x00c0, 0x2f5b: 0x00c0, 0x2f5c: 0x00c0, 0x2f5d: 0x00c0, + 0x2f5e: 0x00c0, 0x2f5f: 0x00c0, 0x2f60: 0x00c0, 0x2f61: 0x00c0, 0x2f62: 0x00c0, 0x2f63: 0x00c0, + 0x2f64: 0x00c0, 0x2f65: 0x00c0, 0x2f66: 0x00c0, 0x2f68: 0x00c0, 0x2f69: 0x00c0, + 0x2f6a: 0x00c0, 0x2f6b: 0x00c0, 0x2f6c: 0x00c0, 0x2f6d: 0x00c0, 0x2f6e: 0x00c0, 0x2f6f: 0x00c0, + 0x2f70: 0x00c0, 0x2f71: 0x00c0, 0x2f72: 0x00c0, 0x2f73: 0x00c0, 0x2f74: 0x00c0, 0x2f75: 0x00c0, + 0x2f76: 0x00c0, 0x2f77: 0x00c0, 0x2f78: 0x00c0, 0x2f79: 0x00c0, 0x2f7a: 0x00c0, + 0x2f7c: 0x00c0, 0x2f7d: 0x00c0, 0x2f7f: 0x00c0, + // Block 0xbe, offset 0x2f80 + 0x2f80: 0x00c0, 0x2f81: 0x00c0, 0x2f82: 0x00c0, 0x2f83: 0x00c0, 0x2f84: 0x00c0, 0x2f85: 0x00c0, + 0x2f86: 0x00c0, 0x2f87: 0x00c0, 0x2f88: 0x00c0, 0x2f89: 0x00c0, 0x2f8a: 0x00c0, 0x2f8b: 0x00c0, + 0x2f8c: 0x00c0, 0x2f8d: 0x00c0, 0x2f90: 0x00c0, 0x2f91: 0x00c0, + 0x2f92: 0x00c0, 0x2f93: 0x00c0, 0x2f94: 0x00c0, 0x2f95: 0x00c0, 0x2f96: 0x00c0, 0x2f97: 0x00c0, + 0x2f98: 0x00c0, 0x2f99: 0x00c0, 0x2f9a: 0x00c0, 0x2f9b: 0x00c0, 0x2f9c: 0x00c0, 0x2f9d: 0x00c0, + // Block 0xbf, offset 0x2fc0 + 0x2fc0: 0x00c0, 0x2fc1: 0x00c0, 0x2fc2: 0x00c0, 0x2fc3: 0x00c0, 0x2fc4: 0x00c0, 0x2fc5: 0x00c0, + 0x2fc6: 0x00c0, 0x2fc7: 0x00c0, 0x2fc8: 0x00c0, 0x2fc9: 0x00c0, 0x2fca: 0x00c0, 0x2fcb: 0x00c0, + 0x2fcc: 0x00c0, 0x2fcd: 0x00c0, 0x2fce: 0x00c0, 0x2fcf: 0x00c0, 0x2fd0: 0x00c0, 0x2fd1: 0x00c0, + 0x2fd2: 0x00c0, 0x2fd3: 0x00c0, 0x2fd4: 0x00c0, 0x2fd5: 0x00c0, 0x2fd6: 0x00c0, 0x2fd7: 0x00c0, + 0x2fd8: 0x00c0, 0x2fd9: 0x00c0, 0x2fda: 0x00c0, 0x2fdb: 0x00c0, 0x2fdc: 0x00c0, 0x2fdd: 0x00c0, + 0x2fde: 0x00c0, 0x2fdf: 0x00c0, 0x2fe0: 0x00c0, 0x2fe1: 0x00c0, 0x2fe2: 0x00c0, 0x2fe3: 0x00c0, + 0x2fe4: 0x00c0, 0x2fe5: 0x00c0, 0x2fe6: 0x00c0, 0x2fe7: 0x00c0, 0x2fe8: 0x00c0, 0x2fe9: 0x00c0, + 0x2fea: 0x00c0, 0x2feb: 0x00c0, 0x2fec: 0x00c0, 0x2fed: 0x00c0, 0x2fee: 0x00c0, 0x2fef: 0x00c0, + 0x2ff0: 0x00c0, 0x2ff1: 0x00c0, 0x2ff2: 0x00c0, 0x2ff3: 0x00c0, 0x2ff4: 0x00c0, 0x2ff5: 0x00c0, + 0x2ff6: 0x00c0, 0x2ff7: 0x00c0, 0x2ff8: 0x00c0, 0x2ff9: 0x00c0, 0x2ffa: 0x00c0, + // Block 0xc0, offset 0x3000 + 0x3000: 0x0080, 0x3001: 0x0080, 0x3002: 0x0080, + 0x3007: 0x0080, 0x3008: 0x0080, 0x3009: 0x0080, 0x300a: 0x0080, 0x300b: 0x0080, + 0x300c: 0x0080, 0x300d: 0x0080, 0x300e: 0x0080, 0x300f: 0x0080, 0x3010: 0x0080, 0x3011: 0x0080, + 0x3012: 0x0080, 0x3013: 0x0080, 0x3014: 0x0080, 0x3015: 0x0080, 0x3016: 0x0080, 0x3017: 0x0080, + 0x3018: 0x0080, 0x3019: 0x0080, 0x301a: 0x0080, 0x301b: 0x0080, 0x301c: 0x0080, 0x301d: 0x0080, + 0x301e: 0x0080, 0x301f: 0x0080, 0x3020: 0x0080, 0x3021: 0x0080, 0x3022: 0x0080, 0x3023: 0x0080, + 0x3024: 0x0080, 0x3025: 0x0080, 0x3026: 0x0080, 0x3027: 0x0080, 0x3028: 0x0080, 0x3029: 0x0080, + 0x302a: 0x0080, 0x302b: 0x0080, 0x302c: 0x0080, 0x302d: 0x0080, 0x302e: 0x0080, 0x302f: 0x0080, + 0x3030: 0x0080, 0x3031: 0x0080, 0x3032: 0x0080, 0x3033: 0x0080, + 0x3037: 0x0080, 0x3038: 0x0080, 0x3039: 0x0080, 0x303a: 0x0080, 0x303b: 0x0080, + 0x303c: 0x0080, 0x303d: 0x0080, 0x303e: 0x0080, 0x303f: 0x0080, + // Block 0xc1, offset 0x3040 + 0x3040: 0x0088, 0x3041: 0x0088, 0x3042: 0x0088, 0x3043: 0x0088, 0x3044: 0x0088, 0x3045: 0x0088, + 0x3046: 0x0088, 0x3047: 0x0088, 0x3048: 0x0088, 0x3049: 0x0088, 0x304a: 0x0088, 0x304b: 0x0088, + 0x304c: 0x0088, 0x304d: 0x0088, 0x304e: 0x0088, 0x304f: 0x0088, 0x3050: 0x0088, 0x3051: 0x0088, + 0x3052: 0x0088, 0x3053: 0x0088, 0x3054: 0x0088, 0x3055: 0x0088, 0x3056: 0x0088, 0x3057: 0x0088, + 0x3058: 0x0088, 0x3059: 0x0088, 0x305a: 0x0088, 0x305b: 0x0088, 0x305c: 0x0088, 0x305d: 0x0088, + 0x305e: 0x0088, 0x305f: 0x0088, 0x3060: 0x0088, 0x3061: 0x0088, 0x3062: 0x0088, 0x3063: 0x0088, + 0x3064: 0x0088, 0x3065: 0x0088, 0x3066: 0x0088, 0x3067: 0x0088, 0x3068: 0x0088, 0x3069: 0x0088, + 0x306a: 0x0088, 0x306b: 0x0088, 0x306c: 0x0088, 0x306d: 0x0088, 0x306e: 0x0088, 0x306f: 0x0088, + 0x3070: 0x0088, 0x3071: 0x0088, 0x3072: 0x0088, 0x3073: 0x0088, 0x3074: 0x0088, 0x3075: 0x0088, + 0x3076: 0x0088, 0x3077: 0x0088, 0x3078: 0x0088, 0x3079: 0x0088, 0x307a: 0x0088, 0x307b: 0x0088, + 0x307c: 0x0088, 0x307d: 0x0088, 0x307e: 0x0088, 0x307f: 0x0088, + // Block 0xc2, offset 0x3080 + 0x3080: 0x0088, 0x3081: 0x0088, 0x3082: 0x0088, 0x3083: 0x0088, 0x3084: 0x0088, 0x3085: 0x0088, + 0x3086: 0x0088, 0x3087: 0x0088, 0x3088: 0x0088, 0x3089: 0x0088, 0x308a: 0x0088, 0x308b: 0x0088, + 0x308c: 0x0088, 0x308d: 0x0088, 0x308e: 0x0088, 0x3090: 0x0080, 0x3091: 0x0080, + 0x3092: 0x0080, 0x3093: 0x0080, 0x3094: 0x0080, 0x3095: 0x0080, 0x3096: 0x0080, 0x3097: 0x0080, + 0x3098: 0x0080, 0x3099: 0x0080, 0x309a: 0x0080, 0x309b: 0x0080, + 0x30a0: 0x0088, + // Block 0xc3, offset 0x30c0 + 0x30d0: 0x0080, 0x30d1: 0x0080, + 0x30d2: 0x0080, 0x30d3: 0x0080, 0x30d4: 0x0080, 0x30d5: 0x0080, 0x30d6: 0x0080, 0x30d7: 0x0080, + 0x30d8: 0x0080, 0x30d9: 0x0080, 0x30da: 0x0080, 0x30db: 0x0080, 0x30dc: 0x0080, 0x30dd: 0x0080, + 0x30de: 0x0080, 0x30df: 0x0080, 0x30e0: 0x0080, 0x30e1: 0x0080, 0x30e2: 0x0080, 0x30e3: 0x0080, + 0x30e4: 0x0080, 0x30e5: 0x0080, 0x30e6: 0x0080, 0x30e7: 0x0080, 0x30e8: 0x0080, 0x30e9: 0x0080, + 0x30ea: 0x0080, 0x30eb: 0x0080, 0x30ec: 0x0080, 0x30ed: 0x0080, 0x30ee: 0x0080, 0x30ef: 0x0080, + 0x30f0: 0x0080, 0x30f1: 0x0080, 0x30f2: 0x0080, 0x30f3: 0x0080, 0x30f4: 0x0080, 0x30f5: 0x0080, + 0x30f6: 0x0080, 0x30f7: 0x0080, 0x30f8: 0x0080, 0x30f9: 0x0080, 0x30fa: 0x0080, 0x30fb: 0x0080, + 0x30fc: 0x0080, 0x30fd: 0x00c3, + // Block 0xc4, offset 0x3100 + 0x3100: 0x00c0, 0x3101: 0x00c0, 0x3102: 0x00c0, 0x3103: 0x00c0, 0x3104: 0x00c0, 0x3105: 0x00c0, + 0x3106: 0x00c0, 0x3107: 0x00c0, 0x3108: 0x00c0, 0x3109: 0x00c0, 0x310a: 0x00c0, 0x310b: 0x00c0, + 0x310c: 0x00c0, 0x310d: 0x00c0, 0x310e: 0x00c0, 0x310f: 0x00c0, 0x3110: 0x00c0, 0x3111: 0x00c0, + 0x3112: 0x00c0, 0x3113: 0x00c0, 0x3114: 0x00c0, 0x3115: 0x00c0, 0x3116: 0x00c0, 0x3117: 0x00c0, + 0x3118: 0x00c0, 0x3119: 0x00c0, 0x311a: 0x00c0, 0x311b: 0x00c0, 0x311c: 0x00c0, + 0x3120: 0x00c0, 0x3121: 0x00c0, 0x3122: 0x00c0, 0x3123: 0x00c0, + 0x3124: 0x00c0, 0x3125: 0x00c0, 0x3126: 0x00c0, 0x3127: 0x00c0, 0x3128: 0x00c0, 0x3129: 0x00c0, + 0x312a: 0x00c0, 0x312b: 0x00c0, 0x312c: 0x00c0, 0x312d: 0x00c0, 0x312e: 0x00c0, 0x312f: 0x00c0, + 0x3130: 0x00c0, 0x3131: 0x00c0, 0x3132: 0x00c0, 0x3133: 0x00c0, 0x3134: 0x00c0, 0x3135: 0x00c0, + 0x3136: 0x00c0, 0x3137: 0x00c0, 0x3138: 0x00c0, 0x3139: 0x00c0, 0x313a: 0x00c0, 0x313b: 0x00c0, + 0x313c: 0x00c0, 0x313d: 0x00c0, 0x313e: 0x00c0, 0x313f: 0x00c0, + // Block 0xc5, offset 0x3140 + 0x3140: 0x00c0, 0x3141: 0x00c0, 0x3142: 0x00c0, 0x3143: 0x00c0, 0x3144: 0x00c0, 0x3145: 0x00c0, + 0x3146: 0x00c0, 0x3147: 0x00c0, 0x3148: 0x00c0, 0x3149: 0x00c0, 0x314a: 0x00c0, 0x314b: 0x00c0, + 0x314c: 0x00c0, 0x314d: 0x00c0, 0x314e: 0x00c0, 0x314f: 0x00c0, 0x3150: 0x00c0, + 0x3160: 0x00c3, 0x3161: 0x0080, 0x3162: 0x0080, 0x3163: 0x0080, + 0x3164: 0x0080, 0x3165: 0x0080, 0x3166: 0x0080, 0x3167: 0x0080, 0x3168: 0x0080, 0x3169: 0x0080, + 0x316a: 0x0080, 0x316b: 0x0080, 0x316c: 0x0080, 0x316d: 0x0080, 0x316e: 0x0080, 0x316f: 0x0080, + 0x3170: 0x0080, 0x3171: 0x0080, 0x3172: 0x0080, 0x3173: 0x0080, 0x3174: 0x0080, 0x3175: 0x0080, + 0x3176: 0x0080, 0x3177: 0x0080, 0x3178: 0x0080, 0x3179: 0x0080, 0x317a: 0x0080, 0x317b: 0x0080, + // Block 0xc6, offset 0x3180 + 0x3180: 0x00c0, 0x3181: 0x00c0, 0x3182: 0x00c0, 0x3183: 0x00c0, 0x3184: 0x00c0, 0x3185: 0x00c0, + 0x3186: 0x00c0, 0x3187: 0x00c0, 0x3188: 0x00c0, 0x3189: 0x00c0, 0x318a: 0x00c0, 0x318b: 0x00c0, + 0x318c: 0x00c0, 0x318d: 0x00c0, 0x318e: 0x00c0, 0x318f: 0x00c0, 0x3190: 0x00c0, 0x3191: 0x00c0, + 0x3192: 0x00c0, 0x3193: 0x00c0, 0x3194: 0x00c0, 0x3195: 0x00c0, 0x3196: 0x00c0, 0x3197: 0x00c0, + 0x3198: 0x00c0, 0x3199: 0x00c0, 0x319a: 0x00c0, 0x319b: 0x00c0, 0x319c: 0x00c0, 0x319d: 0x00c0, + 0x319e: 0x00c0, 0x319f: 0x00c0, 0x31a0: 0x0080, 0x31a1: 0x0080, 0x31a2: 0x0080, 0x31a3: 0x0080, + 0x31ad: 0x00c0, 0x31ae: 0x00c0, 0x31af: 0x00c0, + 0x31b0: 0x00c0, 0x31b1: 0x00c0, 0x31b2: 0x00c0, 0x31b3: 0x00c0, 0x31b4: 0x00c0, 0x31b5: 0x00c0, + 0x31b6: 0x00c0, 0x31b7: 0x00c0, 0x31b8: 0x00c0, 0x31b9: 0x00c0, 0x31ba: 0x00c0, 0x31bb: 0x00c0, + 0x31bc: 0x00c0, 0x31bd: 0x00c0, 0x31be: 0x00c0, 0x31bf: 0x00c0, + // Block 0xc7, offset 0x31c0 + 0x31c0: 0x00c0, 0x31c1: 0x0080, 0x31c2: 0x00c0, 0x31c3: 0x00c0, 0x31c4: 0x00c0, 0x31c5: 0x00c0, + 0x31c6: 0x00c0, 0x31c7: 0x00c0, 0x31c8: 0x00c0, 0x31c9: 0x00c0, 0x31ca: 0x0080, + 0x31d0: 0x00c0, 0x31d1: 0x00c0, + 0x31d2: 0x00c0, 0x31d3: 0x00c0, 0x31d4: 0x00c0, 0x31d5: 0x00c0, 0x31d6: 0x00c0, 0x31d7: 0x00c0, + 0x31d8: 0x00c0, 0x31d9: 0x00c0, 0x31da: 0x00c0, 0x31db: 0x00c0, 0x31dc: 0x00c0, 0x31dd: 0x00c0, + 0x31de: 0x00c0, 0x31df: 0x00c0, 0x31e0: 0x00c0, 0x31e1: 0x00c0, 0x31e2: 0x00c0, 0x31e3: 0x00c0, + 0x31e4: 0x00c0, 0x31e5: 0x00c0, 0x31e6: 0x00c0, 0x31e7: 0x00c0, 0x31e8: 0x00c0, 0x31e9: 0x00c0, + 0x31ea: 0x00c0, 0x31eb: 0x00c0, 0x31ec: 0x00c0, 0x31ed: 0x00c0, 0x31ee: 0x00c0, 0x31ef: 0x00c0, + 0x31f0: 0x00c0, 0x31f1: 0x00c0, 0x31f2: 0x00c0, 0x31f3: 0x00c0, 0x31f4: 0x00c0, 0x31f5: 0x00c0, + 0x31f6: 0x00c3, 0x31f7: 0x00c3, 0x31f8: 0x00c3, 0x31f9: 0x00c3, 0x31fa: 0x00c3, + // Block 0xc8, offset 0x3200 + 0x3200: 0x00c0, 0x3201: 0x00c0, 0x3202: 0x00c0, 0x3203: 0x00c0, 0x3204: 0x00c0, 0x3205: 0x00c0, + 0x3206: 0x00c0, 0x3207: 0x00c0, 0x3208: 0x00c0, 0x3209: 0x00c0, 0x320a: 0x00c0, 0x320b: 0x00c0, + 0x320c: 0x00c0, 0x320d: 0x00c0, 0x320e: 0x00c0, 0x320f: 0x00c0, 0x3210: 0x00c0, 0x3211: 0x00c0, + 0x3212: 0x00c0, 0x3213: 0x00c0, 0x3214: 0x00c0, 0x3215: 0x00c0, 0x3216: 0x00c0, 0x3217: 0x00c0, + 0x3218: 0x00c0, 0x3219: 0x00c0, 0x321a: 0x00c0, 0x321b: 0x00c0, 0x321c: 0x00c0, 0x321d: 0x00c0, + 0x321f: 0x0080, 0x3220: 0x00c0, 0x3221: 0x00c0, 0x3222: 0x00c0, 0x3223: 0x00c0, + 0x3224: 0x00c0, 0x3225: 0x00c0, 0x3226: 0x00c0, 0x3227: 0x00c0, 0x3228: 0x00c0, 0x3229: 0x00c0, + 0x322a: 0x00c0, 0x322b: 0x00c0, 0x322c: 0x00c0, 0x322d: 0x00c0, 0x322e: 0x00c0, 0x322f: 0x00c0, + 0x3230: 0x00c0, 0x3231: 0x00c0, 0x3232: 0x00c0, 0x3233: 0x00c0, 0x3234: 0x00c0, 0x3235: 0x00c0, + 0x3236: 0x00c0, 0x3237: 0x00c0, 0x3238: 0x00c0, 0x3239: 0x00c0, 0x323a: 0x00c0, 0x323b: 0x00c0, + 0x323c: 0x00c0, 0x323d: 0x00c0, 0x323e: 0x00c0, 0x323f: 0x00c0, + // Block 0xc9, offset 0x3240 + 0x3240: 0x00c0, 0x3241: 0x00c0, 0x3242: 0x00c0, 0x3243: 0x00c0, + 0x3248: 0x00c0, 0x3249: 0x00c0, 0x324a: 0x00c0, 0x324b: 0x00c0, + 0x324c: 0x00c0, 0x324d: 0x00c0, 0x324e: 0x00c0, 0x324f: 0x00c0, 0x3250: 0x0080, 0x3251: 0x0080, + 0x3252: 0x0080, 0x3253: 0x0080, 0x3254: 0x0080, 0x3255: 0x0080, + // Block 0xca, offset 0x3280 + 0x3280: 0x00c0, 0x3281: 0x00c0, 0x3282: 0x00c0, 0x3283: 0x00c0, 0x3284: 0x00c0, 0x3285: 0x00c0, + 0x3286: 0x00c0, 0x3287: 0x00c0, 0x3288: 0x00c0, 0x3289: 0x00c0, 0x328a: 0x00c0, 0x328b: 0x00c0, + 0x328c: 0x00c0, 0x328d: 0x00c0, 0x328e: 0x00c0, 0x328f: 0x00c0, 0x3290: 0x00c0, 0x3291: 0x00c0, + 0x3292: 0x00c0, 0x3293: 0x00c0, 0x3294: 0x00c0, 0x3295: 0x00c0, 0x3296: 0x00c0, 0x3297: 0x00c0, + 0x3298: 0x00c0, 0x3299: 0x00c0, 0x329a: 0x00c0, 0x329b: 0x00c0, 0x329c: 0x00c0, 0x329d: 0x00c0, + 0x32a0: 0x00c0, 0x32a1: 0x00c0, 0x32a2: 0x00c0, 0x32a3: 0x00c0, + 0x32a4: 0x00c0, 0x32a5: 0x00c0, 0x32a6: 0x00c0, 0x32a7: 0x00c0, 0x32a8: 0x00c0, 0x32a9: 0x00c0, + 0x32b0: 0x00c0, 0x32b1: 0x00c0, 0x32b2: 0x00c0, 0x32b3: 0x00c0, 0x32b4: 0x00c0, 0x32b5: 0x00c0, + 0x32b6: 0x00c0, 0x32b7: 0x00c0, 0x32b8: 0x00c0, 0x32b9: 0x00c0, 0x32ba: 0x00c0, 0x32bb: 0x00c0, + 0x32bc: 0x00c0, 0x32bd: 0x00c0, 0x32be: 0x00c0, 0x32bf: 0x00c0, + // Block 0xcb, offset 0x32c0 + 0x32c0: 0x00c0, 0x32c1: 0x00c0, 0x32c2: 0x00c0, 0x32c3: 0x00c0, 0x32c4: 0x00c0, 0x32c5: 0x00c0, + 0x32c6: 0x00c0, 0x32c7: 0x00c0, 0x32c8: 0x00c0, 0x32c9: 0x00c0, 0x32ca: 0x00c0, 0x32cb: 0x00c0, + 0x32cc: 0x00c0, 0x32cd: 0x00c0, 0x32ce: 0x00c0, 0x32cf: 0x00c0, 0x32d0: 0x00c0, 0x32d1: 0x00c0, + 0x32d2: 0x00c0, 0x32d3: 0x00c0, + 0x32d8: 0x00c0, 0x32d9: 0x00c0, 0x32da: 0x00c0, 0x32db: 0x00c0, 0x32dc: 0x00c0, 0x32dd: 0x00c0, + 0x32de: 0x00c0, 0x32df: 0x00c0, 0x32e0: 0x00c0, 0x32e1: 0x00c0, 0x32e2: 0x00c0, 0x32e3: 0x00c0, + 0x32e4: 0x00c0, 0x32e5: 0x00c0, 0x32e6: 0x00c0, 0x32e7: 0x00c0, 0x32e8: 0x00c0, 0x32e9: 0x00c0, + 0x32ea: 0x00c0, 0x32eb: 0x00c0, 0x32ec: 0x00c0, 0x32ed: 0x00c0, 0x32ee: 0x00c0, 0x32ef: 0x00c0, + 0x32f0: 0x00c0, 0x32f1: 0x00c0, 0x32f2: 0x00c0, 0x32f3: 0x00c0, 0x32f4: 0x00c0, 0x32f5: 0x00c0, + 0x32f6: 0x00c0, 0x32f7: 0x00c0, 0x32f8: 0x00c0, 0x32f9: 0x00c0, 0x32fa: 0x00c0, 0x32fb: 0x00c0, + // Block 0xcc, offset 0x3300 + 0x3300: 0x00c0, 0x3301: 0x00c0, 0x3302: 0x00c0, 0x3303: 0x00c0, 0x3304: 0x00c0, 0x3305: 0x00c0, + 0x3306: 0x00c0, 0x3307: 0x00c0, 0x3308: 0x00c0, 0x3309: 0x00c0, 0x330a: 0x00c0, 0x330b: 0x00c0, + 0x330c: 0x00c0, 0x330d: 0x00c0, 0x330e: 0x00c0, 0x330f: 0x00c0, 0x3310: 0x00c0, 0x3311: 0x00c0, + 0x3312: 0x00c0, 0x3313: 0x00c0, 0x3314: 0x00c0, 0x3315: 0x00c0, 0x3316: 0x00c0, 0x3317: 0x00c0, + 0x3318: 0x00c0, 0x3319: 0x00c0, 0x331a: 0x00c0, 0x331b: 0x00c0, 0x331c: 0x00c0, 0x331d: 0x00c0, + 0x331e: 0x00c0, 0x331f: 0x00c0, 0x3320: 0x00c0, 0x3321: 0x00c0, 0x3322: 0x00c0, 0x3323: 0x00c0, + 0x3324: 0x00c0, 0x3325: 0x00c0, 0x3326: 0x00c0, 0x3327: 0x00c0, + 0x3330: 0x00c0, 0x3331: 0x00c0, 0x3332: 0x00c0, 0x3333: 0x00c0, 0x3334: 0x00c0, 0x3335: 0x00c0, + 0x3336: 0x00c0, 0x3337: 0x00c0, 0x3338: 0x00c0, 0x3339: 0x00c0, 0x333a: 0x00c0, 0x333b: 0x00c0, + 0x333c: 0x00c0, 0x333d: 0x00c0, 0x333e: 0x00c0, 0x333f: 0x00c0, + // Block 0xcd, offset 0x3340 + 0x3340: 0x00c0, 0x3341: 0x00c0, 0x3342: 0x00c0, 0x3343: 0x00c0, 0x3344: 0x00c0, 0x3345: 0x00c0, + 0x3346: 0x00c0, 0x3347: 0x00c0, 0x3348: 0x00c0, 0x3349: 0x00c0, 0x334a: 0x00c0, 0x334b: 0x00c0, + 0x334c: 0x00c0, 0x334d: 0x00c0, 0x334e: 0x00c0, 0x334f: 0x00c0, 0x3350: 0x00c0, 0x3351: 0x00c0, + 0x3352: 0x00c0, 0x3353: 0x00c0, 0x3354: 0x00c0, 0x3355: 0x00c0, 0x3356: 0x00c0, 0x3357: 0x00c0, + 0x3358: 0x00c0, 0x3359: 0x00c0, 0x335a: 0x00c0, 0x335b: 0x00c0, 0x335c: 0x00c0, 0x335d: 0x00c0, + 0x335e: 0x00c0, 0x335f: 0x00c0, 0x3360: 0x00c0, 0x3361: 0x00c0, 0x3362: 0x00c0, 0x3363: 0x00c0, + 0x336f: 0x0080, + // Block 0xce, offset 0x3380 + 0x3380: 0x00c0, 0x3381: 0x00c0, 0x3382: 0x00c0, 0x3383: 0x00c0, 0x3384: 0x00c0, 0x3385: 0x00c0, + 0x3386: 0x00c0, 0x3387: 0x00c0, 0x3388: 0x00c0, 0x3389: 0x00c0, 0x338a: 0x00c0, 0x338b: 0x00c0, + 0x338c: 0x00c0, 0x338d: 0x00c0, 0x338e: 0x00c0, 0x338f: 0x00c0, 0x3390: 0x00c0, 0x3391: 0x00c0, + 0x3392: 0x00c0, 0x3393: 0x00c0, 0x3394: 0x00c0, 0x3395: 0x00c0, 0x3396: 0x00c0, 0x3397: 0x00c0, + 0x3398: 0x00c0, 0x3399: 0x00c0, 0x339a: 0x00c0, 0x339b: 0x00c0, 0x339c: 0x00c0, 0x339d: 0x00c0, + 0x339e: 0x00c0, 0x339f: 0x00c0, 0x33a0: 0x00c0, 0x33a1: 0x00c0, 0x33a2: 0x00c0, 0x33a3: 0x00c0, + 0x33a4: 0x00c0, 0x33a5: 0x00c0, 0x33a6: 0x00c0, 0x33a7: 0x00c0, 0x33a8: 0x00c0, 0x33a9: 0x00c0, + 0x33aa: 0x00c0, 0x33ab: 0x00c0, 0x33ac: 0x00c0, 0x33ad: 0x00c0, 0x33ae: 0x00c0, 0x33af: 0x00c0, + 0x33b0: 0x00c0, 0x33b1: 0x00c0, 0x33b2: 0x00c0, 0x33b3: 0x00c0, 0x33b4: 0x00c0, 0x33b5: 0x00c0, + 0x33b6: 0x00c0, + // Block 0xcf, offset 0x33c0 + 0x33c0: 0x00c0, 0x33c1: 0x00c0, 0x33c2: 0x00c0, 0x33c3: 0x00c0, 0x33c4: 0x00c0, 0x33c5: 0x00c0, + 0x33c6: 0x00c0, 0x33c7: 0x00c0, 0x33c8: 0x00c0, 0x33c9: 0x00c0, 0x33ca: 0x00c0, 0x33cb: 0x00c0, + 0x33cc: 0x00c0, 0x33cd: 0x00c0, 0x33ce: 0x00c0, 0x33cf: 0x00c0, 0x33d0: 0x00c0, 0x33d1: 0x00c0, + 0x33d2: 0x00c0, 0x33d3: 0x00c0, 0x33d4: 0x00c0, 0x33d5: 0x00c0, + 0x33e0: 0x00c0, 0x33e1: 0x00c0, 0x33e2: 0x00c0, 0x33e3: 0x00c0, + 0x33e4: 0x00c0, 0x33e5: 0x00c0, 0x33e6: 0x00c0, 0x33e7: 0x00c0, + // Block 0xd0, offset 0x3400 + 0x3400: 0x00c0, 0x3401: 0x00c0, 0x3402: 0x00c0, 0x3403: 0x00c0, 0x3404: 0x00c0, 0x3405: 0x00c0, + 0x3408: 0x00c0, 0x340a: 0x00c0, 0x340b: 0x00c0, + 0x340c: 0x00c0, 0x340d: 0x00c0, 0x340e: 0x00c0, 0x340f: 0x00c0, 0x3410: 0x00c0, 0x3411: 0x00c0, + 0x3412: 0x00c0, 0x3413: 0x00c0, 0x3414: 0x00c0, 0x3415: 0x00c0, 0x3416: 0x00c0, 0x3417: 0x00c0, + 0x3418: 0x00c0, 0x3419: 0x00c0, 0x341a: 0x00c0, 0x341b: 0x00c0, 0x341c: 0x00c0, 0x341d: 0x00c0, + 0x341e: 0x00c0, 0x341f: 0x00c0, 0x3420: 0x00c0, 0x3421: 0x00c0, 0x3422: 0x00c0, 0x3423: 0x00c0, + 0x3424: 0x00c0, 0x3425: 0x00c0, 0x3426: 0x00c0, 0x3427: 0x00c0, 0x3428: 0x00c0, 0x3429: 0x00c0, + 0x342a: 0x00c0, 0x342b: 0x00c0, 0x342c: 0x00c0, 0x342d: 0x00c0, 0x342e: 0x00c0, 0x342f: 0x00c0, + 0x3430: 0x00c0, 0x3431: 0x00c0, 0x3432: 0x00c0, 0x3433: 0x00c0, 0x3434: 0x00c0, 0x3435: 0x00c0, + 0x3437: 0x00c0, 0x3438: 0x00c0, + 0x343c: 0x00c0, 0x343f: 0x00c0, + // Block 0xd1, offset 0x3440 + 0x3440: 0x00c0, 0x3441: 0x00c0, 0x3442: 0x00c0, 0x3443: 0x00c0, 0x3444: 0x00c0, 0x3445: 0x00c0, + 0x3446: 0x00c0, 0x3447: 0x00c0, 0x3448: 0x00c0, 0x3449: 0x00c0, 0x344a: 0x00c0, 0x344b: 0x00c0, + 0x344c: 0x00c0, 0x344d: 0x00c0, 0x344e: 0x00c0, 0x344f: 0x00c0, 0x3450: 0x00c0, 0x3451: 0x00c0, + 0x3452: 0x00c0, 0x3453: 0x00c0, 0x3454: 0x00c0, 0x3455: 0x00c0, 0x3457: 0x0080, + 0x3458: 0x0080, 0x3459: 0x0080, 0x345a: 0x0080, 0x345b: 0x0080, 0x345c: 0x0080, 0x345d: 0x0080, + 0x345e: 0x0080, 0x345f: 0x0080, 0x3460: 0x00c0, 0x3461: 0x00c0, 0x3462: 0x00c0, 0x3463: 0x00c0, + 0x3464: 0x00c0, 0x3465: 0x00c0, 0x3466: 0x00c0, 0x3467: 0x00c0, 0x3468: 0x00c0, 0x3469: 0x00c0, + 0x346a: 0x00c0, 0x346b: 0x00c0, 0x346c: 0x00c0, 0x346d: 0x00c0, 0x346e: 0x00c0, 0x346f: 0x00c0, + 0x3470: 0x00c0, 0x3471: 0x00c0, 0x3472: 0x00c0, 0x3473: 0x00c0, 0x3474: 0x00c0, 0x3475: 0x00c0, + 0x3476: 0x00c0, 0x3477: 0x0080, 0x3478: 0x0080, 0x3479: 0x0080, 0x347a: 0x0080, 0x347b: 0x0080, + 0x347c: 0x0080, 0x347d: 0x0080, 0x347e: 0x0080, 0x347f: 0x0080, + // Block 0xd2, offset 0x3480 + 0x3480: 0x00c0, 0x3481: 0x00c0, 0x3482: 0x00c0, 0x3483: 0x00c0, 0x3484: 0x00c0, 0x3485: 0x00c0, + 0x3486: 0x00c0, 0x3487: 0x00c0, 0x3488: 0x00c0, 0x3489: 0x00c0, 0x348a: 0x00c0, 0x348b: 0x00c0, + 0x348c: 0x00c0, 0x348d: 0x00c0, 0x348e: 0x00c0, 0x348f: 0x00c0, 0x3490: 0x00c0, 0x3491: 0x00c0, + 0x3492: 0x00c0, 0x3493: 0x00c0, 0x3494: 0x00c0, 0x3495: 0x00c0, 0x3496: 0x00c0, 0x3497: 0x00c0, + 0x3498: 0x00c0, 0x3499: 0x00c0, 0x349a: 0x00c0, 0x349b: 0x00c0, 0x349c: 0x00c0, 0x349d: 0x00c0, + 0x349e: 0x00c0, + 0x34a7: 0x0080, 0x34a8: 0x0080, 0x34a9: 0x0080, + 0x34aa: 0x0080, 0x34ab: 0x0080, 0x34ac: 0x0080, 0x34ad: 0x0080, 0x34ae: 0x0080, 0x34af: 0x0080, + // Block 0xd3, offset 0x34c0 + 0x34e0: 0x00c0, 0x34e1: 0x00c0, 0x34e2: 0x00c0, 0x34e3: 0x00c0, + 0x34e4: 0x00c0, 0x34e5: 0x00c0, 0x34e6: 0x00c0, 0x34e7: 0x00c0, 0x34e8: 0x00c0, 0x34e9: 0x00c0, + 0x34ea: 0x00c0, 0x34eb: 0x00c0, 0x34ec: 0x00c0, 0x34ed: 0x00c0, 0x34ee: 0x00c0, 0x34ef: 0x00c0, + 0x34f0: 0x00c0, 0x34f1: 0x00c0, 0x34f2: 0x00c0, 0x34f4: 0x00c0, 0x34f5: 0x00c0, + 0x34fb: 0x0080, + 0x34fc: 0x0080, 0x34fd: 0x0080, 0x34fe: 0x0080, 0x34ff: 0x0080, + // Block 0xd4, offset 0x3500 + 0x3500: 0x00c0, 0x3501: 0x00c0, 0x3502: 0x00c0, 0x3503: 0x00c0, 0x3504: 0x00c0, 0x3505: 0x00c0, + 0x3506: 0x00c0, 0x3507: 0x00c0, 0x3508: 0x00c0, 0x3509: 0x00c0, 0x350a: 0x00c0, 0x350b: 0x00c0, + 0x350c: 0x00c0, 0x350d: 0x00c0, 0x350e: 0x00c0, 0x350f: 0x00c0, 0x3510: 0x00c0, 0x3511: 0x00c0, + 0x3512: 0x00c0, 0x3513: 0x00c0, 0x3514: 0x00c0, 0x3515: 0x00c0, 0x3516: 0x0080, 0x3517: 0x0080, + 0x3518: 0x0080, 0x3519: 0x0080, 0x351a: 0x0080, 0x351b: 0x0080, + 0x351f: 0x0080, 0x3520: 0x00c0, 0x3521: 0x00c0, 0x3522: 0x00c0, 0x3523: 0x00c0, + 0x3524: 0x00c0, 0x3525: 0x00c0, 0x3526: 0x00c0, 0x3527: 0x00c0, 0x3528: 0x00c0, 0x3529: 0x00c0, + 0x352a: 0x00c0, 0x352b: 0x00c0, 0x352c: 0x00c0, 0x352d: 0x00c0, 0x352e: 0x00c0, 0x352f: 0x00c0, + 0x3530: 0x00c0, 0x3531: 0x00c0, 0x3532: 0x00c0, 0x3533: 0x00c0, 0x3534: 0x00c0, 0x3535: 0x00c0, + 0x3536: 0x00c0, 0x3537: 0x00c0, 0x3538: 0x00c0, 0x3539: 0x00c0, + 0x353f: 0x0080, + // Block 0xd5, offset 0x3540 + 0x3540: 0x00c0, 0x3541: 0x00c0, 0x3542: 0x00c0, 0x3543: 0x00c0, 0x3544: 0x00c0, 0x3545: 0x00c0, + 0x3546: 0x00c0, 0x3547: 0x00c0, 0x3548: 0x00c0, 0x3549: 0x00c0, 0x354a: 0x00c0, 0x354b: 0x00c0, + 0x354c: 0x00c0, 0x354d: 0x00c0, 0x354e: 0x00c0, 0x354f: 0x00c0, 0x3550: 0x00c0, 0x3551: 0x00c0, + 0x3552: 0x00c0, 0x3553: 0x00c0, 0x3554: 0x00c0, 0x3555: 0x00c0, 0x3556: 0x00c0, 0x3557: 0x00c0, + 0x3558: 0x00c0, 0x3559: 0x00c0, 0x355a: 0x00c0, 0x355b: 0x00c0, 0x355c: 0x00c0, 0x355d: 0x00c0, + 0x355e: 0x00c0, 0x355f: 0x00c0, 0x3560: 0x00c0, 0x3561: 0x00c0, 0x3562: 0x00c0, 0x3563: 0x00c0, + 0x3564: 0x00c0, 0x3565: 0x00c0, 0x3566: 0x00c0, 0x3567: 0x00c0, 0x3568: 0x00c0, 0x3569: 0x00c0, + 0x356a: 0x00c0, 0x356b: 0x00c0, 0x356c: 0x00c0, 0x356d: 0x00c0, 0x356e: 0x00c0, 0x356f: 0x00c0, + 0x3570: 0x00c0, 0x3571: 0x00c0, 0x3572: 0x00c0, 0x3573: 0x00c0, 0x3574: 0x00c0, 0x3575: 0x00c0, + 0x3576: 0x00c0, 0x3577: 0x00c0, + 0x357c: 0x0080, 0x357d: 0x0080, 0x357e: 0x00c0, 0x357f: 0x00c0, + // Block 0xd6, offset 0x3580 + 0x3580: 0x00c0, 0x3581: 0x00c3, 0x3582: 0x00c3, 0x3583: 0x00c3, 0x3585: 0x00c3, + 0x3586: 0x00c3, + 0x358c: 0x00c3, 0x358d: 0x00c3, 0x358e: 0x00c3, 0x358f: 0x00c3, 0x3590: 0x00c0, 0x3591: 0x00c0, + 0x3592: 0x00c0, 0x3593: 0x00c0, 0x3595: 0x00c0, 0x3596: 0x00c0, 0x3597: 0x00c0, + 0x3599: 0x00c0, 0x359a: 0x00c0, 0x359b: 0x00c0, 0x359c: 0x00c0, 0x359d: 0x00c0, + 0x359e: 0x00c0, 0x359f: 0x00c0, 0x35a0: 0x00c0, 0x35a1: 0x00c0, 0x35a2: 0x00c0, 0x35a3: 0x00c0, + 0x35a4: 0x00c0, 0x35a5: 0x00c0, 0x35a6: 0x00c0, 0x35a7: 0x00c0, 0x35a8: 0x00c0, 0x35a9: 0x00c0, + 0x35aa: 0x00c0, 0x35ab: 0x00c0, 0x35ac: 0x00c0, 0x35ad: 0x00c0, 0x35ae: 0x00c0, 0x35af: 0x00c0, + 0x35b0: 0x00c0, 0x35b1: 0x00c0, 0x35b2: 0x00c0, 0x35b3: 0x00c0, 0x35b4: 0x00c0, 0x35b5: 0x00c0, + 0x35b8: 0x00c3, 0x35b9: 0x00c3, 0x35ba: 0x00c3, + 0x35bf: 0x00c6, + // Block 0xd7, offset 0x35c0 + 0x35c0: 0x0080, 0x35c1: 0x0080, 0x35c2: 0x0080, 0x35c3: 0x0080, 0x35c4: 0x0080, 0x35c5: 0x0080, + 0x35c6: 0x0080, 0x35c7: 0x0080, 0x35c8: 0x0080, + 0x35d0: 0x0080, 0x35d1: 0x0080, + 0x35d2: 0x0080, 0x35d3: 0x0080, 0x35d4: 0x0080, 0x35d5: 0x0080, 0x35d6: 0x0080, 0x35d7: 0x0080, + 0x35d8: 0x0080, + 0x35e0: 0x00c0, 0x35e1: 0x00c0, 0x35e2: 0x00c0, 0x35e3: 0x00c0, + 0x35e4: 0x00c0, 0x35e5: 0x00c0, 0x35e6: 0x00c0, 0x35e7: 0x00c0, 0x35e8: 0x00c0, 0x35e9: 0x00c0, + 0x35ea: 0x00c0, 0x35eb: 0x00c0, 0x35ec: 0x00c0, 0x35ed: 0x00c0, 0x35ee: 0x00c0, 0x35ef: 0x00c0, + 0x35f0: 0x00c0, 0x35f1: 0x00c0, 0x35f2: 0x00c0, 0x35f3: 0x00c0, 0x35f4: 0x00c0, 0x35f5: 0x00c0, + 0x35f6: 0x00c0, 0x35f7: 0x00c0, 0x35f8: 0x00c0, 0x35f9: 0x00c0, 0x35fa: 0x00c0, 0x35fb: 0x00c0, + 0x35fc: 0x00c0, 0x35fd: 0x0080, 0x35fe: 0x0080, 0x35ff: 0x0080, + // Block 0xd8, offset 0x3600 + 0x3600: 0x00c0, 0x3601: 0x00c0, 0x3602: 0x00c0, 0x3603: 0x00c0, 0x3604: 0x00c0, 0x3605: 0x00c0, + 0x3606: 0x00c0, 0x3607: 0x00c0, 0x3608: 0x00c0, 0x3609: 0x00c0, 0x360a: 0x00c0, 0x360b: 0x00c0, + 0x360c: 0x00c0, 0x360d: 0x00c0, 0x360e: 0x00c0, 0x360f: 0x00c0, 0x3610: 0x00c0, 0x3611: 0x00c0, + 0x3612: 0x00c0, 0x3613: 0x00c0, 0x3614: 0x00c0, 0x3615: 0x00c0, 0x3616: 0x00c0, 0x3617: 0x00c0, + 0x3618: 0x00c0, 0x3619: 0x00c0, 0x361a: 0x00c0, 0x361b: 0x00c0, 0x361c: 0x00c0, 0x361d: 0x0080, + 0x361e: 0x0080, 0x361f: 0x0080, + // Block 0xd9, offset 0x3640 + 0x3640: 0x00c2, 0x3641: 0x00c2, 0x3642: 0x00c2, 0x3643: 0x00c2, 0x3644: 0x00c2, 0x3645: 0x00c4, + 0x3646: 0x00c0, 0x3647: 0x00c4, 0x3648: 0x0080, 0x3649: 0x00c4, 0x364a: 0x00c4, 0x364b: 0x00c0, + 0x364c: 0x00c0, 0x364d: 0x00c1, 0x364e: 0x00c4, 0x364f: 0x00c4, 0x3650: 0x00c4, 0x3651: 0x00c4, + 0x3652: 0x00c4, 0x3653: 0x00c2, 0x3654: 0x00c2, 0x3655: 0x00c2, 0x3656: 0x00c2, 0x3657: 0x00c1, + 0x3658: 0x00c2, 0x3659: 0x00c2, 0x365a: 0x00c2, 0x365b: 0x00c2, 0x365c: 0x00c2, 0x365d: 0x00c4, + 0x365e: 0x00c2, 0x365f: 0x00c2, 0x3660: 0x00c2, 0x3661: 0x00c4, 0x3662: 0x00c0, 0x3663: 0x00c0, + 0x3664: 0x00c4, 0x3665: 0x00c3, 0x3666: 0x00c3, + 0x366b: 0x0082, 0x366c: 0x0082, 0x366d: 0x0082, 0x366e: 0x0082, 0x366f: 0x0084, + 0x3670: 0x0080, 0x3671: 0x0080, 0x3672: 0x0080, 0x3673: 0x0080, 0x3674: 0x0080, 0x3675: 0x0080, + 0x3676: 0x0080, + // Block 0xda, offset 0x3680 + 0x3680: 0x00c0, 0x3681: 0x00c0, 0x3682: 0x00c0, 0x3683: 0x00c0, 0x3684: 0x00c0, 0x3685: 0x00c0, + 0x3686: 0x00c0, 0x3687: 0x00c0, 0x3688: 0x00c0, 0x3689: 0x00c0, 0x368a: 0x00c0, 0x368b: 0x00c0, + 0x368c: 0x00c0, 0x368d: 0x00c0, 0x368e: 0x00c0, 0x368f: 0x00c0, 0x3690: 0x00c0, 0x3691: 0x00c0, + 0x3692: 0x00c0, 0x3693: 0x00c0, 0x3694: 0x00c0, 0x3695: 0x00c0, 0x3696: 0x00c0, 0x3697: 0x00c0, + 0x3698: 0x00c0, 0x3699: 0x00c0, 0x369a: 0x00c0, 0x369b: 0x00c0, 0x369c: 0x00c0, 0x369d: 0x00c0, + 0x369e: 0x00c0, 0x369f: 0x00c0, 0x36a0: 0x00c0, 0x36a1: 0x00c0, 0x36a2: 0x00c0, 0x36a3: 0x00c0, + 0x36a4: 0x00c0, 0x36a5: 0x00c0, 0x36a6: 0x00c0, 0x36a7: 0x00c0, 0x36a8: 0x00c0, 0x36a9: 0x00c0, + 0x36aa: 0x00c0, 0x36ab: 0x00c0, 0x36ac: 0x00c0, 0x36ad: 0x00c0, 0x36ae: 0x00c0, 0x36af: 0x00c0, + 0x36b0: 0x00c0, 0x36b1: 0x00c0, 0x36b2: 0x00c0, 0x36b3: 0x00c0, 0x36b4: 0x00c0, 0x36b5: 0x00c0, + 0x36b9: 0x0080, 0x36ba: 0x0080, 0x36bb: 0x0080, + 0x36bc: 0x0080, 0x36bd: 0x0080, 0x36be: 0x0080, 0x36bf: 0x0080, + // Block 0xdb, offset 0x36c0 + 0x36c0: 0x00c0, 0x36c1: 0x00c0, 0x36c2: 0x00c0, 0x36c3: 0x00c0, 0x36c4: 0x00c0, 0x36c5: 0x00c0, + 0x36c6: 0x00c0, 0x36c7: 0x00c0, 0x36c8: 0x00c0, 0x36c9: 0x00c0, 0x36ca: 0x00c0, 0x36cb: 0x00c0, + 0x36cc: 0x00c0, 0x36cd: 0x00c0, 0x36ce: 0x00c0, 0x36cf: 0x00c0, 0x36d0: 0x00c0, 0x36d1: 0x00c0, + 0x36d2: 0x00c0, 0x36d3: 0x00c0, 0x36d4: 0x00c0, 0x36d5: 0x00c0, + 0x36d8: 0x0080, 0x36d9: 0x0080, 0x36da: 0x0080, 0x36db: 0x0080, 0x36dc: 0x0080, 0x36dd: 0x0080, + 0x36de: 0x0080, 0x36df: 0x0080, 0x36e0: 0x00c0, 0x36e1: 0x00c0, 0x36e2: 0x00c0, 0x36e3: 0x00c0, + 0x36e4: 0x00c0, 0x36e5: 0x00c0, 0x36e6: 0x00c0, 0x36e7: 0x00c0, 0x36e8: 0x00c0, 0x36e9: 0x00c0, + 0x36ea: 0x00c0, 0x36eb: 0x00c0, 0x36ec: 0x00c0, 0x36ed: 0x00c0, 0x36ee: 0x00c0, 0x36ef: 0x00c0, + 0x36f0: 0x00c0, 0x36f1: 0x00c0, 0x36f2: 0x00c0, + 0x36f8: 0x0080, 0x36f9: 0x0080, 0x36fa: 0x0080, 0x36fb: 0x0080, + 0x36fc: 0x0080, 0x36fd: 0x0080, 0x36fe: 0x0080, 0x36ff: 0x0080, + // Block 0xdc, offset 0x3700 + 0x3700: 0x00c2, 0x3701: 0x00c4, 0x3702: 0x00c2, 0x3703: 0x00c4, 0x3704: 0x00c4, 0x3705: 0x00c4, + 0x3706: 0x00c2, 0x3707: 0x00c2, 0x3708: 0x00c2, 0x3709: 0x00c4, 0x370a: 0x00c2, 0x370b: 0x00c2, + 0x370c: 0x00c4, 0x370d: 0x00c2, 0x370e: 0x00c4, 0x370f: 0x00c4, 0x3710: 0x00c2, 0x3711: 0x00c4, + 0x3719: 0x0080, 0x371a: 0x0080, 0x371b: 0x0080, 0x371c: 0x0080, + 0x3729: 0x0084, + 0x372a: 0x0084, 0x372b: 0x0084, 0x372c: 0x0084, 0x372d: 0x0082, 0x372e: 0x0082, 0x372f: 0x0080, + // Block 0xdd, offset 0x3740 + 0x3740: 0x00c0, 0x3741: 0x00c0, 0x3742: 0x00c0, 0x3743: 0x00c0, 0x3744: 0x00c0, 0x3745: 0x00c0, + 0x3746: 0x00c0, 0x3747: 0x00c0, 0x3748: 0x00c0, + // Block 0xde, offset 0x3780 + 0x3780: 0x00c0, 0x3781: 0x00c0, 0x3782: 0x00c0, 0x3783: 0x00c0, 0x3784: 0x00c0, 0x3785: 0x00c0, + 0x3786: 0x00c0, 0x3787: 0x00c0, 0x3788: 0x00c0, 0x3789: 0x00c0, 0x378a: 0x00c0, 0x378b: 0x00c0, + 0x378c: 0x00c0, 0x378d: 0x00c0, 0x378e: 0x00c0, 0x378f: 0x00c0, 0x3790: 0x00c0, 0x3791: 0x00c0, + 0x3792: 0x00c0, 0x3793: 0x00c0, 0x3794: 0x00c0, 0x3795: 0x00c0, 0x3796: 0x00c0, 0x3797: 0x00c0, + 0x3798: 0x00c0, 0x3799: 0x00c0, 0x379a: 0x00c0, 0x379b: 0x00c0, 0x379c: 0x00c0, 0x379d: 0x00c0, + 0x379e: 0x00c0, 0x379f: 0x00c0, 0x37a0: 0x00c0, 0x37a1: 0x00c0, 0x37a2: 0x00c0, 0x37a3: 0x00c0, + 0x37a4: 0x00c0, 0x37a5: 0x00c0, 0x37a6: 0x00c0, 0x37a7: 0x00c0, 0x37a8: 0x00c0, 0x37a9: 0x00c0, + 0x37aa: 0x00c0, 0x37ab: 0x00c0, 0x37ac: 0x00c0, 0x37ad: 0x00c0, 0x37ae: 0x00c0, 0x37af: 0x00c0, + 0x37b0: 0x00c0, 0x37b1: 0x00c0, 0x37b2: 0x00c0, + // Block 0xdf, offset 0x37c0 + 0x37c0: 0x00c0, 0x37c1: 0x00c0, 0x37c2: 0x00c0, 0x37c3: 0x00c0, 0x37c4: 0x00c0, 0x37c5: 0x00c0, + 0x37c6: 0x00c0, 0x37c7: 0x00c0, 0x37c8: 0x00c0, 0x37c9: 0x00c0, 0x37ca: 0x00c0, 0x37cb: 0x00c0, + 0x37cc: 0x00c0, 0x37cd: 0x00c0, 0x37ce: 0x00c0, 0x37cf: 0x00c0, 0x37d0: 0x00c0, 0x37d1: 0x00c0, + 0x37d2: 0x00c0, 0x37d3: 0x00c0, 0x37d4: 0x00c0, 0x37d5: 0x00c0, 0x37d6: 0x00c0, 0x37d7: 0x00c0, + 0x37d8: 0x00c0, 0x37d9: 0x00c0, 0x37da: 0x00c0, 0x37db: 0x00c0, 0x37dc: 0x00c0, 0x37dd: 0x00c0, + 0x37de: 0x00c0, 0x37df: 0x00c0, 0x37e0: 0x00c0, 0x37e1: 0x00c0, 0x37e2: 0x00c0, 0x37e3: 0x00c0, + 0x37e4: 0x00c0, 0x37e5: 0x00c0, 0x37e6: 0x00c0, 0x37e7: 0x00c0, 0x37e8: 0x00c0, 0x37e9: 0x00c0, + 0x37ea: 0x00c0, 0x37eb: 0x00c0, 0x37ec: 0x00c0, 0x37ed: 0x00c0, 0x37ee: 0x00c0, 0x37ef: 0x00c0, + 0x37f0: 0x00c0, 0x37f1: 0x00c0, 0x37f2: 0x00c0, + 0x37fa: 0x0080, 0x37fb: 0x0080, + 0x37fc: 0x0080, 0x37fd: 0x0080, 0x37fe: 0x0080, 0x37ff: 0x0080, + // Block 0xe0, offset 0x3800 + 0x3800: 0x00c1, 0x3801: 0x00c2, 0x3802: 0x00c2, 0x3803: 0x00c2, 0x3804: 0x00c2, 0x3805: 0x00c2, + 0x3806: 0x00c2, 0x3807: 0x00c2, 0x3808: 0x00c2, 0x3809: 0x00c2, 0x380a: 0x00c2, 0x380b: 0x00c2, + 0x380c: 0x00c2, 0x380d: 0x00c2, 0x380e: 0x00c2, 0x380f: 0x00c2, 0x3810: 0x00c2, 0x3811: 0x00c2, + 0x3812: 0x00c2, 0x3813: 0x00c2, 0x3814: 0x00c2, 0x3815: 0x00c2, 0x3816: 0x00c2, 0x3817: 0x00c2, + 0x3818: 0x00c2, 0x3819: 0x00c2, 0x381a: 0x00c2, 0x381b: 0x00c2, 0x381c: 0x00c2, 0x381d: 0x00c2, + 0x381e: 0x00c2, 0x381f: 0x00c2, 0x3820: 0x00c2, 0x3821: 0x00c2, 0x3822: 0x00c4, 0x3823: 0x00c2, + 0x3824: 0x00c3, 0x3825: 0x00c3, 0x3826: 0x00c3, 0x3827: 0x00c3, + 0x3830: 0x00c0, 0x3831: 0x00c0, 0x3832: 0x00c0, 0x3833: 0x00c0, 0x3834: 0x00c0, 0x3835: 0x00c0, + 0x3836: 0x00c0, 0x3837: 0x00c0, 0x3838: 0x00c0, 0x3839: 0x00c0, + // Block 0xe1, offset 0x3840 + 0x3860: 0x0080, 0x3861: 0x0080, 0x3862: 0x0080, 0x3863: 0x0080, + 0x3864: 0x0080, 0x3865: 0x0080, 0x3866: 0x0080, 0x3867: 0x0080, 0x3868: 0x0080, 0x3869: 0x0080, + 0x386a: 0x0080, 0x386b: 0x0080, 0x386c: 0x0080, 0x386d: 0x0080, 0x386e: 0x0080, 0x386f: 0x0080, + 0x3870: 0x0080, 0x3871: 0x0080, 0x3872: 0x0080, 0x3873: 0x0080, 0x3874: 0x0080, 0x3875: 0x0080, + 0x3876: 0x0080, 0x3877: 0x0080, 0x3878: 0x0080, 0x3879: 0x0080, 0x387a: 0x0080, 0x387b: 0x0080, + 0x387c: 0x0080, 0x387d: 0x0080, 0x387e: 0x0080, + // Block 0xe2, offset 0x3880 + 0x3880: 0x00c0, 0x3881: 0x00c0, 0x3882: 0x00c0, 0x3883: 0x00c0, 0x3884: 0x00c0, 0x3885: 0x00c0, + 0x3886: 0x00c0, 0x3887: 0x00c0, 0x3888: 0x00c0, 0x3889: 0x00c0, 0x388a: 0x00c0, 0x388b: 0x00c0, + 0x388c: 0x00c0, 0x388d: 0x00c0, 0x388e: 0x00c0, 0x388f: 0x00c0, 0x3890: 0x00c0, 0x3891: 0x00c0, + 0x3892: 0x00c0, 0x3893: 0x00c0, 0x3894: 0x00c0, 0x3895: 0x00c0, 0x3896: 0x00c0, 0x3897: 0x00c0, + 0x3898: 0x00c0, 0x3899: 0x00c0, 0x389a: 0x00c0, 0x389b: 0x00c0, 0x389c: 0x00c0, 0x389d: 0x0080, + 0x389e: 0x0080, 0x389f: 0x0080, 0x38a0: 0x0080, 0x38a1: 0x0080, 0x38a2: 0x0080, 0x38a3: 0x0080, + 0x38a4: 0x0080, 0x38a5: 0x0080, 0x38a6: 0x0080, 0x38a7: 0x00c0, + 0x38b0: 0x00c2, 0x38b1: 0x00c2, 0x38b2: 0x00c2, 0x38b3: 0x00c4, 0x38b4: 0x00c2, 0x38b5: 0x00c2, + 0x38b6: 0x00c2, 0x38b7: 0x00c2, 0x38b8: 0x00c2, 0x38b9: 0x00c2, 0x38ba: 0x00c2, 0x38bb: 0x00c2, + 0x38bc: 0x00c2, 0x38bd: 0x00c2, 0x38be: 0x00c2, 0x38bf: 0x00c2, + // Block 0xe3, offset 0x38c0 + 0x38c0: 0x00c2, 0x38c1: 0x00c2, 0x38c2: 0x00c2, 0x38c3: 0x00c2, 0x38c4: 0x00c2, 0x38c5: 0x00c0, + 0x38c6: 0x00c3, 0x38c7: 0x00c3, 0x38c8: 0x00c3, 0x38c9: 0x00c3, 0x38ca: 0x00c3, 0x38cb: 0x00c3, + 0x38cc: 0x00c3, 0x38cd: 0x00c3, 0x38ce: 0x00c3, 0x38cf: 0x00c3, 0x38d0: 0x00c3, 0x38d1: 0x0082, + 0x38d2: 0x0082, 0x38d3: 0x0082, 0x38d4: 0x0084, 0x38d5: 0x0080, 0x38d6: 0x0080, 0x38d7: 0x0080, + 0x38d8: 0x0080, 0x38d9: 0x0080, + // Block 0xe4, offset 0x3900 + 0x3920: 0x00c0, 0x3921: 0x00c0, 0x3922: 0x00c0, 0x3923: 0x00c0, + 0x3924: 0x00c0, 0x3925: 0x00c0, 0x3926: 0x00c0, 0x3927: 0x00c0, 0x3928: 0x00c0, 0x3929: 0x00c0, + 0x392a: 0x00c0, 0x392b: 0x00c0, 0x392c: 0x00c0, 0x392d: 0x00c0, 0x392e: 0x00c0, 0x392f: 0x00c0, + 0x3930: 0x00c0, 0x3931: 0x00c0, 0x3932: 0x00c0, 0x3933: 0x00c0, 0x3934: 0x00c0, 0x3935: 0x00c0, + 0x3936: 0x00c0, + // Block 0xe5, offset 0x3940 + 0x3940: 0x00c0, 0x3941: 0x00c3, 0x3942: 0x00c0, 0x3943: 0x00c0, 0x3944: 0x00c0, 0x3945: 0x00c0, + 0x3946: 0x00c0, 0x3947: 0x00c0, 0x3948: 0x00c0, 0x3949: 0x00c0, 0x394a: 0x00c0, 0x394b: 0x00c0, + 0x394c: 0x00c0, 0x394d: 0x00c0, 0x394e: 0x00c0, 0x394f: 0x00c0, 0x3950: 0x00c0, 0x3951: 0x00c0, + 0x3952: 0x00c0, 0x3953: 0x00c0, 0x3954: 0x00c0, 0x3955: 0x00c0, 0x3956: 0x00c0, 0x3957: 0x00c0, + 0x3958: 0x00c0, 0x3959: 0x00c0, 0x395a: 0x00c0, 0x395b: 0x00c0, 0x395c: 0x00c0, 0x395d: 0x00c0, + 0x395e: 0x00c0, 0x395f: 0x00c0, 0x3960: 0x00c0, 0x3961: 0x00c0, 0x3962: 0x00c0, 0x3963: 0x00c0, + 0x3964: 0x00c0, 0x3965: 0x00c0, 0x3966: 0x00c0, 0x3967: 0x00c0, 0x3968: 0x00c0, 0x3969: 0x00c0, + 0x396a: 0x00c0, 0x396b: 0x00c0, 0x396c: 0x00c0, 0x396d: 0x00c0, 0x396e: 0x00c0, 0x396f: 0x00c0, + 0x3970: 0x00c0, 0x3971: 0x00c0, 0x3972: 0x00c0, 0x3973: 0x00c0, 0x3974: 0x00c0, 0x3975: 0x00c0, + 0x3976: 0x00c0, 0x3977: 0x00c0, 0x3978: 0x00c3, 0x3979: 0x00c3, 0x397a: 0x00c3, 0x397b: 0x00c3, + 0x397c: 0x00c3, 0x397d: 0x00c3, 0x397e: 0x00c3, 0x397f: 0x00c3, + // Block 0xe6, offset 0x3980 + 0x3980: 0x00c3, 0x3981: 0x00c3, 0x3982: 0x00c3, 0x3983: 0x00c3, 0x3984: 0x00c3, 0x3985: 0x00c3, + 0x3986: 0x00c6, 0x3987: 0x0080, 0x3988: 0x0080, 0x3989: 0x0080, 0x398a: 0x0080, 0x398b: 0x0080, + 0x398c: 0x0080, 0x398d: 0x0080, + 0x3992: 0x0080, 0x3993: 0x0080, 0x3994: 0x0080, 0x3995: 0x0080, 0x3996: 0x0080, 0x3997: 0x0080, + 0x3998: 0x0080, 0x3999: 0x0080, 0x399a: 0x0080, 0x399b: 0x0080, 0x399c: 0x0080, 0x399d: 0x0080, + 0x399e: 0x0080, 0x399f: 0x0080, 0x39a0: 0x0080, 0x39a1: 0x0080, 0x39a2: 0x0080, 0x39a3: 0x0080, + 0x39a4: 0x0080, 0x39a5: 0x0080, 0x39a6: 0x00c0, 0x39a7: 0x00c0, 0x39a8: 0x00c0, 0x39a9: 0x00c0, + 0x39aa: 0x00c0, 0x39ab: 0x00c0, 0x39ac: 0x00c0, 0x39ad: 0x00c0, 0x39ae: 0x00c0, 0x39af: 0x00c0, + 0x39bf: 0x00c6, + // Block 0xe7, offset 0x39c0 + 0x39c0: 0x00c3, 0x39c1: 0x00c3, 0x39c2: 0x00c0, 0x39c3: 0x00c0, 0x39c4: 0x00c0, 0x39c5: 0x00c0, + 0x39c6: 0x00c0, 0x39c7: 0x00c0, 0x39c8: 0x00c0, 0x39c9: 0x00c0, 0x39ca: 0x00c0, 0x39cb: 0x00c0, + 0x39cc: 0x00c0, 0x39cd: 0x00c0, 0x39ce: 0x00c0, 0x39cf: 0x00c0, 0x39d0: 0x00c0, 0x39d1: 0x00c0, + 0x39d2: 0x00c0, 0x39d3: 0x00c0, 0x39d4: 0x00c0, 0x39d5: 0x00c0, 0x39d6: 0x00c0, 0x39d7: 0x00c0, + 0x39d8: 0x00c0, 0x39d9: 0x00c0, 0x39da: 0x00c0, 0x39db: 0x00c0, 0x39dc: 0x00c0, 0x39dd: 0x00c0, + 0x39de: 0x00c0, 0x39df: 0x00c0, 0x39e0: 0x00c0, 0x39e1: 0x00c0, 0x39e2: 0x00c0, 0x39e3: 0x00c0, + 0x39e4: 0x00c0, 0x39e5: 0x00c0, 0x39e6: 0x00c0, 0x39e7: 0x00c0, 0x39e8: 0x00c0, 0x39e9: 0x00c0, + 0x39ea: 0x00c0, 0x39eb: 0x00c0, 0x39ec: 0x00c0, 0x39ed: 0x00c0, 0x39ee: 0x00c0, 0x39ef: 0x00c0, + 0x39f0: 0x00c0, 0x39f1: 0x00c0, 0x39f2: 0x00c0, 0x39f3: 0x00c3, 0x39f4: 0x00c3, 0x39f5: 0x00c3, + 0x39f6: 0x00c3, 0x39f7: 0x00c0, 0x39f8: 0x00c0, 0x39f9: 0x00c6, 0x39fa: 0x00c3, 0x39fb: 0x0080, + 0x39fc: 0x0080, 0x39fd: 0x0040, 0x39fe: 0x0080, 0x39ff: 0x0080, + // Block 0xe8, offset 0x3a00 + 0x3a00: 0x0080, 0x3a01: 0x0080, + 0x3a0d: 0x0040, 0x3a10: 0x00c0, 0x3a11: 0x00c0, + 0x3a12: 0x00c0, 0x3a13: 0x00c0, 0x3a14: 0x00c0, 0x3a15: 0x00c0, 0x3a16: 0x00c0, 0x3a17: 0x00c0, + 0x3a18: 0x00c0, 0x3a19: 0x00c0, 0x3a1a: 0x00c0, 0x3a1b: 0x00c0, 0x3a1c: 0x00c0, 0x3a1d: 0x00c0, + 0x3a1e: 0x00c0, 0x3a1f: 0x00c0, 0x3a20: 0x00c0, 0x3a21: 0x00c0, 0x3a22: 0x00c0, 0x3a23: 0x00c0, + 0x3a24: 0x00c0, 0x3a25: 0x00c0, 0x3a26: 0x00c0, 0x3a27: 0x00c0, 0x3a28: 0x00c0, + 0x3a30: 0x00c0, 0x3a31: 0x00c0, 0x3a32: 0x00c0, 0x3a33: 0x00c0, 0x3a34: 0x00c0, 0x3a35: 0x00c0, + 0x3a36: 0x00c0, 0x3a37: 0x00c0, 0x3a38: 0x00c0, 0x3a39: 0x00c0, + // Block 0xe9, offset 0x3a40 + 0x3a40: 0x00c3, 0x3a41: 0x00c3, 0x3a42: 0x00c3, 0x3a43: 0x00c0, 0x3a44: 0x00c0, 0x3a45: 0x00c0, + 0x3a46: 0x00c0, 0x3a47: 0x00c0, 0x3a48: 0x00c0, 0x3a49: 0x00c0, 0x3a4a: 0x00c0, 0x3a4b: 0x00c0, + 0x3a4c: 0x00c0, 0x3a4d: 0x00c0, 0x3a4e: 0x00c0, 0x3a4f: 0x00c0, 0x3a50: 0x00c0, 0x3a51: 0x00c0, + 0x3a52: 0x00c0, 0x3a53: 0x00c0, 0x3a54: 0x00c0, 0x3a55: 0x00c0, 0x3a56: 0x00c0, 0x3a57: 0x00c0, + 0x3a58: 0x00c0, 0x3a59: 0x00c0, 0x3a5a: 0x00c0, 0x3a5b: 0x00c0, 0x3a5c: 0x00c0, 0x3a5d: 0x00c0, + 0x3a5e: 0x00c0, 0x3a5f: 0x00c0, 0x3a60: 0x00c0, 0x3a61: 0x00c0, 0x3a62: 0x00c0, 0x3a63: 0x00c0, + 0x3a64: 0x00c0, 0x3a65: 0x00c0, 0x3a66: 0x00c0, 0x3a67: 0x00c3, 0x3a68: 0x00c3, 0x3a69: 0x00c3, + 0x3a6a: 0x00c3, 0x3a6b: 0x00c3, 0x3a6c: 0x00c0, 0x3a6d: 0x00c3, 0x3a6e: 0x00c3, 0x3a6f: 0x00c3, + 0x3a70: 0x00c3, 0x3a71: 0x00c3, 0x3a72: 0x00c3, 0x3a73: 0x00c6, 0x3a74: 0x00c6, + 0x3a76: 0x00c0, 0x3a77: 0x00c0, 0x3a78: 0x00c0, 0x3a79: 0x00c0, 0x3a7a: 0x00c0, 0x3a7b: 0x00c0, + 0x3a7c: 0x00c0, 0x3a7d: 0x00c0, 0x3a7e: 0x00c0, 0x3a7f: 0x00c0, + // Block 0xea, offset 0x3a80 + 0x3a80: 0x0080, 0x3a81: 0x0080, 0x3a82: 0x0080, 0x3a83: 0x0080, 0x3a84: 0x00c0, 0x3a85: 0x00c0, + 0x3a86: 0x00c0, + 0x3a90: 0x00c0, 0x3a91: 0x00c0, + 0x3a92: 0x00c0, 0x3a93: 0x00c0, 0x3a94: 0x00c0, 0x3a95: 0x00c0, 0x3a96: 0x00c0, 0x3a97: 0x00c0, + 0x3a98: 0x00c0, 0x3a99: 0x00c0, 0x3a9a: 0x00c0, 0x3a9b: 0x00c0, 0x3a9c: 0x00c0, 0x3a9d: 0x00c0, + 0x3a9e: 0x00c0, 0x3a9f: 0x00c0, 0x3aa0: 0x00c0, 0x3aa1: 0x00c0, 0x3aa2: 0x00c0, 0x3aa3: 0x00c0, + 0x3aa4: 0x00c0, 0x3aa5: 0x00c0, 0x3aa6: 0x00c0, 0x3aa7: 0x00c0, 0x3aa8: 0x00c0, 0x3aa9: 0x00c0, + 0x3aaa: 0x00c0, 0x3aab: 0x00c0, 0x3aac: 0x00c0, 0x3aad: 0x00c0, 0x3aae: 0x00c0, 0x3aaf: 0x00c0, + 0x3ab0: 0x00c0, 0x3ab1: 0x00c0, 0x3ab2: 0x00c0, 0x3ab3: 0x00c3, 0x3ab4: 0x0080, 0x3ab5: 0x0080, + 0x3ab6: 0x00c0, + // Block 0xeb, offset 0x3ac0 + 0x3ac0: 0x00c3, 0x3ac1: 0x00c3, 0x3ac2: 0x00c0, 0x3ac3: 0x00c0, 0x3ac4: 0x00c0, 0x3ac5: 0x00c0, + 0x3ac6: 0x00c0, 0x3ac7: 0x00c0, 0x3ac8: 0x00c0, 0x3ac9: 0x00c0, 0x3aca: 0x00c0, 0x3acb: 0x00c0, + 0x3acc: 0x00c0, 0x3acd: 0x00c0, 0x3ace: 0x00c0, 0x3acf: 0x00c0, 0x3ad0: 0x00c0, 0x3ad1: 0x00c0, + 0x3ad2: 0x00c0, 0x3ad3: 0x00c0, 0x3ad4: 0x00c0, 0x3ad5: 0x00c0, 0x3ad6: 0x00c0, 0x3ad7: 0x00c0, + 0x3ad8: 0x00c0, 0x3ad9: 0x00c0, 0x3ada: 0x00c0, 0x3adb: 0x00c0, 0x3adc: 0x00c0, 0x3add: 0x00c0, + 0x3ade: 0x00c0, 0x3adf: 0x00c0, 0x3ae0: 0x00c0, 0x3ae1: 0x00c0, 0x3ae2: 0x00c0, 0x3ae3: 0x00c0, + 0x3ae4: 0x00c0, 0x3ae5: 0x00c0, 0x3ae6: 0x00c0, 0x3ae7: 0x00c0, 0x3ae8: 0x00c0, 0x3ae9: 0x00c0, + 0x3aea: 0x00c0, 0x3aeb: 0x00c0, 0x3aec: 0x00c0, 0x3aed: 0x00c0, 0x3aee: 0x00c0, 0x3aef: 0x00c0, + 0x3af0: 0x00c0, 0x3af1: 0x00c0, 0x3af2: 0x00c0, 0x3af3: 0x00c0, 0x3af4: 0x00c0, 0x3af5: 0x00c0, + 0x3af6: 0x00c3, 0x3af7: 0x00c3, 0x3af8: 0x00c3, 0x3af9: 0x00c3, 0x3afa: 0x00c3, 0x3afb: 0x00c3, + 0x3afc: 0x00c3, 0x3afd: 0x00c3, 0x3afe: 0x00c3, 0x3aff: 0x00c0, + // Block 0xec, offset 0x3b00 + 0x3b00: 0x00c5, 0x3b01: 0x00c0, 0x3b02: 0x00c0, 0x3b03: 0x00c0, 0x3b04: 0x00c0, 0x3b05: 0x0080, + 0x3b06: 0x0080, 0x3b07: 0x0080, 0x3b08: 0x0080, 0x3b09: 0x00c3, 0x3b0a: 0x00c3, 0x3b0b: 0x00c3, + 0x3b0c: 0x00c3, 0x3b0d: 0x0080, 0x3b10: 0x00c0, 0x3b11: 0x00c0, + 0x3b12: 0x00c0, 0x3b13: 0x00c0, 0x3b14: 0x00c0, 0x3b15: 0x00c0, 0x3b16: 0x00c0, 0x3b17: 0x00c0, + 0x3b18: 0x00c0, 0x3b19: 0x00c0, 0x3b1a: 0x00c0, 0x3b1b: 0x0080, 0x3b1c: 0x00c0, 0x3b1d: 0x0080, + 0x3b1e: 0x0080, 0x3b1f: 0x0080, 0x3b21: 0x0080, 0x3b22: 0x0080, 0x3b23: 0x0080, + 0x3b24: 0x0080, 0x3b25: 0x0080, 0x3b26: 0x0080, 0x3b27: 0x0080, 0x3b28: 0x0080, 0x3b29: 0x0080, + 0x3b2a: 0x0080, 0x3b2b: 0x0080, 0x3b2c: 0x0080, 0x3b2d: 0x0080, 0x3b2e: 0x0080, 0x3b2f: 0x0080, + 0x3b30: 0x0080, 0x3b31: 0x0080, 0x3b32: 0x0080, 0x3b33: 0x0080, 0x3b34: 0x0080, + // Block 0xed, offset 0x3b40 + 0x3b40: 0x00c0, 0x3b41: 0x00c0, 0x3b42: 0x00c0, 0x3b43: 0x00c0, 0x3b44: 0x00c0, 0x3b45: 0x00c0, + 0x3b46: 0x00c0, 0x3b47: 0x00c0, 0x3b48: 0x00c0, 0x3b49: 0x00c0, 0x3b4a: 0x00c0, 0x3b4b: 0x00c0, + 0x3b4c: 0x00c0, 0x3b4d: 0x00c0, 0x3b4e: 0x00c0, 0x3b4f: 0x00c0, 0x3b50: 0x00c0, 0x3b51: 0x00c0, + 0x3b53: 0x00c0, 0x3b54: 0x00c0, 0x3b55: 0x00c0, 0x3b56: 0x00c0, 0x3b57: 0x00c0, + 0x3b58: 0x00c0, 0x3b59: 0x00c0, 0x3b5a: 0x00c0, 0x3b5b: 0x00c0, 0x3b5c: 0x00c0, 0x3b5d: 0x00c0, + 0x3b5e: 0x00c0, 0x3b5f: 0x00c0, 0x3b60: 0x00c0, 0x3b61: 0x00c0, 0x3b62: 0x00c0, 0x3b63: 0x00c0, + 0x3b64: 0x00c0, 0x3b65: 0x00c0, 0x3b66: 0x00c0, 0x3b67: 0x00c0, 0x3b68: 0x00c0, 0x3b69: 0x00c0, + 0x3b6a: 0x00c0, 0x3b6b: 0x00c0, 0x3b6c: 0x00c0, 0x3b6d: 0x00c0, 0x3b6e: 0x00c0, 0x3b6f: 0x00c3, + 0x3b70: 0x00c3, 0x3b71: 0x00c3, 0x3b72: 0x00c0, 0x3b73: 0x00c0, 0x3b74: 0x00c3, 0x3b75: 0x00c5, + 0x3b76: 0x00c3, 0x3b77: 0x00c3, 0x3b78: 0x0080, 0x3b79: 0x0080, 0x3b7a: 0x0080, 0x3b7b: 0x0080, + 0x3b7c: 0x0080, 0x3b7d: 0x0080, 0x3b7e: 0x00c3, + // Block 0xee, offset 0x3b80 + 0x3b80: 0x00c0, 0x3b81: 0x00c0, 0x3b82: 0x00c0, 0x3b83: 0x00c0, 0x3b84: 0x00c0, 0x3b85: 0x00c0, + 0x3b86: 0x00c0, 0x3b88: 0x00c0, 0x3b8a: 0x00c0, 0x3b8b: 0x00c0, + 0x3b8c: 0x00c0, 0x3b8d: 0x00c0, 0x3b8f: 0x00c0, 0x3b90: 0x00c0, 0x3b91: 0x00c0, + 0x3b92: 0x00c0, 0x3b93: 0x00c0, 0x3b94: 0x00c0, 0x3b95: 0x00c0, 0x3b96: 0x00c0, 0x3b97: 0x00c0, + 0x3b98: 0x00c0, 0x3b99: 0x00c0, 0x3b9a: 0x00c0, 0x3b9b: 0x00c0, 0x3b9c: 0x00c0, 0x3b9d: 0x00c0, + 0x3b9f: 0x00c0, 0x3ba0: 0x00c0, 0x3ba1: 0x00c0, 0x3ba2: 0x00c0, 0x3ba3: 0x00c0, + 0x3ba4: 0x00c0, 0x3ba5: 0x00c0, 0x3ba6: 0x00c0, 0x3ba7: 0x00c0, 0x3ba8: 0x00c0, 0x3ba9: 0x0080, + 0x3bb0: 0x00c0, 0x3bb1: 0x00c0, 0x3bb2: 0x00c0, 0x3bb3: 0x00c0, 0x3bb4: 0x00c0, 0x3bb5: 0x00c0, + 0x3bb6: 0x00c0, 0x3bb7: 0x00c0, 0x3bb8: 0x00c0, 0x3bb9: 0x00c0, 0x3bba: 0x00c0, 0x3bbb: 0x00c0, + 0x3bbc: 0x00c0, 0x3bbd: 0x00c0, 0x3bbe: 0x00c0, 0x3bbf: 0x00c0, + // Block 0xef, offset 0x3bc0 + 0x3bc0: 0x00c0, 0x3bc1: 0x00c0, 0x3bc2: 0x00c0, 0x3bc3: 0x00c0, 0x3bc4: 0x00c0, 0x3bc5: 0x00c0, + 0x3bc6: 0x00c0, 0x3bc7: 0x00c0, 0x3bc8: 0x00c0, 0x3bc9: 0x00c0, 0x3bca: 0x00c0, 0x3bcb: 0x00c0, + 0x3bcc: 0x00c0, 0x3bcd: 0x00c0, 0x3bce: 0x00c0, 0x3bcf: 0x00c0, 0x3bd0: 0x00c0, 0x3bd1: 0x00c0, + 0x3bd2: 0x00c0, 0x3bd3: 0x00c0, 0x3bd4: 0x00c0, 0x3bd5: 0x00c0, 0x3bd6: 0x00c0, 0x3bd7: 0x00c0, + 0x3bd8: 0x00c0, 0x3bd9: 0x00c0, 0x3bda: 0x00c0, 0x3bdb: 0x00c0, 0x3bdc: 0x00c0, 0x3bdd: 0x00c0, + 0x3bde: 0x00c0, 0x3bdf: 0x00c3, 0x3be0: 0x00c0, 0x3be1: 0x00c0, 0x3be2: 0x00c0, 0x3be3: 0x00c3, + 0x3be4: 0x00c3, 0x3be5: 0x00c3, 0x3be6: 0x00c3, 0x3be7: 0x00c3, 0x3be8: 0x00c3, 0x3be9: 0x00c3, + 0x3bea: 0x00c6, + 0x3bf0: 0x00c0, 0x3bf1: 0x00c0, 0x3bf2: 0x00c0, 0x3bf3: 0x00c0, 0x3bf4: 0x00c0, 0x3bf5: 0x00c0, + 0x3bf6: 0x00c0, 0x3bf7: 0x00c0, 0x3bf8: 0x00c0, 0x3bf9: 0x00c0, + // Block 0xf0, offset 0x3c00 + 0x3c00: 0x00c3, 0x3c01: 0x00c3, 0x3c02: 0x00c0, 0x3c03: 0x00c0, 0x3c05: 0x00c0, + 0x3c06: 0x00c0, 0x3c07: 0x00c0, 0x3c08: 0x00c0, 0x3c09: 0x00c0, 0x3c0a: 0x00c0, 0x3c0b: 0x00c0, + 0x3c0c: 0x00c0, 0x3c0f: 0x00c0, 0x3c10: 0x00c0, + 0x3c13: 0x00c0, 0x3c14: 0x00c0, 0x3c15: 0x00c0, 0x3c16: 0x00c0, 0x3c17: 0x00c0, + 0x3c18: 0x00c0, 0x3c19: 0x00c0, 0x3c1a: 0x00c0, 0x3c1b: 0x00c0, 0x3c1c: 0x00c0, 0x3c1d: 0x00c0, + 0x3c1e: 0x00c0, 0x3c1f: 0x00c0, 0x3c20: 0x00c0, 0x3c21: 0x00c0, 0x3c22: 0x00c0, 0x3c23: 0x00c0, + 0x3c24: 0x00c0, 0x3c25: 0x00c0, 0x3c26: 0x00c0, 0x3c27: 0x00c0, 0x3c28: 0x00c0, + 0x3c2a: 0x00c0, 0x3c2b: 0x00c0, 0x3c2c: 0x00c0, 0x3c2d: 0x00c0, 0x3c2e: 0x00c0, 0x3c2f: 0x00c0, + 0x3c30: 0x00c0, 0x3c32: 0x00c0, 0x3c33: 0x00c0, 0x3c35: 0x00c0, + 0x3c36: 0x00c0, 0x3c37: 0x00c0, 0x3c38: 0x00c0, 0x3c39: 0x00c0, 0x3c3b: 0x00c3, + 0x3c3c: 0x00c3, 0x3c3d: 0x00c0, 0x3c3e: 0x00c0, 0x3c3f: 0x00c0, + // Block 0xf1, offset 0x3c40 + 0x3c40: 0x00c3, 0x3c41: 0x00c0, 0x3c42: 0x00c0, 0x3c43: 0x00c0, 0x3c44: 0x00c0, + 0x3c47: 0x00c0, 0x3c48: 0x00c0, 0x3c4b: 0x00c0, + 0x3c4c: 0x00c0, 0x3c4d: 0x00c5, 0x3c50: 0x00c0, + 0x3c57: 0x00c0, + 0x3c5d: 0x00c0, + 0x3c5e: 0x00c0, 0x3c5f: 0x00c0, 0x3c60: 0x00c0, 0x3c61: 0x00c0, 0x3c62: 0x00c0, 0x3c63: 0x00c0, + 0x3c66: 0x00c3, 0x3c67: 0x00c3, 0x3c68: 0x00c3, 0x3c69: 0x00c3, + 0x3c6a: 0x00c3, 0x3c6b: 0x00c3, 0x3c6c: 0x00c3, + 0x3c70: 0x00c3, 0x3c71: 0x00c3, 0x3c72: 0x00c3, 0x3c73: 0x00c3, 0x3c74: 0x00c3, + // Block 0xf2, offset 0x3c80 + 0x3c80: 0x00c0, 0x3c81: 0x00c0, 0x3c82: 0x00c0, 0x3c83: 0x00c0, 0x3c84: 0x00c0, 0x3c85: 0x00c0, + 0x3c86: 0x00c0, 0x3c87: 0x00c0, 0x3c88: 0x00c0, 0x3c89: 0x00c0, 0x3c8a: 0x00c0, 0x3c8b: 0x00c0, + 0x3c8c: 0x00c0, 0x3c8d: 0x00c0, 0x3c8e: 0x00c0, 0x3c8f: 0x00c0, 0x3c90: 0x00c0, 0x3c91: 0x00c0, + 0x3c92: 0x00c0, 0x3c93: 0x00c0, 0x3c94: 0x00c0, 0x3c95: 0x00c0, 0x3c96: 0x00c0, 0x3c97: 0x00c0, + 0x3c98: 0x00c0, 0x3c99: 0x00c0, 0x3c9a: 0x00c0, 0x3c9b: 0x00c0, 0x3c9c: 0x00c0, 0x3c9d: 0x00c0, + 0x3c9e: 0x00c0, 0x3c9f: 0x00c0, 0x3ca0: 0x00c0, 0x3ca1: 0x00c0, 0x3ca2: 0x00c0, 0x3ca3: 0x00c0, + 0x3ca4: 0x00c0, 0x3ca5: 0x00c0, 0x3ca6: 0x00c0, 0x3ca7: 0x00c0, 0x3ca8: 0x00c0, 0x3ca9: 0x00c0, + 0x3caa: 0x00c0, 0x3cab: 0x00c0, 0x3cac: 0x00c0, 0x3cad: 0x00c0, 0x3cae: 0x00c0, 0x3caf: 0x00c0, + 0x3cb0: 0x00c0, 0x3cb1: 0x00c0, 0x3cb2: 0x00c0, 0x3cb3: 0x00c0, 0x3cb4: 0x00c0, 0x3cb5: 0x00c0, + 0x3cb6: 0x00c0, 0x3cb7: 0x00c0, 0x3cb8: 0x00c3, 0x3cb9: 0x00c3, 0x3cba: 0x00c3, 0x3cbb: 0x00c3, + 0x3cbc: 0x00c3, 0x3cbd: 0x00c3, 0x3cbe: 0x00c3, 0x3cbf: 0x00c3, + // Block 0xf3, offset 0x3cc0 + 0x3cc0: 0x00c0, 0x3cc1: 0x00c0, 0x3cc2: 0x00c6, 0x3cc3: 0x00c3, 0x3cc4: 0x00c3, 0x3cc5: 0x00c0, + 0x3cc6: 0x00c3, 0x3cc7: 0x00c0, 0x3cc8: 0x00c0, 0x3cc9: 0x00c0, 0x3cca: 0x00c0, 0x3ccb: 0x0080, + 0x3ccc: 0x0080, 0x3ccd: 0x0080, 0x3cce: 0x0080, 0x3ccf: 0x0080, 0x3cd0: 0x00c0, 0x3cd1: 0x00c0, + 0x3cd2: 0x00c0, 0x3cd3: 0x00c0, 0x3cd4: 0x00c0, 0x3cd5: 0x00c0, 0x3cd6: 0x00c0, 0x3cd7: 0x00c0, + 0x3cd8: 0x00c0, 0x3cd9: 0x00c0, 0x3cdb: 0x0080, 0x3cdd: 0x0080, + 0x3cde: 0x00c3, 0x3cdf: 0x00c0, + // Block 0xf4, offset 0x3d00 + 0x3d00: 0x00c0, 0x3d01: 0x00c0, 0x3d02: 0x00c0, 0x3d03: 0x00c0, 0x3d04: 0x00c0, 0x3d05: 0x00c0, + 0x3d06: 0x00c0, 0x3d07: 0x00c0, 0x3d08: 0x00c0, 0x3d09: 0x00c0, 0x3d0a: 0x00c0, 0x3d0b: 0x00c0, + 0x3d0c: 0x00c0, 0x3d0d: 0x00c0, 0x3d0e: 0x00c0, 0x3d0f: 0x00c0, 0x3d10: 0x00c0, 0x3d11: 0x00c0, + 0x3d12: 0x00c0, 0x3d13: 0x00c0, 0x3d14: 0x00c0, 0x3d15: 0x00c0, 0x3d16: 0x00c0, 0x3d17: 0x00c0, + 0x3d18: 0x00c0, 0x3d19: 0x00c0, 0x3d1a: 0x00c0, 0x3d1b: 0x00c0, 0x3d1c: 0x00c0, 0x3d1d: 0x00c0, + 0x3d1e: 0x00c0, 0x3d1f: 0x00c0, 0x3d20: 0x00c0, 0x3d21: 0x00c0, 0x3d22: 0x00c0, 0x3d23: 0x00c0, + 0x3d24: 0x00c0, 0x3d25: 0x00c0, 0x3d26: 0x00c0, 0x3d27: 0x00c0, 0x3d28: 0x00c0, 0x3d29: 0x00c0, + 0x3d2a: 0x00c0, 0x3d2b: 0x00c0, 0x3d2c: 0x00c0, 0x3d2d: 0x00c0, 0x3d2e: 0x00c0, 0x3d2f: 0x00c0, + 0x3d30: 0x00c0, 0x3d31: 0x00c0, 0x3d32: 0x00c0, 0x3d33: 0x00c3, 0x3d34: 0x00c3, 0x3d35: 0x00c3, + 0x3d36: 0x00c3, 0x3d37: 0x00c3, 0x3d38: 0x00c3, 0x3d39: 0x00c0, 0x3d3a: 0x00c3, 0x3d3b: 0x00c0, + 0x3d3c: 0x00c0, 0x3d3d: 0x00c0, 0x3d3e: 0x00c0, 0x3d3f: 0x00c3, + // Block 0xf5, offset 0x3d40 + 0x3d40: 0x00c3, 0x3d41: 0x00c0, 0x3d42: 0x00c6, 0x3d43: 0x00c3, 0x3d44: 0x00c0, 0x3d45: 0x00c0, + 0x3d46: 0x0080, 0x3d47: 0x00c0, + 0x3d50: 0x00c0, 0x3d51: 0x00c0, + 0x3d52: 0x00c0, 0x3d53: 0x00c0, 0x3d54: 0x00c0, 0x3d55: 0x00c0, 0x3d56: 0x00c0, 0x3d57: 0x00c0, + 0x3d58: 0x00c0, 0x3d59: 0x00c0, + // Block 0xf6, offset 0x3d80 + 0x3d80: 0x00c0, 0x3d81: 0x00c0, 0x3d82: 0x00c0, 0x3d83: 0x00c0, 0x3d84: 0x00c0, 0x3d85: 0x00c0, + 0x3d86: 0x00c0, 0x3d87: 0x00c0, 0x3d88: 0x00c0, 0x3d89: 0x00c0, 0x3d8a: 0x00c0, 0x3d8b: 0x00c0, + 0x3d8c: 0x00c0, 0x3d8d: 0x00c0, 0x3d8e: 0x00c0, 0x3d8f: 0x00c0, 0x3d90: 0x00c0, 0x3d91: 0x00c0, + 0x3d92: 0x00c0, 0x3d93: 0x00c0, 0x3d94: 0x00c0, 0x3d95: 0x00c0, 0x3d96: 0x00c0, 0x3d97: 0x00c0, + 0x3d98: 0x00c0, 0x3d99: 0x00c0, 0x3d9a: 0x00c0, 0x3d9b: 0x00c0, 0x3d9c: 0x00c0, 0x3d9d: 0x00c0, + 0x3d9e: 0x00c0, 0x3d9f: 0x00c0, 0x3da0: 0x00c0, 0x3da1: 0x00c0, 0x3da2: 0x00c0, 0x3da3: 0x00c0, + 0x3da4: 0x00c0, 0x3da5: 0x00c0, 0x3da6: 0x00c0, 0x3da7: 0x00c0, 0x3da8: 0x00c0, 0x3da9: 0x00c0, + 0x3daa: 0x00c0, 0x3dab: 0x00c0, 0x3dac: 0x00c0, 0x3dad: 0x00c0, 0x3dae: 0x00c0, 0x3daf: 0x00c0, + 0x3db0: 0x00c0, 0x3db1: 0x00c0, 0x3db2: 0x00c3, 0x3db3: 0x00c3, 0x3db4: 0x00c3, 0x3db5: 0x00c3, + 0x3db8: 0x00c0, 0x3db9: 0x00c0, 0x3dba: 0x00c0, 0x3dbb: 0x00c0, + 0x3dbc: 0x00c3, 0x3dbd: 0x00c3, 0x3dbe: 0x00c0, 0x3dbf: 0x00c6, + // Block 0xf7, offset 0x3dc0 + 0x3dc0: 0x00c3, 0x3dc1: 0x0080, 0x3dc2: 0x0080, 0x3dc3: 0x0080, 0x3dc4: 0x0080, 0x3dc5: 0x0080, + 0x3dc6: 0x0080, 0x3dc7: 0x0080, 0x3dc8: 0x0080, 0x3dc9: 0x0080, 0x3dca: 0x0080, 0x3dcb: 0x0080, + 0x3dcc: 0x0080, 0x3dcd: 0x0080, 0x3dce: 0x0080, 0x3dcf: 0x0080, 0x3dd0: 0x0080, 0x3dd1: 0x0080, + 0x3dd2: 0x0080, 0x3dd3: 0x0080, 0x3dd4: 0x0080, 0x3dd5: 0x0080, 0x3dd6: 0x0080, 0x3dd7: 0x0080, + 0x3dd8: 0x00c0, 0x3dd9: 0x00c0, 0x3dda: 0x00c0, 0x3ddb: 0x00c0, 0x3ddc: 0x00c3, 0x3ddd: 0x00c3, + // Block 0xf8, offset 0x3e00 + 0x3e00: 0x00c0, 0x3e01: 0x00c0, 0x3e02: 0x00c0, 0x3e03: 0x00c0, 0x3e04: 0x00c0, 0x3e05: 0x00c0, + 0x3e06: 0x00c0, 0x3e07: 0x00c0, 0x3e08: 0x00c0, 0x3e09: 0x00c0, 0x3e0a: 0x00c0, 0x3e0b: 0x00c0, + 0x3e0c: 0x00c0, 0x3e0d: 0x00c0, 0x3e0e: 0x00c0, 0x3e0f: 0x00c0, 0x3e10: 0x00c0, 0x3e11: 0x00c0, + 0x3e12: 0x00c0, 0x3e13: 0x00c0, 0x3e14: 0x00c0, 0x3e15: 0x00c0, 0x3e16: 0x00c0, 0x3e17: 0x00c0, + 0x3e18: 0x00c0, 0x3e19: 0x00c0, 0x3e1a: 0x00c0, 0x3e1b: 0x00c0, 0x3e1c: 0x00c0, 0x3e1d: 0x00c0, + 0x3e1e: 0x00c0, 0x3e1f: 0x00c0, 0x3e20: 0x00c0, 0x3e21: 0x00c0, 0x3e22: 0x00c0, 0x3e23: 0x00c0, + 0x3e24: 0x00c0, 0x3e25: 0x00c0, 0x3e26: 0x00c0, 0x3e27: 0x00c0, 0x3e28: 0x00c0, 0x3e29: 0x00c0, + 0x3e2a: 0x00c0, 0x3e2b: 0x00c0, 0x3e2c: 0x00c0, 0x3e2d: 0x00c0, 0x3e2e: 0x00c0, 0x3e2f: 0x00c0, + 0x3e30: 0x00c0, 0x3e31: 0x00c0, 0x3e32: 0x00c0, 0x3e33: 0x00c3, 0x3e34: 0x00c3, 0x3e35: 0x00c3, + 0x3e36: 0x00c3, 0x3e37: 0x00c3, 0x3e38: 0x00c3, 0x3e39: 0x00c3, 0x3e3a: 0x00c3, 0x3e3b: 0x00c0, + 0x3e3c: 0x00c0, 0x3e3d: 0x00c3, 0x3e3e: 0x00c0, 0x3e3f: 0x00c6, + // Block 0xf9, offset 0x3e40 + 0x3e40: 0x00c3, 0x3e41: 0x0080, 0x3e42: 0x0080, 0x3e43: 0x0080, 0x3e44: 0x00c0, + 0x3e50: 0x00c0, 0x3e51: 0x00c0, + 0x3e52: 0x00c0, 0x3e53: 0x00c0, 0x3e54: 0x00c0, 0x3e55: 0x00c0, 0x3e56: 0x00c0, 0x3e57: 0x00c0, + 0x3e58: 0x00c0, 0x3e59: 0x00c0, + 0x3e60: 0x0080, 0x3e61: 0x0080, 0x3e62: 0x0080, 0x3e63: 0x0080, + 0x3e64: 0x0080, 0x3e65: 0x0080, 0x3e66: 0x0080, 0x3e67: 0x0080, 0x3e68: 0x0080, 0x3e69: 0x0080, + 0x3e6a: 0x0080, 0x3e6b: 0x0080, 0x3e6c: 0x0080, + // Block 0xfa, offset 0x3e80 + 0x3e80: 0x00c0, 0x3e81: 0x00c0, 0x3e82: 0x00c0, 0x3e83: 0x00c0, 0x3e84: 0x00c0, 0x3e85: 0x00c0, + 0x3e86: 0x00c0, 0x3e87: 0x00c0, 0x3e88: 0x00c0, 0x3e89: 0x00c0, 0x3e8a: 0x00c0, 0x3e8b: 0x00c0, + 0x3e8c: 0x00c0, 0x3e8d: 0x00c0, 0x3e8e: 0x00c0, 0x3e8f: 0x00c0, 0x3e90: 0x00c0, 0x3e91: 0x00c0, + 0x3e92: 0x00c0, 0x3e93: 0x00c0, 0x3e94: 0x00c0, 0x3e95: 0x00c0, 0x3e96: 0x00c0, 0x3e97: 0x00c0, + 0x3e98: 0x00c0, 0x3e99: 0x00c0, 0x3e9a: 0x00c0, 0x3e9b: 0x00c0, 0x3e9c: 0x00c0, 0x3e9d: 0x00c0, + 0x3e9e: 0x00c0, 0x3e9f: 0x00c0, 0x3ea0: 0x00c0, 0x3ea1: 0x00c0, 0x3ea2: 0x00c0, 0x3ea3: 0x00c0, + 0x3ea4: 0x00c0, 0x3ea5: 0x00c0, 0x3ea6: 0x00c0, 0x3ea7: 0x00c0, 0x3ea8: 0x00c0, 0x3ea9: 0x00c0, + 0x3eaa: 0x00c0, 0x3eab: 0x00c3, 0x3eac: 0x00c0, 0x3ead: 0x00c3, 0x3eae: 0x00c0, 0x3eaf: 0x00c0, + 0x3eb0: 0x00c3, 0x3eb1: 0x00c3, 0x3eb2: 0x00c3, 0x3eb3: 0x00c3, 0x3eb4: 0x00c3, 0x3eb5: 0x00c3, + 0x3eb6: 0x00c5, 0x3eb7: 0x00c3, 0x3eb8: 0x00c0, + // Block 0xfb, offset 0x3ec0 + 0x3ec0: 0x00c0, 0x3ec1: 0x00c0, 0x3ec2: 0x00c0, 0x3ec3: 0x00c0, 0x3ec4: 0x00c0, 0x3ec5: 0x00c0, + 0x3ec6: 0x00c0, 0x3ec7: 0x00c0, 0x3ec8: 0x00c0, 0x3ec9: 0x00c0, + // Block 0xfc, offset 0x3f00 + 0x3f00: 0x00c0, 0x3f01: 0x00c0, 0x3f02: 0x00c0, 0x3f03: 0x00c0, 0x3f04: 0x00c0, 0x3f05: 0x00c0, + 0x3f06: 0x00c0, 0x3f07: 0x00c0, 0x3f08: 0x00c0, 0x3f09: 0x00c0, 0x3f0a: 0x00c0, 0x3f0b: 0x00c0, + 0x3f0c: 0x00c0, 0x3f0d: 0x00c0, 0x3f0e: 0x00c0, 0x3f0f: 0x00c0, 0x3f10: 0x00c0, 0x3f11: 0x00c0, + 0x3f12: 0x00c0, 0x3f13: 0x00c0, 0x3f14: 0x00c0, 0x3f15: 0x00c0, 0x3f16: 0x00c0, 0x3f17: 0x00c0, + 0x3f18: 0x00c0, 0x3f19: 0x00c0, 0x3f1a: 0x00c0, 0x3f1d: 0x00c3, + 0x3f1e: 0x00c3, 0x3f1f: 0x00c3, 0x3f20: 0x00c0, 0x3f21: 0x00c0, 0x3f22: 0x00c3, 0x3f23: 0x00c3, + 0x3f24: 0x00c3, 0x3f25: 0x00c3, 0x3f26: 0x00c0, 0x3f27: 0x00c3, 0x3f28: 0x00c3, 0x3f29: 0x00c3, + 0x3f2a: 0x00c3, 0x3f2b: 0x00c6, + 0x3f30: 0x00c0, 0x3f31: 0x00c0, 0x3f32: 0x00c0, 0x3f33: 0x00c0, 0x3f34: 0x00c0, 0x3f35: 0x00c0, + 0x3f36: 0x00c0, 0x3f37: 0x00c0, 0x3f38: 0x00c0, 0x3f39: 0x00c0, 0x3f3a: 0x0080, 0x3f3b: 0x0080, + 0x3f3c: 0x0080, 0x3f3d: 0x0080, 0x3f3e: 0x0080, 0x3f3f: 0x0080, + // Block 0xfd, offset 0x3f40 + 0x3f40: 0x00c0, 0x3f41: 0x00c0, 0x3f42: 0x00c0, 0x3f43: 0x00c0, 0x3f44: 0x00c0, 0x3f45: 0x00c0, + 0x3f46: 0x00c0, 0x3f47: 0x00c0, 0x3f48: 0x00c0, 0x3f49: 0x00c0, 0x3f4a: 0x00c0, 0x3f4b: 0x00c0, + 0x3f4c: 0x00c0, 0x3f4d: 0x00c0, 0x3f4e: 0x00c0, 0x3f4f: 0x00c0, 0x3f50: 0x00c0, 0x3f51: 0x00c0, + 0x3f52: 0x00c0, 0x3f53: 0x00c0, 0x3f54: 0x00c0, 0x3f55: 0x00c0, 0x3f56: 0x00c0, 0x3f57: 0x00c0, + 0x3f58: 0x00c0, 0x3f59: 0x00c0, 0x3f5a: 0x00c0, 0x3f5b: 0x00c0, 0x3f5c: 0x00c0, 0x3f5d: 0x00c0, + 0x3f5e: 0x00c0, 0x3f5f: 0x00c0, 0x3f60: 0x00c0, 0x3f61: 0x00c0, 0x3f62: 0x00c0, 0x3f63: 0x00c0, + 0x3f64: 0x00c0, 0x3f65: 0x00c0, 0x3f66: 0x00c0, 0x3f67: 0x00c0, 0x3f68: 0x00c0, 0x3f69: 0x00c0, + 0x3f6a: 0x00c0, 0x3f6b: 0x00c0, 0x3f6c: 0x00c0, 0x3f6d: 0x00c0, 0x3f6e: 0x00c0, 0x3f6f: 0x00c3, + 0x3f70: 0x00c3, 0x3f71: 0x00c3, 0x3f72: 0x00c3, 0x3f73: 0x00c3, 0x3f74: 0x00c3, 0x3f75: 0x00c3, + 0x3f76: 0x00c3, 0x3f77: 0x00c3, 0x3f78: 0x00c0, 0x3f79: 0x00c6, 0x3f7a: 0x00c3, 0x3f7b: 0x0080, + // Block 0xfe, offset 0x3f80 + 0x3fa0: 0x00c0, 0x3fa1: 0x00c0, 0x3fa2: 0x00c0, 0x3fa3: 0x00c0, + 0x3fa4: 0x00c0, 0x3fa5: 0x00c0, 0x3fa6: 0x00c0, 0x3fa7: 0x00c0, 0x3fa8: 0x00c0, 0x3fa9: 0x00c0, + 0x3faa: 0x00c0, 0x3fab: 0x00c0, 0x3fac: 0x00c0, 0x3fad: 0x00c0, 0x3fae: 0x00c0, 0x3faf: 0x00c0, + 0x3fb0: 0x00c0, 0x3fb1: 0x00c0, 0x3fb2: 0x00c0, 0x3fb3: 0x00c0, 0x3fb4: 0x00c0, 0x3fb5: 0x00c0, + 0x3fb6: 0x00c0, 0x3fb7: 0x00c0, 0x3fb8: 0x00c0, 0x3fb9: 0x00c0, 0x3fba: 0x00c0, 0x3fbb: 0x00c0, + 0x3fbc: 0x00c0, 0x3fbd: 0x00c0, 0x3fbe: 0x00c0, 0x3fbf: 0x00c0, + // Block 0xff, offset 0x3fc0 + 0x3fc0: 0x00c0, 0x3fc1: 0x00c0, 0x3fc2: 0x00c0, 0x3fc3: 0x00c0, 0x3fc4: 0x00c0, 0x3fc5: 0x00c0, + 0x3fc6: 0x00c0, 0x3fc7: 0x00c0, 0x3fc8: 0x00c0, 0x3fc9: 0x00c0, 0x3fca: 0x00c0, 0x3fcb: 0x00c0, + 0x3fcc: 0x00c0, 0x3fcd: 0x00c0, 0x3fce: 0x00c0, 0x3fcf: 0x00c0, 0x3fd0: 0x00c0, 0x3fd1: 0x00c0, + 0x3fd2: 0x00c0, 0x3fd3: 0x00c0, 0x3fd4: 0x00c0, 0x3fd5: 0x00c0, 0x3fd6: 0x00c0, 0x3fd7: 0x00c0, + 0x3fd8: 0x00c0, 0x3fd9: 0x00c0, 0x3fda: 0x00c0, 0x3fdb: 0x00c0, 0x3fdc: 0x00c0, 0x3fdd: 0x00c0, + 0x3fde: 0x00c0, 0x3fdf: 0x00c0, 0x3fe0: 0x00c0, 0x3fe1: 0x00c0, 0x3fe2: 0x00c0, 0x3fe3: 0x00c0, + 0x3fe4: 0x00c0, 0x3fe5: 0x00c0, 0x3fe6: 0x00c0, 0x3fe7: 0x00c0, 0x3fe8: 0x00c0, 0x3fe9: 0x00c0, + 0x3fea: 0x0080, 0x3feb: 0x0080, 0x3fec: 0x0080, 0x3fed: 0x0080, 0x3fee: 0x0080, 0x3fef: 0x0080, + 0x3ff0: 0x0080, 0x3ff1: 0x0080, 0x3ff2: 0x0080, + 0x3fff: 0x00c0, + // Block 0x100, offset 0x4000 + 0x4020: 0x00c0, 0x4021: 0x00c0, 0x4022: 0x00c0, 0x4023: 0x00c0, + 0x4024: 0x00c0, 0x4025: 0x00c0, 0x4026: 0x00c0, 0x4027: 0x00c0, + 0x402a: 0x00c0, 0x402b: 0x00c0, 0x402c: 0x00c0, 0x402d: 0x00c0, 0x402e: 0x00c0, 0x402f: 0x00c0, + 0x4030: 0x00c0, 0x4031: 0x00c0, 0x4032: 0x00c0, 0x4033: 0x00c0, 0x4034: 0x00c0, 0x4035: 0x00c0, + 0x4036: 0x00c0, 0x4037: 0x00c0, 0x4038: 0x00c0, 0x4039: 0x00c0, 0x403a: 0x00c0, 0x403b: 0x00c0, + 0x403c: 0x00c0, 0x403d: 0x00c0, 0x403e: 0x00c0, 0x403f: 0x00c0, + // Block 0x101, offset 0x4040 + 0x4040: 0x00c0, 0x4041: 0x00c0, 0x4042: 0x00c0, 0x4043: 0x00c0, 0x4044: 0x00c0, 0x4045: 0x00c0, + 0x4046: 0x00c0, 0x4047: 0x00c0, 0x4048: 0x00c0, 0x4049: 0x00c0, 0x404a: 0x00c0, 0x404b: 0x00c0, + 0x404c: 0x00c0, 0x404d: 0x00c0, 0x404e: 0x00c0, 0x404f: 0x00c0, 0x4050: 0x00c0, 0x4051: 0x00c0, + 0x4052: 0x00c0, 0x4053: 0x00c0, 0x4054: 0x00c3, 0x4055: 0x00c3, 0x4056: 0x00c3, 0x4057: 0x00c3, + 0x405a: 0x00c3, 0x405b: 0x00c3, 0x405c: 0x00c0, 0x405d: 0x00c0, + 0x405e: 0x00c0, 0x405f: 0x00c0, 0x4060: 0x00c6, 0x4061: 0x00c0, 0x4062: 0x0080, 0x4063: 0x00c0, + 0x4064: 0x00c0, + // Block 0x102, offset 0x4080 + 0x4080: 0x00c0, 0x4081: 0x00c3, 0x4082: 0x00c3, 0x4083: 0x00c3, 0x4084: 0x00c3, 0x4085: 0x00c3, + 0x4086: 0x00c3, 0x4087: 0x00c3, 0x4088: 0x00c3, 0x4089: 0x00c3, 0x408a: 0x00c3, 0x408b: 0x00c0, + 0x408c: 0x00c0, 0x408d: 0x00c0, 0x408e: 0x00c0, 0x408f: 0x00c0, 0x4090: 0x00c0, 0x4091: 0x00c0, + 0x4092: 0x00c0, 0x4093: 0x00c0, 0x4094: 0x00c0, 0x4095: 0x00c0, 0x4096: 0x00c0, 0x4097: 0x00c0, + 0x4098: 0x00c0, 0x4099: 0x00c0, 0x409a: 0x00c0, 0x409b: 0x00c0, 0x409c: 0x00c0, 0x409d: 0x00c0, + 0x409e: 0x00c0, 0x409f: 0x00c0, 0x40a0: 0x00c0, 0x40a1: 0x00c0, 0x40a2: 0x00c0, 0x40a3: 0x00c0, + 0x40a4: 0x00c0, 0x40a5: 0x00c0, 0x40a6: 0x00c0, 0x40a7: 0x00c0, 0x40a8: 0x00c0, 0x40a9: 0x00c0, + 0x40aa: 0x00c0, 0x40ab: 0x00c0, 0x40ac: 0x00c0, 0x40ad: 0x00c0, 0x40ae: 0x00c0, 0x40af: 0x00c0, + 0x40b0: 0x00c0, 0x40b1: 0x00c0, 0x40b2: 0x00c0, 0x40b3: 0x00c3, 0x40b4: 0x00c6, 0x40b5: 0x00c3, + 0x40b6: 0x00c3, 0x40b7: 0x00c3, 0x40b8: 0x00c3, 0x40b9: 0x00c0, 0x40ba: 0x00c0, 0x40bb: 0x00c3, + 0x40bc: 0x00c3, 0x40bd: 0x00c3, 0x40be: 0x00c3, 0x40bf: 0x0080, + // Block 0x103, offset 0x40c0 + 0x40c0: 0x0080, 0x40c1: 0x0080, 0x40c2: 0x0080, 0x40c3: 0x0080, 0x40c4: 0x0080, 0x40c5: 0x0080, + 0x40c6: 0x0080, 0x40c7: 0x00c6, + 0x40d0: 0x00c0, 0x40d1: 0x00c3, + 0x40d2: 0x00c3, 0x40d3: 0x00c3, 0x40d4: 0x00c3, 0x40d5: 0x00c3, 0x40d6: 0x00c3, 0x40d7: 0x00c0, + 0x40d8: 0x00c0, 0x40d9: 0x00c3, 0x40da: 0x00c3, 0x40db: 0x00c3, 0x40dc: 0x00c0, 0x40dd: 0x00c0, + 0x40de: 0x00c0, 0x40df: 0x00c0, 0x40e0: 0x00c0, 0x40e1: 0x00c0, 0x40e2: 0x00c0, 0x40e3: 0x00c0, + 0x40e4: 0x00c0, 0x40e5: 0x00c0, 0x40e6: 0x00c0, 0x40e7: 0x00c0, 0x40e8: 0x00c0, 0x40e9: 0x00c0, + 0x40ea: 0x00c0, 0x40eb: 0x00c0, 0x40ec: 0x00c0, 0x40ed: 0x00c0, 0x40ee: 0x00c0, 0x40ef: 0x00c0, + 0x40f0: 0x00c0, 0x40f1: 0x00c0, 0x40f2: 0x00c0, 0x40f3: 0x00c0, 0x40f4: 0x00c0, 0x40f5: 0x00c0, + 0x40f6: 0x00c0, 0x40f7: 0x00c0, 0x40f8: 0x00c0, 0x40f9: 0x00c0, 0x40fa: 0x00c0, 0x40fb: 0x00c0, + 0x40fc: 0x00c0, 0x40fd: 0x00c0, 0x40fe: 0x00c0, 0x40ff: 0x00c0, + // Block 0x104, offset 0x4100 + 0x4100: 0x00c0, 0x4101: 0x00c0, 0x4102: 0x00c0, 0x4103: 0x00c0, 0x4104: 0x00c0, 0x4105: 0x00c0, + 0x4106: 0x00c0, 0x4107: 0x00c0, 0x4108: 0x00c0, 0x4109: 0x00c0, 0x410a: 0x00c3, 0x410b: 0x00c3, + 0x410c: 0x00c3, 0x410d: 0x00c3, 0x410e: 0x00c3, 0x410f: 0x00c3, 0x4110: 0x00c3, 0x4111: 0x00c3, + 0x4112: 0x00c3, 0x4113: 0x00c3, 0x4114: 0x00c3, 0x4115: 0x00c3, 0x4116: 0x00c3, 0x4117: 0x00c0, + 0x4118: 0x00c3, 0x4119: 0x00c6, 0x411a: 0x0080, 0x411b: 0x0080, 0x411c: 0x0080, 0x411d: 0x00c0, + 0x411e: 0x0080, 0x411f: 0x0080, 0x4120: 0x0080, 0x4121: 0x0080, 0x4122: 0x0080, + // Block 0x105, offset 0x4140 + 0x4140: 0x00c0, 0x4141: 0x00c0, 0x4142: 0x00c0, 0x4143: 0x00c0, 0x4144: 0x00c0, 0x4145: 0x00c0, + 0x4146: 0x00c0, 0x4147: 0x00c0, 0x4148: 0x00c0, 0x4149: 0x00c0, 0x414a: 0x00c0, 0x414b: 0x00c0, + 0x414c: 0x00c0, 0x414d: 0x00c0, 0x414e: 0x00c0, 0x414f: 0x00c0, 0x4150: 0x00c0, 0x4151: 0x00c0, + 0x4152: 0x00c0, 0x4153: 0x00c0, 0x4154: 0x00c0, 0x4155: 0x00c0, 0x4156: 0x00c0, 0x4157: 0x00c0, + 0x4158: 0x00c0, 0x4159: 0x00c0, 0x415a: 0x00c0, 0x415b: 0x00c0, 0x415c: 0x00c0, 0x415d: 0x00c0, + 0x415e: 0x00c0, 0x415f: 0x00c0, 0x4160: 0x00c0, 0x4161: 0x00c0, 0x4162: 0x00c0, 0x4163: 0x00c0, + 0x4164: 0x00c0, 0x4165: 0x00c0, 0x4166: 0x00c0, 0x4167: 0x00c0, 0x4168: 0x00c0, 0x4169: 0x00c0, + 0x416a: 0x00c0, 0x416b: 0x00c0, 0x416c: 0x00c0, 0x416d: 0x00c0, 0x416e: 0x00c0, 0x416f: 0x00c0, + 0x4170: 0x00c0, 0x4171: 0x00c0, 0x4172: 0x00c0, 0x4173: 0x00c0, 0x4174: 0x00c0, 0x4175: 0x00c0, + 0x4176: 0x00c0, 0x4177: 0x00c0, 0x4178: 0x00c0, + // Block 0x106, offset 0x4180 + 0x4180: 0x00c0, 0x4181: 0x00c0, 0x4182: 0x00c0, 0x4183: 0x00c0, 0x4184: 0x00c0, 0x4185: 0x00c0, + 0x4186: 0x00c0, 0x4187: 0x00c0, 0x4188: 0x00c0, 0x418a: 0x00c0, 0x418b: 0x00c0, + 0x418c: 0x00c0, 0x418d: 0x00c0, 0x418e: 0x00c0, 0x418f: 0x00c0, 0x4190: 0x00c0, 0x4191: 0x00c0, + 0x4192: 0x00c0, 0x4193: 0x00c0, 0x4194: 0x00c0, 0x4195: 0x00c0, 0x4196: 0x00c0, 0x4197: 0x00c0, + 0x4198: 0x00c0, 0x4199: 0x00c0, 0x419a: 0x00c0, 0x419b: 0x00c0, 0x419c: 0x00c0, 0x419d: 0x00c0, + 0x419e: 0x00c0, 0x419f: 0x00c0, 0x41a0: 0x00c0, 0x41a1: 0x00c0, 0x41a2: 0x00c0, 0x41a3: 0x00c0, + 0x41a4: 0x00c0, 0x41a5: 0x00c0, 0x41a6: 0x00c0, 0x41a7: 0x00c0, 0x41a8: 0x00c0, 0x41a9: 0x00c0, + 0x41aa: 0x00c0, 0x41ab: 0x00c0, 0x41ac: 0x00c0, 0x41ad: 0x00c0, 0x41ae: 0x00c0, 0x41af: 0x00c0, + 0x41b0: 0x00c3, 0x41b1: 0x00c3, 0x41b2: 0x00c3, 0x41b3: 0x00c3, 0x41b4: 0x00c3, 0x41b5: 0x00c3, + 0x41b6: 0x00c3, 0x41b8: 0x00c3, 0x41b9: 0x00c3, 0x41ba: 0x00c3, 0x41bb: 0x00c3, + 0x41bc: 0x00c3, 0x41bd: 0x00c3, 0x41be: 0x00c0, 0x41bf: 0x00c6, + // Block 0x107, offset 0x41c0 + 0x41c0: 0x00c0, 0x41c1: 0x0080, 0x41c2: 0x0080, 0x41c3: 0x0080, 0x41c4: 0x0080, 0x41c5: 0x0080, + 0x41d0: 0x00c0, 0x41d1: 0x00c0, + 0x41d2: 0x00c0, 0x41d3: 0x00c0, 0x41d4: 0x00c0, 0x41d5: 0x00c0, 0x41d6: 0x00c0, 0x41d7: 0x00c0, + 0x41d8: 0x00c0, 0x41d9: 0x00c0, 0x41da: 0x0080, 0x41db: 0x0080, 0x41dc: 0x0080, 0x41dd: 0x0080, + 0x41de: 0x0080, 0x41df: 0x0080, 0x41e0: 0x0080, 0x41e1: 0x0080, 0x41e2: 0x0080, 0x41e3: 0x0080, + 0x41e4: 0x0080, 0x41e5: 0x0080, 0x41e6: 0x0080, 0x41e7: 0x0080, 0x41e8: 0x0080, 0x41e9: 0x0080, + 0x41ea: 0x0080, 0x41eb: 0x0080, 0x41ec: 0x0080, + 0x41f0: 0x0080, 0x41f1: 0x0080, 0x41f2: 0x00c0, 0x41f3: 0x00c0, 0x41f4: 0x00c0, 0x41f5: 0x00c0, + 0x41f6: 0x00c0, 0x41f7: 0x00c0, 0x41f8: 0x00c0, 0x41f9: 0x00c0, 0x41fa: 0x00c0, 0x41fb: 0x00c0, + 0x41fc: 0x00c0, 0x41fd: 0x00c0, 0x41fe: 0x00c0, 0x41ff: 0x00c0, + // Block 0x108, offset 0x4200 + 0x4200: 0x00c0, 0x4201: 0x00c0, 0x4202: 0x00c0, 0x4203: 0x00c0, 0x4204: 0x00c0, 0x4205: 0x00c0, + 0x4206: 0x00c0, 0x4207: 0x00c0, 0x4208: 0x00c0, 0x4209: 0x00c0, 0x420a: 0x00c0, 0x420b: 0x00c0, + 0x420c: 0x00c0, 0x420d: 0x00c0, 0x420e: 0x00c0, 0x420f: 0x00c0, + 0x4212: 0x00c3, 0x4213: 0x00c3, 0x4214: 0x00c3, 0x4215: 0x00c3, 0x4216: 0x00c3, 0x4217: 0x00c3, + 0x4218: 0x00c3, 0x4219: 0x00c3, 0x421a: 0x00c3, 0x421b: 0x00c3, 0x421c: 0x00c3, 0x421d: 0x00c3, + 0x421e: 0x00c3, 0x421f: 0x00c3, 0x4220: 0x00c3, 0x4221: 0x00c3, 0x4222: 0x00c3, 0x4223: 0x00c3, + 0x4224: 0x00c3, 0x4225: 0x00c3, 0x4226: 0x00c3, 0x4227: 0x00c3, 0x4229: 0x00c0, + 0x422a: 0x00c3, 0x422b: 0x00c3, 0x422c: 0x00c3, 0x422d: 0x00c3, 0x422e: 0x00c3, 0x422f: 0x00c3, + 0x4230: 0x00c3, 0x4231: 0x00c0, 0x4232: 0x00c3, 0x4233: 0x00c3, 0x4234: 0x00c0, 0x4235: 0x00c3, + 0x4236: 0x00c3, + // Block 0x109, offset 0x4240 + 0x4240: 0x00c0, 0x4241: 0x00c0, 0x4242: 0x00c0, 0x4243: 0x00c0, 0x4244: 0x00c0, 0x4245: 0x00c0, + 0x4246: 0x00c0, 0x4248: 0x00c0, 0x4249: 0x00c0, 0x424b: 0x00c0, + 0x424c: 0x00c0, 0x424d: 0x00c0, 0x424e: 0x00c0, 0x424f: 0x00c0, 0x4250: 0x00c0, 0x4251: 0x00c0, + 0x4252: 0x00c0, 0x4253: 0x00c0, 0x4254: 0x00c0, 0x4255: 0x00c0, 0x4256: 0x00c0, 0x4257: 0x00c0, + 0x4258: 0x00c0, 0x4259: 0x00c0, 0x425a: 0x00c0, 0x425b: 0x00c0, 0x425c: 0x00c0, 0x425d: 0x00c0, + 0x425e: 0x00c0, 0x425f: 0x00c0, 0x4260: 0x00c0, 0x4261: 0x00c0, 0x4262: 0x00c0, 0x4263: 0x00c0, + 0x4264: 0x00c0, 0x4265: 0x00c0, 0x4266: 0x00c0, 0x4267: 0x00c0, 0x4268: 0x00c0, 0x4269: 0x00c0, + 0x426a: 0x00c0, 0x426b: 0x00c0, 0x426c: 0x00c0, 0x426d: 0x00c0, 0x426e: 0x00c0, 0x426f: 0x00c0, + 0x4270: 0x00c0, 0x4271: 0x00c3, 0x4272: 0x00c3, 0x4273: 0x00c3, 0x4274: 0x00c3, 0x4275: 0x00c3, + 0x4276: 0x00c3, 0x427a: 0x00c3, + 0x427c: 0x00c3, 0x427d: 0x00c3, 0x427f: 0x00c3, + // Block 0x10a, offset 0x4280 + 0x4280: 0x00c3, 0x4281: 0x00c3, 0x4282: 0x00c3, 0x4283: 0x00c3, 0x4284: 0x00c6, 0x4285: 0x00c6, + 0x4286: 0x00c0, 0x4287: 0x00c3, + 0x4290: 0x00c0, 0x4291: 0x00c0, + 0x4292: 0x00c0, 0x4293: 0x00c0, 0x4294: 0x00c0, 0x4295: 0x00c0, 0x4296: 0x00c0, 0x4297: 0x00c0, + 0x4298: 0x00c0, 0x4299: 0x00c0, + 0x42a0: 0x00c0, 0x42a1: 0x00c0, 0x42a2: 0x00c0, 0x42a3: 0x00c0, + 0x42a4: 0x00c0, 0x42a5: 0x00c0, 0x42a7: 0x00c0, 0x42a8: 0x00c0, + 0x42aa: 0x00c0, 0x42ab: 0x00c0, 0x42ac: 0x00c0, 0x42ad: 0x00c0, 0x42ae: 0x00c0, 0x42af: 0x00c0, + 0x42b0: 0x00c0, 0x42b1: 0x00c0, 0x42b2: 0x00c0, 0x42b3: 0x00c0, 0x42b4: 0x00c0, 0x42b5: 0x00c0, + 0x42b6: 0x00c0, 0x42b7: 0x00c0, 0x42b8: 0x00c0, 0x42b9: 0x00c0, 0x42ba: 0x00c0, 0x42bb: 0x00c0, + 0x42bc: 0x00c0, 0x42bd: 0x00c0, 0x42be: 0x00c0, 0x42bf: 0x00c0, + // Block 0x10b, offset 0x42c0 + 0x42c0: 0x00c0, 0x42c1: 0x00c0, 0x42c2: 0x00c0, 0x42c3: 0x00c0, 0x42c4: 0x00c0, 0x42c5: 0x00c0, + 0x42c6: 0x00c0, 0x42c7: 0x00c0, 0x42c8: 0x00c0, 0x42c9: 0x00c0, 0x42ca: 0x00c0, 0x42cb: 0x00c0, + 0x42cc: 0x00c0, 0x42cd: 0x00c0, 0x42ce: 0x00c0, 0x42d0: 0x00c3, 0x42d1: 0x00c3, + 0x42d3: 0x00c0, 0x42d4: 0x00c0, 0x42d5: 0x00c3, 0x42d6: 0x00c0, 0x42d7: 0x00c6, + 0x42d8: 0x00c0, + 0x42e0: 0x00c0, 0x42e1: 0x00c0, 0x42e2: 0x00c0, 0x42e3: 0x00c0, + 0x42e4: 0x00c0, 0x42e5: 0x00c0, 0x42e6: 0x00c0, 0x42e7: 0x00c0, 0x42e8: 0x00c0, 0x42e9: 0x00c0, + // Block 0x10c, offset 0x4300 + 0x4320: 0x00c0, 0x4321: 0x00c0, 0x4322: 0x00c0, 0x4323: 0x00c0, + 0x4324: 0x00c0, 0x4325: 0x00c0, 0x4326: 0x00c0, 0x4327: 0x00c0, 0x4328: 0x00c0, 0x4329: 0x00c0, + 0x432a: 0x00c0, 0x432b: 0x00c0, 0x432c: 0x00c0, 0x432d: 0x00c0, 0x432e: 0x00c0, 0x432f: 0x00c0, + 0x4330: 0x00c0, 0x4331: 0x00c0, 0x4332: 0x00c0, 0x4333: 0x00c3, 0x4334: 0x00c3, 0x4335: 0x00c0, + 0x4336: 0x00c0, 0x4337: 0x0080, 0x4338: 0x0080, + // Block 0x10d, offset 0x4340 + 0x4340: 0x0080, 0x4341: 0x0080, 0x4342: 0x0080, 0x4343: 0x0080, 0x4344: 0x0080, 0x4345: 0x0080, + 0x4346: 0x0080, 0x4347: 0x0080, 0x4348: 0x0080, 0x4349: 0x0080, 0x434a: 0x0080, 0x434b: 0x0080, + 0x434c: 0x0080, 0x434d: 0x0080, 0x434e: 0x0080, 0x434f: 0x0080, 0x4350: 0x0080, 0x4351: 0x0080, + 0x4352: 0x0080, 0x4353: 0x0080, 0x4354: 0x0080, 0x4355: 0x0080, 0x4356: 0x0080, 0x4357: 0x0080, + 0x4358: 0x0080, 0x4359: 0x0080, 0x435a: 0x0080, 0x435b: 0x0080, 0x435c: 0x0080, 0x435d: 0x0080, + 0x435e: 0x0080, 0x435f: 0x0080, 0x4360: 0x0080, 0x4361: 0x0080, 0x4362: 0x0080, 0x4363: 0x0080, + 0x4364: 0x0080, 0x4365: 0x0080, 0x4366: 0x0080, 0x4367: 0x0080, 0x4368: 0x0080, 0x4369: 0x0080, + 0x436a: 0x0080, 0x436b: 0x0080, 0x436c: 0x0080, 0x436d: 0x0080, 0x436e: 0x0080, 0x436f: 0x0080, + 0x4370: 0x0080, 0x4371: 0x0080, + 0x437f: 0x0080, + // Block 0x10e, offset 0x4380 + 0x4380: 0x00c0, 0x4381: 0x00c0, 0x4382: 0x00c0, 0x4383: 0x00c0, 0x4384: 0x00c0, 0x4385: 0x00c0, + 0x4386: 0x00c0, 0x4387: 0x00c0, 0x4388: 0x00c0, 0x4389: 0x00c0, 0x438a: 0x00c0, 0x438b: 0x00c0, + 0x438c: 0x00c0, 0x438d: 0x00c0, 0x438e: 0x00c0, 0x438f: 0x00c0, 0x4390: 0x00c0, 0x4391: 0x00c0, + 0x4392: 0x00c0, 0x4393: 0x00c0, 0x4394: 0x00c0, 0x4395: 0x00c0, 0x4396: 0x00c0, 0x4397: 0x00c0, + 0x4398: 0x00c0, 0x4399: 0x00c0, + // Block 0x10f, offset 0x43c0 + 0x43c0: 0x0080, 0x43c1: 0x0080, 0x43c2: 0x0080, 0x43c3: 0x0080, 0x43c4: 0x0080, 0x43c5: 0x0080, + 0x43c6: 0x0080, 0x43c7: 0x0080, 0x43c8: 0x0080, 0x43c9: 0x0080, 0x43ca: 0x0080, 0x43cb: 0x0080, + 0x43cc: 0x0080, 0x43cd: 0x0080, 0x43ce: 0x0080, 0x43cf: 0x0080, 0x43d0: 0x0080, 0x43d1: 0x0080, + 0x43d2: 0x0080, 0x43d3: 0x0080, 0x43d4: 0x0080, 0x43d5: 0x0080, 0x43d6: 0x0080, 0x43d7: 0x0080, + 0x43d8: 0x0080, 0x43d9: 0x0080, 0x43da: 0x0080, 0x43db: 0x0080, 0x43dc: 0x0080, 0x43dd: 0x0080, + 0x43de: 0x0080, 0x43df: 0x0080, 0x43e0: 0x0080, 0x43e1: 0x0080, 0x43e2: 0x0080, 0x43e3: 0x0080, + 0x43e4: 0x0080, 0x43e5: 0x0080, 0x43e6: 0x0080, 0x43e7: 0x0080, 0x43e8: 0x0080, 0x43e9: 0x0080, + 0x43ea: 0x0080, 0x43eb: 0x0080, 0x43ec: 0x0080, 0x43ed: 0x0080, 0x43ee: 0x0080, + 0x43f0: 0x0080, 0x43f1: 0x0080, 0x43f2: 0x0080, 0x43f3: 0x0080, 0x43f4: 0x0080, + // Block 0x110, offset 0x4400 + 0x4400: 0x00c0, 0x4401: 0x00c0, 0x4402: 0x00c0, 0x4403: 0x00c0, + // Block 0x111, offset 0x4440 + 0x4440: 0x00c0, 0x4441: 0x00c0, 0x4442: 0x00c0, 0x4443: 0x00c0, 0x4444: 0x00c0, 0x4445: 0x00c0, + 0x4446: 0x00c0, 0x4447: 0x00c0, 0x4448: 0x00c0, 0x4449: 0x00c0, 0x444a: 0x00c0, 0x444b: 0x00c0, + 0x444c: 0x00c0, 0x444d: 0x00c0, 0x444e: 0x00c0, 0x444f: 0x00c0, 0x4450: 0x00c0, 0x4451: 0x00c0, + 0x4452: 0x00c0, 0x4453: 0x00c0, 0x4454: 0x00c0, 0x4455: 0x00c0, 0x4456: 0x00c0, 0x4457: 0x00c0, + 0x4458: 0x00c0, 0x4459: 0x00c0, 0x445a: 0x00c0, 0x445b: 0x00c0, 0x445c: 0x00c0, 0x445d: 0x00c0, + 0x445e: 0x00c0, 0x445f: 0x00c0, 0x4460: 0x00c0, 0x4461: 0x00c0, 0x4462: 0x00c0, 0x4463: 0x00c0, + 0x4464: 0x00c0, 0x4465: 0x00c0, 0x4466: 0x00c0, 0x4467: 0x00c0, 0x4468: 0x00c0, 0x4469: 0x00c0, + 0x446a: 0x00c0, 0x446b: 0x00c0, 0x446c: 0x00c0, 0x446d: 0x00c0, 0x446e: 0x00c0, + 0x4470: 0x0040, 0x4471: 0x0040, 0x4472: 0x0040, 0x4473: 0x0040, 0x4474: 0x0040, 0x4475: 0x0040, + 0x4476: 0x0040, 0x4477: 0x0040, 0x4478: 0x0040, + // Block 0x112, offset 0x4480 + 0x4480: 0x00c0, 0x4481: 0x00c0, 0x4482: 0x00c0, 0x4483: 0x00c0, 0x4484: 0x00c0, 0x4485: 0x00c0, + 0x4486: 0x00c0, + // Block 0x113, offset 0x44c0 + 0x44c0: 0x00c0, 0x44c1: 0x00c0, 0x44c2: 0x00c0, 0x44c3: 0x00c0, 0x44c4: 0x00c0, 0x44c5: 0x00c0, + 0x44c6: 0x00c0, 0x44c7: 0x00c0, 0x44c8: 0x00c0, 0x44c9: 0x00c0, 0x44ca: 0x00c0, 0x44cb: 0x00c0, + 0x44cc: 0x00c0, 0x44cd: 0x00c0, 0x44ce: 0x00c0, 0x44cf: 0x00c0, 0x44d0: 0x00c0, 0x44d1: 0x00c0, + 0x44d2: 0x00c0, 0x44d3: 0x00c0, 0x44d4: 0x00c0, 0x44d5: 0x00c0, 0x44d6: 0x00c0, 0x44d7: 0x00c0, + 0x44d8: 0x00c0, 0x44d9: 0x00c0, 0x44da: 0x00c0, 0x44db: 0x00c0, 0x44dc: 0x00c0, 0x44dd: 0x00c0, + 0x44de: 0x00c0, 0x44e0: 0x00c0, 0x44e1: 0x00c0, 0x44e2: 0x00c0, 0x44e3: 0x00c0, + 0x44e4: 0x00c0, 0x44e5: 0x00c0, 0x44e6: 0x00c0, 0x44e7: 0x00c0, 0x44e8: 0x00c0, 0x44e9: 0x00c0, + 0x44ee: 0x0080, 0x44ef: 0x0080, + // Block 0x114, offset 0x4500 + 0x4510: 0x00c0, 0x4511: 0x00c0, + 0x4512: 0x00c0, 0x4513: 0x00c0, 0x4514: 0x00c0, 0x4515: 0x00c0, 0x4516: 0x00c0, 0x4517: 0x00c0, + 0x4518: 0x00c0, 0x4519: 0x00c0, 0x451a: 0x00c0, 0x451b: 0x00c0, 0x451c: 0x00c0, 0x451d: 0x00c0, + 0x451e: 0x00c0, 0x451f: 0x00c0, 0x4520: 0x00c0, 0x4521: 0x00c0, 0x4522: 0x00c0, 0x4523: 0x00c0, + 0x4524: 0x00c0, 0x4525: 0x00c0, 0x4526: 0x00c0, 0x4527: 0x00c0, 0x4528: 0x00c0, 0x4529: 0x00c0, + 0x452a: 0x00c0, 0x452b: 0x00c0, 0x452c: 0x00c0, 0x452d: 0x00c0, + 0x4530: 0x00c3, 0x4531: 0x00c3, 0x4532: 0x00c3, 0x4533: 0x00c3, 0x4534: 0x00c3, 0x4535: 0x0080, + // Block 0x115, offset 0x4540 + 0x4540: 0x00c0, 0x4541: 0x00c0, 0x4542: 0x00c0, 0x4543: 0x00c0, 0x4544: 0x00c0, 0x4545: 0x00c0, + 0x4546: 0x00c0, 0x4547: 0x00c0, 0x4548: 0x00c0, 0x4549: 0x00c0, 0x454a: 0x00c0, 0x454b: 0x00c0, + 0x454c: 0x00c0, 0x454d: 0x00c0, 0x454e: 0x00c0, 0x454f: 0x00c0, 0x4550: 0x00c0, 0x4551: 0x00c0, + 0x4552: 0x00c0, 0x4553: 0x00c0, 0x4554: 0x00c0, 0x4555: 0x00c0, 0x4556: 0x00c0, 0x4557: 0x00c0, + 0x4558: 0x00c0, 0x4559: 0x00c0, 0x455a: 0x00c0, 0x455b: 0x00c0, 0x455c: 0x00c0, 0x455d: 0x00c0, + 0x455e: 0x00c0, 0x455f: 0x00c0, 0x4560: 0x00c0, 0x4561: 0x00c0, 0x4562: 0x00c0, 0x4563: 0x00c0, + 0x4564: 0x00c0, 0x4565: 0x00c0, 0x4566: 0x00c0, 0x4567: 0x00c0, 0x4568: 0x00c0, 0x4569: 0x00c0, + 0x456a: 0x00c0, 0x456b: 0x00c0, 0x456c: 0x00c0, 0x456d: 0x00c0, 0x456e: 0x00c0, 0x456f: 0x00c0, + 0x4570: 0x00c3, 0x4571: 0x00c3, 0x4572: 0x00c3, 0x4573: 0x00c3, 0x4574: 0x00c3, 0x4575: 0x00c3, + 0x4576: 0x00c3, 0x4577: 0x0080, 0x4578: 0x0080, 0x4579: 0x0080, 0x457a: 0x0080, 0x457b: 0x0080, + 0x457c: 0x0080, 0x457d: 0x0080, 0x457e: 0x0080, 0x457f: 0x0080, + // Block 0x116, offset 0x4580 + 0x4580: 0x00c0, 0x4581: 0x00c0, 0x4582: 0x00c0, 0x4583: 0x00c0, 0x4584: 0x0080, 0x4585: 0x0080, + 0x4590: 0x00c0, 0x4591: 0x00c0, + 0x4592: 0x00c0, 0x4593: 0x00c0, 0x4594: 0x00c0, 0x4595: 0x00c0, 0x4596: 0x00c0, 0x4597: 0x00c0, + 0x4598: 0x00c0, 0x4599: 0x00c0, 0x459b: 0x0080, 0x459c: 0x0080, 0x459d: 0x0080, + 0x459e: 0x0080, 0x459f: 0x0080, 0x45a0: 0x0080, 0x45a1: 0x0080, 0x45a3: 0x00c0, + 0x45a4: 0x00c0, 0x45a5: 0x00c0, 0x45a6: 0x00c0, 0x45a7: 0x00c0, 0x45a8: 0x00c0, 0x45a9: 0x00c0, + 0x45aa: 0x00c0, 0x45ab: 0x00c0, 0x45ac: 0x00c0, 0x45ad: 0x00c0, 0x45ae: 0x00c0, 0x45af: 0x00c0, + 0x45b0: 0x00c0, 0x45b1: 0x00c0, 0x45b2: 0x00c0, 0x45b3: 0x00c0, 0x45b4: 0x00c0, 0x45b5: 0x00c0, + 0x45b6: 0x00c0, 0x45b7: 0x00c0, + 0x45bd: 0x00c0, 0x45be: 0x00c0, 0x45bf: 0x00c0, + // Block 0x117, offset 0x45c0 + 0x45c0: 0x00c0, 0x45c1: 0x00c0, 0x45c2: 0x00c0, 0x45c3: 0x00c0, 0x45c4: 0x00c0, 0x45c5: 0x00c0, + 0x45c6: 0x00c0, 0x45c7: 0x00c0, 0x45c8: 0x00c0, 0x45c9: 0x00c0, 0x45ca: 0x00c0, 0x45cb: 0x00c0, + 0x45cc: 0x00c0, 0x45cd: 0x00c0, 0x45ce: 0x00c0, 0x45cf: 0x00c0, + // Block 0x118, offset 0x4600 + 0x4600: 0x0080, 0x4601: 0x0080, 0x4602: 0x0080, 0x4603: 0x0080, 0x4604: 0x0080, 0x4605: 0x0080, + 0x4606: 0x0080, 0x4607: 0x0080, 0x4608: 0x0080, 0x4609: 0x0080, 0x460a: 0x0080, 0x460b: 0x0080, + 0x460c: 0x0080, 0x460d: 0x0080, 0x460e: 0x0080, 0x460f: 0x0080, 0x4610: 0x0080, 0x4611: 0x0080, + 0x4612: 0x0080, 0x4613: 0x0080, 0x4614: 0x0080, 0x4615: 0x0080, 0x4616: 0x0080, 0x4617: 0x0080, + 0x4618: 0x0080, 0x4619: 0x0080, 0x461a: 0x0080, + // Block 0x119, offset 0x4640 + 0x4640: 0x00c0, 0x4641: 0x00c0, 0x4642: 0x00c0, 0x4643: 0x00c0, 0x4644: 0x00c0, 0x4645: 0x00c0, + 0x4646: 0x00c0, 0x4647: 0x00c0, 0x4648: 0x00c0, 0x4649: 0x00c0, 0x464a: 0x00c0, + 0x464f: 0x00c3, 0x4650: 0x00c0, 0x4651: 0x00c0, + 0x4652: 0x00c0, 0x4653: 0x00c0, 0x4654: 0x00c0, 0x4655: 0x00c0, 0x4656: 0x00c0, 0x4657: 0x00c0, + 0x4658: 0x00c0, 0x4659: 0x00c0, 0x465a: 0x00c0, 0x465b: 0x00c0, 0x465c: 0x00c0, 0x465d: 0x00c0, + 0x465e: 0x00c0, 0x465f: 0x00c0, 0x4660: 0x00c0, 0x4661: 0x00c0, 0x4662: 0x00c0, 0x4663: 0x00c0, + 0x4664: 0x00c0, 0x4665: 0x00c0, 0x4666: 0x00c0, 0x4667: 0x00c0, 0x4668: 0x00c0, 0x4669: 0x00c0, + 0x466a: 0x00c0, 0x466b: 0x00c0, 0x466c: 0x00c0, 0x466d: 0x00c0, 0x466e: 0x00c0, 0x466f: 0x00c0, + 0x4670: 0x00c0, 0x4671: 0x00c0, 0x4672: 0x00c0, 0x4673: 0x00c0, 0x4674: 0x00c0, 0x4675: 0x00c0, + 0x4676: 0x00c0, 0x4677: 0x00c0, 0x4678: 0x00c0, 0x4679: 0x00c0, 0x467a: 0x00c0, 0x467b: 0x00c0, + 0x467c: 0x00c0, 0x467d: 0x00c0, 0x467e: 0x00c0, 0x467f: 0x00c0, + // Block 0x11a, offset 0x4680 + 0x4680: 0x00c0, 0x4681: 0x00c0, 0x4682: 0x00c0, 0x4683: 0x00c0, 0x4684: 0x00c0, 0x4685: 0x00c0, + 0x4686: 0x00c0, 0x4687: 0x00c0, + 0x468f: 0x00c3, 0x4690: 0x00c3, 0x4691: 0x00c3, + 0x4692: 0x00c3, 0x4693: 0x00c0, 0x4694: 0x00c0, 0x4695: 0x00c0, 0x4696: 0x00c0, 0x4697: 0x00c0, + 0x4698: 0x00c0, 0x4699: 0x00c0, 0x469a: 0x00c0, 0x469b: 0x00c0, 0x469c: 0x00c0, 0x469d: 0x00c0, + 0x469e: 0x00c0, 0x469f: 0x00c0, + // Block 0x11b, offset 0x46c0 + 0x46e0: 0x00c0, 0x46e1: 0x00c0, 0x46e2: 0x0080, 0x46e3: 0x00c0, + // Block 0x11c, offset 0x4700 + 0x4700: 0x00c0, 0x4701: 0x00c0, 0x4702: 0x00c0, 0x4703: 0x00c0, 0x4704: 0x00c0, 0x4705: 0x00c0, + 0x4706: 0x00c0, 0x4707: 0x00c0, 0x4708: 0x00c0, 0x4709: 0x00c0, 0x470a: 0x00c0, 0x470b: 0x00c0, + 0x470c: 0x00c0, 0x470d: 0x00c0, 0x470e: 0x00c0, 0x470f: 0x00c0, 0x4710: 0x00c0, 0x4711: 0x00c0, + 0x4712: 0x00c0, 0x4713: 0x00c0, 0x4714: 0x00c0, 0x4715: 0x00c0, 0x4716: 0x00c0, 0x4717: 0x00c0, + 0x4718: 0x00c0, 0x4719: 0x00c0, 0x471a: 0x00c0, 0x471b: 0x00c0, 0x471c: 0x00c0, 0x471d: 0x00c0, + 0x471e: 0x00c0, 0x471f: 0x00c0, 0x4720: 0x00c0, 0x4721: 0x00c0, 0x4722: 0x00c0, 0x4723: 0x00c0, + 0x4724: 0x00c0, 0x4725: 0x00c0, 0x4726: 0x00c0, 0x4727: 0x00c0, 0x4728: 0x00c0, 0x4729: 0x00c0, + 0x472a: 0x00c0, 0x472b: 0x00c0, 0x472c: 0x00c0, 0x472d: 0x00c0, 0x472e: 0x00c0, 0x472f: 0x00c0, + 0x4730: 0x00c0, 0x4731: 0x00c0, 0x4732: 0x00c0, 0x4733: 0x00c0, 0x4734: 0x00c0, 0x4735: 0x00c0, + 0x4736: 0x00c0, 0x4737: 0x00c0, + // Block 0x11d, offset 0x4740 + 0x4740: 0x00cc, 0x4741: 0x00cc, 0x4742: 0x00cc, 0x4743: 0x00cc, 0x4744: 0x00cc, 0x4745: 0x00cc, + 0x4746: 0x00cc, 0x4747: 0x00cc, 0x4748: 0x00cc, 0x4749: 0x00cc, 0x474a: 0x00cc, 0x474b: 0x00cc, + 0x474c: 0x00cc, 0x474d: 0x00cc, 0x474e: 0x00cc, 0x474f: 0x00cc, 0x4750: 0x00cc, 0x4751: 0x00cc, + 0x4752: 0x00cc, 0x4753: 0x00cc, 0x4754: 0x00cc, 0x4755: 0x00cc, 0x4756: 0x00cc, 0x4757: 0x00cc, + 0x4758: 0x00cc, 0x4759: 0x00cc, 0x475a: 0x00cc, 0x475b: 0x00cc, 0x475c: 0x00cc, 0x475d: 0x00cc, + 0x475e: 0x00cc, + // Block 0x11e, offset 0x4780 + 0x4790: 0x00cc, 0x4791: 0x00cc, + 0x4792: 0x00cc, + 0x47a4: 0x00cc, 0x47a5: 0x00cc, 0x47a6: 0x00cc, 0x47a7: 0x00cc, + 0x47b0: 0x00c0, 0x47b1: 0x00c0, 0x47b2: 0x00c0, 0x47b3: 0x00c0, 0x47b4: 0x00c0, 0x47b5: 0x00c0, + 0x47b6: 0x00c0, 0x47b7: 0x00c0, 0x47b8: 0x00c0, 0x47b9: 0x00c0, 0x47ba: 0x00c0, 0x47bb: 0x00c0, + 0x47bc: 0x00c0, 0x47bd: 0x00c0, 0x47be: 0x00c0, 0x47bf: 0x00c0, + // Block 0x11f, offset 0x47c0 + 0x47c0: 0x00c0, 0x47c1: 0x00c0, 0x47c2: 0x00c0, 0x47c3: 0x00c0, 0x47c4: 0x00c0, 0x47c5: 0x00c0, + 0x47c6: 0x00c0, 0x47c7: 0x00c0, 0x47c8: 0x00c0, 0x47c9: 0x00c0, 0x47ca: 0x00c0, 0x47cb: 0x00c0, + 0x47cc: 0x00c0, 0x47cd: 0x00c0, 0x47ce: 0x00c0, 0x47cf: 0x00c0, 0x47d0: 0x00c0, 0x47d1: 0x00c0, + 0x47d2: 0x00c0, 0x47d3: 0x00c0, 0x47d4: 0x00c0, 0x47d5: 0x00c0, 0x47d6: 0x00c0, 0x47d7: 0x00c0, + 0x47d8: 0x00c0, 0x47d9: 0x00c0, 0x47da: 0x00c0, 0x47db: 0x00c0, 0x47dc: 0x00c0, 0x47dd: 0x00c0, + 0x47de: 0x00c0, 0x47df: 0x00c0, 0x47e0: 0x00c0, 0x47e1: 0x00c0, 0x47e2: 0x00c0, 0x47e3: 0x00c0, + 0x47e4: 0x00c0, 0x47e5: 0x00c0, 0x47e6: 0x00c0, 0x47e7: 0x00c0, 0x47e8: 0x00c0, 0x47e9: 0x00c0, + 0x47ea: 0x00c0, 0x47eb: 0x00c0, 0x47ec: 0x00c0, 0x47ed: 0x00c0, 0x47ee: 0x00c0, 0x47ef: 0x00c0, + 0x47f0: 0x00c0, 0x47f1: 0x00c0, 0x47f2: 0x00c0, 0x47f3: 0x00c0, 0x47f4: 0x00c0, 0x47f5: 0x00c0, + 0x47f6: 0x00c0, 0x47f7: 0x00c0, 0x47f8: 0x00c0, 0x47f9: 0x00c0, 0x47fa: 0x00c0, 0x47fb: 0x00c0, + // Block 0x120, offset 0x4800 + 0x4800: 0x00c0, 0x4801: 0x00c0, 0x4802: 0x00c0, 0x4803: 0x00c0, 0x4804: 0x00c0, 0x4805: 0x00c0, + 0x4806: 0x00c0, 0x4807: 0x00c0, 0x4808: 0x00c0, 0x4809: 0x00c0, 0x480a: 0x00c0, 0x480b: 0x00c0, + 0x480c: 0x00c0, 0x480d: 0x00c0, 0x480e: 0x00c0, 0x480f: 0x00c0, 0x4810: 0x00c0, 0x4811: 0x00c0, + 0x4812: 0x00c0, 0x4813: 0x00c0, 0x4814: 0x00c0, 0x4815: 0x00c0, 0x4816: 0x00c0, 0x4817: 0x00c0, + 0x4818: 0x00c0, 0x4819: 0x00c0, 0x481a: 0x00c0, 0x481b: 0x00c0, 0x481c: 0x00c0, 0x481d: 0x00c0, + 0x481e: 0x00c0, 0x481f: 0x00c0, 0x4820: 0x00c0, 0x4821: 0x00c0, 0x4822: 0x00c0, 0x4823: 0x00c0, + 0x4824: 0x00c0, 0x4825: 0x00c0, 0x4826: 0x00c0, 0x4827: 0x00c0, 0x4828: 0x00c0, 0x4829: 0x00c0, + 0x482a: 0x00c0, + 0x4830: 0x00c0, 0x4831: 0x00c0, 0x4832: 0x00c0, 0x4833: 0x00c0, 0x4834: 0x00c0, 0x4835: 0x00c0, + 0x4836: 0x00c0, 0x4837: 0x00c0, 0x4838: 0x00c0, 0x4839: 0x00c0, 0x483a: 0x00c0, 0x483b: 0x00c0, + 0x483c: 0x00c0, + // Block 0x121, offset 0x4840 + 0x4840: 0x00c0, 0x4841: 0x00c0, 0x4842: 0x00c0, 0x4843: 0x00c0, 0x4844: 0x00c0, 0x4845: 0x00c0, + 0x4846: 0x00c0, 0x4847: 0x00c0, 0x4848: 0x00c0, + 0x4850: 0x00c0, 0x4851: 0x00c0, + 0x4852: 0x00c0, 0x4853: 0x00c0, 0x4854: 0x00c0, 0x4855: 0x00c0, 0x4856: 0x00c0, 0x4857: 0x00c0, + 0x4858: 0x00c0, 0x4859: 0x00c0, 0x485c: 0x0080, 0x485d: 0x00c3, + 0x485e: 0x00c3, 0x485f: 0x0080, 0x4860: 0x0040, 0x4861: 0x0040, 0x4862: 0x0040, 0x4863: 0x0040, + // Block 0x122, offset 0x4880 + 0x4880: 0x0080, 0x4881: 0x0080, 0x4882: 0x0080, 0x4883: 0x0080, 0x4884: 0x0080, 0x4885: 0x0080, + 0x4886: 0x0080, 0x4887: 0x0080, 0x4888: 0x0080, 0x4889: 0x0080, 0x488a: 0x0080, 0x488b: 0x0080, + 0x488c: 0x0080, 0x488d: 0x0080, 0x488e: 0x0080, 0x488f: 0x0080, 0x4890: 0x0080, 0x4891: 0x0080, + 0x4892: 0x0080, 0x4893: 0x0080, 0x4894: 0x0080, 0x4895: 0x0080, 0x4896: 0x0080, 0x4897: 0x0080, + 0x4898: 0x0080, 0x4899: 0x0080, 0x489a: 0x0080, 0x489b: 0x0080, 0x489c: 0x0080, 0x489d: 0x0080, + 0x489e: 0x0080, 0x489f: 0x0080, 0x48a0: 0x0080, 0x48a1: 0x0080, 0x48a2: 0x0080, 0x48a3: 0x0080, + 0x48a4: 0x0080, 0x48a5: 0x0080, 0x48a6: 0x0080, 0x48a7: 0x0080, 0x48a8: 0x0080, 0x48a9: 0x0080, + 0x48aa: 0x0080, 0x48ab: 0x0080, 0x48ac: 0x0080, 0x48ad: 0x0080, 0x48ae: 0x0080, 0x48af: 0x0080, + 0x48b0: 0x0080, 0x48b1: 0x0080, 0x48b2: 0x0080, 0x48b3: 0x0080, 0x48b4: 0x0080, 0x48b5: 0x0080, + // Block 0x123, offset 0x48c0 + 0x48c0: 0x0080, 0x48c1: 0x0080, 0x48c2: 0x0080, 0x48c3: 0x0080, 0x48c4: 0x0080, 0x48c5: 0x0080, + 0x48c6: 0x0080, 0x48c7: 0x0080, 0x48c8: 0x0080, 0x48c9: 0x0080, 0x48ca: 0x0080, 0x48cb: 0x0080, + 0x48cc: 0x0080, 0x48cd: 0x0080, 0x48ce: 0x0080, 0x48cf: 0x0080, 0x48d0: 0x0080, 0x48d1: 0x0080, + 0x48d2: 0x0080, 0x48d3: 0x0080, 0x48d4: 0x0080, 0x48d5: 0x0080, 0x48d6: 0x0080, 0x48d7: 0x0080, + 0x48d8: 0x0080, 0x48d9: 0x0080, 0x48da: 0x0080, 0x48db: 0x0080, 0x48dc: 0x0080, 0x48dd: 0x0080, + 0x48de: 0x0080, 0x48df: 0x0080, 0x48e0: 0x0080, 0x48e1: 0x0080, 0x48e2: 0x0080, 0x48e3: 0x0080, + 0x48e4: 0x0080, 0x48e5: 0x0080, 0x48e6: 0x0080, 0x48e9: 0x0080, + 0x48ea: 0x0080, 0x48eb: 0x0080, 0x48ec: 0x0080, 0x48ed: 0x0080, 0x48ee: 0x0080, 0x48ef: 0x0080, + 0x48f0: 0x0080, 0x48f1: 0x0080, 0x48f2: 0x0080, 0x48f3: 0x0080, 0x48f4: 0x0080, 0x48f5: 0x0080, + 0x48f6: 0x0080, 0x48f7: 0x0080, 0x48f8: 0x0080, 0x48f9: 0x0080, 0x48fa: 0x0080, 0x48fb: 0x0080, + 0x48fc: 0x0080, 0x48fd: 0x0080, 0x48fe: 0x0080, 0x48ff: 0x0080, + // Block 0x124, offset 0x4900 + 0x4900: 0x0080, 0x4901: 0x0080, 0x4902: 0x0080, 0x4903: 0x0080, 0x4904: 0x0080, 0x4905: 0x0080, + 0x4906: 0x0080, 0x4907: 0x0080, 0x4908: 0x0080, 0x4909: 0x0080, 0x490a: 0x0080, 0x490b: 0x0080, + 0x490c: 0x0080, 0x490d: 0x0080, 0x490e: 0x0080, 0x490f: 0x0080, 0x4910: 0x0080, 0x4911: 0x0080, + 0x4912: 0x0080, 0x4913: 0x0080, 0x4914: 0x0080, 0x4915: 0x0080, 0x4916: 0x0080, 0x4917: 0x0080, + 0x4918: 0x0080, 0x4919: 0x0080, 0x491a: 0x0080, 0x491b: 0x0080, 0x491c: 0x0080, 0x491d: 0x0080, + 0x491e: 0x0080, 0x491f: 0x0080, 0x4920: 0x0080, 0x4921: 0x0080, 0x4922: 0x0080, 0x4923: 0x0080, + 0x4924: 0x0080, 0x4925: 0x00c0, 0x4926: 0x00c0, 0x4927: 0x00c3, 0x4928: 0x00c3, 0x4929: 0x00c3, + 0x492a: 0x0080, 0x492b: 0x0080, 0x492c: 0x0080, 0x492d: 0x00c0, 0x492e: 0x00c0, 0x492f: 0x00c0, + 0x4930: 0x00c0, 0x4931: 0x00c0, 0x4932: 0x00c0, 0x4933: 0x0040, 0x4934: 0x0040, 0x4935: 0x0040, + 0x4936: 0x0040, 0x4937: 0x0040, 0x4938: 0x0040, 0x4939: 0x0040, 0x493a: 0x0040, 0x493b: 0x00c3, + 0x493c: 0x00c3, 0x493d: 0x00c3, 0x493e: 0x00c3, 0x493f: 0x00c3, + // Block 0x125, offset 0x4940 + 0x4940: 0x00c3, 0x4941: 0x00c3, 0x4942: 0x00c3, 0x4943: 0x0080, 0x4944: 0x0080, 0x4945: 0x00c3, + 0x4946: 0x00c3, 0x4947: 0x00c3, 0x4948: 0x00c3, 0x4949: 0x00c3, 0x494a: 0x00c3, 0x494b: 0x00c3, + 0x494c: 0x0080, 0x494d: 0x0080, 0x494e: 0x0080, 0x494f: 0x0080, 0x4950: 0x0080, 0x4951: 0x0080, + 0x4952: 0x0080, 0x4953: 0x0080, 0x4954: 0x0080, 0x4955: 0x0080, 0x4956: 0x0080, 0x4957: 0x0080, + 0x4958: 0x0080, 0x4959: 0x0080, 0x495a: 0x0080, 0x495b: 0x0080, 0x495c: 0x0080, 0x495d: 0x0080, + 0x495e: 0x0080, 0x495f: 0x0080, 0x4960: 0x0080, 0x4961: 0x0080, 0x4962: 0x0080, 0x4963: 0x0080, + 0x4964: 0x0080, 0x4965: 0x0080, 0x4966: 0x0080, 0x4967: 0x0080, 0x4968: 0x0080, 0x4969: 0x0080, + 0x496a: 0x00c3, 0x496b: 0x00c3, 0x496c: 0x00c3, 0x496d: 0x00c3, 0x496e: 0x0080, 0x496f: 0x0080, + 0x4970: 0x0080, 0x4971: 0x0080, 0x4972: 0x0080, 0x4973: 0x0080, 0x4974: 0x0080, 0x4975: 0x0080, + 0x4976: 0x0080, 0x4977: 0x0080, 0x4978: 0x0080, 0x4979: 0x0080, 0x497a: 0x0080, 0x497b: 0x0080, + 0x497c: 0x0080, 0x497d: 0x0080, 0x497e: 0x0080, 0x497f: 0x0080, + // Block 0x126, offset 0x4980 + 0x4980: 0x0080, 0x4981: 0x0080, 0x4982: 0x0080, 0x4983: 0x0080, 0x4984: 0x0080, 0x4985: 0x0080, + 0x4986: 0x0080, 0x4987: 0x0080, 0x4988: 0x0080, 0x4989: 0x0080, 0x498a: 0x0080, 0x498b: 0x0080, + 0x498c: 0x0080, 0x498d: 0x0080, 0x498e: 0x0080, 0x498f: 0x0080, 0x4990: 0x0080, 0x4991: 0x0080, + 0x4992: 0x0080, 0x4993: 0x0080, 0x4994: 0x0080, 0x4995: 0x0080, 0x4996: 0x0080, 0x4997: 0x0080, + 0x4998: 0x0080, 0x4999: 0x0080, 0x499a: 0x0080, 0x499b: 0x0080, 0x499c: 0x0080, 0x499d: 0x0080, + 0x499e: 0x0080, 0x499f: 0x0080, 0x49a0: 0x0080, 0x49a1: 0x0080, 0x49a2: 0x0080, 0x49a3: 0x0080, + 0x49a4: 0x0080, 0x49a5: 0x0080, 0x49a6: 0x0080, 0x49a7: 0x0080, 0x49a8: 0x0080, + // Block 0x127, offset 0x49c0 + 0x49c0: 0x0088, 0x49c1: 0x0088, 0x49c2: 0x00c9, 0x49c3: 0x00c9, 0x49c4: 0x00c9, 0x49c5: 0x0088, + // Block 0x128, offset 0x4a00 + 0x4a20: 0x0080, 0x4a21: 0x0080, 0x4a22: 0x0080, 0x4a23: 0x0080, + 0x4a24: 0x0080, 0x4a25: 0x0080, 0x4a26: 0x0080, 0x4a27: 0x0080, 0x4a28: 0x0080, 0x4a29: 0x0080, + 0x4a2a: 0x0080, 0x4a2b: 0x0080, 0x4a2c: 0x0080, 0x4a2d: 0x0080, 0x4a2e: 0x0080, 0x4a2f: 0x0080, + 0x4a30: 0x0080, 0x4a31: 0x0080, 0x4a32: 0x0080, 0x4a33: 0x0080, + // Block 0x129, offset 0x4a40 + 0x4a40: 0x0080, 0x4a41: 0x0080, 0x4a42: 0x0080, 0x4a43: 0x0080, 0x4a44: 0x0080, 0x4a45: 0x0080, + 0x4a46: 0x0080, 0x4a47: 0x0080, 0x4a48: 0x0080, 0x4a49: 0x0080, 0x4a4a: 0x0080, 0x4a4b: 0x0080, + 0x4a4c: 0x0080, 0x4a4d: 0x0080, 0x4a4e: 0x0080, 0x4a4f: 0x0080, 0x4a50: 0x0080, 0x4a51: 0x0080, + 0x4a52: 0x0080, 0x4a53: 0x0080, 0x4a54: 0x0080, 0x4a55: 0x0080, 0x4a56: 0x0080, + 0x4a60: 0x0080, 0x4a61: 0x0080, 0x4a62: 0x0080, 0x4a63: 0x0080, + 0x4a64: 0x0080, 0x4a65: 0x0080, 0x4a66: 0x0080, 0x4a67: 0x0080, 0x4a68: 0x0080, 0x4a69: 0x0080, + 0x4a6a: 0x0080, 0x4a6b: 0x0080, 0x4a6c: 0x0080, 0x4a6d: 0x0080, 0x4a6e: 0x0080, 0x4a6f: 0x0080, + 0x4a70: 0x0080, 0x4a71: 0x0080, 0x4a72: 0x0080, 0x4a73: 0x0080, 0x4a74: 0x0080, 0x4a75: 0x0080, + 0x4a76: 0x0080, 0x4a77: 0x0080, 0x4a78: 0x0080, + // Block 0x12a, offset 0x4a80 + 0x4a80: 0x0080, 0x4a81: 0x0080, 0x4a82: 0x0080, 0x4a83: 0x0080, 0x4a84: 0x0080, 0x4a85: 0x0080, + 0x4a86: 0x0080, 0x4a87: 0x0080, 0x4a88: 0x0080, 0x4a89: 0x0080, 0x4a8a: 0x0080, 0x4a8b: 0x0080, + 0x4a8c: 0x0080, 0x4a8d: 0x0080, 0x4a8e: 0x0080, 0x4a8f: 0x0080, 0x4a90: 0x0080, 0x4a91: 0x0080, + 0x4a92: 0x0080, 0x4a93: 0x0080, 0x4a94: 0x0080, 0x4a96: 0x0080, 0x4a97: 0x0080, + 0x4a98: 0x0080, 0x4a99: 0x0080, 0x4a9a: 0x0080, 0x4a9b: 0x0080, 0x4a9c: 0x0080, 0x4a9d: 0x0080, + 0x4a9e: 0x0080, 0x4a9f: 0x0080, 0x4aa0: 0x0080, 0x4aa1: 0x0080, 0x4aa2: 0x0080, 0x4aa3: 0x0080, + 0x4aa4: 0x0080, 0x4aa5: 0x0080, 0x4aa6: 0x0080, 0x4aa7: 0x0080, 0x4aa8: 0x0080, 0x4aa9: 0x0080, + 0x4aaa: 0x0080, 0x4aab: 0x0080, 0x4aac: 0x0080, 0x4aad: 0x0080, 0x4aae: 0x0080, 0x4aaf: 0x0080, + 0x4ab0: 0x0080, 0x4ab1: 0x0080, 0x4ab2: 0x0080, 0x4ab3: 0x0080, 0x4ab4: 0x0080, 0x4ab5: 0x0080, + 0x4ab6: 0x0080, 0x4ab7: 0x0080, 0x4ab8: 0x0080, 0x4ab9: 0x0080, 0x4aba: 0x0080, 0x4abb: 0x0080, + 0x4abc: 0x0080, 0x4abd: 0x0080, 0x4abe: 0x0080, 0x4abf: 0x0080, + // Block 0x12b, offset 0x4ac0 + 0x4ac0: 0x0080, 0x4ac1: 0x0080, 0x4ac2: 0x0080, 0x4ac3: 0x0080, 0x4ac4: 0x0080, 0x4ac5: 0x0080, + 0x4ac6: 0x0080, 0x4ac7: 0x0080, 0x4ac8: 0x0080, 0x4ac9: 0x0080, 0x4aca: 0x0080, 0x4acb: 0x0080, + 0x4acc: 0x0080, 0x4acd: 0x0080, 0x4ace: 0x0080, 0x4acf: 0x0080, 0x4ad0: 0x0080, 0x4ad1: 0x0080, + 0x4ad2: 0x0080, 0x4ad3: 0x0080, 0x4ad4: 0x0080, 0x4ad5: 0x0080, 0x4ad6: 0x0080, 0x4ad7: 0x0080, + 0x4ad8: 0x0080, 0x4ad9: 0x0080, 0x4ada: 0x0080, 0x4adb: 0x0080, 0x4adc: 0x0080, + 0x4ade: 0x0080, 0x4adf: 0x0080, 0x4ae2: 0x0080, + 0x4ae5: 0x0080, 0x4ae6: 0x0080, 0x4ae9: 0x0080, + 0x4aea: 0x0080, 0x4aeb: 0x0080, 0x4aec: 0x0080, 0x4aee: 0x0080, 0x4aef: 0x0080, + 0x4af0: 0x0080, 0x4af1: 0x0080, 0x4af2: 0x0080, 0x4af3: 0x0080, 0x4af4: 0x0080, 0x4af5: 0x0080, + 0x4af6: 0x0080, 0x4af7: 0x0080, 0x4af8: 0x0080, 0x4af9: 0x0080, 0x4afb: 0x0080, + 0x4afd: 0x0080, 0x4afe: 0x0080, 0x4aff: 0x0080, + // Block 0x12c, offset 0x4b00 + 0x4b00: 0x0080, 0x4b01: 0x0080, 0x4b02: 0x0080, 0x4b03: 0x0080, 0x4b05: 0x0080, + 0x4b06: 0x0080, 0x4b07: 0x0080, 0x4b08: 0x0080, 0x4b09: 0x0080, 0x4b0a: 0x0080, 0x4b0b: 0x0080, + 0x4b0c: 0x0080, 0x4b0d: 0x0080, 0x4b0e: 0x0080, 0x4b0f: 0x0080, 0x4b10: 0x0080, 0x4b11: 0x0080, + 0x4b12: 0x0080, 0x4b13: 0x0080, 0x4b14: 0x0080, 0x4b15: 0x0080, 0x4b16: 0x0080, 0x4b17: 0x0080, + 0x4b18: 0x0080, 0x4b19: 0x0080, 0x4b1a: 0x0080, 0x4b1b: 0x0080, 0x4b1c: 0x0080, 0x4b1d: 0x0080, + 0x4b1e: 0x0080, 0x4b1f: 0x0080, 0x4b20: 0x0080, 0x4b21: 0x0080, 0x4b22: 0x0080, 0x4b23: 0x0080, + 0x4b24: 0x0080, 0x4b25: 0x0080, 0x4b26: 0x0080, 0x4b27: 0x0080, 0x4b28: 0x0080, 0x4b29: 0x0080, + 0x4b2a: 0x0080, 0x4b2b: 0x0080, 0x4b2c: 0x0080, 0x4b2d: 0x0080, 0x4b2e: 0x0080, 0x4b2f: 0x0080, + 0x4b30: 0x0080, 0x4b31: 0x0080, 0x4b32: 0x0080, 0x4b33: 0x0080, 0x4b34: 0x0080, 0x4b35: 0x0080, + 0x4b36: 0x0080, 0x4b37: 0x0080, 0x4b38: 0x0080, 0x4b39: 0x0080, 0x4b3a: 0x0080, 0x4b3b: 0x0080, + 0x4b3c: 0x0080, 0x4b3d: 0x0080, 0x4b3e: 0x0080, 0x4b3f: 0x0080, + // Block 0x12d, offset 0x4b40 + 0x4b40: 0x0080, 0x4b41: 0x0080, 0x4b42: 0x0080, 0x4b43: 0x0080, 0x4b44: 0x0080, 0x4b45: 0x0080, + 0x4b47: 0x0080, 0x4b48: 0x0080, 0x4b49: 0x0080, 0x4b4a: 0x0080, + 0x4b4d: 0x0080, 0x4b4e: 0x0080, 0x4b4f: 0x0080, 0x4b50: 0x0080, 0x4b51: 0x0080, + 0x4b52: 0x0080, 0x4b53: 0x0080, 0x4b54: 0x0080, 0x4b56: 0x0080, 0x4b57: 0x0080, + 0x4b58: 0x0080, 0x4b59: 0x0080, 0x4b5a: 0x0080, 0x4b5b: 0x0080, 0x4b5c: 0x0080, + 0x4b5e: 0x0080, 0x4b5f: 0x0080, 0x4b60: 0x0080, 0x4b61: 0x0080, 0x4b62: 0x0080, 0x4b63: 0x0080, + 0x4b64: 0x0080, 0x4b65: 0x0080, 0x4b66: 0x0080, 0x4b67: 0x0080, 0x4b68: 0x0080, 0x4b69: 0x0080, + 0x4b6a: 0x0080, 0x4b6b: 0x0080, 0x4b6c: 0x0080, 0x4b6d: 0x0080, 0x4b6e: 0x0080, 0x4b6f: 0x0080, + 0x4b70: 0x0080, 0x4b71: 0x0080, 0x4b72: 0x0080, 0x4b73: 0x0080, 0x4b74: 0x0080, 0x4b75: 0x0080, + 0x4b76: 0x0080, 0x4b77: 0x0080, 0x4b78: 0x0080, 0x4b79: 0x0080, 0x4b7b: 0x0080, + 0x4b7c: 0x0080, 0x4b7d: 0x0080, 0x4b7e: 0x0080, + // Block 0x12e, offset 0x4b80 + 0x4b80: 0x0080, 0x4b81: 0x0080, 0x4b82: 0x0080, 0x4b83: 0x0080, 0x4b84: 0x0080, + 0x4b86: 0x0080, 0x4b8a: 0x0080, 0x4b8b: 0x0080, + 0x4b8c: 0x0080, 0x4b8d: 0x0080, 0x4b8e: 0x0080, 0x4b8f: 0x0080, 0x4b90: 0x0080, + 0x4b92: 0x0080, 0x4b93: 0x0080, 0x4b94: 0x0080, 0x4b95: 0x0080, 0x4b96: 0x0080, 0x4b97: 0x0080, + 0x4b98: 0x0080, 0x4b99: 0x0080, 0x4b9a: 0x0080, 0x4b9b: 0x0080, 0x4b9c: 0x0080, 0x4b9d: 0x0080, + 0x4b9e: 0x0080, 0x4b9f: 0x0080, 0x4ba0: 0x0080, 0x4ba1: 0x0080, 0x4ba2: 0x0080, 0x4ba3: 0x0080, + 0x4ba4: 0x0080, 0x4ba5: 0x0080, 0x4ba6: 0x0080, 0x4ba7: 0x0080, 0x4ba8: 0x0080, 0x4ba9: 0x0080, + 0x4baa: 0x0080, 0x4bab: 0x0080, 0x4bac: 0x0080, 0x4bad: 0x0080, 0x4bae: 0x0080, 0x4baf: 0x0080, + 0x4bb0: 0x0080, 0x4bb1: 0x0080, 0x4bb2: 0x0080, 0x4bb3: 0x0080, 0x4bb4: 0x0080, 0x4bb5: 0x0080, + 0x4bb6: 0x0080, 0x4bb7: 0x0080, 0x4bb8: 0x0080, 0x4bb9: 0x0080, 0x4bba: 0x0080, 0x4bbb: 0x0080, + 0x4bbc: 0x0080, 0x4bbd: 0x0080, 0x4bbe: 0x0080, 0x4bbf: 0x0080, + // Block 0x12f, offset 0x4bc0 + 0x4bc0: 0x0080, 0x4bc1: 0x0080, 0x4bc2: 0x0080, 0x4bc3: 0x0080, 0x4bc4: 0x0080, 0x4bc5: 0x0080, + 0x4bc6: 0x0080, 0x4bc7: 0x0080, 0x4bc8: 0x0080, 0x4bc9: 0x0080, 0x4bca: 0x0080, 0x4bcb: 0x0080, + 0x4bcc: 0x0080, 0x4bcd: 0x0080, 0x4bce: 0x0080, 0x4bcf: 0x0080, 0x4bd0: 0x0080, 0x4bd1: 0x0080, + 0x4bd2: 0x0080, 0x4bd3: 0x0080, 0x4bd4: 0x0080, 0x4bd5: 0x0080, 0x4bd6: 0x0080, 0x4bd7: 0x0080, + 0x4bd8: 0x0080, 0x4bd9: 0x0080, 0x4bda: 0x0080, 0x4bdb: 0x0080, 0x4bdc: 0x0080, 0x4bdd: 0x0080, + 0x4bde: 0x0080, 0x4bdf: 0x0080, 0x4be0: 0x0080, 0x4be1: 0x0080, 0x4be2: 0x0080, 0x4be3: 0x0080, + 0x4be4: 0x0080, 0x4be5: 0x0080, 0x4be8: 0x0080, 0x4be9: 0x0080, + 0x4bea: 0x0080, 0x4beb: 0x0080, 0x4bec: 0x0080, 0x4bed: 0x0080, 0x4bee: 0x0080, 0x4bef: 0x0080, + 0x4bf0: 0x0080, 0x4bf1: 0x0080, 0x4bf2: 0x0080, 0x4bf3: 0x0080, 0x4bf4: 0x0080, 0x4bf5: 0x0080, + 0x4bf6: 0x0080, 0x4bf7: 0x0080, 0x4bf8: 0x0080, 0x4bf9: 0x0080, 0x4bfa: 0x0080, 0x4bfb: 0x0080, + 0x4bfc: 0x0080, 0x4bfd: 0x0080, 0x4bfe: 0x0080, 0x4bff: 0x0080, + // Block 0x130, offset 0x4c00 + 0x4c00: 0x0080, 0x4c01: 0x0080, 0x4c02: 0x0080, 0x4c03: 0x0080, 0x4c04: 0x0080, 0x4c05: 0x0080, + 0x4c06: 0x0080, 0x4c07: 0x0080, 0x4c08: 0x0080, 0x4c09: 0x0080, 0x4c0a: 0x0080, 0x4c0b: 0x0080, + 0x4c0e: 0x0080, 0x4c0f: 0x0080, 0x4c10: 0x0080, 0x4c11: 0x0080, + 0x4c12: 0x0080, 0x4c13: 0x0080, 0x4c14: 0x0080, 0x4c15: 0x0080, 0x4c16: 0x0080, 0x4c17: 0x0080, + 0x4c18: 0x0080, 0x4c19: 0x0080, 0x4c1a: 0x0080, 0x4c1b: 0x0080, 0x4c1c: 0x0080, 0x4c1d: 0x0080, + 0x4c1e: 0x0080, 0x4c1f: 0x0080, 0x4c20: 0x0080, 0x4c21: 0x0080, 0x4c22: 0x0080, 0x4c23: 0x0080, + 0x4c24: 0x0080, 0x4c25: 0x0080, 0x4c26: 0x0080, 0x4c27: 0x0080, 0x4c28: 0x0080, 0x4c29: 0x0080, + 0x4c2a: 0x0080, 0x4c2b: 0x0080, 0x4c2c: 0x0080, 0x4c2d: 0x0080, 0x4c2e: 0x0080, 0x4c2f: 0x0080, + 0x4c30: 0x0080, 0x4c31: 0x0080, 0x4c32: 0x0080, 0x4c33: 0x0080, 0x4c34: 0x0080, 0x4c35: 0x0080, + 0x4c36: 0x0080, 0x4c37: 0x0080, 0x4c38: 0x0080, 0x4c39: 0x0080, 0x4c3a: 0x0080, 0x4c3b: 0x0080, + 0x4c3c: 0x0080, 0x4c3d: 0x0080, 0x4c3e: 0x0080, 0x4c3f: 0x0080, + // Block 0x131, offset 0x4c40 + 0x4c40: 0x00c3, 0x4c41: 0x00c3, 0x4c42: 0x00c3, 0x4c43: 0x00c3, 0x4c44: 0x00c3, 0x4c45: 0x00c3, + 0x4c46: 0x00c3, 0x4c47: 0x00c3, 0x4c48: 0x00c3, 0x4c49: 0x00c3, 0x4c4a: 0x00c3, 0x4c4b: 0x00c3, + 0x4c4c: 0x00c3, 0x4c4d: 0x00c3, 0x4c4e: 0x00c3, 0x4c4f: 0x00c3, 0x4c50: 0x00c3, 0x4c51: 0x00c3, + 0x4c52: 0x00c3, 0x4c53: 0x00c3, 0x4c54: 0x00c3, 0x4c55: 0x00c3, 0x4c56: 0x00c3, 0x4c57: 0x00c3, + 0x4c58: 0x00c3, 0x4c59: 0x00c3, 0x4c5a: 0x00c3, 0x4c5b: 0x00c3, 0x4c5c: 0x00c3, 0x4c5d: 0x00c3, + 0x4c5e: 0x00c3, 0x4c5f: 0x00c3, 0x4c60: 0x00c3, 0x4c61: 0x00c3, 0x4c62: 0x00c3, 0x4c63: 0x00c3, + 0x4c64: 0x00c3, 0x4c65: 0x00c3, 0x4c66: 0x00c3, 0x4c67: 0x00c3, 0x4c68: 0x00c3, 0x4c69: 0x00c3, + 0x4c6a: 0x00c3, 0x4c6b: 0x00c3, 0x4c6c: 0x00c3, 0x4c6d: 0x00c3, 0x4c6e: 0x00c3, 0x4c6f: 0x00c3, + 0x4c70: 0x00c3, 0x4c71: 0x00c3, 0x4c72: 0x00c3, 0x4c73: 0x00c3, 0x4c74: 0x00c3, 0x4c75: 0x00c3, + 0x4c76: 0x00c3, 0x4c77: 0x0080, 0x4c78: 0x0080, 0x4c79: 0x0080, 0x4c7a: 0x0080, 0x4c7b: 0x00c3, + 0x4c7c: 0x00c3, 0x4c7d: 0x00c3, 0x4c7e: 0x00c3, 0x4c7f: 0x00c3, + // Block 0x132, offset 0x4c80 + 0x4c80: 0x00c3, 0x4c81: 0x00c3, 0x4c82: 0x00c3, 0x4c83: 0x00c3, 0x4c84: 0x00c3, 0x4c85: 0x00c3, + 0x4c86: 0x00c3, 0x4c87: 0x00c3, 0x4c88: 0x00c3, 0x4c89: 0x00c3, 0x4c8a: 0x00c3, 0x4c8b: 0x00c3, + 0x4c8c: 0x00c3, 0x4c8d: 0x00c3, 0x4c8e: 0x00c3, 0x4c8f: 0x00c3, 0x4c90: 0x00c3, 0x4c91: 0x00c3, + 0x4c92: 0x00c3, 0x4c93: 0x00c3, 0x4c94: 0x00c3, 0x4c95: 0x00c3, 0x4c96: 0x00c3, 0x4c97: 0x00c3, + 0x4c98: 0x00c3, 0x4c99: 0x00c3, 0x4c9a: 0x00c3, 0x4c9b: 0x00c3, 0x4c9c: 0x00c3, 0x4c9d: 0x00c3, + 0x4c9e: 0x00c3, 0x4c9f: 0x00c3, 0x4ca0: 0x00c3, 0x4ca1: 0x00c3, 0x4ca2: 0x00c3, 0x4ca3: 0x00c3, + 0x4ca4: 0x00c3, 0x4ca5: 0x00c3, 0x4ca6: 0x00c3, 0x4ca7: 0x00c3, 0x4ca8: 0x00c3, 0x4ca9: 0x00c3, + 0x4caa: 0x00c3, 0x4cab: 0x00c3, 0x4cac: 0x00c3, 0x4cad: 0x0080, 0x4cae: 0x0080, 0x4caf: 0x0080, + 0x4cb0: 0x0080, 0x4cb1: 0x0080, 0x4cb2: 0x0080, 0x4cb3: 0x0080, 0x4cb4: 0x0080, 0x4cb5: 0x00c3, + 0x4cb6: 0x0080, 0x4cb7: 0x0080, 0x4cb8: 0x0080, 0x4cb9: 0x0080, 0x4cba: 0x0080, 0x4cbb: 0x0080, + 0x4cbc: 0x0080, 0x4cbd: 0x0080, 0x4cbe: 0x0080, 0x4cbf: 0x0080, + // Block 0x133, offset 0x4cc0 + 0x4cc0: 0x0080, 0x4cc1: 0x0080, 0x4cc2: 0x0080, 0x4cc3: 0x0080, 0x4cc4: 0x00c3, 0x4cc5: 0x0080, + 0x4cc6: 0x0080, 0x4cc7: 0x0080, 0x4cc8: 0x0080, 0x4cc9: 0x0080, 0x4cca: 0x0080, 0x4ccb: 0x0080, + 0x4cdb: 0x00c3, 0x4cdc: 0x00c3, 0x4cdd: 0x00c3, + 0x4cde: 0x00c3, 0x4cdf: 0x00c3, 0x4ce1: 0x00c3, 0x4ce2: 0x00c3, 0x4ce3: 0x00c3, + 0x4ce4: 0x00c3, 0x4ce5: 0x00c3, 0x4ce6: 0x00c3, 0x4ce7: 0x00c3, 0x4ce8: 0x00c3, 0x4ce9: 0x00c3, + 0x4cea: 0x00c3, 0x4ceb: 0x00c3, 0x4cec: 0x00c3, 0x4ced: 0x00c3, 0x4cee: 0x00c3, 0x4cef: 0x00c3, + // Block 0x134, offset 0x4d00 + 0x4d00: 0x00c3, 0x4d01: 0x00c3, 0x4d02: 0x00c3, 0x4d03: 0x00c3, 0x4d04: 0x00c3, 0x4d05: 0x00c3, + 0x4d06: 0x00c3, 0x4d08: 0x00c3, 0x4d09: 0x00c3, 0x4d0a: 0x00c3, 0x4d0b: 0x00c3, + 0x4d0c: 0x00c3, 0x4d0d: 0x00c3, 0x4d0e: 0x00c3, 0x4d0f: 0x00c3, 0x4d10: 0x00c3, 0x4d11: 0x00c3, + 0x4d12: 0x00c3, 0x4d13: 0x00c3, 0x4d14: 0x00c3, 0x4d15: 0x00c3, 0x4d16: 0x00c3, 0x4d17: 0x00c3, + 0x4d18: 0x00c3, 0x4d1b: 0x00c3, 0x4d1c: 0x00c3, 0x4d1d: 0x00c3, + 0x4d1e: 0x00c3, 0x4d1f: 0x00c3, 0x4d20: 0x00c3, 0x4d21: 0x00c3, 0x4d23: 0x00c3, + 0x4d24: 0x00c3, 0x4d26: 0x00c3, 0x4d27: 0x00c3, 0x4d28: 0x00c3, 0x4d29: 0x00c3, + 0x4d2a: 0x00c3, + // Block 0x135, offset 0x4d40 + 0x4d40: 0x00c0, 0x4d41: 0x00c0, 0x4d42: 0x00c0, 0x4d43: 0x00c0, 0x4d44: 0x00c0, 0x4d45: 0x00c0, + 0x4d46: 0x00c0, 0x4d47: 0x00c0, 0x4d48: 0x00c0, 0x4d49: 0x00c0, 0x4d4a: 0x00c0, 0x4d4b: 0x00c0, + 0x4d4c: 0x00c0, 0x4d4d: 0x00c0, 0x4d4e: 0x00c0, 0x4d4f: 0x00c0, 0x4d50: 0x00c0, 0x4d51: 0x00c0, + 0x4d52: 0x00c0, 0x4d53: 0x00c0, 0x4d54: 0x00c0, 0x4d55: 0x00c0, 0x4d56: 0x00c0, 0x4d57: 0x00c0, + 0x4d58: 0x00c0, 0x4d59: 0x00c0, 0x4d5a: 0x00c0, 0x4d5b: 0x00c0, 0x4d5c: 0x00c0, 0x4d5d: 0x00c0, + 0x4d5e: 0x00c0, 0x4d5f: 0x00c0, 0x4d60: 0x00c0, 0x4d61: 0x00c0, 0x4d62: 0x00c0, 0x4d63: 0x00c0, + 0x4d64: 0x00c0, 0x4d65: 0x00c0, 0x4d66: 0x00c0, 0x4d67: 0x00c0, 0x4d68: 0x00c0, 0x4d69: 0x00c0, + 0x4d6a: 0x00c0, 0x4d6b: 0x00c0, 0x4d6c: 0x00c0, + 0x4d70: 0x00c3, 0x4d71: 0x00c3, 0x4d72: 0x00c3, 0x4d73: 0x00c3, 0x4d74: 0x00c3, 0x4d75: 0x00c3, + 0x4d76: 0x00c3, 0x4d77: 0x00c0, 0x4d78: 0x00c0, 0x4d79: 0x00c0, 0x4d7a: 0x00c0, 0x4d7b: 0x00c0, + 0x4d7c: 0x00c0, 0x4d7d: 0x00c0, + // Block 0x136, offset 0x4d80 + 0x4d80: 0x00c0, 0x4d81: 0x00c0, 0x4d82: 0x00c0, 0x4d83: 0x00c0, 0x4d84: 0x00c0, 0x4d85: 0x00c0, + 0x4d86: 0x00c0, 0x4d87: 0x00c0, 0x4d88: 0x00c0, 0x4d89: 0x00c0, + 0x4d8e: 0x00c0, 0x4d8f: 0x0080, + // Block 0x137, offset 0x4dc0 + 0x4dc0: 0x00c0, 0x4dc1: 0x00c0, 0x4dc2: 0x00c0, 0x4dc3: 0x00c0, 0x4dc4: 0x00c0, 0x4dc5: 0x00c0, + 0x4dc6: 0x00c0, 0x4dc7: 0x00c0, 0x4dc8: 0x00c0, 0x4dc9: 0x00c0, 0x4dca: 0x00c0, 0x4dcb: 0x00c0, + 0x4dcc: 0x00c0, 0x4dcd: 0x00c0, 0x4dce: 0x00c0, 0x4dcf: 0x00c0, 0x4dd0: 0x00c0, 0x4dd1: 0x00c0, + 0x4dd2: 0x00c0, 0x4dd3: 0x00c0, 0x4dd4: 0x00c0, 0x4dd5: 0x00c0, 0x4dd6: 0x00c0, 0x4dd7: 0x00c0, + 0x4dd8: 0x00c0, 0x4dd9: 0x00c0, 0x4dda: 0x00c0, 0x4ddb: 0x00c0, 0x4ddc: 0x00c0, 0x4ddd: 0x00c0, + 0x4dde: 0x00c0, 0x4ddf: 0x00c0, 0x4de0: 0x00c0, 0x4de1: 0x00c0, 0x4de2: 0x00c0, 0x4de3: 0x00c0, + 0x4de4: 0x00c0, 0x4de5: 0x00c0, 0x4de6: 0x00c0, 0x4de7: 0x00c0, 0x4de8: 0x00c0, 0x4de9: 0x00c0, + 0x4dea: 0x00c0, 0x4deb: 0x00c0, 0x4dec: 0x00c3, 0x4ded: 0x00c3, 0x4dee: 0x00c3, 0x4def: 0x00c3, + 0x4df0: 0x00c0, 0x4df1: 0x00c0, 0x4df2: 0x00c0, 0x4df3: 0x00c0, 0x4df4: 0x00c0, 0x4df5: 0x00c0, + 0x4df6: 0x00c0, 0x4df7: 0x00c0, 0x4df8: 0x00c0, 0x4df9: 0x00c0, + 0x4dff: 0x0080, + // Block 0x138, offset 0x4e00 + 0x4e00: 0x00c0, 0x4e01: 0x00c0, 0x4e02: 0x00c0, 0x4e03: 0x00c0, 0x4e04: 0x00c0, + 0x4e07: 0x0080, 0x4e08: 0x0080, 0x4e09: 0x0080, 0x4e0a: 0x0080, 0x4e0b: 0x0080, + 0x4e0c: 0x0080, 0x4e0d: 0x0080, 0x4e0e: 0x0080, 0x4e0f: 0x0080, 0x4e10: 0x00c3, 0x4e11: 0x00c3, + 0x4e12: 0x00c3, 0x4e13: 0x00c3, 0x4e14: 0x00c3, 0x4e15: 0x00c3, 0x4e16: 0x00c3, + // Block 0x139, offset 0x4e40 + 0x4e40: 0x00c2, 0x4e41: 0x00c2, 0x4e42: 0x00c2, 0x4e43: 0x00c2, 0x4e44: 0x00c2, 0x4e45: 0x00c2, + 0x4e46: 0x00c2, 0x4e47: 0x00c2, 0x4e48: 0x00c2, 0x4e49: 0x00c2, 0x4e4a: 0x00c2, 0x4e4b: 0x00c2, + 0x4e4c: 0x00c2, 0x4e4d: 0x00c2, 0x4e4e: 0x00c2, 0x4e4f: 0x00c2, 0x4e50: 0x00c2, 0x4e51: 0x00c2, + 0x4e52: 0x00c2, 0x4e53: 0x00c2, 0x4e54: 0x00c2, 0x4e55: 0x00c2, 0x4e56: 0x00c2, 0x4e57: 0x00c2, + 0x4e58: 0x00c2, 0x4e59: 0x00c2, 0x4e5a: 0x00c2, 0x4e5b: 0x00c2, 0x4e5c: 0x00c2, 0x4e5d: 0x00c2, + 0x4e5e: 0x00c2, 0x4e5f: 0x00c2, 0x4e60: 0x00c2, 0x4e61: 0x00c2, 0x4e62: 0x00c2, 0x4e63: 0x00c2, + 0x4e64: 0x00c2, 0x4e65: 0x00c2, 0x4e66: 0x00c2, 0x4e67: 0x00c2, 0x4e68: 0x00c2, 0x4e69: 0x00c2, + 0x4e6a: 0x00c2, 0x4e6b: 0x00c2, 0x4e6c: 0x00c2, 0x4e6d: 0x00c2, 0x4e6e: 0x00c2, 0x4e6f: 0x00c2, + 0x4e70: 0x00c2, 0x4e71: 0x00c2, 0x4e72: 0x00c2, 0x4e73: 0x00c2, 0x4e74: 0x00c2, 0x4e75: 0x00c2, + 0x4e76: 0x00c2, 0x4e77: 0x00c2, 0x4e78: 0x00c2, 0x4e79: 0x00c2, 0x4e7a: 0x00c2, 0x4e7b: 0x00c2, + 0x4e7c: 0x00c2, 0x4e7d: 0x00c2, 0x4e7e: 0x00c2, 0x4e7f: 0x00c2, + // Block 0x13a, offset 0x4e80 + 0x4e80: 0x00c2, 0x4e81: 0x00c2, 0x4e82: 0x00c2, 0x4e83: 0x00c2, 0x4e84: 0x00c3, 0x4e85: 0x00c3, + 0x4e86: 0x00c3, 0x4e87: 0x00c3, 0x4e88: 0x00c3, 0x4e89: 0x00c3, 0x4e8a: 0x00c3, 0x4e8b: 0x00c3, + 0x4e90: 0x00c0, 0x4e91: 0x00c0, + 0x4e92: 0x00c0, 0x4e93: 0x00c0, 0x4e94: 0x00c0, 0x4e95: 0x00c0, 0x4e96: 0x00c0, 0x4e97: 0x00c0, + 0x4e98: 0x00c0, 0x4e99: 0x00c0, + 0x4e9e: 0x0080, 0x4e9f: 0x0080, + // Block 0x13b, offset 0x4ec0 + 0x4ef1: 0x0080, 0x4ef2: 0x0080, 0x4ef3: 0x0080, 0x4ef4: 0x0080, 0x4ef5: 0x0080, + 0x4ef6: 0x0080, 0x4ef7: 0x0080, 0x4ef8: 0x0080, 0x4ef9: 0x0080, 0x4efa: 0x0080, 0x4efb: 0x0080, + 0x4efc: 0x0080, 0x4efd: 0x0080, 0x4efe: 0x0080, 0x4eff: 0x0080, + // Block 0x13c, offset 0x4f00 + 0x4f00: 0x0080, 0x4f01: 0x0080, 0x4f02: 0x0080, 0x4f03: 0x0080, 0x4f04: 0x0080, 0x4f05: 0x0080, + 0x4f06: 0x0080, 0x4f07: 0x0080, 0x4f08: 0x0080, 0x4f09: 0x0080, 0x4f0a: 0x0080, 0x4f0b: 0x0080, + 0x4f0c: 0x0080, 0x4f0d: 0x0080, 0x4f0e: 0x0080, 0x4f0f: 0x0080, 0x4f10: 0x0080, 0x4f11: 0x0080, + 0x4f12: 0x0080, 0x4f13: 0x0080, 0x4f14: 0x0080, 0x4f15: 0x0080, 0x4f16: 0x0080, 0x4f17: 0x0080, + 0x4f18: 0x0080, 0x4f19: 0x0080, 0x4f1a: 0x0080, 0x4f1b: 0x0080, 0x4f1c: 0x0080, 0x4f1d: 0x0080, + 0x4f1e: 0x0080, 0x4f1f: 0x0080, 0x4f20: 0x0080, 0x4f21: 0x0080, 0x4f22: 0x0080, 0x4f23: 0x0080, + 0x4f24: 0x0080, 0x4f25: 0x0080, 0x4f26: 0x0080, 0x4f27: 0x0080, 0x4f28: 0x0080, 0x4f29: 0x0080, + 0x4f2a: 0x0080, 0x4f2b: 0x0080, 0x4f2c: 0x0080, 0x4f2d: 0x0080, 0x4f2e: 0x0080, 0x4f2f: 0x0080, + 0x4f30: 0x0080, 0x4f31: 0x0080, 0x4f32: 0x0080, 0x4f33: 0x0080, 0x4f34: 0x0080, + // Block 0x13d, offset 0x4f40 + 0x4f41: 0x0080, 0x4f42: 0x0080, 0x4f43: 0x0080, 0x4f44: 0x0080, 0x4f45: 0x0080, + 0x4f46: 0x0080, 0x4f47: 0x0080, 0x4f48: 0x0080, 0x4f49: 0x0080, 0x4f4a: 0x0080, 0x4f4b: 0x0080, + 0x4f4c: 0x0080, 0x4f4d: 0x0080, 0x4f4e: 0x0080, 0x4f4f: 0x0080, 0x4f50: 0x0080, 0x4f51: 0x0080, + 0x4f52: 0x0080, 0x4f53: 0x0080, 0x4f54: 0x0080, 0x4f55: 0x0080, 0x4f56: 0x0080, 0x4f57: 0x0080, + 0x4f58: 0x0080, 0x4f59: 0x0080, 0x4f5a: 0x0080, 0x4f5b: 0x0080, 0x4f5c: 0x0080, 0x4f5d: 0x0080, + 0x4f5e: 0x0080, 0x4f5f: 0x0080, 0x4f60: 0x0080, 0x4f61: 0x0080, 0x4f62: 0x0080, 0x4f63: 0x0080, + 0x4f64: 0x0080, 0x4f65: 0x0080, 0x4f66: 0x0080, 0x4f67: 0x0080, 0x4f68: 0x0080, 0x4f69: 0x0080, + 0x4f6a: 0x0080, 0x4f6b: 0x0080, 0x4f6c: 0x0080, 0x4f6d: 0x0080, 0x4f6e: 0x0080, 0x4f6f: 0x0080, + 0x4f70: 0x0080, 0x4f71: 0x0080, 0x4f72: 0x0080, 0x4f73: 0x0080, 0x4f74: 0x0080, 0x4f75: 0x0080, + 0x4f76: 0x0080, 0x4f77: 0x0080, 0x4f78: 0x0080, 0x4f79: 0x0080, 0x4f7a: 0x0080, 0x4f7b: 0x0080, + 0x4f7c: 0x0080, 0x4f7d: 0x0080, + // Block 0x13e, offset 0x4f80 + 0x4f80: 0x0080, 0x4f81: 0x0080, 0x4f82: 0x0080, 0x4f83: 0x0080, 0x4f85: 0x0080, + 0x4f86: 0x0080, 0x4f87: 0x0080, 0x4f88: 0x0080, 0x4f89: 0x0080, 0x4f8a: 0x0080, 0x4f8b: 0x0080, + 0x4f8c: 0x0080, 0x4f8d: 0x0080, 0x4f8e: 0x0080, 0x4f8f: 0x0080, 0x4f90: 0x0080, 0x4f91: 0x0080, + 0x4f92: 0x0080, 0x4f93: 0x0080, 0x4f94: 0x0080, 0x4f95: 0x0080, 0x4f96: 0x0080, 0x4f97: 0x0080, + 0x4f98: 0x0080, 0x4f99: 0x0080, 0x4f9a: 0x0080, 0x4f9b: 0x0080, 0x4f9c: 0x0080, 0x4f9d: 0x0080, + 0x4f9e: 0x0080, 0x4f9f: 0x0080, 0x4fa1: 0x0080, 0x4fa2: 0x0080, + 0x4fa4: 0x0080, 0x4fa7: 0x0080, 0x4fa9: 0x0080, + 0x4faa: 0x0080, 0x4fab: 0x0080, 0x4fac: 0x0080, 0x4fad: 0x0080, 0x4fae: 0x0080, 0x4faf: 0x0080, + 0x4fb0: 0x0080, 0x4fb1: 0x0080, 0x4fb2: 0x0080, 0x4fb4: 0x0080, 0x4fb5: 0x0080, + 0x4fb6: 0x0080, 0x4fb7: 0x0080, 0x4fb9: 0x0080, 0x4fbb: 0x0080, + // Block 0x13f, offset 0x4fc0 + 0x4fc2: 0x0080, + 0x4fc7: 0x0080, 0x4fc9: 0x0080, 0x4fcb: 0x0080, + 0x4fcd: 0x0080, 0x4fce: 0x0080, 0x4fcf: 0x0080, 0x4fd1: 0x0080, + 0x4fd2: 0x0080, 0x4fd4: 0x0080, 0x4fd7: 0x0080, + 0x4fd9: 0x0080, 0x4fdb: 0x0080, 0x4fdd: 0x0080, + 0x4fdf: 0x0080, 0x4fe1: 0x0080, 0x4fe2: 0x0080, + 0x4fe4: 0x0080, 0x4fe7: 0x0080, 0x4fe8: 0x0080, 0x4fe9: 0x0080, + 0x4fea: 0x0080, 0x4fec: 0x0080, 0x4fed: 0x0080, 0x4fee: 0x0080, 0x4fef: 0x0080, + 0x4ff0: 0x0080, 0x4ff1: 0x0080, 0x4ff2: 0x0080, 0x4ff4: 0x0080, 0x4ff5: 0x0080, + 0x4ff6: 0x0080, 0x4ff7: 0x0080, 0x4ff9: 0x0080, 0x4ffa: 0x0080, 0x4ffb: 0x0080, + 0x4ffc: 0x0080, 0x4ffe: 0x0080, + // Block 0x140, offset 0x5000 + 0x5000: 0x0080, 0x5001: 0x0080, 0x5002: 0x0080, 0x5003: 0x0080, 0x5004: 0x0080, 0x5005: 0x0080, + 0x5006: 0x0080, 0x5007: 0x0080, 0x5008: 0x0080, 0x5009: 0x0080, 0x500b: 0x0080, + 0x500c: 0x0080, 0x500d: 0x0080, 0x500e: 0x0080, 0x500f: 0x0080, 0x5010: 0x0080, 0x5011: 0x0080, + 0x5012: 0x0080, 0x5013: 0x0080, 0x5014: 0x0080, 0x5015: 0x0080, 0x5016: 0x0080, 0x5017: 0x0080, + 0x5018: 0x0080, 0x5019: 0x0080, 0x501a: 0x0080, 0x501b: 0x0080, + 0x5021: 0x0080, 0x5022: 0x0080, 0x5023: 0x0080, + 0x5025: 0x0080, 0x5026: 0x0080, 0x5027: 0x0080, 0x5028: 0x0080, 0x5029: 0x0080, + 0x502b: 0x0080, 0x502c: 0x0080, 0x502d: 0x0080, 0x502e: 0x0080, 0x502f: 0x0080, + 0x5030: 0x0080, 0x5031: 0x0080, 0x5032: 0x0080, 0x5033: 0x0080, 0x5034: 0x0080, 0x5035: 0x0080, + 0x5036: 0x0080, 0x5037: 0x0080, 0x5038: 0x0080, 0x5039: 0x0080, 0x503a: 0x0080, 0x503b: 0x0080, + // Block 0x141, offset 0x5040 + 0x5070: 0x0080, 0x5071: 0x0080, + // Block 0x142, offset 0x5080 + 0x5080: 0x0080, 0x5081: 0x0080, 0x5082: 0x0080, 0x5083: 0x0080, 0x5084: 0x0080, 0x5085: 0x0080, + 0x5086: 0x0080, 0x5087: 0x0080, 0x5088: 0x0080, 0x5089: 0x0080, 0x508a: 0x0080, 0x508b: 0x0080, + 0x508c: 0x0080, 0x508d: 0x0080, 0x508e: 0x0080, 0x508f: 0x0080, 0x5090: 0x0080, 0x5091: 0x0080, + 0x5092: 0x0080, 0x5093: 0x0080, 0x5094: 0x0080, 0x5095: 0x0080, 0x5096: 0x0080, 0x5097: 0x0080, + 0x5098: 0x0080, 0x5099: 0x0080, 0x509a: 0x0080, 0x509b: 0x0080, 0x509c: 0x0080, 0x509d: 0x0080, + 0x509e: 0x0080, 0x509f: 0x0080, 0x50a0: 0x0080, 0x50a1: 0x0080, 0x50a2: 0x0080, 0x50a3: 0x0080, + 0x50a4: 0x0080, 0x50a5: 0x0080, 0x50a6: 0x0080, 0x50a7: 0x0080, 0x50a8: 0x0080, 0x50a9: 0x0080, + 0x50aa: 0x0080, 0x50ab: 0x0080, + 0x50b0: 0x0080, 0x50b1: 0x0080, 0x50b2: 0x0080, 0x50b3: 0x0080, 0x50b4: 0x0080, 0x50b5: 0x0080, + 0x50b6: 0x0080, 0x50b7: 0x0080, 0x50b8: 0x0080, 0x50b9: 0x0080, 0x50ba: 0x0080, 0x50bb: 0x0080, + 0x50bc: 0x0080, 0x50bd: 0x0080, 0x50be: 0x0080, 0x50bf: 0x0080, + // Block 0x143, offset 0x50c0 + 0x50c0: 0x0080, 0x50c1: 0x0080, 0x50c2: 0x0080, 0x50c3: 0x0080, 0x50c4: 0x0080, 0x50c5: 0x0080, + 0x50c6: 0x0080, 0x50c7: 0x0080, 0x50c8: 0x0080, 0x50c9: 0x0080, 0x50ca: 0x0080, 0x50cb: 0x0080, + 0x50cc: 0x0080, 0x50cd: 0x0080, 0x50ce: 0x0080, 0x50cf: 0x0080, 0x50d0: 0x0080, 0x50d1: 0x0080, + 0x50d2: 0x0080, 0x50d3: 0x0080, + 0x50e0: 0x0080, 0x50e1: 0x0080, 0x50e2: 0x0080, 0x50e3: 0x0080, + 0x50e4: 0x0080, 0x50e5: 0x0080, 0x50e6: 0x0080, 0x50e7: 0x0080, 0x50e8: 0x0080, 0x50e9: 0x0080, + 0x50ea: 0x0080, 0x50eb: 0x0080, 0x50ec: 0x0080, 0x50ed: 0x0080, 0x50ee: 0x0080, + 0x50f1: 0x0080, 0x50f2: 0x0080, 0x50f3: 0x0080, 0x50f4: 0x0080, 0x50f5: 0x0080, + 0x50f6: 0x0080, 0x50f7: 0x0080, 0x50f8: 0x0080, 0x50f9: 0x0080, 0x50fa: 0x0080, 0x50fb: 0x0080, + 0x50fc: 0x0080, 0x50fd: 0x0080, 0x50fe: 0x0080, 0x50ff: 0x0080, + // Block 0x144, offset 0x5100 + 0x5101: 0x0080, 0x5102: 0x0080, 0x5103: 0x0080, 0x5104: 0x0080, 0x5105: 0x0080, + 0x5106: 0x0080, 0x5107: 0x0080, 0x5108: 0x0080, 0x5109: 0x0080, 0x510a: 0x0080, 0x510b: 0x0080, + 0x510c: 0x0080, 0x510d: 0x0080, 0x510e: 0x0080, 0x510f: 0x0080, 0x5111: 0x0080, + 0x5112: 0x0080, 0x5113: 0x0080, 0x5114: 0x0080, 0x5115: 0x0080, 0x5116: 0x0080, 0x5117: 0x0080, + 0x5118: 0x0080, 0x5119: 0x0080, 0x511a: 0x0080, 0x511b: 0x0080, 0x511c: 0x0080, 0x511d: 0x0080, + 0x511e: 0x0080, 0x511f: 0x0080, 0x5120: 0x0080, 0x5121: 0x0080, 0x5122: 0x0080, 0x5123: 0x0080, + 0x5124: 0x0080, 0x5125: 0x0080, 0x5126: 0x0080, 0x5127: 0x0080, 0x5128: 0x0080, 0x5129: 0x0080, + 0x512a: 0x0080, 0x512b: 0x0080, 0x512c: 0x0080, 0x512d: 0x0080, 0x512e: 0x0080, 0x512f: 0x0080, + 0x5130: 0x0080, 0x5131: 0x0080, 0x5132: 0x0080, 0x5133: 0x0080, 0x5134: 0x0080, 0x5135: 0x0080, + // Block 0x145, offset 0x5140 + 0x5140: 0x0080, 0x5141: 0x0080, 0x5142: 0x0080, 0x5143: 0x0080, 0x5144: 0x0080, 0x5145: 0x0080, + 0x5146: 0x0080, 0x5147: 0x0080, 0x5148: 0x0080, 0x5149: 0x0080, 0x514a: 0x0080, 0x514b: 0x0080, + 0x514c: 0x0080, 0x5150: 0x0080, 0x5151: 0x0080, + 0x5152: 0x0080, 0x5153: 0x0080, 0x5154: 0x0080, 0x5155: 0x0080, 0x5156: 0x0080, 0x5157: 0x0080, + 0x5158: 0x0080, 0x5159: 0x0080, 0x515a: 0x0080, 0x515b: 0x0080, 0x515c: 0x0080, 0x515d: 0x0080, + 0x515e: 0x0080, 0x515f: 0x0080, 0x5160: 0x0080, 0x5161: 0x0080, 0x5162: 0x0080, 0x5163: 0x0080, + 0x5164: 0x0080, 0x5165: 0x0080, 0x5166: 0x0080, 0x5167: 0x0080, 0x5168: 0x0080, 0x5169: 0x0080, + 0x516a: 0x0080, 0x516b: 0x0080, 0x516c: 0x0080, 0x516d: 0x0080, 0x516e: 0x0080, 0x516f: 0x0080, + 0x5170: 0x0080, 0x5171: 0x0080, 0x5172: 0x0080, 0x5173: 0x0080, 0x5174: 0x0080, 0x5175: 0x0080, + 0x5176: 0x0080, 0x5177: 0x0080, 0x5178: 0x0080, 0x5179: 0x0080, 0x517a: 0x0080, 0x517b: 0x0080, + 0x517c: 0x0080, 0x517d: 0x0080, 0x517e: 0x0080, 0x517f: 0x0080, + // Block 0x146, offset 0x5180 + 0x5180: 0x0080, 0x5181: 0x0080, 0x5182: 0x0080, 0x5183: 0x0080, 0x5184: 0x0080, 0x5185: 0x0080, + 0x5186: 0x0080, 0x5187: 0x0080, 0x5188: 0x0080, 0x5189: 0x0080, 0x518a: 0x0080, 0x518b: 0x0080, + 0x518c: 0x0080, 0x518d: 0x0080, 0x518e: 0x0080, 0x518f: 0x0080, 0x5190: 0x0080, 0x5191: 0x0080, + 0x5192: 0x0080, 0x5193: 0x0080, 0x5194: 0x0080, 0x5195: 0x0080, 0x5196: 0x0080, 0x5197: 0x0080, + 0x5198: 0x0080, 0x5199: 0x0080, 0x519a: 0x0080, 0x519b: 0x0080, 0x519c: 0x0080, 0x519d: 0x0080, + 0x519e: 0x0080, 0x519f: 0x0080, 0x51a0: 0x0080, 0x51a1: 0x0080, 0x51a2: 0x0080, 0x51a3: 0x0080, + 0x51a4: 0x0080, 0x51a5: 0x0080, 0x51a6: 0x0080, 0x51a7: 0x0080, 0x51a8: 0x0080, 0x51a9: 0x0080, + 0x51aa: 0x0080, 0x51ab: 0x0080, 0x51ac: 0x0080, + 0x51b0: 0x0080, 0x51b1: 0x0080, 0x51b2: 0x0080, 0x51b3: 0x0080, 0x51b4: 0x0080, 0x51b5: 0x0080, + 0x51b6: 0x0080, 0x51b7: 0x0080, 0x51b8: 0x0080, 0x51b9: 0x0080, 0x51ba: 0x0080, 0x51bb: 0x0080, + 0x51bc: 0x0080, 0x51bd: 0x0080, 0x51be: 0x0080, 0x51bf: 0x0080, + // Block 0x147, offset 0x51c0 + 0x51c0: 0x0080, 0x51c1: 0x0080, 0x51c2: 0x0080, 0x51c3: 0x0080, 0x51c4: 0x0080, 0x51c5: 0x0080, + 0x51c6: 0x0080, 0x51c7: 0x0080, 0x51c8: 0x0080, 0x51c9: 0x0080, 0x51ca: 0x0080, 0x51cb: 0x0080, + 0x51cc: 0x0080, 0x51cd: 0x0080, 0x51ce: 0x0080, 0x51cf: 0x0080, 0x51d0: 0x0080, 0x51d1: 0x0080, + 0x51d2: 0x0080, 0x51d3: 0x0080, 0x51d4: 0x0080, 0x51d5: 0x0080, 0x51d6: 0x0080, 0x51d7: 0x0080, + 0x51d8: 0x0080, 0x51d9: 0x0080, 0x51da: 0x0080, 0x51db: 0x0080, 0x51dc: 0x0080, 0x51dd: 0x0080, + 0x51de: 0x0080, 0x51df: 0x0080, 0x51e0: 0x0080, 0x51e1: 0x0080, 0x51e2: 0x0080, 0x51e3: 0x0080, + 0x51e4: 0x0080, 0x51e5: 0x0080, 0x51e6: 0x0080, 0x51e7: 0x0080, 0x51e8: 0x0080, 0x51e9: 0x0080, + 0x51ea: 0x0080, 0x51eb: 0x0080, 0x51ec: 0x0080, + // Block 0x148, offset 0x5200 + 0x5226: 0x0080, 0x5227: 0x0080, 0x5228: 0x0080, 0x5229: 0x0080, + 0x522a: 0x0080, 0x522b: 0x0080, 0x522c: 0x0080, 0x522d: 0x0080, 0x522e: 0x0080, 0x522f: 0x0080, + 0x5230: 0x0080, 0x5231: 0x0080, 0x5232: 0x0080, 0x5233: 0x0080, 0x5234: 0x0080, 0x5235: 0x0080, + 0x5236: 0x0080, 0x5237: 0x0080, 0x5238: 0x0080, 0x5239: 0x0080, 0x523a: 0x0080, 0x523b: 0x0080, + 0x523c: 0x0080, 0x523d: 0x0080, 0x523e: 0x0080, 0x523f: 0x0080, + // Block 0x149, offset 0x5240 + 0x5240: 0x008c, 0x5241: 0x0080, 0x5242: 0x0080, + 0x5250: 0x0080, 0x5251: 0x0080, + 0x5252: 0x0080, 0x5253: 0x0080, 0x5254: 0x0080, 0x5255: 0x0080, 0x5256: 0x0080, 0x5257: 0x0080, + 0x5258: 0x0080, 0x5259: 0x0080, 0x525a: 0x0080, 0x525b: 0x0080, 0x525c: 0x0080, 0x525d: 0x0080, + 0x525e: 0x0080, 0x525f: 0x0080, 0x5260: 0x0080, 0x5261: 0x0080, 0x5262: 0x0080, 0x5263: 0x0080, + 0x5264: 0x0080, 0x5265: 0x0080, 0x5266: 0x0080, 0x5267: 0x0080, 0x5268: 0x0080, 0x5269: 0x0080, + 0x526a: 0x0080, 0x526b: 0x0080, 0x526c: 0x0080, 0x526d: 0x0080, 0x526e: 0x0080, 0x526f: 0x0080, + 0x5270: 0x0080, 0x5271: 0x0080, 0x5272: 0x0080, 0x5273: 0x0080, 0x5274: 0x0080, 0x5275: 0x0080, + 0x5276: 0x0080, 0x5277: 0x0080, 0x5278: 0x0080, 0x5279: 0x0080, 0x527a: 0x0080, 0x527b: 0x0080, + // Block 0x14a, offset 0x5280 + 0x5280: 0x0080, 0x5281: 0x0080, 0x5282: 0x0080, 0x5283: 0x0080, 0x5284: 0x0080, 0x5285: 0x0080, + 0x5286: 0x0080, 0x5287: 0x0080, 0x5288: 0x0080, + 0x5290: 0x0080, 0x5291: 0x0080, + 0x52a0: 0x0080, 0x52a1: 0x0080, 0x52a2: 0x0080, 0x52a3: 0x0080, + 0x52a4: 0x0080, 0x52a5: 0x0080, + // Block 0x14b, offset 0x52c0 + 0x52c0: 0x0080, 0x52c1: 0x0080, 0x52c2: 0x0080, 0x52c3: 0x0080, 0x52c4: 0x0080, 0x52c5: 0x0080, + 0x52c6: 0x0080, 0x52c7: 0x0080, 0x52c8: 0x0080, 0x52c9: 0x0080, 0x52ca: 0x0080, 0x52cb: 0x0080, + 0x52cc: 0x0080, 0x52cd: 0x0080, 0x52ce: 0x0080, 0x52cf: 0x0080, 0x52d0: 0x0080, 0x52d1: 0x0080, + 0x52d2: 0x0080, 0x52d3: 0x0080, 0x52d4: 0x0080, 0x52d5: 0x0080, + 0x52e0: 0x0080, 0x52e1: 0x0080, 0x52e2: 0x0080, 0x52e3: 0x0080, + 0x52e4: 0x0080, 0x52e5: 0x0080, 0x52e6: 0x0080, 0x52e7: 0x0080, 0x52e8: 0x0080, 0x52e9: 0x0080, + 0x52ea: 0x0080, 0x52eb: 0x0080, 0x52ec: 0x0080, + 0x52f0: 0x0080, 0x52f1: 0x0080, 0x52f2: 0x0080, 0x52f3: 0x0080, 0x52f4: 0x0080, 0x52f5: 0x0080, + 0x52f6: 0x0080, 0x52f7: 0x0080, 0x52f8: 0x0080, 0x52f9: 0x0080, 0x52fa: 0x0080, + // Block 0x14c, offset 0x5300 + 0x5300: 0x0080, 0x5301: 0x0080, 0x5302: 0x0080, 0x5303: 0x0080, 0x5304: 0x0080, 0x5305: 0x0080, + 0x5306: 0x0080, 0x5307: 0x0080, 0x5308: 0x0080, 0x5309: 0x0080, 0x530a: 0x0080, 0x530b: 0x0080, + 0x530c: 0x0080, 0x530d: 0x0080, 0x530e: 0x0080, 0x530f: 0x0080, 0x5310: 0x0080, 0x5311: 0x0080, + 0x5312: 0x0080, 0x5313: 0x0080, 0x5314: 0x0080, 0x5315: 0x0080, 0x5316: 0x0080, 0x5317: 0x0080, + 0x5318: 0x0080, 0x5319: 0x0080, 0x531a: 0x0080, 0x531b: 0x0080, 0x531c: 0x0080, 0x531d: 0x0080, + 0x531e: 0x0080, 0x531f: 0x0080, 0x5320: 0x0080, 0x5321: 0x0080, 0x5322: 0x0080, 0x5323: 0x0080, + 0x5324: 0x0080, 0x5325: 0x0080, 0x5326: 0x0080, 0x5327: 0x0080, 0x5328: 0x0080, 0x5329: 0x0080, + 0x532a: 0x0080, 0x532b: 0x0080, 0x532c: 0x0080, 0x532d: 0x0080, 0x532e: 0x0080, 0x532f: 0x0080, + 0x5330: 0x0080, 0x5331: 0x0080, 0x5332: 0x0080, 0x5333: 0x0080, + // Block 0x14d, offset 0x5340 + 0x5340: 0x0080, 0x5341: 0x0080, 0x5342: 0x0080, 0x5343: 0x0080, 0x5344: 0x0080, 0x5345: 0x0080, + 0x5346: 0x0080, 0x5347: 0x0080, 0x5348: 0x0080, 0x5349: 0x0080, 0x534a: 0x0080, 0x534b: 0x0080, + 0x534c: 0x0080, 0x534d: 0x0080, 0x534e: 0x0080, 0x534f: 0x0080, 0x5350: 0x0080, 0x5351: 0x0080, + 0x5352: 0x0080, 0x5353: 0x0080, 0x5354: 0x0080, 0x5355: 0x0080, 0x5356: 0x0080, 0x5357: 0x0080, + 0x5358: 0x0080, + 0x5360: 0x0080, 0x5361: 0x0080, 0x5362: 0x0080, 0x5363: 0x0080, + 0x5364: 0x0080, 0x5365: 0x0080, 0x5366: 0x0080, 0x5367: 0x0080, 0x5368: 0x0080, 0x5369: 0x0080, + 0x536a: 0x0080, 0x536b: 0x0080, + // Block 0x14e, offset 0x5380 + 0x5380: 0x0080, 0x5381: 0x0080, 0x5382: 0x0080, 0x5383: 0x0080, 0x5384: 0x0080, 0x5385: 0x0080, + 0x5386: 0x0080, 0x5387: 0x0080, 0x5388: 0x0080, 0x5389: 0x0080, 0x538a: 0x0080, 0x538b: 0x0080, + 0x5390: 0x0080, 0x5391: 0x0080, + 0x5392: 0x0080, 0x5393: 0x0080, 0x5394: 0x0080, 0x5395: 0x0080, 0x5396: 0x0080, 0x5397: 0x0080, + 0x5398: 0x0080, 0x5399: 0x0080, 0x539a: 0x0080, 0x539b: 0x0080, 0x539c: 0x0080, 0x539d: 0x0080, + 0x539e: 0x0080, 0x539f: 0x0080, 0x53a0: 0x0080, 0x53a1: 0x0080, 0x53a2: 0x0080, 0x53a3: 0x0080, + 0x53a4: 0x0080, 0x53a5: 0x0080, 0x53a6: 0x0080, 0x53a7: 0x0080, 0x53a8: 0x0080, 0x53a9: 0x0080, + 0x53aa: 0x0080, 0x53ab: 0x0080, 0x53ac: 0x0080, 0x53ad: 0x0080, 0x53ae: 0x0080, 0x53af: 0x0080, + 0x53b0: 0x0080, 0x53b1: 0x0080, 0x53b2: 0x0080, 0x53b3: 0x0080, 0x53b4: 0x0080, 0x53b5: 0x0080, + 0x53b6: 0x0080, 0x53b7: 0x0080, 0x53b8: 0x0080, 0x53b9: 0x0080, 0x53ba: 0x0080, 0x53bb: 0x0080, + 0x53bc: 0x0080, 0x53bd: 0x0080, 0x53be: 0x0080, 0x53bf: 0x0080, + // Block 0x14f, offset 0x53c0 + 0x53c0: 0x0080, 0x53c1: 0x0080, 0x53c2: 0x0080, 0x53c3: 0x0080, 0x53c4: 0x0080, 0x53c5: 0x0080, + 0x53c6: 0x0080, 0x53c7: 0x0080, + 0x53d0: 0x0080, 0x53d1: 0x0080, + 0x53d2: 0x0080, 0x53d3: 0x0080, 0x53d4: 0x0080, 0x53d5: 0x0080, 0x53d6: 0x0080, 0x53d7: 0x0080, + 0x53d8: 0x0080, 0x53d9: 0x0080, + 0x53e0: 0x0080, 0x53e1: 0x0080, 0x53e2: 0x0080, 0x53e3: 0x0080, + 0x53e4: 0x0080, 0x53e5: 0x0080, 0x53e6: 0x0080, 0x53e7: 0x0080, 0x53e8: 0x0080, 0x53e9: 0x0080, + 0x53ea: 0x0080, 0x53eb: 0x0080, 0x53ec: 0x0080, 0x53ed: 0x0080, 0x53ee: 0x0080, 0x53ef: 0x0080, + 0x53f0: 0x0080, 0x53f1: 0x0080, 0x53f2: 0x0080, 0x53f3: 0x0080, 0x53f4: 0x0080, 0x53f5: 0x0080, + 0x53f6: 0x0080, 0x53f7: 0x0080, 0x53f8: 0x0080, 0x53f9: 0x0080, 0x53fa: 0x0080, 0x53fb: 0x0080, + 0x53fc: 0x0080, 0x53fd: 0x0080, 0x53fe: 0x0080, 0x53ff: 0x0080, + // Block 0x150, offset 0x5400 + 0x5400: 0x0080, 0x5401: 0x0080, 0x5402: 0x0080, 0x5403: 0x0080, 0x5404: 0x0080, 0x5405: 0x0080, + 0x5406: 0x0080, 0x5407: 0x0080, + 0x5410: 0x0080, 0x5411: 0x0080, + 0x5412: 0x0080, 0x5413: 0x0080, 0x5414: 0x0080, 0x5415: 0x0080, 0x5416: 0x0080, 0x5417: 0x0080, + 0x5418: 0x0080, 0x5419: 0x0080, 0x541a: 0x0080, 0x541b: 0x0080, 0x541c: 0x0080, 0x541d: 0x0080, + 0x541e: 0x0080, 0x541f: 0x0080, 0x5420: 0x0080, 0x5421: 0x0080, 0x5422: 0x0080, 0x5423: 0x0080, + 0x5424: 0x0080, 0x5425: 0x0080, 0x5426: 0x0080, 0x5427: 0x0080, 0x5428: 0x0080, 0x5429: 0x0080, + 0x542a: 0x0080, 0x542b: 0x0080, 0x542c: 0x0080, 0x542d: 0x0080, + // Block 0x151, offset 0x5440 + 0x5440: 0x0080, 0x5441: 0x0080, 0x5442: 0x0080, 0x5443: 0x0080, 0x5444: 0x0080, 0x5445: 0x0080, + 0x5446: 0x0080, 0x5447: 0x0080, 0x5448: 0x0080, 0x5449: 0x0080, 0x544a: 0x0080, 0x544b: 0x0080, + 0x544d: 0x0080, 0x544e: 0x0080, 0x544f: 0x0080, 0x5450: 0x0080, 0x5451: 0x0080, + 0x5452: 0x0080, 0x5453: 0x0080, 0x5454: 0x0080, 0x5455: 0x0080, 0x5456: 0x0080, 0x5457: 0x0080, + 0x5458: 0x0080, 0x5459: 0x0080, 0x545a: 0x0080, 0x545b: 0x0080, 0x545c: 0x0080, 0x545d: 0x0080, + 0x545e: 0x0080, 0x545f: 0x0080, 0x5460: 0x0080, 0x5461: 0x0080, 0x5462: 0x0080, 0x5463: 0x0080, + 0x5464: 0x0080, 0x5465: 0x0080, 0x5466: 0x0080, 0x5467: 0x0080, 0x5468: 0x0080, 0x5469: 0x0080, + 0x546a: 0x0080, 0x546b: 0x0080, 0x546c: 0x0080, 0x546d: 0x0080, 0x546e: 0x0080, 0x546f: 0x0080, + 0x5470: 0x0080, 0x5471: 0x0080, 0x5472: 0x0080, 0x5473: 0x0080, 0x5474: 0x0080, 0x5475: 0x0080, + 0x5476: 0x0080, 0x5477: 0x0080, 0x5478: 0x0080, 0x5479: 0x0080, 0x547a: 0x0080, 0x547b: 0x0080, + 0x547c: 0x0080, 0x547d: 0x0080, 0x547e: 0x0080, 0x547f: 0x0080, + // Block 0x152, offset 0x5480 + 0x5480: 0x0080, 0x5481: 0x0080, 0x5482: 0x0080, 0x5483: 0x0080, 0x5484: 0x0080, 0x5485: 0x0080, + 0x5486: 0x0080, 0x5487: 0x0080, 0x5488: 0x0080, 0x5489: 0x0080, 0x548a: 0x0080, 0x548b: 0x0080, + 0x548c: 0x0080, 0x548d: 0x0080, 0x548e: 0x0080, 0x548f: 0x0080, 0x5490: 0x0080, 0x5491: 0x0080, + 0x5492: 0x0080, 0x5493: 0x0080, 0x5494: 0x0080, 0x5495: 0x0080, 0x5496: 0x0080, 0x5497: 0x0080, + 0x5498: 0x0080, 0x5499: 0x0080, 0x549a: 0x0080, 0x549b: 0x0080, 0x549c: 0x0080, 0x549d: 0x0080, + 0x549e: 0x0080, 0x549f: 0x0080, 0x54a0: 0x0080, 0x54a1: 0x0080, 0x54a2: 0x0080, 0x54a3: 0x0080, + 0x54a4: 0x0080, 0x54a5: 0x0080, 0x54a6: 0x0080, 0x54a7: 0x0080, 0x54a8: 0x0080, 0x54a9: 0x0080, + 0x54aa: 0x0080, 0x54ab: 0x0080, 0x54ac: 0x0080, 0x54ad: 0x0080, 0x54ae: 0x0080, 0x54af: 0x0080, + 0x54b0: 0x0080, 0x54b1: 0x0080, 0x54b3: 0x0080, 0x54b4: 0x0080, 0x54b5: 0x0080, + 0x54b6: 0x0080, 0x54ba: 0x0080, 0x54bb: 0x0080, + 0x54bc: 0x0080, 0x54bd: 0x0080, 0x54be: 0x0080, 0x54bf: 0x0080, + // Block 0x153, offset 0x54c0 + 0x54c0: 0x0080, 0x54c1: 0x0080, 0x54c2: 0x0080, 0x54c3: 0x0080, 0x54c4: 0x0080, 0x54c5: 0x0080, + 0x54c6: 0x0080, 0x54c7: 0x0080, 0x54c8: 0x0080, 0x54c9: 0x0080, 0x54ca: 0x0080, 0x54cb: 0x0080, + 0x54cc: 0x0080, 0x54cd: 0x0080, 0x54ce: 0x0080, 0x54cf: 0x0080, 0x54d0: 0x0080, 0x54d1: 0x0080, + 0x54d2: 0x0080, 0x54d3: 0x0080, 0x54d4: 0x0080, 0x54d5: 0x0080, 0x54d6: 0x0080, 0x54d7: 0x0080, + 0x54d8: 0x0080, 0x54d9: 0x0080, 0x54da: 0x0080, 0x54db: 0x0080, 0x54dc: 0x0080, 0x54dd: 0x0080, + 0x54de: 0x0080, 0x54df: 0x0080, 0x54e0: 0x0080, 0x54e1: 0x0080, 0x54e2: 0x0080, + 0x54e5: 0x0080, 0x54e6: 0x0080, 0x54e7: 0x0080, 0x54e8: 0x0080, 0x54e9: 0x0080, + 0x54ea: 0x0080, 0x54ee: 0x0080, 0x54ef: 0x0080, + 0x54f0: 0x0080, 0x54f1: 0x0080, 0x54f2: 0x0080, 0x54f3: 0x0080, 0x54f4: 0x0080, 0x54f5: 0x0080, + 0x54f6: 0x0080, 0x54f7: 0x0080, 0x54f8: 0x0080, 0x54f9: 0x0080, 0x54fa: 0x0080, 0x54fb: 0x0080, + 0x54fc: 0x0080, 0x54fd: 0x0080, 0x54fe: 0x0080, 0x54ff: 0x0080, + // Block 0x154, offset 0x5500 + 0x5500: 0x0080, 0x5501: 0x0080, 0x5502: 0x0080, 0x5503: 0x0080, 0x5504: 0x0080, 0x5505: 0x0080, + 0x5506: 0x0080, 0x5507: 0x0080, 0x5508: 0x0080, 0x5509: 0x0080, 0x550a: 0x0080, + 0x550d: 0x0080, 0x550e: 0x0080, 0x550f: 0x0080, 0x5510: 0x0080, 0x5511: 0x0080, + 0x5512: 0x0080, 0x5513: 0x0080, 0x5514: 0x0080, 0x5515: 0x0080, 0x5516: 0x0080, 0x5517: 0x0080, + 0x5518: 0x0080, 0x5519: 0x0080, 0x551a: 0x0080, 0x551b: 0x0080, 0x551c: 0x0080, 0x551d: 0x0080, + 0x551e: 0x0080, 0x551f: 0x0080, 0x5520: 0x0080, 0x5521: 0x0080, 0x5522: 0x0080, 0x5523: 0x0080, + 0x5524: 0x0080, 0x5525: 0x0080, 0x5526: 0x0080, 0x5527: 0x0080, 0x5528: 0x0080, 0x5529: 0x0080, + 0x552a: 0x0080, 0x552b: 0x0080, 0x552c: 0x0080, 0x552d: 0x0080, 0x552e: 0x0080, 0x552f: 0x0080, + 0x5530: 0x0080, 0x5531: 0x0080, 0x5532: 0x0080, 0x5533: 0x0080, 0x5534: 0x0080, 0x5535: 0x0080, + 0x5536: 0x0080, 0x5537: 0x0080, 0x5538: 0x0080, 0x5539: 0x0080, 0x553a: 0x0080, 0x553b: 0x0080, + 0x553c: 0x0080, 0x553d: 0x0080, 0x553e: 0x0080, 0x553f: 0x0080, + // Block 0x155, offset 0x5540 + 0x5540: 0x0080, 0x5541: 0x0080, 0x5542: 0x0080, 0x5543: 0x0080, 0x5544: 0x0080, 0x5545: 0x0080, + 0x5546: 0x0080, 0x5547: 0x0080, 0x5548: 0x0080, 0x5549: 0x0080, 0x554a: 0x0080, 0x554b: 0x0080, + 0x554c: 0x0080, 0x554d: 0x0080, 0x554e: 0x0080, 0x554f: 0x0080, 0x5550: 0x0080, 0x5551: 0x0080, + 0x5552: 0x0080, 0x5553: 0x0080, + 0x5560: 0x0080, 0x5561: 0x0080, 0x5562: 0x0080, 0x5563: 0x0080, + 0x5564: 0x0080, 0x5565: 0x0080, 0x5566: 0x0080, 0x5567: 0x0080, 0x5568: 0x0080, 0x5569: 0x0080, + 0x556a: 0x0080, 0x556b: 0x0080, 0x556c: 0x0080, 0x556d: 0x0080, + 0x5570: 0x0080, 0x5571: 0x0080, 0x5572: 0x0080, 0x5573: 0x0080, + 0x5578: 0x0080, 0x5579: 0x0080, 0x557a: 0x0080, + // Block 0x156, offset 0x5580 + 0x5580: 0x0080, 0x5581: 0x0080, 0x5582: 0x0080, + 0x5590: 0x0080, 0x5591: 0x0080, + 0x5592: 0x0080, 0x5593: 0x0080, 0x5594: 0x0080, 0x5595: 0x0080, + // Block 0x157, offset 0x55c0 + 0x55c0: 0x00cc, 0x55c1: 0x00cc, 0x55c2: 0x00cc, 0x55c3: 0x00cc, 0x55c4: 0x00cc, 0x55c5: 0x00cc, + 0x55c6: 0x00cc, 0x55c7: 0x00cc, 0x55c8: 0x00cc, 0x55c9: 0x00cc, 0x55ca: 0x00cc, 0x55cb: 0x00cc, + 0x55cc: 0x00cc, 0x55cd: 0x00cc, 0x55ce: 0x00cc, 0x55cf: 0x00cc, 0x55d0: 0x00cc, 0x55d1: 0x00cc, + 0x55d2: 0x00cc, 0x55d3: 0x00cc, 0x55d4: 0x00cc, 0x55d5: 0x00cc, 0x55d6: 0x00cc, + // Block 0x158, offset 0x5600 + 0x5600: 0x00cc, 0x5601: 0x00cc, 0x5602: 0x00cc, 0x5603: 0x00cc, 0x5604: 0x00cc, 0x5605: 0x00cc, + 0x5606: 0x00cc, 0x5607: 0x00cc, 0x5608: 0x00cc, 0x5609: 0x00cc, 0x560a: 0x00cc, 0x560b: 0x00cc, + 0x560c: 0x00cc, 0x560d: 0x00cc, 0x560e: 0x00cc, 0x560f: 0x00cc, 0x5610: 0x00cc, 0x5611: 0x00cc, + 0x5612: 0x00cc, 0x5613: 0x00cc, 0x5614: 0x00cc, 0x5615: 0x00cc, 0x5616: 0x00cc, 0x5617: 0x00cc, + 0x5618: 0x00cc, 0x5619: 0x00cc, 0x561a: 0x00cc, 0x561b: 0x00cc, 0x561c: 0x00cc, 0x561d: 0x00cc, + 0x561e: 0x00cc, 0x561f: 0x00cc, 0x5620: 0x00cc, 0x5621: 0x00cc, 0x5622: 0x00cc, 0x5623: 0x00cc, + 0x5624: 0x00cc, 0x5625: 0x00cc, 0x5626: 0x00cc, 0x5627: 0x00cc, 0x5628: 0x00cc, 0x5629: 0x00cc, + 0x562a: 0x00cc, 0x562b: 0x00cc, 0x562c: 0x00cc, 0x562d: 0x00cc, 0x562e: 0x00cc, 0x562f: 0x00cc, + 0x5630: 0x00cc, 0x5631: 0x00cc, 0x5632: 0x00cc, 0x5633: 0x00cc, 0x5634: 0x00cc, + // Block 0x159, offset 0x5640 + 0x5640: 0x00cc, 0x5641: 0x00cc, 0x5642: 0x00cc, 0x5643: 0x00cc, 0x5644: 0x00cc, 0x5645: 0x00cc, + 0x5646: 0x00cc, 0x5647: 0x00cc, 0x5648: 0x00cc, 0x5649: 0x00cc, 0x564a: 0x00cc, 0x564b: 0x00cc, + 0x564c: 0x00cc, 0x564d: 0x00cc, 0x564e: 0x00cc, 0x564f: 0x00cc, 0x5650: 0x00cc, 0x5651: 0x00cc, + 0x5652: 0x00cc, 0x5653: 0x00cc, 0x5654: 0x00cc, 0x5655: 0x00cc, 0x5656: 0x00cc, 0x5657: 0x00cc, + 0x5658: 0x00cc, 0x5659: 0x00cc, 0x565a: 0x00cc, 0x565b: 0x00cc, 0x565c: 0x00cc, 0x565d: 0x00cc, + 0x5660: 0x00cc, 0x5661: 0x00cc, 0x5662: 0x00cc, 0x5663: 0x00cc, + 0x5664: 0x00cc, 0x5665: 0x00cc, 0x5666: 0x00cc, 0x5667: 0x00cc, 0x5668: 0x00cc, 0x5669: 0x00cc, + 0x566a: 0x00cc, 0x566b: 0x00cc, 0x566c: 0x00cc, 0x566d: 0x00cc, 0x566e: 0x00cc, 0x566f: 0x00cc, + 0x5670: 0x00cc, 0x5671: 0x00cc, 0x5672: 0x00cc, 0x5673: 0x00cc, 0x5674: 0x00cc, 0x5675: 0x00cc, + 0x5676: 0x00cc, 0x5677: 0x00cc, 0x5678: 0x00cc, 0x5679: 0x00cc, 0x567a: 0x00cc, 0x567b: 0x00cc, + 0x567c: 0x00cc, 0x567d: 0x00cc, 0x567e: 0x00cc, 0x567f: 0x00cc, + // Block 0x15a, offset 0x5680 + 0x5680: 0x00cc, 0x5681: 0x00cc, 0x5682: 0x00cc, 0x5683: 0x00cc, 0x5684: 0x00cc, 0x5685: 0x00cc, + 0x5686: 0x00cc, 0x5687: 0x00cc, 0x5688: 0x00cc, 0x5689: 0x00cc, 0x568a: 0x00cc, 0x568b: 0x00cc, + 0x568c: 0x00cc, 0x568d: 0x00cc, 0x568e: 0x00cc, 0x568f: 0x00cc, 0x5690: 0x00cc, 0x5691: 0x00cc, + 0x5692: 0x00cc, 0x5693: 0x00cc, 0x5694: 0x00cc, 0x5695: 0x00cc, 0x5696: 0x00cc, 0x5697: 0x00cc, + 0x5698: 0x00cc, 0x5699: 0x00cc, 0x569a: 0x00cc, 0x569b: 0x00cc, 0x569c: 0x00cc, 0x569d: 0x00cc, + 0x569e: 0x00cc, 0x569f: 0x00cc, 0x56a0: 0x00cc, 0x56a1: 0x00cc, + 0x56b0: 0x00cc, 0x56b1: 0x00cc, 0x56b2: 0x00cc, 0x56b3: 0x00cc, 0x56b4: 0x00cc, 0x56b5: 0x00cc, + 0x56b6: 0x00cc, 0x56b7: 0x00cc, 0x56b8: 0x00cc, 0x56b9: 0x00cc, 0x56ba: 0x00cc, 0x56bb: 0x00cc, + 0x56bc: 0x00cc, 0x56bd: 0x00cc, 0x56be: 0x00cc, 0x56bf: 0x00cc, + // Block 0x15b, offset 0x56c0 + 0x56c0: 0x00cc, 0x56c1: 0x00cc, 0x56c2: 0x00cc, 0x56c3: 0x00cc, 0x56c4: 0x00cc, 0x56c5: 0x00cc, + 0x56c6: 0x00cc, 0x56c7: 0x00cc, 0x56c8: 0x00cc, 0x56c9: 0x00cc, 0x56ca: 0x00cc, 0x56cb: 0x00cc, + 0x56cc: 0x00cc, 0x56cd: 0x00cc, 0x56ce: 0x00cc, 0x56cf: 0x00cc, 0x56d0: 0x00cc, 0x56d1: 0x00cc, + 0x56d2: 0x00cc, 0x56d3: 0x00cc, 0x56d4: 0x00cc, 0x56d5: 0x00cc, 0x56d6: 0x00cc, 0x56d7: 0x00cc, + 0x56d8: 0x00cc, 0x56d9: 0x00cc, 0x56da: 0x00cc, 0x56db: 0x00cc, 0x56dc: 0x00cc, 0x56dd: 0x00cc, + 0x56de: 0x00cc, 0x56df: 0x00cc, 0x56e0: 0x00cc, + // Block 0x15c, offset 0x5700 + 0x5700: 0x008c, 0x5701: 0x008c, 0x5702: 0x008c, 0x5703: 0x008c, 0x5704: 0x008c, 0x5705: 0x008c, + 0x5706: 0x008c, 0x5707: 0x008c, 0x5708: 0x008c, 0x5709: 0x008c, 0x570a: 0x008c, 0x570b: 0x008c, + 0x570c: 0x008c, 0x570d: 0x008c, 0x570e: 0x008c, 0x570f: 0x008c, 0x5710: 0x008c, 0x5711: 0x008c, + 0x5712: 0x008c, 0x5713: 0x008c, 0x5714: 0x008c, 0x5715: 0x008c, 0x5716: 0x008c, 0x5717: 0x008c, + 0x5718: 0x008c, 0x5719: 0x008c, 0x571a: 0x008c, 0x571b: 0x008c, 0x571c: 0x008c, 0x571d: 0x008c, + // Block 0x15d, offset 0x5740 + 0x5741: 0x0040, + 0x5760: 0x0040, 0x5761: 0x0040, 0x5762: 0x0040, 0x5763: 0x0040, + 0x5764: 0x0040, 0x5765: 0x0040, 0x5766: 0x0040, 0x5767: 0x0040, 0x5768: 0x0040, 0x5769: 0x0040, + 0x576a: 0x0040, 0x576b: 0x0040, 0x576c: 0x0040, 0x576d: 0x0040, 0x576e: 0x0040, 0x576f: 0x0040, + 0x5770: 0x0040, 0x5771: 0x0040, 0x5772: 0x0040, 0x5773: 0x0040, 0x5774: 0x0040, 0x5775: 0x0040, + 0x5776: 0x0040, 0x5777: 0x0040, 0x5778: 0x0040, 0x5779: 0x0040, 0x577a: 0x0040, 0x577b: 0x0040, + 0x577c: 0x0040, 0x577d: 0x0040, 0x577e: 0x0040, 0x577f: 0x0040, + // Block 0x15e, offset 0x5780 + 0x5780: 0x0040, 0x5781: 0x0040, 0x5782: 0x0040, 0x5783: 0x0040, 0x5784: 0x0040, 0x5785: 0x0040, + 0x5786: 0x0040, 0x5787: 0x0040, 0x5788: 0x0040, 0x5789: 0x0040, 0x578a: 0x0040, 0x578b: 0x0040, + 0x578c: 0x0040, 0x578d: 0x0040, 0x578e: 0x0040, 0x578f: 0x0040, 0x5790: 0x0040, 0x5791: 0x0040, + 0x5792: 0x0040, 0x5793: 0x0040, 0x5794: 0x0040, 0x5795: 0x0040, 0x5796: 0x0040, 0x5797: 0x0040, + 0x5798: 0x0040, 0x5799: 0x0040, 0x579a: 0x0040, 0x579b: 0x0040, 0x579c: 0x0040, 0x579d: 0x0040, + 0x579e: 0x0040, 0x579f: 0x0040, 0x57a0: 0x0040, 0x57a1: 0x0040, 0x57a2: 0x0040, 0x57a3: 0x0040, + 0x57a4: 0x0040, 0x57a5: 0x0040, 0x57a6: 0x0040, 0x57a7: 0x0040, 0x57a8: 0x0040, 0x57a9: 0x0040, + 0x57aa: 0x0040, 0x57ab: 0x0040, 0x57ac: 0x0040, 0x57ad: 0x0040, 0x57ae: 0x0040, 0x57af: 0x0040, + // Block 0x15f, offset 0x57c0 + 0x57c0: 0x0040, 0x57c1: 0x0040, 0x57c2: 0x0040, 0x57c3: 0x0040, 0x57c4: 0x0040, 0x57c5: 0x0040, + 0x57c6: 0x0040, 0x57c7: 0x0040, 0x57c8: 0x0040, 0x57c9: 0x0040, 0x57ca: 0x0040, 0x57cb: 0x0040, + 0x57cc: 0x0040, 0x57cd: 0x0040, 0x57ce: 0x0040, 0x57cf: 0x0040, 0x57d0: 0x0040, 0x57d1: 0x0040, + 0x57d2: 0x0040, 0x57d3: 0x0040, 0x57d4: 0x0040, 0x57d5: 0x0040, 0x57d6: 0x0040, 0x57d7: 0x0040, + 0x57d8: 0x0040, 0x57d9: 0x0040, 0x57da: 0x0040, 0x57db: 0x0040, 0x57dc: 0x0040, 0x57dd: 0x0040, + 0x57de: 0x0040, 0x57df: 0x0040, 0x57e0: 0x0040, 0x57e1: 0x0040, 0x57e2: 0x0040, 0x57e3: 0x0040, + 0x57e4: 0x0040, 0x57e5: 0x0040, 0x57e6: 0x0040, 0x57e7: 0x0040, 0x57e8: 0x0040, 0x57e9: 0x0040, + 0x57ea: 0x0040, 0x57eb: 0x0040, 0x57ec: 0x0040, 0x57ed: 0x0040, 0x57ee: 0x0040, 0x57ef: 0x0040, + 0x57f0: 0x0040, 0x57f1: 0x0040, 0x57f2: 0x0040, 0x57f3: 0x0040, 0x57f4: 0x0040, 0x57f5: 0x0040, + 0x57f6: 0x0040, 0x57f7: 0x0040, 0x57f8: 0x0040, 0x57f9: 0x0040, 0x57fa: 0x0040, 0x57fb: 0x0040, + 0x57fc: 0x0040, 0x57fd: 0x0040, +} + +// derivedPropertiesIndex: 37 blocks, 2368 entries, 4736 bytes +// Block 0 is the zero block. +var derivedPropertiesIndex = [2368]uint16{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xc2: 0x01, 0xc3: 0x02, 0xc4: 0x03, 0xc5: 0x04, 0xc6: 0x05, 0xc7: 0x06, + 0xc8: 0x05, 0xc9: 0x05, 0xca: 0x07, 0xcb: 0x08, 0xcc: 0x09, 0xcd: 0x0a, 0xce: 0x0b, 0xcf: 0x0c, + 0xd0: 0x05, 0xd1: 0x05, 0xd2: 0x0d, 0xd3: 0x05, 0xd4: 0x0e, 0xd5: 0x0f, 0xd6: 0x10, 0xd7: 0x11, + 0xd8: 0x12, 0xd9: 0x13, 0xda: 0x14, 0xdb: 0x15, 0xdc: 0x16, 0xdd: 0x17, 0xde: 0x18, 0xdf: 0x19, + 0xe0: 0x02, 0xe1: 0x03, 0xe2: 0x04, 0xe3: 0x05, 0xe4: 0x06, 0xe5: 0x07, 0xe6: 0x07, 0xe7: 0x07, + 0xe8: 0x07, 0xe9: 0x08, 0xea: 0x09, 0xeb: 0x0a, 0xec: 0x0a, 0xed: 0x0b, 0xee: 0x0c, 0xef: 0x0d, + 0xf0: 0x1e, 0xf3: 0x21, 0xf4: 0x22, + // Block 0x4, offset 0x100 + 0x120: 0x1a, 0x121: 0x1b, 0x122: 0x1c, 0x123: 0x1d, 0x124: 0x1e, 0x125: 0x1f, 0x126: 0x20, 0x127: 0x21, + 0x128: 0x22, 0x129: 0x23, 0x12a: 0x24, 0x12b: 0x25, 0x12c: 0x26, 0x12d: 0x27, 0x12e: 0x28, 0x12f: 0x29, + 0x130: 0x2a, 0x131: 0x2b, 0x132: 0x2c, 0x133: 0x2d, 0x134: 0x2e, 0x135: 0x2f, 0x136: 0x30, 0x137: 0x31, + 0x138: 0x32, 0x139: 0x33, 0x13a: 0x34, 0x13b: 0x35, 0x13c: 0x36, 0x13d: 0x37, 0x13e: 0x38, 0x13f: 0x39, + // Block 0x5, offset 0x140 + 0x140: 0x3a, 0x141: 0x3b, 0x142: 0x3c, 0x143: 0x3d, 0x144: 0x3e, 0x145: 0x3e, 0x146: 0x3e, 0x147: 0x3e, + 0x148: 0x05, 0x149: 0x3f, 0x14a: 0x40, 0x14b: 0x41, 0x14c: 0x42, 0x14d: 0x43, 0x14e: 0x44, 0x14f: 0x45, + 0x150: 0x46, 0x151: 0x05, 0x152: 0x05, 0x153: 0x05, 0x154: 0x05, 0x155: 0x05, 0x156: 0x05, 0x157: 0x05, + 0x158: 0x05, 0x159: 0x47, 0x15a: 0x48, 0x15b: 0x49, 0x15c: 0x4a, 0x15d: 0x4b, 0x15e: 0x4c, 0x15f: 0x4d, + 0x160: 0x4e, 0x161: 0x4f, 0x162: 0x50, 0x163: 0x51, 0x164: 0x52, 0x165: 0x53, 0x166: 0x54, 0x167: 0x55, + 0x168: 0x56, 0x169: 0x57, 0x16a: 0x58, 0x16c: 0x59, 0x16d: 0x5a, 0x16e: 0x5b, 0x16f: 0x5c, + 0x170: 0x5d, 0x171: 0x5e, 0x172: 0x5f, 0x173: 0x60, 0x174: 0x61, 0x175: 0x62, 0x176: 0x63, 0x177: 0x64, + 0x178: 0x05, 0x179: 0x05, 0x17a: 0x65, 0x17b: 0x05, 0x17c: 0x66, 0x17d: 0x67, 0x17e: 0x68, 0x17f: 0x69, + // Block 0x6, offset 0x180 + 0x180: 0x6a, 0x181: 0x6b, 0x182: 0x6c, 0x183: 0x6d, 0x184: 0x6e, 0x185: 0x6f, 0x186: 0x70, 0x187: 0x71, + 0x188: 0x71, 0x189: 0x71, 0x18a: 0x71, 0x18b: 0x71, 0x18c: 0x71, 0x18d: 0x71, 0x18e: 0x71, 0x18f: 0x71, + 0x190: 0x72, 0x191: 0x73, 0x192: 0x71, 0x193: 0x71, 0x194: 0x71, 0x195: 0x71, 0x196: 0x71, 0x197: 0x71, + 0x198: 0x71, 0x199: 0x71, 0x19a: 0x71, 0x19b: 0x71, 0x19c: 0x71, 0x19d: 0x71, 0x19e: 0x71, 0x19f: 0x71, + 0x1a0: 0x71, 0x1a1: 0x71, 0x1a2: 0x71, 0x1a3: 0x71, 0x1a4: 0x71, 0x1a5: 0x71, 0x1a6: 0x71, 0x1a7: 0x71, + 0x1a8: 0x71, 0x1a9: 0x71, 0x1aa: 0x71, 0x1ab: 0x71, 0x1ac: 0x71, 0x1ad: 0x74, 0x1ae: 0x75, 0x1af: 0x71, + 0x1b0: 0x76, 0x1b1: 0x77, 0x1b2: 0x05, 0x1b3: 0x78, 0x1b4: 0x79, 0x1b5: 0x7a, 0x1b6: 0x7b, 0x1b7: 0x7c, + 0x1b8: 0x7d, 0x1b9: 0x7e, 0x1ba: 0x7f, 0x1bb: 0x80, 0x1bc: 0x81, 0x1bd: 0x81, 0x1be: 0x81, 0x1bf: 0x82, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x83, 0x1c1: 0x84, 0x1c2: 0x85, 0x1c3: 0x86, 0x1c4: 0x87, 0x1c5: 0x88, 0x1c6: 0x89, 0x1c7: 0x8a, + 0x1c8: 0x8b, 0x1c9: 0x71, 0x1ca: 0x71, 0x1cb: 0x8c, 0x1cc: 0x81, 0x1cd: 0x8d, 0x1ce: 0x71, 0x1cf: 0x71, + 0x1d0: 0x8e, 0x1d1: 0x8e, 0x1d2: 0x8e, 0x1d3: 0x8e, 0x1d4: 0x8e, 0x1d5: 0x8e, 0x1d6: 0x8e, 0x1d7: 0x8e, + 0x1d8: 0x8e, 0x1d9: 0x8e, 0x1da: 0x8e, 0x1db: 0x8e, 0x1dc: 0x8e, 0x1dd: 0x8e, 0x1de: 0x8e, 0x1df: 0x8e, + 0x1e0: 0x8e, 0x1e1: 0x8e, 0x1e2: 0x8e, 0x1e3: 0x8e, 0x1e4: 0x8e, 0x1e5: 0x8e, 0x1e6: 0x8e, 0x1e7: 0x8e, + 0x1e8: 0x8e, 0x1e9: 0x8e, 0x1ea: 0x8e, 0x1eb: 0x8e, 0x1ec: 0x8e, 0x1ed: 0x8e, 0x1ee: 0x8e, 0x1ef: 0x8e, + 0x1f0: 0x8e, 0x1f1: 0x8e, 0x1f2: 0x8e, 0x1f3: 0x8e, 0x1f4: 0x8e, 0x1f5: 0x8e, 0x1f6: 0x8e, 0x1f7: 0x8e, + 0x1f8: 0x8e, 0x1f9: 0x8e, 0x1fa: 0x8e, 0x1fb: 0x8e, 0x1fc: 0x8e, 0x1fd: 0x8e, 0x1fe: 0x8e, 0x1ff: 0x8e, + // Block 0x8, offset 0x200 + 0x200: 0x8e, 0x201: 0x8e, 0x202: 0x8e, 0x203: 0x8e, 0x204: 0x8e, 0x205: 0x8e, 0x206: 0x8e, 0x207: 0x8e, + 0x208: 0x8e, 0x209: 0x8e, 0x20a: 0x8e, 0x20b: 0x8e, 0x20c: 0x8e, 0x20d: 0x8e, 0x20e: 0x8e, 0x20f: 0x8e, + 0x210: 0x8e, 0x211: 0x8e, 0x212: 0x8e, 0x213: 0x8e, 0x214: 0x8e, 0x215: 0x8e, 0x216: 0x8e, 0x217: 0x8e, + 0x218: 0x8e, 0x219: 0x8e, 0x21a: 0x8e, 0x21b: 0x8e, 0x21c: 0x8e, 0x21d: 0x8e, 0x21e: 0x8e, 0x21f: 0x8e, + 0x220: 0x8e, 0x221: 0x8e, 0x222: 0x8e, 0x223: 0x8e, 0x224: 0x8e, 0x225: 0x8e, 0x226: 0x8e, 0x227: 0x8e, + 0x228: 0x8e, 0x229: 0x8e, 0x22a: 0x8e, 0x22b: 0x8e, 0x22c: 0x8e, 0x22d: 0x8e, 0x22e: 0x8e, 0x22f: 0x8e, + 0x230: 0x8e, 0x231: 0x8e, 0x232: 0x8e, 0x233: 0x8e, 0x234: 0x8e, 0x235: 0x8e, 0x236: 0x8f, 0x237: 0x71, + 0x238: 0x8e, 0x239: 0x8e, 0x23a: 0x8e, 0x23b: 0x8e, 0x23c: 0x8e, 0x23d: 0x8e, 0x23e: 0x8e, 0x23f: 0x8e, + // Block 0x9, offset 0x240 + 0x240: 0x8e, 0x241: 0x8e, 0x242: 0x8e, 0x243: 0x8e, 0x244: 0x8e, 0x245: 0x8e, 0x246: 0x8e, 0x247: 0x8e, + 0x248: 0x8e, 0x249: 0x8e, 0x24a: 0x8e, 0x24b: 0x8e, 0x24c: 0x8e, 0x24d: 0x8e, 0x24e: 0x8e, 0x24f: 0x8e, + 0x250: 0x8e, 0x251: 0x8e, 0x252: 0x8e, 0x253: 0x8e, 0x254: 0x8e, 0x255: 0x8e, 0x256: 0x8e, 0x257: 0x8e, + 0x258: 0x8e, 0x259: 0x8e, 0x25a: 0x8e, 0x25b: 0x8e, 0x25c: 0x8e, 0x25d: 0x8e, 0x25e: 0x8e, 0x25f: 0x8e, + 0x260: 0x8e, 0x261: 0x8e, 0x262: 0x8e, 0x263: 0x8e, 0x264: 0x8e, 0x265: 0x8e, 0x266: 0x8e, 0x267: 0x8e, + 0x268: 0x8e, 0x269: 0x8e, 0x26a: 0x8e, 0x26b: 0x8e, 0x26c: 0x8e, 0x26d: 0x8e, 0x26e: 0x8e, 0x26f: 0x8e, + 0x270: 0x8e, 0x271: 0x8e, 0x272: 0x8e, 0x273: 0x8e, 0x274: 0x8e, 0x275: 0x8e, 0x276: 0x8e, 0x277: 0x8e, + 0x278: 0x8e, 0x279: 0x8e, 0x27a: 0x8e, 0x27b: 0x8e, 0x27c: 0x8e, 0x27d: 0x8e, 0x27e: 0x8e, 0x27f: 0x8e, + // Block 0xa, offset 0x280 + 0x280: 0x8e, 0x281: 0x8e, 0x282: 0x8e, 0x283: 0x8e, 0x284: 0x8e, 0x285: 0x8e, 0x286: 0x8e, 0x287: 0x8e, + 0x288: 0x8e, 0x289: 0x8e, 0x28a: 0x8e, 0x28b: 0x8e, 0x28c: 0x8e, 0x28d: 0x8e, 0x28e: 0x8e, 0x28f: 0x8e, + 0x290: 0x8e, 0x291: 0x8e, 0x292: 0x8e, 0x293: 0x8e, 0x294: 0x8e, 0x295: 0x8e, 0x296: 0x8e, 0x297: 0x8e, + 0x298: 0x8e, 0x299: 0x8e, 0x29a: 0x8e, 0x29b: 0x8e, 0x29c: 0x8e, 0x29d: 0x8e, 0x29e: 0x8e, 0x29f: 0x8e, + 0x2a0: 0x8e, 0x2a1: 0x8e, 0x2a2: 0x8e, 0x2a3: 0x8e, 0x2a4: 0x8e, 0x2a5: 0x8e, 0x2a6: 0x8e, 0x2a7: 0x8e, + 0x2a8: 0x8e, 0x2a9: 0x8e, 0x2aa: 0x8e, 0x2ab: 0x8e, 0x2ac: 0x8e, 0x2ad: 0x8e, 0x2ae: 0x8e, 0x2af: 0x8e, + 0x2b0: 0x8e, 0x2b1: 0x8e, 0x2b2: 0x8e, 0x2b3: 0x8e, 0x2b4: 0x8e, 0x2b5: 0x8e, 0x2b6: 0x8e, 0x2b7: 0x8e, + 0x2b8: 0x8e, 0x2b9: 0x8e, 0x2ba: 0x8e, 0x2bb: 0x8e, 0x2bc: 0x8e, 0x2bd: 0x8e, 0x2be: 0x8e, 0x2bf: 0x90, + // Block 0xb, offset 0x2c0 + 0x2c0: 0x05, 0x2c1: 0x05, 0x2c2: 0x05, 0x2c3: 0x05, 0x2c4: 0x05, 0x2c5: 0x05, 0x2c6: 0x05, 0x2c7: 0x05, + 0x2c8: 0x05, 0x2c9: 0x05, 0x2ca: 0x05, 0x2cb: 0x05, 0x2cc: 0x05, 0x2cd: 0x05, 0x2ce: 0x05, 0x2cf: 0x05, + 0x2d0: 0x05, 0x2d1: 0x05, 0x2d2: 0x91, 0x2d3: 0x92, 0x2d4: 0x05, 0x2d5: 0x05, 0x2d6: 0x05, 0x2d7: 0x05, + 0x2d8: 0x93, 0x2d9: 0x94, 0x2da: 0x95, 0x2db: 0x96, 0x2dc: 0x97, 0x2dd: 0x98, 0x2de: 0x99, 0x2df: 0x9a, + 0x2e0: 0x9b, 0x2e1: 0x9c, 0x2e2: 0x05, 0x2e3: 0x9d, 0x2e4: 0x9e, 0x2e5: 0x9f, 0x2e6: 0xa0, 0x2e7: 0xa1, + 0x2e8: 0xa2, 0x2e9: 0xa3, 0x2ea: 0xa4, 0x2eb: 0xa5, 0x2ec: 0xa6, 0x2ed: 0xa7, 0x2ee: 0x05, 0x2ef: 0xa8, + 0x2f0: 0x05, 0x2f1: 0x05, 0x2f2: 0x05, 0x2f3: 0x05, 0x2f4: 0x05, 0x2f5: 0x05, 0x2f6: 0x05, 0x2f7: 0x05, + 0x2f8: 0x05, 0x2f9: 0x05, 0x2fa: 0x05, 0x2fb: 0x05, 0x2fc: 0x05, 0x2fd: 0x05, 0x2fe: 0x05, 0x2ff: 0x05, + // Block 0xc, offset 0x300 + 0x300: 0x05, 0x301: 0x05, 0x302: 0x05, 0x303: 0x05, 0x304: 0x05, 0x305: 0x05, 0x306: 0x05, 0x307: 0x05, + 0x308: 0x05, 0x309: 0x05, 0x30a: 0x05, 0x30b: 0x05, 0x30c: 0x05, 0x30d: 0x05, 0x30e: 0x05, 0x30f: 0x05, + 0x310: 0x05, 0x311: 0x05, 0x312: 0x05, 0x313: 0x05, 0x314: 0x05, 0x315: 0x05, 0x316: 0x05, 0x317: 0x05, + 0x318: 0x05, 0x319: 0x05, 0x31a: 0x05, 0x31b: 0x05, 0x31c: 0x05, 0x31d: 0x05, 0x31e: 0x05, 0x31f: 0x05, + 0x320: 0x05, 0x321: 0x05, 0x322: 0x05, 0x323: 0x05, 0x324: 0x05, 0x325: 0x05, 0x326: 0x05, 0x327: 0x05, + 0x328: 0x05, 0x329: 0x05, 0x32a: 0x05, 0x32b: 0x05, 0x32c: 0x05, 0x32d: 0x05, 0x32e: 0x05, 0x32f: 0x05, + 0x330: 0x05, 0x331: 0x05, 0x332: 0x05, 0x333: 0x05, 0x334: 0x05, 0x335: 0x05, 0x336: 0x05, 0x337: 0x05, + 0x338: 0x05, 0x339: 0x05, 0x33a: 0x05, 0x33b: 0x05, 0x33c: 0x05, 0x33d: 0x05, 0x33e: 0x05, 0x33f: 0x05, + // Block 0xd, offset 0x340 + 0x340: 0x05, 0x341: 0x05, 0x342: 0x05, 0x343: 0x05, 0x344: 0x05, 0x345: 0x05, 0x346: 0x05, 0x347: 0x05, + 0x348: 0x05, 0x349: 0x05, 0x34a: 0x05, 0x34b: 0x05, 0x34c: 0x05, 0x34d: 0x05, 0x34e: 0x05, 0x34f: 0x05, + 0x350: 0x05, 0x351: 0x05, 0x352: 0x05, 0x353: 0x05, 0x354: 0x05, 0x355: 0x05, 0x356: 0x05, 0x357: 0x05, + 0x358: 0x05, 0x359: 0x05, 0x35a: 0x05, 0x35b: 0x05, 0x35c: 0x05, 0x35d: 0x05, 0x35e: 0xa9, 0x35f: 0xaa, + // Block 0xe, offset 0x380 + 0x380: 0x3e, 0x381: 0x3e, 0x382: 0x3e, 0x383: 0x3e, 0x384: 0x3e, 0x385: 0x3e, 0x386: 0x3e, 0x387: 0x3e, + 0x388: 0x3e, 0x389: 0x3e, 0x38a: 0x3e, 0x38b: 0x3e, 0x38c: 0x3e, 0x38d: 0x3e, 0x38e: 0x3e, 0x38f: 0x3e, + 0x390: 0x3e, 0x391: 0x3e, 0x392: 0x3e, 0x393: 0x3e, 0x394: 0x3e, 0x395: 0x3e, 0x396: 0x3e, 0x397: 0x3e, + 0x398: 0x3e, 0x399: 0x3e, 0x39a: 0x3e, 0x39b: 0x3e, 0x39c: 0x3e, 0x39d: 0x3e, 0x39e: 0x3e, 0x39f: 0x3e, + 0x3a0: 0x3e, 0x3a1: 0x3e, 0x3a2: 0x3e, 0x3a3: 0x3e, 0x3a4: 0x3e, 0x3a5: 0x3e, 0x3a6: 0x3e, 0x3a7: 0x3e, + 0x3a8: 0x3e, 0x3a9: 0x3e, 0x3aa: 0x3e, 0x3ab: 0x3e, 0x3ac: 0x3e, 0x3ad: 0x3e, 0x3ae: 0x3e, 0x3af: 0x3e, + 0x3b0: 0x3e, 0x3b1: 0x3e, 0x3b2: 0x3e, 0x3b3: 0x3e, 0x3b4: 0x3e, 0x3b5: 0x3e, 0x3b6: 0x3e, 0x3b7: 0x3e, + 0x3b8: 0x3e, 0x3b9: 0x3e, 0x3ba: 0x3e, 0x3bb: 0x3e, 0x3bc: 0x3e, 0x3bd: 0x3e, 0x3be: 0x3e, 0x3bf: 0x3e, + // Block 0xf, offset 0x3c0 + 0x3c0: 0x3e, 0x3c1: 0x3e, 0x3c2: 0x3e, 0x3c3: 0x3e, 0x3c4: 0x3e, 0x3c5: 0x3e, 0x3c6: 0x3e, 0x3c7: 0x3e, + 0x3c8: 0x3e, 0x3c9: 0x3e, 0x3ca: 0x3e, 0x3cb: 0x3e, 0x3cc: 0x3e, 0x3cd: 0x3e, 0x3ce: 0x3e, 0x3cf: 0x3e, + 0x3d0: 0x3e, 0x3d1: 0x3e, 0x3d2: 0x3e, 0x3d3: 0x3e, 0x3d4: 0x3e, 0x3d5: 0x3e, 0x3d6: 0x3e, 0x3d7: 0x3e, + 0x3d8: 0x3e, 0x3d9: 0x3e, 0x3da: 0x3e, 0x3db: 0x3e, 0x3dc: 0x3e, 0x3dd: 0x3e, 0x3de: 0x3e, 0x3df: 0x3e, + 0x3e0: 0x3e, 0x3e1: 0x3e, 0x3e2: 0x3e, 0x3e3: 0x3e, 0x3e4: 0x81, 0x3e5: 0x81, 0x3e6: 0x81, 0x3e7: 0x81, + 0x3e8: 0xab, 0x3e9: 0xac, 0x3ea: 0x81, 0x3eb: 0xad, 0x3ec: 0xae, 0x3ed: 0xaf, 0x3ee: 0x71, 0x3ef: 0xb0, + 0x3f0: 0x71, 0x3f1: 0x71, 0x3f2: 0x71, 0x3f3: 0x71, 0x3f4: 0x71, 0x3f5: 0xb1, 0x3f6: 0xb2, 0x3f7: 0xb3, + 0x3f8: 0xb4, 0x3f9: 0xb5, 0x3fa: 0x71, 0x3fb: 0xb6, 0x3fc: 0xb7, 0x3fd: 0xb8, 0x3fe: 0xb9, 0x3ff: 0xba, + // Block 0x10, offset 0x400 + 0x400: 0xbb, 0x401: 0xbc, 0x402: 0x05, 0x403: 0xbd, 0x404: 0xbe, 0x405: 0xbf, 0x406: 0xc0, 0x407: 0xc1, + 0x40a: 0xc2, 0x40b: 0xc3, 0x40c: 0xc4, 0x40d: 0xc5, 0x40e: 0xc6, 0x40f: 0xc7, + 0x410: 0x05, 0x411: 0x05, 0x412: 0xc8, 0x413: 0xc9, 0x414: 0xca, 0x415: 0xcb, + 0x418: 0x05, 0x419: 0x05, 0x41a: 0x05, 0x41b: 0x05, 0x41c: 0xcc, 0x41d: 0xcd, + 0x420: 0xce, 0x421: 0xcf, 0x422: 0xd0, 0x423: 0xd1, 0x424: 0xd2, 0x426: 0xd3, 0x427: 0xb2, + 0x428: 0xd4, 0x429: 0xd5, 0x42a: 0xd6, 0x42b: 0xd7, 0x42c: 0xd8, 0x42d: 0xd9, 0x42e: 0xda, + 0x430: 0x05, 0x431: 0xdb, 0x432: 0xdc, 0x433: 0xdd, 0x434: 0xde, + 0x439: 0xdf, 0x43c: 0xe0, 0x43d: 0xe1, 0x43f: 0xe2, + // Block 0x11, offset 0x440 + 0x440: 0xe3, 0x441: 0xe4, 0x442: 0xe5, 0x443: 0xe6, 0x444: 0xe7, 0x445: 0xe8, 0x446: 0xe9, 0x447: 0xea, + 0x448: 0xeb, 0x44a: 0xec, 0x44b: 0xed, 0x44c: 0xee, 0x44d: 0xef, + 0x450: 0xf0, 0x451: 0xf1, 0x452: 0xf2, 0x453: 0xf3, 0x456: 0xf4, 0x457: 0xf5, + 0x458: 0xf6, 0x459: 0xf7, 0x45a: 0xf8, 0x45b: 0xf9, 0x45c: 0xfa, + 0x460: 0xfb, 0x462: 0xfc, 0x463: 0xfd, 0x466: 0xfe, 0x467: 0xff, + 0x468: 0x100, 0x469: 0x101, 0x46a: 0x102, 0x46b: 0x103, + 0x470: 0x104, 0x471: 0x105, 0x472: 0x106, 0x474: 0x107, 0x475: 0x108, 0x476: 0x109, + 0x47b: 0x10a, 0x47f: 0x10b, + // Block 0x12, offset 0x480 + 0x480: 0x05, 0x481: 0x05, 0x482: 0x05, 0x483: 0x05, 0x484: 0x05, 0x485: 0x05, 0x486: 0x05, 0x487: 0x05, + 0x488: 0x05, 0x489: 0x05, 0x48a: 0x05, 0x48b: 0x05, 0x48c: 0x05, 0x48d: 0x05, 0x48e: 0x10c, + 0x490: 0x71, 0x491: 0x10d, 0x492: 0x05, 0x493: 0x05, 0x494: 0x05, 0x495: 0x10e, + // Block 0x13, offset 0x4c0 + 0x4c0: 0x05, 0x4c1: 0x05, 0x4c2: 0x05, 0x4c3: 0x05, 0x4c4: 0x05, 0x4c5: 0x05, 0x4c6: 0x05, 0x4c7: 0x05, + 0x4c8: 0x05, 0x4c9: 0x05, 0x4ca: 0x05, 0x4cb: 0x05, 0x4cc: 0x05, 0x4cd: 0x05, 0x4ce: 0x05, 0x4cf: 0x05, + 0x4d0: 0x10f, + // Block 0x14, offset 0x500 + 0x510: 0x05, 0x511: 0x05, 0x512: 0x05, 0x513: 0x05, 0x514: 0x05, 0x515: 0x05, 0x516: 0x05, 0x517: 0x05, + 0x518: 0x05, 0x519: 0x110, + // Block 0x15, offset 0x540 + 0x560: 0x05, 0x561: 0x05, 0x562: 0x05, 0x563: 0x05, 0x564: 0x05, 0x565: 0x05, 0x566: 0x05, 0x567: 0x05, + 0x568: 0x103, 0x569: 0x111, 0x56b: 0x112, 0x56c: 0x113, 0x56d: 0x114, 0x56e: 0x115, + 0x579: 0x05, 0x57a: 0x116, 0x57c: 0x05, 0x57d: 0x117, 0x57e: 0x118, 0x57f: 0x119, + // Block 0x16, offset 0x580 + 0x580: 0x05, 0x581: 0x05, 0x582: 0x05, 0x583: 0x05, 0x584: 0x05, 0x585: 0x05, 0x586: 0x05, 0x587: 0x05, + 0x588: 0x05, 0x589: 0x05, 0x58a: 0x05, 0x58b: 0x05, 0x58c: 0x05, 0x58d: 0x05, 0x58e: 0x05, 0x58f: 0x05, + 0x590: 0x05, 0x591: 0x05, 0x592: 0x05, 0x593: 0x05, 0x594: 0x05, 0x595: 0x05, 0x596: 0x05, 0x597: 0x05, + 0x598: 0x05, 0x599: 0x05, 0x59a: 0x05, 0x59b: 0x05, 0x59c: 0x05, 0x59d: 0x05, 0x59e: 0x05, 0x59f: 0x11a, + 0x5a0: 0x05, 0x5a1: 0x05, 0x5a2: 0x05, 0x5a3: 0x05, 0x5a4: 0x05, 0x5a5: 0x05, 0x5a6: 0x05, 0x5a7: 0x05, + 0x5a8: 0x05, 0x5a9: 0x05, 0x5aa: 0x05, 0x5ab: 0xdc, + // Block 0x17, offset 0x5c0 + 0x5c0: 0x8e, 0x5c1: 0x8e, 0x5c2: 0x8e, 0x5c3: 0x8e, 0x5c4: 0x11b, 0x5c5: 0x11c, 0x5c6: 0x05, 0x5c7: 0x05, + 0x5c8: 0x05, 0x5c9: 0x05, 0x5ca: 0x05, 0x5cb: 0x11d, + 0x5f0: 0x05, 0x5f1: 0x11e, 0x5f2: 0x11f, + // Block 0x18, offset 0x600 + 0x600: 0x71, 0x601: 0x71, 0x602: 0x71, 0x603: 0x120, 0x604: 0x121, 0x605: 0x122, 0x606: 0x123, 0x607: 0x124, + 0x608: 0xbf, 0x609: 0x125, 0x60b: 0x126, 0x60c: 0x71, 0x60d: 0x127, + 0x610: 0x71, 0x611: 0x128, 0x612: 0x129, 0x613: 0x12a, 0x614: 0x12b, 0x615: 0x12c, 0x616: 0x71, 0x617: 0x71, + 0x618: 0x71, 0x619: 0x71, 0x61a: 0x12d, 0x61b: 0x71, 0x61c: 0x71, 0x61d: 0x71, 0x61e: 0x71, 0x61f: 0x12e, + 0x620: 0x71, 0x621: 0x71, 0x622: 0x71, 0x623: 0x71, 0x624: 0x71, 0x625: 0x71, 0x626: 0x71, 0x627: 0x71, + 0x628: 0x12f, 0x629: 0x130, 0x62a: 0x131, + // Block 0x19, offset 0x640 + 0x640: 0x132, 0x644: 0x133, 0x645: 0x134, + 0x64b: 0x135, + 0x660: 0x05, 0x661: 0x05, 0x662: 0x05, 0x663: 0x136, 0x664: 0x137, 0x665: 0x138, + 0x671: 0x139, 0x672: 0x13a, 0x674: 0x13b, + 0x678: 0x13c, 0x679: 0x13d, 0x67a: 0x13e, 0x67b: 0x13f, + // Block 0x1a, offset 0x680 + 0x680: 0x140, 0x681: 0x71, 0x682: 0x141, 0x683: 0x142, 0x684: 0x143, 0x685: 0x144, 0x686: 0x145, 0x687: 0x146, + 0x688: 0x147, 0x689: 0x148, 0x68c: 0x71, 0x68d: 0x71, 0x68e: 0x71, 0x68f: 0x71, + 0x690: 0x71, 0x691: 0x71, 0x692: 0x71, 0x693: 0x71, 0x694: 0x71, 0x695: 0x71, 0x696: 0x71, 0x697: 0x71, + 0x698: 0x71, 0x699: 0x71, 0x69a: 0x71, 0x69b: 0x149, 0x69c: 0x71, 0x69d: 0x14a, 0x69e: 0x71, 0x69f: 0x14b, + 0x6a0: 0x14c, 0x6a1: 0x14d, 0x6a2: 0x14e, 0x6a4: 0x14f, 0x6a5: 0x150, 0x6a6: 0x151, 0x6a7: 0x152, + 0x6a8: 0x71, 0x6a9: 0x153, 0x6aa: 0x154, + // Block 0x1b, offset 0x6c0 + 0x6c0: 0x8e, 0x6c1: 0x8e, 0x6c2: 0x8e, 0x6c3: 0x8e, 0x6c4: 0x8e, 0x6c5: 0x8e, 0x6c6: 0x8e, 0x6c7: 0x8e, + 0x6c8: 0x8e, 0x6c9: 0x8e, 0x6ca: 0x8e, 0x6cb: 0x8e, 0x6cc: 0x8e, 0x6cd: 0x8e, 0x6ce: 0x8e, 0x6cf: 0x8e, + 0x6d0: 0x8e, 0x6d1: 0x8e, 0x6d2: 0x8e, 0x6d3: 0x8e, 0x6d4: 0x8e, 0x6d5: 0x8e, 0x6d6: 0x8e, 0x6d7: 0x8e, + 0x6d8: 0x8e, 0x6d9: 0x8e, 0x6da: 0x8e, 0x6db: 0x155, 0x6dc: 0x8e, 0x6dd: 0x8e, 0x6de: 0x8e, 0x6df: 0x8e, + 0x6e0: 0x8e, 0x6e1: 0x8e, 0x6e2: 0x8e, 0x6e3: 0x8e, 0x6e4: 0x8e, 0x6e5: 0x8e, 0x6e6: 0x8e, 0x6e7: 0x8e, + 0x6e8: 0x8e, 0x6e9: 0x8e, 0x6ea: 0x8e, 0x6eb: 0x8e, 0x6ec: 0x8e, 0x6ed: 0x8e, 0x6ee: 0x8e, 0x6ef: 0x8e, + 0x6f0: 0x8e, 0x6f1: 0x8e, 0x6f2: 0x8e, 0x6f3: 0x8e, 0x6f4: 0x8e, 0x6f5: 0x8e, 0x6f6: 0x8e, 0x6f7: 0x8e, + 0x6f8: 0x8e, 0x6f9: 0x8e, 0x6fa: 0x8e, 0x6fb: 0x8e, 0x6fc: 0x8e, 0x6fd: 0x8e, 0x6fe: 0x8e, 0x6ff: 0x8e, + // Block 0x1c, offset 0x700 + 0x700: 0x8e, 0x701: 0x8e, 0x702: 0x8e, 0x703: 0x8e, 0x704: 0x8e, 0x705: 0x8e, 0x706: 0x8e, 0x707: 0x8e, + 0x708: 0x8e, 0x709: 0x8e, 0x70a: 0x8e, 0x70b: 0x8e, 0x70c: 0x8e, 0x70d: 0x8e, 0x70e: 0x8e, 0x70f: 0x8e, + 0x710: 0x8e, 0x711: 0x8e, 0x712: 0x8e, 0x713: 0x8e, 0x714: 0x8e, 0x715: 0x8e, 0x716: 0x8e, 0x717: 0x8e, + 0x718: 0x8e, 0x719: 0x8e, 0x71a: 0x8e, 0x71b: 0x8e, 0x71c: 0x156, 0x71d: 0x8e, 0x71e: 0x8e, 0x71f: 0x8e, + 0x720: 0x157, 0x721: 0x8e, 0x722: 0x8e, 0x723: 0x8e, 0x724: 0x8e, 0x725: 0x8e, 0x726: 0x8e, 0x727: 0x8e, + 0x728: 0x8e, 0x729: 0x8e, 0x72a: 0x8e, 0x72b: 0x8e, 0x72c: 0x8e, 0x72d: 0x8e, 0x72e: 0x8e, 0x72f: 0x8e, + 0x730: 0x8e, 0x731: 0x8e, 0x732: 0x8e, 0x733: 0x8e, 0x734: 0x8e, 0x735: 0x8e, 0x736: 0x8e, 0x737: 0x8e, + 0x738: 0x8e, 0x739: 0x8e, 0x73a: 0x8e, 0x73b: 0x8e, 0x73c: 0x8e, 0x73d: 0x8e, 0x73e: 0x8e, 0x73f: 0x8e, + // Block 0x1d, offset 0x740 + 0x740: 0x8e, 0x741: 0x8e, 0x742: 0x8e, 0x743: 0x8e, 0x744: 0x8e, 0x745: 0x8e, 0x746: 0x8e, 0x747: 0x8e, + 0x748: 0x8e, 0x749: 0x8e, 0x74a: 0x8e, 0x74b: 0x8e, 0x74c: 0x8e, 0x74d: 0x8e, 0x74e: 0x8e, 0x74f: 0x8e, + 0x750: 0x8e, 0x751: 0x8e, 0x752: 0x8e, 0x753: 0x8e, 0x754: 0x8e, 0x755: 0x8e, 0x756: 0x8e, 0x757: 0x8e, + 0x758: 0x8e, 0x759: 0x8e, 0x75a: 0x8e, 0x75b: 0x8e, 0x75c: 0x8e, 0x75d: 0x8e, 0x75e: 0x8e, 0x75f: 0x8e, + 0x760: 0x8e, 0x761: 0x8e, 0x762: 0x8e, 0x763: 0x8e, 0x764: 0x8e, 0x765: 0x8e, 0x766: 0x8e, 0x767: 0x8e, + 0x768: 0x8e, 0x769: 0x8e, 0x76a: 0x8e, 0x76b: 0x8e, 0x76c: 0x8e, 0x76d: 0x8e, 0x76e: 0x8e, 0x76f: 0x8e, + 0x770: 0x8e, 0x771: 0x8e, 0x772: 0x8e, 0x773: 0x8e, 0x774: 0x8e, 0x775: 0x8e, 0x776: 0x8e, 0x777: 0x8e, + 0x778: 0x8e, 0x779: 0x8e, 0x77a: 0x158, 0x77b: 0x8e, 0x77c: 0x8e, 0x77d: 0x8e, 0x77e: 0x8e, 0x77f: 0x8e, + // Block 0x1e, offset 0x780 + 0x780: 0x8e, 0x781: 0x8e, 0x782: 0x8e, 0x783: 0x8e, 0x784: 0x8e, 0x785: 0x8e, 0x786: 0x8e, 0x787: 0x8e, + 0x788: 0x8e, 0x789: 0x8e, 0x78a: 0x8e, 0x78b: 0x8e, 0x78c: 0x8e, 0x78d: 0x8e, 0x78e: 0x8e, 0x78f: 0x8e, + 0x790: 0x8e, 0x791: 0x8e, 0x792: 0x8e, 0x793: 0x8e, 0x794: 0x8e, 0x795: 0x8e, 0x796: 0x8e, 0x797: 0x8e, + 0x798: 0x8e, 0x799: 0x8e, 0x79a: 0x8e, 0x79b: 0x8e, 0x79c: 0x8e, 0x79d: 0x8e, 0x79e: 0x8e, 0x79f: 0x8e, + 0x7a0: 0x8e, 0x7a1: 0x8e, 0x7a2: 0x8e, 0x7a3: 0x8e, 0x7a4: 0x8e, 0x7a5: 0x8e, 0x7a6: 0x8e, 0x7a7: 0x8e, + 0x7a8: 0x8e, 0x7a9: 0x8e, 0x7aa: 0x8e, 0x7ab: 0x8e, 0x7ac: 0x8e, 0x7ad: 0x8e, 0x7ae: 0x8e, 0x7af: 0x159, + // Block 0x1f, offset 0x7c0 + 0x7e0: 0x81, 0x7e1: 0x81, 0x7e2: 0x81, 0x7e3: 0x81, 0x7e4: 0x81, 0x7e5: 0x81, 0x7e6: 0x81, 0x7e7: 0x81, + 0x7e8: 0x15a, + // Block 0x20, offset 0x800 + 0x810: 0x0e, 0x811: 0x0f, 0x812: 0x10, 0x813: 0x11, 0x814: 0x12, 0x816: 0x13, 0x817: 0x0a, + 0x818: 0x14, 0x81b: 0x15, 0x81d: 0x16, 0x81e: 0x17, 0x81f: 0x18, + 0x820: 0x07, 0x821: 0x07, 0x822: 0x07, 0x823: 0x07, 0x824: 0x07, 0x825: 0x07, 0x826: 0x07, 0x827: 0x07, + 0x828: 0x07, 0x829: 0x07, 0x82a: 0x19, 0x82b: 0x1a, 0x82c: 0x1b, 0x82d: 0x07, 0x82e: 0x1c, 0x82f: 0x1d, + // Block 0x21, offset 0x840 + 0x840: 0x15b, 0x841: 0x3e, 0x844: 0x3e, 0x845: 0x3e, 0x846: 0x3e, 0x847: 0x15c, + // Block 0x22, offset 0x880 + 0x880: 0x3e, 0x881: 0x3e, 0x882: 0x3e, 0x883: 0x3e, 0x884: 0x3e, 0x885: 0x3e, 0x886: 0x3e, 0x887: 0x3e, + 0x888: 0x3e, 0x889: 0x3e, 0x88a: 0x3e, 0x88b: 0x3e, 0x88c: 0x3e, 0x88d: 0x3e, 0x88e: 0x3e, 0x88f: 0x3e, + 0x890: 0x3e, 0x891: 0x3e, 0x892: 0x3e, 0x893: 0x3e, 0x894: 0x3e, 0x895: 0x3e, 0x896: 0x3e, 0x897: 0x3e, + 0x898: 0x3e, 0x899: 0x3e, 0x89a: 0x3e, 0x89b: 0x3e, 0x89c: 0x3e, 0x89d: 0x3e, 0x89e: 0x3e, 0x89f: 0x3e, + 0x8a0: 0x3e, 0x8a1: 0x3e, 0x8a2: 0x3e, 0x8a3: 0x3e, 0x8a4: 0x3e, 0x8a5: 0x3e, 0x8a6: 0x3e, 0x8a7: 0x3e, + 0x8a8: 0x3e, 0x8a9: 0x3e, 0x8aa: 0x3e, 0x8ab: 0x3e, 0x8ac: 0x3e, 0x8ad: 0x3e, 0x8ae: 0x3e, 0x8af: 0x3e, + 0x8b0: 0x3e, 0x8b1: 0x3e, 0x8b2: 0x3e, 0x8b3: 0x3e, 0x8b4: 0x3e, 0x8b5: 0x3e, 0x8b6: 0x3e, 0x8b7: 0x3e, + 0x8b8: 0x3e, 0x8b9: 0x3e, 0x8ba: 0x3e, 0x8bb: 0x3e, 0x8bc: 0x3e, 0x8bd: 0x3e, 0x8be: 0x3e, 0x8bf: 0x15d, + // Block 0x23, offset 0x8c0 + 0x8e0: 0x1f, + 0x8f0: 0x0c, 0x8f1: 0x0c, 0x8f2: 0x0c, 0x8f3: 0x0c, 0x8f4: 0x0c, 0x8f5: 0x0c, 0x8f6: 0x0c, 0x8f7: 0x0c, + 0x8f8: 0x0c, 0x8f9: 0x0c, 0x8fa: 0x0c, 0x8fb: 0x0c, 0x8fc: 0x0c, 0x8fd: 0x0c, 0x8fe: 0x0c, 0x8ff: 0x20, + // Block 0x24, offset 0x900 + 0x900: 0x0c, 0x901: 0x0c, 0x902: 0x0c, 0x903: 0x0c, 0x904: 0x0c, 0x905: 0x0c, 0x906: 0x0c, 0x907: 0x0c, + 0x908: 0x0c, 0x909: 0x0c, 0x90a: 0x0c, 0x90b: 0x0c, 0x90c: 0x0c, 0x90d: 0x0c, 0x90e: 0x0c, 0x90f: 0x20, +} + +// Total table size 27264 bytes (26KiB); checksum: 811C9DC5 diff --git a/golang.org/x/text/transform/transform.go b/golang.org/x/text/transform/transform.go index 520b9ada0e..48ec64b40c 100644 --- a/golang.org/x/text/transform/transform.go +++ b/golang.org/x/text/transform/transform.go @@ -648,7 +648,8 @@ func String(t Transformer, s string) (result string, n int, err error) { // Transform the remaining input, growing dst and src buffers as necessary. for { n := copy(src, s[pSrc:]) - nDst, nSrc, err := t.Transform(dst[pDst:], src[:n], pSrc+n == len(s)) + atEOF := pSrc+n == len(s) + nDst, nSrc, err := t.Transform(dst[pDst:], src[:n], atEOF) pDst += nDst pSrc += nSrc @@ -659,6 +660,9 @@ func String(t Transformer, s string) (result string, n int, err error) { dst = grow(dst, pDst) } } else if err == ErrShortSrc { + if atEOF { + return string(dst[:pDst]), pSrc, err + } if nSrc == 0 { src = grow(src, 0) } diff --git a/golang.org/x/text/unicode/bidi/core.go b/golang.org/x/text/unicode/bidi/core.go index 48d144008a..50deb6600a 100644 --- a/golang.org/x/text/unicode/bidi/core.go +++ b/golang.org/x/text/unicode/bidi/core.go @@ -480,15 +480,15 @@ func (s *isolatingRunSequence) resolveWeakTypes() { // Rule W1. // Changes all NSMs. - preceedingCharacterType := s.sos + precedingCharacterType := s.sos for i, t := range s.types { if t == NSM { - s.types[i] = preceedingCharacterType + s.types[i] = precedingCharacterType } else { if t.in(LRI, RLI, FSI, PDI) { - preceedingCharacterType = ON + precedingCharacterType = ON } - preceedingCharacterType = t + precedingCharacterType = t } } diff --git a/golang.org/x/text/unicode/bidi/tables11.0.0.go b/golang.org/x/text/unicode/bidi/tables11.0.0.go index 022e3c6909..16b11db538 100644 --- a/golang.org/x/text/unicode/bidi/tables11.0.0.go +++ b/golang.org/x/text/unicode/bidi/tables11.0.0.go @@ -1,6 +1,6 @@ // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. -// +build go1.13 +// +build go1.13,!go1.14 package bidi diff --git a/golang.org/x/text/unicode/bidi/tables12.0.0.go b/golang.org/x/text/unicode/bidi/tables12.0.0.go new file mode 100644 index 0000000000..7ffa365121 --- /dev/null +++ b/golang.org/x/text/unicode/bidi/tables12.0.0.go @@ -0,0 +1,1923 @@ +// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. + +// +build go1.14 + +package bidi + +// UnicodeVersion is the Unicode version from which the tables in this package are derived. +const UnicodeVersion = "12.0.0" + +// xorMasks contains masks to be xor-ed with brackets to get the reverse +// version. +var xorMasks = []int32{ // 8 elements + 0, 1, 6, 7, 3, 15, 29, 63, +} // Size: 56 bytes + +// lookup returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *bidiTrie) lookup(s []byte) (v uint8, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return bidiValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = bidiIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = bidiIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = bidiIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *bidiTrie) lookupUnsafe(s []byte) uint8 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return bidiValues[c0] + } + i := bidiIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = bidiIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = bidiIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// lookupString returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *bidiTrie) lookupString(s string) (v uint8, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return bidiValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = bidiIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := bidiIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = bidiIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = bidiIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *bidiTrie) lookupStringUnsafe(s string) uint8 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return bidiValues[c0] + } + i := bidiIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = bidiIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = bidiIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// bidiTrie. Total size: 16896 bytes (16.50 KiB). Checksum: 6f0927067913dc6d. +type bidiTrie struct{} + +func newBidiTrie(i int) *bidiTrie { + return &bidiTrie{} +} + +// lookupValue determines the type of block n and looks up the value for b. +func (t *bidiTrie) lookupValue(n uint32, b byte) uint8 { + switch { + default: + return uint8(bidiValues[n<<6+uint32(b)]) + } +} + +// bidiValues: 240 blocks, 15360 entries, 15360 bytes +// The third block is the zero block. +var bidiValues = [15360]uint8{ + // Block 0x0, offset 0x0 + 0x00: 0x000b, 0x01: 0x000b, 0x02: 0x000b, 0x03: 0x000b, 0x04: 0x000b, 0x05: 0x000b, + 0x06: 0x000b, 0x07: 0x000b, 0x08: 0x000b, 0x09: 0x0008, 0x0a: 0x0007, 0x0b: 0x0008, + 0x0c: 0x0009, 0x0d: 0x0007, 0x0e: 0x000b, 0x0f: 0x000b, 0x10: 0x000b, 0x11: 0x000b, + 0x12: 0x000b, 0x13: 0x000b, 0x14: 0x000b, 0x15: 0x000b, 0x16: 0x000b, 0x17: 0x000b, + 0x18: 0x000b, 0x19: 0x000b, 0x1a: 0x000b, 0x1b: 0x000b, 0x1c: 0x0007, 0x1d: 0x0007, + 0x1e: 0x0007, 0x1f: 0x0008, 0x20: 0x0009, 0x21: 0x000a, 0x22: 0x000a, 0x23: 0x0004, + 0x24: 0x0004, 0x25: 0x0004, 0x26: 0x000a, 0x27: 0x000a, 0x28: 0x003a, 0x29: 0x002a, + 0x2a: 0x000a, 0x2b: 0x0003, 0x2c: 0x0006, 0x2d: 0x0003, 0x2e: 0x0006, 0x2f: 0x0006, + 0x30: 0x0002, 0x31: 0x0002, 0x32: 0x0002, 0x33: 0x0002, 0x34: 0x0002, 0x35: 0x0002, + 0x36: 0x0002, 0x37: 0x0002, 0x38: 0x0002, 0x39: 0x0002, 0x3a: 0x0006, 0x3b: 0x000a, + 0x3c: 0x000a, 0x3d: 0x000a, 0x3e: 0x000a, 0x3f: 0x000a, + // Block 0x1, offset 0x40 + 0x40: 0x000a, + 0x5b: 0x005a, 0x5c: 0x000a, 0x5d: 0x004a, + 0x5e: 0x000a, 0x5f: 0x000a, 0x60: 0x000a, + 0x7b: 0x005a, + 0x7c: 0x000a, 0x7d: 0x004a, 0x7e: 0x000a, 0x7f: 0x000b, + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xc0: 0x000b, 0xc1: 0x000b, 0xc2: 0x000b, 0xc3: 0x000b, 0xc4: 0x000b, 0xc5: 0x0007, + 0xc6: 0x000b, 0xc7: 0x000b, 0xc8: 0x000b, 0xc9: 0x000b, 0xca: 0x000b, 0xcb: 0x000b, + 0xcc: 0x000b, 0xcd: 0x000b, 0xce: 0x000b, 0xcf: 0x000b, 0xd0: 0x000b, 0xd1: 0x000b, + 0xd2: 0x000b, 0xd3: 0x000b, 0xd4: 0x000b, 0xd5: 0x000b, 0xd6: 0x000b, 0xd7: 0x000b, + 0xd8: 0x000b, 0xd9: 0x000b, 0xda: 0x000b, 0xdb: 0x000b, 0xdc: 0x000b, 0xdd: 0x000b, + 0xde: 0x000b, 0xdf: 0x000b, 0xe0: 0x0006, 0xe1: 0x000a, 0xe2: 0x0004, 0xe3: 0x0004, + 0xe4: 0x0004, 0xe5: 0x0004, 0xe6: 0x000a, 0xe7: 0x000a, 0xe8: 0x000a, 0xe9: 0x000a, + 0xeb: 0x000a, 0xec: 0x000a, 0xed: 0x000b, 0xee: 0x000a, 0xef: 0x000a, + 0xf0: 0x0004, 0xf1: 0x0004, 0xf2: 0x0002, 0xf3: 0x0002, 0xf4: 0x000a, + 0xf6: 0x000a, 0xf7: 0x000a, 0xf8: 0x000a, 0xf9: 0x0002, 0xfb: 0x000a, + 0xfc: 0x000a, 0xfd: 0x000a, 0xfe: 0x000a, 0xff: 0x000a, + // Block 0x4, offset 0x100 + 0x117: 0x000a, + 0x137: 0x000a, + // Block 0x5, offset 0x140 + 0x179: 0x000a, 0x17a: 0x000a, + // Block 0x6, offset 0x180 + 0x182: 0x000a, 0x183: 0x000a, 0x184: 0x000a, 0x185: 0x000a, + 0x186: 0x000a, 0x187: 0x000a, 0x188: 0x000a, 0x189: 0x000a, 0x18a: 0x000a, 0x18b: 0x000a, + 0x18c: 0x000a, 0x18d: 0x000a, 0x18e: 0x000a, 0x18f: 0x000a, + 0x192: 0x000a, 0x193: 0x000a, 0x194: 0x000a, 0x195: 0x000a, 0x196: 0x000a, 0x197: 0x000a, + 0x198: 0x000a, 0x199: 0x000a, 0x19a: 0x000a, 0x19b: 0x000a, 0x19c: 0x000a, 0x19d: 0x000a, + 0x19e: 0x000a, 0x19f: 0x000a, + 0x1a5: 0x000a, 0x1a6: 0x000a, 0x1a7: 0x000a, 0x1a8: 0x000a, 0x1a9: 0x000a, + 0x1aa: 0x000a, 0x1ab: 0x000a, 0x1ac: 0x000a, 0x1ad: 0x000a, 0x1af: 0x000a, + 0x1b0: 0x000a, 0x1b1: 0x000a, 0x1b2: 0x000a, 0x1b3: 0x000a, 0x1b4: 0x000a, 0x1b5: 0x000a, + 0x1b6: 0x000a, 0x1b7: 0x000a, 0x1b8: 0x000a, 0x1b9: 0x000a, 0x1ba: 0x000a, 0x1bb: 0x000a, + 0x1bc: 0x000a, 0x1bd: 0x000a, 0x1be: 0x000a, 0x1bf: 0x000a, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x000c, 0x1c1: 0x000c, 0x1c2: 0x000c, 0x1c3: 0x000c, 0x1c4: 0x000c, 0x1c5: 0x000c, + 0x1c6: 0x000c, 0x1c7: 0x000c, 0x1c8: 0x000c, 0x1c9: 0x000c, 0x1ca: 0x000c, 0x1cb: 0x000c, + 0x1cc: 0x000c, 0x1cd: 0x000c, 0x1ce: 0x000c, 0x1cf: 0x000c, 0x1d0: 0x000c, 0x1d1: 0x000c, + 0x1d2: 0x000c, 0x1d3: 0x000c, 0x1d4: 0x000c, 0x1d5: 0x000c, 0x1d6: 0x000c, 0x1d7: 0x000c, + 0x1d8: 0x000c, 0x1d9: 0x000c, 0x1da: 0x000c, 0x1db: 0x000c, 0x1dc: 0x000c, 0x1dd: 0x000c, + 0x1de: 0x000c, 0x1df: 0x000c, 0x1e0: 0x000c, 0x1e1: 0x000c, 0x1e2: 0x000c, 0x1e3: 0x000c, + 0x1e4: 0x000c, 0x1e5: 0x000c, 0x1e6: 0x000c, 0x1e7: 0x000c, 0x1e8: 0x000c, 0x1e9: 0x000c, + 0x1ea: 0x000c, 0x1eb: 0x000c, 0x1ec: 0x000c, 0x1ed: 0x000c, 0x1ee: 0x000c, 0x1ef: 0x000c, + 0x1f0: 0x000c, 0x1f1: 0x000c, 0x1f2: 0x000c, 0x1f3: 0x000c, 0x1f4: 0x000c, 0x1f5: 0x000c, + 0x1f6: 0x000c, 0x1f7: 0x000c, 0x1f8: 0x000c, 0x1f9: 0x000c, 0x1fa: 0x000c, 0x1fb: 0x000c, + 0x1fc: 0x000c, 0x1fd: 0x000c, 0x1fe: 0x000c, 0x1ff: 0x000c, + // Block 0x8, offset 0x200 + 0x200: 0x000c, 0x201: 0x000c, 0x202: 0x000c, 0x203: 0x000c, 0x204: 0x000c, 0x205: 0x000c, + 0x206: 0x000c, 0x207: 0x000c, 0x208: 0x000c, 0x209: 0x000c, 0x20a: 0x000c, 0x20b: 0x000c, + 0x20c: 0x000c, 0x20d: 0x000c, 0x20e: 0x000c, 0x20f: 0x000c, 0x210: 0x000c, 0x211: 0x000c, + 0x212: 0x000c, 0x213: 0x000c, 0x214: 0x000c, 0x215: 0x000c, 0x216: 0x000c, 0x217: 0x000c, + 0x218: 0x000c, 0x219: 0x000c, 0x21a: 0x000c, 0x21b: 0x000c, 0x21c: 0x000c, 0x21d: 0x000c, + 0x21e: 0x000c, 0x21f: 0x000c, 0x220: 0x000c, 0x221: 0x000c, 0x222: 0x000c, 0x223: 0x000c, + 0x224: 0x000c, 0x225: 0x000c, 0x226: 0x000c, 0x227: 0x000c, 0x228: 0x000c, 0x229: 0x000c, + 0x22a: 0x000c, 0x22b: 0x000c, 0x22c: 0x000c, 0x22d: 0x000c, 0x22e: 0x000c, 0x22f: 0x000c, + 0x234: 0x000a, 0x235: 0x000a, + 0x23e: 0x000a, + // Block 0x9, offset 0x240 + 0x244: 0x000a, 0x245: 0x000a, + 0x247: 0x000a, + // Block 0xa, offset 0x280 + 0x2b6: 0x000a, + // Block 0xb, offset 0x2c0 + 0x2c3: 0x000c, 0x2c4: 0x000c, 0x2c5: 0x000c, + 0x2c6: 0x000c, 0x2c7: 0x000c, 0x2c8: 0x000c, 0x2c9: 0x000c, + // Block 0xc, offset 0x300 + 0x30a: 0x000a, + 0x30d: 0x000a, 0x30e: 0x000a, 0x30f: 0x0004, 0x310: 0x0001, 0x311: 0x000c, + 0x312: 0x000c, 0x313: 0x000c, 0x314: 0x000c, 0x315: 0x000c, 0x316: 0x000c, 0x317: 0x000c, + 0x318: 0x000c, 0x319: 0x000c, 0x31a: 0x000c, 0x31b: 0x000c, 0x31c: 0x000c, 0x31d: 0x000c, + 0x31e: 0x000c, 0x31f: 0x000c, 0x320: 0x000c, 0x321: 0x000c, 0x322: 0x000c, 0x323: 0x000c, + 0x324: 0x000c, 0x325: 0x000c, 0x326: 0x000c, 0x327: 0x000c, 0x328: 0x000c, 0x329: 0x000c, + 0x32a: 0x000c, 0x32b: 0x000c, 0x32c: 0x000c, 0x32d: 0x000c, 0x32e: 0x000c, 0x32f: 0x000c, + 0x330: 0x000c, 0x331: 0x000c, 0x332: 0x000c, 0x333: 0x000c, 0x334: 0x000c, 0x335: 0x000c, + 0x336: 0x000c, 0x337: 0x000c, 0x338: 0x000c, 0x339: 0x000c, 0x33a: 0x000c, 0x33b: 0x000c, + 0x33c: 0x000c, 0x33d: 0x000c, 0x33e: 0x0001, 0x33f: 0x000c, + // Block 0xd, offset 0x340 + 0x340: 0x0001, 0x341: 0x000c, 0x342: 0x000c, 0x343: 0x0001, 0x344: 0x000c, 0x345: 0x000c, + 0x346: 0x0001, 0x347: 0x000c, 0x348: 0x0001, 0x349: 0x0001, 0x34a: 0x0001, 0x34b: 0x0001, + 0x34c: 0x0001, 0x34d: 0x0001, 0x34e: 0x0001, 0x34f: 0x0001, 0x350: 0x0001, 0x351: 0x0001, + 0x352: 0x0001, 0x353: 0x0001, 0x354: 0x0001, 0x355: 0x0001, 0x356: 0x0001, 0x357: 0x0001, + 0x358: 0x0001, 0x359: 0x0001, 0x35a: 0x0001, 0x35b: 0x0001, 0x35c: 0x0001, 0x35d: 0x0001, + 0x35e: 0x0001, 0x35f: 0x0001, 0x360: 0x0001, 0x361: 0x0001, 0x362: 0x0001, 0x363: 0x0001, + 0x364: 0x0001, 0x365: 0x0001, 0x366: 0x0001, 0x367: 0x0001, 0x368: 0x0001, 0x369: 0x0001, + 0x36a: 0x0001, 0x36b: 0x0001, 0x36c: 0x0001, 0x36d: 0x0001, 0x36e: 0x0001, 0x36f: 0x0001, + 0x370: 0x0001, 0x371: 0x0001, 0x372: 0x0001, 0x373: 0x0001, 0x374: 0x0001, 0x375: 0x0001, + 0x376: 0x0001, 0x377: 0x0001, 0x378: 0x0001, 0x379: 0x0001, 0x37a: 0x0001, 0x37b: 0x0001, + 0x37c: 0x0001, 0x37d: 0x0001, 0x37e: 0x0001, 0x37f: 0x0001, + // Block 0xe, offset 0x380 + 0x380: 0x0005, 0x381: 0x0005, 0x382: 0x0005, 0x383: 0x0005, 0x384: 0x0005, 0x385: 0x0005, + 0x386: 0x000a, 0x387: 0x000a, 0x388: 0x000d, 0x389: 0x0004, 0x38a: 0x0004, 0x38b: 0x000d, + 0x38c: 0x0006, 0x38d: 0x000d, 0x38e: 0x000a, 0x38f: 0x000a, 0x390: 0x000c, 0x391: 0x000c, + 0x392: 0x000c, 0x393: 0x000c, 0x394: 0x000c, 0x395: 0x000c, 0x396: 0x000c, 0x397: 0x000c, + 0x398: 0x000c, 0x399: 0x000c, 0x39a: 0x000c, 0x39b: 0x000d, 0x39c: 0x000d, 0x39d: 0x000d, + 0x39e: 0x000d, 0x39f: 0x000d, 0x3a0: 0x000d, 0x3a1: 0x000d, 0x3a2: 0x000d, 0x3a3: 0x000d, + 0x3a4: 0x000d, 0x3a5: 0x000d, 0x3a6: 0x000d, 0x3a7: 0x000d, 0x3a8: 0x000d, 0x3a9: 0x000d, + 0x3aa: 0x000d, 0x3ab: 0x000d, 0x3ac: 0x000d, 0x3ad: 0x000d, 0x3ae: 0x000d, 0x3af: 0x000d, + 0x3b0: 0x000d, 0x3b1: 0x000d, 0x3b2: 0x000d, 0x3b3: 0x000d, 0x3b4: 0x000d, 0x3b5: 0x000d, + 0x3b6: 0x000d, 0x3b7: 0x000d, 0x3b8: 0x000d, 0x3b9: 0x000d, 0x3ba: 0x000d, 0x3bb: 0x000d, + 0x3bc: 0x000d, 0x3bd: 0x000d, 0x3be: 0x000d, 0x3bf: 0x000d, + // Block 0xf, offset 0x3c0 + 0x3c0: 0x000d, 0x3c1: 0x000d, 0x3c2: 0x000d, 0x3c3: 0x000d, 0x3c4: 0x000d, 0x3c5: 0x000d, + 0x3c6: 0x000d, 0x3c7: 0x000d, 0x3c8: 0x000d, 0x3c9: 0x000d, 0x3ca: 0x000d, 0x3cb: 0x000c, + 0x3cc: 0x000c, 0x3cd: 0x000c, 0x3ce: 0x000c, 0x3cf: 0x000c, 0x3d0: 0x000c, 0x3d1: 0x000c, + 0x3d2: 0x000c, 0x3d3: 0x000c, 0x3d4: 0x000c, 0x3d5: 0x000c, 0x3d6: 0x000c, 0x3d7: 0x000c, + 0x3d8: 0x000c, 0x3d9: 0x000c, 0x3da: 0x000c, 0x3db: 0x000c, 0x3dc: 0x000c, 0x3dd: 0x000c, + 0x3de: 0x000c, 0x3df: 0x000c, 0x3e0: 0x0005, 0x3e1: 0x0005, 0x3e2: 0x0005, 0x3e3: 0x0005, + 0x3e4: 0x0005, 0x3e5: 0x0005, 0x3e6: 0x0005, 0x3e7: 0x0005, 0x3e8: 0x0005, 0x3e9: 0x0005, + 0x3ea: 0x0004, 0x3eb: 0x0005, 0x3ec: 0x0005, 0x3ed: 0x000d, 0x3ee: 0x000d, 0x3ef: 0x000d, + 0x3f0: 0x000c, 0x3f1: 0x000d, 0x3f2: 0x000d, 0x3f3: 0x000d, 0x3f4: 0x000d, 0x3f5: 0x000d, + 0x3f6: 0x000d, 0x3f7: 0x000d, 0x3f8: 0x000d, 0x3f9: 0x000d, 0x3fa: 0x000d, 0x3fb: 0x000d, + 0x3fc: 0x000d, 0x3fd: 0x000d, 0x3fe: 0x000d, 0x3ff: 0x000d, + // Block 0x10, offset 0x400 + 0x400: 0x000d, 0x401: 0x000d, 0x402: 0x000d, 0x403: 0x000d, 0x404: 0x000d, 0x405: 0x000d, + 0x406: 0x000d, 0x407: 0x000d, 0x408: 0x000d, 0x409: 0x000d, 0x40a: 0x000d, 0x40b: 0x000d, + 0x40c: 0x000d, 0x40d: 0x000d, 0x40e: 0x000d, 0x40f: 0x000d, 0x410: 0x000d, 0x411: 0x000d, + 0x412: 0x000d, 0x413: 0x000d, 0x414: 0x000d, 0x415: 0x000d, 0x416: 0x000d, 0x417: 0x000d, + 0x418: 0x000d, 0x419: 0x000d, 0x41a: 0x000d, 0x41b: 0x000d, 0x41c: 0x000d, 0x41d: 0x000d, + 0x41e: 0x000d, 0x41f: 0x000d, 0x420: 0x000d, 0x421: 0x000d, 0x422: 0x000d, 0x423: 0x000d, + 0x424: 0x000d, 0x425: 0x000d, 0x426: 0x000d, 0x427: 0x000d, 0x428: 0x000d, 0x429: 0x000d, + 0x42a: 0x000d, 0x42b: 0x000d, 0x42c: 0x000d, 0x42d: 0x000d, 0x42e: 0x000d, 0x42f: 0x000d, + 0x430: 0x000d, 0x431: 0x000d, 0x432: 0x000d, 0x433: 0x000d, 0x434: 0x000d, 0x435: 0x000d, + 0x436: 0x000d, 0x437: 0x000d, 0x438: 0x000d, 0x439: 0x000d, 0x43a: 0x000d, 0x43b: 0x000d, + 0x43c: 0x000d, 0x43d: 0x000d, 0x43e: 0x000d, 0x43f: 0x000d, + // Block 0x11, offset 0x440 + 0x440: 0x000d, 0x441: 0x000d, 0x442: 0x000d, 0x443: 0x000d, 0x444: 0x000d, 0x445: 0x000d, + 0x446: 0x000d, 0x447: 0x000d, 0x448: 0x000d, 0x449: 0x000d, 0x44a: 0x000d, 0x44b: 0x000d, + 0x44c: 0x000d, 0x44d: 0x000d, 0x44e: 0x000d, 0x44f: 0x000d, 0x450: 0x000d, 0x451: 0x000d, + 0x452: 0x000d, 0x453: 0x000d, 0x454: 0x000d, 0x455: 0x000d, 0x456: 0x000c, 0x457: 0x000c, + 0x458: 0x000c, 0x459: 0x000c, 0x45a: 0x000c, 0x45b: 0x000c, 0x45c: 0x000c, 0x45d: 0x0005, + 0x45e: 0x000a, 0x45f: 0x000c, 0x460: 0x000c, 0x461: 0x000c, 0x462: 0x000c, 0x463: 0x000c, + 0x464: 0x000c, 0x465: 0x000d, 0x466: 0x000d, 0x467: 0x000c, 0x468: 0x000c, 0x469: 0x000a, + 0x46a: 0x000c, 0x46b: 0x000c, 0x46c: 0x000c, 0x46d: 0x000c, 0x46e: 0x000d, 0x46f: 0x000d, + 0x470: 0x0002, 0x471: 0x0002, 0x472: 0x0002, 0x473: 0x0002, 0x474: 0x0002, 0x475: 0x0002, + 0x476: 0x0002, 0x477: 0x0002, 0x478: 0x0002, 0x479: 0x0002, 0x47a: 0x000d, 0x47b: 0x000d, + 0x47c: 0x000d, 0x47d: 0x000d, 0x47e: 0x000d, 0x47f: 0x000d, + // Block 0x12, offset 0x480 + 0x480: 0x000d, 0x481: 0x000d, 0x482: 0x000d, 0x483: 0x000d, 0x484: 0x000d, 0x485: 0x000d, + 0x486: 0x000d, 0x487: 0x000d, 0x488: 0x000d, 0x489: 0x000d, 0x48a: 0x000d, 0x48b: 0x000d, + 0x48c: 0x000d, 0x48d: 0x000d, 0x48e: 0x000d, 0x48f: 0x000d, 0x490: 0x000d, 0x491: 0x000c, + 0x492: 0x000d, 0x493: 0x000d, 0x494: 0x000d, 0x495: 0x000d, 0x496: 0x000d, 0x497: 0x000d, + 0x498: 0x000d, 0x499: 0x000d, 0x49a: 0x000d, 0x49b: 0x000d, 0x49c: 0x000d, 0x49d: 0x000d, + 0x49e: 0x000d, 0x49f: 0x000d, 0x4a0: 0x000d, 0x4a1: 0x000d, 0x4a2: 0x000d, 0x4a3: 0x000d, + 0x4a4: 0x000d, 0x4a5: 0x000d, 0x4a6: 0x000d, 0x4a7: 0x000d, 0x4a8: 0x000d, 0x4a9: 0x000d, + 0x4aa: 0x000d, 0x4ab: 0x000d, 0x4ac: 0x000d, 0x4ad: 0x000d, 0x4ae: 0x000d, 0x4af: 0x000d, + 0x4b0: 0x000c, 0x4b1: 0x000c, 0x4b2: 0x000c, 0x4b3: 0x000c, 0x4b4: 0x000c, 0x4b5: 0x000c, + 0x4b6: 0x000c, 0x4b7: 0x000c, 0x4b8: 0x000c, 0x4b9: 0x000c, 0x4ba: 0x000c, 0x4bb: 0x000c, + 0x4bc: 0x000c, 0x4bd: 0x000c, 0x4be: 0x000c, 0x4bf: 0x000c, + // Block 0x13, offset 0x4c0 + 0x4c0: 0x000c, 0x4c1: 0x000c, 0x4c2: 0x000c, 0x4c3: 0x000c, 0x4c4: 0x000c, 0x4c5: 0x000c, + 0x4c6: 0x000c, 0x4c7: 0x000c, 0x4c8: 0x000c, 0x4c9: 0x000c, 0x4ca: 0x000c, 0x4cb: 0x000d, + 0x4cc: 0x000d, 0x4cd: 0x000d, 0x4ce: 0x000d, 0x4cf: 0x000d, 0x4d0: 0x000d, 0x4d1: 0x000d, + 0x4d2: 0x000d, 0x4d3: 0x000d, 0x4d4: 0x000d, 0x4d5: 0x000d, 0x4d6: 0x000d, 0x4d7: 0x000d, + 0x4d8: 0x000d, 0x4d9: 0x000d, 0x4da: 0x000d, 0x4db: 0x000d, 0x4dc: 0x000d, 0x4dd: 0x000d, + 0x4de: 0x000d, 0x4df: 0x000d, 0x4e0: 0x000d, 0x4e1: 0x000d, 0x4e2: 0x000d, 0x4e3: 0x000d, + 0x4e4: 0x000d, 0x4e5: 0x000d, 0x4e6: 0x000d, 0x4e7: 0x000d, 0x4e8: 0x000d, 0x4e9: 0x000d, + 0x4ea: 0x000d, 0x4eb: 0x000d, 0x4ec: 0x000d, 0x4ed: 0x000d, 0x4ee: 0x000d, 0x4ef: 0x000d, + 0x4f0: 0x000d, 0x4f1: 0x000d, 0x4f2: 0x000d, 0x4f3: 0x000d, 0x4f4: 0x000d, 0x4f5: 0x000d, + 0x4f6: 0x000d, 0x4f7: 0x000d, 0x4f8: 0x000d, 0x4f9: 0x000d, 0x4fa: 0x000d, 0x4fb: 0x000d, + 0x4fc: 0x000d, 0x4fd: 0x000d, 0x4fe: 0x000d, 0x4ff: 0x000d, + // Block 0x14, offset 0x500 + 0x500: 0x000d, 0x501: 0x000d, 0x502: 0x000d, 0x503: 0x000d, 0x504: 0x000d, 0x505: 0x000d, + 0x506: 0x000d, 0x507: 0x000d, 0x508: 0x000d, 0x509: 0x000d, 0x50a: 0x000d, 0x50b: 0x000d, + 0x50c: 0x000d, 0x50d: 0x000d, 0x50e: 0x000d, 0x50f: 0x000d, 0x510: 0x000d, 0x511: 0x000d, + 0x512: 0x000d, 0x513: 0x000d, 0x514: 0x000d, 0x515: 0x000d, 0x516: 0x000d, 0x517: 0x000d, + 0x518: 0x000d, 0x519: 0x000d, 0x51a: 0x000d, 0x51b: 0x000d, 0x51c: 0x000d, 0x51d: 0x000d, + 0x51e: 0x000d, 0x51f: 0x000d, 0x520: 0x000d, 0x521: 0x000d, 0x522: 0x000d, 0x523: 0x000d, + 0x524: 0x000d, 0x525: 0x000d, 0x526: 0x000c, 0x527: 0x000c, 0x528: 0x000c, 0x529: 0x000c, + 0x52a: 0x000c, 0x52b: 0x000c, 0x52c: 0x000c, 0x52d: 0x000c, 0x52e: 0x000c, 0x52f: 0x000c, + 0x530: 0x000c, 0x531: 0x000d, 0x532: 0x000d, 0x533: 0x000d, 0x534: 0x000d, 0x535: 0x000d, + 0x536: 0x000d, 0x537: 0x000d, 0x538: 0x000d, 0x539: 0x000d, 0x53a: 0x000d, 0x53b: 0x000d, + 0x53c: 0x000d, 0x53d: 0x000d, 0x53e: 0x000d, 0x53f: 0x000d, + // Block 0x15, offset 0x540 + 0x540: 0x0001, 0x541: 0x0001, 0x542: 0x0001, 0x543: 0x0001, 0x544: 0x0001, 0x545: 0x0001, + 0x546: 0x0001, 0x547: 0x0001, 0x548: 0x0001, 0x549: 0x0001, 0x54a: 0x0001, 0x54b: 0x0001, + 0x54c: 0x0001, 0x54d: 0x0001, 0x54e: 0x0001, 0x54f: 0x0001, 0x550: 0x0001, 0x551: 0x0001, + 0x552: 0x0001, 0x553: 0x0001, 0x554: 0x0001, 0x555: 0x0001, 0x556: 0x0001, 0x557: 0x0001, + 0x558: 0x0001, 0x559: 0x0001, 0x55a: 0x0001, 0x55b: 0x0001, 0x55c: 0x0001, 0x55d: 0x0001, + 0x55e: 0x0001, 0x55f: 0x0001, 0x560: 0x0001, 0x561: 0x0001, 0x562: 0x0001, 0x563: 0x0001, + 0x564: 0x0001, 0x565: 0x0001, 0x566: 0x0001, 0x567: 0x0001, 0x568: 0x0001, 0x569: 0x0001, + 0x56a: 0x0001, 0x56b: 0x000c, 0x56c: 0x000c, 0x56d: 0x000c, 0x56e: 0x000c, 0x56f: 0x000c, + 0x570: 0x000c, 0x571: 0x000c, 0x572: 0x000c, 0x573: 0x000c, 0x574: 0x0001, 0x575: 0x0001, + 0x576: 0x000a, 0x577: 0x000a, 0x578: 0x000a, 0x579: 0x000a, 0x57a: 0x0001, 0x57b: 0x0001, + 0x57c: 0x0001, 0x57d: 0x000c, 0x57e: 0x0001, 0x57f: 0x0001, + // Block 0x16, offset 0x580 + 0x580: 0x0001, 0x581: 0x0001, 0x582: 0x0001, 0x583: 0x0001, 0x584: 0x0001, 0x585: 0x0001, + 0x586: 0x0001, 0x587: 0x0001, 0x588: 0x0001, 0x589: 0x0001, 0x58a: 0x0001, 0x58b: 0x0001, + 0x58c: 0x0001, 0x58d: 0x0001, 0x58e: 0x0001, 0x58f: 0x0001, 0x590: 0x0001, 0x591: 0x0001, + 0x592: 0x0001, 0x593: 0x0001, 0x594: 0x0001, 0x595: 0x0001, 0x596: 0x000c, 0x597: 0x000c, + 0x598: 0x000c, 0x599: 0x000c, 0x59a: 0x0001, 0x59b: 0x000c, 0x59c: 0x000c, 0x59d: 0x000c, + 0x59e: 0x000c, 0x59f: 0x000c, 0x5a0: 0x000c, 0x5a1: 0x000c, 0x5a2: 0x000c, 0x5a3: 0x000c, + 0x5a4: 0x0001, 0x5a5: 0x000c, 0x5a6: 0x000c, 0x5a7: 0x000c, 0x5a8: 0x0001, 0x5a9: 0x000c, + 0x5aa: 0x000c, 0x5ab: 0x000c, 0x5ac: 0x000c, 0x5ad: 0x000c, 0x5ae: 0x0001, 0x5af: 0x0001, + 0x5b0: 0x0001, 0x5b1: 0x0001, 0x5b2: 0x0001, 0x5b3: 0x0001, 0x5b4: 0x0001, 0x5b5: 0x0001, + 0x5b6: 0x0001, 0x5b7: 0x0001, 0x5b8: 0x0001, 0x5b9: 0x0001, 0x5ba: 0x0001, 0x5bb: 0x0001, + 0x5bc: 0x0001, 0x5bd: 0x0001, 0x5be: 0x0001, 0x5bf: 0x0001, + // Block 0x17, offset 0x5c0 + 0x5c0: 0x0001, 0x5c1: 0x0001, 0x5c2: 0x0001, 0x5c3: 0x0001, 0x5c4: 0x0001, 0x5c5: 0x0001, + 0x5c6: 0x0001, 0x5c7: 0x0001, 0x5c8: 0x0001, 0x5c9: 0x0001, 0x5ca: 0x0001, 0x5cb: 0x0001, + 0x5cc: 0x0001, 0x5cd: 0x0001, 0x5ce: 0x0001, 0x5cf: 0x0001, 0x5d0: 0x0001, 0x5d1: 0x0001, + 0x5d2: 0x0001, 0x5d3: 0x0001, 0x5d4: 0x0001, 0x5d5: 0x0001, 0x5d6: 0x0001, 0x5d7: 0x0001, + 0x5d8: 0x0001, 0x5d9: 0x000c, 0x5da: 0x000c, 0x5db: 0x000c, 0x5dc: 0x0001, 0x5dd: 0x0001, + 0x5de: 0x0001, 0x5df: 0x0001, 0x5e0: 0x000d, 0x5e1: 0x000d, 0x5e2: 0x000d, 0x5e3: 0x000d, + 0x5e4: 0x000d, 0x5e5: 0x000d, 0x5e6: 0x000d, 0x5e7: 0x000d, 0x5e8: 0x000d, 0x5e9: 0x000d, + 0x5ea: 0x000d, 0x5eb: 0x000d, 0x5ec: 0x000d, 0x5ed: 0x000d, 0x5ee: 0x000d, 0x5ef: 0x000d, + 0x5f0: 0x0001, 0x5f1: 0x0001, 0x5f2: 0x0001, 0x5f3: 0x0001, 0x5f4: 0x0001, 0x5f5: 0x0001, + 0x5f6: 0x0001, 0x5f7: 0x0001, 0x5f8: 0x0001, 0x5f9: 0x0001, 0x5fa: 0x0001, 0x5fb: 0x0001, + 0x5fc: 0x0001, 0x5fd: 0x0001, 0x5fe: 0x0001, 0x5ff: 0x0001, + // Block 0x18, offset 0x600 + 0x600: 0x0001, 0x601: 0x0001, 0x602: 0x0001, 0x603: 0x0001, 0x604: 0x0001, 0x605: 0x0001, + 0x606: 0x0001, 0x607: 0x0001, 0x608: 0x0001, 0x609: 0x0001, 0x60a: 0x0001, 0x60b: 0x0001, + 0x60c: 0x0001, 0x60d: 0x0001, 0x60e: 0x0001, 0x60f: 0x0001, 0x610: 0x0001, 0x611: 0x0001, + 0x612: 0x0001, 0x613: 0x0001, 0x614: 0x0001, 0x615: 0x0001, 0x616: 0x0001, 0x617: 0x0001, + 0x618: 0x0001, 0x619: 0x0001, 0x61a: 0x0001, 0x61b: 0x0001, 0x61c: 0x0001, 0x61d: 0x0001, + 0x61e: 0x0001, 0x61f: 0x0001, 0x620: 0x000d, 0x621: 0x000d, 0x622: 0x000d, 0x623: 0x000d, + 0x624: 0x000d, 0x625: 0x000d, 0x626: 0x000d, 0x627: 0x000d, 0x628: 0x000d, 0x629: 0x000d, + 0x62a: 0x000d, 0x62b: 0x000d, 0x62c: 0x000d, 0x62d: 0x000d, 0x62e: 0x000d, 0x62f: 0x000d, + 0x630: 0x000d, 0x631: 0x000d, 0x632: 0x000d, 0x633: 0x000d, 0x634: 0x000d, 0x635: 0x000d, + 0x636: 0x000d, 0x637: 0x000d, 0x638: 0x000d, 0x639: 0x000d, 0x63a: 0x000d, 0x63b: 0x000d, + 0x63c: 0x000d, 0x63d: 0x000d, 0x63e: 0x000d, 0x63f: 0x000d, + // Block 0x19, offset 0x640 + 0x640: 0x000d, 0x641: 0x000d, 0x642: 0x000d, 0x643: 0x000d, 0x644: 0x000d, 0x645: 0x000d, + 0x646: 0x000d, 0x647: 0x000d, 0x648: 0x000d, 0x649: 0x000d, 0x64a: 0x000d, 0x64b: 0x000d, + 0x64c: 0x000d, 0x64d: 0x000d, 0x64e: 0x000d, 0x64f: 0x000d, 0x650: 0x000d, 0x651: 0x000d, + 0x652: 0x000d, 0x653: 0x000c, 0x654: 0x000c, 0x655: 0x000c, 0x656: 0x000c, 0x657: 0x000c, + 0x658: 0x000c, 0x659: 0x000c, 0x65a: 0x000c, 0x65b: 0x000c, 0x65c: 0x000c, 0x65d: 0x000c, + 0x65e: 0x000c, 0x65f: 0x000c, 0x660: 0x000c, 0x661: 0x000c, 0x662: 0x0005, 0x663: 0x000c, + 0x664: 0x000c, 0x665: 0x000c, 0x666: 0x000c, 0x667: 0x000c, 0x668: 0x000c, 0x669: 0x000c, + 0x66a: 0x000c, 0x66b: 0x000c, 0x66c: 0x000c, 0x66d: 0x000c, 0x66e: 0x000c, 0x66f: 0x000c, + 0x670: 0x000c, 0x671: 0x000c, 0x672: 0x000c, 0x673: 0x000c, 0x674: 0x000c, 0x675: 0x000c, + 0x676: 0x000c, 0x677: 0x000c, 0x678: 0x000c, 0x679: 0x000c, 0x67a: 0x000c, 0x67b: 0x000c, + 0x67c: 0x000c, 0x67d: 0x000c, 0x67e: 0x000c, 0x67f: 0x000c, + // Block 0x1a, offset 0x680 + 0x680: 0x000c, 0x681: 0x000c, 0x682: 0x000c, + 0x6ba: 0x000c, + 0x6bc: 0x000c, + // Block 0x1b, offset 0x6c0 + 0x6c1: 0x000c, 0x6c2: 0x000c, 0x6c3: 0x000c, 0x6c4: 0x000c, 0x6c5: 0x000c, + 0x6c6: 0x000c, 0x6c7: 0x000c, 0x6c8: 0x000c, + 0x6cd: 0x000c, 0x6d1: 0x000c, + 0x6d2: 0x000c, 0x6d3: 0x000c, 0x6d4: 0x000c, 0x6d5: 0x000c, 0x6d6: 0x000c, 0x6d7: 0x000c, + 0x6e2: 0x000c, 0x6e3: 0x000c, + // Block 0x1c, offset 0x700 + 0x701: 0x000c, + 0x73c: 0x000c, + // Block 0x1d, offset 0x740 + 0x741: 0x000c, 0x742: 0x000c, 0x743: 0x000c, 0x744: 0x000c, + 0x74d: 0x000c, + 0x762: 0x000c, 0x763: 0x000c, + 0x772: 0x0004, 0x773: 0x0004, + 0x77b: 0x0004, + 0x77e: 0x000c, + // Block 0x1e, offset 0x780 + 0x781: 0x000c, 0x782: 0x000c, + 0x7bc: 0x000c, + // Block 0x1f, offset 0x7c0 + 0x7c1: 0x000c, 0x7c2: 0x000c, + 0x7c7: 0x000c, 0x7c8: 0x000c, 0x7cb: 0x000c, + 0x7cc: 0x000c, 0x7cd: 0x000c, 0x7d1: 0x000c, + 0x7f0: 0x000c, 0x7f1: 0x000c, 0x7f5: 0x000c, + // Block 0x20, offset 0x800 + 0x801: 0x000c, 0x802: 0x000c, 0x803: 0x000c, 0x804: 0x000c, 0x805: 0x000c, + 0x807: 0x000c, 0x808: 0x000c, + 0x80d: 0x000c, + 0x822: 0x000c, 0x823: 0x000c, + 0x831: 0x0004, + 0x83a: 0x000c, 0x83b: 0x000c, + 0x83c: 0x000c, 0x83d: 0x000c, 0x83e: 0x000c, 0x83f: 0x000c, + // Block 0x21, offset 0x840 + 0x841: 0x000c, + 0x87c: 0x000c, 0x87f: 0x000c, + // Block 0x22, offset 0x880 + 0x881: 0x000c, 0x882: 0x000c, 0x883: 0x000c, 0x884: 0x000c, + 0x88d: 0x000c, + 0x896: 0x000c, + 0x8a2: 0x000c, 0x8a3: 0x000c, + // Block 0x23, offset 0x8c0 + 0x8c2: 0x000c, + // Block 0x24, offset 0x900 + 0x900: 0x000c, + 0x90d: 0x000c, + 0x933: 0x000a, 0x934: 0x000a, 0x935: 0x000a, + 0x936: 0x000a, 0x937: 0x000a, 0x938: 0x000a, 0x939: 0x0004, 0x93a: 0x000a, + // Block 0x25, offset 0x940 + 0x940: 0x000c, 0x944: 0x000c, + 0x97e: 0x000c, 0x97f: 0x000c, + // Block 0x26, offset 0x980 + 0x980: 0x000c, + 0x986: 0x000c, 0x987: 0x000c, 0x988: 0x000c, 0x98a: 0x000c, 0x98b: 0x000c, + 0x98c: 0x000c, 0x98d: 0x000c, + 0x995: 0x000c, 0x996: 0x000c, + 0x9a2: 0x000c, 0x9a3: 0x000c, + 0x9b8: 0x000a, 0x9b9: 0x000a, 0x9ba: 0x000a, 0x9bb: 0x000a, + 0x9bc: 0x000a, 0x9bd: 0x000a, 0x9be: 0x000a, + // Block 0x27, offset 0x9c0 + 0x9cc: 0x000c, 0x9cd: 0x000c, + 0x9e2: 0x000c, 0x9e3: 0x000c, + // Block 0x28, offset 0xa00 + 0xa00: 0x000c, 0xa01: 0x000c, + 0xa3b: 0x000c, + 0xa3c: 0x000c, + // Block 0x29, offset 0xa40 + 0xa41: 0x000c, 0xa42: 0x000c, 0xa43: 0x000c, 0xa44: 0x000c, + 0xa4d: 0x000c, + 0xa62: 0x000c, 0xa63: 0x000c, + // Block 0x2a, offset 0xa80 + 0xa8a: 0x000c, + 0xa92: 0x000c, 0xa93: 0x000c, 0xa94: 0x000c, 0xa96: 0x000c, + // Block 0x2b, offset 0xac0 + 0xaf1: 0x000c, 0xaf4: 0x000c, 0xaf5: 0x000c, + 0xaf6: 0x000c, 0xaf7: 0x000c, 0xaf8: 0x000c, 0xaf9: 0x000c, 0xafa: 0x000c, + 0xaff: 0x0004, + // Block 0x2c, offset 0xb00 + 0xb07: 0x000c, 0xb08: 0x000c, 0xb09: 0x000c, 0xb0a: 0x000c, 0xb0b: 0x000c, + 0xb0c: 0x000c, 0xb0d: 0x000c, 0xb0e: 0x000c, + // Block 0x2d, offset 0xb40 + 0xb71: 0x000c, 0xb74: 0x000c, 0xb75: 0x000c, + 0xb76: 0x000c, 0xb77: 0x000c, 0xb78: 0x000c, 0xb79: 0x000c, 0xb7a: 0x000c, 0xb7b: 0x000c, + 0xb7c: 0x000c, + // Block 0x2e, offset 0xb80 + 0xb88: 0x000c, 0xb89: 0x000c, 0xb8a: 0x000c, 0xb8b: 0x000c, + 0xb8c: 0x000c, 0xb8d: 0x000c, + // Block 0x2f, offset 0xbc0 + 0xbd8: 0x000c, 0xbd9: 0x000c, + 0xbf5: 0x000c, + 0xbf7: 0x000c, 0xbf9: 0x000c, 0xbfa: 0x003a, 0xbfb: 0x002a, + 0xbfc: 0x003a, 0xbfd: 0x002a, + // Block 0x30, offset 0xc00 + 0xc31: 0x000c, 0xc32: 0x000c, 0xc33: 0x000c, 0xc34: 0x000c, 0xc35: 0x000c, + 0xc36: 0x000c, 0xc37: 0x000c, 0xc38: 0x000c, 0xc39: 0x000c, 0xc3a: 0x000c, 0xc3b: 0x000c, + 0xc3c: 0x000c, 0xc3d: 0x000c, 0xc3e: 0x000c, + // Block 0x31, offset 0xc40 + 0xc40: 0x000c, 0xc41: 0x000c, 0xc42: 0x000c, 0xc43: 0x000c, 0xc44: 0x000c, + 0xc46: 0x000c, 0xc47: 0x000c, + 0xc4d: 0x000c, 0xc4e: 0x000c, 0xc4f: 0x000c, 0xc50: 0x000c, 0xc51: 0x000c, + 0xc52: 0x000c, 0xc53: 0x000c, 0xc54: 0x000c, 0xc55: 0x000c, 0xc56: 0x000c, 0xc57: 0x000c, + 0xc59: 0x000c, 0xc5a: 0x000c, 0xc5b: 0x000c, 0xc5c: 0x000c, 0xc5d: 0x000c, + 0xc5e: 0x000c, 0xc5f: 0x000c, 0xc60: 0x000c, 0xc61: 0x000c, 0xc62: 0x000c, 0xc63: 0x000c, + 0xc64: 0x000c, 0xc65: 0x000c, 0xc66: 0x000c, 0xc67: 0x000c, 0xc68: 0x000c, 0xc69: 0x000c, + 0xc6a: 0x000c, 0xc6b: 0x000c, 0xc6c: 0x000c, 0xc6d: 0x000c, 0xc6e: 0x000c, 0xc6f: 0x000c, + 0xc70: 0x000c, 0xc71: 0x000c, 0xc72: 0x000c, 0xc73: 0x000c, 0xc74: 0x000c, 0xc75: 0x000c, + 0xc76: 0x000c, 0xc77: 0x000c, 0xc78: 0x000c, 0xc79: 0x000c, 0xc7a: 0x000c, 0xc7b: 0x000c, + 0xc7c: 0x000c, + // Block 0x32, offset 0xc80 + 0xc86: 0x000c, + // Block 0x33, offset 0xcc0 + 0xced: 0x000c, 0xcee: 0x000c, 0xcef: 0x000c, + 0xcf0: 0x000c, 0xcf2: 0x000c, 0xcf3: 0x000c, 0xcf4: 0x000c, 0xcf5: 0x000c, + 0xcf6: 0x000c, 0xcf7: 0x000c, 0xcf9: 0x000c, 0xcfa: 0x000c, + 0xcfd: 0x000c, 0xcfe: 0x000c, + // Block 0x34, offset 0xd00 + 0xd18: 0x000c, 0xd19: 0x000c, + 0xd1e: 0x000c, 0xd1f: 0x000c, 0xd20: 0x000c, + 0xd31: 0x000c, 0xd32: 0x000c, 0xd33: 0x000c, 0xd34: 0x000c, + // Block 0x35, offset 0xd40 + 0xd42: 0x000c, 0xd45: 0x000c, + 0xd46: 0x000c, + 0xd4d: 0x000c, + 0xd5d: 0x000c, + // Block 0x36, offset 0xd80 + 0xd9d: 0x000c, + 0xd9e: 0x000c, 0xd9f: 0x000c, + // Block 0x37, offset 0xdc0 + 0xdd0: 0x000a, 0xdd1: 0x000a, + 0xdd2: 0x000a, 0xdd3: 0x000a, 0xdd4: 0x000a, 0xdd5: 0x000a, 0xdd6: 0x000a, 0xdd7: 0x000a, + 0xdd8: 0x000a, 0xdd9: 0x000a, + // Block 0x38, offset 0xe00 + 0xe00: 0x000a, + // Block 0x39, offset 0xe40 + 0xe40: 0x0009, + 0xe5b: 0x007a, 0xe5c: 0x006a, + // Block 0x3a, offset 0xe80 + 0xe92: 0x000c, 0xe93: 0x000c, 0xe94: 0x000c, + 0xeb2: 0x000c, 0xeb3: 0x000c, 0xeb4: 0x000c, + // Block 0x3b, offset 0xec0 + 0xed2: 0x000c, 0xed3: 0x000c, + 0xef2: 0x000c, 0xef3: 0x000c, + // Block 0x3c, offset 0xf00 + 0xf34: 0x000c, 0xf35: 0x000c, + 0xf37: 0x000c, 0xf38: 0x000c, 0xf39: 0x000c, 0xf3a: 0x000c, 0xf3b: 0x000c, + 0xf3c: 0x000c, 0xf3d: 0x000c, + // Block 0x3d, offset 0xf40 + 0xf46: 0x000c, 0xf49: 0x000c, 0xf4a: 0x000c, 0xf4b: 0x000c, + 0xf4c: 0x000c, 0xf4d: 0x000c, 0xf4e: 0x000c, 0xf4f: 0x000c, 0xf50: 0x000c, 0xf51: 0x000c, + 0xf52: 0x000c, 0xf53: 0x000c, + 0xf5b: 0x0004, 0xf5d: 0x000c, + 0xf70: 0x000a, 0xf71: 0x000a, 0xf72: 0x000a, 0xf73: 0x000a, 0xf74: 0x000a, 0xf75: 0x000a, + 0xf76: 0x000a, 0xf77: 0x000a, 0xf78: 0x000a, 0xf79: 0x000a, + // Block 0x3e, offset 0xf80 + 0xf80: 0x000a, 0xf81: 0x000a, 0xf82: 0x000a, 0xf83: 0x000a, 0xf84: 0x000a, 0xf85: 0x000a, + 0xf86: 0x000a, 0xf87: 0x000a, 0xf88: 0x000a, 0xf89: 0x000a, 0xf8a: 0x000a, 0xf8b: 0x000c, + 0xf8c: 0x000c, 0xf8d: 0x000c, 0xf8e: 0x000b, + // Block 0x3f, offset 0xfc0 + 0xfc5: 0x000c, + 0xfc6: 0x000c, + 0xfe9: 0x000c, + // Block 0x40, offset 0x1000 + 0x1020: 0x000c, 0x1021: 0x000c, 0x1022: 0x000c, + 0x1027: 0x000c, 0x1028: 0x000c, + 0x1032: 0x000c, + 0x1039: 0x000c, 0x103a: 0x000c, 0x103b: 0x000c, + // Block 0x41, offset 0x1040 + 0x1040: 0x000a, 0x1044: 0x000a, 0x1045: 0x000a, + // Block 0x42, offset 0x1080 + 0x109e: 0x000a, 0x109f: 0x000a, 0x10a0: 0x000a, 0x10a1: 0x000a, 0x10a2: 0x000a, 0x10a3: 0x000a, + 0x10a4: 0x000a, 0x10a5: 0x000a, 0x10a6: 0x000a, 0x10a7: 0x000a, 0x10a8: 0x000a, 0x10a9: 0x000a, + 0x10aa: 0x000a, 0x10ab: 0x000a, 0x10ac: 0x000a, 0x10ad: 0x000a, 0x10ae: 0x000a, 0x10af: 0x000a, + 0x10b0: 0x000a, 0x10b1: 0x000a, 0x10b2: 0x000a, 0x10b3: 0x000a, 0x10b4: 0x000a, 0x10b5: 0x000a, + 0x10b6: 0x000a, 0x10b7: 0x000a, 0x10b8: 0x000a, 0x10b9: 0x000a, 0x10ba: 0x000a, 0x10bb: 0x000a, + 0x10bc: 0x000a, 0x10bd: 0x000a, 0x10be: 0x000a, 0x10bf: 0x000a, + // Block 0x43, offset 0x10c0 + 0x10d7: 0x000c, + 0x10d8: 0x000c, 0x10db: 0x000c, + // Block 0x44, offset 0x1100 + 0x1116: 0x000c, + 0x1118: 0x000c, 0x1119: 0x000c, 0x111a: 0x000c, 0x111b: 0x000c, 0x111c: 0x000c, 0x111d: 0x000c, + 0x111e: 0x000c, 0x1120: 0x000c, 0x1122: 0x000c, + 0x1125: 0x000c, 0x1126: 0x000c, 0x1127: 0x000c, 0x1128: 0x000c, 0x1129: 0x000c, + 0x112a: 0x000c, 0x112b: 0x000c, 0x112c: 0x000c, + 0x1133: 0x000c, 0x1134: 0x000c, 0x1135: 0x000c, + 0x1136: 0x000c, 0x1137: 0x000c, 0x1138: 0x000c, 0x1139: 0x000c, 0x113a: 0x000c, 0x113b: 0x000c, + 0x113c: 0x000c, 0x113f: 0x000c, + // Block 0x45, offset 0x1140 + 0x1170: 0x000c, 0x1171: 0x000c, 0x1172: 0x000c, 0x1173: 0x000c, 0x1174: 0x000c, 0x1175: 0x000c, + 0x1176: 0x000c, 0x1177: 0x000c, 0x1178: 0x000c, 0x1179: 0x000c, 0x117a: 0x000c, 0x117b: 0x000c, + 0x117c: 0x000c, 0x117d: 0x000c, 0x117e: 0x000c, + // Block 0x46, offset 0x1180 + 0x1180: 0x000c, 0x1181: 0x000c, 0x1182: 0x000c, 0x1183: 0x000c, + 0x11b4: 0x000c, + 0x11b6: 0x000c, 0x11b7: 0x000c, 0x11b8: 0x000c, 0x11b9: 0x000c, 0x11ba: 0x000c, + 0x11bc: 0x000c, + // Block 0x47, offset 0x11c0 + 0x11c2: 0x000c, + 0x11eb: 0x000c, 0x11ec: 0x000c, 0x11ed: 0x000c, 0x11ee: 0x000c, 0x11ef: 0x000c, + 0x11f0: 0x000c, 0x11f1: 0x000c, 0x11f2: 0x000c, 0x11f3: 0x000c, + // Block 0x48, offset 0x1200 + 0x1200: 0x000c, 0x1201: 0x000c, + 0x1222: 0x000c, 0x1223: 0x000c, + 0x1224: 0x000c, 0x1225: 0x000c, 0x1228: 0x000c, 0x1229: 0x000c, + 0x122b: 0x000c, 0x122c: 0x000c, 0x122d: 0x000c, + // Block 0x49, offset 0x1240 + 0x1266: 0x000c, 0x1268: 0x000c, 0x1269: 0x000c, + 0x126d: 0x000c, 0x126f: 0x000c, + 0x1270: 0x000c, 0x1271: 0x000c, + // Block 0x4a, offset 0x1280 + 0x12ac: 0x000c, 0x12ad: 0x000c, 0x12ae: 0x000c, 0x12af: 0x000c, + 0x12b0: 0x000c, 0x12b1: 0x000c, 0x12b2: 0x000c, 0x12b3: 0x000c, + 0x12b6: 0x000c, 0x12b7: 0x000c, + // Block 0x4b, offset 0x12c0 + 0x12d0: 0x000c, 0x12d1: 0x000c, + 0x12d2: 0x000c, 0x12d4: 0x000c, 0x12d5: 0x000c, 0x12d6: 0x000c, 0x12d7: 0x000c, + 0x12d8: 0x000c, 0x12d9: 0x000c, 0x12da: 0x000c, 0x12db: 0x000c, 0x12dc: 0x000c, 0x12dd: 0x000c, + 0x12de: 0x000c, 0x12df: 0x000c, 0x12e0: 0x000c, 0x12e2: 0x000c, 0x12e3: 0x000c, + 0x12e4: 0x000c, 0x12e5: 0x000c, 0x12e6: 0x000c, 0x12e7: 0x000c, 0x12e8: 0x000c, + 0x12ed: 0x000c, + 0x12f4: 0x000c, + 0x12f8: 0x000c, 0x12f9: 0x000c, + // Block 0x4c, offset 0x1300 + 0x1300: 0x000c, 0x1301: 0x000c, 0x1302: 0x000c, 0x1303: 0x000c, 0x1304: 0x000c, 0x1305: 0x000c, + 0x1306: 0x000c, 0x1307: 0x000c, 0x1308: 0x000c, 0x1309: 0x000c, 0x130a: 0x000c, 0x130b: 0x000c, + 0x130c: 0x000c, 0x130d: 0x000c, 0x130e: 0x000c, 0x130f: 0x000c, 0x1310: 0x000c, 0x1311: 0x000c, + 0x1312: 0x000c, 0x1313: 0x000c, 0x1314: 0x000c, 0x1315: 0x000c, 0x1316: 0x000c, 0x1317: 0x000c, + 0x1318: 0x000c, 0x1319: 0x000c, 0x131a: 0x000c, 0x131b: 0x000c, 0x131c: 0x000c, 0x131d: 0x000c, + 0x131e: 0x000c, 0x131f: 0x000c, 0x1320: 0x000c, 0x1321: 0x000c, 0x1322: 0x000c, 0x1323: 0x000c, + 0x1324: 0x000c, 0x1325: 0x000c, 0x1326: 0x000c, 0x1327: 0x000c, 0x1328: 0x000c, 0x1329: 0x000c, + 0x132a: 0x000c, 0x132b: 0x000c, 0x132c: 0x000c, 0x132d: 0x000c, 0x132e: 0x000c, 0x132f: 0x000c, + 0x1330: 0x000c, 0x1331: 0x000c, 0x1332: 0x000c, 0x1333: 0x000c, 0x1334: 0x000c, 0x1335: 0x000c, + 0x1336: 0x000c, 0x1337: 0x000c, 0x1338: 0x000c, 0x1339: 0x000c, 0x133b: 0x000c, + 0x133c: 0x000c, 0x133d: 0x000c, 0x133e: 0x000c, 0x133f: 0x000c, + // Block 0x4d, offset 0x1340 + 0x137d: 0x000a, 0x137f: 0x000a, + // Block 0x4e, offset 0x1380 + 0x1380: 0x000a, 0x1381: 0x000a, + 0x138d: 0x000a, 0x138e: 0x000a, 0x138f: 0x000a, + 0x139d: 0x000a, + 0x139e: 0x000a, 0x139f: 0x000a, + 0x13ad: 0x000a, 0x13ae: 0x000a, 0x13af: 0x000a, + 0x13bd: 0x000a, 0x13be: 0x000a, + // Block 0x4f, offset 0x13c0 + 0x13c0: 0x0009, 0x13c1: 0x0009, 0x13c2: 0x0009, 0x13c3: 0x0009, 0x13c4: 0x0009, 0x13c5: 0x0009, + 0x13c6: 0x0009, 0x13c7: 0x0009, 0x13c8: 0x0009, 0x13c9: 0x0009, 0x13ca: 0x0009, 0x13cb: 0x000b, + 0x13cc: 0x000b, 0x13cd: 0x000b, 0x13cf: 0x0001, 0x13d0: 0x000a, 0x13d1: 0x000a, + 0x13d2: 0x000a, 0x13d3: 0x000a, 0x13d4: 0x000a, 0x13d5: 0x000a, 0x13d6: 0x000a, 0x13d7: 0x000a, + 0x13d8: 0x000a, 0x13d9: 0x000a, 0x13da: 0x000a, 0x13db: 0x000a, 0x13dc: 0x000a, 0x13dd: 0x000a, + 0x13de: 0x000a, 0x13df: 0x000a, 0x13e0: 0x000a, 0x13e1: 0x000a, 0x13e2: 0x000a, 0x13e3: 0x000a, + 0x13e4: 0x000a, 0x13e5: 0x000a, 0x13e6: 0x000a, 0x13e7: 0x000a, 0x13e8: 0x0009, 0x13e9: 0x0007, + 0x13ea: 0x000e, 0x13eb: 0x000e, 0x13ec: 0x000e, 0x13ed: 0x000e, 0x13ee: 0x000e, 0x13ef: 0x0006, + 0x13f0: 0x0004, 0x13f1: 0x0004, 0x13f2: 0x0004, 0x13f3: 0x0004, 0x13f4: 0x0004, 0x13f5: 0x000a, + 0x13f6: 0x000a, 0x13f7: 0x000a, 0x13f8: 0x000a, 0x13f9: 0x000a, 0x13fa: 0x000a, 0x13fb: 0x000a, + 0x13fc: 0x000a, 0x13fd: 0x000a, 0x13fe: 0x000a, 0x13ff: 0x000a, + // Block 0x50, offset 0x1400 + 0x1400: 0x000a, 0x1401: 0x000a, 0x1402: 0x000a, 0x1403: 0x000a, 0x1404: 0x0006, 0x1405: 0x009a, + 0x1406: 0x008a, 0x1407: 0x000a, 0x1408: 0x000a, 0x1409: 0x000a, 0x140a: 0x000a, 0x140b: 0x000a, + 0x140c: 0x000a, 0x140d: 0x000a, 0x140e: 0x000a, 0x140f: 0x000a, 0x1410: 0x000a, 0x1411: 0x000a, + 0x1412: 0x000a, 0x1413: 0x000a, 0x1414: 0x000a, 0x1415: 0x000a, 0x1416: 0x000a, 0x1417: 0x000a, + 0x1418: 0x000a, 0x1419: 0x000a, 0x141a: 0x000a, 0x141b: 0x000a, 0x141c: 0x000a, 0x141d: 0x000a, + 0x141e: 0x000a, 0x141f: 0x0009, 0x1420: 0x000b, 0x1421: 0x000b, 0x1422: 0x000b, 0x1423: 0x000b, + 0x1424: 0x000b, 0x1425: 0x000b, 0x1426: 0x000e, 0x1427: 0x000e, 0x1428: 0x000e, 0x1429: 0x000e, + 0x142a: 0x000b, 0x142b: 0x000b, 0x142c: 0x000b, 0x142d: 0x000b, 0x142e: 0x000b, 0x142f: 0x000b, + 0x1430: 0x0002, 0x1434: 0x0002, 0x1435: 0x0002, + 0x1436: 0x0002, 0x1437: 0x0002, 0x1438: 0x0002, 0x1439: 0x0002, 0x143a: 0x0003, 0x143b: 0x0003, + 0x143c: 0x000a, 0x143d: 0x009a, 0x143e: 0x008a, + // Block 0x51, offset 0x1440 + 0x1440: 0x0002, 0x1441: 0x0002, 0x1442: 0x0002, 0x1443: 0x0002, 0x1444: 0x0002, 0x1445: 0x0002, + 0x1446: 0x0002, 0x1447: 0x0002, 0x1448: 0x0002, 0x1449: 0x0002, 0x144a: 0x0003, 0x144b: 0x0003, + 0x144c: 0x000a, 0x144d: 0x009a, 0x144e: 0x008a, + 0x1460: 0x0004, 0x1461: 0x0004, 0x1462: 0x0004, 0x1463: 0x0004, + 0x1464: 0x0004, 0x1465: 0x0004, 0x1466: 0x0004, 0x1467: 0x0004, 0x1468: 0x0004, 0x1469: 0x0004, + 0x146a: 0x0004, 0x146b: 0x0004, 0x146c: 0x0004, 0x146d: 0x0004, 0x146e: 0x0004, 0x146f: 0x0004, + 0x1470: 0x0004, 0x1471: 0x0004, 0x1472: 0x0004, 0x1473: 0x0004, 0x1474: 0x0004, 0x1475: 0x0004, + 0x1476: 0x0004, 0x1477: 0x0004, 0x1478: 0x0004, 0x1479: 0x0004, 0x147a: 0x0004, 0x147b: 0x0004, + 0x147c: 0x0004, 0x147d: 0x0004, 0x147e: 0x0004, 0x147f: 0x0004, + // Block 0x52, offset 0x1480 + 0x1480: 0x0004, 0x1481: 0x0004, 0x1482: 0x0004, 0x1483: 0x0004, 0x1484: 0x0004, 0x1485: 0x0004, + 0x1486: 0x0004, 0x1487: 0x0004, 0x1488: 0x0004, 0x1489: 0x0004, 0x148a: 0x0004, 0x148b: 0x0004, + 0x148c: 0x0004, 0x148d: 0x0004, 0x148e: 0x0004, 0x148f: 0x0004, 0x1490: 0x000c, 0x1491: 0x000c, + 0x1492: 0x000c, 0x1493: 0x000c, 0x1494: 0x000c, 0x1495: 0x000c, 0x1496: 0x000c, 0x1497: 0x000c, + 0x1498: 0x000c, 0x1499: 0x000c, 0x149a: 0x000c, 0x149b: 0x000c, 0x149c: 0x000c, 0x149d: 0x000c, + 0x149e: 0x000c, 0x149f: 0x000c, 0x14a0: 0x000c, 0x14a1: 0x000c, 0x14a2: 0x000c, 0x14a3: 0x000c, + 0x14a4: 0x000c, 0x14a5: 0x000c, 0x14a6: 0x000c, 0x14a7: 0x000c, 0x14a8: 0x000c, 0x14a9: 0x000c, + 0x14aa: 0x000c, 0x14ab: 0x000c, 0x14ac: 0x000c, 0x14ad: 0x000c, 0x14ae: 0x000c, 0x14af: 0x000c, + 0x14b0: 0x000c, + // Block 0x53, offset 0x14c0 + 0x14c0: 0x000a, 0x14c1: 0x000a, 0x14c3: 0x000a, 0x14c4: 0x000a, 0x14c5: 0x000a, + 0x14c6: 0x000a, 0x14c8: 0x000a, 0x14c9: 0x000a, + 0x14d4: 0x000a, 0x14d6: 0x000a, 0x14d7: 0x000a, + 0x14d8: 0x000a, + 0x14de: 0x000a, 0x14df: 0x000a, 0x14e0: 0x000a, 0x14e1: 0x000a, 0x14e2: 0x000a, 0x14e3: 0x000a, + 0x14e5: 0x000a, 0x14e7: 0x000a, 0x14e9: 0x000a, + 0x14ee: 0x0004, + 0x14fa: 0x000a, 0x14fb: 0x000a, + // Block 0x54, offset 0x1500 + 0x1500: 0x000a, 0x1501: 0x000a, 0x1502: 0x000a, 0x1503: 0x000a, 0x1504: 0x000a, + 0x150a: 0x000a, 0x150b: 0x000a, + 0x150c: 0x000a, 0x150d: 0x000a, 0x1510: 0x000a, 0x1511: 0x000a, + 0x1512: 0x000a, 0x1513: 0x000a, 0x1514: 0x000a, 0x1515: 0x000a, 0x1516: 0x000a, 0x1517: 0x000a, + 0x1518: 0x000a, 0x1519: 0x000a, 0x151a: 0x000a, 0x151b: 0x000a, 0x151c: 0x000a, 0x151d: 0x000a, + 0x151e: 0x000a, 0x151f: 0x000a, + // Block 0x55, offset 0x1540 + 0x1549: 0x000a, 0x154a: 0x000a, 0x154b: 0x000a, + 0x1550: 0x000a, 0x1551: 0x000a, + 0x1552: 0x000a, 0x1553: 0x000a, 0x1554: 0x000a, 0x1555: 0x000a, 0x1556: 0x000a, 0x1557: 0x000a, + 0x1558: 0x000a, 0x1559: 0x000a, 0x155a: 0x000a, 0x155b: 0x000a, 0x155c: 0x000a, 0x155d: 0x000a, + 0x155e: 0x000a, 0x155f: 0x000a, 0x1560: 0x000a, 0x1561: 0x000a, 0x1562: 0x000a, 0x1563: 0x000a, + 0x1564: 0x000a, 0x1565: 0x000a, 0x1566: 0x000a, 0x1567: 0x000a, 0x1568: 0x000a, 0x1569: 0x000a, + 0x156a: 0x000a, 0x156b: 0x000a, 0x156c: 0x000a, 0x156d: 0x000a, 0x156e: 0x000a, 0x156f: 0x000a, + 0x1570: 0x000a, 0x1571: 0x000a, 0x1572: 0x000a, 0x1573: 0x000a, 0x1574: 0x000a, 0x1575: 0x000a, + 0x1576: 0x000a, 0x1577: 0x000a, 0x1578: 0x000a, 0x1579: 0x000a, 0x157a: 0x000a, 0x157b: 0x000a, + 0x157c: 0x000a, 0x157d: 0x000a, 0x157e: 0x000a, 0x157f: 0x000a, + // Block 0x56, offset 0x1580 + 0x1580: 0x000a, 0x1581: 0x000a, 0x1582: 0x000a, 0x1583: 0x000a, 0x1584: 0x000a, 0x1585: 0x000a, + 0x1586: 0x000a, 0x1587: 0x000a, 0x1588: 0x000a, 0x1589: 0x000a, 0x158a: 0x000a, 0x158b: 0x000a, + 0x158c: 0x000a, 0x158d: 0x000a, 0x158e: 0x000a, 0x158f: 0x000a, 0x1590: 0x000a, 0x1591: 0x000a, + 0x1592: 0x000a, 0x1593: 0x000a, 0x1594: 0x000a, 0x1595: 0x000a, 0x1596: 0x000a, 0x1597: 0x000a, + 0x1598: 0x000a, 0x1599: 0x000a, 0x159a: 0x000a, 0x159b: 0x000a, 0x159c: 0x000a, 0x159d: 0x000a, + 0x159e: 0x000a, 0x159f: 0x000a, 0x15a0: 0x000a, 0x15a1: 0x000a, 0x15a2: 0x000a, 0x15a3: 0x000a, + 0x15a4: 0x000a, 0x15a5: 0x000a, 0x15a6: 0x000a, 0x15a7: 0x000a, 0x15a8: 0x000a, 0x15a9: 0x000a, + 0x15aa: 0x000a, 0x15ab: 0x000a, 0x15ac: 0x000a, 0x15ad: 0x000a, 0x15ae: 0x000a, 0x15af: 0x000a, + 0x15b0: 0x000a, 0x15b1: 0x000a, 0x15b2: 0x000a, 0x15b3: 0x000a, 0x15b4: 0x000a, 0x15b5: 0x000a, + 0x15b6: 0x000a, 0x15b7: 0x000a, 0x15b8: 0x000a, 0x15b9: 0x000a, 0x15ba: 0x000a, 0x15bb: 0x000a, + 0x15bc: 0x000a, 0x15bd: 0x000a, 0x15be: 0x000a, 0x15bf: 0x000a, + // Block 0x57, offset 0x15c0 + 0x15c0: 0x000a, 0x15c1: 0x000a, 0x15c2: 0x000a, 0x15c3: 0x000a, 0x15c4: 0x000a, 0x15c5: 0x000a, + 0x15c6: 0x000a, 0x15c7: 0x000a, 0x15c8: 0x000a, 0x15c9: 0x000a, 0x15ca: 0x000a, 0x15cb: 0x000a, + 0x15cc: 0x000a, 0x15cd: 0x000a, 0x15ce: 0x000a, 0x15cf: 0x000a, 0x15d0: 0x000a, 0x15d1: 0x000a, + 0x15d2: 0x0003, 0x15d3: 0x0004, 0x15d4: 0x000a, 0x15d5: 0x000a, 0x15d6: 0x000a, 0x15d7: 0x000a, + 0x15d8: 0x000a, 0x15d9: 0x000a, 0x15da: 0x000a, 0x15db: 0x000a, 0x15dc: 0x000a, 0x15dd: 0x000a, + 0x15de: 0x000a, 0x15df: 0x000a, 0x15e0: 0x000a, 0x15e1: 0x000a, 0x15e2: 0x000a, 0x15e3: 0x000a, + 0x15e4: 0x000a, 0x15e5: 0x000a, 0x15e6: 0x000a, 0x15e7: 0x000a, 0x15e8: 0x000a, 0x15e9: 0x000a, + 0x15ea: 0x000a, 0x15eb: 0x000a, 0x15ec: 0x000a, 0x15ed: 0x000a, 0x15ee: 0x000a, 0x15ef: 0x000a, + 0x15f0: 0x000a, 0x15f1: 0x000a, 0x15f2: 0x000a, 0x15f3: 0x000a, 0x15f4: 0x000a, 0x15f5: 0x000a, + 0x15f6: 0x000a, 0x15f7: 0x000a, 0x15f8: 0x000a, 0x15f9: 0x000a, 0x15fa: 0x000a, 0x15fb: 0x000a, + 0x15fc: 0x000a, 0x15fd: 0x000a, 0x15fe: 0x000a, 0x15ff: 0x000a, + // Block 0x58, offset 0x1600 + 0x1600: 0x000a, 0x1601: 0x000a, 0x1602: 0x000a, 0x1603: 0x000a, 0x1604: 0x000a, 0x1605: 0x000a, + 0x1606: 0x000a, 0x1607: 0x000a, 0x1608: 0x003a, 0x1609: 0x002a, 0x160a: 0x003a, 0x160b: 0x002a, + 0x160c: 0x000a, 0x160d: 0x000a, 0x160e: 0x000a, 0x160f: 0x000a, 0x1610: 0x000a, 0x1611: 0x000a, + 0x1612: 0x000a, 0x1613: 0x000a, 0x1614: 0x000a, 0x1615: 0x000a, 0x1616: 0x000a, 0x1617: 0x000a, + 0x1618: 0x000a, 0x1619: 0x000a, 0x161a: 0x000a, 0x161b: 0x000a, 0x161c: 0x000a, 0x161d: 0x000a, + 0x161e: 0x000a, 0x161f: 0x000a, 0x1620: 0x000a, 0x1621: 0x000a, 0x1622: 0x000a, 0x1623: 0x000a, + 0x1624: 0x000a, 0x1625: 0x000a, 0x1626: 0x000a, 0x1627: 0x000a, 0x1628: 0x000a, 0x1629: 0x009a, + 0x162a: 0x008a, 0x162b: 0x000a, 0x162c: 0x000a, 0x162d: 0x000a, 0x162e: 0x000a, 0x162f: 0x000a, + 0x1630: 0x000a, 0x1631: 0x000a, 0x1632: 0x000a, 0x1633: 0x000a, 0x1634: 0x000a, 0x1635: 0x000a, + // Block 0x59, offset 0x1640 + 0x167b: 0x000a, + 0x167c: 0x000a, 0x167d: 0x000a, 0x167e: 0x000a, 0x167f: 0x000a, + // Block 0x5a, offset 0x1680 + 0x1680: 0x000a, 0x1681: 0x000a, 0x1682: 0x000a, 0x1683: 0x000a, 0x1684: 0x000a, 0x1685: 0x000a, + 0x1686: 0x000a, 0x1687: 0x000a, 0x1688: 0x000a, 0x1689: 0x000a, 0x168a: 0x000a, 0x168b: 0x000a, + 0x168c: 0x000a, 0x168d: 0x000a, 0x168e: 0x000a, 0x168f: 0x000a, 0x1690: 0x000a, 0x1691: 0x000a, + 0x1692: 0x000a, 0x1693: 0x000a, 0x1694: 0x000a, 0x1696: 0x000a, 0x1697: 0x000a, + 0x1698: 0x000a, 0x1699: 0x000a, 0x169a: 0x000a, 0x169b: 0x000a, 0x169c: 0x000a, 0x169d: 0x000a, + 0x169e: 0x000a, 0x169f: 0x000a, 0x16a0: 0x000a, 0x16a1: 0x000a, 0x16a2: 0x000a, 0x16a3: 0x000a, + 0x16a4: 0x000a, 0x16a5: 0x000a, 0x16a6: 0x000a, 0x16a7: 0x000a, 0x16a8: 0x000a, 0x16a9: 0x000a, + 0x16aa: 0x000a, 0x16ab: 0x000a, 0x16ac: 0x000a, 0x16ad: 0x000a, 0x16ae: 0x000a, 0x16af: 0x000a, + 0x16b0: 0x000a, 0x16b1: 0x000a, 0x16b2: 0x000a, 0x16b3: 0x000a, 0x16b4: 0x000a, 0x16b5: 0x000a, + 0x16b6: 0x000a, 0x16b7: 0x000a, 0x16b8: 0x000a, 0x16b9: 0x000a, 0x16ba: 0x000a, 0x16bb: 0x000a, + 0x16bc: 0x000a, 0x16bd: 0x000a, 0x16be: 0x000a, 0x16bf: 0x000a, + // Block 0x5b, offset 0x16c0 + 0x16c0: 0x000a, 0x16c1: 0x000a, 0x16c2: 0x000a, 0x16c3: 0x000a, 0x16c4: 0x000a, 0x16c5: 0x000a, + 0x16c6: 0x000a, 0x16c7: 0x000a, 0x16c8: 0x000a, 0x16c9: 0x000a, 0x16ca: 0x000a, 0x16cb: 0x000a, + 0x16cc: 0x000a, 0x16cd: 0x000a, 0x16ce: 0x000a, 0x16cf: 0x000a, 0x16d0: 0x000a, 0x16d1: 0x000a, + 0x16d2: 0x000a, 0x16d3: 0x000a, 0x16d4: 0x000a, 0x16d5: 0x000a, 0x16d6: 0x000a, 0x16d7: 0x000a, + 0x16d8: 0x000a, 0x16d9: 0x000a, 0x16da: 0x000a, 0x16db: 0x000a, 0x16dc: 0x000a, 0x16dd: 0x000a, + 0x16de: 0x000a, 0x16df: 0x000a, 0x16e0: 0x000a, 0x16e1: 0x000a, 0x16e2: 0x000a, 0x16e3: 0x000a, + 0x16e4: 0x000a, 0x16e5: 0x000a, 0x16e6: 0x000a, + // Block 0x5c, offset 0x1700 + 0x1700: 0x000a, 0x1701: 0x000a, 0x1702: 0x000a, 0x1703: 0x000a, 0x1704: 0x000a, 0x1705: 0x000a, + 0x1706: 0x000a, 0x1707: 0x000a, 0x1708: 0x000a, 0x1709: 0x000a, 0x170a: 0x000a, + 0x1720: 0x000a, 0x1721: 0x000a, 0x1722: 0x000a, 0x1723: 0x000a, + 0x1724: 0x000a, 0x1725: 0x000a, 0x1726: 0x000a, 0x1727: 0x000a, 0x1728: 0x000a, 0x1729: 0x000a, + 0x172a: 0x000a, 0x172b: 0x000a, 0x172c: 0x000a, 0x172d: 0x000a, 0x172e: 0x000a, 0x172f: 0x000a, + 0x1730: 0x000a, 0x1731: 0x000a, 0x1732: 0x000a, 0x1733: 0x000a, 0x1734: 0x000a, 0x1735: 0x000a, + 0x1736: 0x000a, 0x1737: 0x000a, 0x1738: 0x000a, 0x1739: 0x000a, 0x173a: 0x000a, 0x173b: 0x000a, + 0x173c: 0x000a, 0x173d: 0x000a, 0x173e: 0x000a, 0x173f: 0x000a, + // Block 0x5d, offset 0x1740 + 0x1740: 0x000a, 0x1741: 0x000a, 0x1742: 0x000a, 0x1743: 0x000a, 0x1744: 0x000a, 0x1745: 0x000a, + 0x1746: 0x000a, 0x1747: 0x000a, 0x1748: 0x0002, 0x1749: 0x0002, 0x174a: 0x0002, 0x174b: 0x0002, + 0x174c: 0x0002, 0x174d: 0x0002, 0x174e: 0x0002, 0x174f: 0x0002, 0x1750: 0x0002, 0x1751: 0x0002, + 0x1752: 0x0002, 0x1753: 0x0002, 0x1754: 0x0002, 0x1755: 0x0002, 0x1756: 0x0002, 0x1757: 0x0002, + 0x1758: 0x0002, 0x1759: 0x0002, 0x175a: 0x0002, 0x175b: 0x0002, + // Block 0x5e, offset 0x1780 + 0x17aa: 0x000a, 0x17ab: 0x000a, 0x17ac: 0x000a, 0x17ad: 0x000a, 0x17ae: 0x000a, 0x17af: 0x000a, + 0x17b0: 0x000a, 0x17b1: 0x000a, 0x17b2: 0x000a, 0x17b3: 0x000a, 0x17b4: 0x000a, 0x17b5: 0x000a, + 0x17b6: 0x000a, 0x17b7: 0x000a, 0x17b8: 0x000a, 0x17b9: 0x000a, 0x17ba: 0x000a, 0x17bb: 0x000a, + 0x17bc: 0x000a, 0x17bd: 0x000a, 0x17be: 0x000a, 0x17bf: 0x000a, + // Block 0x5f, offset 0x17c0 + 0x17c0: 0x000a, 0x17c1: 0x000a, 0x17c2: 0x000a, 0x17c3: 0x000a, 0x17c4: 0x000a, 0x17c5: 0x000a, + 0x17c6: 0x000a, 0x17c7: 0x000a, 0x17c8: 0x000a, 0x17c9: 0x000a, 0x17ca: 0x000a, 0x17cb: 0x000a, + 0x17cc: 0x000a, 0x17cd: 0x000a, 0x17ce: 0x000a, 0x17cf: 0x000a, 0x17d0: 0x000a, 0x17d1: 0x000a, + 0x17d2: 0x000a, 0x17d3: 0x000a, 0x17d4: 0x000a, 0x17d5: 0x000a, 0x17d6: 0x000a, 0x17d7: 0x000a, + 0x17d8: 0x000a, 0x17d9: 0x000a, 0x17da: 0x000a, 0x17db: 0x000a, 0x17dc: 0x000a, 0x17dd: 0x000a, + 0x17de: 0x000a, 0x17df: 0x000a, 0x17e0: 0x000a, 0x17e1: 0x000a, 0x17e2: 0x000a, 0x17e3: 0x000a, + 0x17e4: 0x000a, 0x17e5: 0x000a, 0x17e6: 0x000a, 0x17e7: 0x000a, 0x17e8: 0x000a, 0x17e9: 0x000a, + 0x17ea: 0x000a, 0x17eb: 0x000a, 0x17ed: 0x000a, 0x17ee: 0x000a, 0x17ef: 0x000a, + 0x17f0: 0x000a, 0x17f1: 0x000a, 0x17f2: 0x000a, 0x17f3: 0x000a, 0x17f4: 0x000a, 0x17f5: 0x000a, + 0x17f6: 0x000a, 0x17f7: 0x000a, 0x17f8: 0x000a, 0x17f9: 0x000a, 0x17fa: 0x000a, 0x17fb: 0x000a, + 0x17fc: 0x000a, 0x17fd: 0x000a, 0x17fe: 0x000a, 0x17ff: 0x000a, + // Block 0x60, offset 0x1800 + 0x1800: 0x000a, 0x1801: 0x000a, 0x1802: 0x000a, 0x1803: 0x000a, 0x1804: 0x000a, 0x1805: 0x000a, + 0x1806: 0x000a, 0x1807: 0x000a, 0x1808: 0x000a, 0x1809: 0x000a, 0x180a: 0x000a, 0x180b: 0x000a, + 0x180c: 0x000a, 0x180d: 0x000a, 0x180e: 0x000a, 0x180f: 0x000a, 0x1810: 0x000a, 0x1811: 0x000a, + 0x1812: 0x000a, 0x1813: 0x000a, 0x1814: 0x000a, 0x1815: 0x000a, 0x1816: 0x000a, 0x1817: 0x000a, + 0x1818: 0x000a, 0x1819: 0x000a, 0x181a: 0x000a, 0x181b: 0x000a, 0x181c: 0x000a, 0x181d: 0x000a, + 0x181e: 0x000a, 0x181f: 0x000a, 0x1820: 0x000a, 0x1821: 0x000a, 0x1822: 0x000a, 0x1823: 0x000a, + 0x1824: 0x000a, 0x1825: 0x000a, 0x1826: 0x000a, 0x1827: 0x000a, 0x1828: 0x003a, 0x1829: 0x002a, + 0x182a: 0x003a, 0x182b: 0x002a, 0x182c: 0x003a, 0x182d: 0x002a, 0x182e: 0x003a, 0x182f: 0x002a, + 0x1830: 0x003a, 0x1831: 0x002a, 0x1832: 0x003a, 0x1833: 0x002a, 0x1834: 0x003a, 0x1835: 0x002a, + 0x1836: 0x000a, 0x1837: 0x000a, 0x1838: 0x000a, 0x1839: 0x000a, 0x183a: 0x000a, 0x183b: 0x000a, + 0x183c: 0x000a, 0x183d: 0x000a, 0x183e: 0x000a, 0x183f: 0x000a, + // Block 0x61, offset 0x1840 + 0x1840: 0x000a, 0x1841: 0x000a, 0x1842: 0x000a, 0x1843: 0x000a, 0x1844: 0x000a, 0x1845: 0x009a, + 0x1846: 0x008a, 0x1847: 0x000a, 0x1848: 0x000a, 0x1849: 0x000a, 0x184a: 0x000a, 0x184b: 0x000a, + 0x184c: 0x000a, 0x184d: 0x000a, 0x184e: 0x000a, 0x184f: 0x000a, 0x1850: 0x000a, 0x1851: 0x000a, + 0x1852: 0x000a, 0x1853: 0x000a, 0x1854: 0x000a, 0x1855: 0x000a, 0x1856: 0x000a, 0x1857: 0x000a, + 0x1858: 0x000a, 0x1859: 0x000a, 0x185a: 0x000a, 0x185b: 0x000a, 0x185c: 0x000a, 0x185d: 0x000a, + 0x185e: 0x000a, 0x185f: 0x000a, 0x1860: 0x000a, 0x1861: 0x000a, 0x1862: 0x000a, 0x1863: 0x000a, + 0x1864: 0x000a, 0x1865: 0x000a, 0x1866: 0x003a, 0x1867: 0x002a, 0x1868: 0x003a, 0x1869: 0x002a, + 0x186a: 0x003a, 0x186b: 0x002a, 0x186c: 0x003a, 0x186d: 0x002a, 0x186e: 0x003a, 0x186f: 0x002a, + 0x1870: 0x000a, 0x1871: 0x000a, 0x1872: 0x000a, 0x1873: 0x000a, 0x1874: 0x000a, 0x1875: 0x000a, + 0x1876: 0x000a, 0x1877: 0x000a, 0x1878: 0x000a, 0x1879: 0x000a, 0x187a: 0x000a, 0x187b: 0x000a, + 0x187c: 0x000a, 0x187d: 0x000a, 0x187e: 0x000a, 0x187f: 0x000a, + // Block 0x62, offset 0x1880 + 0x1880: 0x000a, 0x1881: 0x000a, 0x1882: 0x000a, 0x1883: 0x007a, 0x1884: 0x006a, 0x1885: 0x009a, + 0x1886: 0x008a, 0x1887: 0x00ba, 0x1888: 0x00aa, 0x1889: 0x009a, 0x188a: 0x008a, 0x188b: 0x007a, + 0x188c: 0x006a, 0x188d: 0x00da, 0x188e: 0x002a, 0x188f: 0x003a, 0x1890: 0x00ca, 0x1891: 0x009a, + 0x1892: 0x008a, 0x1893: 0x007a, 0x1894: 0x006a, 0x1895: 0x009a, 0x1896: 0x008a, 0x1897: 0x00ba, + 0x1898: 0x00aa, 0x1899: 0x000a, 0x189a: 0x000a, 0x189b: 0x000a, 0x189c: 0x000a, 0x189d: 0x000a, + 0x189e: 0x000a, 0x189f: 0x000a, 0x18a0: 0x000a, 0x18a1: 0x000a, 0x18a2: 0x000a, 0x18a3: 0x000a, + 0x18a4: 0x000a, 0x18a5: 0x000a, 0x18a6: 0x000a, 0x18a7: 0x000a, 0x18a8: 0x000a, 0x18a9: 0x000a, + 0x18aa: 0x000a, 0x18ab: 0x000a, 0x18ac: 0x000a, 0x18ad: 0x000a, 0x18ae: 0x000a, 0x18af: 0x000a, + 0x18b0: 0x000a, 0x18b1: 0x000a, 0x18b2: 0x000a, 0x18b3: 0x000a, 0x18b4: 0x000a, 0x18b5: 0x000a, + 0x18b6: 0x000a, 0x18b7: 0x000a, 0x18b8: 0x000a, 0x18b9: 0x000a, 0x18ba: 0x000a, 0x18bb: 0x000a, + 0x18bc: 0x000a, 0x18bd: 0x000a, 0x18be: 0x000a, 0x18bf: 0x000a, + // Block 0x63, offset 0x18c0 + 0x18c0: 0x000a, 0x18c1: 0x000a, 0x18c2: 0x000a, 0x18c3: 0x000a, 0x18c4: 0x000a, 0x18c5: 0x000a, + 0x18c6: 0x000a, 0x18c7: 0x000a, 0x18c8: 0x000a, 0x18c9: 0x000a, 0x18ca: 0x000a, 0x18cb: 0x000a, + 0x18cc: 0x000a, 0x18cd: 0x000a, 0x18ce: 0x000a, 0x18cf: 0x000a, 0x18d0: 0x000a, 0x18d1: 0x000a, + 0x18d2: 0x000a, 0x18d3: 0x000a, 0x18d4: 0x000a, 0x18d5: 0x000a, 0x18d6: 0x000a, 0x18d7: 0x000a, + 0x18d8: 0x003a, 0x18d9: 0x002a, 0x18da: 0x003a, 0x18db: 0x002a, 0x18dc: 0x000a, 0x18dd: 0x000a, + 0x18de: 0x000a, 0x18df: 0x000a, 0x18e0: 0x000a, 0x18e1: 0x000a, 0x18e2: 0x000a, 0x18e3: 0x000a, + 0x18e4: 0x000a, 0x18e5: 0x000a, 0x18e6: 0x000a, 0x18e7: 0x000a, 0x18e8: 0x000a, 0x18e9: 0x000a, + 0x18ea: 0x000a, 0x18eb: 0x000a, 0x18ec: 0x000a, 0x18ed: 0x000a, 0x18ee: 0x000a, 0x18ef: 0x000a, + 0x18f0: 0x000a, 0x18f1: 0x000a, 0x18f2: 0x000a, 0x18f3: 0x000a, 0x18f4: 0x000a, 0x18f5: 0x000a, + 0x18f6: 0x000a, 0x18f7: 0x000a, 0x18f8: 0x000a, 0x18f9: 0x000a, 0x18fa: 0x000a, 0x18fb: 0x000a, + 0x18fc: 0x003a, 0x18fd: 0x002a, 0x18fe: 0x000a, 0x18ff: 0x000a, + // Block 0x64, offset 0x1900 + 0x1900: 0x000a, 0x1901: 0x000a, 0x1902: 0x000a, 0x1903: 0x000a, 0x1904: 0x000a, 0x1905: 0x000a, + 0x1906: 0x000a, 0x1907: 0x000a, 0x1908: 0x000a, 0x1909: 0x000a, 0x190a: 0x000a, 0x190b: 0x000a, + 0x190c: 0x000a, 0x190d: 0x000a, 0x190e: 0x000a, 0x190f: 0x000a, 0x1910: 0x000a, 0x1911: 0x000a, + 0x1912: 0x000a, 0x1913: 0x000a, 0x1914: 0x000a, 0x1915: 0x000a, 0x1916: 0x000a, 0x1917: 0x000a, + 0x1918: 0x000a, 0x1919: 0x000a, 0x191a: 0x000a, 0x191b: 0x000a, 0x191c: 0x000a, 0x191d: 0x000a, + 0x191e: 0x000a, 0x191f: 0x000a, 0x1920: 0x000a, 0x1921: 0x000a, 0x1922: 0x000a, 0x1923: 0x000a, + 0x1924: 0x000a, 0x1925: 0x000a, 0x1926: 0x000a, 0x1927: 0x000a, 0x1928: 0x000a, 0x1929: 0x000a, + 0x192a: 0x000a, 0x192b: 0x000a, 0x192c: 0x000a, 0x192d: 0x000a, 0x192e: 0x000a, 0x192f: 0x000a, + 0x1930: 0x000a, 0x1931: 0x000a, 0x1932: 0x000a, 0x1933: 0x000a, + 0x1936: 0x000a, 0x1937: 0x000a, 0x1938: 0x000a, 0x1939: 0x000a, 0x193a: 0x000a, 0x193b: 0x000a, + 0x193c: 0x000a, 0x193d: 0x000a, 0x193e: 0x000a, 0x193f: 0x000a, + // Block 0x65, offset 0x1940 + 0x1940: 0x000a, 0x1941: 0x000a, 0x1942: 0x000a, 0x1943: 0x000a, 0x1944: 0x000a, 0x1945: 0x000a, + 0x1946: 0x000a, 0x1947: 0x000a, 0x1948: 0x000a, 0x1949: 0x000a, 0x194a: 0x000a, 0x194b: 0x000a, + 0x194c: 0x000a, 0x194d: 0x000a, 0x194e: 0x000a, 0x194f: 0x000a, 0x1950: 0x000a, 0x1951: 0x000a, + 0x1952: 0x000a, 0x1953: 0x000a, 0x1954: 0x000a, 0x1955: 0x000a, + 0x1958: 0x000a, 0x1959: 0x000a, 0x195a: 0x000a, 0x195b: 0x000a, 0x195c: 0x000a, 0x195d: 0x000a, + 0x195e: 0x000a, 0x195f: 0x000a, 0x1960: 0x000a, 0x1961: 0x000a, 0x1962: 0x000a, 0x1963: 0x000a, + 0x1964: 0x000a, 0x1965: 0x000a, 0x1966: 0x000a, 0x1967: 0x000a, 0x1968: 0x000a, 0x1969: 0x000a, + 0x196a: 0x000a, 0x196b: 0x000a, 0x196c: 0x000a, 0x196d: 0x000a, 0x196e: 0x000a, 0x196f: 0x000a, + 0x1970: 0x000a, 0x1971: 0x000a, 0x1972: 0x000a, 0x1973: 0x000a, 0x1974: 0x000a, 0x1975: 0x000a, + 0x1976: 0x000a, 0x1977: 0x000a, 0x1978: 0x000a, 0x1979: 0x000a, 0x197a: 0x000a, 0x197b: 0x000a, + 0x197c: 0x000a, 0x197d: 0x000a, 0x197e: 0x000a, 0x197f: 0x000a, + // Block 0x66, offset 0x1980 + 0x19a5: 0x000a, 0x19a6: 0x000a, 0x19a7: 0x000a, 0x19a8: 0x000a, 0x19a9: 0x000a, + 0x19aa: 0x000a, 0x19af: 0x000c, + 0x19b0: 0x000c, 0x19b1: 0x000c, + 0x19b9: 0x000a, 0x19ba: 0x000a, 0x19bb: 0x000a, + 0x19bc: 0x000a, 0x19bd: 0x000a, 0x19be: 0x000a, 0x19bf: 0x000a, + // Block 0x67, offset 0x19c0 + 0x19ff: 0x000c, + // Block 0x68, offset 0x1a00 + 0x1a20: 0x000c, 0x1a21: 0x000c, 0x1a22: 0x000c, 0x1a23: 0x000c, + 0x1a24: 0x000c, 0x1a25: 0x000c, 0x1a26: 0x000c, 0x1a27: 0x000c, 0x1a28: 0x000c, 0x1a29: 0x000c, + 0x1a2a: 0x000c, 0x1a2b: 0x000c, 0x1a2c: 0x000c, 0x1a2d: 0x000c, 0x1a2e: 0x000c, 0x1a2f: 0x000c, + 0x1a30: 0x000c, 0x1a31: 0x000c, 0x1a32: 0x000c, 0x1a33: 0x000c, 0x1a34: 0x000c, 0x1a35: 0x000c, + 0x1a36: 0x000c, 0x1a37: 0x000c, 0x1a38: 0x000c, 0x1a39: 0x000c, 0x1a3a: 0x000c, 0x1a3b: 0x000c, + 0x1a3c: 0x000c, 0x1a3d: 0x000c, 0x1a3e: 0x000c, 0x1a3f: 0x000c, + // Block 0x69, offset 0x1a40 + 0x1a40: 0x000a, 0x1a41: 0x000a, 0x1a42: 0x000a, 0x1a43: 0x000a, 0x1a44: 0x000a, 0x1a45: 0x000a, + 0x1a46: 0x000a, 0x1a47: 0x000a, 0x1a48: 0x000a, 0x1a49: 0x000a, 0x1a4a: 0x000a, 0x1a4b: 0x000a, + 0x1a4c: 0x000a, 0x1a4d: 0x000a, 0x1a4e: 0x000a, 0x1a4f: 0x000a, 0x1a50: 0x000a, 0x1a51: 0x000a, + 0x1a52: 0x000a, 0x1a53: 0x000a, 0x1a54: 0x000a, 0x1a55: 0x000a, 0x1a56: 0x000a, 0x1a57: 0x000a, + 0x1a58: 0x000a, 0x1a59: 0x000a, 0x1a5a: 0x000a, 0x1a5b: 0x000a, 0x1a5c: 0x000a, 0x1a5d: 0x000a, + 0x1a5e: 0x000a, 0x1a5f: 0x000a, 0x1a60: 0x000a, 0x1a61: 0x000a, 0x1a62: 0x003a, 0x1a63: 0x002a, + 0x1a64: 0x003a, 0x1a65: 0x002a, 0x1a66: 0x003a, 0x1a67: 0x002a, 0x1a68: 0x003a, 0x1a69: 0x002a, + 0x1a6a: 0x000a, 0x1a6b: 0x000a, 0x1a6c: 0x000a, 0x1a6d: 0x000a, 0x1a6e: 0x000a, 0x1a6f: 0x000a, + 0x1a70: 0x000a, 0x1a71: 0x000a, 0x1a72: 0x000a, 0x1a73: 0x000a, 0x1a74: 0x000a, 0x1a75: 0x000a, + 0x1a76: 0x000a, 0x1a77: 0x000a, 0x1a78: 0x000a, 0x1a79: 0x000a, 0x1a7a: 0x000a, 0x1a7b: 0x000a, + 0x1a7c: 0x000a, 0x1a7d: 0x000a, 0x1a7e: 0x000a, 0x1a7f: 0x000a, + // Block 0x6a, offset 0x1a80 + 0x1a80: 0x000a, 0x1a81: 0x000a, 0x1a82: 0x000a, 0x1a83: 0x000a, 0x1a84: 0x000a, 0x1a85: 0x000a, + 0x1a86: 0x000a, 0x1a87: 0x000a, 0x1a88: 0x000a, 0x1a89: 0x000a, 0x1a8a: 0x000a, 0x1a8b: 0x000a, + 0x1a8c: 0x000a, 0x1a8d: 0x000a, 0x1a8e: 0x000a, 0x1a8f: 0x000a, + // Block 0x6b, offset 0x1ac0 + 0x1ac0: 0x000a, 0x1ac1: 0x000a, 0x1ac2: 0x000a, 0x1ac3: 0x000a, 0x1ac4: 0x000a, 0x1ac5: 0x000a, + 0x1ac6: 0x000a, 0x1ac7: 0x000a, 0x1ac8: 0x000a, 0x1ac9: 0x000a, 0x1aca: 0x000a, 0x1acb: 0x000a, + 0x1acc: 0x000a, 0x1acd: 0x000a, 0x1ace: 0x000a, 0x1acf: 0x000a, 0x1ad0: 0x000a, 0x1ad1: 0x000a, + 0x1ad2: 0x000a, 0x1ad3: 0x000a, 0x1ad4: 0x000a, 0x1ad5: 0x000a, 0x1ad6: 0x000a, 0x1ad7: 0x000a, + 0x1ad8: 0x000a, 0x1ad9: 0x000a, 0x1adb: 0x000a, 0x1adc: 0x000a, 0x1add: 0x000a, + 0x1ade: 0x000a, 0x1adf: 0x000a, 0x1ae0: 0x000a, 0x1ae1: 0x000a, 0x1ae2: 0x000a, 0x1ae3: 0x000a, + 0x1ae4: 0x000a, 0x1ae5: 0x000a, 0x1ae6: 0x000a, 0x1ae7: 0x000a, 0x1ae8: 0x000a, 0x1ae9: 0x000a, + 0x1aea: 0x000a, 0x1aeb: 0x000a, 0x1aec: 0x000a, 0x1aed: 0x000a, 0x1aee: 0x000a, 0x1aef: 0x000a, + 0x1af0: 0x000a, 0x1af1: 0x000a, 0x1af2: 0x000a, 0x1af3: 0x000a, 0x1af4: 0x000a, 0x1af5: 0x000a, + 0x1af6: 0x000a, 0x1af7: 0x000a, 0x1af8: 0x000a, 0x1af9: 0x000a, 0x1afa: 0x000a, 0x1afb: 0x000a, + 0x1afc: 0x000a, 0x1afd: 0x000a, 0x1afe: 0x000a, 0x1aff: 0x000a, + // Block 0x6c, offset 0x1b00 + 0x1b00: 0x000a, 0x1b01: 0x000a, 0x1b02: 0x000a, 0x1b03: 0x000a, 0x1b04: 0x000a, 0x1b05: 0x000a, + 0x1b06: 0x000a, 0x1b07: 0x000a, 0x1b08: 0x000a, 0x1b09: 0x000a, 0x1b0a: 0x000a, 0x1b0b: 0x000a, + 0x1b0c: 0x000a, 0x1b0d: 0x000a, 0x1b0e: 0x000a, 0x1b0f: 0x000a, 0x1b10: 0x000a, 0x1b11: 0x000a, + 0x1b12: 0x000a, 0x1b13: 0x000a, 0x1b14: 0x000a, 0x1b15: 0x000a, 0x1b16: 0x000a, 0x1b17: 0x000a, + 0x1b18: 0x000a, 0x1b19: 0x000a, 0x1b1a: 0x000a, 0x1b1b: 0x000a, 0x1b1c: 0x000a, 0x1b1d: 0x000a, + 0x1b1e: 0x000a, 0x1b1f: 0x000a, 0x1b20: 0x000a, 0x1b21: 0x000a, 0x1b22: 0x000a, 0x1b23: 0x000a, + 0x1b24: 0x000a, 0x1b25: 0x000a, 0x1b26: 0x000a, 0x1b27: 0x000a, 0x1b28: 0x000a, 0x1b29: 0x000a, + 0x1b2a: 0x000a, 0x1b2b: 0x000a, 0x1b2c: 0x000a, 0x1b2d: 0x000a, 0x1b2e: 0x000a, 0x1b2f: 0x000a, + 0x1b30: 0x000a, 0x1b31: 0x000a, 0x1b32: 0x000a, 0x1b33: 0x000a, + // Block 0x6d, offset 0x1b40 + 0x1b40: 0x000a, 0x1b41: 0x000a, 0x1b42: 0x000a, 0x1b43: 0x000a, 0x1b44: 0x000a, 0x1b45: 0x000a, + 0x1b46: 0x000a, 0x1b47: 0x000a, 0x1b48: 0x000a, 0x1b49: 0x000a, 0x1b4a: 0x000a, 0x1b4b: 0x000a, + 0x1b4c: 0x000a, 0x1b4d: 0x000a, 0x1b4e: 0x000a, 0x1b4f: 0x000a, 0x1b50: 0x000a, 0x1b51: 0x000a, + 0x1b52: 0x000a, 0x1b53: 0x000a, 0x1b54: 0x000a, 0x1b55: 0x000a, + 0x1b70: 0x000a, 0x1b71: 0x000a, 0x1b72: 0x000a, 0x1b73: 0x000a, 0x1b74: 0x000a, 0x1b75: 0x000a, + 0x1b76: 0x000a, 0x1b77: 0x000a, 0x1b78: 0x000a, 0x1b79: 0x000a, 0x1b7a: 0x000a, 0x1b7b: 0x000a, + // Block 0x6e, offset 0x1b80 + 0x1b80: 0x0009, 0x1b81: 0x000a, 0x1b82: 0x000a, 0x1b83: 0x000a, 0x1b84: 0x000a, + 0x1b88: 0x003a, 0x1b89: 0x002a, 0x1b8a: 0x003a, 0x1b8b: 0x002a, + 0x1b8c: 0x003a, 0x1b8d: 0x002a, 0x1b8e: 0x003a, 0x1b8f: 0x002a, 0x1b90: 0x003a, 0x1b91: 0x002a, + 0x1b92: 0x000a, 0x1b93: 0x000a, 0x1b94: 0x003a, 0x1b95: 0x002a, 0x1b96: 0x003a, 0x1b97: 0x002a, + 0x1b98: 0x003a, 0x1b99: 0x002a, 0x1b9a: 0x003a, 0x1b9b: 0x002a, 0x1b9c: 0x000a, 0x1b9d: 0x000a, + 0x1b9e: 0x000a, 0x1b9f: 0x000a, 0x1ba0: 0x000a, + 0x1baa: 0x000c, 0x1bab: 0x000c, 0x1bac: 0x000c, 0x1bad: 0x000c, + 0x1bb0: 0x000a, + 0x1bb6: 0x000a, 0x1bb7: 0x000a, + 0x1bbd: 0x000a, 0x1bbe: 0x000a, 0x1bbf: 0x000a, + // Block 0x6f, offset 0x1bc0 + 0x1bd9: 0x000c, 0x1bda: 0x000c, 0x1bdb: 0x000a, 0x1bdc: 0x000a, + 0x1be0: 0x000a, + // Block 0x70, offset 0x1c00 + 0x1c3b: 0x000a, + // Block 0x71, offset 0x1c40 + 0x1c40: 0x000a, 0x1c41: 0x000a, 0x1c42: 0x000a, 0x1c43: 0x000a, 0x1c44: 0x000a, 0x1c45: 0x000a, + 0x1c46: 0x000a, 0x1c47: 0x000a, 0x1c48: 0x000a, 0x1c49: 0x000a, 0x1c4a: 0x000a, 0x1c4b: 0x000a, + 0x1c4c: 0x000a, 0x1c4d: 0x000a, 0x1c4e: 0x000a, 0x1c4f: 0x000a, 0x1c50: 0x000a, 0x1c51: 0x000a, + 0x1c52: 0x000a, 0x1c53: 0x000a, 0x1c54: 0x000a, 0x1c55: 0x000a, 0x1c56: 0x000a, 0x1c57: 0x000a, + 0x1c58: 0x000a, 0x1c59: 0x000a, 0x1c5a: 0x000a, 0x1c5b: 0x000a, 0x1c5c: 0x000a, 0x1c5d: 0x000a, + 0x1c5e: 0x000a, 0x1c5f: 0x000a, 0x1c60: 0x000a, 0x1c61: 0x000a, 0x1c62: 0x000a, 0x1c63: 0x000a, + // Block 0x72, offset 0x1c80 + 0x1c9d: 0x000a, + 0x1c9e: 0x000a, + // Block 0x73, offset 0x1cc0 + 0x1cd0: 0x000a, 0x1cd1: 0x000a, + 0x1cd2: 0x000a, 0x1cd3: 0x000a, 0x1cd4: 0x000a, 0x1cd5: 0x000a, 0x1cd6: 0x000a, 0x1cd7: 0x000a, + 0x1cd8: 0x000a, 0x1cd9: 0x000a, 0x1cda: 0x000a, 0x1cdb: 0x000a, 0x1cdc: 0x000a, 0x1cdd: 0x000a, + 0x1cde: 0x000a, 0x1cdf: 0x000a, + 0x1cfc: 0x000a, 0x1cfd: 0x000a, 0x1cfe: 0x000a, + // Block 0x74, offset 0x1d00 + 0x1d31: 0x000a, 0x1d32: 0x000a, 0x1d33: 0x000a, 0x1d34: 0x000a, 0x1d35: 0x000a, + 0x1d36: 0x000a, 0x1d37: 0x000a, 0x1d38: 0x000a, 0x1d39: 0x000a, 0x1d3a: 0x000a, 0x1d3b: 0x000a, + 0x1d3c: 0x000a, 0x1d3d: 0x000a, 0x1d3e: 0x000a, 0x1d3f: 0x000a, + // Block 0x75, offset 0x1d40 + 0x1d4c: 0x000a, 0x1d4d: 0x000a, 0x1d4e: 0x000a, 0x1d4f: 0x000a, + // Block 0x76, offset 0x1d80 + 0x1db7: 0x000a, 0x1db8: 0x000a, 0x1db9: 0x000a, 0x1dba: 0x000a, + // Block 0x77, offset 0x1dc0 + 0x1dde: 0x000a, 0x1ddf: 0x000a, + 0x1dff: 0x000a, + // Block 0x78, offset 0x1e00 + 0x1e10: 0x000a, 0x1e11: 0x000a, + 0x1e12: 0x000a, 0x1e13: 0x000a, 0x1e14: 0x000a, 0x1e15: 0x000a, 0x1e16: 0x000a, 0x1e17: 0x000a, + 0x1e18: 0x000a, 0x1e19: 0x000a, 0x1e1a: 0x000a, 0x1e1b: 0x000a, 0x1e1c: 0x000a, 0x1e1d: 0x000a, + 0x1e1e: 0x000a, 0x1e1f: 0x000a, 0x1e20: 0x000a, 0x1e21: 0x000a, 0x1e22: 0x000a, 0x1e23: 0x000a, + 0x1e24: 0x000a, 0x1e25: 0x000a, 0x1e26: 0x000a, 0x1e27: 0x000a, 0x1e28: 0x000a, 0x1e29: 0x000a, + 0x1e2a: 0x000a, 0x1e2b: 0x000a, 0x1e2c: 0x000a, 0x1e2d: 0x000a, 0x1e2e: 0x000a, 0x1e2f: 0x000a, + 0x1e30: 0x000a, 0x1e31: 0x000a, 0x1e32: 0x000a, 0x1e33: 0x000a, 0x1e34: 0x000a, 0x1e35: 0x000a, + 0x1e36: 0x000a, 0x1e37: 0x000a, 0x1e38: 0x000a, 0x1e39: 0x000a, 0x1e3a: 0x000a, 0x1e3b: 0x000a, + 0x1e3c: 0x000a, 0x1e3d: 0x000a, 0x1e3e: 0x000a, 0x1e3f: 0x000a, + // Block 0x79, offset 0x1e40 + 0x1e40: 0x000a, 0x1e41: 0x000a, 0x1e42: 0x000a, 0x1e43: 0x000a, 0x1e44: 0x000a, 0x1e45: 0x000a, + 0x1e46: 0x000a, + // Block 0x7a, offset 0x1e80 + 0x1e8d: 0x000a, 0x1e8e: 0x000a, 0x1e8f: 0x000a, + // Block 0x7b, offset 0x1ec0 + 0x1eef: 0x000c, + 0x1ef0: 0x000c, 0x1ef1: 0x000c, 0x1ef2: 0x000c, 0x1ef3: 0x000a, 0x1ef4: 0x000c, 0x1ef5: 0x000c, + 0x1ef6: 0x000c, 0x1ef7: 0x000c, 0x1ef8: 0x000c, 0x1ef9: 0x000c, 0x1efa: 0x000c, 0x1efb: 0x000c, + 0x1efc: 0x000c, 0x1efd: 0x000c, 0x1efe: 0x000a, 0x1eff: 0x000a, + // Block 0x7c, offset 0x1f00 + 0x1f1e: 0x000c, 0x1f1f: 0x000c, + // Block 0x7d, offset 0x1f40 + 0x1f70: 0x000c, 0x1f71: 0x000c, + // Block 0x7e, offset 0x1f80 + 0x1f80: 0x000a, 0x1f81: 0x000a, 0x1f82: 0x000a, 0x1f83: 0x000a, 0x1f84: 0x000a, 0x1f85: 0x000a, + 0x1f86: 0x000a, 0x1f87: 0x000a, 0x1f88: 0x000a, 0x1f89: 0x000a, 0x1f8a: 0x000a, 0x1f8b: 0x000a, + 0x1f8c: 0x000a, 0x1f8d: 0x000a, 0x1f8e: 0x000a, 0x1f8f: 0x000a, 0x1f90: 0x000a, 0x1f91: 0x000a, + 0x1f92: 0x000a, 0x1f93: 0x000a, 0x1f94: 0x000a, 0x1f95: 0x000a, 0x1f96: 0x000a, 0x1f97: 0x000a, + 0x1f98: 0x000a, 0x1f99: 0x000a, 0x1f9a: 0x000a, 0x1f9b: 0x000a, 0x1f9c: 0x000a, 0x1f9d: 0x000a, + 0x1f9e: 0x000a, 0x1f9f: 0x000a, 0x1fa0: 0x000a, 0x1fa1: 0x000a, + // Block 0x7f, offset 0x1fc0 + 0x1fc8: 0x000a, + // Block 0x80, offset 0x2000 + 0x2002: 0x000c, + 0x2006: 0x000c, 0x200b: 0x000c, + 0x2025: 0x000c, 0x2026: 0x000c, 0x2028: 0x000a, 0x2029: 0x000a, + 0x202a: 0x000a, 0x202b: 0x000a, + 0x2038: 0x0004, 0x2039: 0x0004, + // Block 0x81, offset 0x2040 + 0x2074: 0x000a, 0x2075: 0x000a, + 0x2076: 0x000a, 0x2077: 0x000a, + // Block 0x82, offset 0x2080 + 0x2084: 0x000c, 0x2085: 0x000c, + 0x20a0: 0x000c, 0x20a1: 0x000c, 0x20a2: 0x000c, 0x20a3: 0x000c, + 0x20a4: 0x000c, 0x20a5: 0x000c, 0x20a6: 0x000c, 0x20a7: 0x000c, 0x20a8: 0x000c, 0x20a9: 0x000c, + 0x20aa: 0x000c, 0x20ab: 0x000c, 0x20ac: 0x000c, 0x20ad: 0x000c, 0x20ae: 0x000c, 0x20af: 0x000c, + 0x20b0: 0x000c, 0x20b1: 0x000c, + 0x20bf: 0x000c, + // Block 0x83, offset 0x20c0 + 0x20e6: 0x000c, 0x20e7: 0x000c, 0x20e8: 0x000c, 0x20e9: 0x000c, + 0x20ea: 0x000c, 0x20eb: 0x000c, 0x20ec: 0x000c, 0x20ed: 0x000c, + // Block 0x84, offset 0x2100 + 0x2107: 0x000c, 0x2108: 0x000c, 0x2109: 0x000c, 0x210a: 0x000c, 0x210b: 0x000c, + 0x210c: 0x000c, 0x210d: 0x000c, 0x210e: 0x000c, 0x210f: 0x000c, 0x2110: 0x000c, 0x2111: 0x000c, + // Block 0x85, offset 0x2140 + 0x2140: 0x000c, 0x2141: 0x000c, 0x2142: 0x000c, + 0x2173: 0x000c, + 0x2176: 0x000c, 0x2177: 0x000c, 0x2178: 0x000c, 0x2179: 0x000c, + 0x217c: 0x000c, 0x217d: 0x000c, + // Block 0x86, offset 0x2180 + 0x21a5: 0x000c, + // Block 0x87, offset 0x21c0 + 0x21e9: 0x000c, + 0x21ea: 0x000c, 0x21eb: 0x000c, 0x21ec: 0x000c, 0x21ed: 0x000c, 0x21ee: 0x000c, + 0x21f1: 0x000c, 0x21f2: 0x000c, 0x21f5: 0x000c, + 0x21f6: 0x000c, + // Block 0x88, offset 0x2200 + 0x2203: 0x000c, + 0x220c: 0x000c, + 0x223c: 0x000c, + // Block 0x89, offset 0x2240 + 0x2270: 0x000c, 0x2272: 0x000c, 0x2273: 0x000c, 0x2274: 0x000c, + 0x2277: 0x000c, 0x2278: 0x000c, + 0x227e: 0x000c, 0x227f: 0x000c, + // Block 0x8a, offset 0x2280 + 0x2281: 0x000c, + 0x22ac: 0x000c, 0x22ad: 0x000c, + 0x22b6: 0x000c, + // Block 0x8b, offset 0x22c0 + 0x22e5: 0x000c, 0x22e8: 0x000c, + 0x22ed: 0x000c, + // Block 0x8c, offset 0x2300 + 0x231d: 0x0001, + 0x231e: 0x000c, 0x231f: 0x0001, 0x2320: 0x0001, 0x2321: 0x0001, 0x2322: 0x0001, 0x2323: 0x0001, + 0x2324: 0x0001, 0x2325: 0x0001, 0x2326: 0x0001, 0x2327: 0x0001, 0x2328: 0x0001, 0x2329: 0x0003, + 0x232a: 0x0001, 0x232b: 0x0001, 0x232c: 0x0001, 0x232d: 0x0001, 0x232e: 0x0001, 0x232f: 0x0001, + 0x2330: 0x0001, 0x2331: 0x0001, 0x2332: 0x0001, 0x2333: 0x0001, 0x2334: 0x0001, 0x2335: 0x0001, + 0x2336: 0x0001, 0x2337: 0x0001, 0x2338: 0x0001, 0x2339: 0x0001, 0x233a: 0x0001, 0x233b: 0x0001, + 0x233c: 0x0001, 0x233d: 0x0001, 0x233e: 0x0001, 0x233f: 0x0001, + // Block 0x8d, offset 0x2340 + 0x2340: 0x0001, 0x2341: 0x0001, 0x2342: 0x0001, 0x2343: 0x0001, 0x2344: 0x0001, 0x2345: 0x0001, + 0x2346: 0x0001, 0x2347: 0x0001, 0x2348: 0x0001, 0x2349: 0x0001, 0x234a: 0x0001, 0x234b: 0x0001, + 0x234c: 0x0001, 0x234d: 0x0001, 0x234e: 0x0001, 0x234f: 0x0001, 0x2350: 0x000d, 0x2351: 0x000d, + 0x2352: 0x000d, 0x2353: 0x000d, 0x2354: 0x000d, 0x2355: 0x000d, 0x2356: 0x000d, 0x2357: 0x000d, + 0x2358: 0x000d, 0x2359: 0x000d, 0x235a: 0x000d, 0x235b: 0x000d, 0x235c: 0x000d, 0x235d: 0x000d, + 0x235e: 0x000d, 0x235f: 0x000d, 0x2360: 0x000d, 0x2361: 0x000d, 0x2362: 0x000d, 0x2363: 0x000d, + 0x2364: 0x000d, 0x2365: 0x000d, 0x2366: 0x000d, 0x2367: 0x000d, 0x2368: 0x000d, 0x2369: 0x000d, + 0x236a: 0x000d, 0x236b: 0x000d, 0x236c: 0x000d, 0x236d: 0x000d, 0x236e: 0x000d, 0x236f: 0x000d, + 0x2370: 0x000d, 0x2371: 0x000d, 0x2372: 0x000d, 0x2373: 0x000d, 0x2374: 0x000d, 0x2375: 0x000d, + 0x2376: 0x000d, 0x2377: 0x000d, 0x2378: 0x000d, 0x2379: 0x000d, 0x237a: 0x000d, 0x237b: 0x000d, + 0x237c: 0x000d, 0x237d: 0x000d, 0x237e: 0x000d, 0x237f: 0x000d, + // Block 0x8e, offset 0x2380 + 0x2380: 0x000d, 0x2381: 0x000d, 0x2382: 0x000d, 0x2383: 0x000d, 0x2384: 0x000d, 0x2385: 0x000d, + 0x2386: 0x000d, 0x2387: 0x000d, 0x2388: 0x000d, 0x2389: 0x000d, 0x238a: 0x000d, 0x238b: 0x000d, + 0x238c: 0x000d, 0x238d: 0x000d, 0x238e: 0x000d, 0x238f: 0x000d, 0x2390: 0x000d, 0x2391: 0x000d, + 0x2392: 0x000d, 0x2393: 0x000d, 0x2394: 0x000d, 0x2395: 0x000d, 0x2396: 0x000d, 0x2397: 0x000d, + 0x2398: 0x000d, 0x2399: 0x000d, 0x239a: 0x000d, 0x239b: 0x000d, 0x239c: 0x000d, 0x239d: 0x000d, + 0x239e: 0x000d, 0x239f: 0x000d, 0x23a0: 0x000d, 0x23a1: 0x000d, 0x23a2: 0x000d, 0x23a3: 0x000d, + 0x23a4: 0x000d, 0x23a5: 0x000d, 0x23a6: 0x000d, 0x23a7: 0x000d, 0x23a8: 0x000d, 0x23a9: 0x000d, + 0x23aa: 0x000d, 0x23ab: 0x000d, 0x23ac: 0x000d, 0x23ad: 0x000d, 0x23ae: 0x000d, 0x23af: 0x000d, + 0x23b0: 0x000d, 0x23b1: 0x000d, 0x23b2: 0x000d, 0x23b3: 0x000d, 0x23b4: 0x000d, 0x23b5: 0x000d, + 0x23b6: 0x000d, 0x23b7: 0x000d, 0x23b8: 0x000d, 0x23b9: 0x000d, 0x23ba: 0x000d, 0x23bb: 0x000d, + 0x23bc: 0x000d, 0x23bd: 0x000d, 0x23be: 0x000a, 0x23bf: 0x000a, + // Block 0x8f, offset 0x23c0 + 0x23c0: 0x000d, 0x23c1: 0x000d, 0x23c2: 0x000d, 0x23c3: 0x000d, 0x23c4: 0x000d, 0x23c5: 0x000d, + 0x23c6: 0x000d, 0x23c7: 0x000d, 0x23c8: 0x000d, 0x23c9: 0x000d, 0x23ca: 0x000d, 0x23cb: 0x000d, + 0x23cc: 0x000d, 0x23cd: 0x000d, 0x23ce: 0x000d, 0x23cf: 0x000d, 0x23d0: 0x000b, 0x23d1: 0x000b, + 0x23d2: 0x000b, 0x23d3: 0x000b, 0x23d4: 0x000b, 0x23d5: 0x000b, 0x23d6: 0x000b, 0x23d7: 0x000b, + 0x23d8: 0x000b, 0x23d9: 0x000b, 0x23da: 0x000b, 0x23db: 0x000b, 0x23dc: 0x000b, 0x23dd: 0x000b, + 0x23de: 0x000b, 0x23df: 0x000b, 0x23e0: 0x000b, 0x23e1: 0x000b, 0x23e2: 0x000b, 0x23e3: 0x000b, + 0x23e4: 0x000b, 0x23e5: 0x000b, 0x23e6: 0x000b, 0x23e7: 0x000b, 0x23e8: 0x000b, 0x23e9: 0x000b, + 0x23ea: 0x000b, 0x23eb: 0x000b, 0x23ec: 0x000b, 0x23ed: 0x000b, 0x23ee: 0x000b, 0x23ef: 0x000b, + 0x23f0: 0x000d, 0x23f1: 0x000d, 0x23f2: 0x000d, 0x23f3: 0x000d, 0x23f4: 0x000d, 0x23f5: 0x000d, + 0x23f6: 0x000d, 0x23f7: 0x000d, 0x23f8: 0x000d, 0x23f9: 0x000d, 0x23fa: 0x000d, 0x23fb: 0x000d, + 0x23fc: 0x000d, 0x23fd: 0x000a, 0x23fe: 0x000d, 0x23ff: 0x000d, + // Block 0x90, offset 0x2400 + 0x2400: 0x000c, 0x2401: 0x000c, 0x2402: 0x000c, 0x2403: 0x000c, 0x2404: 0x000c, 0x2405: 0x000c, + 0x2406: 0x000c, 0x2407: 0x000c, 0x2408: 0x000c, 0x2409: 0x000c, 0x240a: 0x000c, 0x240b: 0x000c, + 0x240c: 0x000c, 0x240d: 0x000c, 0x240e: 0x000c, 0x240f: 0x000c, 0x2410: 0x000a, 0x2411: 0x000a, + 0x2412: 0x000a, 0x2413: 0x000a, 0x2414: 0x000a, 0x2415: 0x000a, 0x2416: 0x000a, 0x2417: 0x000a, + 0x2418: 0x000a, 0x2419: 0x000a, + 0x2420: 0x000c, 0x2421: 0x000c, 0x2422: 0x000c, 0x2423: 0x000c, + 0x2424: 0x000c, 0x2425: 0x000c, 0x2426: 0x000c, 0x2427: 0x000c, 0x2428: 0x000c, 0x2429: 0x000c, + 0x242a: 0x000c, 0x242b: 0x000c, 0x242c: 0x000c, 0x242d: 0x000c, 0x242e: 0x000c, 0x242f: 0x000c, + 0x2430: 0x000a, 0x2431: 0x000a, 0x2432: 0x000a, 0x2433: 0x000a, 0x2434: 0x000a, 0x2435: 0x000a, + 0x2436: 0x000a, 0x2437: 0x000a, 0x2438: 0x000a, 0x2439: 0x000a, 0x243a: 0x000a, 0x243b: 0x000a, + 0x243c: 0x000a, 0x243d: 0x000a, 0x243e: 0x000a, 0x243f: 0x000a, + // Block 0x91, offset 0x2440 + 0x2440: 0x000a, 0x2441: 0x000a, 0x2442: 0x000a, 0x2443: 0x000a, 0x2444: 0x000a, 0x2445: 0x000a, + 0x2446: 0x000a, 0x2447: 0x000a, 0x2448: 0x000a, 0x2449: 0x000a, 0x244a: 0x000a, 0x244b: 0x000a, + 0x244c: 0x000a, 0x244d: 0x000a, 0x244e: 0x000a, 0x244f: 0x000a, 0x2450: 0x0006, 0x2451: 0x000a, + 0x2452: 0x0006, 0x2454: 0x000a, 0x2455: 0x0006, 0x2456: 0x000a, 0x2457: 0x000a, + 0x2458: 0x000a, 0x2459: 0x009a, 0x245a: 0x008a, 0x245b: 0x007a, 0x245c: 0x006a, 0x245d: 0x009a, + 0x245e: 0x008a, 0x245f: 0x0004, 0x2460: 0x000a, 0x2461: 0x000a, 0x2462: 0x0003, 0x2463: 0x0003, + 0x2464: 0x000a, 0x2465: 0x000a, 0x2466: 0x000a, 0x2468: 0x000a, 0x2469: 0x0004, + 0x246a: 0x0004, 0x246b: 0x000a, + 0x2470: 0x000d, 0x2471: 0x000d, 0x2472: 0x000d, 0x2473: 0x000d, 0x2474: 0x000d, 0x2475: 0x000d, + 0x2476: 0x000d, 0x2477: 0x000d, 0x2478: 0x000d, 0x2479: 0x000d, 0x247a: 0x000d, 0x247b: 0x000d, + 0x247c: 0x000d, 0x247d: 0x000d, 0x247e: 0x000d, 0x247f: 0x000d, + // Block 0x92, offset 0x2480 + 0x2480: 0x000d, 0x2481: 0x000d, 0x2482: 0x000d, 0x2483: 0x000d, 0x2484: 0x000d, 0x2485: 0x000d, + 0x2486: 0x000d, 0x2487: 0x000d, 0x2488: 0x000d, 0x2489: 0x000d, 0x248a: 0x000d, 0x248b: 0x000d, + 0x248c: 0x000d, 0x248d: 0x000d, 0x248e: 0x000d, 0x248f: 0x000d, 0x2490: 0x000d, 0x2491: 0x000d, + 0x2492: 0x000d, 0x2493: 0x000d, 0x2494: 0x000d, 0x2495: 0x000d, 0x2496: 0x000d, 0x2497: 0x000d, + 0x2498: 0x000d, 0x2499: 0x000d, 0x249a: 0x000d, 0x249b: 0x000d, 0x249c: 0x000d, 0x249d: 0x000d, + 0x249e: 0x000d, 0x249f: 0x000d, 0x24a0: 0x000d, 0x24a1: 0x000d, 0x24a2: 0x000d, 0x24a3: 0x000d, + 0x24a4: 0x000d, 0x24a5: 0x000d, 0x24a6: 0x000d, 0x24a7: 0x000d, 0x24a8: 0x000d, 0x24a9: 0x000d, + 0x24aa: 0x000d, 0x24ab: 0x000d, 0x24ac: 0x000d, 0x24ad: 0x000d, 0x24ae: 0x000d, 0x24af: 0x000d, + 0x24b0: 0x000d, 0x24b1: 0x000d, 0x24b2: 0x000d, 0x24b3: 0x000d, 0x24b4: 0x000d, 0x24b5: 0x000d, + 0x24b6: 0x000d, 0x24b7: 0x000d, 0x24b8: 0x000d, 0x24b9: 0x000d, 0x24ba: 0x000d, 0x24bb: 0x000d, + 0x24bc: 0x000d, 0x24bd: 0x000d, 0x24be: 0x000d, 0x24bf: 0x000b, + // Block 0x93, offset 0x24c0 + 0x24c1: 0x000a, 0x24c2: 0x000a, 0x24c3: 0x0004, 0x24c4: 0x0004, 0x24c5: 0x0004, + 0x24c6: 0x000a, 0x24c7: 0x000a, 0x24c8: 0x003a, 0x24c9: 0x002a, 0x24ca: 0x000a, 0x24cb: 0x0003, + 0x24cc: 0x0006, 0x24cd: 0x0003, 0x24ce: 0x0006, 0x24cf: 0x0006, 0x24d0: 0x0002, 0x24d1: 0x0002, + 0x24d2: 0x0002, 0x24d3: 0x0002, 0x24d4: 0x0002, 0x24d5: 0x0002, 0x24d6: 0x0002, 0x24d7: 0x0002, + 0x24d8: 0x0002, 0x24d9: 0x0002, 0x24da: 0x0006, 0x24db: 0x000a, 0x24dc: 0x000a, 0x24dd: 0x000a, + 0x24de: 0x000a, 0x24df: 0x000a, 0x24e0: 0x000a, + 0x24fb: 0x005a, + 0x24fc: 0x000a, 0x24fd: 0x004a, 0x24fe: 0x000a, 0x24ff: 0x000a, + // Block 0x94, offset 0x2500 + 0x2500: 0x000a, + 0x251b: 0x005a, 0x251c: 0x000a, 0x251d: 0x004a, + 0x251e: 0x000a, 0x251f: 0x00fa, 0x2520: 0x00ea, 0x2521: 0x000a, 0x2522: 0x003a, 0x2523: 0x002a, + 0x2524: 0x000a, 0x2525: 0x000a, + // Block 0x95, offset 0x2540 + 0x2560: 0x0004, 0x2561: 0x0004, 0x2562: 0x000a, 0x2563: 0x000a, + 0x2564: 0x000a, 0x2565: 0x0004, 0x2566: 0x0004, 0x2568: 0x000a, 0x2569: 0x000a, + 0x256a: 0x000a, 0x256b: 0x000a, 0x256c: 0x000a, 0x256d: 0x000a, 0x256e: 0x000a, + 0x2570: 0x000b, 0x2571: 0x000b, 0x2572: 0x000b, 0x2573: 0x000b, 0x2574: 0x000b, 0x2575: 0x000b, + 0x2576: 0x000b, 0x2577: 0x000b, 0x2578: 0x000b, 0x2579: 0x000a, 0x257a: 0x000a, 0x257b: 0x000a, + 0x257c: 0x000a, 0x257d: 0x000a, 0x257e: 0x000b, 0x257f: 0x000b, + // Block 0x96, offset 0x2580 + 0x2581: 0x000a, + // Block 0x97, offset 0x25c0 + 0x25c0: 0x000a, 0x25c1: 0x000a, 0x25c2: 0x000a, 0x25c3: 0x000a, 0x25c4: 0x000a, 0x25c5: 0x000a, + 0x25c6: 0x000a, 0x25c7: 0x000a, 0x25c8: 0x000a, 0x25c9: 0x000a, 0x25ca: 0x000a, 0x25cb: 0x000a, + 0x25cc: 0x000a, 0x25d0: 0x000a, 0x25d1: 0x000a, + 0x25d2: 0x000a, 0x25d3: 0x000a, 0x25d4: 0x000a, 0x25d5: 0x000a, 0x25d6: 0x000a, 0x25d7: 0x000a, + 0x25d8: 0x000a, 0x25d9: 0x000a, 0x25da: 0x000a, 0x25db: 0x000a, + 0x25e0: 0x000a, + // Block 0x98, offset 0x2600 + 0x263d: 0x000c, + // Block 0x99, offset 0x2640 + 0x2660: 0x000c, 0x2661: 0x0002, 0x2662: 0x0002, 0x2663: 0x0002, + 0x2664: 0x0002, 0x2665: 0x0002, 0x2666: 0x0002, 0x2667: 0x0002, 0x2668: 0x0002, 0x2669: 0x0002, + 0x266a: 0x0002, 0x266b: 0x0002, 0x266c: 0x0002, 0x266d: 0x0002, 0x266e: 0x0002, 0x266f: 0x0002, + 0x2670: 0x0002, 0x2671: 0x0002, 0x2672: 0x0002, 0x2673: 0x0002, 0x2674: 0x0002, 0x2675: 0x0002, + 0x2676: 0x0002, 0x2677: 0x0002, 0x2678: 0x0002, 0x2679: 0x0002, 0x267a: 0x0002, 0x267b: 0x0002, + // Block 0x9a, offset 0x2680 + 0x26b6: 0x000c, 0x26b7: 0x000c, 0x26b8: 0x000c, 0x26b9: 0x000c, 0x26ba: 0x000c, + // Block 0x9b, offset 0x26c0 + 0x26c0: 0x0001, 0x26c1: 0x0001, 0x26c2: 0x0001, 0x26c3: 0x0001, 0x26c4: 0x0001, 0x26c5: 0x0001, + 0x26c6: 0x0001, 0x26c7: 0x0001, 0x26c8: 0x0001, 0x26c9: 0x0001, 0x26ca: 0x0001, 0x26cb: 0x0001, + 0x26cc: 0x0001, 0x26cd: 0x0001, 0x26ce: 0x0001, 0x26cf: 0x0001, 0x26d0: 0x0001, 0x26d1: 0x0001, + 0x26d2: 0x0001, 0x26d3: 0x0001, 0x26d4: 0x0001, 0x26d5: 0x0001, 0x26d6: 0x0001, 0x26d7: 0x0001, + 0x26d8: 0x0001, 0x26d9: 0x0001, 0x26da: 0x0001, 0x26db: 0x0001, 0x26dc: 0x0001, 0x26dd: 0x0001, + 0x26de: 0x0001, 0x26df: 0x0001, 0x26e0: 0x0001, 0x26e1: 0x0001, 0x26e2: 0x0001, 0x26e3: 0x0001, + 0x26e4: 0x0001, 0x26e5: 0x0001, 0x26e6: 0x0001, 0x26e7: 0x0001, 0x26e8: 0x0001, 0x26e9: 0x0001, + 0x26ea: 0x0001, 0x26eb: 0x0001, 0x26ec: 0x0001, 0x26ed: 0x0001, 0x26ee: 0x0001, 0x26ef: 0x0001, + 0x26f0: 0x0001, 0x26f1: 0x0001, 0x26f2: 0x0001, 0x26f3: 0x0001, 0x26f4: 0x0001, 0x26f5: 0x0001, + 0x26f6: 0x0001, 0x26f7: 0x0001, 0x26f8: 0x0001, 0x26f9: 0x0001, 0x26fa: 0x0001, 0x26fb: 0x0001, + 0x26fc: 0x0001, 0x26fd: 0x0001, 0x26fe: 0x0001, 0x26ff: 0x0001, + // Block 0x9c, offset 0x2700 + 0x2700: 0x0001, 0x2701: 0x0001, 0x2702: 0x0001, 0x2703: 0x0001, 0x2704: 0x0001, 0x2705: 0x0001, + 0x2706: 0x0001, 0x2707: 0x0001, 0x2708: 0x0001, 0x2709: 0x0001, 0x270a: 0x0001, 0x270b: 0x0001, + 0x270c: 0x0001, 0x270d: 0x0001, 0x270e: 0x0001, 0x270f: 0x0001, 0x2710: 0x0001, 0x2711: 0x0001, + 0x2712: 0x0001, 0x2713: 0x0001, 0x2714: 0x0001, 0x2715: 0x0001, 0x2716: 0x0001, 0x2717: 0x0001, + 0x2718: 0x0001, 0x2719: 0x0001, 0x271a: 0x0001, 0x271b: 0x0001, 0x271c: 0x0001, 0x271d: 0x0001, + 0x271e: 0x0001, 0x271f: 0x000a, 0x2720: 0x0001, 0x2721: 0x0001, 0x2722: 0x0001, 0x2723: 0x0001, + 0x2724: 0x0001, 0x2725: 0x0001, 0x2726: 0x0001, 0x2727: 0x0001, 0x2728: 0x0001, 0x2729: 0x0001, + 0x272a: 0x0001, 0x272b: 0x0001, 0x272c: 0x0001, 0x272d: 0x0001, 0x272e: 0x0001, 0x272f: 0x0001, + 0x2730: 0x0001, 0x2731: 0x0001, 0x2732: 0x0001, 0x2733: 0x0001, 0x2734: 0x0001, 0x2735: 0x0001, + 0x2736: 0x0001, 0x2737: 0x0001, 0x2738: 0x0001, 0x2739: 0x0001, 0x273a: 0x0001, 0x273b: 0x0001, + 0x273c: 0x0001, 0x273d: 0x0001, 0x273e: 0x0001, 0x273f: 0x0001, + // Block 0x9d, offset 0x2740 + 0x2740: 0x0001, 0x2741: 0x000c, 0x2742: 0x000c, 0x2743: 0x000c, 0x2744: 0x0001, 0x2745: 0x000c, + 0x2746: 0x000c, 0x2747: 0x0001, 0x2748: 0x0001, 0x2749: 0x0001, 0x274a: 0x0001, 0x274b: 0x0001, + 0x274c: 0x000c, 0x274d: 0x000c, 0x274e: 0x000c, 0x274f: 0x000c, 0x2750: 0x0001, 0x2751: 0x0001, + 0x2752: 0x0001, 0x2753: 0x0001, 0x2754: 0x0001, 0x2755: 0x0001, 0x2756: 0x0001, 0x2757: 0x0001, + 0x2758: 0x0001, 0x2759: 0x0001, 0x275a: 0x0001, 0x275b: 0x0001, 0x275c: 0x0001, 0x275d: 0x0001, + 0x275e: 0x0001, 0x275f: 0x0001, 0x2760: 0x0001, 0x2761: 0x0001, 0x2762: 0x0001, 0x2763: 0x0001, + 0x2764: 0x0001, 0x2765: 0x0001, 0x2766: 0x0001, 0x2767: 0x0001, 0x2768: 0x0001, 0x2769: 0x0001, + 0x276a: 0x0001, 0x276b: 0x0001, 0x276c: 0x0001, 0x276d: 0x0001, 0x276e: 0x0001, 0x276f: 0x0001, + 0x2770: 0x0001, 0x2771: 0x0001, 0x2772: 0x0001, 0x2773: 0x0001, 0x2774: 0x0001, 0x2775: 0x0001, + 0x2776: 0x0001, 0x2777: 0x0001, 0x2778: 0x000c, 0x2779: 0x000c, 0x277a: 0x000c, 0x277b: 0x0001, + 0x277c: 0x0001, 0x277d: 0x0001, 0x277e: 0x0001, 0x277f: 0x000c, + // Block 0x9e, offset 0x2780 + 0x2780: 0x0001, 0x2781: 0x0001, 0x2782: 0x0001, 0x2783: 0x0001, 0x2784: 0x0001, 0x2785: 0x0001, + 0x2786: 0x0001, 0x2787: 0x0001, 0x2788: 0x0001, 0x2789: 0x0001, 0x278a: 0x0001, 0x278b: 0x0001, + 0x278c: 0x0001, 0x278d: 0x0001, 0x278e: 0x0001, 0x278f: 0x0001, 0x2790: 0x0001, 0x2791: 0x0001, + 0x2792: 0x0001, 0x2793: 0x0001, 0x2794: 0x0001, 0x2795: 0x0001, 0x2796: 0x0001, 0x2797: 0x0001, + 0x2798: 0x0001, 0x2799: 0x0001, 0x279a: 0x0001, 0x279b: 0x0001, 0x279c: 0x0001, 0x279d: 0x0001, + 0x279e: 0x0001, 0x279f: 0x0001, 0x27a0: 0x0001, 0x27a1: 0x0001, 0x27a2: 0x0001, 0x27a3: 0x0001, + 0x27a4: 0x0001, 0x27a5: 0x000c, 0x27a6: 0x000c, 0x27a7: 0x0001, 0x27a8: 0x0001, 0x27a9: 0x0001, + 0x27aa: 0x0001, 0x27ab: 0x0001, 0x27ac: 0x0001, 0x27ad: 0x0001, 0x27ae: 0x0001, 0x27af: 0x0001, + 0x27b0: 0x0001, 0x27b1: 0x0001, 0x27b2: 0x0001, 0x27b3: 0x0001, 0x27b4: 0x0001, 0x27b5: 0x0001, + 0x27b6: 0x0001, 0x27b7: 0x0001, 0x27b8: 0x0001, 0x27b9: 0x0001, 0x27ba: 0x0001, 0x27bb: 0x0001, + 0x27bc: 0x0001, 0x27bd: 0x0001, 0x27be: 0x0001, 0x27bf: 0x0001, + // Block 0x9f, offset 0x27c0 + 0x27c0: 0x0001, 0x27c1: 0x0001, 0x27c2: 0x0001, 0x27c3: 0x0001, 0x27c4: 0x0001, 0x27c5: 0x0001, + 0x27c6: 0x0001, 0x27c7: 0x0001, 0x27c8: 0x0001, 0x27c9: 0x0001, 0x27ca: 0x0001, 0x27cb: 0x0001, + 0x27cc: 0x0001, 0x27cd: 0x0001, 0x27ce: 0x0001, 0x27cf: 0x0001, 0x27d0: 0x0001, 0x27d1: 0x0001, + 0x27d2: 0x0001, 0x27d3: 0x0001, 0x27d4: 0x0001, 0x27d5: 0x0001, 0x27d6: 0x0001, 0x27d7: 0x0001, + 0x27d8: 0x0001, 0x27d9: 0x0001, 0x27da: 0x0001, 0x27db: 0x0001, 0x27dc: 0x0001, 0x27dd: 0x0001, + 0x27de: 0x0001, 0x27df: 0x0001, 0x27e0: 0x0001, 0x27e1: 0x0001, 0x27e2: 0x0001, 0x27e3: 0x0001, + 0x27e4: 0x0001, 0x27e5: 0x0001, 0x27e6: 0x0001, 0x27e7: 0x0001, 0x27e8: 0x0001, 0x27e9: 0x0001, + 0x27ea: 0x0001, 0x27eb: 0x0001, 0x27ec: 0x0001, 0x27ed: 0x0001, 0x27ee: 0x0001, 0x27ef: 0x0001, + 0x27f0: 0x0001, 0x27f1: 0x0001, 0x27f2: 0x0001, 0x27f3: 0x0001, 0x27f4: 0x0001, 0x27f5: 0x0001, + 0x27f6: 0x0001, 0x27f7: 0x0001, 0x27f8: 0x0001, 0x27f9: 0x000a, 0x27fa: 0x000a, 0x27fb: 0x000a, + 0x27fc: 0x000a, 0x27fd: 0x000a, 0x27fe: 0x000a, 0x27ff: 0x000a, + // Block 0xa0, offset 0x2800 + 0x2800: 0x000d, 0x2801: 0x000d, 0x2802: 0x000d, 0x2803: 0x000d, 0x2804: 0x000d, 0x2805: 0x000d, + 0x2806: 0x000d, 0x2807: 0x000d, 0x2808: 0x000d, 0x2809: 0x000d, 0x280a: 0x000d, 0x280b: 0x000d, + 0x280c: 0x000d, 0x280d: 0x000d, 0x280e: 0x000d, 0x280f: 0x000d, 0x2810: 0x000d, 0x2811: 0x000d, + 0x2812: 0x000d, 0x2813: 0x000d, 0x2814: 0x000d, 0x2815: 0x000d, 0x2816: 0x000d, 0x2817: 0x000d, + 0x2818: 0x000d, 0x2819: 0x000d, 0x281a: 0x000d, 0x281b: 0x000d, 0x281c: 0x000d, 0x281d: 0x000d, + 0x281e: 0x000d, 0x281f: 0x000d, 0x2820: 0x000d, 0x2821: 0x000d, 0x2822: 0x000d, 0x2823: 0x000d, + 0x2824: 0x000c, 0x2825: 0x000c, 0x2826: 0x000c, 0x2827: 0x000c, 0x2828: 0x000d, 0x2829: 0x000d, + 0x282a: 0x000d, 0x282b: 0x000d, 0x282c: 0x000d, 0x282d: 0x000d, 0x282e: 0x000d, 0x282f: 0x000d, + 0x2830: 0x0005, 0x2831: 0x0005, 0x2832: 0x0005, 0x2833: 0x0005, 0x2834: 0x0005, 0x2835: 0x0005, + 0x2836: 0x0005, 0x2837: 0x0005, 0x2838: 0x0005, 0x2839: 0x0005, 0x283a: 0x000d, 0x283b: 0x000d, + 0x283c: 0x000d, 0x283d: 0x000d, 0x283e: 0x000d, 0x283f: 0x000d, + // Block 0xa1, offset 0x2840 + 0x2840: 0x0001, 0x2841: 0x0001, 0x2842: 0x0001, 0x2843: 0x0001, 0x2844: 0x0001, 0x2845: 0x0001, + 0x2846: 0x0001, 0x2847: 0x0001, 0x2848: 0x0001, 0x2849: 0x0001, 0x284a: 0x0001, 0x284b: 0x0001, + 0x284c: 0x0001, 0x284d: 0x0001, 0x284e: 0x0001, 0x284f: 0x0001, 0x2850: 0x0001, 0x2851: 0x0001, + 0x2852: 0x0001, 0x2853: 0x0001, 0x2854: 0x0001, 0x2855: 0x0001, 0x2856: 0x0001, 0x2857: 0x0001, + 0x2858: 0x0001, 0x2859: 0x0001, 0x285a: 0x0001, 0x285b: 0x0001, 0x285c: 0x0001, 0x285d: 0x0001, + 0x285e: 0x0001, 0x285f: 0x0001, 0x2860: 0x0005, 0x2861: 0x0005, 0x2862: 0x0005, 0x2863: 0x0005, + 0x2864: 0x0005, 0x2865: 0x0005, 0x2866: 0x0005, 0x2867: 0x0005, 0x2868: 0x0005, 0x2869: 0x0005, + 0x286a: 0x0005, 0x286b: 0x0005, 0x286c: 0x0005, 0x286d: 0x0005, 0x286e: 0x0005, 0x286f: 0x0005, + 0x2870: 0x0005, 0x2871: 0x0005, 0x2872: 0x0005, 0x2873: 0x0005, 0x2874: 0x0005, 0x2875: 0x0005, + 0x2876: 0x0005, 0x2877: 0x0005, 0x2878: 0x0005, 0x2879: 0x0005, 0x287a: 0x0005, 0x287b: 0x0005, + 0x287c: 0x0005, 0x287d: 0x0005, 0x287e: 0x0005, 0x287f: 0x0001, + // Block 0xa2, offset 0x2880 + 0x2880: 0x0001, 0x2881: 0x0001, 0x2882: 0x0001, 0x2883: 0x0001, 0x2884: 0x0001, 0x2885: 0x0001, + 0x2886: 0x0001, 0x2887: 0x0001, 0x2888: 0x0001, 0x2889: 0x0001, 0x288a: 0x0001, 0x288b: 0x0001, + 0x288c: 0x0001, 0x288d: 0x0001, 0x288e: 0x0001, 0x288f: 0x0001, 0x2890: 0x0001, 0x2891: 0x0001, + 0x2892: 0x0001, 0x2893: 0x0001, 0x2894: 0x0001, 0x2895: 0x0001, 0x2896: 0x0001, 0x2897: 0x0001, + 0x2898: 0x0001, 0x2899: 0x0001, 0x289a: 0x0001, 0x289b: 0x0001, 0x289c: 0x0001, 0x289d: 0x0001, + 0x289e: 0x0001, 0x289f: 0x0001, 0x28a0: 0x0001, 0x28a1: 0x0001, 0x28a2: 0x0001, 0x28a3: 0x0001, + 0x28a4: 0x0001, 0x28a5: 0x0001, 0x28a6: 0x0001, 0x28a7: 0x0001, 0x28a8: 0x0001, 0x28a9: 0x0001, + 0x28aa: 0x0001, 0x28ab: 0x0001, 0x28ac: 0x0001, 0x28ad: 0x0001, 0x28ae: 0x0001, 0x28af: 0x0001, + 0x28b0: 0x000d, 0x28b1: 0x000d, 0x28b2: 0x000d, 0x28b3: 0x000d, 0x28b4: 0x000d, 0x28b5: 0x000d, + 0x28b6: 0x000d, 0x28b7: 0x000d, 0x28b8: 0x000d, 0x28b9: 0x000d, 0x28ba: 0x000d, 0x28bb: 0x000d, + 0x28bc: 0x000d, 0x28bd: 0x000d, 0x28be: 0x000d, 0x28bf: 0x000d, + // Block 0xa3, offset 0x28c0 + 0x28c0: 0x000d, 0x28c1: 0x000d, 0x28c2: 0x000d, 0x28c3: 0x000d, 0x28c4: 0x000d, 0x28c5: 0x000d, + 0x28c6: 0x000c, 0x28c7: 0x000c, 0x28c8: 0x000c, 0x28c9: 0x000c, 0x28ca: 0x000c, 0x28cb: 0x000c, + 0x28cc: 0x000c, 0x28cd: 0x000c, 0x28ce: 0x000c, 0x28cf: 0x000c, 0x28d0: 0x000c, 0x28d1: 0x000d, + 0x28d2: 0x000d, 0x28d3: 0x000d, 0x28d4: 0x000d, 0x28d5: 0x000d, 0x28d6: 0x000d, 0x28d7: 0x000d, + 0x28d8: 0x000d, 0x28d9: 0x000d, 0x28da: 0x000d, 0x28db: 0x000d, 0x28dc: 0x000d, 0x28dd: 0x000d, + 0x28de: 0x000d, 0x28df: 0x000d, 0x28e0: 0x000d, 0x28e1: 0x000d, 0x28e2: 0x000d, 0x28e3: 0x000d, + 0x28e4: 0x000d, 0x28e5: 0x000d, 0x28e6: 0x000d, 0x28e7: 0x000d, 0x28e8: 0x000d, 0x28e9: 0x000d, + 0x28ea: 0x000d, 0x28eb: 0x000d, 0x28ec: 0x000d, 0x28ed: 0x000d, 0x28ee: 0x000d, 0x28ef: 0x000d, + 0x28f0: 0x0001, 0x28f1: 0x0001, 0x28f2: 0x0001, 0x28f3: 0x0001, 0x28f4: 0x0001, 0x28f5: 0x0001, + 0x28f6: 0x0001, 0x28f7: 0x0001, 0x28f8: 0x0001, 0x28f9: 0x0001, 0x28fa: 0x0001, 0x28fb: 0x0001, + 0x28fc: 0x0001, 0x28fd: 0x0001, 0x28fe: 0x0001, 0x28ff: 0x0001, + // Block 0xa4, offset 0x2900 + 0x2901: 0x000c, + 0x2938: 0x000c, 0x2939: 0x000c, 0x293a: 0x000c, 0x293b: 0x000c, + 0x293c: 0x000c, 0x293d: 0x000c, 0x293e: 0x000c, 0x293f: 0x000c, + // Block 0xa5, offset 0x2940 + 0x2940: 0x000c, 0x2941: 0x000c, 0x2942: 0x000c, 0x2943: 0x000c, 0x2944: 0x000c, 0x2945: 0x000c, + 0x2946: 0x000c, + 0x2952: 0x000a, 0x2953: 0x000a, 0x2954: 0x000a, 0x2955: 0x000a, 0x2956: 0x000a, 0x2957: 0x000a, + 0x2958: 0x000a, 0x2959: 0x000a, 0x295a: 0x000a, 0x295b: 0x000a, 0x295c: 0x000a, 0x295d: 0x000a, + 0x295e: 0x000a, 0x295f: 0x000a, 0x2960: 0x000a, 0x2961: 0x000a, 0x2962: 0x000a, 0x2963: 0x000a, + 0x2964: 0x000a, 0x2965: 0x000a, + 0x297f: 0x000c, + // Block 0xa6, offset 0x2980 + 0x2980: 0x000c, 0x2981: 0x000c, + 0x29b3: 0x000c, 0x29b4: 0x000c, 0x29b5: 0x000c, + 0x29b6: 0x000c, 0x29b9: 0x000c, 0x29ba: 0x000c, + // Block 0xa7, offset 0x29c0 + 0x29c0: 0x000c, 0x29c1: 0x000c, 0x29c2: 0x000c, + 0x29e7: 0x000c, 0x29e8: 0x000c, 0x29e9: 0x000c, + 0x29ea: 0x000c, 0x29eb: 0x000c, 0x29ed: 0x000c, 0x29ee: 0x000c, 0x29ef: 0x000c, + 0x29f0: 0x000c, 0x29f1: 0x000c, 0x29f2: 0x000c, 0x29f3: 0x000c, 0x29f4: 0x000c, + // Block 0xa8, offset 0x2a00 + 0x2a33: 0x000c, + // Block 0xa9, offset 0x2a40 + 0x2a40: 0x000c, 0x2a41: 0x000c, + 0x2a76: 0x000c, 0x2a77: 0x000c, 0x2a78: 0x000c, 0x2a79: 0x000c, 0x2a7a: 0x000c, 0x2a7b: 0x000c, + 0x2a7c: 0x000c, 0x2a7d: 0x000c, 0x2a7e: 0x000c, + // Block 0xaa, offset 0x2a80 + 0x2a89: 0x000c, 0x2a8a: 0x000c, 0x2a8b: 0x000c, + 0x2a8c: 0x000c, + // Block 0xab, offset 0x2ac0 + 0x2aef: 0x000c, + 0x2af0: 0x000c, 0x2af1: 0x000c, 0x2af4: 0x000c, + 0x2af6: 0x000c, 0x2af7: 0x000c, + 0x2afe: 0x000c, + // Block 0xac, offset 0x2b00 + 0x2b1f: 0x000c, 0x2b23: 0x000c, + 0x2b24: 0x000c, 0x2b25: 0x000c, 0x2b26: 0x000c, 0x2b27: 0x000c, 0x2b28: 0x000c, 0x2b29: 0x000c, + 0x2b2a: 0x000c, + // Block 0xad, offset 0x2b40 + 0x2b40: 0x000c, + 0x2b66: 0x000c, 0x2b67: 0x000c, 0x2b68: 0x000c, 0x2b69: 0x000c, + 0x2b6a: 0x000c, 0x2b6b: 0x000c, 0x2b6c: 0x000c, + 0x2b70: 0x000c, 0x2b71: 0x000c, 0x2b72: 0x000c, 0x2b73: 0x000c, 0x2b74: 0x000c, + // Block 0xae, offset 0x2b80 + 0x2bb8: 0x000c, 0x2bb9: 0x000c, 0x2bba: 0x000c, 0x2bbb: 0x000c, + 0x2bbc: 0x000c, 0x2bbd: 0x000c, 0x2bbe: 0x000c, 0x2bbf: 0x000c, + // Block 0xaf, offset 0x2bc0 + 0x2bc2: 0x000c, 0x2bc3: 0x000c, 0x2bc4: 0x000c, + 0x2bc6: 0x000c, + 0x2bde: 0x000c, + // Block 0xb0, offset 0x2c00 + 0x2c33: 0x000c, 0x2c34: 0x000c, 0x2c35: 0x000c, + 0x2c36: 0x000c, 0x2c37: 0x000c, 0x2c38: 0x000c, 0x2c3a: 0x000c, + 0x2c3f: 0x000c, + // Block 0xb1, offset 0x2c40 + 0x2c40: 0x000c, 0x2c42: 0x000c, 0x2c43: 0x000c, + // Block 0xb2, offset 0x2c80 + 0x2cb2: 0x000c, 0x2cb3: 0x000c, 0x2cb4: 0x000c, 0x2cb5: 0x000c, + 0x2cbc: 0x000c, 0x2cbd: 0x000c, 0x2cbf: 0x000c, + // Block 0xb3, offset 0x2cc0 + 0x2cc0: 0x000c, + 0x2cdc: 0x000c, 0x2cdd: 0x000c, + // Block 0xb4, offset 0x2d00 + 0x2d33: 0x000c, 0x2d34: 0x000c, 0x2d35: 0x000c, + 0x2d36: 0x000c, 0x2d37: 0x000c, 0x2d38: 0x000c, 0x2d39: 0x000c, 0x2d3a: 0x000c, + 0x2d3d: 0x000c, 0x2d3f: 0x000c, + // Block 0xb5, offset 0x2d40 + 0x2d40: 0x000c, + 0x2d60: 0x000a, 0x2d61: 0x000a, 0x2d62: 0x000a, 0x2d63: 0x000a, + 0x2d64: 0x000a, 0x2d65: 0x000a, 0x2d66: 0x000a, 0x2d67: 0x000a, 0x2d68: 0x000a, 0x2d69: 0x000a, + 0x2d6a: 0x000a, 0x2d6b: 0x000a, 0x2d6c: 0x000a, + // Block 0xb6, offset 0x2d80 + 0x2dab: 0x000c, 0x2dad: 0x000c, + 0x2db0: 0x000c, 0x2db1: 0x000c, 0x2db2: 0x000c, 0x2db3: 0x000c, 0x2db4: 0x000c, 0x2db5: 0x000c, + 0x2db7: 0x000c, + // Block 0xb7, offset 0x2dc0 + 0x2ddd: 0x000c, + 0x2dde: 0x000c, 0x2ddf: 0x000c, 0x2de2: 0x000c, 0x2de3: 0x000c, + 0x2de4: 0x000c, 0x2de5: 0x000c, 0x2de7: 0x000c, 0x2de8: 0x000c, 0x2de9: 0x000c, + 0x2dea: 0x000c, 0x2deb: 0x000c, + // Block 0xb8, offset 0x2e00 + 0x2e2f: 0x000c, + 0x2e30: 0x000c, 0x2e31: 0x000c, 0x2e32: 0x000c, 0x2e33: 0x000c, 0x2e34: 0x000c, 0x2e35: 0x000c, + 0x2e36: 0x000c, 0x2e37: 0x000c, 0x2e39: 0x000c, 0x2e3a: 0x000c, + // Block 0xb9, offset 0x2e40 + 0x2e54: 0x000c, 0x2e55: 0x000c, 0x2e56: 0x000c, 0x2e57: 0x000c, + 0x2e5a: 0x000c, 0x2e5b: 0x000c, + 0x2e60: 0x000c, + // Block 0xba, offset 0x2e80 + 0x2e81: 0x000c, 0x2e82: 0x000c, 0x2e83: 0x000c, 0x2e84: 0x000c, 0x2e85: 0x000c, + 0x2e86: 0x000c, 0x2e89: 0x000c, 0x2e8a: 0x000c, + 0x2eb3: 0x000c, 0x2eb4: 0x000c, 0x2eb5: 0x000c, + 0x2eb6: 0x000c, 0x2eb7: 0x000c, 0x2eb8: 0x000c, 0x2ebb: 0x000c, + 0x2ebc: 0x000c, 0x2ebd: 0x000c, 0x2ebe: 0x000c, + // Block 0xbb, offset 0x2ec0 + 0x2ec7: 0x000c, + 0x2ed1: 0x000c, + 0x2ed2: 0x000c, 0x2ed3: 0x000c, 0x2ed4: 0x000c, 0x2ed5: 0x000c, 0x2ed6: 0x000c, + 0x2ed9: 0x000c, 0x2eda: 0x000c, 0x2edb: 0x000c, + // Block 0xbc, offset 0x2f00 + 0x2f0a: 0x000c, 0x2f0b: 0x000c, + 0x2f0c: 0x000c, 0x2f0d: 0x000c, 0x2f0e: 0x000c, 0x2f0f: 0x000c, 0x2f10: 0x000c, 0x2f11: 0x000c, + 0x2f12: 0x000c, 0x2f13: 0x000c, 0x2f14: 0x000c, 0x2f15: 0x000c, 0x2f16: 0x000c, + 0x2f18: 0x000c, 0x2f19: 0x000c, + // Block 0xbd, offset 0x2f40 + 0x2f70: 0x000c, 0x2f71: 0x000c, 0x2f72: 0x000c, 0x2f73: 0x000c, 0x2f74: 0x000c, 0x2f75: 0x000c, + 0x2f76: 0x000c, 0x2f78: 0x000c, 0x2f79: 0x000c, 0x2f7a: 0x000c, 0x2f7b: 0x000c, + 0x2f7c: 0x000c, 0x2f7d: 0x000c, + // Block 0xbe, offset 0x2f80 + 0x2f92: 0x000c, 0x2f93: 0x000c, 0x2f94: 0x000c, 0x2f95: 0x000c, 0x2f96: 0x000c, 0x2f97: 0x000c, + 0x2f98: 0x000c, 0x2f99: 0x000c, 0x2f9a: 0x000c, 0x2f9b: 0x000c, 0x2f9c: 0x000c, 0x2f9d: 0x000c, + 0x2f9e: 0x000c, 0x2f9f: 0x000c, 0x2fa0: 0x000c, 0x2fa1: 0x000c, 0x2fa2: 0x000c, 0x2fa3: 0x000c, + 0x2fa4: 0x000c, 0x2fa5: 0x000c, 0x2fa6: 0x000c, 0x2fa7: 0x000c, + 0x2faa: 0x000c, 0x2fab: 0x000c, 0x2fac: 0x000c, 0x2fad: 0x000c, 0x2fae: 0x000c, 0x2faf: 0x000c, + 0x2fb0: 0x000c, 0x2fb2: 0x000c, 0x2fb3: 0x000c, 0x2fb5: 0x000c, + 0x2fb6: 0x000c, + // Block 0xbf, offset 0x2fc0 + 0x2ff1: 0x000c, 0x2ff2: 0x000c, 0x2ff3: 0x000c, 0x2ff4: 0x000c, 0x2ff5: 0x000c, + 0x2ff6: 0x000c, 0x2ffa: 0x000c, + 0x2ffc: 0x000c, 0x2ffd: 0x000c, 0x2fff: 0x000c, + // Block 0xc0, offset 0x3000 + 0x3000: 0x000c, 0x3001: 0x000c, 0x3002: 0x000c, 0x3003: 0x000c, 0x3004: 0x000c, 0x3005: 0x000c, + 0x3007: 0x000c, + // Block 0xc1, offset 0x3040 + 0x3050: 0x000c, 0x3051: 0x000c, + 0x3055: 0x000c, 0x3057: 0x000c, + // Block 0xc2, offset 0x3080 + 0x30b3: 0x000c, 0x30b4: 0x000c, + // Block 0xc3, offset 0x30c0 + 0x30d5: 0x000a, 0x30d6: 0x000a, 0x30d7: 0x000a, + 0x30d8: 0x000a, 0x30d9: 0x000a, 0x30da: 0x000a, 0x30db: 0x000a, 0x30dc: 0x000a, 0x30dd: 0x0004, + 0x30de: 0x0004, 0x30df: 0x0004, 0x30e0: 0x0004, 0x30e1: 0x000a, 0x30e2: 0x000a, 0x30e3: 0x000a, + 0x30e4: 0x000a, 0x30e5: 0x000a, 0x30e6: 0x000a, 0x30e7: 0x000a, 0x30e8: 0x000a, 0x30e9: 0x000a, + 0x30ea: 0x000a, 0x30eb: 0x000a, 0x30ec: 0x000a, 0x30ed: 0x000a, 0x30ee: 0x000a, 0x30ef: 0x000a, + 0x30f0: 0x000a, 0x30f1: 0x000a, + // Block 0xc4, offset 0x3100 + 0x3130: 0x000c, 0x3131: 0x000c, 0x3132: 0x000c, 0x3133: 0x000c, 0x3134: 0x000c, + // Block 0xc5, offset 0x3140 + 0x3170: 0x000c, 0x3171: 0x000c, 0x3172: 0x000c, 0x3173: 0x000c, 0x3174: 0x000c, 0x3175: 0x000c, + 0x3176: 0x000c, + // Block 0xc6, offset 0x3180 + 0x318f: 0x000c, + // Block 0xc7, offset 0x31c0 + 0x31cf: 0x000c, 0x31d0: 0x000c, 0x31d1: 0x000c, + 0x31d2: 0x000c, + // Block 0xc8, offset 0x3200 + 0x3222: 0x000a, + // Block 0xc9, offset 0x3240 + 0x325d: 0x000c, + 0x325e: 0x000c, 0x3260: 0x000b, 0x3261: 0x000b, 0x3262: 0x000b, 0x3263: 0x000b, + // Block 0xca, offset 0x3280 + 0x32a7: 0x000c, 0x32a8: 0x000c, 0x32a9: 0x000c, + 0x32b3: 0x000b, 0x32b4: 0x000b, 0x32b5: 0x000b, + 0x32b6: 0x000b, 0x32b7: 0x000b, 0x32b8: 0x000b, 0x32b9: 0x000b, 0x32ba: 0x000b, 0x32bb: 0x000c, + 0x32bc: 0x000c, 0x32bd: 0x000c, 0x32be: 0x000c, 0x32bf: 0x000c, + // Block 0xcb, offset 0x32c0 + 0x32c0: 0x000c, 0x32c1: 0x000c, 0x32c2: 0x000c, 0x32c5: 0x000c, + 0x32c6: 0x000c, 0x32c7: 0x000c, 0x32c8: 0x000c, 0x32c9: 0x000c, 0x32ca: 0x000c, 0x32cb: 0x000c, + 0x32ea: 0x000c, 0x32eb: 0x000c, 0x32ec: 0x000c, 0x32ed: 0x000c, + // Block 0xcc, offset 0x3300 + 0x3300: 0x000a, 0x3301: 0x000a, 0x3302: 0x000c, 0x3303: 0x000c, 0x3304: 0x000c, 0x3305: 0x000a, + // Block 0xcd, offset 0x3340 + 0x3340: 0x000a, 0x3341: 0x000a, 0x3342: 0x000a, 0x3343: 0x000a, 0x3344: 0x000a, 0x3345: 0x000a, + 0x3346: 0x000a, 0x3347: 0x000a, 0x3348: 0x000a, 0x3349: 0x000a, 0x334a: 0x000a, 0x334b: 0x000a, + 0x334c: 0x000a, 0x334d: 0x000a, 0x334e: 0x000a, 0x334f: 0x000a, 0x3350: 0x000a, 0x3351: 0x000a, + 0x3352: 0x000a, 0x3353: 0x000a, 0x3354: 0x000a, 0x3355: 0x000a, 0x3356: 0x000a, + // Block 0xce, offset 0x3380 + 0x339b: 0x000a, + // Block 0xcf, offset 0x33c0 + 0x33d5: 0x000a, + // Block 0xd0, offset 0x3400 + 0x340f: 0x000a, + // Block 0xd1, offset 0x3440 + 0x3449: 0x000a, + // Block 0xd2, offset 0x3480 + 0x3483: 0x000a, + 0x348e: 0x0002, 0x348f: 0x0002, 0x3490: 0x0002, 0x3491: 0x0002, + 0x3492: 0x0002, 0x3493: 0x0002, 0x3494: 0x0002, 0x3495: 0x0002, 0x3496: 0x0002, 0x3497: 0x0002, + 0x3498: 0x0002, 0x3499: 0x0002, 0x349a: 0x0002, 0x349b: 0x0002, 0x349c: 0x0002, 0x349d: 0x0002, + 0x349e: 0x0002, 0x349f: 0x0002, 0x34a0: 0x0002, 0x34a1: 0x0002, 0x34a2: 0x0002, 0x34a3: 0x0002, + 0x34a4: 0x0002, 0x34a5: 0x0002, 0x34a6: 0x0002, 0x34a7: 0x0002, 0x34a8: 0x0002, 0x34a9: 0x0002, + 0x34aa: 0x0002, 0x34ab: 0x0002, 0x34ac: 0x0002, 0x34ad: 0x0002, 0x34ae: 0x0002, 0x34af: 0x0002, + 0x34b0: 0x0002, 0x34b1: 0x0002, 0x34b2: 0x0002, 0x34b3: 0x0002, 0x34b4: 0x0002, 0x34b5: 0x0002, + 0x34b6: 0x0002, 0x34b7: 0x0002, 0x34b8: 0x0002, 0x34b9: 0x0002, 0x34ba: 0x0002, 0x34bb: 0x0002, + 0x34bc: 0x0002, 0x34bd: 0x0002, 0x34be: 0x0002, 0x34bf: 0x0002, + // Block 0xd3, offset 0x34c0 + 0x34c0: 0x000c, 0x34c1: 0x000c, 0x34c2: 0x000c, 0x34c3: 0x000c, 0x34c4: 0x000c, 0x34c5: 0x000c, + 0x34c6: 0x000c, 0x34c7: 0x000c, 0x34c8: 0x000c, 0x34c9: 0x000c, 0x34ca: 0x000c, 0x34cb: 0x000c, + 0x34cc: 0x000c, 0x34cd: 0x000c, 0x34ce: 0x000c, 0x34cf: 0x000c, 0x34d0: 0x000c, 0x34d1: 0x000c, + 0x34d2: 0x000c, 0x34d3: 0x000c, 0x34d4: 0x000c, 0x34d5: 0x000c, 0x34d6: 0x000c, 0x34d7: 0x000c, + 0x34d8: 0x000c, 0x34d9: 0x000c, 0x34da: 0x000c, 0x34db: 0x000c, 0x34dc: 0x000c, 0x34dd: 0x000c, + 0x34de: 0x000c, 0x34df: 0x000c, 0x34e0: 0x000c, 0x34e1: 0x000c, 0x34e2: 0x000c, 0x34e3: 0x000c, + 0x34e4: 0x000c, 0x34e5: 0x000c, 0x34e6: 0x000c, 0x34e7: 0x000c, 0x34e8: 0x000c, 0x34e9: 0x000c, + 0x34ea: 0x000c, 0x34eb: 0x000c, 0x34ec: 0x000c, 0x34ed: 0x000c, 0x34ee: 0x000c, 0x34ef: 0x000c, + 0x34f0: 0x000c, 0x34f1: 0x000c, 0x34f2: 0x000c, 0x34f3: 0x000c, 0x34f4: 0x000c, 0x34f5: 0x000c, + 0x34f6: 0x000c, 0x34fb: 0x000c, + 0x34fc: 0x000c, 0x34fd: 0x000c, 0x34fe: 0x000c, 0x34ff: 0x000c, + // Block 0xd4, offset 0x3500 + 0x3500: 0x000c, 0x3501: 0x000c, 0x3502: 0x000c, 0x3503: 0x000c, 0x3504: 0x000c, 0x3505: 0x000c, + 0x3506: 0x000c, 0x3507: 0x000c, 0x3508: 0x000c, 0x3509: 0x000c, 0x350a: 0x000c, 0x350b: 0x000c, + 0x350c: 0x000c, 0x350d: 0x000c, 0x350e: 0x000c, 0x350f: 0x000c, 0x3510: 0x000c, 0x3511: 0x000c, + 0x3512: 0x000c, 0x3513: 0x000c, 0x3514: 0x000c, 0x3515: 0x000c, 0x3516: 0x000c, 0x3517: 0x000c, + 0x3518: 0x000c, 0x3519: 0x000c, 0x351a: 0x000c, 0x351b: 0x000c, 0x351c: 0x000c, 0x351d: 0x000c, + 0x351e: 0x000c, 0x351f: 0x000c, 0x3520: 0x000c, 0x3521: 0x000c, 0x3522: 0x000c, 0x3523: 0x000c, + 0x3524: 0x000c, 0x3525: 0x000c, 0x3526: 0x000c, 0x3527: 0x000c, 0x3528: 0x000c, 0x3529: 0x000c, + 0x352a: 0x000c, 0x352b: 0x000c, 0x352c: 0x000c, + 0x3535: 0x000c, + // Block 0xd5, offset 0x3540 + 0x3544: 0x000c, + 0x355b: 0x000c, 0x355c: 0x000c, 0x355d: 0x000c, + 0x355e: 0x000c, 0x355f: 0x000c, 0x3561: 0x000c, 0x3562: 0x000c, 0x3563: 0x000c, + 0x3564: 0x000c, 0x3565: 0x000c, 0x3566: 0x000c, 0x3567: 0x000c, 0x3568: 0x000c, 0x3569: 0x000c, + 0x356a: 0x000c, 0x356b: 0x000c, 0x356c: 0x000c, 0x356d: 0x000c, 0x356e: 0x000c, 0x356f: 0x000c, + // Block 0xd6, offset 0x3580 + 0x3580: 0x000c, 0x3581: 0x000c, 0x3582: 0x000c, 0x3583: 0x000c, 0x3584: 0x000c, 0x3585: 0x000c, + 0x3586: 0x000c, 0x3588: 0x000c, 0x3589: 0x000c, 0x358a: 0x000c, 0x358b: 0x000c, + 0x358c: 0x000c, 0x358d: 0x000c, 0x358e: 0x000c, 0x358f: 0x000c, 0x3590: 0x000c, 0x3591: 0x000c, + 0x3592: 0x000c, 0x3593: 0x000c, 0x3594: 0x000c, 0x3595: 0x000c, 0x3596: 0x000c, 0x3597: 0x000c, + 0x3598: 0x000c, 0x359b: 0x000c, 0x359c: 0x000c, 0x359d: 0x000c, + 0x359e: 0x000c, 0x359f: 0x000c, 0x35a0: 0x000c, 0x35a1: 0x000c, 0x35a3: 0x000c, + 0x35a4: 0x000c, 0x35a6: 0x000c, 0x35a7: 0x000c, 0x35a8: 0x000c, 0x35a9: 0x000c, + 0x35aa: 0x000c, + // Block 0xd7, offset 0x35c0 + 0x35ec: 0x000c, 0x35ed: 0x000c, 0x35ee: 0x000c, 0x35ef: 0x000c, + 0x35ff: 0x0004, + // Block 0xd8, offset 0x3600 + 0x3600: 0x0001, 0x3601: 0x0001, 0x3602: 0x0001, 0x3603: 0x0001, 0x3604: 0x0001, 0x3605: 0x0001, + 0x3606: 0x0001, 0x3607: 0x0001, 0x3608: 0x0001, 0x3609: 0x0001, 0x360a: 0x0001, 0x360b: 0x0001, + 0x360c: 0x0001, 0x360d: 0x0001, 0x360e: 0x0001, 0x360f: 0x0001, 0x3610: 0x000c, 0x3611: 0x000c, + 0x3612: 0x000c, 0x3613: 0x000c, 0x3614: 0x000c, 0x3615: 0x000c, 0x3616: 0x000c, 0x3617: 0x0001, + 0x3618: 0x0001, 0x3619: 0x0001, 0x361a: 0x0001, 0x361b: 0x0001, 0x361c: 0x0001, 0x361d: 0x0001, + 0x361e: 0x0001, 0x361f: 0x0001, 0x3620: 0x0001, 0x3621: 0x0001, 0x3622: 0x0001, 0x3623: 0x0001, + 0x3624: 0x0001, 0x3625: 0x0001, 0x3626: 0x0001, 0x3627: 0x0001, 0x3628: 0x0001, 0x3629: 0x0001, + 0x362a: 0x0001, 0x362b: 0x0001, 0x362c: 0x0001, 0x362d: 0x0001, 0x362e: 0x0001, 0x362f: 0x0001, + 0x3630: 0x0001, 0x3631: 0x0001, 0x3632: 0x0001, 0x3633: 0x0001, 0x3634: 0x0001, 0x3635: 0x0001, + 0x3636: 0x0001, 0x3637: 0x0001, 0x3638: 0x0001, 0x3639: 0x0001, 0x363a: 0x0001, 0x363b: 0x0001, + 0x363c: 0x0001, 0x363d: 0x0001, 0x363e: 0x0001, 0x363f: 0x0001, + // Block 0xd9, offset 0x3640 + 0x3640: 0x0001, 0x3641: 0x0001, 0x3642: 0x0001, 0x3643: 0x0001, 0x3644: 0x000c, 0x3645: 0x000c, + 0x3646: 0x000c, 0x3647: 0x000c, 0x3648: 0x000c, 0x3649: 0x000c, 0x364a: 0x000c, 0x364b: 0x0001, + 0x364c: 0x0001, 0x364d: 0x0001, 0x364e: 0x0001, 0x364f: 0x0001, 0x3650: 0x0001, 0x3651: 0x0001, + 0x3652: 0x0001, 0x3653: 0x0001, 0x3654: 0x0001, 0x3655: 0x0001, 0x3656: 0x0001, 0x3657: 0x0001, + 0x3658: 0x0001, 0x3659: 0x0001, 0x365a: 0x0001, 0x365b: 0x0001, 0x365c: 0x0001, 0x365d: 0x0001, + 0x365e: 0x0001, 0x365f: 0x0001, 0x3660: 0x0001, 0x3661: 0x0001, 0x3662: 0x0001, 0x3663: 0x0001, + 0x3664: 0x0001, 0x3665: 0x0001, 0x3666: 0x0001, 0x3667: 0x0001, 0x3668: 0x0001, 0x3669: 0x0001, + 0x366a: 0x0001, 0x366b: 0x0001, 0x366c: 0x0001, 0x366d: 0x0001, 0x366e: 0x0001, 0x366f: 0x0001, + 0x3670: 0x0001, 0x3671: 0x0001, 0x3672: 0x0001, 0x3673: 0x0001, 0x3674: 0x0001, 0x3675: 0x0001, + 0x3676: 0x0001, 0x3677: 0x0001, 0x3678: 0x0001, 0x3679: 0x0001, 0x367a: 0x0001, 0x367b: 0x0001, + 0x367c: 0x0001, 0x367d: 0x0001, 0x367e: 0x0001, 0x367f: 0x0001, + // Block 0xda, offset 0x3680 + 0x3680: 0x000d, 0x3681: 0x000d, 0x3682: 0x000d, 0x3683: 0x000d, 0x3684: 0x000d, 0x3685: 0x000d, + 0x3686: 0x000d, 0x3687: 0x000d, 0x3688: 0x000d, 0x3689: 0x000d, 0x368a: 0x000d, 0x368b: 0x000d, + 0x368c: 0x000d, 0x368d: 0x000d, 0x368e: 0x000d, 0x368f: 0x000d, 0x3690: 0x0001, 0x3691: 0x0001, + 0x3692: 0x0001, 0x3693: 0x0001, 0x3694: 0x0001, 0x3695: 0x0001, 0x3696: 0x0001, 0x3697: 0x0001, + 0x3698: 0x0001, 0x3699: 0x0001, 0x369a: 0x0001, 0x369b: 0x0001, 0x369c: 0x0001, 0x369d: 0x0001, + 0x369e: 0x0001, 0x369f: 0x0001, 0x36a0: 0x0001, 0x36a1: 0x0001, 0x36a2: 0x0001, 0x36a3: 0x0001, + 0x36a4: 0x0001, 0x36a5: 0x0001, 0x36a6: 0x0001, 0x36a7: 0x0001, 0x36a8: 0x0001, 0x36a9: 0x0001, + 0x36aa: 0x0001, 0x36ab: 0x0001, 0x36ac: 0x0001, 0x36ad: 0x0001, 0x36ae: 0x0001, 0x36af: 0x0001, + 0x36b0: 0x0001, 0x36b1: 0x0001, 0x36b2: 0x0001, 0x36b3: 0x0001, 0x36b4: 0x0001, 0x36b5: 0x0001, + 0x36b6: 0x0001, 0x36b7: 0x0001, 0x36b8: 0x0001, 0x36b9: 0x0001, 0x36ba: 0x0001, 0x36bb: 0x0001, + 0x36bc: 0x0001, 0x36bd: 0x0001, 0x36be: 0x0001, 0x36bf: 0x0001, + // Block 0xdb, offset 0x36c0 + 0x36c0: 0x000d, 0x36c1: 0x000d, 0x36c2: 0x000d, 0x36c3: 0x000d, 0x36c4: 0x000d, 0x36c5: 0x000d, + 0x36c6: 0x000d, 0x36c7: 0x000d, 0x36c8: 0x000d, 0x36c9: 0x000d, 0x36ca: 0x000d, 0x36cb: 0x000d, + 0x36cc: 0x000d, 0x36cd: 0x000d, 0x36ce: 0x000d, 0x36cf: 0x000d, 0x36d0: 0x000d, 0x36d1: 0x000d, + 0x36d2: 0x000d, 0x36d3: 0x000d, 0x36d4: 0x000d, 0x36d5: 0x000d, 0x36d6: 0x000d, 0x36d7: 0x000d, + 0x36d8: 0x000d, 0x36d9: 0x000d, 0x36da: 0x000d, 0x36db: 0x000d, 0x36dc: 0x000d, 0x36dd: 0x000d, + 0x36de: 0x000d, 0x36df: 0x000d, 0x36e0: 0x000d, 0x36e1: 0x000d, 0x36e2: 0x000d, 0x36e3: 0x000d, + 0x36e4: 0x000d, 0x36e5: 0x000d, 0x36e6: 0x000d, 0x36e7: 0x000d, 0x36e8: 0x000d, 0x36e9: 0x000d, + 0x36ea: 0x000d, 0x36eb: 0x000d, 0x36ec: 0x000d, 0x36ed: 0x000d, 0x36ee: 0x000d, 0x36ef: 0x000d, + 0x36f0: 0x000a, 0x36f1: 0x000a, 0x36f2: 0x000d, 0x36f3: 0x000d, 0x36f4: 0x000d, 0x36f5: 0x000d, + 0x36f6: 0x000d, 0x36f7: 0x000d, 0x36f8: 0x000d, 0x36f9: 0x000d, 0x36fa: 0x000d, 0x36fb: 0x000d, + 0x36fc: 0x000d, 0x36fd: 0x000d, 0x36fe: 0x000d, 0x36ff: 0x000d, + // Block 0xdc, offset 0x3700 + 0x3700: 0x000a, 0x3701: 0x000a, 0x3702: 0x000a, 0x3703: 0x000a, 0x3704: 0x000a, 0x3705: 0x000a, + 0x3706: 0x000a, 0x3707: 0x000a, 0x3708: 0x000a, 0x3709: 0x000a, 0x370a: 0x000a, 0x370b: 0x000a, + 0x370c: 0x000a, 0x370d: 0x000a, 0x370e: 0x000a, 0x370f: 0x000a, 0x3710: 0x000a, 0x3711: 0x000a, + 0x3712: 0x000a, 0x3713: 0x000a, 0x3714: 0x000a, 0x3715: 0x000a, 0x3716: 0x000a, 0x3717: 0x000a, + 0x3718: 0x000a, 0x3719: 0x000a, 0x371a: 0x000a, 0x371b: 0x000a, 0x371c: 0x000a, 0x371d: 0x000a, + 0x371e: 0x000a, 0x371f: 0x000a, 0x3720: 0x000a, 0x3721: 0x000a, 0x3722: 0x000a, 0x3723: 0x000a, + 0x3724: 0x000a, 0x3725: 0x000a, 0x3726: 0x000a, 0x3727: 0x000a, 0x3728: 0x000a, 0x3729: 0x000a, + 0x372a: 0x000a, 0x372b: 0x000a, + 0x3730: 0x000a, 0x3731: 0x000a, 0x3732: 0x000a, 0x3733: 0x000a, 0x3734: 0x000a, 0x3735: 0x000a, + 0x3736: 0x000a, 0x3737: 0x000a, 0x3738: 0x000a, 0x3739: 0x000a, 0x373a: 0x000a, 0x373b: 0x000a, + 0x373c: 0x000a, 0x373d: 0x000a, 0x373e: 0x000a, 0x373f: 0x000a, + // Block 0xdd, offset 0x3740 + 0x3740: 0x000a, 0x3741: 0x000a, 0x3742: 0x000a, 0x3743: 0x000a, 0x3744: 0x000a, 0x3745: 0x000a, + 0x3746: 0x000a, 0x3747: 0x000a, 0x3748: 0x000a, 0x3749: 0x000a, 0x374a: 0x000a, 0x374b: 0x000a, + 0x374c: 0x000a, 0x374d: 0x000a, 0x374e: 0x000a, 0x374f: 0x000a, 0x3750: 0x000a, 0x3751: 0x000a, + 0x3752: 0x000a, 0x3753: 0x000a, + 0x3760: 0x000a, 0x3761: 0x000a, 0x3762: 0x000a, 0x3763: 0x000a, + 0x3764: 0x000a, 0x3765: 0x000a, 0x3766: 0x000a, 0x3767: 0x000a, 0x3768: 0x000a, 0x3769: 0x000a, + 0x376a: 0x000a, 0x376b: 0x000a, 0x376c: 0x000a, 0x376d: 0x000a, 0x376e: 0x000a, + 0x3771: 0x000a, 0x3772: 0x000a, 0x3773: 0x000a, 0x3774: 0x000a, 0x3775: 0x000a, + 0x3776: 0x000a, 0x3777: 0x000a, 0x3778: 0x000a, 0x3779: 0x000a, 0x377a: 0x000a, 0x377b: 0x000a, + 0x377c: 0x000a, 0x377d: 0x000a, 0x377e: 0x000a, 0x377f: 0x000a, + // Block 0xde, offset 0x3780 + 0x3781: 0x000a, 0x3782: 0x000a, 0x3783: 0x000a, 0x3784: 0x000a, 0x3785: 0x000a, + 0x3786: 0x000a, 0x3787: 0x000a, 0x3788: 0x000a, 0x3789: 0x000a, 0x378a: 0x000a, 0x378b: 0x000a, + 0x378c: 0x000a, 0x378d: 0x000a, 0x378e: 0x000a, 0x378f: 0x000a, 0x3791: 0x000a, + 0x3792: 0x000a, 0x3793: 0x000a, 0x3794: 0x000a, 0x3795: 0x000a, 0x3796: 0x000a, 0x3797: 0x000a, + 0x3798: 0x000a, 0x3799: 0x000a, 0x379a: 0x000a, 0x379b: 0x000a, 0x379c: 0x000a, 0x379d: 0x000a, + 0x379e: 0x000a, 0x379f: 0x000a, 0x37a0: 0x000a, 0x37a1: 0x000a, 0x37a2: 0x000a, 0x37a3: 0x000a, + 0x37a4: 0x000a, 0x37a5: 0x000a, 0x37a6: 0x000a, 0x37a7: 0x000a, 0x37a8: 0x000a, 0x37a9: 0x000a, + 0x37aa: 0x000a, 0x37ab: 0x000a, 0x37ac: 0x000a, 0x37ad: 0x000a, 0x37ae: 0x000a, 0x37af: 0x000a, + 0x37b0: 0x000a, 0x37b1: 0x000a, 0x37b2: 0x000a, 0x37b3: 0x000a, 0x37b4: 0x000a, 0x37b5: 0x000a, + // Block 0xdf, offset 0x37c0 + 0x37c0: 0x0002, 0x37c1: 0x0002, 0x37c2: 0x0002, 0x37c3: 0x0002, 0x37c4: 0x0002, 0x37c5: 0x0002, + 0x37c6: 0x0002, 0x37c7: 0x0002, 0x37c8: 0x0002, 0x37c9: 0x0002, 0x37ca: 0x0002, 0x37cb: 0x000a, + 0x37cc: 0x000a, + 0x37ef: 0x000a, + // Block 0xe0, offset 0x3800 + 0x382a: 0x000a, 0x382b: 0x000a, 0x382c: 0x000a, + // Block 0xe1, offset 0x3840 + 0x3860: 0x000a, 0x3861: 0x000a, 0x3862: 0x000a, 0x3863: 0x000a, + 0x3864: 0x000a, 0x3865: 0x000a, + // Block 0xe2, offset 0x3880 + 0x3880: 0x000a, 0x3881: 0x000a, 0x3882: 0x000a, 0x3883: 0x000a, 0x3884: 0x000a, 0x3885: 0x000a, + 0x3886: 0x000a, 0x3887: 0x000a, 0x3888: 0x000a, 0x3889: 0x000a, 0x388a: 0x000a, 0x388b: 0x000a, + 0x388c: 0x000a, 0x388d: 0x000a, 0x388e: 0x000a, 0x388f: 0x000a, 0x3890: 0x000a, 0x3891: 0x000a, + 0x3892: 0x000a, 0x3893: 0x000a, 0x3894: 0x000a, 0x3895: 0x000a, + 0x38a0: 0x000a, 0x38a1: 0x000a, 0x38a2: 0x000a, 0x38a3: 0x000a, + 0x38a4: 0x000a, 0x38a5: 0x000a, 0x38a6: 0x000a, 0x38a7: 0x000a, 0x38a8: 0x000a, 0x38a9: 0x000a, + 0x38aa: 0x000a, 0x38ab: 0x000a, 0x38ac: 0x000a, + 0x38b0: 0x000a, 0x38b1: 0x000a, 0x38b2: 0x000a, 0x38b3: 0x000a, 0x38b4: 0x000a, 0x38b5: 0x000a, + 0x38b6: 0x000a, 0x38b7: 0x000a, 0x38b8: 0x000a, 0x38b9: 0x000a, 0x38ba: 0x000a, + // Block 0xe3, offset 0x38c0 + 0x38c0: 0x000a, 0x38c1: 0x000a, 0x38c2: 0x000a, 0x38c3: 0x000a, 0x38c4: 0x000a, 0x38c5: 0x000a, + 0x38c6: 0x000a, 0x38c7: 0x000a, 0x38c8: 0x000a, 0x38c9: 0x000a, 0x38ca: 0x000a, 0x38cb: 0x000a, + 0x38cc: 0x000a, 0x38cd: 0x000a, 0x38ce: 0x000a, 0x38cf: 0x000a, 0x38d0: 0x000a, 0x38d1: 0x000a, + 0x38d2: 0x000a, 0x38d3: 0x000a, 0x38d4: 0x000a, 0x38d5: 0x000a, 0x38d6: 0x000a, 0x38d7: 0x000a, + 0x38d8: 0x000a, + 0x38e0: 0x000a, 0x38e1: 0x000a, 0x38e2: 0x000a, 0x38e3: 0x000a, + 0x38e4: 0x000a, 0x38e5: 0x000a, 0x38e6: 0x000a, 0x38e7: 0x000a, 0x38e8: 0x000a, 0x38e9: 0x000a, + 0x38ea: 0x000a, 0x38eb: 0x000a, + // Block 0xe4, offset 0x3900 + 0x3900: 0x000a, 0x3901: 0x000a, 0x3902: 0x000a, 0x3903: 0x000a, 0x3904: 0x000a, 0x3905: 0x000a, + 0x3906: 0x000a, 0x3907: 0x000a, 0x3908: 0x000a, 0x3909: 0x000a, 0x390a: 0x000a, 0x390b: 0x000a, + 0x3910: 0x000a, 0x3911: 0x000a, + 0x3912: 0x000a, 0x3913: 0x000a, 0x3914: 0x000a, 0x3915: 0x000a, 0x3916: 0x000a, 0x3917: 0x000a, + 0x3918: 0x000a, 0x3919: 0x000a, 0x391a: 0x000a, 0x391b: 0x000a, 0x391c: 0x000a, 0x391d: 0x000a, + 0x391e: 0x000a, 0x391f: 0x000a, 0x3920: 0x000a, 0x3921: 0x000a, 0x3922: 0x000a, 0x3923: 0x000a, + 0x3924: 0x000a, 0x3925: 0x000a, 0x3926: 0x000a, 0x3927: 0x000a, 0x3928: 0x000a, 0x3929: 0x000a, + 0x392a: 0x000a, 0x392b: 0x000a, 0x392c: 0x000a, 0x392d: 0x000a, 0x392e: 0x000a, 0x392f: 0x000a, + 0x3930: 0x000a, 0x3931: 0x000a, 0x3932: 0x000a, 0x3933: 0x000a, 0x3934: 0x000a, 0x3935: 0x000a, + 0x3936: 0x000a, 0x3937: 0x000a, 0x3938: 0x000a, 0x3939: 0x000a, 0x393a: 0x000a, 0x393b: 0x000a, + 0x393c: 0x000a, 0x393d: 0x000a, 0x393e: 0x000a, 0x393f: 0x000a, + // Block 0xe5, offset 0x3940 + 0x3940: 0x000a, 0x3941: 0x000a, 0x3942: 0x000a, 0x3943: 0x000a, 0x3944: 0x000a, 0x3945: 0x000a, + 0x3946: 0x000a, 0x3947: 0x000a, + 0x3950: 0x000a, 0x3951: 0x000a, + 0x3952: 0x000a, 0x3953: 0x000a, 0x3954: 0x000a, 0x3955: 0x000a, 0x3956: 0x000a, 0x3957: 0x000a, + 0x3958: 0x000a, 0x3959: 0x000a, + 0x3960: 0x000a, 0x3961: 0x000a, 0x3962: 0x000a, 0x3963: 0x000a, + 0x3964: 0x000a, 0x3965: 0x000a, 0x3966: 0x000a, 0x3967: 0x000a, 0x3968: 0x000a, 0x3969: 0x000a, + 0x396a: 0x000a, 0x396b: 0x000a, 0x396c: 0x000a, 0x396d: 0x000a, 0x396e: 0x000a, 0x396f: 0x000a, + 0x3970: 0x000a, 0x3971: 0x000a, 0x3972: 0x000a, 0x3973: 0x000a, 0x3974: 0x000a, 0x3975: 0x000a, + 0x3976: 0x000a, 0x3977: 0x000a, 0x3978: 0x000a, 0x3979: 0x000a, 0x397a: 0x000a, 0x397b: 0x000a, + 0x397c: 0x000a, 0x397d: 0x000a, 0x397e: 0x000a, 0x397f: 0x000a, + // Block 0xe6, offset 0x3980 + 0x3980: 0x000a, 0x3981: 0x000a, 0x3982: 0x000a, 0x3983: 0x000a, 0x3984: 0x000a, 0x3985: 0x000a, + 0x3986: 0x000a, 0x3987: 0x000a, + 0x3990: 0x000a, 0x3991: 0x000a, + 0x3992: 0x000a, 0x3993: 0x000a, 0x3994: 0x000a, 0x3995: 0x000a, 0x3996: 0x000a, 0x3997: 0x000a, + 0x3998: 0x000a, 0x3999: 0x000a, 0x399a: 0x000a, 0x399b: 0x000a, 0x399c: 0x000a, 0x399d: 0x000a, + 0x399e: 0x000a, 0x399f: 0x000a, 0x39a0: 0x000a, 0x39a1: 0x000a, 0x39a2: 0x000a, 0x39a3: 0x000a, + 0x39a4: 0x000a, 0x39a5: 0x000a, 0x39a6: 0x000a, 0x39a7: 0x000a, 0x39a8: 0x000a, 0x39a9: 0x000a, + 0x39aa: 0x000a, 0x39ab: 0x000a, 0x39ac: 0x000a, 0x39ad: 0x000a, + // Block 0xe7, offset 0x39c0 + 0x39c0: 0x000a, 0x39c1: 0x000a, 0x39c2: 0x000a, 0x39c3: 0x000a, 0x39c4: 0x000a, 0x39c5: 0x000a, + 0x39c6: 0x000a, 0x39c7: 0x000a, 0x39c8: 0x000a, 0x39c9: 0x000a, 0x39ca: 0x000a, 0x39cb: 0x000a, + 0x39cd: 0x000a, 0x39ce: 0x000a, 0x39cf: 0x000a, 0x39d0: 0x000a, 0x39d1: 0x000a, + 0x39d2: 0x000a, 0x39d3: 0x000a, 0x39d4: 0x000a, 0x39d5: 0x000a, 0x39d6: 0x000a, 0x39d7: 0x000a, + 0x39d8: 0x000a, 0x39d9: 0x000a, 0x39da: 0x000a, 0x39db: 0x000a, 0x39dc: 0x000a, 0x39dd: 0x000a, + 0x39de: 0x000a, 0x39df: 0x000a, 0x39e0: 0x000a, 0x39e1: 0x000a, 0x39e2: 0x000a, 0x39e3: 0x000a, + 0x39e4: 0x000a, 0x39e5: 0x000a, 0x39e6: 0x000a, 0x39e7: 0x000a, 0x39e8: 0x000a, 0x39e9: 0x000a, + 0x39ea: 0x000a, 0x39eb: 0x000a, 0x39ec: 0x000a, 0x39ed: 0x000a, 0x39ee: 0x000a, 0x39ef: 0x000a, + 0x39f0: 0x000a, 0x39f1: 0x000a, 0x39f2: 0x000a, 0x39f3: 0x000a, 0x39f4: 0x000a, 0x39f5: 0x000a, + 0x39f6: 0x000a, 0x39f7: 0x000a, 0x39f8: 0x000a, 0x39f9: 0x000a, 0x39fa: 0x000a, 0x39fb: 0x000a, + 0x39fc: 0x000a, 0x39fd: 0x000a, 0x39fe: 0x000a, 0x39ff: 0x000a, + // Block 0xe8, offset 0x3a00 + 0x3a00: 0x000a, 0x3a01: 0x000a, 0x3a02: 0x000a, 0x3a03: 0x000a, 0x3a04: 0x000a, 0x3a05: 0x000a, + 0x3a06: 0x000a, 0x3a07: 0x000a, 0x3a08: 0x000a, 0x3a09: 0x000a, 0x3a0a: 0x000a, 0x3a0b: 0x000a, + 0x3a0c: 0x000a, 0x3a0d: 0x000a, 0x3a0e: 0x000a, 0x3a0f: 0x000a, 0x3a10: 0x000a, 0x3a11: 0x000a, + 0x3a12: 0x000a, 0x3a13: 0x000a, 0x3a14: 0x000a, 0x3a15: 0x000a, 0x3a16: 0x000a, 0x3a17: 0x000a, + 0x3a18: 0x000a, 0x3a19: 0x000a, 0x3a1a: 0x000a, 0x3a1b: 0x000a, 0x3a1c: 0x000a, 0x3a1d: 0x000a, + 0x3a1e: 0x000a, 0x3a1f: 0x000a, 0x3a20: 0x000a, 0x3a21: 0x000a, 0x3a22: 0x000a, 0x3a23: 0x000a, + 0x3a24: 0x000a, 0x3a25: 0x000a, 0x3a26: 0x000a, 0x3a27: 0x000a, 0x3a28: 0x000a, 0x3a29: 0x000a, + 0x3a2a: 0x000a, 0x3a2b: 0x000a, 0x3a2c: 0x000a, 0x3a2d: 0x000a, 0x3a2e: 0x000a, 0x3a2f: 0x000a, + 0x3a30: 0x000a, 0x3a31: 0x000a, 0x3a33: 0x000a, 0x3a34: 0x000a, 0x3a35: 0x000a, + 0x3a36: 0x000a, 0x3a3a: 0x000a, 0x3a3b: 0x000a, + 0x3a3c: 0x000a, 0x3a3d: 0x000a, 0x3a3e: 0x000a, 0x3a3f: 0x000a, + // Block 0xe9, offset 0x3a40 + 0x3a40: 0x000a, 0x3a41: 0x000a, 0x3a42: 0x000a, 0x3a43: 0x000a, 0x3a44: 0x000a, 0x3a45: 0x000a, + 0x3a46: 0x000a, 0x3a47: 0x000a, 0x3a48: 0x000a, 0x3a49: 0x000a, 0x3a4a: 0x000a, 0x3a4b: 0x000a, + 0x3a4c: 0x000a, 0x3a4d: 0x000a, 0x3a4e: 0x000a, 0x3a4f: 0x000a, 0x3a50: 0x000a, 0x3a51: 0x000a, + 0x3a52: 0x000a, 0x3a53: 0x000a, 0x3a54: 0x000a, 0x3a55: 0x000a, 0x3a56: 0x000a, 0x3a57: 0x000a, + 0x3a58: 0x000a, 0x3a59: 0x000a, 0x3a5a: 0x000a, 0x3a5b: 0x000a, 0x3a5c: 0x000a, 0x3a5d: 0x000a, + 0x3a5e: 0x000a, 0x3a5f: 0x000a, 0x3a60: 0x000a, 0x3a61: 0x000a, 0x3a62: 0x000a, + 0x3a65: 0x000a, 0x3a66: 0x000a, 0x3a67: 0x000a, 0x3a68: 0x000a, 0x3a69: 0x000a, + 0x3a6a: 0x000a, 0x3a6e: 0x000a, 0x3a6f: 0x000a, + 0x3a70: 0x000a, 0x3a71: 0x000a, 0x3a72: 0x000a, 0x3a73: 0x000a, 0x3a74: 0x000a, 0x3a75: 0x000a, + 0x3a76: 0x000a, 0x3a77: 0x000a, 0x3a78: 0x000a, 0x3a79: 0x000a, 0x3a7a: 0x000a, 0x3a7b: 0x000a, + 0x3a7c: 0x000a, 0x3a7d: 0x000a, 0x3a7e: 0x000a, 0x3a7f: 0x000a, + // Block 0xea, offset 0x3a80 + 0x3a80: 0x000a, 0x3a81: 0x000a, 0x3a82: 0x000a, 0x3a83: 0x000a, 0x3a84: 0x000a, 0x3a85: 0x000a, + 0x3a86: 0x000a, 0x3a87: 0x000a, 0x3a88: 0x000a, 0x3a89: 0x000a, 0x3a8a: 0x000a, + 0x3a8d: 0x000a, 0x3a8e: 0x000a, 0x3a8f: 0x000a, 0x3a90: 0x000a, 0x3a91: 0x000a, + 0x3a92: 0x000a, 0x3a93: 0x000a, 0x3a94: 0x000a, 0x3a95: 0x000a, 0x3a96: 0x000a, 0x3a97: 0x000a, + 0x3a98: 0x000a, 0x3a99: 0x000a, 0x3a9a: 0x000a, 0x3a9b: 0x000a, 0x3a9c: 0x000a, 0x3a9d: 0x000a, + 0x3a9e: 0x000a, 0x3a9f: 0x000a, 0x3aa0: 0x000a, 0x3aa1: 0x000a, 0x3aa2: 0x000a, 0x3aa3: 0x000a, + 0x3aa4: 0x000a, 0x3aa5: 0x000a, 0x3aa6: 0x000a, 0x3aa7: 0x000a, 0x3aa8: 0x000a, 0x3aa9: 0x000a, + 0x3aaa: 0x000a, 0x3aab: 0x000a, 0x3aac: 0x000a, 0x3aad: 0x000a, 0x3aae: 0x000a, 0x3aaf: 0x000a, + 0x3ab0: 0x000a, 0x3ab1: 0x000a, 0x3ab2: 0x000a, 0x3ab3: 0x000a, 0x3ab4: 0x000a, 0x3ab5: 0x000a, + 0x3ab6: 0x000a, 0x3ab7: 0x000a, 0x3ab8: 0x000a, 0x3ab9: 0x000a, 0x3aba: 0x000a, 0x3abb: 0x000a, + 0x3abc: 0x000a, 0x3abd: 0x000a, 0x3abe: 0x000a, 0x3abf: 0x000a, + // Block 0xeb, offset 0x3ac0 + 0x3ac0: 0x000a, 0x3ac1: 0x000a, 0x3ac2: 0x000a, 0x3ac3: 0x000a, 0x3ac4: 0x000a, 0x3ac5: 0x000a, + 0x3ac6: 0x000a, 0x3ac7: 0x000a, 0x3ac8: 0x000a, 0x3ac9: 0x000a, 0x3aca: 0x000a, 0x3acb: 0x000a, + 0x3acc: 0x000a, 0x3acd: 0x000a, 0x3ace: 0x000a, 0x3acf: 0x000a, 0x3ad0: 0x000a, 0x3ad1: 0x000a, + 0x3ad2: 0x000a, 0x3ad3: 0x000a, + 0x3ae0: 0x000a, 0x3ae1: 0x000a, 0x3ae2: 0x000a, 0x3ae3: 0x000a, + 0x3ae4: 0x000a, 0x3ae5: 0x000a, 0x3ae6: 0x000a, 0x3ae7: 0x000a, 0x3ae8: 0x000a, 0x3ae9: 0x000a, + 0x3aea: 0x000a, 0x3aeb: 0x000a, 0x3aec: 0x000a, 0x3aed: 0x000a, + 0x3af0: 0x000a, 0x3af1: 0x000a, 0x3af2: 0x000a, 0x3af3: 0x000a, + 0x3af8: 0x000a, 0x3af9: 0x000a, 0x3afa: 0x000a, + // Block 0xec, offset 0x3b00 + 0x3b00: 0x000a, 0x3b01: 0x000a, 0x3b02: 0x000a, + 0x3b10: 0x000a, 0x3b11: 0x000a, + 0x3b12: 0x000a, 0x3b13: 0x000a, 0x3b14: 0x000a, 0x3b15: 0x000a, + // Block 0xed, offset 0x3b40 + 0x3b7e: 0x000b, 0x3b7f: 0x000b, + // Block 0xee, offset 0x3b80 + 0x3b80: 0x000b, 0x3b81: 0x000b, 0x3b82: 0x000b, 0x3b83: 0x000b, 0x3b84: 0x000b, 0x3b85: 0x000b, + 0x3b86: 0x000b, 0x3b87: 0x000b, 0x3b88: 0x000b, 0x3b89: 0x000b, 0x3b8a: 0x000b, 0x3b8b: 0x000b, + 0x3b8c: 0x000b, 0x3b8d: 0x000b, 0x3b8e: 0x000b, 0x3b8f: 0x000b, 0x3b90: 0x000b, 0x3b91: 0x000b, + 0x3b92: 0x000b, 0x3b93: 0x000b, 0x3b94: 0x000b, 0x3b95: 0x000b, 0x3b96: 0x000b, 0x3b97: 0x000b, + 0x3b98: 0x000b, 0x3b99: 0x000b, 0x3b9a: 0x000b, 0x3b9b: 0x000b, 0x3b9c: 0x000b, 0x3b9d: 0x000b, + 0x3b9e: 0x000b, 0x3b9f: 0x000b, 0x3ba0: 0x000b, 0x3ba1: 0x000b, 0x3ba2: 0x000b, 0x3ba3: 0x000b, + 0x3ba4: 0x000b, 0x3ba5: 0x000b, 0x3ba6: 0x000b, 0x3ba7: 0x000b, 0x3ba8: 0x000b, 0x3ba9: 0x000b, + 0x3baa: 0x000b, 0x3bab: 0x000b, 0x3bac: 0x000b, 0x3bad: 0x000b, 0x3bae: 0x000b, 0x3baf: 0x000b, + 0x3bb0: 0x000b, 0x3bb1: 0x000b, 0x3bb2: 0x000b, 0x3bb3: 0x000b, 0x3bb4: 0x000b, 0x3bb5: 0x000b, + 0x3bb6: 0x000b, 0x3bb7: 0x000b, 0x3bb8: 0x000b, 0x3bb9: 0x000b, 0x3bba: 0x000b, 0x3bbb: 0x000b, + 0x3bbc: 0x000b, 0x3bbd: 0x000b, 0x3bbe: 0x000b, 0x3bbf: 0x000b, + // Block 0xef, offset 0x3bc0 + 0x3bc0: 0x000c, 0x3bc1: 0x000c, 0x3bc2: 0x000c, 0x3bc3: 0x000c, 0x3bc4: 0x000c, 0x3bc5: 0x000c, + 0x3bc6: 0x000c, 0x3bc7: 0x000c, 0x3bc8: 0x000c, 0x3bc9: 0x000c, 0x3bca: 0x000c, 0x3bcb: 0x000c, + 0x3bcc: 0x000c, 0x3bcd: 0x000c, 0x3bce: 0x000c, 0x3bcf: 0x000c, 0x3bd0: 0x000c, 0x3bd1: 0x000c, + 0x3bd2: 0x000c, 0x3bd3: 0x000c, 0x3bd4: 0x000c, 0x3bd5: 0x000c, 0x3bd6: 0x000c, 0x3bd7: 0x000c, + 0x3bd8: 0x000c, 0x3bd9: 0x000c, 0x3bda: 0x000c, 0x3bdb: 0x000c, 0x3bdc: 0x000c, 0x3bdd: 0x000c, + 0x3bde: 0x000c, 0x3bdf: 0x000c, 0x3be0: 0x000c, 0x3be1: 0x000c, 0x3be2: 0x000c, 0x3be3: 0x000c, + 0x3be4: 0x000c, 0x3be5: 0x000c, 0x3be6: 0x000c, 0x3be7: 0x000c, 0x3be8: 0x000c, 0x3be9: 0x000c, + 0x3bea: 0x000c, 0x3beb: 0x000c, 0x3bec: 0x000c, 0x3bed: 0x000c, 0x3bee: 0x000c, 0x3bef: 0x000c, + 0x3bf0: 0x000b, 0x3bf1: 0x000b, 0x3bf2: 0x000b, 0x3bf3: 0x000b, 0x3bf4: 0x000b, 0x3bf5: 0x000b, + 0x3bf6: 0x000b, 0x3bf7: 0x000b, 0x3bf8: 0x000b, 0x3bf9: 0x000b, 0x3bfa: 0x000b, 0x3bfb: 0x000b, + 0x3bfc: 0x000b, 0x3bfd: 0x000b, 0x3bfe: 0x000b, 0x3bff: 0x000b, +} + +// bidiIndex: 24 blocks, 1536 entries, 1536 bytes +// Block 0 is the zero block. +var bidiIndex = [1536]uint8{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xc2: 0x01, 0xc3: 0x02, + 0xca: 0x03, 0xcb: 0x04, 0xcc: 0x05, 0xcd: 0x06, 0xce: 0x07, 0xcf: 0x08, + 0xd2: 0x09, 0xd6: 0x0a, 0xd7: 0x0b, + 0xd8: 0x0c, 0xd9: 0x0d, 0xda: 0x0e, 0xdb: 0x0f, 0xdc: 0x10, 0xdd: 0x11, 0xde: 0x12, 0xdf: 0x13, + 0xe0: 0x02, 0xe1: 0x03, 0xe2: 0x04, 0xe3: 0x05, 0xe4: 0x06, + 0xea: 0x07, 0xef: 0x08, + 0xf0: 0x11, 0xf1: 0x12, 0xf2: 0x12, 0xf3: 0x14, 0xf4: 0x15, + // Block 0x4, offset 0x100 + 0x120: 0x14, 0x121: 0x15, 0x122: 0x16, 0x123: 0x17, 0x124: 0x18, 0x125: 0x19, 0x126: 0x1a, 0x127: 0x1b, + 0x128: 0x1c, 0x129: 0x1d, 0x12a: 0x1c, 0x12b: 0x1e, 0x12c: 0x1f, 0x12d: 0x20, 0x12e: 0x21, 0x12f: 0x22, + 0x130: 0x23, 0x131: 0x24, 0x132: 0x1a, 0x133: 0x25, 0x134: 0x26, 0x135: 0x27, 0x137: 0x28, + 0x138: 0x29, 0x139: 0x2a, 0x13a: 0x2b, 0x13b: 0x2c, 0x13c: 0x2d, 0x13d: 0x2e, 0x13e: 0x2f, 0x13f: 0x30, + // Block 0x5, offset 0x140 + 0x140: 0x31, 0x141: 0x32, 0x142: 0x33, + 0x14d: 0x34, 0x14e: 0x35, + 0x150: 0x36, + 0x15a: 0x37, 0x15c: 0x38, 0x15d: 0x39, 0x15e: 0x3a, 0x15f: 0x3b, + 0x160: 0x3c, 0x162: 0x3d, 0x164: 0x3e, 0x165: 0x3f, 0x167: 0x40, + 0x168: 0x41, 0x169: 0x42, 0x16a: 0x43, 0x16c: 0x44, 0x16d: 0x45, 0x16e: 0x46, 0x16f: 0x47, + 0x170: 0x48, 0x173: 0x49, 0x177: 0x4a, + 0x17e: 0x4b, 0x17f: 0x4c, + // Block 0x6, offset 0x180 + 0x180: 0x4d, 0x181: 0x4e, 0x182: 0x4f, 0x183: 0x50, 0x184: 0x51, 0x185: 0x52, 0x186: 0x53, 0x187: 0x54, + 0x188: 0x55, 0x189: 0x54, 0x18a: 0x54, 0x18b: 0x54, 0x18c: 0x56, 0x18d: 0x57, 0x18e: 0x58, 0x18f: 0x54, + 0x190: 0x59, 0x191: 0x5a, 0x192: 0x5b, 0x193: 0x5c, 0x194: 0x54, 0x195: 0x54, 0x196: 0x54, 0x197: 0x54, + 0x198: 0x54, 0x199: 0x54, 0x19a: 0x5d, 0x19b: 0x54, 0x19c: 0x54, 0x19d: 0x5e, 0x19e: 0x54, 0x19f: 0x5f, + 0x1a4: 0x54, 0x1a5: 0x54, 0x1a6: 0x60, 0x1a7: 0x61, + 0x1a8: 0x54, 0x1a9: 0x54, 0x1aa: 0x54, 0x1ab: 0x54, 0x1ac: 0x54, 0x1ad: 0x62, 0x1ae: 0x63, 0x1af: 0x54, + 0x1b3: 0x64, 0x1b5: 0x65, 0x1b7: 0x66, + 0x1b8: 0x67, 0x1b9: 0x68, 0x1ba: 0x69, 0x1bb: 0x6a, 0x1bc: 0x54, 0x1bd: 0x54, 0x1be: 0x54, 0x1bf: 0x6b, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x6c, 0x1c2: 0x6d, 0x1c3: 0x6e, 0x1c7: 0x6f, + 0x1c8: 0x70, 0x1c9: 0x71, 0x1ca: 0x72, 0x1cb: 0x73, 0x1cd: 0x74, 0x1cf: 0x75, + // Block 0x8, offset 0x200 + 0x237: 0x54, + // Block 0x9, offset 0x240 + 0x252: 0x76, 0x253: 0x77, + 0x258: 0x78, 0x259: 0x79, 0x25a: 0x7a, 0x25b: 0x7b, 0x25c: 0x7c, 0x25e: 0x7d, + 0x260: 0x7e, 0x261: 0x7f, 0x263: 0x80, 0x264: 0x81, 0x265: 0x82, 0x266: 0x83, 0x267: 0x84, + 0x268: 0x85, 0x269: 0x86, 0x26a: 0x87, 0x26b: 0x88, 0x26f: 0x89, + // Block 0xa, offset 0x280 + 0x2ac: 0x8a, 0x2ad: 0x8b, 0x2ae: 0x0e, 0x2af: 0x0e, + 0x2b0: 0x0e, 0x2b1: 0x0e, 0x2b2: 0x0e, 0x2b3: 0x0e, 0x2b4: 0x8c, 0x2b5: 0x0e, 0x2b6: 0x0e, 0x2b7: 0x8d, + 0x2b8: 0x8e, 0x2b9: 0x8f, 0x2ba: 0x0e, 0x2bb: 0x90, 0x2bc: 0x91, 0x2bd: 0x92, 0x2bf: 0x93, + // Block 0xb, offset 0x2c0 + 0x2c4: 0x94, 0x2c5: 0x54, 0x2c6: 0x95, 0x2c7: 0x96, + 0x2cb: 0x97, 0x2cd: 0x98, + 0x2e0: 0x99, 0x2e1: 0x99, 0x2e2: 0x99, 0x2e3: 0x99, 0x2e4: 0x9a, 0x2e5: 0x99, 0x2e6: 0x99, 0x2e7: 0x99, + 0x2e8: 0x9b, 0x2e9: 0x99, 0x2ea: 0x99, 0x2eb: 0x9c, 0x2ec: 0x9d, 0x2ed: 0x99, 0x2ee: 0x99, 0x2ef: 0x99, + 0x2f0: 0x99, 0x2f1: 0x99, 0x2f2: 0x99, 0x2f3: 0x99, 0x2f4: 0x9e, 0x2f5: 0x99, 0x2f6: 0x99, 0x2f7: 0x99, + 0x2f8: 0x99, 0x2f9: 0x9f, 0x2fa: 0x99, 0x2fb: 0x99, 0x2fc: 0xa0, 0x2fd: 0xa1, 0x2fe: 0x99, 0x2ff: 0x99, + // Block 0xc, offset 0x300 + 0x300: 0xa2, 0x301: 0xa3, 0x302: 0xa4, 0x304: 0xa5, 0x305: 0xa6, 0x306: 0xa7, 0x307: 0xa8, + 0x308: 0xa9, 0x30b: 0xaa, 0x30c: 0x26, 0x30d: 0xab, + 0x310: 0xac, 0x311: 0xad, 0x312: 0xae, 0x313: 0xaf, 0x316: 0xb0, 0x317: 0xb1, + 0x318: 0xb2, 0x319: 0xb3, 0x31a: 0xb4, 0x31c: 0xb5, + 0x320: 0xb6, 0x327: 0xb7, + 0x328: 0xb8, 0x329: 0xb9, 0x32a: 0xba, + 0x330: 0xbb, 0x332: 0xbc, 0x334: 0xbd, 0x335: 0xbe, 0x336: 0xbf, + 0x33b: 0xc0, 0x33f: 0xc1, + // Block 0xd, offset 0x340 + 0x36b: 0xc2, 0x36c: 0xc3, + 0x37d: 0xc4, 0x37e: 0xc5, 0x37f: 0xc6, + // Block 0xe, offset 0x380 + 0x3b2: 0xc7, + // Block 0xf, offset 0x3c0 + 0x3c5: 0xc8, 0x3c6: 0xc9, + 0x3c8: 0x54, 0x3c9: 0xca, 0x3cc: 0x54, 0x3cd: 0xcb, + 0x3db: 0xcc, 0x3dc: 0xcd, 0x3dd: 0xce, 0x3de: 0xcf, 0x3df: 0xd0, + 0x3e8: 0xd1, 0x3e9: 0xd2, 0x3ea: 0xd3, + // Block 0x10, offset 0x400 + 0x400: 0xd4, 0x404: 0xc3, + 0x40b: 0xd5, + 0x420: 0x99, 0x421: 0x99, 0x422: 0x99, 0x423: 0xd6, 0x424: 0x99, 0x425: 0xd7, 0x426: 0x99, 0x427: 0x99, + 0x428: 0x99, 0x429: 0x99, 0x42a: 0x99, 0x42b: 0x99, 0x42c: 0x99, 0x42d: 0x99, 0x42e: 0x99, 0x42f: 0x99, + 0x430: 0x99, 0x431: 0xa0, 0x432: 0x0e, 0x433: 0x99, 0x434: 0x0e, 0x435: 0xd8, 0x436: 0x99, 0x437: 0x99, + 0x438: 0x0e, 0x439: 0x0e, 0x43a: 0x0e, 0x43b: 0xd9, 0x43c: 0x99, 0x43d: 0x99, 0x43e: 0x99, 0x43f: 0x99, + // Block 0x11, offset 0x440 + 0x440: 0xda, 0x441: 0x54, 0x442: 0xdb, 0x443: 0xdc, 0x444: 0xdd, 0x445: 0xde, + 0x449: 0xdf, 0x44c: 0x54, 0x44d: 0x54, 0x44e: 0x54, 0x44f: 0x54, + 0x450: 0x54, 0x451: 0x54, 0x452: 0x54, 0x453: 0x54, 0x454: 0x54, 0x455: 0x54, 0x456: 0x54, 0x457: 0x54, + 0x458: 0x54, 0x459: 0x54, 0x45a: 0x54, 0x45b: 0xe0, 0x45c: 0x54, 0x45d: 0x6a, 0x45e: 0x54, 0x45f: 0xe1, + 0x460: 0xe2, 0x461: 0xe3, 0x462: 0xe4, 0x464: 0xe5, 0x465: 0xe6, 0x466: 0xe7, 0x467: 0xe8, + 0x468: 0x54, 0x469: 0xe9, 0x46a: 0xea, + 0x47f: 0xeb, + // Block 0x12, offset 0x480 + 0x4bf: 0xeb, + // Block 0x13, offset 0x4c0 + 0x4d0: 0x09, 0x4d1: 0x0a, 0x4d6: 0x0b, + 0x4db: 0x0c, 0x4dd: 0x0d, 0x4de: 0x0e, 0x4df: 0x0f, + 0x4ef: 0x10, + 0x4ff: 0x10, + // Block 0x14, offset 0x500 + 0x50f: 0x10, + 0x51f: 0x10, + 0x52f: 0x10, + 0x53f: 0x10, + // Block 0x15, offset 0x540 + 0x540: 0xec, 0x541: 0xec, 0x542: 0xec, 0x543: 0xec, 0x544: 0x05, 0x545: 0x05, 0x546: 0x05, 0x547: 0xed, + 0x548: 0xec, 0x549: 0xec, 0x54a: 0xec, 0x54b: 0xec, 0x54c: 0xec, 0x54d: 0xec, 0x54e: 0xec, 0x54f: 0xec, + 0x550: 0xec, 0x551: 0xec, 0x552: 0xec, 0x553: 0xec, 0x554: 0xec, 0x555: 0xec, 0x556: 0xec, 0x557: 0xec, + 0x558: 0xec, 0x559: 0xec, 0x55a: 0xec, 0x55b: 0xec, 0x55c: 0xec, 0x55d: 0xec, 0x55e: 0xec, 0x55f: 0xec, + 0x560: 0xec, 0x561: 0xec, 0x562: 0xec, 0x563: 0xec, 0x564: 0xec, 0x565: 0xec, 0x566: 0xec, 0x567: 0xec, + 0x568: 0xec, 0x569: 0xec, 0x56a: 0xec, 0x56b: 0xec, 0x56c: 0xec, 0x56d: 0xec, 0x56e: 0xec, 0x56f: 0xec, + 0x570: 0xec, 0x571: 0xec, 0x572: 0xec, 0x573: 0xec, 0x574: 0xec, 0x575: 0xec, 0x576: 0xec, 0x577: 0xec, + 0x578: 0xec, 0x579: 0xec, 0x57a: 0xec, 0x57b: 0xec, 0x57c: 0xec, 0x57d: 0xec, 0x57e: 0xec, 0x57f: 0xec, + // Block 0x16, offset 0x580 + 0x58f: 0x10, + 0x59f: 0x10, + 0x5a0: 0x13, + 0x5af: 0x10, + 0x5bf: 0x10, + // Block 0x17, offset 0x5c0 + 0x5cf: 0x10, +} + +// Total table size 16952 bytes (16KiB); checksum: F50EF68C diff --git a/golang.org/x/text/unicode/norm/tables11.0.0.go b/golang.org/x/text/unicode/norm/tables11.0.0.go index 7297cce32b..2c58f09baa 100644 --- a/golang.org/x/text/unicode/norm/tables11.0.0.go +++ b/golang.org/x/text/unicode/norm/tables11.0.0.go @@ -1,6 +1,6 @@ // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. -// +build go1.13 +// +build go1.13,!go1.14 package norm diff --git a/golang.org/x/text/unicode/norm/tables12.0.0.go b/golang.org/x/text/unicode/norm/tables12.0.0.go new file mode 100644 index 0000000000..10f5202c69 --- /dev/null +++ b/golang.org/x/text/unicode/norm/tables12.0.0.go @@ -0,0 +1,7710 @@ +// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. + +// +build go1.14 + +package norm + +import "sync" + +const ( + // Version is the Unicode edition from which the tables are derived. + Version = "12.0.0" + + // MaxTransformChunkSize indicates the maximum number of bytes that Transform + // may need to write atomically for any Form. Making a destination buffer at + // least this size ensures that Transform can always make progress and that + // the user does not need to grow the buffer on an ErrShortDst. + MaxTransformChunkSize = 35 + maxNonStarters*4 +) + +var ccc = [55]uint8{ + 0, 1, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 26, 27, 28, + 29, 30, 31, 32, 33, 34, 35, 36, + 84, 91, 103, 107, 118, 122, 129, 130, + 132, 202, 214, 216, 218, 220, 222, 224, + 226, 228, 230, 232, 233, 234, 240, +} + +const ( + firstMulti = 0x186D + firstCCC = 0x2CA1 + endMulti = 0x2F63 + firstLeadingCCC = 0x49B1 + firstCCCZeroExcept = 0x4A7B + firstStarterWithNLead = 0x4AA2 + lastDecomp = 0x4AA4 + maxDecomp = 0x8000 +) + +// decomps: 19108 bytes +var decomps = [...]byte{ + // Bytes 0 - 3f + 0x00, 0x41, 0x20, 0x41, 0x21, 0x41, 0x22, 0x41, + 0x23, 0x41, 0x24, 0x41, 0x25, 0x41, 0x26, 0x41, + 0x27, 0x41, 0x28, 0x41, 0x29, 0x41, 0x2A, 0x41, + 0x2B, 0x41, 0x2C, 0x41, 0x2D, 0x41, 0x2E, 0x41, + 0x2F, 0x41, 0x30, 0x41, 0x31, 0x41, 0x32, 0x41, + 0x33, 0x41, 0x34, 0x41, 0x35, 0x41, 0x36, 0x41, + 0x37, 0x41, 0x38, 0x41, 0x39, 0x41, 0x3A, 0x41, + 0x3B, 0x41, 0x3C, 0x41, 0x3D, 0x41, 0x3E, 0x41, + // Bytes 40 - 7f + 0x3F, 0x41, 0x40, 0x41, 0x41, 0x41, 0x42, 0x41, + 0x43, 0x41, 0x44, 0x41, 0x45, 0x41, 0x46, 0x41, + 0x47, 0x41, 0x48, 0x41, 0x49, 0x41, 0x4A, 0x41, + 0x4B, 0x41, 0x4C, 0x41, 0x4D, 0x41, 0x4E, 0x41, + 0x4F, 0x41, 0x50, 0x41, 0x51, 0x41, 0x52, 0x41, + 0x53, 0x41, 0x54, 0x41, 0x55, 0x41, 0x56, 0x41, + 0x57, 0x41, 0x58, 0x41, 0x59, 0x41, 0x5A, 0x41, + 0x5B, 0x41, 0x5C, 0x41, 0x5D, 0x41, 0x5E, 0x41, + // Bytes 80 - bf + 0x5F, 0x41, 0x60, 0x41, 0x61, 0x41, 0x62, 0x41, + 0x63, 0x41, 0x64, 0x41, 0x65, 0x41, 0x66, 0x41, + 0x67, 0x41, 0x68, 0x41, 0x69, 0x41, 0x6A, 0x41, + 0x6B, 0x41, 0x6C, 0x41, 0x6D, 0x41, 0x6E, 0x41, + 0x6F, 0x41, 0x70, 0x41, 0x71, 0x41, 0x72, 0x41, + 0x73, 0x41, 0x74, 0x41, 0x75, 0x41, 0x76, 0x41, + 0x77, 0x41, 0x78, 0x41, 0x79, 0x41, 0x7A, 0x41, + 0x7B, 0x41, 0x7C, 0x41, 0x7D, 0x41, 0x7E, 0x42, + // Bytes c0 - ff + 0xC2, 0xA2, 0x42, 0xC2, 0xA3, 0x42, 0xC2, 0xA5, + 0x42, 0xC2, 0xA6, 0x42, 0xC2, 0xAC, 0x42, 0xC2, + 0xB7, 0x42, 0xC3, 0x86, 0x42, 0xC3, 0xB0, 0x42, + 0xC4, 0xA6, 0x42, 0xC4, 0xA7, 0x42, 0xC4, 0xB1, + 0x42, 0xC5, 0x8B, 0x42, 0xC5, 0x93, 0x42, 0xC6, + 0x8E, 0x42, 0xC6, 0x90, 0x42, 0xC6, 0xAB, 0x42, + 0xC8, 0xA2, 0x42, 0xC8, 0xB7, 0x42, 0xC9, 0x90, + 0x42, 0xC9, 0x91, 0x42, 0xC9, 0x92, 0x42, 0xC9, + // Bytes 100 - 13f + 0x94, 0x42, 0xC9, 0x95, 0x42, 0xC9, 0x99, 0x42, + 0xC9, 0x9B, 0x42, 0xC9, 0x9C, 0x42, 0xC9, 0x9F, + 0x42, 0xC9, 0xA1, 0x42, 0xC9, 0xA3, 0x42, 0xC9, + 0xA5, 0x42, 0xC9, 0xA6, 0x42, 0xC9, 0xA8, 0x42, + 0xC9, 0xA9, 0x42, 0xC9, 0xAA, 0x42, 0xC9, 0xAB, + 0x42, 0xC9, 0xAD, 0x42, 0xC9, 0xAF, 0x42, 0xC9, + 0xB0, 0x42, 0xC9, 0xB1, 0x42, 0xC9, 0xB2, 0x42, + 0xC9, 0xB3, 0x42, 0xC9, 0xB4, 0x42, 0xC9, 0xB5, + // Bytes 140 - 17f + 0x42, 0xC9, 0xB8, 0x42, 0xC9, 0xB9, 0x42, 0xC9, + 0xBB, 0x42, 0xCA, 0x81, 0x42, 0xCA, 0x82, 0x42, + 0xCA, 0x83, 0x42, 0xCA, 0x89, 0x42, 0xCA, 0x8A, + 0x42, 0xCA, 0x8B, 0x42, 0xCA, 0x8C, 0x42, 0xCA, + 0x90, 0x42, 0xCA, 0x91, 0x42, 0xCA, 0x92, 0x42, + 0xCA, 0x95, 0x42, 0xCA, 0x9D, 0x42, 0xCA, 0x9F, + 0x42, 0xCA, 0xB9, 0x42, 0xCE, 0x91, 0x42, 0xCE, + 0x92, 0x42, 0xCE, 0x93, 0x42, 0xCE, 0x94, 0x42, + // Bytes 180 - 1bf + 0xCE, 0x95, 0x42, 0xCE, 0x96, 0x42, 0xCE, 0x97, + 0x42, 0xCE, 0x98, 0x42, 0xCE, 0x99, 0x42, 0xCE, + 0x9A, 0x42, 0xCE, 0x9B, 0x42, 0xCE, 0x9C, 0x42, + 0xCE, 0x9D, 0x42, 0xCE, 0x9E, 0x42, 0xCE, 0x9F, + 0x42, 0xCE, 0xA0, 0x42, 0xCE, 0xA1, 0x42, 0xCE, + 0xA3, 0x42, 0xCE, 0xA4, 0x42, 0xCE, 0xA5, 0x42, + 0xCE, 0xA6, 0x42, 0xCE, 0xA7, 0x42, 0xCE, 0xA8, + 0x42, 0xCE, 0xA9, 0x42, 0xCE, 0xB1, 0x42, 0xCE, + // Bytes 1c0 - 1ff + 0xB2, 0x42, 0xCE, 0xB3, 0x42, 0xCE, 0xB4, 0x42, + 0xCE, 0xB5, 0x42, 0xCE, 0xB6, 0x42, 0xCE, 0xB7, + 0x42, 0xCE, 0xB8, 0x42, 0xCE, 0xB9, 0x42, 0xCE, + 0xBA, 0x42, 0xCE, 0xBB, 0x42, 0xCE, 0xBC, 0x42, + 0xCE, 0xBD, 0x42, 0xCE, 0xBE, 0x42, 0xCE, 0xBF, + 0x42, 0xCF, 0x80, 0x42, 0xCF, 0x81, 0x42, 0xCF, + 0x82, 0x42, 0xCF, 0x83, 0x42, 0xCF, 0x84, 0x42, + 0xCF, 0x85, 0x42, 0xCF, 0x86, 0x42, 0xCF, 0x87, + // Bytes 200 - 23f + 0x42, 0xCF, 0x88, 0x42, 0xCF, 0x89, 0x42, 0xCF, + 0x9C, 0x42, 0xCF, 0x9D, 0x42, 0xD0, 0xBD, 0x42, + 0xD1, 0x8A, 0x42, 0xD1, 0x8C, 0x42, 0xD7, 0x90, + 0x42, 0xD7, 0x91, 0x42, 0xD7, 0x92, 0x42, 0xD7, + 0x93, 0x42, 0xD7, 0x94, 0x42, 0xD7, 0x9B, 0x42, + 0xD7, 0x9C, 0x42, 0xD7, 0x9D, 0x42, 0xD7, 0xA2, + 0x42, 0xD7, 0xA8, 0x42, 0xD7, 0xAA, 0x42, 0xD8, + 0xA1, 0x42, 0xD8, 0xA7, 0x42, 0xD8, 0xA8, 0x42, + // Bytes 240 - 27f + 0xD8, 0xA9, 0x42, 0xD8, 0xAA, 0x42, 0xD8, 0xAB, + 0x42, 0xD8, 0xAC, 0x42, 0xD8, 0xAD, 0x42, 0xD8, + 0xAE, 0x42, 0xD8, 0xAF, 0x42, 0xD8, 0xB0, 0x42, + 0xD8, 0xB1, 0x42, 0xD8, 0xB2, 0x42, 0xD8, 0xB3, + 0x42, 0xD8, 0xB4, 0x42, 0xD8, 0xB5, 0x42, 0xD8, + 0xB6, 0x42, 0xD8, 0xB7, 0x42, 0xD8, 0xB8, 0x42, + 0xD8, 0xB9, 0x42, 0xD8, 0xBA, 0x42, 0xD9, 0x81, + 0x42, 0xD9, 0x82, 0x42, 0xD9, 0x83, 0x42, 0xD9, + // Bytes 280 - 2bf + 0x84, 0x42, 0xD9, 0x85, 0x42, 0xD9, 0x86, 0x42, + 0xD9, 0x87, 0x42, 0xD9, 0x88, 0x42, 0xD9, 0x89, + 0x42, 0xD9, 0x8A, 0x42, 0xD9, 0xAE, 0x42, 0xD9, + 0xAF, 0x42, 0xD9, 0xB1, 0x42, 0xD9, 0xB9, 0x42, + 0xD9, 0xBA, 0x42, 0xD9, 0xBB, 0x42, 0xD9, 0xBE, + 0x42, 0xD9, 0xBF, 0x42, 0xDA, 0x80, 0x42, 0xDA, + 0x83, 0x42, 0xDA, 0x84, 0x42, 0xDA, 0x86, 0x42, + 0xDA, 0x87, 0x42, 0xDA, 0x88, 0x42, 0xDA, 0x8C, + // Bytes 2c0 - 2ff + 0x42, 0xDA, 0x8D, 0x42, 0xDA, 0x8E, 0x42, 0xDA, + 0x91, 0x42, 0xDA, 0x98, 0x42, 0xDA, 0xA1, 0x42, + 0xDA, 0xA4, 0x42, 0xDA, 0xA6, 0x42, 0xDA, 0xA9, + 0x42, 0xDA, 0xAD, 0x42, 0xDA, 0xAF, 0x42, 0xDA, + 0xB1, 0x42, 0xDA, 0xB3, 0x42, 0xDA, 0xBA, 0x42, + 0xDA, 0xBB, 0x42, 0xDA, 0xBE, 0x42, 0xDB, 0x81, + 0x42, 0xDB, 0x85, 0x42, 0xDB, 0x86, 0x42, 0xDB, + 0x87, 0x42, 0xDB, 0x88, 0x42, 0xDB, 0x89, 0x42, + // Bytes 300 - 33f + 0xDB, 0x8B, 0x42, 0xDB, 0x8C, 0x42, 0xDB, 0x90, + 0x42, 0xDB, 0x92, 0x43, 0xE0, 0xBC, 0x8B, 0x43, + 0xE1, 0x83, 0x9C, 0x43, 0xE1, 0x84, 0x80, 0x43, + 0xE1, 0x84, 0x81, 0x43, 0xE1, 0x84, 0x82, 0x43, + 0xE1, 0x84, 0x83, 0x43, 0xE1, 0x84, 0x84, 0x43, + 0xE1, 0x84, 0x85, 0x43, 0xE1, 0x84, 0x86, 0x43, + 0xE1, 0x84, 0x87, 0x43, 0xE1, 0x84, 0x88, 0x43, + 0xE1, 0x84, 0x89, 0x43, 0xE1, 0x84, 0x8A, 0x43, + // Bytes 340 - 37f + 0xE1, 0x84, 0x8B, 0x43, 0xE1, 0x84, 0x8C, 0x43, + 0xE1, 0x84, 0x8D, 0x43, 0xE1, 0x84, 0x8E, 0x43, + 0xE1, 0x84, 0x8F, 0x43, 0xE1, 0x84, 0x90, 0x43, + 0xE1, 0x84, 0x91, 0x43, 0xE1, 0x84, 0x92, 0x43, + 0xE1, 0x84, 0x94, 0x43, 0xE1, 0x84, 0x95, 0x43, + 0xE1, 0x84, 0x9A, 0x43, 0xE1, 0x84, 0x9C, 0x43, + 0xE1, 0x84, 0x9D, 0x43, 0xE1, 0x84, 0x9E, 0x43, + 0xE1, 0x84, 0xA0, 0x43, 0xE1, 0x84, 0xA1, 0x43, + // Bytes 380 - 3bf + 0xE1, 0x84, 0xA2, 0x43, 0xE1, 0x84, 0xA3, 0x43, + 0xE1, 0x84, 0xA7, 0x43, 0xE1, 0x84, 0xA9, 0x43, + 0xE1, 0x84, 0xAB, 0x43, 0xE1, 0x84, 0xAC, 0x43, + 0xE1, 0x84, 0xAD, 0x43, 0xE1, 0x84, 0xAE, 0x43, + 0xE1, 0x84, 0xAF, 0x43, 0xE1, 0x84, 0xB2, 0x43, + 0xE1, 0x84, 0xB6, 0x43, 0xE1, 0x85, 0x80, 0x43, + 0xE1, 0x85, 0x87, 0x43, 0xE1, 0x85, 0x8C, 0x43, + 0xE1, 0x85, 0x97, 0x43, 0xE1, 0x85, 0x98, 0x43, + // Bytes 3c0 - 3ff + 0xE1, 0x85, 0x99, 0x43, 0xE1, 0x85, 0xA0, 0x43, + 0xE1, 0x86, 0x84, 0x43, 0xE1, 0x86, 0x85, 0x43, + 0xE1, 0x86, 0x88, 0x43, 0xE1, 0x86, 0x91, 0x43, + 0xE1, 0x86, 0x92, 0x43, 0xE1, 0x86, 0x94, 0x43, + 0xE1, 0x86, 0x9E, 0x43, 0xE1, 0x86, 0xA1, 0x43, + 0xE1, 0x87, 0x87, 0x43, 0xE1, 0x87, 0x88, 0x43, + 0xE1, 0x87, 0x8C, 0x43, 0xE1, 0x87, 0x8E, 0x43, + 0xE1, 0x87, 0x93, 0x43, 0xE1, 0x87, 0x97, 0x43, + // Bytes 400 - 43f + 0xE1, 0x87, 0x99, 0x43, 0xE1, 0x87, 0x9D, 0x43, + 0xE1, 0x87, 0x9F, 0x43, 0xE1, 0x87, 0xB1, 0x43, + 0xE1, 0x87, 0xB2, 0x43, 0xE1, 0xB4, 0x82, 0x43, + 0xE1, 0xB4, 0x96, 0x43, 0xE1, 0xB4, 0x97, 0x43, + 0xE1, 0xB4, 0x9C, 0x43, 0xE1, 0xB4, 0x9D, 0x43, + 0xE1, 0xB4, 0xA5, 0x43, 0xE1, 0xB5, 0xBB, 0x43, + 0xE1, 0xB6, 0x85, 0x43, 0xE2, 0x80, 0x82, 0x43, + 0xE2, 0x80, 0x83, 0x43, 0xE2, 0x80, 0x90, 0x43, + // Bytes 440 - 47f + 0xE2, 0x80, 0x93, 0x43, 0xE2, 0x80, 0x94, 0x43, + 0xE2, 0x82, 0xA9, 0x43, 0xE2, 0x86, 0x90, 0x43, + 0xE2, 0x86, 0x91, 0x43, 0xE2, 0x86, 0x92, 0x43, + 0xE2, 0x86, 0x93, 0x43, 0xE2, 0x88, 0x82, 0x43, + 0xE2, 0x88, 0x87, 0x43, 0xE2, 0x88, 0x91, 0x43, + 0xE2, 0x88, 0x92, 0x43, 0xE2, 0x94, 0x82, 0x43, + 0xE2, 0x96, 0xA0, 0x43, 0xE2, 0x97, 0x8B, 0x43, + 0xE2, 0xA6, 0x85, 0x43, 0xE2, 0xA6, 0x86, 0x43, + // Bytes 480 - 4bf + 0xE2, 0xB5, 0xA1, 0x43, 0xE3, 0x80, 0x81, 0x43, + 0xE3, 0x80, 0x82, 0x43, 0xE3, 0x80, 0x88, 0x43, + 0xE3, 0x80, 0x89, 0x43, 0xE3, 0x80, 0x8A, 0x43, + 0xE3, 0x80, 0x8B, 0x43, 0xE3, 0x80, 0x8C, 0x43, + 0xE3, 0x80, 0x8D, 0x43, 0xE3, 0x80, 0x8E, 0x43, + 0xE3, 0x80, 0x8F, 0x43, 0xE3, 0x80, 0x90, 0x43, + 0xE3, 0x80, 0x91, 0x43, 0xE3, 0x80, 0x92, 0x43, + 0xE3, 0x80, 0x94, 0x43, 0xE3, 0x80, 0x95, 0x43, + // Bytes 4c0 - 4ff + 0xE3, 0x80, 0x96, 0x43, 0xE3, 0x80, 0x97, 0x43, + 0xE3, 0x82, 0xA1, 0x43, 0xE3, 0x82, 0xA2, 0x43, + 0xE3, 0x82, 0xA3, 0x43, 0xE3, 0x82, 0xA4, 0x43, + 0xE3, 0x82, 0xA5, 0x43, 0xE3, 0x82, 0xA6, 0x43, + 0xE3, 0x82, 0xA7, 0x43, 0xE3, 0x82, 0xA8, 0x43, + 0xE3, 0x82, 0xA9, 0x43, 0xE3, 0x82, 0xAA, 0x43, + 0xE3, 0x82, 0xAB, 0x43, 0xE3, 0x82, 0xAD, 0x43, + 0xE3, 0x82, 0xAF, 0x43, 0xE3, 0x82, 0xB1, 0x43, + // Bytes 500 - 53f + 0xE3, 0x82, 0xB3, 0x43, 0xE3, 0x82, 0xB5, 0x43, + 0xE3, 0x82, 0xB7, 0x43, 0xE3, 0x82, 0xB9, 0x43, + 0xE3, 0x82, 0xBB, 0x43, 0xE3, 0x82, 0xBD, 0x43, + 0xE3, 0x82, 0xBF, 0x43, 0xE3, 0x83, 0x81, 0x43, + 0xE3, 0x83, 0x83, 0x43, 0xE3, 0x83, 0x84, 0x43, + 0xE3, 0x83, 0x86, 0x43, 0xE3, 0x83, 0x88, 0x43, + 0xE3, 0x83, 0x8A, 0x43, 0xE3, 0x83, 0x8B, 0x43, + 0xE3, 0x83, 0x8C, 0x43, 0xE3, 0x83, 0x8D, 0x43, + // Bytes 540 - 57f + 0xE3, 0x83, 0x8E, 0x43, 0xE3, 0x83, 0x8F, 0x43, + 0xE3, 0x83, 0x92, 0x43, 0xE3, 0x83, 0x95, 0x43, + 0xE3, 0x83, 0x98, 0x43, 0xE3, 0x83, 0x9B, 0x43, + 0xE3, 0x83, 0x9E, 0x43, 0xE3, 0x83, 0x9F, 0x43, + 0xE3, 0x83, 0xA0, 0x43, 0xE3, 0x83, 0xA1, 0x43, + 0xE3, 0x83, 0xA2, 0x43, 0xE3, 0x83, 0xA3, 0x43, + 0xE3, 0x83, 0xA4, 0x43, 0xE3, 0x83, 0xA5, 0x43, + 0xE3, 0x83, 0xA6, 0x43, 0xE3, 0x83, 0xA7, 0x43, + // Bytes 580 - 5bf + 0xE3, 0x83, 0xA8, 0x43, 0xE3, 0x83, 0xA9, 0x43, + 0xE3, 0x83, 0xAA, 0x43, 0xE3, 0x83, 0xAB, 0x43, + 0xE3, 0x83, 0xAC, 0x43, 0xE3, 0x83, 0xAD, 0x43, + 0xE3, 0x83, 0xAF, 0x43, 0xE3, 0x83, 0xB0, 0x43, + 0xE3, 0x83, 0xB1, 0x43, 0xE3, 0x83, 0xB2, 0x43, + 0xE3, 0x83, 0xB3, 0x43, 0xE3, 0x83, 0xBB, 0x43, + 0xE3, 0x83, 0xBC, 0x43, 0xE3, 0x92, 0x9E, 0x43, + 0xE3, 0x92, 0xB9, 0x43, 0xE3, 0x92, 0xBB, 0x43, + // Bytes 5c0 - 5ff + 0xE3, 0x93, 0x9F, 0x43, 0xE3, 0x94, 0x95, 0x43, + 0xE3, 0x9B, 0xAE, 0x43, 0xE3, 0x9B, 0xBC, 0x43, + 0xE3, 0x9E, 0x81, 0x43, 0xE3, 0xA0, 0xAF, 0x43, + 0xE3, 0xA1, 0xA2, 0x43, 0xE3, 0xA1, 0xBC, 0x43, + 0xE3, 0xA3, 0x87, 0x43, 0xE3, 0xA3, 0xA3, 0x43, + 0xE3, 0xA4, 0x9C, 0x43, 0xE3, 0xA4, 0xBA, 0x43, + 0xE3, 0xA8, 0xAE, 0x43, 0xE3, 0xA9, 0xAC, 0x43, + 0xE3, 0xAB, 0xA4, 0x43, 0xE3, 0xAC, 0x88, 0x43, + // Bytes 600 - 63f + 0xE3, 0xAC, 0x99, 0x43, 0xE3, 0xAD, 0x89, 0x43, + 0xE3, 0xAE, 0x9D, 0x43, 0xE3, 0xB0, 0x98, 0x43, + 0xE3, 0xB1, 0x8E, 0x43, 0xE3, 0xB4, 0xB3, 0x43, + 0xE3, 0xB6, 0x96, 0x43, 0xE3, 0xBA, 0xAC, 0x43, + 0xE3, 0xBA, 0xB8, 0x43, 0xE3, 0xBC, 0x9B, 0x43, + 0xE3, 0xBF, 0xBC, 0x43, 0xE4, 0x80, 0x88, 0x43, + 0xE4, 0x80, 0x98, 0x43, 0xE4, 0x80, 0xB9, 0x43, + 0xE4, 0x81, 0x86, 0x43, 0xE4, 0x82, 0x96, 0x43, + // Bytes 640 - 67f + 0xE4, 0x83, 0xA3, 0x43, 0xE4, 0x84, 0xAF, 0x43, + 0xE4, 0x88, 0x82, 0x43, 0xE4, 0x88, 0xA7, 0x43, + 0xE4, 0x8A, 0xA0, 0x43, 0xE4, 0x8C, 0x81, 0x43, + 0xE4, 0x8C, 0xB4, 0x43, 0xE4, 0x8D, 0x99, 0x43, + 0xE4, 0x8F, 0x95, 0x43, 0xE4, 0x8F, 0x99, 0x43, + 0xE4, 0x90, 0x8B, 0x43, 0xE4, 0x91, 0xAB, 0x43, + 0xE4, 0x94, 0xAB, 0x43, 0xE4, 0x95, 0x9D, 0x43, + 0xE4, 0x95, 0xA1, 0x43, 0xE4, 0x95, 0xAB, 0x43, + // Bytes 680 - 6bf + 0xE4, 0x97, 0x97, 0x43, 0xE4, 0x97, 0xB9, 0x43, + 0xE4, 0x98, 0xB5, 0x43, 0xE4, 0x9A, 0xBE, 0x43, + 0xE4, 0x9B, 0x87, 0x43, 0xE4, 0xA6, 0x95, 0x43, + 0xE4, 0xA7, 0xA6, 0x43, 0xE4, 0xA9, 0xAE, 0x43, + 0xE4, 0xA9, 0xB6, 0x43, 0xE4, 0xAA, 0xB2, 0x43, + 0xE4, 0xAC, 0xB3, 0x43, 0xE4, 0xAF, 0x8E, 0x43, + 0xE4, 0xB3, 0x8E, 0x43, 0xE4, 0xB3, 0xAD, 0x43, + 0xE4, 0xB3, 0xB8, 0x43, 0xE4, 0xB5, 0x96, 0x43, + // Bytes 6c0 - 6ff + 0xE4, 0xB8, 0x80, 0x43, 0xE4, 0xB8, 0x81, 0x43, + 0xE4, 0xB8, 0x83, 0x43, 0xE4, 0xB8, 0x89, 0x43, + 0xE4, 0xB8, 0x8A, 0x43, 0xE4, 0xB8, 0x8B, 0x43, + 0xE4, 0xB8, 0x8D, 0x43, 0xE4, 0xB8, 0x99, 0x43, + 0xE4, 0xB8, 0xA6, 0x43, 0xE4, 0xB8, 0xA8, 0x43, + 0xE4, 0xB8, 0xAD, 0x43, 0xE4, 0xB8, 0xB2, 0x43, + 0xE4, 0xB8, 0xB6, 0x43, 0xE4, 0xB8, 0xB8, 0x43, + 0xE4, 0xB8, 0xB9, 0x43, 0xE4, 0xB8, 0xBD, 0x43, + // Bytes 700 - 73f + 0xE4, 0xB8, 0xBF, 0x43, 0xE4, 0xB9, 0x81, 0x43, + 0xE4, 0xB9, 0x99, 0x43, 0xE4, 0xB9, 0x9D, 0x43, + 0xE4, 0xBA, 0x82, 0x43, 0xE4, 0xBA, 0x85, 0x43, + 0xE4, 0xBA, 0x86, 0x43, 0xE4, 0xBA, 0x8C, 0x43, + 0xE4, 0xBA, 0x94, 0x43, 0xE4, 0xBA, 0xA0, 0x43, + 0xE4, 0xBA, 0xA4, 0x43, 0xE4, 0xBA, 0xAE, 0x43, + 0xE4, 0xBA, 0xBA, 0x43, 0xE4, 0xBB, 0x80, 0x43, + 0xE4, 0xBB, 0x8C, 0x43, 0xE4, 0xBB, 0xA4, 0x43, + // Bytes 740 - 77f + 0xE4, 0xBC, 0x81, 0x43, 0xE4, 0xBC, 0x91, 0x43, + 0xE4, 0xBD, 0xA0, 0x43, 0xE4, 0xBE, 0x80, 0x43, + 0xE4, 0xBE, 0x86, 0x43, 0xE4, 0xBE, 0x8B, 0x43, + 0xE4, 0xBE, 0xAE, 0x43, 0xE4, 0xBE, 0xBB, 0x43, + 0xE4, 0xBE, 0xBF, 0x43, 0xE5, 0x80, 0x82, 0x43, + 0xE5, 0x80, 0xAB, 0x43, 0xE5, 0x81, 0xBA, 0x43, + 0xE5, 0x82, 0x99, 0x43, 0xE5, 0x83, 0x8F, 0x43, + 0xE5, 0x83, 0x9A, 0x43, 0xE5, 0x83, 0xA7, 0x43, + // Bytes 780 - 7bf + 0xE5, 0x84, 0xAA, 0x43, 0xE5, 0x84, 0xBF, 0x43, + 0xE5, 0x85, 0x80, 0x43, 0xE5, 0x85, 0x85, 0x43, + 0xE5, 0x85, 0x8D, 0x43, 0xE5, 0x85, 0x94, 0x43, + 0xE5, 0x85, 0xA4, 0x43, 0xE5, 0x85, 0xA5, 0x43, + 0xE5, 0x85, 0xA7, 0x43, 0xE5, 0x85, 0xA8, 0x43, + 0xE5, 0x85, 0xA9, 0x43, 0xE5, 0x85, 0xAB, 0x43, + 0xE5, 0x85, 0xAD, 0x43, 0xE5, 0x85, 0xB7, 0x43, + 0xE5, 0x86, 0x80, 0x43, 0xE5, 0x86, 0x82, 0x43, + // Bytes 7c0 - 7ff + 0xE5, 0x86, 0x8D, 0x43, 0xE5, 0x86, 0x92, 0x43, + 0xE5, 0x86, 0x95, 0x43, 0xE5, 0x86, 0x96, 0x43, + 0xE5, 0x86, 0x97, 0x43, 0xE5, 0x86, 0x99, 0x43, + 0xE5, 0x86, 0xA4, 0x43, 0xE5, 0x86, 0xAB, 0x43, + 0xE5, 0x86, 0xAC, 0x43, 0xE5, 0x86, 0xB5, 0x43, + 0xE5, 0x86, 0xB7, 0x43, 0xE5, 0x87, 0x89, 0x43, + 0xE5, 0x87, 0x8C, 0x43, 0xE5, 0x87, 0x9C, 0x43, + 0xE5, 0x87, 0x9E, 0x43, 0xE5, 0x87, 0xA0, 0x43, + // Bytes 800 - 83f + 0xE5, 0x87, 0xB5, 0x43, 0xE5, 0x88, 0x80, 0x43, + 0xE5, 0x88, 0x83, 0x43, 0xE5, 0x88, 0x87, 0x43, + 0xE5, 0x88, 0x97, 0x43, 0xE5, 0x88, 0x9D, 0x43, + 0xE5, 0x88, 0xA9, 0x43, 0xE5, 0x88, 0xBA, 0x43, + 0xE5, 0x88, 0xBB, 0x43, 0xE5, 0x89, 0x86, 0x43, + 0xE5, 0x89, 0x8D, 0x43, 0xE5, 0x89, 0xB2, 0x43, + 0xE5, 0x89, 0xB7, 0x43, 0xE5, 0x8A, 0x89, 0x43, + 0xE5, 0x8A, 0x9B, 0x43, 0xE5, 0x8A, 0xA3, 0x43, + // Bytes 840 - 87f + 0xE5, 0x8A, 0xB3, 0x43, 0xE5, 0x8A, 0xB4, 0x43, + 0xE5, 0x8B, 0x87, 0x43, 0xE5, 0x8B, 0x89, 0x43, + 0xE5, 0x8B, 0x92, 0x43, 0xE5, 0x8B, 0x9E, 0x43, + 0xE5, 0x8B, 0xA4, 0x43, 0xE5, 0x8B, 0xB5, 0x43, + 0xE5, 0x8B, 0xB9, 0x43, 0xE5, 0x8B, 0xBA, 0x43, + 0xE5, 0x8C, 0x85, 0x43, 0xE5, 0x8C, 0x86, 0x43, + 0xE5, 0x8C, 0x95, 0x43, 0xE5, 0x8C, 0x97, 0x43, + 0xE5, 0x8C, 0x9A, 0x43, 0xE5, 0x8C, 0xB8, 0x43, + // Bytes 880 - 8bf + 0xE5, 0x8C, 0xBB, 0x43, 0xE5, 0x8C, 0xBF, 0x43, + 0xE5, 0x8D, 0x81, 0x43, 0xE5, 0x8D, 0x84, 0x43, + 0xE5, 0x8D, 0x85, 0x43, 0xE5, 0x8D, 0x89, 0x43, + 0xE5, 0x8D, 0x91, 0x43, 0xE5, 0x8D, 0x94, 0x43, + 0xE5, 0x8D, 0x9A, 0x43, 0xE5, 0x8D, 0x9C, 0x43, + 0xE5, 0x8D, 0xA9, 0x43, 0xE5, 0x8D, 0xB0, 0x43, + 0xE5, 0x8D, 0xB3, 0x43, 0xE5, 0x8D, 0xB5, 0x43, + 0xE5, 0x8D, 0xBD, 0x43, 0xE5, 0x8D, 0xBF, 0x43, + // Bytes 8c0 - 8ff + 0xE5, 0x8E, 0x82, 0x43, 0xE5, 0x8E, 0xB6, 0x43, + 0xE5, 0x8F, 0x83, 0x43, 0xE5, 0x8F, 0x88, 0x43, + 0xE5, 0x8F, 0x8A, 0x43, 0xE5, 0x8F, 0x8C, 0x43, + 0xE5, 0x8F, 0x9F, 0x43, 0xE5, 0x8F, 0xA3, 0x43, + 0xE5, 0x8F, 0xA5, 0x43, 0xE5, 0x8F, 0xAB, 0x43, + 0xE5, 0x8F, 0xAF, 0x43, 0xE5, 0x8F, 0xB1, 0x43, + 0xE5, 0x8F, 0xB3, 0x43, 0xE5, 0x90, 0x86, 0x43, + 0xE5, 0x90, 0x88, 0x43, 0xE5, 0x90, 0x8D, 0x43, + // Bytes 900 - 93f + 0xE5, 0x90, 0x8F, 0x43, 0xE5, 0x90, 0x9D, 0x43, + 0xE5, 0x90, 0xB8, 0x43, 0xE5, 0x90, 0xB9, 0x43, + 0xE5, 0x91, 0x82, 0x43, 0xE5, 0x91, 0x88, 0x43, + 0xE5, 0x91, 0xA8, 0x43, 0xE5, 0x92, 0x9E, 0x43, + 0xE5, 0x92, 0xA2, 0x43, 0xE5, 0x92, 0xBD, 0x43, + 0xE5, 0x93, 0xB6, 0x43, 0xE5, 0x94, 0x90, 0x43, + 0xE5, 0x95, 0x8F, 0x43, 0xE5, 0x95, 0x93, 0x43, + 0xE5, 0x95, 0x95, 0x43, 0xE5, 0x95, 0xA3, 0x43, + // Bytes 940 - 97f + 0xE5, 0x96, 0x84, 0x43, 0xE5, 0x96, 0x87, 0x43, + 0xE5, 0x96, 0x99, 0x43, 0xE5, 0x96, 0x9D, 0x43, + 0xE5, 0x96, 0xAB, 0x43, 0xE5, 0x96, 0xB3, 0x43, + 0xE5, 0x96, 0xB6, 0x43, 0xE5, 0x97, 0x80, 0x43, + 0xE5, 0x97, 0x82, 0x43, 0xE5, 0x97, 0xA2, 0x43, + 0xE5, 0x98, 0x86, 0x43, 0xE5, 0x99, 0x91, 0x43, + 0xE5, 0x99, 0xA8, 0x43, 0xE5, 0x99, 0xB4, 0x43, + 0xE5, 0x9B, 0x97, 0x43, 0xE5, 0x9B, 0x9B, 0x43, + // Bytes 980 - 9bf + 0xE5, 0x9B, 0xB9, 0x43, 0xE5, 0x9C, 0x96, 0x43, + 0xE5, 0x9C, 0x97, 0x43, 0xE5, 0x9C, 0x9F, 0x43, + 0xE5, 0x9C, 0xB0, 0x43, 0xE5, 0x9E, 0x8B, 0x43, + 0xE5, 0x9F, 0x8E, 0x43, 0xE5, 0x9F, 0xB4, 0x43, + 0xE5, 0xA0, 0x8D, 0x43, 0xE5, 0xA0, 0xB1, 0x43, + 0xE5, 0xA0, 0xB2, 0x43, 0xE5, 0xA1, 0x80, 0x43, + 0xE5, 0xA1, 0x9A, 0x43, 0xE5, 0xA1, 0x9E, 0x43, + 0xE5, 0xA2, 0xA8, 0x43, 0xE5, 0xA2, 0xAC, 0x43, + // Bytes 9c0 - 9ff + 0xE5, 0xA2, 0xB3, 0x43, 0xE5, 0xA3, 0x98, 0x43, + 0xE5, 0xA3, 0x9F, 0x43, 0xE5, 0xA3, 0xAB, 0x43, + 0xE5, 0xA3, 0xAE, 0x43, 0xE5, 0xA3, 0xB0, 0x43, + 0xE5, 0xA3, 0xB2, 0x43, 0xE5, 0xA3, 0xB7, 0x43, + 0xE5, 0xA4, 0x82, 0x43, 0xE5, 0xA4, 0x86, 0x43, + 0xE5, 0xA4, 0x8A, 0x43, 0xE5, 0xA4, 0x95, 0x43, + 0xE5, 0xA4, 0x9A, 0x43, 0xE5, 0xA4, 0x9C, 0x43, + 0xE5, 0xA4, 0xA2, 0x43, 0xE5, 0xA4, 0xA7, 0x43, + // Bytes a00 - a3f + 0xE5, 0xA4, 0xA9, 0x43, 0xE5, 0xA5, 0x84, 0x43, + 0xE5, 0xA5, 0x88, 0x43, 0xE5, 0xA5, 0x91, 0x43, + 0xE5, 0xA5, 0x94, 0x43, 0xE5, 0xA5, 0xA2, 0x43, + 0xE5, 0xA5, 0xB3, 0x43, 0xE5, 0xA7, 0x98, 0x43, + 0xE5, 0xA7, 0xAC, 0x43, 0xE5, 0xA8, 0x9B, 0x43, + 0xE5, 0xA8, 0xA7, 0x43, 0xE5, 0xA9, 0xA2, 0x43, + 0xE5, 0xA9, 0xA6, 0x43, 0xE5, 0xAA, 0xB5, 0x43, + 0xE5, 0xAC, 0x88, 0x43, 0xE5, 0xAC, 0xA8, 0x43, + // Bytes a40 - a7f + 0xE5, 0xAC, 0xBE, 0x43, 0xE5, 0xAD, 0x90, 0x43, + 0xE5, 0xAD, 0x97, 0x43, 0xE5, 0xAD, 0xA6, 0x43, + 0xE5, 0xAE, 0x80, 0x43, 0xE5, 0xAE, 0x85, 0x43, + 0xE5, 0xAE, 0x97, 0x43, 0xE5, 0xAF, 0x83, 0x43, + 0xE5, 0xAF, 0x98, 0x43, 0xE5, 0xAF, 0xA7, 0x43, + 0xE5, 0xAF, 0xAE, 0x43, 0xE5, 0xAF, 0xB3, 0x43, + 0xE5, 0xAF, 0xB8, 0x43, 0xE5, 0xAF, 0xBF, 0x43, + 0xE5, 0xB0, 0x86, 0x43, 0xE5, 0xB0, 0x8F, 0x43, + // Bytes a80 - abf + 0xE5, 0xB0, 0xA2, 0x43, 0xE5, 0xB0, 0xB8, 0x43, + 0xE5, 0xB0, 0xBF, 0x43, 0xE5, 0xB1, 0xA0, 0x43, + 0xE5, 0xB1, 0xA2, 0x43, 0xE5, 0xB1, 0xA4, 0x43, + 0xE5, 0xB1, 0xA5, 0x43, 0xE5, 0xB1, 0xAE, 0x43, + 0xE5, 0xB1, 0xB1, 0x43, 0xE5, 0xB2, 0x8D, 0x43, + 0xE5, 0xB3, 0x80, 0x43, 0xE5, 0xB4, 0x99, 0x43, + 0xE5, 0xB5, 0x83, 0x43, 0xE5, 0xB5, 0x90, 0x43, + 0xE5, 0xB5, 0xAB, 0x43, 0xE5, 0xB5, 0xAE, 0x43, + // Bytes ac0 - aff + 0xE5, 0xB5, 0xBC, 0x43, 0xE5, 0xB6, 0xB2, 0x43, + 0xE5, 0xB6, 0xBA, 0x43, 0xE5, 0xB7, 0x9B, 0x43, + 0xE5, 0xB7, 0xA1, 0x43, 0xE5, 0xB7, 0xA2, 0x43, + 0xE5, 0xB7, 0xA5, 0x43, 0xE5, 0xB7, 0xA6, 0x43, + 0xE5, 0xB7, 0xB1, 0x43, 0xE5, 0xB7, 0xBD, 0x43, + 0xE5, 0xB7, 0xBE, 0x43, 0xE5, 0xB8, 0xA8, 0x43, + 0xE5, 0xB8, 0xBD, 0x43, 0xE5, 0xB9, 0xA9, 0x43, + 0xE5, 0xB9, 0xB2, 0x43, 0xE5, 0xB9, 0xB4, 0x43, + // Bytes b00 - b3f + 0xE5, 0xB9, 0xBA, 0x43, 0xE5, 0xB9, 0xBC, 0x43, + 0xE5, 0xB9, 0xBF, 0x43, 0xE5, 0xBA, 0xA6, 0x43, + 0xE5, 0xBA, 0xB0, 0x43, 0xE5, 0xBA, 0xB3, 0x43, + 0xE5, 0xBA, 0xB6, 0x43, 0xE5, 0xBB, 0x89, 0x43, + 0xE5, 0xBB, 0x8A, 0x43, 0xE5, 0xBB, 0x92, 0x43, + 0xE5, 0xBB, 0x93, 0x43, 0xE5, 0xBB, 0x99, 0x43, + 0xE5, 0xBB, 0xAC, 0x43, 0xE5, 0xBB, 0xB4, 0x43, + 0xE5, 0xBB, 0xBE, 0x43, 0xE5, 0xBC, 0x84, 0x43, + // Bytes b40 - b7f + 0xE5, 0xBC, 0x8B, 0x43, 0xE5, 0xBC, 0x93, 0x43, + 0xE5, 0xBC, 0xA2, 0x43, 0xE5, 0xBD, 0x90, 0x43, + 0xE5, 0xBD, 0x93, 0x43, 0xE5, 0xBD, 0xA1, 0x43, + 0xE5, 0xBD, 0xA2, 0x43, 0xE5, 0xBD, 0xA9, 0x43, + 0xE5, 0xBD, 0xAB, 0x43, 0xE5, 0xBD, 0xB3, 0x43, + 0xE5, 0xBE, 0x8B, 0x43, 0xE5, 0xBE, 0x8C, 0x43, + 0xE5, 0xBE, 0x97, 0x43, 0xE5, 0xBE, 0x9A, 0x43, + 0xE5, 0xBE, 0xA9, 0x43, 0xE5, 0xBE, 0xAD, 0x43, + // Bytes b80 - bbf + 0xE5, 0xBF, 0x83, 0x43, 0xE5, 0xBF, 0x8D, 0x43, + 0xE5, 0xBF, 0x97, 0x43, 0xE5, 0xBF, 0xB5, 0x43, + 0xE5, 0xBF, 0xB9, 0x43, 0xE6, 0x80, 0x92, 0x43, + 0xE6, 0x80, 0x9C, 0x43, 0xE6, 0x81, 0xB5, 0x43, + 0xE6, 0x82, 0x81, 0x43, 0xE6, 0x82, 0x94, 0x43, + 0xE6, 0x83, 0x87, 0x43, 0xE6, 0x83, 0x98, 0x43, + 0xE6, 0x83, 0xA1, 0x43, 0xE6, 0x84, 0x88, 0x43, + 0xE6, 0x85, 0x84, 0x43, 0xE6, 0x85, 0x88, 0x43, + // Bytes bc0 - bff + 0xE6, 0x85, 0x8C, 0x43, 0xE6, 0x85, 0x8E, 0x43, + 0xE6, 0x85, 0xA0, 0x43, 0xE6, 0x85, 0xA8, 0x43, + 0xE6, 0x85, 0xBA, 0x43, 0xE6, 0x86, 0x8E, 0x43, + 0xE6, 0x86, 0x90, 0x43, 0xE6, 0x86, 0xA4, 0x43, + 0xE6, 0x86, 0xAF, 0x43, 0xE6, 0x86, 0xB2, 0x43, + 0xE6, 0x87, 0x9E, 0x43, 0xE6, 0x87, 0xB2, 0x43, + 0xE6, 0x87, 0xB6, 0x43, 0xE6, 0x88, 0x80, 0x43, + 0xE6, 0x88, 0x88, 0x43, 0xE6, 0x88, 0x90, 0x43, + // Bytes c00 - c3f + 0xE6, 0x88, 0x9B, 0x43, 0xE6, 0x88, 0xAE, 0x43, + 0xE6, 0x88, 0xB4, 0x43, 0xE6, 0x88, 0xB6, 0x43, + 0xE6, 0x89, 0x8B, 0x43, 0xE6, 0x89, 0x93, 0x43, + 0xE6, 0x89, 0x9D, 0x43, 0xE6, 0x8A, 0x95, 0x43, + 0xE6, 0x8A, 0xB1, 0x43, 0xE6, 0x8B, 0x89, 0x43, + 0xE6, 0x8B, 0x8F, 0x43, 0xE6, 0x8B, 0x93, 0x43, + 0xE6, 0x8B, 0x94, 0x43, 0xE6, 0x8B, 0xBC, 0x43, + 0xE6, 0x8B, 0xBE, 0x43, 0xE6, 0x8C, 0x87, 0x43, + // Bytes c40 - c7f + 0xE6, 0x8C, 0xBD, 0x43, 0xE6, 0x8D, 0x90, 0x43, + 0xE6, 0x8D, 0x95, 0x43, 0xE6, 0x8D, 0xA8, 0x43, + 0xE6, 0x8D, 0xBB, 0x43, 0xE6, 0x8E, 0x83, 0x43, + 0xE6, 0x8E, 0xA0, 0x43, 0xE6, 0x8E, 0xA9, 0x43, + 0xE6, 0x8F, 0x84, 0x43, 0xE6, 0x8F, 0x85, 0x43, + 0xE6, 0x8F, 0xA4, 0x43, 0xE6, 0x90, 0x9C, 0x43, + 0xE6, 0x90, 0xA2, 0x43, 0xE6, 0x91, 0x92, 0x43, + 0xE6, 0x91, 0xA9, 0x43, 0xE6, 0x91, 0xB7, 0x43, + // Bytes c80 - cbf + 0xE6, 0x91, 0xBE, 0x43, 0xE6, 0x92, 0x9A, 0x43, + 0xE6, 0x92, 0x9D, 0x43, 0xE6, 0x93, 0x84, 0x43, + 0xE6, 0x94, 0xAF, 0x43, 0xE6, 0x94, 0xB4, 0x43, + 0xE6, 0x95, 0x8F, 0x43, 0xE6, 0x95, 0x96, 0x43, + 0xE6, 0x95, 0xAC, 0x43, 0xE6, 0x95, 0xB8, 0x43, + 0xE6, 0x96, 0x87, 0x43, 0xE6, 0x96, 0x97, 0x43, + 0xE6, 0x96, 0x99, 0x43, 0xE6, 0x96, 0xA4, 0x43, + 0xE6, 0x96, 0xB0, 0x43, 0xE6, 0x96, 0xB9, 0x43, + // Bytes cc0 - cff + 0xE6, 0x97, 0x85, 0x43, 0xE6, 0x97, 0xA0, 0x43, + 0xE6, 0x97, 0xA2, 0x43, 0xE6, 0x97, 0xA3, 0x43, + 0xE6, 0x97, 0xA5, 0x43, 0xE6, 0x98, 0x93, 0x43, + 0xE6, 0x98, 0xA0, 0x43, 0xE6, 0x99, 0x89, 0x43, + 0xE6, 0x99, 0xB4, 0x43, 0xE6, 0x9A, 0x88, 0x43, + 0xE6, 0x9A, 0x91, 0x43, 0xE6, 0x9A, 0x9C, 0x43, + 0xE6, 0x9A, 0xB4, 0x43, 0xE6, 0x9B, 0x86, 0x43, + 0xE6, 0x9B, 0xB0, 0x43, 0xE6, 0x9B, 0xB4, 0x43, + // Bytes d00 - d3f + 0xE6, 0x9B, 0xB8, 0x43, 0xE6, 0x9C, 0x80, 0x43, + 0xE6, 0x9C, 0x88, 0x43, 0xE6, 0x9C, 0x89, 0x43, + 0xE6, 0x9C, 0x97, 0x43, 0xE6, 0x9C, 0x9B, 0x43, + 0xE6, 0x9C, 0xA1, 0x43, 0xE6, 0x9C, 0xA8, 0x43, + 0xE6, 0x9D, 0x8E, 0x43, 0xE6, 0x9D, 0x93, 0x43, + 0xE6, 0x9D, 0x96, 0x43, 0xE6, 0x9D, 0x9E, 0x43, + 0xE6, 0x9D, 0xBB, 0x43, 0xE6, 0x9E, 0x85, 0x43, + 0xE6, 0x9E, 0x97, 0x43, 0xE6, 0x9F, 0xB3, 0x43, + // Bytes d40 - d7f + 0xE6, 0x9F, 0xBA, 0x43, 0xE6, 0xA0, 0x97, 0x43, + 0xE6, 0xA0, 0x9F, 0x43, 0xE6, 0xA0, 0xAA, 0x43, + 0xE6, 0xA1, 0x92, 0x43, 0xE6, 0xA2, 0x81, 0x43, + 0xE6, 0xA2, 0x85, 0x43, 0xE6, 0xA2, 0x8E, 0x43, + 0xE6, 0xA2, 0xA8, 0x43, 0xE6, 0xA4, 0x94, 0x43, + 0xE6, 0xA5, 0x82, 0x43, 0xE6, 0xA6, 0xA3, 0x43, + 0xE6, 0xA7, 0xAA, 0x43, 0xE6, 0xA8, 0x82, 0x43, + 0xE6, 0xA8, 0x93, 0x43, 0xE6, 0xAA, 0xA8, 0x43, + // Bytes d80 - dbf + 0xE6, 0xAB, 0x93, 0x43, 0xE6, 0xAB, 0x9B, 0x43, + 0xE6, 0xAC, 0x84, 0x43, 0xE6, 0xAC, 0xA0, 0x43, + 0xE6, 0xAC, 0xA1, 0x43, 0xE6, 0xAD, 0x94, 0x43, + 0xE6, 0xAD, 0xA2, 0x43, 0xE6, 0xAD, 0xA3, 0x43, + 0xE6, 0xAD, 0xB2, 0x43, 0xE6, 0xAD, 0xB7, 0x43, + 0xE6, 0xAD, 0xB9, 0x43, 0xE6, 0xAE, 0x9F, 0x43, + 0xE6, 0xAE, 0xAE, 0x43, 0xE6, 0xAE, 0xB3, 0x43, + 0xE6, 0xAE, 0xBA, 0x43, 0xE6, 0xAE, 0xBB, 0x43, + // Bytes dc0 - dff + 0xE6, 0xAF, 0x8B, 0x43, 0xE6, 0xAF, 0x8D, 0x43, + 0xE6, 0xAF, 0x94, 0x43, 0xE6, 0xAF, 0x9B, 0x43, + 0xE6, 0xB0, 0x8F, 0x43, 0xE6, 0xB0, 0x94, 0x43, + 0xE6, 0xB0, 0xB4, 0x43, 0xE6, 0xB1, 0x8E, 0x43, + 0xE6, 0xB1, 0xA7, 0x43, 0xE6, 0xB2, 0x88, 0x43, + 0xE6, 0xB2, 0xBF, 0x43, 0xE6, 0xB3, 0x8C, 0x43, + 0xE6, 0xB3, 0x8D, 0x43, 0xE6, 0xB3, 0xA5, 0x43, + 0xE6, 0xB3, 0xA8, 0x43, 0xE6, 0xB4, 0x96, 0x43, + // Bytes e00 - e3f + 0xE6, 0xB4, 0x9B, 0x43, 0xE6, 0xB4, 0x9E, 0x43, + 0xE6, 0xB4, 0xB4, 0x43, 0xE6, 0xB4, 0xBE, 0x43, + 0xE6, 0xB5, 0x81, 0x43, 0xE6, 0xB5, 0xA9, 0x43, + 0xE6, 0xB5, 0xAA, 0x43, 0xE6, 0xB5, 0xB7, 0x43, + 0xE6, 0xB5, 0xB8, 0x43, 0xE6, 0xB6, 0x85, 0x43, + 0xE6, 0xB7, 0x8B, 0x43, 0xE6, 0xB7, 0x9A, 0x43, + 0xE6, 0xB7, 0xAA, 0x43, 0xE6, 0xB7, 0xB9, 0x43, + 0xE6, 0xB8, 0x9A, 0x43, 0xE6, 0xB8, 0xAF, 0x43, + // Bytes e40 - e7f + 0xE6, 0xB9, 0xAE, 0x43, 0xE6, 0xBA, 0x80, 0x43, + 0xE6, 0xBA, 0x9C, 0x43, 0xE6, 0xBA, 0xBA, 0x43, + 0xE6, 0xBB, 0x87, 0x43, 0xE6, 0xBB, 0x8B, 0x43, + 0xE6, 0xBB, 0x91, 0x43, 0xE6, 0xBB, 0x9B, 0x43, + 0xE6, 0xBC, 0x8F, 0x43, 0xE6, 0xBC, 0x94, 0x43, + 0xE6, 0xBC, 0xA2, 0x43, 0xE6, 0xBC, 0xA3, 0x43, + 0xE6, 0xBD, 0xAE, 0x43, 0xE6, 0xBF, 0x86, 0x43, + 0xE6, 0xBF, 0xAB, 0x43, 0xE6, 0xBF, 0xBE, 0x43, + // Bytes e80 - ebf + 0xE7, 0x80, 0x9B, 0x43, 0xE7, 0x80, 0x9E, 0x43, + 0xE7, 0x80, 0xB9, 0x43, 0xE7, 0x81, 0x8A, 0x43, + 0xE7, 0x81, 0xAB, 0x43, 0xE7, 0x81, 0xB0, 0x43, + 0xE7, 0x81, 0xB7, 0x43, 0xE7, 0x81, 0xBD, 0x43, + 0xE7, 0x82, 0x99, 0x43, 0xE7, 0x82, 0xAD, 0x43, + 0xE7, 0x83, 0x88, 0x43, 0xE7, 0x83, 0x99, 0x43, + 0xE7, 0x84, 0xA1, 0x43, 0xE7, 0x85, 0x85, 0x43, + 0xE7, 0x85, 0x89, 0x43, 0xE7, 0x85, 0xAE, 0x43, + // Bytes ec0 - eff + 0xE7, 0x86, 0x9C, 0x43, 0xE7, 0x87, 0x8E, 0x43, + 0xE7, 0x87, 0x90, 0x43, 0xE7, 0x88, 0x90, 0x43, + 0xE7, 0x88, 0x9B, 0x43, 0xE7, 0x88, 0xA8, 0x43, + 0xE7, 0x88, 0xAA, 0x43, 0xE7, 0x88, 0xAB, 0x43, + 0xE7, 0x88, 0xB5, 0x43, 0xE7, 0x88, 0xB6, 0x43, + 0xE7, 0x88, 0xBB, 0x43, 0xE7, 0x88, 0xBF, 0x43, + 0xE7, 0x89, 0x87, 0x43, 0xE7, 0x89, 0x90, 0x43, + 0xE7, 0x89, 0x99, 0x43, 0xE7, 0x89, 0x9B, 0x43, + // Bytes f00 - f3f + 0xE7, 0x89, 0xA2, 0x43, 0xE7, 0x89, 0xB9, 0x43, + 0xE7, 0x8A, 0x80, 0x43, 0xE7, 0x8A, 0x95, 0x43, + 0xE7, 0x8A, 0xAC, 0x43, 0xE7, 0x8A, 0xAF, 0x43, + 0xE7, 0x8B, 0x80, 0x43, 0xE7, 0x8B, 0xBC, 0x43, + 0xE7, 0x8C, 0xAA, 0x43, 0xE7, 0x8D, 0xB5, 0x43, + 0xE7, 0x8D, 0xBA, 0x43, 0xE7, 0x8E, 0x84, 0x43, + 0xE7, 0x8E, 0x87, 0x43, 0xE7, 0x8E, 0x89, 0x43, + 0xE7, 0x8E, 0x8B, 0x43, 0xE7, 0x8E, 0xA5, 0x43, + // Bytes f40 - f7f + 0xE7, 0x8E, 0xB2, 0x43, 0xE7, 0x8F, 0x9E, 0x43, + 0xE7, 0x90, 0x86, 0x43, 0xE7, 0x90, 0x89, 0x43, + 0xE7, 0x90, 0xA2, 0x43, 0xE7, 0x91, 0x87, 0x43, + 0xE7, 0x91, 0x9C, 0x43, 0xE7, 0x91, 0xA9, 0x43, + 0xE7, 0x91, 0xB1, 0x43, 0xE7, 0x92, 0x85, 0x43, + 0xE7, 0x92, 0x89, 0x43, 0xE7, 0x92, 0x98, 0x43, + 0xE7, 0x93, 0x8A, 0x43, 0xE7, 0x93, 0x9C, 0x43, + 0xE7, 0x93, 0xA6, 0x43, 0xE7, 0x94, 0x86, 0x43, + // Bytes f80 - fbf + 0xE7, 0x94, 0x98, 0x43, 0xE7, 0x94, 0x9F, 0x43, + 0xE7, 0x94, 0xA4, 0x43, 0xE7, 0x94, 0xA8, 0x43, + 0xE7, 0x94, 0xB0, 0x43, 0xE7, 0x94, 0xB2, 0x43, + 0xE7, 0x94, 0xB3, 0x43, 0xE7, 0x94, 0xB7, 0x43, + 0xE7, 0x94, 0xBB, 0x43, 0xE7, 0x94, 0xBE, 0x43, + 0xE7, 0x95, 0x99, 0x43, 0xE7, 0x95, 0xA5, 0x43, + 0xE7, 0x95, 0xB0, 0x43, 0xE7, 0x96, 0x8B, 0x43, + 0xE7, 0x96, 0x92, 0x43, 0xE7, 0x97, 0xA2, 0x43, + // Bytes fc0 - fff + 0xE7, 0x98, 0x90, 0x43, 0xE7, 0x98, 0x9D, 0x43, + 0xE7, 0x98, 0x9F, 0x43, 0xE7, 0x99, 0x82, 0x43, + 0xE7, 0x99, 0xA9, 0x43, 0xE7, 0x99, 0xB6, 0x43, + 0xE7, 0x99, 0xBD, 0x43, 0xE7, 0x9A, 0xAE, 0x43, + 0xE7, 0x9A, 0xBF, 0x43, 0xE7, 0x9B, 0x8A, 0x43, + 0xE7, 0x9B, 0x9B, 0x43, 0xE7, 0x9B, 0xA3, 0x43, + 0xE7, 0x9B, 0xA7, 0x43, 0xE7, 0x9B, 0xAE, 0x43, + 0xE7, 0x9B, 0xB4, 0x43, 0xE7, 0x9C, 0x81, 0x43, + // Bytes 1000 - 103f + 0xE7, 0x9C, 0x9E, 0x43, 0xE7, 0x9C, 0x9F, 0x43, + 0xE7, 0x9D, 0x80, 0x43, 0xE7, 0x9D, 0x8A, 0x43, + 0xE7, 0x9E, 0x8B, 0x43, 0xE7, 0x9E, 0xA7, 0x43, + 0xE7, 0x9F, 0x9B, 0x43, 0xE7, 0x9F, 0xA2, 0x43, + 0xE7, 0x9F, 0xB3, 0x43, 0xE7, 0xA1, 0x8E, 0x43, + 0xE7, 0xA1, 0xAB, 0x43, 0xE7, 0xA2, 0x8C, 0x43, + 0xE7, 0xA2, 0x91, 0x43, 0xE7, 0xA3, 0x8A, 0x43, + 0xE7, 0xA3, 0x8C, 0x43, 0xE7, 0xA3, 0xBB, 0x43, + // Bytes 1040 - 107f + 0xE7, 0xA4, 0xAA, 0x43, 0xE7, 0xA4, 0xBA, 0x43, + 0xE7, 0xA4, 0xBC, 0x43, 0xE7, 0xA4, 0xBE, 0x43, + 0xE7, 0xA5, 0x88, 0x43, 0xE7, 0xA5, 0x89, 0x43, + 0xE7, 0xA5, 0x90, 0x43, 0xE7, 0xA5, 0x96, 0x43, + 0xE7, 0xA5, 0x9D, 0x43, 0xE7, 0xA5, 0x9E, 0x43, + 0xE7, 0xA5, 0xA5, 0x43, 0xE7, 0xA5, 0xBF, 0x43, + 0xE7, 0xA6, 0x81, 0x43, 0xE7, 0xA6, 0x8D, 0x43, + 0xE7, 0xA6, 0x8E, 0x43, 0xE7, 0xA6, 0x8F, 0x43, + // Bytes 1080 - 10bf + 0xE7, 0xA6, 0xAE, 0x43, 0xE7, 0xA6, 0xB8, 0x43, + 0xE7, 0xA6, 0xBE, 0x43, 0xE7, 0xA7, 0x8A, 0x43, + 0xE7, 0xA7, 0x98, 0x43, 0xE7, 0xA7, 0xAB, 0x43, + 0xE7, 0xA8, 0x9C, 0x43, 0xE7, 0xA9, 0x80, 0x43, + 0xE7, 0xA9, 0x8A, 0x43, 0xE7, 0xA9, 0x8F, 0x43, + 0xE7, 0xA9, 0xB4, 0x43, 0xE7, 0xA9, 0xBA, 0x43, + 0xE7, 0xAA, 0x81, 0x43, 0xE7, 0xAA, 0xB1, 0x43, + 0xE7, 0xAB, 0x8B, 0x43, 0xE7, 0xAB, 0xAE, 0x43, + // Bytes 10c0 - 10ff + 0xE7, 0xAB, 0xB9, 0x43, 0xE7, 0xAC, 0xA0, 0x43, + 0xE7, 0xAE, 0x8F, 0x43, 0xE7, 0xAF, 0x80, 0x43, + 0xE7, 0xAF, 0x86, 0x43, 0xE7, 0xAF, 0x89, 0x43, + 0xE7, 0xB0, 0xBE, 0x43, 0xE7, 0xB1, 0xA0, 0x43, + 0xE7, 0xB1, 0xB3, 0x43, 0xE7, 0xB1, 0xBB, 0x43, + 0xE7, 0xB2, 0x92, 0x43, 0xE7, 0xB2, 0xBE, 0x43, + 0xE7, 0xB3, 0x92, 0x43, 0xE7, 0xB3, 0x96, 0x43, + 0xE7, 0xB3, 0xA3, 0x43, 0xE7, 0xB3, 0xA7, 0x43, + // Bytes 1100 - 113f + 0xE7, 0xB3, 0xA8, 0x43, 0xE7, 0xB3, 0xB8, 0x43, + 0xE7, 0xB4, 0x80, 0x43, 0xE7, 0xB4, 0x90, 0x43, + 0xE7, 0xB4, 0xA2, 0x43, 0xE7, 0xB4, 0xAF, 0x43, + 0xE7, 0xB5, 0x82, 0x43, 0xE7, 0xB5, 0x9B, 0x43, + 0xE7, 0xB5, 0xA3, 0x43, 0xE7, 0xB6, 0xA0, 0x43, + 0xE7, 0xB6, 0xBE, 0x43, 0xE7, 0xB7, 0x87, 0x43, + 0xE7, 0xB7, 0xB4, 0x43, 0xE7, 0xB8, 0x82, 0x43, + 0xE7, 0xB8, 0x89, 0x43, 0xE7, 0xB8, 0xB7, 0x43, + // Bytes 1140 - 117f + 0xE7, 0xB9, 0x81, 0x43, 0xE7, 0xB9, 0x85, 0x43, + 0xE7, 0xBC, 0xB6, 0x43, 0xE7, 0xBC, 0xBE, 0x43, + 0xE7, 0xBD, 0x91, 0x43, 0xE7, 0xBD, 0xB2, 0x43, + 0xE7, 0xBD, 0xB9, 0x43, 0xE7, 0xBD, 0xBA, 0x43, + 0xE7, 0xBE, 0x85, 0x43, 0xE7, 0xBE, 0x8A, 0x43, + 0xE7, 0xBE, 0x95, 0x43, 0xE7, 0xBE, 0x9A, 0x43, + 0xE7, 0xBE, 0xBD, 0x43, 0xE7, 0xBF, 0xBA, 0x43, + 0xE8, 0x80, 0x81, 0x43, 0xE8, 0x80, 0x85, 0x43, + // Bytes 1180 - 11bf + 0xE8, 0x80, 0x8C, 0x43, 0xE8, 0x80, 0x92, 0x43, + 0xE8, 0x80, 0xB3, 0x43, 0xE8, 0x81, 0x86, 0x43, + 0xE8, 0x81, 0xA0, 0x43, 0xE8, 0x81, 0xAF, 0x43, + 0xE8, 0x81, 0xB0, 0x43, 0xE8, 0x81, 0xBE, 0x43, + 0xE8, 0x81, 0xBF, 0x43, 0xE8, 0x82, 0x89, 0x43, + 0xE8, 0x82, 0x8B, 0x43, 0xE8, 0x82, 0xAD, 0x43, + 0xE8, 0x82, 0xB2, 0x43, 0xE8, 0x84, 0x83, 0x43, + 0xE8, 0x84, 0xBE, 0x43, 0xE8, 0x87, 0x98, 0x43, + // Bytes 11c0 - 11ff + 0xE8, 0x87, 0xA3, 0x43, 0xE8, 0x87, 0xA8, 0x43, + 0xE8, 0x87, 0xAA, 0x43, 0xE8, 0x87, 0xAD, 0x43, + 0xE8, 0x87, 0xB3, 0x43, 0xE8, 0x87, 0xBC, 0x43, + 0xE8, 0x88, 0x81, 0x43, 0xE8, 0x88, 0x84, 0x43, + 0xE8, 0x88, 0x8C, 0x43, 0xE8, 0x88, 0x98, 0x43, + 0xE8, 0x88, 0x9B, 0x43, 0xE8, 0x88, 0x9F, 0x43, + 0xE8, 0x89, 0xAE, 0x43, 0xE8, 0x89, 0xAF, 0x43, + 0xE8, 0x89, 0xB2, 0x43, 0xE8, 0x89, 0xB8, 0x43, + // Bytes 1200 - 123f + 0xE8, 0x89, 0xB9, 0x43, 0xE8, 0x8A, 0x8B, 0x43, + 0xE8, 0x8A, 0x91, 0x43, 0xE8, 0x8A, 0x9D, 0x43, + 0xE8, 0x8A, 0xB1, 0x43, 0xE8, 0x8A, 0xB3, 0x43, + 0xE8, 0x8A, 0xBD, 0x43, 0xE8, 0x8B, 0xA5, 0x43, + 0xE8, 0x8B, 0xA6, 0x43, 0xE8, 0x8C, 0x9D, 0x43, + 0xE8, 0x8C, 0xA3, 0x43, 0xE8, 0x8C, 0xB6, 0x43, + 0xE8, 0x8D, 0x92, 0x43, 0xE8, 0x8D, 0x93, 0x43, + 0xE8, 0x8D, 0xA3, 0x43, 0xE8, 0x8E, 0xAD, 0x43, + // Bytes 1240 - 127f + 0xE8, 0x8E, 0xBD, 0x43, 0xE8, 0x8F, 0x89, 0x43, + 0xE8, 0x8F, 0x8A, 0x43, 0xE8, 0x8F, 0x8C, 0x43, + 0xE8, 0x8F, 0x9C, 0x43, 0xE8, 0x8F, 0xA7, 0x43, + 0xE8, 0x8F, 0xAF, 0x43, 0xE8, 0x8F, 0xB1, 0x43, + 0xE8, 0x90, 0xBD, 0x43, 0xE8, 0x91, 0x89, 0x43, + 0xE8, 0x91, 0x97, 0x43, 0xE8, 0x93, 0xAE, 0x43, + 0xE8, 0x93, 0xB1, 0x43, 0xE8, 0x93, 0xB3, 0x43, + 0xE8, 0x93, 0xBC, 0x43, 0xE8, 0x94, 0x96, 0x43, + // Bytes 1280 - 12bf + 0xE8, 0x95, 0xA4, 0x43, 0xE8, 0x97, 0x8D, 0x43, + 0xE8, 0x97, 0xBA, 0x43, 0xE8, 0x98, 0x86, 0x43, + 0xE8, 0x98, 0x92, 0x43, 0xE8, 0x98, 0xAD, 0x43, + 0xE8, 0x98, 0xBF, 0x43, 0xE8, 0x99, 0x8D, 0x43, + 0xE8, 0x99, 0x90, 0x43, 0xE8, 0x99, 0x9C, 0x43, + 0xE8, 0x99, 0xA7, 0x43, 0xE8, 0x99, 0xA9, 0x43, + 0xE8, 0x99, 0xAB, 0x43, 0xE8, 0x9A, 0x88, 0x43, + 0xE8, 0x9A, 0xA9, 0x43, 0xE8, 0x9B, 0xA2, 0x43, + // Bytes 12c0 - 12ff + 0xE8, 0x9C, 0x8E, 0x43, 0xE8, 0x9C, 0xA8, 0x43, + 0xE8, 0x9D, 0xAB, 0x43, 0xE8, 0x9D, 0xB9, 0x43, + 0xE8, 0x9E, 0x86, 0x43, 0xE8, 0x9E, 0xBA, 0x43, + 0xE8, 0x9F, 0xA1, 0x43, 0xE8, 0xA0, 0x81, 0x43, + 0xE8, 0xA0, 0x9F, 0x43, 0xE8, 0xA1, 0x80, 0x43, + 0xE8, 0xA1, 0x8C, 0x43, 0xE8, 0xA1, 0xA0, 0x43, + 0xE8, 0xA1, 0xA3, 0x43, 0xE8, 0xA3, 0x82, 0x43, + 0xE8, 0xA3, 0x8F, 0x43, 0xE8, 0xA3, 0x97, 0x43, + // Bytes 1300 - 133f + 0xE8, 0xA3, 0x9E, 0x43, 0xE8, 0xA3, 0xA1, 0x43, + 0xE8, 0xA3, 0xB8, 0x43, 0xE8, 0xA3, 0xBA, 0x43, + 0xE8, 0xA4, 0x90, 0x43, 0xE8, 0xA5, 0x81, 0x43, + 0xE8, 0xA5, 0xA4, 0x43, 0xE8, 0xA5, 0xBE, 0x43, + 0xE8, 0xA6, 0x86, 0x43, 0xE8, 0xA6, 0x8B, 0x43, + 0xE8, 0xA6, 0x96, 0x43, 0xE8, 0xA7, 0x92, 0x43, + 0xE8, 0xA7, 0xA3, 0x43, 0xE8, 0xA8, 0x80, 0x43, + 0xE8, 0xAA, 0xA0, 0x43, 0xE8, 0xAA, 0xAA, 0x43, + // Bytes 1340 - 137f + 0xE8, 0xAA, 0xBF, 0x43, 0xE8, 0xAB, 0x8B, 0x43, + 0xE8, 0xAB, 0x92, 0x43, 0xE8, 0xAB, 0x96, 0x43, + 0xE8, 0xAB, 0xAD, 0x43, 0xE8, 0xAB, 0xB8, 0x43, + 0xE8, 0xAB, 0xBE, 0x43, 0xE8, 0xAC, 0x81, 0x43, + 0xE8, 0xAC, 0xB9, 0x43, 0xE8, 0xAD, 0x98, 0x43, + 0xE8, 0xAE, 0x80, 0x43, 0xE8, 0xAE, 0x8A, 0x43, + 0xE8, 0xB0, 0xB7, 0x43, 0xE8, 0xB1, 0x86, 0x43, + 0xE8, 0xB1, 0x88, 0x43, 0xE8, 0xB1, 0x95, 0x43, + // Bytes 1380 - 13bf + 0xE8, 0xB1, 0xB8, 0x43, 0xE8, 0xB2, 0x9D, 0x43, + 0xE8, 0xB2, 0xA1, 0x43, 0xE8, 0xB2, 0xA9, 0x43, + 0xE8, 0xB2, 0xAB, 0x43, 0xE8, 0xB3, 0x81, 0x43, + 0xE8, 0xB3, 0x82, 0x43, 0xE8, 0xB3, 0x87, 0x43, + 0xE8, 0xB3, 0x88, 0x43, 0xE8, 0xB3, 0x93, 0x43, + 0xE8, 0xB4, 0x88, 0x43, 0xE8, 0xB4, 0x9B, 0x43, + 0xE8, 0xB5, 0xA4, 0x43, 0xE8, 0xB5, 0xB0, 0x43, + 0xE8, 0xB5, 0xB7, 0x43, 0xE8, 0xB6, 0xB3, 0x43, + // Bytes 13c0 - 13ff + 0xE8, 0xB6, 0xBC, 0x43, 0xE8, 0xB7, 0x8B, 0x43, + 0xE8, 0xB7, 0xAF, 0x43, 0xE8, 0xB7, 0xB0, 0x43, + 0xE8, 0xBA, 0xAB, 0x43, 0xE8, 0xBB, 0x8A, 0x43, + 0xE8, 0xBB, 0x94, 0x43, 0xE8, 0xBC, 0xA6, 0x43, + 0xE8, 0xBC, 0xAA, 0x43, 0xE8, 0xBC, 0xB8, 0x43, + 0xE8, 0xBC, 0xBB, 0x43, 0xE8, 0xBD, 0xA2, 0x43, + 0xE8, 0xBE, 0x9B, 0x43, 0xE8, 0xBE, 0x9E, 0x43, + 0xE8, 0xBE, 0xB0, 0x43, 0xE8, 0xBE, 0xB5, 0x43, + // Bytes 1400 - 143f + 0xE8, 0xBE, 0xB6, 0x43, 0xE9, 0x80, 0xA3, 0x43, + 0xE9, 0x80, 0xB8, 0x43, 0xE9, 0x81, 0x8A, 0x43, + 0xE9, 0x81, 0xA9, 0x43, 0xE9, 0x81, 0xB2, 0x43, + 0xE9, 0x81, 0xBC, 0x43, 0xE9, 0x82, 0x8F, 0x43, + 0xE9, 0x82, 0x91, 0x43, 0xE9, 0x82, 0x94, 0x43, + 0xE9, 0x83, 0x8E, 0x43, 0xE9, 0x83, 0x9E, 0x43, + 0xE9, 0x83, 0xB1, 0x43, 0xE9, 0x83, 0xBD, 0x43, + 0xE9, 0x84, 0x91, 0x43, 0xE9, 0x84, 0x9B, 0x43, + // Bytes 1440 - 147f + 0xE9, 0x85, 0x89, 0x43, 0xE9, 0x85, 0x8D, 0x43, + 0xE9, 0x85, 0xAA, 0x43, 0xE9, 0x86, 0x99, 0x43, + 0xE9, 0x86, 0xB4, 0x43, 0xE9, 0x87, 0x86, 0x43, + 0xE9, 0x87, 0x8C, 0x43, 0xE9, 0x87, 0x8F, 0x43, + 0xE9, 0x87, 0x91, 0x43, 0xE9, 0x88, 0xB4, 0x43, + 0xE9, 0x88, 0xB8, 0x43, 0xE9, 0x89, 0xB6, 0x43, + 0xE9, 0x89, 0xBC, 0x43, 0xE9, 0x8B, 0x97, 0x43, + 0xE9, 0x8B, 0x98, 0x43, 0xE9, 0x8C, 0x84, 0x43, + // Bytes 1480 - 14bf + 0xE9, 0x8D, 0x8A, 0x43, 0xE9, 0x8F, 0xB9, 0x43, + 0xE9, 0x90, 0x95, 0x43, 0xE9, 0x95, 0xB7, 0x43, + 0xE9, 0x96, 0x80, 0x43, 0xE9, 0x96, 0x8B, 0x43, + 0xE9, 0x96, 0xAD, 0x43, 0xE9, 0x96, 0xB7, 0x43, + 0xE9, 0x98, 0x9C, 0x43, 0xE9, 0x98, 0xAE, 0x43, + 0xE9, 0x99, 0x8B, 0x43, 0xE9, 0x99, 0x8D, 0x43, + 0xE9, 0x99, 0xB5, 0x43, 0xE9, 0x99, 0xB8, 0x43, + 0xE9, 0x99, 0xBC, 0x43, 0xE9, 0x9A, 0x86, 0x43, + // Bytes 14c0 - 14ff + 0xE9, 0x9A, 0xA3, 0x43, 0xE9, 0x9A, 0xB6, 0x43, + 0xE9, 0x9A, 0xB7, 0x43, 0xE9, 0x9A, 0xB8, 0x43, + 0xE9, 0x9A, 0xB9, 0x43, 0xE9, 0x9B, 0x83, 0x43, + 0xE9, 0x9B, 0xA2, 0x43, 0xE9, 0x9B, 0xA3, 0x43, + 0xE9, 0x9B, 0xA8, 0x43, 0xE9, 0x9B, 0xB6, 0x43, + 0xE9, 0x9B, 0xB7, 0x43, 0xE9, 0x9C, 0xA3, 0x43, + 0xE9, 0x9C, 0xB2, 0x43, 0xE9, 0x9D, 0x88, 0x43, + 0xE9, 0x9D, 0x91, 0x43, 0xE9, 0x9D, 0x96, 0x43, + // Bytes 1500 - 153f + 0xE9, 0x9D, 0x9E, 0x43, 0xE9, 0x9D, 0xA2, 0x43, + 0xE9, 0x9D, 0xA9, 0x43, 0xE9, 0x9F, 0x8B, 0x43, + 0xE9, 0x9F, 0x9B, 0x43, 0xE9, 0x9F, 0xA0, 0x43, + 0xE9, 0x9F, 0xAD, 0x43, 0xE9, 0x9F, 0xB3, 0x43, + 0xE9, 0x9F, 0xBF, 0x43, 0xE9, 0xA0, 0x81, 0x43, + 0xE9, 0xA0, 0x85, 0x43, 0xE9, 0xA0, 0x8B, 0x43, + 0xE9, 0xA0, 0x98, 0x43, 0xE9, 0xA0, 0xA9, 0x43, + 0xE9, 0xA0, 0xBB, 0x43, 0xE9, 0xA1, 0x9E, 0x43, + // Bytes 1540 - 157f + 0xE9, 0xA2, 0xA8, 0x43, 0xE9, 0xA3, 0x9B, 0x43, + 0xE9, 0xA3, 0x9F, 0x43, 0xE9, 0xA3, 0xA2, 0x43, + 0xE9, 0xA3, 0xAF, 0x43, 0xE9, 0xA3, 0xBC, 0x43, + 0xE9, 0xA4, 0xA8, 0x43, 0xE9, 0xA4, 0xA9, 0x43, + 0xE9, 0xA6, 0x96, 0x43, 0xE9, 0xA6, 0x99, 0x43, + 0xE9, 0xA6, 0xA7, 0x43, 0xE9, 0xA6, 0xAC, 0x43, + 0xE9, 0xA7, 0x82, 0x43, 0xE9, 0xA7, 0xB1, 0x43, + 0xE9, 0xA7, 0xBE, 0x43, 0xE9, 0xA9, 0xAA, 0x43, + // Bytes 1580 - 15bf + 0xE9, 0xAA, 0xA8, 0x43, 0xE9, 0xAB, 0x98, 0x43, + 0xE9, 0xAB, 0x9F, 0x43, 0xE9, 0xAC, 0x92, 0x43, + 0xE9, 0xAC, 0xA5, 0x43, 0xE9, 0xAC, 0xAF, 0x43, + 0xE9, 0xAC, 0xB2, 0x43, 0xE9, 0xAC, 0xBC, 0x43, + 0xE9, 0xAD, 0x9A, 0x43, 0xE9, 0xAD, 0xAF, 0x43, + 0xE9, 0xB1, 0x80, 0x43, 0xE9, 0xB1, 0x97, 0x43, + 0xE9, 0xB3, 0xA5, 0x43, 0xE9, 0xB3, 0xBD, 0x43, + 0xE9, 0xB5, 0xA7, 0x43, 0xE9, 0xB6, 0xB4, 0x43, + // Bytes 15c0 - 15ff + 0xE9, 0xB7, 0xBA, 0x43, 0xE9, 0xB8, 0x9E, 0x43, + 0xE9, 0xB9, 0xB5, 0x43, 0xE9, 0xB9, 0xBF, 0x43, + 0xE9, 0xBA, 0x97, 0x43, 0xE9, 0xBA, 0x9F, 0x43, + 0xE9, 0xBA, 0xA5, 0x43, 0xE9, 0xBA, 0xBB, 0x43, + 0xE9, 0xBB, 0x83, 0x43, 0xE9, 0xBB, 0x8D, 0x43, + 0xE9, 0xBB, 0x8E, 0x43, 0xE9, 0xBB, 0x91, 0x43, + 0xE9, 0xBB, 0xB9, 0x43, 0xE9, 0xBB, 0xBD, 0x43, + 0xE9, 0xBB, 0xBE, 0x43, 0xE9, 0xBC, 0x85, 0x43, + // Bytes 1600 - 163f + 0xE9, 0xBC, 0x8E, 0x43, 0xE9, 0xBC, 0x8F, 0x43, + 0xE9, 0xBC, 0x93, 0x43, 0xE9, 0xBC, 0x96, 0x43, + 0xE9, 0xBC, 0xA0, 0x43, 0xE9, 0xBC, 0xBB, 0x43, + 0xE9, 0xBD, 0x83, 0x43, 0xE9, 0xBD, 0x8A, 0x43, + 0xE9, 0xBD, 0x92, 0x43, 0xE9, 0xBE, 0x8D, 0x43, + 0xE9, 0xBE, 0x8E, 0x43, 0xE9, 0xBE, 0x9C, 0x43, + 0xE9, 0xBE, 0x9F, 0x43, 0xE9, 0xBE, 0xA0, 0x43, + 0xEA, 0x9C, 0xA7, 0x43, 0xEA, 0x9D, 0xAF, 0x43, + // Bytes 1640 - 167f + 0xEA, 0xAC, 0xB7, 0x43, 0xEA, 0xAD, 0x92, 0x44, + 0xF0, 0xA0, 0x84, 0xA2, 0x44, 0xF0, 0xA0, 0x94, + 0x9C, 0x44, 0xF0, 0xA0, 0x94, 0xA5, 0x44, 0xF0, + 0xA0, 0x95, 0x8B, 0x44, 0xF0, 0xA0, 0x98, 0xBA, + 0x44, 0xF0, 0xA0, 0xA0, 0x84, 0x44, 0xF0, 0xA0, + 0xA3, 0x9E, 0x44, 0xF0, 0xA0, 0xA8, 0xAC, 0x44, + 0xF0, 0xA0, 0xAD, 0xA3, 0x44, 0xF0, 0xA1, 0x93, + 0xA4, 0x44, 0xF0, 0xA1, 0x9A, 0xA8, 0x44, 0xF0, + // Bytes 1680 - 16bf + 0xA1, 0x9B, 0xAA, 0x44, 0xF0, 0xA1, 0xA7, 0x88, + 0x44, 0xF0, 0xA1, 0xAC, 0x98, 0x44, 0xF0, 0xA1, + 0xB4, 0x8B, 0x44, 0xF0, 0xA1, 0xB7, 0xA4, 0x44, + 0xF0, 0xA1, 0xB7, 0xA6, 0x44, 0xF0, 0xA2, 0x86, + 0x83, 0x44, 0xF0, 0xA2, 0x86, 0x9F, 0x44, 0xF0, + 0xA2, 0x8C, 0xB1, 0x44, 0xF0, 0xA2, 0x9B, 0x94, + 0x44, 0xF0, 0xA2, 0xA1, 0x84, 0x44, 0xF0, 0xA2, + 0xA1, 0x8A, 0x44, 0xF0, 0xA2, 0xAC, 0x8C, 0x44, + // Bytes 16c0 - 16ff + 0xF0, 0xA2, 0xAF, 0xB1, 0x44, 0xF0, 0xA3, 0x80, + 0x8A, 0x44, 0xF0, 0xA3, 0x8A, 0xB8, 0x44, 0xF0, + 0xA3, 0x8D, 0x9F, 0x44, 0xF0, 0xA3, 0x8E, 0x93, + 0x44, 0xF0, 0xA3, 0x8E, 0x9C, 0x44, 0xF0, 0xA3, + 0x8F, 0x83, 0x44, 0xF0, 0xA3, 0x8F, 0x95, 0x44, + 0xF0, 0xA3, 0x91, 0xAD, 0x44, 0xF0, 0xA3, 0x9A, + 0xA3, 0x44, 0xF0, 0xA3, 0xA2, 0xA7, 0x44, 0xF0, + 0xA3, 0xAA, 0x8D, 0x44, 0xF0, 0xA3, 0xAB, 0xBA, + // Bytes 1700 - 173f + 0x44, 0xF0, 0xA3, 0xB2, 0xBC, 0x44, 0xF0, 0xA3, + 0xB4, 0x9E, 0x44, 0xF0, 0xA3, 0xBB, 0x91, 0x44, + 0xF0, 0xA3, 0xBD, 0x9E, 0x44, 0xF0, 0xA3, 0xBE, + 0x8E, 0x44, 0xF0, 0xA4, 0x89, 0xA3, 0x44, 0xF0, + 0xA4, 0x8B, 0xAE, 0x44, 0xF0, 0xA4, 0x8E, 0xAB, + 0x44, 0xF0, 0xA4, 0x98, 0x88, 0x44, 0xF0, 0xA4, + 0x9C, 0xB5, 0x44, 0xF0, 0xA4, 0xA0, 0x94, 0x44, + 0xF0, 0xA4, 0xB0, 0xB6, 0x44, 0xF0, 0xA4, 0xB2, + // Bytes 1740 - 177f + 0x92, 0x44, 0xF0, 0xA4, 0xBE, 0xA1, 0x44, 0xF0, + 0xA4, 0xBE, 0xB8, 0x44, 0xF0, 0xA5, 0x81, 0x84, + 0x44, 0xF0, 0xA5, 0x83, 0xB2, 0x44, 0xF0, 0xA5, + 0x83, 0xB3, 0x44, 0xF0, 0xA5, 0x84, 0x99, 0x44, + 0xF0, 0xA5, 0x84, 0xB3, 0x44, 0xF0, 0xA5, 0x89, + 0x89, 0x44, 0xF0, 0xA5, 0x90, 0x9D, 0x44, 0xF0, + 0xA5, 0x98, 0xA6, 0x44, 0xF0, 0xA5, 0x9A, 0x9A, + 0x44, 0xF0, 0xA5, 0x9B, 0x85, 0x44, 0xF0, 0xA5, + // Bytes 1780 - 17bf + 0xA5, 0xBC, 0x44, 0xF0, 0xA5, 0xAA, 0xA7, 0x44, + 0xF0, 0xA5, 0xAE, 0xAB, 0x44, 0xF0, 0xA5, 0xB2, + 0x80, 0x44, 0xF0, 0xA5, 0xB3, 0x90, 0x44, 0xF0, + 0xA5, 0xBE, 0x86, 0x44, 0xF0, 0xA6, 0x87, 0x9A, + 0x44, 0xF0, 0xA6, 0x88, 0xA8, 0x44, 0xF0, 0xA6, + 0x89, 0x87, 0x44, 0xF0, 0xA6, 0x8B, 0x99, 0x44, + 0xF0, 0xA6, 0x8C, 0xBE, 0x44, 0xF0, 0xA6, 0x93, + 0x9A, 0x44, 0xF0, 0xA6, 0x94, 0xA3, 0x44, 0xF0, + // Bytes 17c0 - 17ff + 0xA6, 0x96, 0xA8, 0x44, 0xF0, 0xA6, 0x9E, 0xA7, + 0x44, 0xF0, 0xA6, 0x9E, 0xB5, 0x44, 0xF0, 0xA6, + 0xAC, 0xBC, 0x44, 0xF0, 0xA6, 0xB0, 0xB6, 0x44, + 0xF0, 0xA6, 0xB3, 0x95, 0x44, 0xF0, 0xA6, 0xB5, + 0xAB, 0x44, 0xF0, 0xA6, 0xBC, 0xAC, 0x44, 0xF0, + 0xA6, 0xBE, 0xB1, 0x44, 0xF0, 0xA7, 0x83, 0x92, + 0x44, 0xF0, 0xA7, 0x8F, 0x8A, 0x44, 0xF0, 0xA7, + 0x99, 0xA7, 0x44, 0xF0, 0xA7, 0xA2, 0xAE, 0x44, + // Bytes 1800 - 183f + 0xF0, 0xA7, 0xA5, 0xA6, 0x44, 0xF0, 0xA7, 0xB2, + 0xA8, 0x44, 0xF0, 0xA7, 0xBB, 0x93, 0x44, 0xF0, + 0xA7, 0xBC, 0xAF, 0x44, 0xF0, 0xA8, 0x97, 0x92, + 0x44, 0xF0, 0xA8, 0x97, 0xAD, 0x44, 0xF0, 0xA8, + 0x9C, 0xAE, 0x44, 0xF0, 0xA8, 0xAF, 0xBA, 0x44, + 0xF0, 0xA8, 0xB5, 0xB7, 0x44, 0xF0, 0xA9, 0x85, + 0x85, 0x44, 0xF0, 0xA9, 0x87, 0x9F, 0x44, 0xF0, + 0xA9, 0x88, 0x9A, 0x44, 0xF0, 0xA9, 0x90, 0x8A, + // Bytes 1840 - 187f + 0x44, 0xF0, 0xA9, 0x92, 0x96, 0x44, 0xF0, 0xA9, + 0x96, 0xB6, 0x44, 0xF0, 0xA9, 0xAC, 0xB0, 0x44, + 0xF0, 0xAA, 0x83, 0x8E, 0x44, 0xF0, 0xAA, 0x84, + 0x85, 0x44, 0xF0, 0xAA, 0x88, 0x8E, 0x44, 0xF0, + 0xAA, 0x8A, 0x91, 0x44, 0xF0, 0xAA, 0x8E, 0x92, + 0x44, 0xF0, 0xAA, 0x98, 0x80, 0x42, 0x21, 0x21, + 0x42, 0x21, 0x3F, 0x42, 0x2E, 0x2E, 0x42, 0x30, + 0x2C, 0x42, 0x30, 0x2E, 0x42, 0x31, 0x2C, 0x42, + // Bytes 1880 - 18bf + 0x31, 0x2E, 0x42, 0x31, 0x30, 0x42, 0x31, 0x31, + 0x42, 0x31, 0x32, 0x42, 0x31, 0x33, 0x42, 0x31, + 0x34, 0x42, 0x31, 0x35, 0x42, 0x31, 0x36, 0x42, + 0x31, 0x37, 0x42, 0x31, 0x38, 0x42, 0x31, 0x39, + 0x42, 0x32, 0x2C, 0x42, 0x32, 0x2E, 0x42, 0x32, + 0x30, 0x42, 0x32, 0x31, 0x42, 0x32, 0x32, 0x42, + 0x32, 0x33, 0x42, 0x32, 0x34, 0x42, 0x32, 0x35, + 0x42, 0x32, 0x36, 0x42, 0x32, 0x37, 0x42, 0x32, + // Bytes 18c0 - 18ff + 0x38, 0x42, 0x32, 0x39, 0x42, 0x33, 0x2C, 0x42, + 0x33, 0x2E, 0x42, 0x33, 0x30, 0x42, 0x33, 0x31, + 0x42, 0x33, 0x32, 0x42, 0x33, 0x33, 0x42, 0x33, + 0x34, 0x42, 0x33, 0x35, 0x42, 0x33, 0x36, 0x42, + 0x33, 0x37, 0x42, 0x33, 0x38, 0x42, 0x33, 0x39, + 0x42, 0x34, 0x2C, 0x42, 0x34, 0x2E, 0x42, 0x34, + 0x30, 0x42, 0x34, 0x31, 0x42, 0x34, 0x32, 0x42, + 0x34, 0x33, 0x42, 0x34, 0x34, 0x42, 0x34, 0x35, + // Bytes 1900 - 193f + 0x42, 0x34, 0x36, 0x42, 0x34, 0x37, 0x42, 0x34, + 0x38, 0x42, 0x34, 0x39, 0x42, 0x35, 0x2C, 0x42, + 0x35, 0x2E, 0x42, 0x35, 0x30, 0x42, 0x36, 0x2C, + 0x42, 0x36, 0x2E, 0x42, 0x37, 0x2C, 0x42, 0x37, + 0x2E, 0x42, 0x38, 0x2C, 0x42, 0x38, 0x2E, 0x42, + 0x39, 0x2C, 0x42, 0x39, 0x2E, 0x42, 0x3D, 0x3D, + 0x42, 0x3F, 0x21, 0x42, 0x3F, 0x3F, 0x42, 0x41, + 0x55, 0x42, 0x42, 0x71, 0x42, 0x43, 0x44, 0x42, + // Bytes 1940 - 197f + 0x44, 0x4A, 0x42, 0x44, 0x5A, 0x42, 0x44, 0x7A, + 0x42, 0x47, 0x42, 0x42, 0x47, 0x79, 0x42, 0x48, + 0x50, 0x42, 0x48, 0x56, 0x42, 0x48, 0x67, 0x42, + 0x48, 0x7A, 0x42, 0x49, 0x49, 0x42, 0x49, 0x4A, + 0x42, 0x49, 0x55, 0x42, 0x49, 0x56, 0x42, 0x49, + 0x58, 0x42, 0x4B, 0x42, 0x42, 0x4B, 0x4B, 0x42, + 0x4B, 0x4D, 0x42, 0x4C, 0x4A, 0x42, 0x4C, 0x6A, + 0x42, 0x4D, 0x42, 0x42, 0x4D, 0x43, 0x42, 0x4D, + // Bytes 1980 - 19bf + 0x44, 0x42, 0x4D, 0x52, 0x42, 0x4D, 0x56, 0x42, + 0x4D, 0x57, 0x42, 0x4E, 0x4A, 0x42, 0x4E, 0x6A, + 0x42, 0x4E, 0x6F, 0x42, 0x50, 0x48, 0x42, 0x50, + 0x52, 0x42, 0x50, 0x61, 0x42, 0x52, 0x73, 0x42, + 0x53, 0x44, 0x42, 0x53, 0x4D, 0x42, 0x53, 0x53, + 0x42, 0x53, 0x76, 0x42, 0x54, 0x4D, 0x42, 0x56, + 0x49, 0x42, 0x57, 0x43, 0x42, 0x57, 0x5A, 0x42, + 0x57, 0x62, 0x42, 0x58, 0x49, 0x42, 0x63, 0x63, + // Bytes 19c0 - 19ff + 0x42, 0x63, 0x64, 0x42, 0x63, 0x6D, 0x42, 0x64, + 0x42, 0x42, 0x64, 0x61, 0x42, 0x64, 0x6C, 0x42, + 0x64, 0x6D, 0x42, 0x64, 0x7A, 0x42, 0x65, 0x56, + 0x42, 0x66, 0x66, 0x42, 0x66, 0x69, 0x42, 0x66, + 0x6C, 0x42, 0x66, 0x6D, 0x42, 0x68, 0x61, 0x42, + 0x69, 0x69, 0x42, 0x69, 0x6A, 0x42, 0x69, 0x6E, + 0x42, 0x69, 0x76, 0x42, 0x69, 0x78, 0x42, 0x6B, + 0x41, 0x42, 0x6B, 0x56, 0x42, 0x6B, 0x57, 0x42, + // Bytes 1a00 - 1a3f + 0x6B, 0x67, 0x42, 0x6B, 0x6C, 0x42, 0x6B, 0x6D, + 0x42, 0x6B, 0x74, 0x42, 0x6C, 0x6A, 0x42, 0x6C, + 0x6D, 0x42, 0x6C, 0x6E, 0x42, 0x6C, 0x78, 0x42, + 0x6D, 0x32, 0x42, 0x6D, 0x33, 0x42, 0x6D, 0x41, + 0x42, 0x6D, 0x56, 0x42, 0x6D, 0x57, 0x42, 0x6D, + 0x62, 0x42, 0x6D, 0x67, 0x42, 0x6D, 0x6C, 0x42, + 0x6D, 0x6D, 0x42, 0x6D, 0x73, 0x42, 0x6E, 0x41, + 0x42, 0x6E, 0x46, 0x42, 0x6E, 0x56, 0x42, 0x6E, + // Bytes 1a40 - 1a7f + 0x57, 0x42, 0x6E, 0x6A, 0x42, 0x6E, 0x6D, 0x42, + 0x6E, 0x73, 0x42, 0x6F, 0x56, 0x42, 0x70, 0x41, + 0x42, 0x70, 0x46, 0x42, 0x70, 0x56, 0x42, 0x70, + 0x57, 0x42, 0x70, 0x63, 0x42, 0x70, 0x73, 0x42, + 0x73, 0x72, 0x42, 0x73, 0x74, 0x42, 0x76, 0x69, + 0x42, 0x78, 0x69, 0x43, 0x28, 0x31, 0x29, 0x43, + 0x28, 0x32, 0x29, 0x43, 0x28, 0x33, 0x29, 0x43, + 0x28, 0x34, 0x29, 0x43, 0x28, 0x35, 0x29, 0x43, + // Bytes 1a80 - 1abf + 0x28, 0x36, 0x29, 0x43, 0x28, 0x37, 0x29, 0x43, + 0x28, 0x38, 0x29, 0x43, 0x28, 0x39, 0x29, 0x43, + 0x28, 0x41, 0x29, 0x43, 0x28, 0x42, 0x29, 0x43, + 0x28, 0x43, 0x29, 0x43, 0x28, 0x44, 0x29, 0x43, + 0x28, 0x45, 0x29, 0x43, 0x28, 0x46, 0x29, 0x43, + 0x28, 0x47, 0x29, 0x43, 0x28, 0x48, 0x29, 0x43, + 0x28, 0x49, 0x29, 0x43, 0x28, 0x4A, 0x29, 0x43, + 0x28, 0x4B, 0x29, 0x43, 0x28, 0x4C, 0x29, 0x43, + // Bytes 1ac0 - 1aff + 0x28, 0x4D, 0x29, 0x43, 0x28, 0x4E, 0x29, 0x43, + 0x28, 0x4F, 0x29, 0x43, 0x28, 0x50, 0x29, 0x43, + 0x28, 0x51, 0x29, 0x43, 0x28, 0x52, 0x29, 0x43, + 0x28, 0x53, 0x29, 0x43, 0x28, 0x54, 0x29, 0x43, + 0x28, 0x55, 0x29, 0x43, 0x28, 0x56, 0x29, 0x43, + 0x28, 0x57, 0x29, 0x43, 0x28, 0x58, 0x29, 0x43, + 0x28, 0x59, 0x29, 0x43, 0x28, 0x5A, 0x29, 0x43, + 0x28, 0x61, 0x29, 0x43, 0x28, 0x62, 0x29, 0x43, + // Bytes 1b00 - 1b3f + 0x28, 0x63, 0x29, 0x43, 0x28, 0x64, 0x29, 0x43, + 0x28, 0x65, 0x29, 0x43, 0x28, 0x66, 0x29, 0x43, + 0x28, 0x67, 0x29, 0x43, 0x28, 0x68, 0x29, 0x43, + 0x28, 0x69, 0x29, 0x43, 0x28, 0x6A, 0x29, 0x43, + 0x28, 0x6B, 0x29, 0x43, 0x28, 0x6C, 0x29, 0x43, + 0x28, 0x6D, 0x29, 0x43, 0x28, 0x6E, 0x29, 0x43, + 0x28, 0x6F, 0x29, 0x43, 0x28, 0x70, 0x29, 0x43, + 0x28, 0x71, 0x29, 0x43, 0x28, 0x72, 0x29, 0x43, + // Bytes 1b40 - 1b7f + 0x28, 0x73, 0x29, 0x43, 0x28, 0x74, 0x29, 0x43, + 0x28, 0x75, 0x29, 0x43, 0x28, 0x76, 0x29, 0x43, + 0x28, 0x77, 0x29, 0x43, 0x28, 0x78, 0x29, 0x43, + 0x28, 0x79, 0x29, 0x43, 0x28, 0x7A, 0x29, 0x43, + 0x2E, 0x2E, 0x2E, 0x43, 0x31, 0x30, 0x2E, 0x43, + 0x31, 0x31, 0x2E, 0x43, 0x31, 0x32, 0x2E, 0x43, + 0x31, 0x33, 0x2E, 0x43, 0x31, 0x34, 0x2E, 0x43, + 0x31, 0x35, 0x2E, 0x43, 0x31, 0x36, 0x2E, 0x43, + // Bytes 1b80 - 1bbf + 0x31, 0x37, 0x2E, 0x43, 0x31, 0x38, 0x2E, 0x43, + 0x31, 0x39, 0x2E, 0x43, 0x32, 0x30, 0x2E, 0x43, + 0x3A, 0x3A, 0x3D, 0x43, 0x3D, 0x3D, 0x3D, 0x43, + 0x43, 0x6F, 0x2E, 0x43, 0x46, 0x41, 0x58, 0x43, + 0x47, 0x48, 0x7A, 0x43, 0x47, 0x50, 0x61, 0x43, + 0x49, 0x49, 0x49, 0x43, 0x4C, 0x54, 0x44, 0x43, + 0x4C, 0xC2, 0xB7, 0x43, 0x4D, 0x48, 0x7A, 0x43, + 0x4D, 0x50, 0x61, 0x43, 0x4D, 0xCE, 0xA9, 0x43, + // Bytes 1bc0 - 1bff + 0x50, 0x50, 0x4D, 0x43, 0x50, 0x50, 0x56, 0x43, + 0x50, 0x54, 0x45, 0x43, 0x54, 0x45, 0x4C, 0x43, + 0x54, 0x48, 0x7A, 0x43, 0x56, 0x49, 0x49, 0x43, + 0x58, 0x49, 0x49, 0x43, 0x61, 0x2F, 0x63, 0x43, + 0x61, 0x2F, 0x73, 0x43, 0x61, 0xCA, 0xBE, 0x43, + 0x62, 0x61, 0x72, 0x43, 0x63, 0x2F, 0x6F, 0x43, + 0x63, 0x2F, 0x75, 0x43, 0x63, 0x61, 0x6C, 0x43, + 0x63, 0x6D, 0x32, 0x43, 0x63, 0x6D, 0x33, 0x43, + // Bytes 1c00 - 1c3f + 0x64, 0x6D, 0x32, 0x43, 0x64, 0x6D, 0x33, 0x43, + 0x65, 0x72, 0x67, 0x43, 0x66, 0x66, 0x69, 0x43, + 0x66, 0x66, 0x6C, 0x43, 0x67, 0x61, 0x6C, 0x43, + 0x68, 0x50, 0x61, 0x43, 0x69, 0x69, 0x69, 0x43, + 0x6B, 0x48, 0x7A, 0x43, 0x6B, 0x50, 0x61, 0x43, + 0x6B, 0x6D, 0x32, 0x43, 0x6B, 0x6D, 0x33, 0x43, + 0x6B, 0xCE, 0xA9, 0x43, 0x6C, 0x6F, 0x67, 0x43, + 0x6C, 0xC2, 0xB7, 0x43, 0x6D, 0x69, 0x6C, 0x43, + // Bytes 1c40 - 1c7f + 0x6D, 0x6D, 0x32, 0x43, 0x6D, 0x6D, 0x33, 0x43, + 0x6D, 0x6F, 0x6C, 0x43, 0x72, 0x61, 0x64, 0x43, + 0x76, 0x69, 0x69, 0x43, 0x78, 0x69, 0x69, 0x43, + 0xC2, 0xB0, 0x43, 0x43, 0xC2, 0xB0, 0x46, 0x43, + 0xCA, 0xBC, 0x6E, 0x43, 0xCE, 0xBC, 0x41, 0x43, + 0xCE, 0xBC, 0x46, 0x43, 0xCE, 0xBC, 0x56, 0x43, + 0xCE, 0xBC, 0x57, 0x43, 0xCE, 0xBC, 0x67, 0x43, + 0xCE, 0xBC, 0x6C, 0x43, 0xCE, 0xBC, 0x6D, 0x43, + // Bytes 1c80 - 1cbf + 0xCE, 0xBC, 0x73, 0x44, 0x28, 0x31, 0x30, 0x29, + 0x44, 0x28, 0x31, 0x31, 0x29, 0x44, 0x28, 0x31, + 0x32, 0x29, 0x44, 0x28, 0x31, 0x33, 0x29, 0x44, + 0x28, 0x31, 0x34, 0x29, 0x44, 0x28, 0x31, 0x35, + 0x29, 0x44, 0x28, 0x31, 0x36, 0x29, 0x44, 0x28, + 0x31, 0x37, 0x29, 0x44, 0x28, 0x31, 0x38, 0x29, + 0x44, 0x28, 0x31, 0x39, 0x29, 0x44, 0x28, 0x32, + 0x30, 0x29, 0x44, 0x30, 0xE7, 0x82, 0xB9, 0x44, + // Bytes 1cc0 - 1cff + 0x31, 0xE2, 0x81, 0x84, 0x44, 0x31, 0xE6, 0x97, + 0xA5, 0x44, 0x31, 0xE6, 0x9C, 0x88, 0x44, 0x31, + 0xE7, 0x82, 0xB9, 0x44, 0x32, 0xE6, 0x97, 0xA5, + 0x44, 0x32, 0xE6, 0x9C, 0x88, 0x44, 0x32, 0xE7, + 0x82, 0xB9, 0x44, 0x33, 0xE6, 0x97, 0xA5, 0x44, + 0x33, 0xE6, 0x9C, 0x88, 0x44, 0x33, 0xE7, 0x82, + 0xB9, 0x44, 0x34, 0xE6, 0x97, 0xA5, 0x44, 0x34, + 0xE6, 0x9C, 0x88, 0x44, 0x34, 0xE7, 0x82, 0xB9, + // Bytes 1d00 - 1d3f + 0x44, 0x35, 0xE6, 0x97, 0xA5, 0x44, 0x35, 0xE6, + 0x9C, 0x88, 0x44, 0x35, 0xE7, 0x82, 0xB9, 0x44, + 0x36, 0xE6, 0x97, 0xA5, 0x44, 0x36, 0xE6, 0x9C, + 0x88, 0x44, 0x36, 0xE7, 0x82, 0xB9, 0x44, 0x37, + 0xE6, 0x97, 0xA5, 0x44, 0x37, 0xE6, 0x9C, 0x88, + 0x44, 0x37, 0xE7, 0x82, 0xB9, 0x44, 0x38, 0xE6, + 0x97, 0xA5, 0x44, 0x38, 0xE6, 0x9C, 0x88, 0x44, + 0x38, 0xE7, 0x82, 0xB9, 0x44, 0x39, 0xE6, 0x97, + // Bytes 1d40 - 1d7f + 0xA5, 0x44, 0x39, 0xE6, 0x9C, 0x88, 0x44, 0x39, + 0xE7, 0x82, 0xB9, 0x44, 0x56, 0x49, 0x49, 0x49, + 0x44, 0x61, 0x2E, 0x6D, 0x2E, 0x44, 0x6B, 0x63, + 0x61, 0x6C, 0x44, 0x70, 0x2E, 0x6D, 0x2E, 0x44, + 0x76, 0x69, 0x69, 0x69, 0x44, 0xD5, 0xA5, 0xD6, + 0x82, 0x44, 0xD5, 0xB4, 0xD5, 0xA5, 0x44, 0xD5, + 0xB4, 0xD5, 0xAB, 0x44, 0xD5, 0xB4, 0xD5, 0xAD, + 0x44, 0xD5, 0xB4, 0xD5, 0xB6, 0x44, 0xD5, 0xBE, + // Bytes 1d80 - 1dbf + 0xD5, 0xB6, 0x44, 0xD7, 0x90, 0xD7, 0x9C, 0x44, + 0xD8, 0xA7, 0xD9, 0xB4, 0x44, 0xD8, 0xA8, 0xD8, + 0xAC, 0x44, 0xD8, 0xA8, 0xD8, 0xAD, 0x44, 0xD8, + 0xA8, 0xD8, 0xAE, 0x44, 0xD8, 0xA8, 0xD8, 0xB1, + 0x44, 0xD8, 0xA8, 0xD8, 0xB2, 0x44, 0xD8, 0xA8, + 0xD9, 0x85, 0x44, 0xD8, 0xA8, 0xD9, 0x86, 0x44, + 0xD8, 0xA8, 0xD9, 0x87, 0x44, 0xD8, 0xA8, 0xD9, + 0x89, 0x44, 0xD8, 0xA8, 0xD9, 0x8A, 0x44, 0xD8, + // Bytes 1dc0 - 1dff + 0xAA, 0xD8, 0xAC, 0x44, 0xD8, 0xAA, 0xD8, 0xAD, + 0x44, 0xD8, 0xAA, 0xD8, 0xAE, 0x44, 0xD8, 0xAA, + 0xD8, 0xB1, 0x44, 0xD8, 0xAA, 0xD8, 0xB2, 0x44, + 0xD8, 0xAA, 0xD9, 0x85, 0x44, 0xD8, 0xAA, 0xD9, + 0x86, 0x44, 0xD8, 0xAA, 0xD9, 0x87, 0x44, 0xD8, + 0xAA, 0xD9, 0x89, 0x44, 0xD8, 0xAA, 0xD9, 0x8A, + 0x44, 0xD8, 0xAB, 0xD8, 0xAC, 0x44, 0xD8, 0xAB, + 0xD8, 0xB1, 0x44, 0xD8, 0xAB, 0xD8, 0xB2, 0x44, + // Bytes 1e00 - 1e3f + 0xD8, 0xAB, 0xD9, 0x85, 0x44, 0xD8, 0xAB, 0xD9, + 0x86, 0x44, 0xD8, 0xAB, 0xD9, 0x87, 0x44, 0xD8, + 0xAB, 0xD9, 0x89, 0x44, 0xD8, 0xAB, 0xD9, 0x8A, + 0x44, 0xD8, 0xAC, 0xD8, 0xAD, 0x44, 0xD8, 0xAC, + 0xD9, 0x85, 0x44, 0xD8, 0xAC, 0xD9, 0x89, 0x44, + 0xD8, 0xAC, 0xD9, 0x8A, 0x44, 0xD8, 0xAD, 0xD8, + 0xAC, 0x44, 0xD8, 0xAD, 0xD9, 0x85, 0x44, 0xD8, + 0xAD, 0xD9, 0x89, 0x44, 0xD8, 0xAD, 0xD9, 0x8A, + // Bytes 1e40 - 1e7f + 0x44, 0xD8, 0xAE, 0xD8, 0xAC, 0x44, 0xD8, 0xAE, + 0xD8, 0xAD, 0x44, 0xD8, 0xAE, 0xD9, 0x85, 0x44, + 0xD8, 0xAE, 0xD9, 0x89, 0x44, 0xD8, 0xAE, 0xD9, + 0x8A, 0x44, 0xD8, 0xB3, 0xD8, 0xAC, 0x44, 0xD8, + 0xB3, 0xD8, 0xAD, 0x44, 0xD8, 0xB3, 0xD8, 0xAE, + 0x44, 0xD8, 0xB3, 0xD8, 0xB1, 0x44, 0xD8, 0xB3, + 0xD9, 0x85, 0x44, 0xD8, 0xB3, 0xD9, 0x87, 0x44, + 0xD8, 0xB3, 0xD9, 0x89, 0x44, 0xD8, 0xB3, 0xD9, + // Bytes 1e80 - 1ebf + 0x8A, 0x44, 0xD8, 0xB4, 0xD8, 0xAC, 0x44, 0xD8, + 0xB4, 0xD8, 0xAD, 0x44, 0xD8, 0xB4, 0xD8, 0xAE, + 0x44, 0xD8, 0xB4, 0xD8, 0xB1, 0x44, 0xD8, 0xB4, + 0xD9, 0x85, 0x44, 0xD8, 0xB4, 0xD9, 0x87, 0x44, + 0xD8, 0xB4, 0xD9, 0x89, 0x44, 0xD8, 0xB4, 0xD9, + 0x8A, 0x44, 0xD8, 0xB5, 0xD8, 0xAD, 0x44, 0xD8, + 0xB5, 0xD8, 0xAE, 0x44, 0xD8, 0xB5, 0xD8, 0xB1, + 0x44, 0xD8, 0xB5, 0xD9, 0x85, 0x44, 0xD8, 0xB5, + // Bytes 1ec0 - 1eff + 0xD9, 0x89, 0x44, 0xD8, 0xB5, 0xD9, 0x8A, 0x44, + 0xD8, 0xB6, 0xD8, 0xAC, 0x44, 0xD8, 0xB6, 0xD8, + 0xAD, 0x44, 0xD8, 0xB6, 0xD8, 0xAE, 0x44, 0xD8, + 0xB6, 0xD8, 0xB1, 0x44, 0xD8, 0xB6, 0xD9, 0x85, + 0x44, 0xD8, 0xB6, 0xD9, 0x89, 0x44, 0xD8, 0xB6, + 0xD9, 0x8A, 0x44, 0xD8, 0xB7, 0xD8, 0xAD, 0x44, + 0xD8, 0xB7, 0xD9, 0x85, 0x44, 0xD8, 0xB7, 0xD9, + 0x89, 0x44, 0xD8, 0xB7, 0xD9, 0x8A, 0x44, 0xD8, + // Bytes 1f00 - 1f3f + 0xB8, 0xD9, 0x85, 0x44, 0xD8, 0xB9, 0xD8, 0xAC, + 0x44, 0xD8, 0xB9, 0xD9, 0x85, 0x44, 0xD8, 0xB9, + 0xD9, 0x89, 0x44, 0xD8, 0xB9, 0xD9, 0x8A, 0x44, + 0xD8, 0xBA, 0xD8, 0xAC, 0x44, 0xD8, 0xBA, 0xD9, + 0x85, 0x44, 0xD8, 0xBA, 0xD9, 0x89, 0x44, 0xD8, + 0xBA, 0xD9, 0x8A, 0x44, 0xD9, 0x81, 0xD8, 0xAC, + 0x44, 0xD9, 0x81, 0xD8, 0xAD, 0x44, 0xD9, 0x81, + 0xD8, 0xAE, 0x44, 0xD9, 0x81, 0xD9, 0x85, 0x44, + // Bytes 1f40 - 1f7f + 0xD9, 0x81, 0xD9, 0x89, 0x44, 0xD9, 0x81, 0xD9, + 0x8A, 0x44, 0xD9, 0x82, 0xD8, 0xAD, 0x44, 0xD9, + 0x82, 0xD9, 0x85, 0x44, 0xD9, 0x82, 0xD9, 0x89, + 0x44, 0xD9, 0x82, 0xD9, 0x8A, 0x44, 0xD9, 0x83, + 0xD8, 0xA7, 0x44, 0xD9, 0x83, 0xD8, 0xAC, 0x44, + 0xD9, 0x83, 0xD8, 0xAD, 0x44, 0xD9, 0x83, 0xD8, + 0xAE, 0x44, 0xD9, 0x83, 0xD9, 0x84, 0x44, 0xD9, + 0x83, 0xD9, 0x85, 0x44, 0xD9, 0x83, 0xD9, 0x89, + // Bytes 1f80 - 1fbf + 0x44, 0xD9, 0x83, 0xD9, 0x8A, 0x44, 0xD9, 0x84, + 0xD8, 0xA7, 0x44, 0xD9, 0x84, 0xD8, 0xAC, 0x44, + 0xD9, 0x84, 0xD8, 0xAD, 0x44, 0xD9, 0x84, 0xD8, + 0xAE, 0x44, 0xD9, 0x84, 0xD9, 0x85, 0x44, 0xD9, + 0x84, 0xD9, 0x87, 0x44, 0xD9, 0x84, 0xD9, 0x89, + 0x44, 0xD9, 0x84, 0xD9, 0x8A, 0x44, 0xD9, 0x85, + 0xD8, 0xA7, 0x44, 0xD9, 0x85, 0xD8, 0xAC, 0x44, + 0xD9, 0x85, 0xD8, 0xAD, 0x44, 0xD9, 0x85, 0xD8, + // Bytes 1fc0 - 1fff + 0xAE, 0x44, 0xD9, 0x85, 0xD9, 0x85, 0x44, 0xD9, + 0x85, 0xD9, 0x89, 0x44, 0xD9, 0x85, 0xD9, 0x8A, + 0x44, 0xD9, 0x86, 0xD8, 0xAC, 0x44, 0xD9, 0x86, + 0xD8, 0xAD, 0x44, 0xD9, 0x86, 0xD8, 0xAE, 0x44, + 0xD9, 0x86, 0xD8, 0xB1, 0x44, 0xD9, 0x86, 0xD8, + 0xB2, 0x44, 0xD9, 0x86, 0xD9, 0x85, 0x44, 0xD9, + 0x86, 0xD9, 0x86, 0x44, 0xD9, 0x86, 0xD9, 0x87, + 0x44, 0xD9, 0x86, 0xD9, 0x89, 0x44, 0xD9, 0x86, + // Bytes 2000 - 203f + 0xD9, 0x8A, 0x44, 0xD9, 0x87, 0xD8, 0xAC, 0x44, + 0xD9, 0x87, 0xD9, 0x85, 0x44, 0xD9, 0x87, 0xD9, + 0x89, 0x44, 0xD9, 0x87, 0xD9, 0x8A, 0x44, 0xD9, + 0x88, 0xD9, 0xB4, 0x44, 0xD9, 0x8A, 0xD8, 0xAC, + 0x44, 0xD9, 0x8A, 0xD8, 0xAD, 0x44, 0xD9, 0x8A, + 0xD8, 0xAE, 0x44, 0xD9, 0x8A, 0xD8, 0xB1, 0x44, + 0xD9, 0x8A, 0xD8, 0xB2, 0x44, 0xD9, 0x8A, 0xD9, + 0x85, 0x44, 0xD9, 0x8A, 0xD9, 0x86, 0x44, 0xD9, + // Bytes 2040 - 207f + 0x8A, 0xD9, 0x87, 0x44, 0xD9, 0x8A, 0xD9, 0x89, + 0x44, 0xD9, 0x8A, 0xD9, 0x8A, 0x44, 0xD9, 0x8A, + 0xD9, 0xB4, 0x44, 0xDB, 0x87, 0xD9, 0xB4, 0x45, + 0x28, 0xE1, 0x84, 0x80, 0x29, 0x45, 0x28, 0xE1, + 0x84, 0x82, 0x29, 0x45, 0x28, 0xE1, 0x84, 0x83, + 0x29, 0x45, 0x28, 0xE1, 0x84, 0x85, 0x29, 0x45, + 0x28, 0xE1, 0x84, 0x86, 0x29, 0x45, 0x28, 0xE1, + 0x84, 0x87, 0x29, 0x45, 0x28, 0xE1, 0x84, 0x89, + // Bytes 2080 - 20bf + 0x29, 0x45, 0x28, 0xE1, 0x84, 0x8B, 0x29, 0x45, + 0x28, 0xE1, 0x84, 0x8C, 0x29, 0x45, 0x28, 0xE1, + 0x84, 0x8E, 0x29, 0x45, 0x28, 0xE1, 0x84, 0x8F, + 0x29, 0x45, 0x28, 0xE1, 0x84, 0x90, 0x29, 0x45, + 0x28, 0xE1, 0x84, 0x91, 0x29, 0x45, 0x28, 0xE1, + 0x84, 0x92, 0x29, 0x45, 0x28, 0xE4, 0xB8, 0x80, + 0x29, 0x45, 0x28, 0xE4, 0xB8, 0x83, 0x29, 0x45, + 0x28, 0xE4, 0xB8, 0x89, 0x29, 0x45, 0x28, 0xE4, + // Bytes 20c0 - 20ff + 0xB9, 0x9D, 0x29, 0x45, 0x28, 0xE4, 0xBA, 0x8C, + 0x29, 0x45, 0x28, 0xE4, 0xBA, 0x94, 0x29, 0x45, + 0x28, 0xE4, 0xBB, 0xA3, 0x29, 0x45, 0x28, 0xE4, + 0xBC, 0x81, 0x29, 0x45, 0x28, 0xE4, 0xBC, 0x91, + 0x29, 0x45, 0x28, 0xE5, 0x85, 0xAB, 0x29, 0x45, + 0x28, 0xE5, 0x85, 0xAD, 0x29, 0x45, 0x28, 0xE5, + 0x8A, 0xB4, 0x29, 0x45, 0x28, 0xE5, 0x8D, 0x81, + 0x29, 0x45, 0x28, 0xE5, 0x8D, 0x94, 0x29, 0x45, + // Bytes 2100 - 213f + 0x28, 0xE5, 0x90, 0x8D, 0x29, 0x45, 0x28, 0xE5, + 0x91, 0xBC, 0x29, 0x45, 0x28, 0xE5, 0x9B, 0x9B, + 0x29, 0x45, 0x28, 0xE5, 0x9C, 0x9F, 0x29, 0x45, + 0x28, 0xE5, 0xAD, 0xA6, 0x29, 0x45, 0x28, 0xE6, + 0x97, 0xA5, 0x29, 0x45, 0x28, 0xE6, 0x9C, 0x88, + 0x29, 0x45, 0x28, 0xE6, 0x9C, 0x89, 0x29, 0x45, + 0x28, 0xE6, 0x9C, 0xA8, 0x29, 0x45, 0x28, 0xE6, + 0xA0, 0xAA, 0x29, 0x45, 0x28, 0xE6, 0xB0, 0xB4, + // Bytes 2140 - 217f + 0x29, 0x45, 0x28, 0xE7, 0x81, 0xAB, 0x29, 0x45, + 0x28, 0xE7, 0x89, 0xB9, 0x29, 0x45, 0x28, 0xE7, + 0x9B, 0xA3, 0x29, 0x45, 0x28, 0xE7, 0xA4, 0xBE, + 0x29, 0x45, 0x28, 0xE7, 0xA5, 0x9D, 0x29, 0x45, + 0x28, 0xE7, 0xA5, 0xAD, 0x29, 0x45, 0x28, 0xE8, + 0x87, 0xAA, 0x29, 0x45, 0x28, 0xE8, 0x87, 0xB3, + 0x29, 0x45, 0x28, 0xE8, 0xB2, 0xA1, 0x29, 0x45, + 0x28, 0xE8, 0xB3, 0x87, 0x29, 0x45, 0x28, 0xE9, + // Bytes 2180 - 21bf + 0x87, 0x91, 0x29, 0x45, 0x30, 0xE2, 0x81, 0x84, + 0x33, 0x45, 0x31, 0x30, 0xE6, 0x97, 0xA5, 0x45, + 0x31, 0x30, 0xE6, 0x9C, 0x88, 0x45, 0x31, 0x30, + 0xE7, 0x82, 0xB9, 0x45, 0x31, 0x31, 0xE6, 0x97, + 0xA5, 0x45, 0x31, 0x31, 0xE6, 0x9C, 0x88, 0x45, + 0x31, 0x31, 0xE7, 0x82, 0xB9, 0x45, 0x31, 0x32, + 0xE6, 0x97, 0xA5, 0x45, 0x31, 0x32, 0xE6, 0x9C, + 0x88, 0x45, 0x31, 0x32, 0xE7, 0x82, 0xB9, 0x45, + // Bytes 21c0 - 21ff + 0x31, 0x33, 0xE6, 0x97, 0xA5, 0x45, 0x31, 0x33, + 0xE7, 0x82, 0xB9, 0x45, 0x31, 0x34, 0xE6, 0x97, + 0xA5, 0x45, 0x31, 0x34, 0xE7, 0x82, 0xB9, 0x45, + 0x31, 0x35, 0xE6, 0x97, 0xA5, 0x45, 0x31, 0x35, + 0xE7, 0x82, 0xB9, 0x45, 0x31, 0x36, 0xE6, 0x97, + 0xA5, 0x45, 0x31, 0x36, 0xE7, 0x82, 0xB9, 0x45, + 0x31, 0x37, 0xE6, 0x97, 0xA5, 0x45, 0x31, 0x37, + 0xE7, 0x82, 0xB9, 0x45, 0x31, 0x38, 0xE6, 0x97, + // Bytes 2200 - 223f + 0xA5, 0x45, 0x31, 0x38, 0xE7, 0x82, 0xB9, 0x45, + 0x31, 0x39, 0xE6, 0x97, 0xA5, 0x45, 0x31, 0x39, + 0xE7, 0x82, 0xB9, 0x45, 0x31, 0xE2, 0x81, 0x84, + 0x32, 0x45, 0x31, 0xE2, 0x81, 0x84, 0x33, 0x45, + 0x31, 0xE2, 0x81, 0x84, 0x34, 0x45, 0x31, 0xE2, + 0x81, 0x84, 0x35, 0x45, 0x31, 0xE2, 0x81, 0x84, + 0x36, 0x45, 0x31, 0xE2, 0x81, 0x84, 0x37, 0x45, + 0x31, 0xE2, 0x81, 0x84, 0x38, 0x45, 0x31, 0xE2, + // Bytes 2240 - 227f + 0x81, 0x84, 0x39, 0x45, 0x32, 0x30, 0xE6, 0x97, + 0xA5, 0x45, 0x32, 0x30, 0xE7, 0x82, 0xB9, 0x45, + 0x32, 0x31, 0xE6, 0x97, 0xA5, 0x45, 0x32, 0x31, + 0xE7, 0x82, 0xB9, 0x45, 0x32, 0x32, 0xE6, 0x97, + 0xA5, 0x45, 0x32, 0x32, 0xE7, 0x82, 0xB9, 0x45, + 0x32, 0x33, 0xE6, 0x97, 0xA5, 0x45, 0x32, 0x33, + 0xE7, 0x82, 0xB9, 0x45, 0x32, 0x34, 0xE6, 0x97, + 0xA5, 0x45, 0x32, 0x34, 0xE7, 0x82, 0xB9, 0x45, + // Bytes 2280 - 22bf + 0x32, 0x35, 0xE6, 0x97, 0xA5, 0x45, 0x32, 0x36, + 0xE6, 0x97, 0xA5, 0x45, 0x32, 0x37, 0xE6, 0x97, + 0xA5, 0x45, 0x32, 0x38, 0xE6, 0x97, 0xA5, 0x45, + 0x32, 0x39, 0xE6, 0x97, 0xA5, 0x45, 0x32, 0xE2, + 0x81, 0x84, 0x33, 0x45, 0x32, 0xE2, 0x81, 0x84, + 0x35, 0x45, 0x33, 0x30, 0xE6, 0x97, 0xA5, 0x45, + 0x33, 0x31, 0xE6, 0x97, 0xA5, 0x45, 0x33, 0xE2, + 0x81, 0x84, 0x34, 0x45, 0x33, 0xE2, 0x81, 0x84, + // Bytes 22c0 - 22ff + 0x35, 0x45, 0x33, 0xE2, 0x81, 0x84, 0x38, 0x45, + 0x34, 0xE2, 0x81, 0x84, 0x35, 0x45, 0x35, 0xE2, + 0x81, 0x84, 0x36, 0x45, 0x35, 0xE2, 0x81, 0x84, + 0x38, 0x45, 0x37, 0xE2, 0x81, 0x84, 0x38, 0x45, + 0x41, 0xE2, 0x88, 0x95, 0x6D, 0x45, 0x56, 0xE2, + 0x88, 0x95, 0x6D, 0x45, 0x6D, 0xE2, 0x88, 0x95, + 0x73, 0x46, 0x31, 0xE2, 0x81, 0x84, 0x31, 0x30, + 0x46, 0x43, 0xE2, 0x88, 0x95, 0x6B, 0x67, 0x46, + // Bytes 2300 - 233f + 0x6D, 0xE2, 0x88, 0x95, 0x73, 0x32, 0x46, 0xD8, + 0xA8, 0xD8, 0xAD, 0xD9, 0x8A, 0x46, 0xD8, 0xA8, + 0xD8, 0xAE, 0xD9, 0x8A, 0x46, 0xD8, 0xAA, 0xD8, + 0xAC, 0xD9, 0x85, 0x46, 0xD8, 0xAA, 0xD8, 0xAC, + 0xD9, 0x89, 0x46, 0xD8, 0xAA, 0xD8, 0xAC, 0xD9, + 0x8A, 0x46, 0xD8, 0xAA, 0xD8, 0xAD, 0xD8, 0xAC, + 0x46, 0xD8, 0xAA, 0xD8, 0xAD, 0xD9, 0x85, 0x46, + 0xD8, 0xAA, 0xD8, 0xAE, 0xD9, 0x85, 0x46, 0xD8, + // Bytes 2340 - 237f + 0xAA, 0xD8, 0xAE, 0xD9, 0x89, 0x46, 0xD8, 0xAA, + 0xD8, 0xAE, 0xD9, 0x8A, 0x46, 0xD8, 0xAA, 0xD9, + 0x85, 0xD8, 0xAC, 0x46, 0xD8, 0xAA, 0xD9, 0x85, + 0xD8, 0xAD, 0x46, 0xD8, 0xAA, 0xD9, 0x85, 0xD8, + 0xAE, 0x46, 0xD8, 0xAA, 0xD9, 0x85, 0xD9, 0x89, + 0x46, 0xD8, 0xAA, 0xD9, 0x85, 0xD9, 0x8A, 0x46, + 0xD8, 0xAC, 0xD8, 0xAD, 0xD9, 0x89, 0x46, 0xD8, + 0xAC, 0xD8, 0xAD, 0xD9, 0x8A, 0x46, 0xD8, 0xAC, + // Bytes 2380 - 23bf + 0xD9, 0x85, 0xD8, 0xAD, 0x46, 0xD8, 0xAC, 0xD9, + 0x85, 0xD9, 0x89, 0x46, 0xD8, 0xAC, 0xD9, 0x85, + 0xD9, 0x8A, 0x46, 0xD8, 0xAD, 0xD8, 0xAC, 0xD9, + 0x8A, 0x46, 0xD8, 0xAD, 0xD9, 0x85, 0xD9, 0x89, + 0x46, 0xD8, 0xAD, 0xD9, 0x85, 0xD9, 0x8A, 0x46, + 0xD8, 0xB3, 0xD8, 0xAC, 0xD8, 0xAD, 0x46, 0xD8, + 0xB3, 0xD8, 0xAC, 0xD9, 0x89, 0x46, 0xD8, 0xB3, + 0xD8, 0xAD, 0xD8, 0xAC, 0x46, 0xD8, 0xB3, 0xD8, + // Bytes 23c0 - 23ff + 0xAE, 0xD9, 0x89, 0x46, 0xD8, 0xB3, 0xD8, 0xAE, + 0xD9, 0x8A, 0x46, 0xD8, 0xB3, 0xD9, 0x85, 0xD8, + 0xAC, 0x46, 0xD8, 0xB3, 0xD9, 0x85, 0xD8, 0xAD, + 0x46, 0xD8, 0xB3, 0xD9, 0x85, 0xD9, 0x85, 0x46, + 0xD8, 0xB4, 0xD8, 0xAC, 0xD9, 0x8A, 0x46, 0xD8, + 0xB4, 0xD8, 0xAD, 0xD9, 0x85, 0x46, 0xD8, 0xB4, + 0xD8, 0xAD, 0xD9, 0x8A, 0x46, 0xD8, 0xB4, 0xD9, + 0x85, 0xD8, 0xAE, 0x46, 0xD8, 0xB4, 0xD9, 0x85, + // Bytes 2400 - 243f + 0xD9, 0x85, 0x46, 0xD8, 0xB5, 0xD8, 0xAD, 0xD8, + 0xAD, 0x46, 0xD8, 0xB5, 0xD8, 0xAD, 0xD9, 0x8A, + 0x46, 0xD8, 0xB5, 0xD9, 0x84, 0xD9, 0x89, 0x46, + 0xD8, 0xB5, 0xD9, 0x84, 0xDB, 0x92, 0x46, 0xD8, + 0xB5, 0xD9, 0x85, 0xD9, 0x85, 0x46, 0xD8, 0xB6, + 0xD8, 0xAD, 0xD9, 0x89, 0x46, 0xD8, 0xB6, 0xD8, + 0xAD, 0xD9, 0x8A, 0x46, 0xD8, 0xB6, 0xD8, 0xAE, + 0xD9, 0x85, 0x46, 0xD8, 0xB7, 0xD9, 0x85, 0xD8, + // Bytes 2440 - 247f + 0xAD, 0x46, 0xD8, 0xB7, 0xD9, 0x85, 0xD9, 0x85, + 0x46, 0xD8, 0xB7, 0xD9, 0x85, 0xD9, 0x8A, 0x46, + 0xD8, 0xB9, 0xD8, 0xAC, 0xD9, 0x85, 0x46, 0xD8, + 0xB9, 0xD9, 0x85, 0xD9, 0x85, 0x46, 0xD8, 0xB9, + 0xD9, 0x85, 0xD9, 0x89, 0x46, 0xD8, 0xB9, 0xD9, + 0x85, 0xD9, 0x8A, 0x46, 0xD8, 0xBA, 0xD9, 0x85, + 0xD9, 0x85, 0x46, 0xD8, 0xBA, 0xD9, 0x85, 0xD9, + 0x89, 0x46, 0xD8, 0xBA, 0xD9, 0x85, 0xD9, 0x8A, + // Bytes 2480 - 24bf + 0x46, 0xD9, 0x81, 0xD8, 0xAE, 0xD9, 0x85, 0x46, + 0xD9, 0x81, 0xD9, 0x85, 0xD9, 0x8A, 0x46, 0xD9, + 0x82, 0xD9, 0x84, 0xDB, 0x92, 0x46, 0xD9, 0x82, + 0xD9, 0x85, 0xD8, 0xAD, 0x46, 0xD9, 0x82, 0xD9, + 0x85, 0xD9, 0x85, 0x46, 0xD9, 0x82, 0xD9, 0x85, + 0xD9, 0x8A, 0x46, 0xD9, 0x83, 0xD9, 0x85, 0xD9, + 0x85, 0x46, 0xD9, 0x83, 0xD9, 0x85, 0xD9, 0x8A, + 0x46, 0xD9, 0x84, 0xD8, 0xAC, 0xD8, 0xAC, 0x46, + // Bytes 24c0 - 24ff + 0xD9, 0x84, 0xD8, 0xAC, 0xD9, 0x85, 0x46, 0xD9, + 0x84, 0xD8, 0xAC, 0xD9, 0x8A, 0x46, 0xD9, 0x84, + 0xD8, 0xAD, 0xD9, 0x85, 0x46, 0xD9, 0x84, 0xD8, + 0xAD, 0xD9, 0x89, 0x46, 0xD9, 0x84, 0xD8, 0xAD, + 0xD9, 0x8A, 0x46, 0xD9, 0x84, 0xD8, 0xAE, 0xD9, + 0x85, 0x46, 0xD9, 0x84, 0xD9, 0x85, 0xD8, 0xAD, + 0x46, 0xD9, 0x84, 0xD9, 0x85, 0xD9, 0x8A, 0x46, + 0xD9, 0x85, 0xD8, 0xAC, 0xD8, 0xAD, 0x46, 0xD9, + // Bytes 2500 - 253f + 0x85, 0xD8, 0xAC, 0xD8, 0xAE, 0x46, 0xD9, 0x85, + 0xD8, 0xAC, 0xD9, 0x85, 0x46, 0xD9, 0x85, 0xD8, + 0xAC, 0xD9, 0x8A, 0x46, 0xD9, 0x85, 0xD8, 0xAD, + 0xD8, 0xAC, 0x46, 0xD9, 0x85, 0xD8, 0xAD, 0xD9, + 0x85, 0x46, 0xD9, 0x85, 0xD8, 0xAD, 0xD9, 0x8A, + 0x46, 0xD9, 0x85, 0xD8, 0xAE, 0xD8, 0xAC, 0x46, + 0xD9, 0x85, 0xD8, 0xAE, 0xD9, 0x85, 0x46, 0xD9, + 0x85, 0xD8, 0xAE, 0xD9, 0x8A, 0x46, 0xD9, 0x85, + // Bytes 2540 - 257f + 0xD9, 0x85, 0xD9, 0x8A, 0x46, 0xD9, 0x86, 0xD8, + 0xAC, 0xD8, 0xAD, 0x46, 0xD9, 0x86, 0xD8, 0xAC, + 0xD9, 0x85, 0x46, 0xD9, 0x86, 0xD8, 0xAC, 0xD9, + 0x89, 0x46, 0xD9, 0x86, 0xD8, 0xAC, 0xD9, 0x8A, + 0x46, 0xD9, 0x86, 0xD8, 0xAD, 0xD9, 0x85, 0x46, + 0xD9, 0x86, 0xD8, 0xAD, 0xD9, 0x89, 0x46, 0xD9, + 0x86, 0xD8, 0xAD, 0xD9, 0x8A, 0x46, 0xD9, 0x86, + 0xD9, 0x85, 0xD9, 0x89, 0x46, 0xD9, 0x86, 0xD9, + // Bytes 2580 - 25bf + 0x85, 0xD9, 0x8A, 0x46, 0xD9, 0x87, 0xD9, 0x85, + 0xD8, 0xAC, 0x46, 0xD9, 0x87, 0xD9, 0x85, 0xD9, + 0x85, 0x46, 0xD9, 0x8A, 0xD8, 0xAC, 0xD9, 0x8A, + 0x46, 0xD9, 0x8A, 0xD8, 0xAD, 0xD9, 0x8A, 0x46, + 0xD9, 0x8A, 0xD9, 0x85, 0xD9, 0x85, 0x46, 0xD9, + 0x8A, 0xD9, 0x85, 0xD9, 0x8A, 0x46, 0xD9, 0x8A, + 0xD9, 0x94, 0xD8, 0xA7, 0x46, 0xD9, 0x8A, 0xD9, + 0x94, 0xD8, 0xAC, 0x46, 0xD9, 0x8A, 0xD9, 0x94, + // Bytes 25c0 - 25ff + 0xD8, 0xAD, 0x46, 0xD9, 0x8A, 0xD9, 0x94, 0xD8, + 0xAE, 0x46, 0xD9, 0x8A, 0xD9, 0x94, 0xD8, 0xB1, + 0x46, 0xD9, 0x8A, 0xD9, 0x94, 0xD8, 0xB2, 0x46, + 0xD9, 0x8A, 0xD9, 0x94, 0xD9, 0x85, 0x46, 0xD9, + 0x8A, 0xD9, 0x94, 0xD9, 0x86, 0x46, 0xD9, 0x8A, + 0xD9, 0x94, 0xD9, 0x87, 0x46, 0xD9, 0x8A, 0xD9, + 0x94, 0xD9, 0x88, 0x46, 0xD9, 0x8A, 0xD9, 0x94, + 0xD9, 0x89, 0x46, 0xD9, 0x8A, 0xD9, 0x94, 0xD9, + // Bytes 2600 - 263f + 0x8A, 0x46, 0xD9, 0x8A, 0xD9, 0x94, 0xDB, 0x86, + 0x46, 0xD9, 0x8A, 0xD9, 0x94, 0xDB, 0x87, 0x46, + 0xD9, 0x8A, 0xD9, 0x94, 0xDB, 0x88, 0x46, 0xD9, + 0x8A, 0xD9, 0x94, 0xDB, 0x90, 0x46, 0xD9, 0x8A, + 0xD9, 0x94, 0xDB, 0x95, 0x46, 0xE0, 0xB9, 0x8D, + 0xE0, 0xB8, 0xB2, 0x46, 0xE0, 0xBA, 0xAB, 0xE0, + 0xBA, 0x99, 0x46, 0xE0, 0xBA, 0xAB, 0xE0, 0xBA, + 0xA1, 0x46, 0xE0, 0xBB, 0x8D, 0xE0, 0xBA, 0xB2, + // Bytes 2640 - 267f + 0x46, 0xE0, 0xBD, 0x80, 0xE0, 0xBE, 0xB5, 0x46, + 0xE0, 0xBD, 0x82, 0xE0, 0xBE, 0xB7, 0x46, 0xE0, + 0xBD, 0x8C, 0xE0, 0xBE, 0xB7, 0x46, 0xE0, 0xBD, + 0x91, 0xE0, 0xBE, 0xB7, 0x46, 0xE0, 0xBD, 0x96, + 0xE0, 0xBE, 0xB7, 0x46, 0xE0, 0xBD, 0x9B, 0xE0, + 0xBE, 0xB7, 0x46, 0xE0, 0xBE, 0x90, 0xE0, 0xBE, + 0xB5, 0x46, 0xE0, 0xBE, 0x92, 0xE0, 0xBE, 0xB7, + 0x46, 0xE0, 0xBE, 0x9C, 0xE0, 0xBE, 0xB7, 0x46, + // Bytes 2680 - 26bf + 0xE0, 0xBE, 0xA1, 0xE0, 0xBE, 0xB7, 0x46, 0xE0, + 0xBE, 0xA6, 0xE0, 0xBE, 0xB7, 0x46, 0xE0, 0xBE, + 0xAB, 0xE0, 0xBE, 0xB7, 0x46, 0xE2, 0x80, 0xB2, + 0xE2, 0x80, 0xB2, 0x46, 0xE2, 0x80, 0xB5, 0xE2, + 0x80, 0xB5, 0x46, 0xE2, 0x88, 0xAB, 0xE2, 0x88, + 0xAB, 0x46, 0xE2, 0x88, 0xAE, 0xE2, 0x88, 0xAE, + 0x46, 0xE3, 0x81, 0xBB, 0xE3, 0x81, 0x8B, 0x46, + 0xE3, 0x82, 0x88, 0xE3, 0x82, 0x8A, 0x46, 0xE3, + // Bytes 26c0 - 26ff + 0x82, 0xAD, 0xE3, 0x83, 0xAD, 0x46, 0xE3, 0x82, + 0xB3, 0xE3, 0x82, 0xB3, 0x46, 0xE3, 0x82, 0xB3, + 0xE3, 0x83, 0x88, 0x46, 0xE3, 0x83, 0x88, 0xE3, + 0x83, 0xB3, 0x46, 0xE3, 0x83, 0x8A, 0xE3, 0x83, + 0x8E, 0x46, 0xE3, 0x83, 0x9B, 0xE3, 0x83, 0xB3, + 0x46, 0xE3, 0x83, 0x9F, 0xE3, 0x83, 0xAA, 0x46, + 0xE3, 0x83, 0xAA, 0xE3, 0x83, 0xA9, 0x46, 0xE3, + 0x83, 0xAC, 0xE3, 0x83, 0xA0, 0x46, 0xE5, 0xA4, + // Bytes 2700 - 273f + 0xA7, 0xE6, 0xAD, 0xA3, 0x46, 0xE5, 0xB9, 0xB3, + 0xE6, 0x88, 0x90, 0x46, 0xE6, 0x98, 0x8E, 0xE6, + 0xB2, 0xBB, 0x46, 0xE6, 0x98, 0xAD, 0xE5, 0x92, + 0x8C, 0x47, 0x72, 0x61, 0x64, 0xE2, 0x88, 0x95, + 0x73, 0x47, 0xE3, 0x80, 0x94, 0x53, 0xE3, 0x80, + 0x95, 0x48, 0x28, 0xE1, 0x84, 0x80, 0xE1, 0x85, + 0xA1, 0x29, 0x48, 0x28, 0xE1, 0x84, 0x82, 0xE1, + 0x85, 0xA1, 0x29, 0x48, 0x28, 0xE1, 0x84, 0x83, + // Bytes 2740 - 277f + 0xE1, 0x85, 0xA1, 0x29, 0x48, 0x28, 0xE1, 0x84, + 0x85, 0xE1, 0x85, 0xA1, 0x29, 0x48, 0x28, 0xE1, + 0x84, 0x86, 0xE1, 0x85, 0xA1, 0x29, 0x48, 0x28, + 0xE1, 0x84, 0x87, 0xE1, 0x85, 0xA1, 0x29, 0x48, + 0x28, 0xE1, 0x84, 0x89, 0xE1, 0x85, 0xA1, 0x29, + 0x48, 0x28, 0xE1, 0x84, 0x8B, 0xE1, 0x85, 0xA1, + 0x29, 0x48, 0x28, 0xE1, 0x84, 0x8C, 0xE1, 0x85, + 0xA1, 0x29, 0x48, 0x28, 0xE1, 0x84, 0x8C, 0xE1, + // Bytes 2780 - 27bf + 0x85, 0xAE, 0x29, 0x48, 0x28, 0xE1, 0x84, 0x8E, + 0xE1, 0x85, 0xA1, 0x29, 0x48, 0x28, 0xE1, 0x84, + 0x8F, 0xE1, 0x85, 0xA1, 0x29, 0x48, 0x28, 0xE1, + 0x84, 0x90, 0xE1, 0x85, 0xA1, 0x29, 0x48, 0x28, + 0xE1, 0x84, 0x91, 0xE1, 0x85, 0xA1, 0x29, 0x48, + 0x28, 0xE1, 0x84, 0x92, 0xE1, 0x85, 0xA1, 0x29, + 0x48, 0x72, 0x61, 0x64, 0xE2, 0x88, 0x95, 0x73, + 0x32, 0x48, 0xD8, 0xA7, 0xD9, 0x83, 0xD8, 0xA8, + // Bytes 27c0 - 27ff + 0xD8, 0xB1, 0x48, 0xD8, 0xA7, 0xD9, 0x84, 0xD9, + 0x84, 0xD9, 0x87, 0x48, 0xD8, 0xB1, 0xD8, 0xB3, + 0xD9, 0x88, 0xD9, 0x84, 0x48, 0xD8, 0xB1, 0xDB, + 0x8C, 0xD8, 0xA7, 0xD9, 0x84, 0x48, 0xD8, 0xB5, + 0xD9, 0x84, 0xD8, 0xB9, 0xD9, 0x85, 0x48, 0xD8, + 0xB9, 0xD9, 0x84, 0xD9, 0x8A, 0xD9, 0x87, 0x48, + 0xD9, 0x85, 0xD8, 0xAD, 0xD9, 0x85, 0xD8, 0xAF, + 0x48, 0xD9, 0x88, 0xD8, 0xB3, 0xD9, 0x84, 0xD9, + // Bytes 2800 - 283f + 0x85, 0x49, 0xE2, 0x80, 0xB2, 0xE2, 0x80, 0xB2, + 0xE2, 0x80, 0xB2, 0x49, 0xE2, 0x80, 0xB5, 0xE2, + 0x80, 0xB5, 0xE2, 0x80, 0xB5, 0x49, 0xE2, 0x88, + 0xAB, 0xE2, 0x88, 0xAB, 0xE2, 0x88, 0xAB, 0x49, + 0xE2, 0x88, 0xAE, 0xE2, 0x88, 0xAE, 0xE2, 0x88, + 0xAE, 0x49, 0xE3, 0x80, 0x94, 0xE4, 0xB8, 0x89, + 0xE3, 0x80, 0x95, 0x49, 0xE3, 0x80, 0x94, 0xE4, + 0xBA, 0x8C, 0xE3, 0x80, 0x95, 0x49, 0xE3, 0x80, + // Bytes 2840 - 287f + 0x94, 0xE5, 0x8B, 0x9D, 0xE3, 0x80, 0x95, 0x49, + 0xE3, 0x80, 0x94, 0xE5, 0xAE, 0x89, 0xE3, 0x80, + 0x95, 0x49, 0xE3, 0x80, 0x94, 0xE6, 0x89, 0x93, + 0xE3, 0x80, 0x95, 0x49, 0xE3, 0x80, 0x94, 0xE6, + 0x95, 0x97, 0xE3, 0x80, 0x95, 0x49, 0xE3, 0x80, + 0x94, 0xE6, 0x9C, 0xAC, 0xE3, 0x80, 0x95, 0x49, + 0xE3, 0x80, 0x94, 0xE7, 0x82, 0xB9, 0xE3, 0x80, + 0x95, 0x49, 0xE3, 0x80, 0x94, 0xE7, 0x9B, 0x97, + // Bytes 2880 - 28bf + 0xE3, 0x80, 0x95, 0x49, 0xE3, 0x82, 0xA2, 0xE3, + 0x83, 0xBC, 0xE3, 0x83, 0xAB, 0x49, 0xE3, 0x82, + 0xA4, 0xE3, 0x83, 0xB3, 0xE3, 0x83, 0x81, 0x49, + 0xE3, 0x82, 0xA6, 0xE3, 0x82, 0xA9, 0xE3, 0x83, + 0xB3, 0x49, 0xE3, 0x82, 0xAA, 0xE3, 0x83, 0xB3, + 0xE3, 0x82, 0xB9, 0x49, 0xE3, 0x82, 0xAA, 0xE3, + 0x83, 0xBC, 0xE3, 0x83, 0xA0, 0x49, 0xE3, 0x82, + 0xAB, 0xE3, 0x82, 0xA4, 0xE3, 0x83, 0xAA, 0x49, + // Bytes 28c0 - 28ff + 0xE3, 0x82, 0xB1, 0xE3, 0x83, 0xBC, 0xE3, 0x82, + 0xB9, 0x49, 0xE3, 0x82, 0xB3, 0xE3, 0x83, 0xAB, + 0xE3, 0x83, 0x8A, 0x49, 0xE3, 0x82, 0xBB, 0xE3, + 0x83, 0xB3, 0xE3, 0x83, 0x81, 0x49, 0xE3, 0x82, + 0xBB, 0xE3, 0x83, 0xB3, 0xE3, 0x83, 0x88, 0x49, + 0xE3, 0x83, 0x86, 0xE3, 0x82, 0x99, 0xE3, 0x82, + 0xB7, 0x49, 0xE3, 0x83, 0x88, 0xE3, 0x82, 0x99, + 0xE3, 0x83, 0xAB, 0x49, 0xE3, 0x83, 0x8E, 0xE3, + // Bytes 2900 - 293f + 0x83, 0x83, 0xE3, 0x83, 0x88, 0x49, 0xE3, 0x83, + 0x8F, 0xE3, 0x82, 0xA4, 0xE3, 0x83, 0x84, 0x49, + 0xE3, 0x83, 0x92, 0xE3, 0x82, 0x99, 0xE3, 0x83, + 0xAB, 0x49, 0xE3, 0x83, 0x92, 0xE3, 0x82, 0x9A, + 0xE3, 0x82, 0xB3, 0x49, 0xE3, 0x83, 0x95, 0xE3, + 0x83, 0xA9, 0xE3, 0x83, 0xB3, 0x49, 0xE3, 0x83, + 0x98, 0xE3, 0x82, 0x9A, 0xE3, 0x82, 0xBD, 0x49, + 0xE3, 0x83, 0x98, 0xE3, 0x83, 0xAB, 0xE3, 0x83, + // Bytes 2940 - 297f + 0x84, 0x49, 0xE3, 0x83, 0x9B, 0xE3, 0x83, 0xBC, + 0xE3, 0x83, 0xAB, 0x49, 0xE3, 0x83, 0x9B, 0xE3, + 0x83, 0xBC, 0xE3, 0x83, 0xB3, 0x49, 0xE3, 0x83, + 0x9E, 0xE3, 0x82, 0xA4, 0xE3, 0x83, 0xAB, 0x49, + 0xE3, 0x83, 0x9E, 0xE3, 0x83, 0x83, 0xE3, 0x83, + 0x8F, 0x49, 0xE3, 0x83, 0x9E, 0xE3, 0x83, 0xAB, + 0xE3, 0x82, 0xAF, 0x49, 0xE3, 0x83, 0xA4, 0xE3, + 0x83, 0xBC, 0xE3, 0x83, 0xAB, 0x49, 0xE3, 0x83, + // Bytes 2980 - 29bf + 0xA6, 0xE3, 0x82, 0xA2, 0xE3, 0x83, 0xB3, 0x49, + 0xE3, 0x83, 0xAF, 0xE3, 0x83, 0x83, 0xE3, 0x83, + 0x88, 0x4C, 0xE2, 0x80, 0xB2, 0xE2, 0x80, 0xB2, + 0xE2, 0x80, 0xB2, 0xE2, 0x80, 0xB2, 0x4C, 0xE2, + 0x88, 0xAB, 0xE2, 0x88, 0xAB, 0xE2, 0x88, 0xAB, + 0xE2, 0x88, 0xAB, 0x4C, 0xE3, 0x82, 0xA2, 0xE3, + 0x83, 0xAB, 0xE3, 0x83, 0x95, 0xE3, 0x82, 0xA1, + 0x4C, 0xE3, 0x82, 0xA8, 0xE3, 0x83, 0xBC, 0xE3, + // Bytes 29c0 - 29ff + 0x82, 0xAB, 0xE3, 0x83, 0xBC, 0x4C, 0xE3, 0x82, + 0xAB, 0xE3, 0x82, 0x99, 0xE3, 0x83, 0xAD, 0xE3, + 0x83, 0xB3, 0x4C, 0xE3, 0x82, 0xAB, 0xE3, 0x82, + 0x99, 0xE3, 0x83, 0xB3, 0xE3, 0x83, 0x9E, 0x4C, + 0xE3, 0x82, 0xAB, 0xE3, 0x83, 0xA9, 0xE3, 0x83, + 0x83, 0xE3, 0x83, 0x88, 0x4C, 0xE3, 0x82, 0xAB, + 0xE3, 0x83, 0xAD, 0xE3, 0x83, 0xAA, 0xE3, 0x83, + 0xBC, 0x4C, 0xE3, 0x82, 0xAD, 0xE3, 0x82, 0x99, + // Bytes 2a00 - 2a3f + 0xE3, 0x83, 0x8B, 0xE3, 0x83, 0xBC, 0x4C, 0xE3, + 0x82, 0xAD, 0xE3, 0x83, 0xA5, 0xE3, 0x83, 0xAA, + 0xE3, 0x83, 0xBC, 0x4C, 0xE3, 0x82, 0xAF, 0xE3, + 0x82, 0x99, 0xE3, 0x83, 0xA9, 0xE3, 0x83, 0xA0, + 0x4C, 0xE3, 0x82, 0xAF, 0xE3, 0x83, 0xAD, 0xE3, + 0x83, 0xBC, 0xE3, 0x83, 0x8D, 0x4C, 0xE3, 0x82, + 0xB5, 0xE3, 0x82, 0xA4, 0xE3, 0x82, 0xAF, 0xE3, + 0x83, 0xAB, 0x4C, 0xE3, 0x82, 0xBF, 0xE3, 0x82, + // Bytes 2a40 - 2a7f + 0x99, 0xE3, 0x83, 0xBC, 0xE3, 0x82, 0xB9, 0x4C, + 0xE3, 0x83, 0x8F, 0xE3, 0x82, 0x9A, 0xE3, 0x83, + 0xBC, 0xE3, 0x83, 0x84, 0x4C, 0xE3, 0x83, 0x92, + 0xE3, 0x82, 0x9A, 0xE3, 0x82, 0xAF, 0xE3, 0x83, + 0xAB, 0x4C, 0xE3, 0x83, 0x95, 0xE3, 0x82, 0xA3, + 0xE3, 0x83, 0xBC, 0xE3, 0x83, 0x88, 0x4C, 0xE3, + 0x83, 0x98, 0xE3, 0x82, 0x99, 0xE3, 0x83, 0xBC, + 0xE3, 0x82, 0xBF, 0x4C, 0xE3, 0x83, 0x98, 0xE3, + // Bytes 2a80 - 2abf + 0x82, 0x9A, 0xE3, 0x83, 0x8B, 0xE3, 0x83, 0x92, + 0x4C, 0xE3, 0x83, 0x98, 0xE3, 0x82, 0x9A, 0xE3, + 0x83, 0xB3, 0xE3, 0x82, 0xB9, 0x4C, 0xE3, 0x83, + 0x9B, 0xE3, 0x82, 0x99, 0xE3, 0x83, 0xAB, 0xE3, + 0x83, 0x88, 0x4C, 0xE3, 0x83, 0x9E, 0xE3, 0x82, + 0xA4, 0xE3, 0x82, 0xAF, 0xE3, 0x83, 0xAD, 0x4C, + 0xE3, 0x83, 0x9F, 0xE3, 0x82, 0xAF, 0xE3, 0x83, + 0xAD, 0xE3, 0x83, 0xB3, 0x4C, 0xE3, 0x83, 0xA1, + // Bytes 2ac0 - 2aff + 0xE3, 0x83, 0xBC, 0xE3, 0x83, 0x88, 0xE3, 0x83, + 0xAB, 0x4C, 0xE3, 0x83, 0xAA, 0xE3, 0x83, 0x83, + 0xE3, 0x83, 0x88, 0xE3, 0x83, 0xAB, 0x4C, 0xE3, + 0x83, 0xAB, 0xE3, 0x83, 0x92, 0xE3, 0x82, 0x9A, + 0xE3, 0x83, 0xBC, 0x4C, 0xE6, 0xA0, 0xAA, 0xE5, + 0xBC, 0x8F, 0xE4, 0xBC, 0x9A, 0xE7, 0xA4, 0xBE, + 0x4E, 0x28, 0xE1, 0x84, 0x8B, 0xE1, 0x85, 0xA9, + 0xE1, 0x84, 0x92, 0xE1, 0x85, 0xAE, 0x29, 0x4F, + // Bytes 2b00 - 2b3f + 0xD8, 0xAC, 0xD9, 0x84, 0x20, 0xD8, 0xAC, 0xD9, + 0x84, 0xD8, 0xA7, 0xD9, 0x84, 0xD9, 0x87, 0x4F, + 0xE3, 0x82, 0xA2, 0xE3, 0x83, 0x8F, 0xE3, 0x82, + 0x9A, 0xE3, 0x83, 0xBC, 0xE3, 0x83, 0x88, 0x4F, + 0xE3, 0x82, 0xA2, 0xE3, 0x83, 0xB3, 0xE3, 0x83, + 0x98, 0xE3, 0x82, 0x9A, 0xE3, 0x82, 0xA2, 0x4F, + 0xE3, 0x82, 0xAD, 0xE3, 0x83, 0xAD, 0xE3, 0x83, + 0xAF, 0xE3, 0x83, 0x83, 0xE3, 0x83, 0x88, 0x4F, + // Bytes 2b40 - 2b7f + 0xE3, 0x82, 0xB5, 0xE3, 0x83, 0xB3, 0xE3, 0x83, + 0x81, 0xE3, 0x83, 0xBC, 0xE3, 0x83, 0xA0, 0x4F, + 0xE3, 0x83, 0x8F, 0xE3, 0x82, 0x99, 0xE3, 0x83, + 0xBC, 0xE3, 0x83, 0xAC, 0xE3, 0x83, 0xAB, 0x4F, + 0xE3, 0x83, 0x98, 0xE3, 0x82, 0xAF, 0xE3, 0x82, + 0xBF, 0xE3, 0x83, 0xBC, 0xE3, 0x83, 0xAB, 0x4F, + 0xE3, 0x83, 0x9B, 0xE3, 0x82, 0x9A, 0xE3, 0x82, + 0xA4, 0xE3, 0x83, 0xB3, 0xE3, 0x83, 0x88, 0x4F, + // Bytes 2b80 - 2bbf + 0xE3, 0x83, 0x9E, 0xE3, 0x83, 0xB3, 0xE3, 0x82, + 0xB7, 0xE3, 0x83, 0xA7, 0xE3, 0x83, 0xB3, 0x4F, + 0xE3, 0x83, 0xA1, 0xE3, 0x82, 0xAB, 0xE3, 0x82, + 0x99, 0xE3, 0x83, 0x88, 0xE3, 0x83, 0xB3, 0x4F, + 0xE3, 0x83, 0xAB, 0xE3, 0x83, 0xBC, 0xE3, 0x83, + 0x95, 0xE3, 0x82, 0x99, 0xE3, 0x83, 0xAB, 0x51, + 0x28, 0xE1, 0x84, 0x8B, 0xE1, 0x85, 0xA9, 0xE1, + 0x84, 0x8C, 0xE1, 0x85, 0xA5, 0xE1, 0x86, 0xAB, + // Bytes 2bc0 - 2bff + 0x29, 0x52, 0xE3, 0x82, 0xAD, 0xE3, 0x82, 0x99, + 0xE3, 0x83, 0xAB, 0xE3, 0x82, 0xBF, 0xE3, 0x82, + 0x99, 0xE3, 0x83, 0xBC, 0x52, 0xE3, 0x82, 0xAD, + 0xE3, 0x83, 0xAD, 0xE3, 0x82, 0xAF, 0xE3, 0x82, + 0x99, 0xE3, 0x83, 0xA9, 0xE3, 0x83, 0xA0, 0x52, + 0xE3, 0x82, 0xAD, 0xE3, 0x83, 0xAD, 0xE3, 0x83, + 0xA1, 0xE3, 0x83, 0xBC, 0xE3, 0x83, 0x88, 0xE3, + 0x83, 0xAB, 0x52, 0xE3, 0x82, 0xAF, 0xE3, 0x82, + // Bytes 2c00 - 2c3f + 0x99, 0xE3, 0x83, 0xA9, 0xE3, 0x83, 0xA0, 0xE3, + 0x83, 0x88, 0xE3, 0x83, 0xB3, 0x52, 0xE3, 0x82, + 0xAF, 0xE3, 0x83, 0xAB, 0xE3, 0x82, 0xBB, 0xE3, + 0x82, 0x99, 0xE3, 0x82, 0xA4, 0xE3, 0x83, 0xAD, + 0x52, 0xE3, 0x83, 0x8F, 0xE3, 0x82, 0x9A, 0xE3, + 0x83, 0xBC, 0xE3, 0x82, 0xBB, 0xE3, 0x83, 0xB3, + 0xE3, 0x83, 0x88, 0x52, 0xE3, 0x83, 0x92, 0xE3, + 0x82, 0x9A, 0xE3, 0x82, 0xA2, 0xE3, 0x82, 0xB9, + // Bytes 2c40 - 2c7f + 0xE3, 0x83, 0x88, 0xE3, 0x83, 0xAB, 0x52, 0xE3, + 0x83, 0x95, 0xE3, 0x82, 0x99, 0xE3, 0x83, 0x83, + 0xE3, 0x82, 0xB7, 0xE3, 0x82, 0xA7, 0xE3, 0x83, + 0xAB, 0x52, 0xE3, 0x83, 0x9F, 0xE3, 0x83, 0xAA, + 0xE3, 0x83, 0x8F, 0xE3, 0x82, 0x99, 0xE3, 0x83, + 0xBC, 0xE3, 0x83, 0xAB, 0x52, 0xE3, 0x83, 0xAC, + 0xE3, 0x83, 0xB3, 0xE3, 0x83, 0x88, 0xE3, 0x82, + 0xB1, 0xE3, 0x82, 0x99, 0xE3, 0x83, 0xB3, 0x61, + // Bytes 2c80 - 2cbf + 0xD8, 0xB5, 0xD9, 0x84, 0xD9, 0x89, 0x20, 0xD8, + 0xA7, 0xD9, 0x84, 0xD9, 0x84, 0xD9, 0x87, 0x20, + 0xD8, 0xB9, 0xD9, 0x84, 0xD9, 0x8A, 0xD9, 0x87, + 0x20, 0xD9, 0x88, 0xD8, 0xB3, 0xD9, 0x84, 0xD9, + 0x85, 0x06, 0xE0, 0xA7, 0x87, 0xE0, 0xA6, 0xBE, + 0x01, 0x06, 0xE0, 0xA7, 0x87, 0xE0, 0xA7, 0x97, + 0x01, 0x06, 0xE0, 0xAD, 0x87, 0xE0, 0xAC, 0xBE, + 0x01, 0x06, 0xE0, 0xAD, 0x87, 0xE0, 0xAD, 0x96, + // Bytes 2cc0 - 2cff + 0x01, 0x06, 0xE0, 0xAD, 0x87, 0xE0, 0xAD, 0x97, + 0x01, 0x06, 0xE0, 0xAE, 0x92, 0xE0, 0xAF, 0x97, + 0x01, 0x06, 0xE0, 0xAF, 0x86, 0xE0, 0xAE, 0xBE, + 0x01, 0x06, 0xE0, 0xAF, 0x86, 0xE0, 0xAF, 0x97, + 0x01, 0x06, 0xE0, 0xAF, 0x87, 0xE0, 0xAE, 0xBE, + 0x01, 0x06, 0xE0, 0xB2, 0xBF, 0xE0, 0xB3, 0x95, + 0x01, 0x06, 0xE0, 0xB3, 0x86, 0xE0, 0xB3, 0x95, + 0x01, 0x06, 0xE0, 0xB3, 0x86, 0xE0, 0xB3, 0x96, + // Bytes 2d00 - 2d3f + 0x01, 0x06, 0xE0, 0xB5, 0x86, 0xE0, 0xB4, 0xBE, + 0x01, 0x06, 0xE0, 0xB5, 0x86, 0xE0, 0xB5, 0x97, + 0x01, 0x06, 0xE0, 0xB5, 0x87, 0xE0, 0xB4, 0xBE, + 0x01, 0x06, 0xE0, 0xB7, 0x99, 0xE0, 0xB7, 0x9F, + 0x01, 0x06, 0xE1, 0x80, 0xA5, 0xE1, 0x80, 0xAE, + 0x01, 0x06, 0xE1, 0xAC, 0x85, 0xE1, 0xAC, 0xB5, + 0x01, 0x06, 0xE1, 0xAC, 0x87, 0xE1, 0xAC, 0xB5, + 0x01, 0x06, 0xE1, 0xAC, 0x89, 0xE1, 0xAC, 0xB5, + // Bytes 2d40 - 2d7f + 0x01, 0x06, 0xE1, 0xAC, 0x8B, 0xE1, 0xAC, 0xB5, + 0x01, 0x06, 0xE1, 0xAC, 0x8D, 0xE1, 0xAC, 0xB5, + 0x01, 0x06, 0xE1, 0xAC, 0x91, 0xE1, 0xAC, 0xB5, + 0x01, 0x06, 0xE1, 0xAC, 0xBA, 0xE1, 0xAC, 0xB5, + 0x01, 0x06, 0xE1, 0xAC, 0xBC, 0xE1, 0xAC, 0xB5, + 0x01, 0x06, 0xE1, 0xAC, 0xBE, 0xE1, 0xAC, 0xB5, + 0x01, 0x06, 0xE1, 0xAC, 0xBF, 0xE1, 0xAC, 0xB5, + 0x01, 0x06, 0xE1, 0xAD, 0x82, 0xE1, 0xAC, 0xB5, + // Bytes 2d80 - 2dbf + 0x01, 0x08, 0xF0, 0x91, 0x84, 0xB1, 0xF0, 0x91, + 0x84, 0xA7, 0x01, 0x08, 0xF0, 0x91, 0x84, 0xB2, + 0xF0, 0x91, 0x84, 0xA7, 0x01, 0x08, 0xF0, 0x91, + 0x8D, 0x87, 0xF0, 0x91, 0x8C, 0xBE, 0x01, 0x08, + 0xF0, 0x91, 0x8D, 0x87, 0xF0, 0x91, 0x8D, 0x97, + 0x01, 0x08, 0xF0, 0x91, 0x92, 0xB9, 0xF0, 0x91, + 0x92, 0xB0, 0x01, 0x08, 0xF0, 0x91, 0x92, 0xB9, + 0xF0, 0x91, 0x92, 0xBA, 0x01, 0x08, 0xF0, 0x91, + // Bytes 2dc0 - 2dff + 0x92, 0xB9, 0xF0, 0x91, 0x92, 0xBD, 0x01, 0x08, + 0xF0, 0x91, 0x96, 0xB8, 0xF0, 0x91, 0x96, 0xAF, + 0x01, 0x08, 0xF0, 0x91, 0x96, 0xB9, 0xF0, 0x91, + 0x96, 0xAF, 0x01, 0x09, 0xE0, 0xB3, 0x86, 0xE0, + 0xB3, 0x82, 0xE0, 0xB3, 0x95, 0x02, 0x09, 0xE0, + 0xB7, 0x99, 0xE0, 0xB7, 0x8F, 0xE0, 0xB7, 0x8A, + 0x12, 0x44, 0x44, 0x5A, 0xCC, 0x8C, 0xC9, 0x44, + 0x44, 0x7A, 0xCC, 0x8C, 0xC9, 0x44, 0x64, 0x7A, + // Bytes 2e00 - 2e3f + 0xCC, 0x8C, 0xC9, 0x46, 0xD9, 0x84, 0xD8, 0xA7, + 0xD9, 0x93, 0xC9, 0x46, 0xD9, 0x84, 0xD8, 0xA7, + 0xD9, 0x94, 0xC9, 0x46, 0xD9, 0x84, 0xD8, 0xA7, + 0xD9, 0x95, 0xB5, 0x46, 0xE1, 0x84, 0x80, 0xE1, + 0x85, 0xA1, 0x01, 0x46, 0xE1, 0x84, 0x82, 0xE1, + 0x85, 0xA1, 0x01, 0x46, 0xE1, 0x84, 0x83, 0xE1, + 0x85, 0xA1, 0x01, 0x46, 0xE1, 0x84, 0x85, 0xE1, + 0x85, 0xA1, 0x01, 0x46, 0xE1, 0x84, 0x86, 0xE1, + // Bytes 2e40 - 2e7f + 0x85, 0xA1, 0x01, 0x46, 0xE1, 0x84, 0x87, 0xE1, + 0x85, 0xA1, 0x01, 0x46, 0xE1, 0x84, 0x89, 0xE1, + 0x85, 0xA1, 0x01, 0x46, 0xE1, 0x84, 0x8B, 0xE1, + 0x85, 0xA1, 0x01, 0x46, 0xE1, 0x84, 0x8B, 0xE1, + 0x85, 0xAE, 0x01, 0x46, 0xE1, 0x84, 0x8C, 0xE1, + 0x85, 0xA1, 0x01, 0x46, 0xE1, 0x84, 0x8E, 0xE1, + 0x85, 0xA1, 0x01, 0x46, 0xE1, 0x84, 0x8F, 0xE1, + 0x85, 0xA1, 0x01, 0x46, 0xE1, 0x84, 0x90, 0xE1, + // Bytes 2e80 - 2ebf + 0x85, 0xA1, 0x01, 0x46, 0xE1, 0x84, 0x91, 0xE1, + 0x85, 0xA1, 0x01, 0x46, 0xE1, 0x84, 0x92, 0xE1, + 0x85, 0xA1, 0x01, 0x49, 0xE3, 0x83, 0xA1, 0xE3, + 0x82, 0xAB, 0xE3, 0x82, 0x99, 0x0D, 0x4C, 0xE1, + 0x84, 0x8C, 0xE1, 0x85, 0xAE, 0xE1, 0x84, 0x8B, + 0xE1, 0x85, 0xB4, 0x01, 0x4C, 0xE3, 0x82, 0xAD, + 0xE3, 0x82, 0x99, 0xE3, 0x82, 0xAB, 0xE3, 0x82, + 0x99, 0x0D, 0x4C, 0xE3, 0x82, 0xB3, 0xE3, 0x83, + // Bytes 2ec0 - 2eff + 0xBC, 0xE3, 0x83, 0x9B, 0xE3, 0x82, 0x9A, 0x0D, + 0x4C, 0xE3, 0x83, 0xA4, 0xE3, 0x83, 0xBC, 0xE3, + 0x83, 0x88, 0xE3, 0x82, 0x99, 0x0D, 0x4F, 0xE1, + 0x84, 0x8E, 0xE1, 0x85, 0xA1, 0xE1, 0x86, 0xB7, + 0xE1, 0x84, 0x80, 0xE1, 0x85, 0xA9, 0x01, 0x4F, + 0xE3, 0x82, 0xA4, 0xE3, 0x83, 0x8B, 0xE3, 0x83, + 0xB3, 0xE3, 0x82, 0xAF, 0xE3, 0x82, 0x99, 0x0D, + 0x4F, 0xE3, 0x82, 0xB7, 0xE3, 0x83, 0xAA, 0xE3, + // Bytes 2f00 - 2f3f + 0x83, 0xB3, 0xE3, 0x82, 0xAF, 0xE3, 0x82, 0x99, + 0x0D, 0x4F, 0xE3, 0x83, 0x98, 0xE3, 0x82, 0x9A, + 0xE3, 0x83, 0xBC, 0xE3, 0x82, 0xB7, 0xE3, 0x82, + 0x99, 0x0D, 0x4F, 0xE3, 0x83, 0x9B, 0xE3, 0x82, + 0x9A, 0xE3, 0x83, 0xB3, 0xE3, 0x83, 0x88, 0xE3, + 0x82, 0x99, 0x0D, 0x52, 0xE3, 0x82, 0xA8, 0xE3, + 0x82, 0xB9, 0xE3, 0x82, 0xAF, 0xE3, 0x83, 0xBC, + 0xE3, 0x83, 0x88, 0xE3, 0x82, 0x99, 0x0D, 0x52, + // Bytes 2f40 - 2f7f + 0xE3, 0x83, 0x95, 0xE3, 0x82, 0xA1, 0xE3, 0x83, + 0xA9, 0xE3, 0x83, 0x83, 0xE3, 0x83, 0x88, 0xE3, + 0x82, 0x99, 0x0D, 0x86, 0xE0, 0xB3, 0x86, 0xE0, + 0xB3, 0x82, 0x01, 0x86, 0xE0, 0xB7, 0x99, 0xE0, + 0xB7, 0x8F, 0x01, 0x03, 0x3C, 0xCC, 0xB8, 0x05, + 0x03, 0x3D, 0xCC, 0xB8, 0x05, 0x03, 0x3E, 0xCC, + 0xB8, 0x05, 0x03, 0x41, 0xCC, 0x80, 0xC9, 0x03, + 0x41, 0xCC, 0x81, 0xC9, 0x03, 0x41, 0xCC, 0x83, + // Bytes 2f80 - 2fbf + 0xC9, 0x03, 0x41, 0xCC, 0x84, 0xC9, 0x03, 0x41, + 0xCC, 0x89, 0xC9, 0x03, 0x41, 0xCC, 0x8C, 0xC9, + 0x03, 0x41, 0xCC, 0x8F, 0xC9, 0x03, 0x41, 0xCC, + 0x91, 0xC9, 0x03, 0x41, 0xCC, 0xA5, 0xB5, 0x03, + 0x41, 0xCC, 0xA8, 0xA5, 0x03, 0x42, 0xCC, 0x87, + 0xC9, 0x03, 0x42, 0xCC, 0xA3, 0xB5, 0x03, 0x42, + 0xCC, 0xB1, 0xB5, 0x03, 0x43, 0xCC, 0x81, 0xC9, + 0x03, 0x43, 0xCC, 0x82, 0xC9, 0x03, 0x43, 0xCC, + // Bytes 2fc0 - 2fff + 0x87, 0xC9, 0x03, 0x43, 0xCC, 0x8C, 0xC9, 0x03, + 0x44, 0xCC, 0x87, 0xC9, 0x03, 0x44, 0xCC, 0x8C, + 0xC9, 0x03, 0x44, 0xCC, 0xA3, 0xB5, 0x03, 0x44, + 0xCC, 0xA7, 0xA5, 0x03, 0x44, 0xCC, 0xAD, 0xB5, + 0x03, 0x44, 0xCC, 0xB1, 0xB5, 0x03, 0x45, 0xCC, + 0x80, 0xC9, 0x03, 0x45, 0xCC, 0x81, 0xC9, 0x03, + 0x45, 0xCC, 0x83, 0xC9, 0x03, 0x45, 0xCC, 0x86, + 0xC9, 0x03, 0x45, 0xCC, 0x87, 0xC9, 0x03, 0x45, + // Bytes 3000 - 303f + 0xCC, 0x88, 0xC9, 0x03, 0x45, 0xCC, 0x89, 0xC9, + 0x03, 0x45, 0xCC, 0x8C, 0xC9, 0x03, 0x45, 0xCC, + 0x8F, 0xC9, 0x03, 0x45, 0xCC, 0x91, 0xC9, 0x03, + 0x45, 0xCC, 0xA8, 0xA5, 0x03, 0x45, 0xCC, 0xAD, + 0xB5, 0x03, 0x45, 0xCC, 0xB0, 0xB5, 0x03, 0x46, + 0xCC, 0x87, 0xC9, 0x03, 0x47, 0xCC, 0x81, 0xC9, + 0x03, 0x47, 0xCC, 0x82, 0xC9, 0x03, 0x47, 0xCC, + 0x84, 0xC9, 0x03, 0x47, 0xCC, 0x86, 0xC9, 0x03, + // Bytes 3040 - 307f + 0x47, 0xCC, 0x87, 0xC9, 0x03, 0x47, 0xCC, 0x8C, + 0xC9, 0x03, 0x47, 0xCC, 0xA7, 0xA5, 0x03, 0x48, + 0xCC, 0x82, 0xC9, 0x03, 0x48, 0xCC, 0x87, 0xC9, + 0x03, 0x48, 0xCC, 0x88, 0xC9, 0x03, 0x48, 0xCC, + 0x8C, 0xC9, 0x03, 0x48, 0xCC, 0xA3, 0xB5, 0x03, + 0x48, 0xCC, 0xA7, 0xA5, 0x03, 0x48, 0xCC, 0xAE, + 0xB5, 0x03, 0x49, 0xCC, 0x80, 0xC9, 0x03, 0x49, + 0xCC, 0x81, 0xC9, 0x03, 0x49, 0xCC, 0x82, 0xC9, + // Bytes 3080 - 30bf + 0x03, 0x49, 0xCC, 0x83, 0xC9, 0x03, 0x49, 0xCC, + 0x84, 0xC9, 0x03, 0x49, 0xCC, 0x86, 0xC9, 0x03, + 0x49, 0xCC, 0x87, 0xC9, 0x03, 0x49, 0xCC, 0x89, + 0xC9, 0x03, 0x49, 0xCC, 0x8C, 0xC9, 0x03, 0x49, + 0xCC, 0x8F, 0xC9, 0x03, 0x49, 0xCC, 0x91, 0xC9, + 0x03, 0x49, 0xCC, 0xA3, 0xB5, 0x03, 0x49, 0xCC, + 0xA8, 0xA5, 0x03, 0x49, 0xCC, 0xB0, 0xB5, 0x03, + 0x4A, 0xCC, 0x82, 0xC9, 0x03, 0x4B, 0xCC, 0x81, + // Bytes 30c0 - 30ff + 0xC9, 0x03, 0x4B, 0xCC, 0x8C, 0xC9, 0x03, 0x4B, + 0xCC, 0xA3, 0xB5, 0x03, 0x4B, 0xCC, 0xA7, 0xA5, + 0x03, 0x4B, 0xCC, 0xB1, 0xB5, 0x03, 0x4C, 0xCC, + 0x81, 0xC9, 0x03, 0x4C, 0xCC, 0x8C, 0xC9, 0x03, + 0x4C, 0xCC, 0xA7, 0xA5, 0x03, 0x4C, 0xCC, 0xAD, + 0xB5, 0x03, 0x4C, 0xCC, 0xB1, 0xB5, 0x03, 0x4D, + 0xCC, 0x81, 0xC9, 0x03, 0x4D, 0xCC, 0x87, 0xC9, + 0x03, 0x4D, 0xCC, 0xA3, 0xB5, 0x03, 0x4E, 0xCC, + // Bytes 3100 - 313f + 0x80, 0xC9, 0x03, 0x4E, 0xCC, 0x81, 0xC9, 0x03, + 0x4E, 0xCC, 0x83, 0xC9, 0x03, 0x4E, 0xCC, 0x87, + 0xC9, 0x03, 0x4E, 0xCC, 0x8C, 0xC9, 0x03, 0x4E, + 0xCC, 0xA3, 0xB5, 0x03, 0x4E, 0xCC, 0xA7, 0xA5, + 0x03, 0x4E, 0xCC, 0xAD, 0xB5, 0x03, 0x4E, 0xCC, + 0xB1, 0xB5, 0x03, 0x4F, 0xCC, 0x80, 0xC9, 0x03, + 0x4F, 0xCC, 0x81, 0xC9, 0x03, 0x4F, 0xCC, 0x86, + 0xC9, 0x03, 0x4F, 0xCC, 0x89, 0xC9, 0x03, 0x4F, + // Bytes 3140 - 317f + 0xCC, 0x8B, 0xC9, 0x03, 0x4F, 0xCC, 0x8C, 0xC9, + 0x03, 0x4F, 0xCC, 0x8F, 0xC9, 0x03, 0x4F, 0xCC, + 0x91, 0xC9, 0x03, 0x50, 0xCC, 0x81, 0xC9, 0x03, + 0x50, 0xCC, 0x87, 0xC9, 0x03, 0x52, 0xCC, 0x81, + 0xC9, 0x03, 0x52, 0xCC, 0x87, 0xC9, 0x03, 0x52, + 0xCC, 0x8C, 0xC9, 0x03, 0x52, 0xCC, 0x8F, 0xC9, + 0x03, 0x52, 0xCC, 0x91, 0xC9, 0x03, 0x52, 0xCC, + 0xA7, 0xA5, 0x03, 0x52, 0xCC, 0xB1, 0xB5, 0x03, + // Bytes 3180 - 31bf + 0x53, 0xCC, 0x82, 0xC9, 0x03, 0x53, 0xCC, 0x87, + 0xC9, 0x03, 0x53, 0xCC, 0xA6, 0xB5, 0x03, 0x53, + 0xCC, 0xA7, 0xA5, 0x03, 0x54, 0xCC, 0x87, 0xC9, + 0x03, 0x54, 0xCC, 0x8C, 0xC9, 0x03, 0x54, 0xCC, + 0xA3, 0xB5, 0x03, 0x54, 0xCC, 0xA6, 0xB5, 0x03, + 0x54, 0xCC, 0xA7, 0xA5, 0x03, 0x54, 0xCC, 0xAD, + 0xB5, 0x03, 0x54, 0xCC, 0xB1, 0xB5, 0x03, 0x55, + 0xCC, 0x80, 0xC9, 0x03, 0x55, 0xCC, 0x81, 0xC9, + // Bytes 31c0 - 31ff + 0x03, 0x55, 0xCC, 0x82, 0xC9, 0x03, 0x55, 0xCC, + 0x86, 0xC9, 0x03, 0x55, 0xCC, 0x89, 0xC9, 0x03, + 0x55, 0xCC, 0x8A, 0xC9, 0x03, 0x55, 0xCC, 0x8B, + 0xC9, 0x03, 0x55, 0xCC, 0x8C, 0xC9, 0x03, 0x55, + 0xCC, 0x8F, 0xC9, 0x03, 0x55, 0xCC, 0x91, 0xC9, + 0x03, 0x55, 0xCC, 0xA3, 0xB5, 0x03, 0x55, 0xCC, + 0xA4, 0xB5, 0x03, 0x55, 0xCC, 0xA8, 0xA5, 0x03, + 0x55, 0xCC, 0xAD, 0xB5, 0x03, 0x55, 0xCC, 0xB0, + // Bytes 3200 - 323f + 0xB5, 0x03, 0x56, 0xCC, 0x83, 0xC9, 0x03, 0x56, + 0xCC, 0xA3, 0xB5, 0x03, 0x57, 0xCC, 0x80, 0xC9, + 0x03, 0x57, 0xCC, 0x81, 0xC9, 0x03, 0x57, 0xCC, + 0x82, 0xC9, 0x03, 0x57, 0xCC, 0x87, 0xC9, 0x03, + 0x57, 0xCC, 0x88, 0xC9, 0x03, 0x57, 0xCC, 0xA3, + 0xB5, 0x03, 0x58, 0xCC, 0x87, 0xC9, 0x03, 0x58, + 0xCC, 0x88, 0xC9, 0x03, 0x59, 0xCC, 0x80, 0xC9, + 0x03, 0x59, 0xCC, 0x81, 0xC9, 0x03, 0x59, 0xCC, + // Bytes 3240 - 327f + 0x82, 0xC9, 0x03, 0x59, 0xCC, 0x83, 0xC9, 0x03, + 0x59, 0xCC, 0x84, 0xC9, 0x03, 0x59, 0xCC, 0x87, + 0xC9, 0x03, 0x59, 0xCC, 0x88, 0xC9, 0x03, 0x59, + 0xCC, 0x89, 0xC9, 0x03, 0x59, 0xCC, 0xA3, 0xB5, + 0x03, 0x5A, 0xCC, 0x81, 0xC9, 0x03, 0x5A, 0xCC, + 0x82, 0xC9, 0x03, 0x5A, 0xCC, 0x87, 0xC9, 0x03, + 0x5A, 0xCC, 0x8C, 0xC9, 0x03, 0x5A, 0xCC, 0xA3, + 0xB5, 0x03, 0x5A, 0xCC, 0xB1, 0xB5, 0x03, 0x61, + // Bytes 3280 - 32bf + 0xCC, 0x80, 0xC9, 0x03, 0x61, 0xCC, 0x81, 0xC9, + 0x03, 0x61, 0xCC, 0x83, 0xC9, 0x03, 0x61, 0xCC, + 0x84, 0xC9, 0x03, 0x61, 0xCC, 0x89, 0xC9, 0x03, + 0x61, 0xCC, 0x8C, 0xC9, 0x03, 0x61, 0xCC, 0x8F, + 0xC9, 0x03, 0x61, 0xCC, 0x91, 0xC9, 0x03, 0x61, + 0xCC, 0xA5, 0xB5, 0x03, 0x61, 0xCC, 0xA8, 0xA5, + 0x03, 0x62, 0xCC, 0x87, 0xC9, 0x03, 0x62, 0xCC, + 0xA3, 0xB5, 0x03, 0x62, 0xCC, 0xB1, 0xB5, 0x03, + // Bytes 32c0 - 32ff + 0x63, 0xCC, 0x81, 0xC9, 0x03, 0x63, 0xCC, 0x82, + 0xC9, 0x03, 0x63, 0xCC, 0x87, 0xC9, 0x03, 0x63, + 0xCC, 0x8C, 0xC9, 0x03, 0x64, 0xCC, 0x87, 0xC9, + 0x03, 0x64, 0xCC, 0x8C, 0xC9, 0x03, 0x64, 0xCC, + 0xA3, 0xB5, 0x03, 0x64, 0xCC, 0xA7, 0xA5, 0x03, + 0x64, 0xCC, 0xAD, 0xB5, 0x03, 0x64, 0xCC, 0xB1, + 0xB5, 0x03, 0x65, 0xCC, 0x80, 0xC9, 0x03, 0x65, + 0xCC, 0x81, 0xC9, 0x03, 0x65, 0xCC, 0x83, 0xC9, + // Bytes 3300 - 333f + 0x03, 0x65, 0xCC, 0x86, 0xC9, 0x03, 0x65, 0xCC, + 0x87, 0xC9, 0x03, 0x65, 0xCC, 0x88, 0xC9, 0x03, + 0x65, 0xCC, 0x89, 0xC9, 0x03, 0x65, 0xCC, 0x8C, + 0xC9, 0x03, 0x65, 0xCC, 0x8F, 0xC9, 0x03, 0x65, + 0xCC, 0x91, 0xC9, 0x03, 0x65, 0xCC, 0xA8, 0xA5, + 0x03, 0x65, 0xCC, 0xAD, 0xB5, 0x03, 0x65, 0xCC, + 0xB0, 0xB5, 0x03, 0x66, 0xCC, 0x87, 0xC9, 0x03, + 0x67, 0xCC, 0x81, 0xC9, 0x03, 0x67, 0xCC, 0x82, + // Bytes 3340 - 337f + 0xC9, 0x03, 0x67, 0xCC, 0x84, 0xC9, 0x03, 0x67, + 0xCC, 0x86, 0xC9, 0x03, 0x67, 0xCC, 0x87, 0xC9, + 0x03, 0x67, 0xCC, 0x8C, 0xC9, 0x03, 0x67, 0xCC, + 0xA7, 0xA5, 0x03, 0x68, 0xCC, 0x82, 0xC9, 0x03, + 0x68, 0xCC, 0x87, 0xC9, 0x03, 0x68, 0xCC, 0x88, + 0xC9, 0x03, 0x68, 0xCC, 0x8C, 0xC9, 0x03, 0x68, + 0xCC, 0xA3, 0xB5, 0x03, 0x68, 0xCC, 0xA7, 0xA5, + 0x03, 0x68, 0xCC, 0xAE, 0xB5, 0x03, 0x68, 0xCC, + // Bytes 3380 - 33bf + 0xB1, 0xB5, 0x03, 0x69, 0xCC, 0x80, 0xC9, 0x03, + 0x69, 0xCC, 0x81, 0xC9, 0x03, 0x69, 0xCC, 0x82, + 0xC9, 0x03, 0x69, 0xCC, 0x83, 0xC9, 0x03, 0x69, + 0xCC, 0x84, 0xC9, 0x03, 0x69, 0xCC, 0x86, 0xC9, + 0x03, 0x69, 0xCC, 0x89, 0xC9, 0x03, 0x69, 0xCC, + 0x8C, 0xC9, 0x03, 0x69, 0xCC, 0x8F, 0xC9, 0x03, + 0x69, 0xCC, 0x91, 0xC9, 0x03, 0x69, 0xCC, 0xA3, + 0xB5, 0x03, 0x69, 0xCC, 0xA8, 0xA5, 0x03, 0x69, + // Bytes 33c0 - 33ff + 0xCC, 0xB0, 0xB5, 0x03, 0x6A, 0xCC, 0x82, 0xC9, + 0x03, 0x6A, 0xCC, 0x8C, 0xC9, 0x03, 0x6B, 0xCC, + 0x81, 0xC9, 0x03, 0x6B, 0xCC, 0x8C, 0xC9, 0x03, + 0x6B, 0xCC, 0xA3, 0xB5, 0x03, 0x6B, 0xCC, 0xA7, + 0xA5, 0x03, 0x6B, 0xCC, 0xB1, 0xB5, 0x03, 0x6C, + 0xCC, 0x81, 0xC9, 0x03, 0x6C, 0xCC, 0x8C, 0xC9, + 0x03, 0x6C, 0xCC, 0xA7, 0xA5, 0x03, 0x6C, 0xCC, + 0xAD, 0xB5, 0x03, 0x6C, 0xCC, 0xB1, 0xB5, 0x03, + // Bytes 3400 - 343f + 0x6D, 0xCC, 0x81, 0xC9, 0x03, 0x6D, 0xCC, 0x87, + 0xC9, 0x03, 0x6D, 0xCC, 0xA3, 0xB5, 0x03, 0x6E, + 0xCC, 0x80, 0xC9, 0x03, 0x6E, 0xCC, 0x81, 0xC9, + 0x03, 0x6E, 0xCC, 0x83, 0xC9, 0x03, 0x6E, 0xCC, + 0x87, 0xC9, 0x03, 0x6E, 0xCC, 0x8C, 0xC9, 0x03, + 0x6E, 0xCC, 0xA3, 0xB5, 0x03, 0x6E, 0xCC, 0xA7, + 0xA5, 0x03, 0x6E, 0xCC, 0xAD, 0xB5, 0x03, 0x6E, + 0xCC, 0xB1, 0xB5, 0x03, 0x6F, 0xCC, 0x80, 0xC9, + // Bytes 3440 - 347f + 0x03, 0x6F, 0xCC, 0x81, 0xC9, 0x03, 0x6F, 0xCC, + 0x86, 0xC9, 0x03, 0x6F, 0xCC, 0x89, 0xC9, 0x03, + 0x6F, 0xCC, 0x8B, 0xC9, 0x03, 0x6F, 0xCC, 0x8C, + 0xC9, 0x03, 0x6F, 0xCC, 0x8F, 0xC9, 0x03, 0x6F, + 0xCC, 0x91, 0xC9, 0x03, 0x70, 0xCC, 0x81, 0xC9, + 0x03, 0x70, 0xCC, 0x87, 0xC9, 0x03, 0x72, 0xCC, + 0x81, 0xC9, 0x03, 0x72, 0xCC, 0x87, 0xC9, 0x03, + 0x72, 0xCC, 0x8C, 0xC9, 0x03, 0x72, 0xCC, 0x8F, + // Bytes 3480 - 34bf + 0xC9, 0x03, 0x72, 0xCC, 0x91, 0xC9, 0x03, 0x72, + 0xCC, 0xA7, 0xA5, 0x03, 0x72, 0xCC, 0xB1, 0xB5, + 0x03, 0x73, 0xCC, 0x82, 0xC9, 0x03, 0x73, 0xCC, + 0x87, 0xC9, 0x03, 0x73, 0xCC, 0xA6, 0xB5, 0x03, + 0x73, 0xCC, 0xA7, 0xA5, 0x03, 0x74, 0xCC, 0x87, + 0xC9, 0x03, 0x74, 0xCC, 0x88, 0xC9, 0x03, 0x74, + 0xCC, 0x8C, 0xC9, 0x03, 0x74, 0xCC, 0xA3, 0xB5, + 0x03, 0x74, 0xCC, 0xA6, 0xB5, 0x03, 0x74, 0xCC, + // Bytes 34c0 - 34ff + 0xA7, 0xA5, 0x03, 0x74, 0xCC, 0xAD, 0xB5, 0x03, + 0x74, 0xCC, 0xB1, 0xB5, 0x03, 0x75, 0xCC, 0x80, + 0xC9, 0x03, 0x75, 0xCC, 0x81, 0xC9, 0x03, 0x75, + 0xCC, 0x82, 0xC9, 0x03, 0x75, 0xCC, 0x86, 0xC9, + 0x03, 0x75, 0xCC, 0x89, 0xC9, 0x03, 0x75, 0xCC, + 0x8A, 0xC9, 0x03, 0x75, 0xCC, 0x8B, 0xC9, 0x03, + 0x75, 0xCC, 0x8C, 0xC9, 0x03, 0x75, 0xCC, 0x8F, + 0xC9, 0x03, 0x75, 0xCC, 0x91, 0xC9, 0x03, 0x75, + // Bytes 3500 - 353f + 0xCC, 0xA3, 0xB5, 0x03, 0x75, 0xCC, 0xA4, 0xB5, + 0x03, 0x75, 0xCC, 0xA8, 0xA5, 0x03, 0x75, 0xCC, + 0xAD, 0xB5, 0x03, 0x75, 0xCC, 0xB0, 0xB5, 0x03, + 0x76, 0xCC, 0x83, 0xC9, 0x03, 0x76, 0xCC, 0xA3, + 0xB5, 0x03, 0x77, 0xCC, 0x80, 0xC9, 0x03, 0x77, + 0xCC, 0x81, 0xC9, 0x03, 0x77, 0xCC, 0x82, 0xC9, + 0x03, 0x77, 0xCC, 0x87, 0xC9, 0x03, 0x77, 0xCC, + 0x88, 0xC9, 0x03, 0x77, 0xCC, 0x8A, 0xC9, 0x03, + // Bytes 3540 - 357f + 0x77, 0xCC, 0xA3, 0xB5, 0x03, 0x78, 0xCC, 0x87, + 0xC9, 0x03, 0x78, 0xCC, 0x88, 0xC9, 0x03, 0x79, + 0xCC, 0x80, 0xC9, 0x03, 0x79, 0xCC, 0x81, 0xC9, + 0x03, 0x79, 0xCC, 0x82, 0xC9, 0x03, 0x79, 0xCC, + 0x83, 0xC9, 0x03, 0x79, 0xCC, 0x84, 0xC9, 0x03, + 0x79, 0xCC, 0x87, 0xC9, 0x03, 0x79, 0xCC, 0x88, + 0xC9, 0x03, 0x79, 0xCC, 0x89, 0xC9, 0x03, 0x79, + 0xCC, 0x8A, 0xC9, 0x03, 0x79, 0xCC, 0xA3, 0xB5, + // Bytes 3580 - 35bf + 0x03, 0x7A, 0xCC, 0x81, 0xC9, 0x03, 0x7A, 0xCC, + 0x82, 0xC9, 0x03, 0x7A, 0xCC, 0x87, 0xC9, 0x03, + 0x7A, 0xCC, 0x8C, 0xC9, 0x03, 0x7A, 0xCC, 0xA3, + 0xB5, 0x03, 0x7A, 0xCC, 0xB1, 0xB5, 0x04, 0xC2, + 0xA8, 0xCC, 0x80, 0xCA, 0x04, 0xC2, 0xA8, 0xCC, + 0x81, 0xCA, 0x04, 0xC2, 0xA8, 0xCD, 0x82, 0xCA, + 0x04, 0xC3, 0x86, 0xCC, 0x81, 0xC9, 0x04, 0xC3, + 0x86, 0xCC, 0x84, 0xC9, 0x04, 0xC3, 0x98, 0xCC, + // Bytes 35c0 - 35ff + 0x81, 0xC9, 0x04, 0xC3, 0xA6, 0xCC, 0x81, 0xC9, + 0x04, 0xC3, 0xA6, 0xCC, 0x84, 0xC9, 0x04, 0xC3, + 0xB8, 0xCC, 0x81, 0xC9, 0x04, 0xC5, 0xBF, 0xCC, + 0x87, 0xC9, 0x04, 0xC6, 0xB7, 0xCC, 0x8C, 0xC9, + 0x04, 0xCA, 0x92, 0xCC, 0x8C, 0xC9, 0x04, 0xCE, + 0x91, 0xCC, 0x80, 0xC9, 0x04, 0xCE, 0x91, 0xCC, + 0x81, 0xC9, 0x04, 0xCE, 0x91, 0xCC, 0x84, 0xC9, + 0x04, 0xCE, 0x91, 0xCC, 0x86, 0xC9, 0x04, 0xCE, + // Bytes 3600 - 363f + 0x91, 0xCD, 0x85, 0xD9, 0x04, 0xCE, 0x95, 0xCC, + 0x80, 0xC9, 0x04, 0xCE, 0x95, 0xCC, 0x81, 0xC9, + 0x04, 0xCE, 0x97, 0xCC, 0x80, 0xC9, 0x04, 0xCE, + 0x97, 0xCC, 0x81, 0xC9, 0x04, 0xCE, 0x97, 0xCD, + 0x85, 0xD9, 0x04, 0xCE, 0x99, 0xCC, 0x80, 0xC9, + 0x04, 0xCE, 0x99, 0xCC, 0x81, 0xC9, 0x04, 0xCE, + 0x99, 0xCC, 0x84, 0xC9, 0x04, 0xCE, 0x99, 0xCC, + 0x86, 0xC9, 0x04, 0xCE, 0x99, 0xCC, 0x88, 0xC9, + // Bytes 3640 - 367f + 0x04, 0xCE, 0x9F, 0xCC, 0x80, 0xC9, 0x04, 0xCE, + 0x9F, 0xCC, 0x81, 0xC9, 0x04, 0xCE, 0xA1, 0xCC, + 0x94, 0xC9, 0x04, 0xCE, 0xA5, 0xCC, 0x80, 0xC9, + 0x04, 0xCE, 0xA5, 0xCC, 0x81, 0xC9, 0x04, 0xCE, + 0xA5, 0xCC, 0x84, 0xC9, 0x04, 0xCE, 0xA5, 0xCC, + 0x86, 0xC9, 0x04, 0xCE, 0xA5, 0xCC, 0x88, 0xC9, + 0x04, 0xCE, 0xA9, 0xCC, 0x80, 0xC9, 0x04, 0xCE, + 0xA9, 0xCC, 0x81, 0xC9, 0x04, 0xCE, 0xA9, 0xCD, + // Bytes 3680 - 36bf + 0x85, 0xD9, 0x04, 0xCE, 0xB1, 0xCC, 0x84, 0xC9, + 0x04, 0xCE, 0xB1, 0xCC, 0x86, 0xC9, 0x04, 0xCE, + 0xB1, 0xCD, 0x85, 0xD9, 0x04, 0xCE, 0xB5, 0xCC, + 0x80, 0xC9, 0x04, 0xCE, 0xB5, 0xCC, 0x81, 0xC9, + 0x04, 0xCE, 0xB7, 0xCD, 0x85, 0xD9, 0x04, 0xCE, + 0xB9, 0xCC, 0x80, 0xC9, 0x04, 0xCE, 0xB9, 0xCC, + 0x81, 0xC9, 0x04, 0xCE, 0xB9, 0xCC, 0x84, 0xC9, + 0x04, 0xCE, 0xB9, 0xCC, 0x86, 0xC9, 0x04, 0xCE, + // Bytes 36c0 - 36ff + 0xB9, 0xCD, 0x82, 0xC9, 0x04, 0xCE, 0xBF, 0xCC, + 0x80, 0xC9, 0x04, 0xCE, 0xBF, 0xCC, 0x81, 0xC9, + 0x04, 0xCF, 0x81, 0xCC, 0x93, 0xC9, 0x04, 0xCF, + 0x81, 0xCC, 0x94, 0xC9, 0x04, 0xCF, 0x85, 0xCC, + 0x80, 0xC9, 0x04, 0xCF, 0x85, 0xCC, 0x81, 0xC9, + 0x04, 0xCF, 0x85, 0xCC, 0x84, 0xC9, 0x04, 0xCF, + 0x85, 0xCC, 0x86, 0xC9, 0x04, 0xCF, 0x85, 0xCD, + 0x82, 0xC9, 0x04, 0xCF, 0x89, 0xCD, 0x85, 0xD9, + // Bytes 3700 - 373f + 0x04, 0xCF, 0x92, 0xCC, 0x81, 0xC9, 0x04, 0xCF, + 0x92, 0xCC, 0x88, 0xC9, 0x04, 0xD0, 0x86, 0xCC, + 0x88, 0xC9, 0x04, 0xD0, 0x90, 0xCC, 0x86, 0xC9, + 0x04, 0xD0, 0x90, 0xCC, 0x88, 0xC9, 0x04, 0xD0, + 0x93, 0xCC, 0x81, 0xC9, 0x04, 0xD0, 0x95, 0xCC, + 0x80, 0xC9, 0x04, 0xD0, 0x95, 0xCC, 0x86, 0xC9, + 0x04, 0xD0, 0x95, 0xCC, 0x88, 0xC9, 0x04, 0xD0, + 0x96, 0xCC, 0x86, 0xC9, 0x04, 0xD0, 0x96, 0xCC, + // Bytes 3740 - 377f + 0x88, 0xC9, 0x04, 0xD0, 0x97, 0xCC, 0x88, 0xC9, + 0x04, 0xD0, 0x98, 0xCC, 0x80, 0xC9, 0x04, 0xD0, + 0x98, 0xCC, 0x84, 0xC9, 0x04, 0xD0, 0x98, 0xCC, + 0x86, 0xC9, 0x04, 0xD0, 0x98, 0xCC, 0x88, 0xC9, + 0x04, 0xD0, 0x9A, 0xCC, 0x81, 0xC9, 0x04, 0xD0, + 0x9E, 0xCC, 0x88, 0xC9, 0x04, 0xD0, 0xA3, 0xCC, + 0x84, 0xC9, 0x04, 0xD0, 0xA3, 0xCC, 0x86, 0xC9, + 0x04, 0xD0, 0xA3, 0xCC, 0x88, 0xC9, 0x04, 0xD0, + // Bytes 3780 - 37bf + 0xA3, 0xCC, 0x8B, 0xC9, 0x04, 0xD0, 0xA7, 0xCC, + 0x88, 0xC9, 0x04, 0xD0, 0xAB, 0xCC, 0x88, 0xC9, + 0x04, 0xD0, 0xAD, 0xCC, 0x88, 0xC9, 0x04, 0xD0, + 0xB0, 0xCC, 0x86, 0xC9, 0x04, 0xD0, 0xB0, 0xCC, + 0x88, 0xC9, 0x04, 0xD0, 0xB3, 0xCC, 0x81, 0xC9, + 0x04, 0xD0, 0xB5, 0xCC, 0x80, 0xC9, 0x04, 0xD0, + 0xB5, 0xCC, 0x86, 0xC9, 0x04, 0xD0, 0xB5, 0xCC, + 0x88, 0xC9, 0x04, 0xD0, 0xB6, 0xCC, 0x86, 0xC9, + // Bytes 37c0 - 37ff + 0x04, 0xD0, 0xB6, 0xCC, 0x88, 0xC9, 0x04, 0xD0, + 0xB7, 0xCC, 0x88, 0xC9, 0x04, 0xD0, 0xB8, 0xCC, + 0x80, 0xC9, 0x04, 0xD0, 0xB8, 0xCC, 0x84, 0xC9, + 0x04, 0xD0, 0xB8, 0xCC, 0x86, 0xC9, 0x04, 0xD0, + 0xB8, 0xCC, 0x88, 0xC9, 0x04, 0xD0, 0xBA, 0xCC, + 0x81, 0xC9, 0x04, 0xD0, 0xBE, 0xCC, 0x88, 0xC9, + 0x04, 0xD1, 0x83, 0xCC, 0x84, 0xC9, 0x04, 0xD1, + 0x83, 0xCC, 0x86, 0xC9, 0x04, 0xD1, 0x83, 0xCC, + // Bytes 3800 - 383f + 0x88, 0xC9, 0x04, 0xD1, 0x83, 0xCC, 0x8B, 0xC9, + 0x04, 0xD1, 0x87, 0xCC, 0x88, 0xC9, 0x04, 0xD1, + 0x8B, 0xCC, 0x88, 0xC9, 0x04, 0xD1, 0x8D, 0xCC, + 0x88, 0xC9, 0x04, 0xD1, 0x96, 0xCC, 0x88, 0xC9, + 0x04, 0xD1, 0xB4, 0xCC, 0x8F, 0xC9, 0x04, 0xD1, + 0xB5, 0xCC, 0x8F, 0xC9, 0x04, 0xD3, 0x98, 0xCC, + 0x88, 0xC9, 0x04, 0xD3, 0x99, 0xCC, 0x88, 0xC9, + 0x04, 0xD3, 0xA8, 0xCC, 0x88, 0xC9, 0x04, 0xD3, + // Bytes 3840 - 387f + 0xA9, 0xCC, 0x88, 0xC9, 0x04, 0xD8, 0xA7, 0xD9, + 0x93, 0xC9, 0x04, 0xD8, 0xA7, 0xD9, 0x94, 0xC9, + 0x04, 0xD8, 0xA7, 0xD9, 0x95, 0xB5, 0x04, 0xD9, + 0x88, 0xD9, 0x94, 0xC9, 0x04, 0xD9, 0x8A, 0xD9, + 0x94, 0xC9, 0x04, 0xDB, 0x81, 0xD9, 0x94, 0xC9, + 0x04, 0xDB, 0x92, 0xD9, 0x94, 0xC9, 0x04, 0xDB, + 0x95, 0xD9, 0x94, 0xC9, 0x05, 0x41, 0xCC, 0x82, + 0xCC, 0x80, 0xCA, 0x05, 0x41, 0xCC, 0x82, 0xCC, + // Bytes 3880 - 38bf + 0x81, 0xCA, 0x05, 0x41, 0xCC, 0x82, 0xCC, 0x83, + 0xCA, 0x05, 0x41, 0xCC, 0x82, 0xCC, 0x89, 0xCA, + 0x05, 0x41, 0xCC, 0x86, 0xCC, 0x80, 0xCA, 0x05, + 0x41, 0xCC, 0x86, 0xCC, 0x81, 0xCA, 0x05, 0x41, + 0xCC, 0x86, 0xCC, 0x83, 0xCA, 0x05, 0x41, 0xCC, + 0x86, 0xCC, 0x89, 0xCA, 0x05, 0x41, 0xCC, 0x87, + 0xCC, 0x84, 0xCA, 0x05, 0x41, 0xCC, 0x88, 0xCC, + 0x84, 0xCA, 0x05, 0x41, 0xCC, 0x8A, 0xCC, 0x81, + // Bytes 38c0 - 38ff + 0xCA, 0x05, 0x41, 0xCC, 0xA3, 0xCC, 0x82, 0xCA, + 0x05, 0x41, 0xCC, 0xA3, 0xCC, 0x86, 0xCA, 0x05, + 0x43, 0xCC, 0xA7, 0xCC, 0x81, 0xCA, 0x05, 0x45, + 0xCC, 0x82, 0xCC, 0x80, 0xCA, 0x05, 0x45, 0xCC, + 0x82, 0xCC, 0x81, 0xCA, 0x05, 0x45, 0xCC, 0x82, + 0xCC, 0x83, 0xCA, 0x05, 0x45, 0xCC, 0x82, 0xCC, + 0x89, 0xCA, 0x05, 0x45, 0xCC, 0x84, 0xCC, 0x80, + 0xCA, 0x05, 0x45, 0xCC, 0x84, 0xCC, 0x81, 0xCA, + // Bytes 3900 - 393f + 0x05, 0x45, 0xCC, 0xA3, 0xCC, 0x82, 0xCA, 0x05, + 0x45, 0xCC, 0xA7, 0xCC, 0x86, 0xCA, 0x05, 0x49, + 0xCC, 0x88, 0xCC, 0x81, 0xCA, 0x05, 0x4C, 0xCC, + 0xA3, 0xCC, 0x84, 0xCA, 0x05, 0x4F, 0xCC, 0x82, + 0xCC, 0x80, 0xCA, 0x05, 0x4F, 0xCC, 0x82, 0xCC, + 0x81, 0xCA, 0x05, 0x4F, 0xCC, 0x82, 0xCC, 0x83, + 0xCA, 0x05, 0x4F, 0xCC, 0x82, 0xCC, 0x89, 0xCA, + 0x05, 0x4F, 0xCC, 0x83, 0xCC, 0x81, 0xCA, 0x05, + // Bytes 3940 - 397f + 0x4F, 0xCC, 0x83, 0xCC, 0x84, 0xCA, 0x05, 0x4F, + 0xCC, 0x83, 0xCC, 0x88, 0xCA, 0x05, 0x4F, 0xCC, + 0x84, 0xCC, 0x80, 0xCA, 0x05, 0x4F, 0xCC, 0x84, + 0xCC, 0x81, 0xCA, 0x05, 0x4F, 0xCC, 0x87, 0xCC, + 0x84, 0xCA, 0x05, 0x4F, 0xCC, 0x88, 0xCC, 0x84, + 0xCA, 0x05, 0x4F, 0xCC, 0x9B, 0xCC, 0x80, 0xCA, + 0x05, 0x4F, 0xCC, 0x9B, 0xCC, 0x81, 0xCA, 0x05, + 0x4F, 0xCC, 0x9B, 0xCC, 0x83, 0xCA, 0x05, 0x4F, + // Bytes 3980 - 39bf + 0xCC, 0x9B, 0xCC, 0x89, 0xCA, 0x05, 0x4F, 0xCC, + 0x9B, 0xCC, 0xA3, 0xB6, 0x05, 0x4F, 0xCC, 0xA3, + 0xCC, 0x82, 0xCA, 0x05, 0x4F, 0xCC, 0xA8, 0xCC, + 0x84, 0xCA, 0x05, 0x52, 0xCC, 0xA3, 0xCC, 0x84, + 0xCA, 0x05, 0x53, 0xCC, 0x81, 0xCC, 0x87, 0xCA, + 0x05, 0x53, 0xCC, 0x8C, 0xCC, 0x87, 0xCA, 0x05, + 0x53, 0xCC, 0xA3, 0xCC, 0x87, 0xCA, 0x05, 0x55, + 0xCC, 0x83, 0xCC, 0x81, 0xCA, 0x05, 0x55, 0xCC, + // Bytes 39c0 - 39ff + 0x84, 0xCC, 0x88, 0xCA, 0x05, 0x55, 0xCC, 0x88, + 0xCC, 0x80, 0xCA, 0x05, 0x55, 0xCC, 0x88, 0xCC, + 0x81, 0xCA, 0x05, 0x55, 0xCC, 0x88, 0xCC, 0x84, + 0xCA, 0x05, 0x55, 0xCC, 0x88, 0xCC, 0x8C, 0xCA, + 0x05, 0x55, 0xCC, 0x9B, 0xCC, 0x80, 0xCA, 0x05, + 0x55, 0xCC, 0x9B, 0xCC, 0x81, 0xCA, 0x05, 0x55, + 0xCC, 0x9B, 0xCC, 0x83, 0xCA, 0x05, 0x55, 0xCC, + 0x9B, 0xCC, 0x89, 0xCA, 0x05, 0x55, 0xCC, 0x9B, + // Bytes 3a00 - 3a3f + 0xCC, 0xA3, 0xB6, 0x05, 0x61, 0xCC, 0x82, 0xCC, + 0x80, 0xCA, 0x05, 0x61, 0xCC, 0x82, 0xCC, 0x81, + 0xCA, 0x05, 0x61, 0xCC, 0x82, 0xCC, 0x83, 0xCA, + 0x05, 0x61, 0xCC, 0x82, 0xCC, 0x89, 0xCA, 0x05, + 0x61, 0xCC, 0x86, 0xCC, 0x80, 0xCA, 0x05, 0x61, + 0xCC, 0x86, 0xCC, 0x81, 0xCA, 0x05, 0x61, 0xCC, + 0x86, 0xCC, 0x83, 0xCA, 0x05, 0x61, 0xCC, 0x86, + 0xCC, 0x89, 0xCA, 0x05, 0x61, 0xCC, 0x87, 0xCC, + // Bytes 3a40 - 3a7f + 0x84, 0xCA, 0x05, 0x61, 0xCC, 0x88, 0xCC, 0x84, + 0xCA, 0x05, 0x61, 0xCC, 0x8A, 0xCC, 0x81, 0xCA, + 0x05, 0x61, 0xCC, 0xA3, 0xCC, 0x82, 0xCA, 0x05, + 0x61, 0xCC, 0xA3, 0xCC, 0x86, 0xCA, 0x05, 0x63, + 0xCC, 0xA7, 0xCC, 0x81, 0xCA, 0x05, 0x65, 0xCC, + 0x82, 0xCC, 0x80, 0xCA, 0x05, 0x65, 0xCC, 0x82, + 0xCC, 0x81, 0xCA, 0x05, 0x65, 0xCC, 0x82, 0xCC, + 0x83, 0xCA, 0x05, 0x65, 0xCC, 0x82, 0xCC, 0x89, + // Bytes 3a80 - 3abf + 0xCA, 0x05, 0x65, 0xCC, 0x84, 0xCC, 0x80, 0xCA, + 0x05, 0x65, 0xCC, 0x84, 0xCC, 0x81, 0xCA, 0x05, + 0x65, 0xCC, 0xA3, 0xCC, 0x82, 0xCA, 0x05, 0x65, + 0xCC, 0xA7, 0xCC, 0x86, 0xCA, 0x05, 0x69, 0xCC, + 0x88, 0xCC, 0x81, 0xCA, 0x05, 0x6C, 0xCC, 0xA3, + 0xCC, 0x84, 0xCA, 0x05, 0x6F, 0xCC, 0x82, 0xCC, + 0x80, 0xCA, 0x05, 0x6F, 0xCC, 0x82, 0xCC, 0x81, + 0xCA, 0x05, 0x6F, 0xCC, 0x82, 0xCC, 0x83, 0xCA, + // Bytes 3ac0 - 3aff + 0x05, 0x6F, 0xCC, 0x82, 0xCC, 0x89, 0xCA, 0x05, + 0x6F, 0xCC, 0x83, 0xCC, 0x81, 0xCA, 0x05, 0x6F, + 0xCC, 0x83, 0xCC, 0x84, 0xCA, 0x05, 0x6F, 0xCC, + 0x83, 0xCC, 0x88, 0xCA, 0x05, 0x6F, 0xCC, 0x84, + 0xCC, 0x80, 0xCA, 0x05, 0x6F, 0xCC, 0x84, 0xCC, + 0x81, 0xCA, 0x05, 0x6F, 0xCC, 0x87, 0xCC, 0x84, + 0xCA, 0x05, 0x6F, 0xCC, 0x88, 0xCC, 0x84, 0xCA, + 0x05, 0x6F, 0xCC, 0x9B, 0xCC, 0x80, 0xCA, 0x05, + // Bytes 3b00 - 3b3f + 0x6F, 0xCC, 0x9B, 0xCC, 0x81, 0xCA, 0x05, 0x6F, + 0xCC, 0x9B, 0xCC, 0x83, 0xCA, 0x05, 0x6F, 0xCC, + 0x9B, 0xCC, 0x89, 0xCA, 0x05, 0x6F, 0xCC, 0x9B, + 0xCC, 0xA3, 0xB6, 0x05, 0x6F, 0xCC, 0xA3, 0xCC, + 0x82, 0xCA, 0x05, 0x6F, 0xCC, 0xA8, 0xCC, 0x84, + 0xCA, 0x05, 0x72, 0xCC, 0xA3, 0xCC, 0x84, 0xCA, + 0x05, 0x73, 0xCC, 0x81, 0xCC, 0x87, 0xCA, 0x05, + 0x73, 0xCC, 0x8C, 0xCC, 0x87, 0xCA, 0x05, 0x73, + // Bytes 3b40 - 3b7f + 0xCC, 0xA3, 0xCC, 0x87, 0xCA, 0x05, 0x75, 0xCC, + 0x83, 0xCC, 0x81, 0xCA, 0x05, 0x75, 0xCC, 0x84, + 0xCC, 0x88, 0xCA, 0x05, 0x75, 0xCC, 0x88, 0xCC, + 0x80, 0xCA, 0x05, 0x75, 0xCC, 0x88, 0xCC, 0x81, + 0xCA, 0x05, 0x75, 0xCC, 0x88, 0xCC, 0x84, 0xCA, + 0x05, 0x75, 0xCC, 0x88, 0xCC, 0x8C, 0xCA, 0x05, + 0x75, 0xCC, 0x9B, 0xCC, 0x80, 0xCA, 0x05, 0x75, + 0xCC, 0x9B, 0xCC, 0x81, 0xCA, 0x05, 0x75, 0xCC, + // Bytes 3b80 - 3bbf + 0x9B, 0xCC, 0x83, 0xCA, 0x05, 0x75, 0xCC, 0x9B, + 0xCC, 0x89, 0xCA, 0x05, 0x75, 0xCC, 0x9B, 0xCC, + 0xA3, 0xB6, 0x05, 0xE1, 0xBE, 0xBF, 0xCC, 0x80, + 0xCA, 0x05, 0xE1, 0xBE, 0xBF, 0xCC, 0x81, 0xCA, + 0x05, 0xE1, 0xBE, 0xBF, 0xCD, 0x82, 0xCA, 0x05, + 0xE1, 0xBF, 0xBE, 0xCC, 0x80, 0xCA, 0x05, 0xE1, + 0xBF, 0xBE, 0xCC, 0x81, 0xCA, 0x05, 0xE1, 0xBF, + 0xBE, 0xCD, 0x82, 0xCA, 0x05, 0xE2, 0x86, 0x90, + // Bytes 3bc0 - 3bff + 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x86, 0x92, 0xCC, + 0xB8, 0x05, 0x05, 0xE2, 0x86, 0x94, 0xCC, 0xB8, + 0x05, 0x05, 0xE2, 0x87, 0x90, 0xCC, 0xB8, 0x05, + 0x05, 0xE2, 0x87, 0x92, 0xCC, 0xB8, 0x05, 0x05, + 0xE2, 0x87, 0x94, 0xCC, 0xB8, 0x05, 0x05, 0xE2, + 0x88, 0x83, 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x88, + 0x88, 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x88, 0x8B, + 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x88, 0xA3, 0xCC, + // Bytes 3c00 - 3c3f + 0xB8, 0x05, 0x05, 0xE2, 0x88, 0xA5, 0xCC, 0xB8, + 0x05, 0x05, 0xE2, 0x88, 0xBC, 0xCC, 0xB8, 0x05, + 0x05, 0xE2, 0x89, 0x83, 0xCC, 0xB8, 0x05, 0x05, + 0xE2, 0x89, 0x85, 0xCC, 0xB8, 0x05, 0x05, 0xE2, + 0x89, 0x88, 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x89, + 0x8D, 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x89, 0xA1, + 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x89, 0xA4, 0xCC, + 0xB8, 0x05, 0x05, 0xE2, 0x89, 0xA5, 0xCC, 0xB8, + // Bytes 3c40 - 3c7f + 0x05, 0x05, 0xE2, 0x89, 0xB2, 0xCC, 0xB8, 0x05, + 0x05, 0xE2, 0x89, 0xB3, 0xCC, 0xB8, 0x05, 0x05, + 0xE2, 0x89, 0xB6, 0xCC, 0xB8, 0x05, 0x05, 0xE2, + 0x89, 0xB7, 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x89, + 0xBA, 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x89, 0xBB, + 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x89, 0xBC, 0xCC, + 0xB8, 0x05, 0x05, 0xE2, 0x89, 0xBD, 0xCC, 0xB8, + 0x05, 0x05, 0xE2, 0x8A, 0x82, 0xCC, 0xB8, 0x05, + // Bytes 3c80 - 3cbf + 0x05, 0xE2, 0x8A, 0x83, 0xCC, 0xB8, 0x05, 0x05, + 0xE2, 0x8A, 0x86, 0xCC, 0xB8, 0x05, 0x05, 0xE2, + 0x8A, 0x87, 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x8A, + 0x91, 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x8A, 0x92, + 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x8A, 0xA2, 0xCC, + 0xB8, 0x05, 0x05, 0xE2, 0x8A, 0xA8, 0xCC, 0xB8, + 0x05, 0x05, 0xE2, 0x8A, 0xA9, 0xCC, 0xB8, 0x05, + 0x05, 0xE2, 0x8A, 0xAB, 0xCC, 0xB8, 0x05, 0x05, + // Bytes 3cc0 - 3cff + 0xE2, 0x8A, 0xB2, 0xCC, 0xB8, 0x05, 0x05, 0xE2, + 0x8A, 0xB3, 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x8A, + 0xB4, 0xCC, 0xB8, 0x05, 0x05, 0xE2, 0x8A, 0xB5, + 0xCC, 0xB8, 0x05, 0x06, 0xCE, 0x91, 0xCC, 0x93, + 0xCD, 0x85, 0xDA, 0x06, 0xCE, 0x91, 0xCC, 0x94, + 0xCD, 0x85, 0xDA, 0x06, 0xCE, 0x95, 0xCC, 0x93, + 0xCC, 0x80, 0xCA, 0x06, 0xCE, 0x95, 0xCC, 0x93, + 0xCC, 0x81, 0xCA, 0x06, 0xCE, 0x95, 0xCC, 0x94, + // Bytes 3d00 - 3d3f + 0xCC, 0x80, 0xCA, 0x06, 0xCE, 0x95, 0xCC, 0x94, + 0xCC, 0x81, 0xCA, 0x06, 0xCE, 0x97, 0xCC, 0x93, + 0xCD, 0x85, 0xDA, 0x06, 0xCE, 0x97, 0xCC, 0x94, + 0xCD, 0x85, 0xDA, 0x06, 0xCE, 0x99, 0xCC, 0x93, + 0xCC, 0x80, 0xCA, 0x06, 0xCE, 0x99, 0xCC, 0x93, + 0xCC, 0x81, 0xCA, 0x06, 0xCE, 0x99, 0xCC, 0x93, + 0xCD, 0x82, 0xCA, 0x06, 0xCE, 0x99, 0xCC, 0x94, + 0xCC, 0x80, 0xCA, 0x06, 0xCE, 0x99, 0xCC, 0x94, + // Bytes 3d40 - 3d7f + 0xCC, 0x81, 0xCA, 0x06, 0xCE, 0x99, 0xCC, 0x94, + 0xCD, 0x82, 0xCA, 0x06, 0xCE, 0x9F, 0xCC, 0x93, + 0xCC, 0x80, 0xCA, 0x06, 0xCE, 0x9F, 0xCC, 0x93, + 0xCC, 0x81, 0xCA, 0x06, 0xCE, 0x9F, 0xCC, 0x94, + 0xCC, 0x80, 0xCA, 0x06, 0xCE, 0x9F, 0xCC, 0x94, + 0xCC, 0x81, 0xCA, 0x06, 0xCE, 0xA5, 0xCC, 0x94, + 0xCC, 0x80, 0xCA, 0x06, 0xCE, 0xA5, 0xCC, 0x94, + 0xCC, 0x81, 0xCA, 0x06, 0xCE, 0xA5, 0xCC, 0x94, + // Bytes 3d80 - 3dbf + 0xCD, 0x82, 0xCA, 0x06, 0xCE, 0xA9, 0xCC, 0x93, + 0xCD, 0x85, 0xDA, 0x06, 0xCE, 0xA9, 0xCC, 0x94, + 0xCD, 0x85, 0xDA, 0x06, 0xCE, 0xB1, 0xCC, 0x80, + 0xCD, 0x85, 0xDA, 0x06, 0xCE, 0xB1, 0xCC, 0x81, + 0xCD, 0x85, 0xDA, 0x06, 0xCE, 0xB1, 0xCC, 0x93, + 0xCD, 0x85, 0xDA, 0x06, 0xCE, 0xB1, 0xCC, 0x94, + 0xCD, 0x85, 0xDA, 0x06, 0xCE, 0xB1, 0xCD, 0x82, + 0xCD, 0x85, 0xDA, 0x06, 0xCE, 0xB5, 0xCC, 0x93, + // Bytes 3dc0 - 3dff + 0xCC, 0x80, 0xCA, 0x06, 0xCE, 0xB5, 0xCC, 0x93, + 0xCC, 0x81, 0xCA, 0x06, 0xCE, 0xB5, 0xCC, 0x94, + 0xCC, 0x80, 0xCA, 0x06, 0xCE, 0xB5, 0xCC, 0x94, + 0xCC, 0x81, 0xCA, 0x06, 0xCE, 0xB7, 0xCC, 0x80, + 0xCD, 0x85, 0xDA, 0x06, 0xCE, 0xB7, 0xCC, 0x81, + 0xCD, 0x85, 0xDA, 0x06, 0xCE, 0xB7, 0xCC, 0x93, + 0xCD, 0x85, 0xDA, 0x06, 0xCE, 0xB7, 0xCC, 0x94, + 0xCD, 0x85, 0xDA, 0x06, 0xCE, 0xB7, 0xCD, 0x82, + // Bytes 3e00 - 3e3f + 0xCD, 0x85, 0xDA, 0x06, 0xCE, 0xB9, 0xCC, 0x88, + 0xCC, 0x80, 0xCA, 0x06, 0xCE, 0xB9, 0xCC, 0x88, + 0xCC, 0x81, 0xCA, 0x06, 0xCE, 0xB9, 0xCC, 0x88, + 0xCD, 0x82, 0xCA, 0x06, 0xCE, 0xB9, 0xCC, 0x93, + 0xCC, 0x80, 0xCA, 0x06, 0xCE, 0xB9, 0xCC, 0x93, + 0xCC, 0x81, 0xCA, 0x06, 0xCE, 0xB9, 0xCC, 0x93, + 0xCD, 0x82, 0xCA, 0x06, 0xCE, 0xB9, 0xCC, 0x94, + 0xCC, 0x80, 0xCA, 0x06, 0xCE, 0xB9, 0xCC, 0x94, + // Bytes 3e40 - 3e7f + 0xCC, 0x81, 0xCA, 0x06, 0xCE, 0xB9, 0xCC, 0x94, + 0xCD, 0x82, 0xCA, 0x06, 0xCE, 0xBF, 0xCC, 0x93, + 0xCC, 0x80, 0xCA, 0x06, 0xCE, 0xBF, 0xCC, 0x93, + 0xCC, 0x81, 0xCA, 0x06, 0xCE, 0xBF, 0xCC, 0x94, + 0xCC, 0x80, 0xCA, 0x06, 0xCE, 0xBF, 0xCC, 0x94, + 0xCC, 0x81, 0xCA, 0x06, 0xCF, 0x85, 0xCC, 0x88, + 0xCC, 0x80, 0xCA, 0x06, 0xCF, 0x85, 0xCC, 0x88, + 0xCC, 0x81, 0xCA, 0x06, 0xCF, 0x85, 0xCC, 0x88, + // Bytes 3e80 - 3ebf + 0xCD, 0x82, 0xCA, 0x06, 0xCF, 0x85, 0xCC, 0x93, + 0xCC, 0x80, 0xCA, 0x06, 0xCF, 0x85, 0xCC, 0x93, + 0xCC, 0x81, 0xCA, 0x06, 0xCF, 0x85, 0xCC, 0x93, + 0xCD, 0x82, 0xCA, 0x06, 0xCF, 0x85, 0xCC, 0x94, + 0xCC, 0x80, 0xCA, 0x06, 0xCF, 0x85, 0xCC, 0x94, + 0xCC, 0x81, 0xCA, 0x06, 0xCF, 0x85, 0xCC, 0x94, + 0xCD, 0x82, 0xCA, 0x06, 0xCF, 0x89, 0xCC, 0x80, + 0xCD, 0x85, 0xDA, 0x06, 0xCF, 0x89, 0xCC, 0x81, + // Bytes 3ec0 - 3eff + 0xCD, 0x85, 0xDA, 0x06, 0xCF, 0x89, 0xCC, 0x93, + 0xCD, 0x85, 0xDA, 0x06, 0xCF, 0x89, 0xCC, 0x94, + 0xCD, 0x85, 0xDA, 0x06, 0xCF, 0x89, 0xCD, 0x82, + 0xCD, 0x85, 0xDA, 0x06, 0xE0, 0xA4, 0xA8, 0xE0, + 0xA4, 0xBC, 0x09, 0x06, 0xE0, 0xA4, 0xB0, 0xE0, + 0xA4, 0xBC, 0x09, 0x06, 0xE0, 0xA4, 0xB3, 0xE0, + 0xA4, 0xBC, 0x09, 0x06, 0xE0, 0xB1, 0x86, 0xE0, + 0xB1, 0x96, 0x85, 0x06, 0xE0, 0xB7, 0x99, 0xE0, + // Bytes 3f00 - 3f3f + 0xB7, 0x8A, 0x11, 0x06, 0xE3, 0x81, 0x86, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x81, 0x8B, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x81, 0x8D, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x81, 0x8F, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x81, 0x91, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x81, 0x93, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x81, 0x95, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x81, 0x97, 0xE3, + // Bytes 3f40 - 3f7f + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x81, 0x99, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x81, 0x9B, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x81, 0x9D, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x81, 0x9F, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x81, 0xA1, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x81, 0xA4, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x81, 0xA6, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x81, 0xA8, 0xE3, + // Bytes 3f80 - 3fbf + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x81, 0xAF, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x81, 0xAF, 0xE3, + 0x82, 0x9A, 0x0D, 0x06, 0xE3, 0x81, 0xB2, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x81, 0xB2, 0xE3, + 0x82, 0x9A, 0x0D, 0x06, 0xE3, 0x81, 0xB5, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x81, 0xB5, 0xE3, + 0x82, 0x9A, 0x0D, 0x06, 0xE3, 0x81, 0xB8, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x81, 0xB8, 0xE3, + // Bytes 3fc0 - 3fff + 0x82, 0x9A, 0x0D, 0x06, 0xE3, 0x81, 0xBB, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x81, 0xBB, 0xE3, + 0x82, 0x9A, 0x0D, 0x06, 0xE3, 0x82, 0x9D, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x82, 0xA6, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x82, 0xAB, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x82, 0xAD, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x82, 0xAF, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x82, 0xB1, 0xE3, + // Bytes 4000 - 403f + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x82, 0xB3, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x82, 0xB5, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x82, 0xB7, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x82, 0xB9, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x82, 0xBB, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x82, 0xBD, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x82, 0xBF, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x83, 0x81, 0xE3, + // Bytes 4040 - 407f + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x83, 0x84, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x83, 0x86, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x83, 0x88, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x83, 0x8F, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x83, 0x8F, 0xE3, + 0x82, 0x9A, 0x0D, 0x06, 0xE3, 0x83, 0x92, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x83, 0x92, 0xE3, + 0x82, 0x9A, 0x0D, 0x06, 0xE3, 0x83, 0x95, 0xE3, + // Bytes 4080 - 40bf + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x83, 0x95, 0xE3, + 0x82, 0x9A, 0x0D, 0x06, 0xE3, 0x83, 0x98, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x83, 0x98, 0xE3, + 0x82, 0x9A, 0x0D, 0x06, 0xE3, 0x83, 0x9B, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x83, 0x9B, 0xE3, + 0x82, 0x9A, 0x0D, 0x06, 0xE3, 0x83, 0xAF, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x83, 0xB0, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x83, 0xB1, 0xE3, + // Bytes 40c0 - 40ff + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x83, 0xB2, 0xE3, + 0x82, 0x99, 0x0D, 0x06, 0xE3, 0x83, 0xBD, 0xE3, + 0x82, 0x99, 0x0D, 0x08, 0xCE, 0x91, 0xCC, 0x93, + 0xCC, 0x80, 0xCD, 0x85, 0xDB, 0x08, 0xCE, 0x91, + 0xCC, 0x93, 0xCC, 0x81, 0xCD, 0x85, 0xDB, 0x08, + 0xCE, 0x91, 0xCC, 0x93, 0xCD, 0x82, 0xCD, 0x85, + 0xDB, 0x08, 0xCE, 0x91, 0xCC, 0x94, 0xCC, 0x80, + 0xCD, 0x85, 0xDB, 0x08, 0xCE, 0x91, 0xCC, 0x94, + // Bytes 4100 - 413f + 0xCC, 0x81, 0xCD, 0x85, 0xDB, 0x08, 0xCE, 0x91, + 0xCC, 0x94, 0xCD, 0x82, 0xCD, 0x85, 0xDB, 0x08, + 0xCE, 0x97, 0xCC, 0x93, 0xCC, 0x80, 0xCD, 0x85, + 0xDB, 0x08, 0xCE, 0x97, 0xCC, 0x93, 0xCC, 0x81, + 0xCD, 0x85, 0xDB, 0x08, 0xCE, 0x97, 0xCC, 0x93, + 0xCD, 0x82, 0xCD, 0x85, 0xDB, 0x08, 0xCE, 0x97, + 0xCC, 0x94, 0xCC, 0x80, 0xCD, 0x85, 0xDB, 0x08, + 0xCE, 0x97, 0xCC, 0x94, 0xCC, 0x81, 0xCD, 0x85, + // Bytes 4140 - 417f + 0xDB, 0x08, 0xCE, 0x97, 0xCC, 0x94, 0xCD, 0x82, + 0xCD, 0x85, 0xDB, 0x08, 0xCE, 0xA9, 0xCC, 0x93, + 0xCC, 0x80, 0xCD, 0x85, 0xDB, 0x08, 0xCE, 0xA9, + 0xCC, 0x93, 0xCC, 0x81, 0xCD, 0x85, 0xDB, 0x08, + 0xCE, 0xA9, 0xCC, 0x93, 0xCD, 0x82, 0xCD, 0x85, + 0xDB, 0x08, 0xCE, 0xA9, 0xCC, 0x94, 0xCC, 0x80, + 0xCD, 0x85, 0xDB, 0x08, 0xCE, 0xA9, 0xCC, 0x94, + 0xCC, 0x81, 0xCD, 0x85, 0xDB, 0x08, 0xCE, 0xA9, + // Bytes 4180 - 41bf + 0xCC, 0x94, 0xCD, 0x82, 0xCD, 0x85, 0xDB, 0x08, + 0xCE, 0xB1, 0xCC, 0x93, 0xCC, 0x80, 0xCD, 0x85, + 0xDB, 0x08, 0xCE, 0xB1, 0xCC, 0x93, 0xCC, 0x81, + 0xCD, 0x85, 0xDB, 0x08, 0xCE, 0xB1, 0xCC, 0x93, + 0xCD, 0x82, 0xCD, 0x85, 0xDB, 0x08, 0xCE, 0xB1, + 0xCC, 0x94, 0xCC, 0x80, 0xCD, 0x85, 0xDB, 0x08, + 0xCE, 0xB1, 0xCC, 0x94, 0xCC, 0x81, 0xCD, 0x85, + 0xDB, 0x08, 0xCE, 0xB1, 0xCC, 0x94, 0xCD, 0x82, + // Bytes 41c0 - 41ff + 0xCD, 0x85, 0xDB, 0x08, 0xCE, 0xB7, 0xCC, 0x93, + 0xCC, 0x80, 0xCD, 0x85, 0xDB, 0x08, 0xCE, 0xB7, + 0xCC, 0x93, 0xCC, 0x81, 0xCD, 0x85, 0xDB, 0x08, + 0xCE, 0xB7, 0xCC, 0x93, 0xCD, 0x82, 0xCD, 0x85, + 0xDB, 0x08, 0xCE, 0xB7, 0xCC, 0x94, 0xCC, 0x80, + 0xCD, 0x85, 0xDB, 0x08, 0xCE, 0xB7, 0xCC, 0x94, + 0xCC, 0x81, 0xCD, 0x85, 0xDB, 0x08, 0xCE, 0xB7, + 0xCC, 0x94, 0xCD, 0x82, 0xCD, 0x85, 0xDB, 0x08, + // Bytes 4200 - 423f + 0xCF, 0x89, 0xCC, 0x93, 0xCC, 0x80, 0xCD, 0x85, + 0xDB, 0x08, 0xCF, 0x89, 0xCC, 0x93, 0xCC, 0x81, + 0xCD, 0x85, 0xDB, 0x08, 0xCF, 0x89, 0xCC, 0x93, + 0xCD, 0x82, 0xCD, 0x85, 0xDB, 0x08, 0xCF, 0x89, + 0xCC, 0x94, 0xCC, 0x80, 0xCD, 0x85, 0xDB, 0x08, + 0xCF, 0x89, 0xCC, 0x94, 0xCC, 0x81, 0xCD, 0x85, + 0xDB, 0x08, 0xCF, 0x89, 0xCC, 0x94, 0xCD, 0x82, + 0xCD, 0x85, 0xDB, 0x08, 0xF0, 0x91, 0x82, 0x99, + // Bytes 4240 - 427f + 0xF0, 0x91, 0x82, 0xBA, 0x09, 0x08, 0xF0, 0x91, + 0x82, 0x9B, 0xF0, 0x91, 0x82, 0xBA, 0x09, 0x08, + 0xF0, 0x91, 0x82, 0xA5, 0xF0, 0x91, 0x82, 0xBA, + 0x09, 0x42, 0xC2, 0xB4, 0x01, 0x43, 0x20, 0xCC, + 0x81, 0xC9, 0x43, 0x20, 0xCC, 0x83, 0xC9, 0x43, + 0x20, 0xCC, 0x84, 0xC9, 0x43, 0x20, 0xCC, 0x85, + 0xC9, 0x43, 0x20, 0xCC, 0x86, 0xC9, 0x43, 0x20, + 0xCC, 0x87, 0xC9, 0x43, 0x20, 0xCC, 0x88, 0xC9, + // Bytes 4280 - 42bf + 0x43, 0x20, 0xCC, 0x8A, 0xC9, 0x43, 0x20, 0xCC, + 0x8B, 0xC9, 0x43, 0x20, 0xCC, 0x93, 0xC9, 0x43, + 0x20, 0xCC, 0x94, 0xC9, 0x43, 0x20, 0xCC, 0xA7, + 0xA5, 0x43, 0x20, 0xCC, 0xA8, 0xA5, 0x43, 0x20, + 0xCC, 0xB3, 0xB5, 0x43, 0x20, 0xCD, 0x82, 0xC9, + 0x43, 0x20, 0xCD, 0x85, 0xD9, 0x43, 0x20, 0xD9, + 0x8B, 0x59, 0x43, 0x20, 0xD9, 0x8C, 0x5D, 0x43, + 0x20, 0xD9, 0x8D, 0x61, 0x43, 0x20, 0xD9, 0x8E, + // Bytes 42c0 - 42ff + 0x65, 0x43, 0x20, 0xD9, 0x8F, 0x69, 0x43, 0x20, + 0xD9, 0x90, 0x6D, 0x43, 0x20, 0xD9, 0x91, 0x71, + 0x43, 0x20, 0xD9, 0x92, 0x75, 0x43, 0x41, 0xCC, + 0x8A, 0xC9, 0x43, 0x73, 0xCC, 0x87, 0xC9, 0x44, + 0x20, 0xE3, 0x82, 0x99, 0x0D, 0x44, 0x20, 0xE3, + 0x82, 0x9A, 0x0D, 0x44, 0xC2, 0xA8, 0xCC, 0x81, + 0xCA, 0x44, 0xCE, 0x91, 0xCC, 0x81, 0xC9, 0x44, + 0xCE, 0x95, 0xCC, 0x81, 0xC9, 0x44, 0xCE, 0x97, + // Bytes 4300 - 433f + 0xCC, 0x81, 0xC9, 0x44, 0xCE, 0x99, 0xCC, 0x81, + 0xC9, 0x44, 0xCE, 0x9F, 0xCC, 0x81, 0xC9, 0x44, + 0xCE, 0xA5, 0xCC, 0x81, 0xC9, 0x44, 0xCE, 0xA5, + 0xCC, 0x88, 0xC9, 0x44, 0xCE, 0xA9, 0xCC, 0x81, + 0xC9, 0x44, 0xCE, 0xB1, 0xCC, 0x81, 0xC9, 0x44, + 0xCE, 0xB5, 0xCC, 0x81, 0xC9, 0x44, 0xCE, 0xB7, + 0xCC, 0x81, 0xC9, 0x44, 0xCE, 0xB9, 0xCC, 0x81, + 0xC9, 0x44, 0xCE, 0xBF, 0xCC, 0x81, 0xC9, 0x44, + // Bytes 4340 - 437f + 0xCF, 0x85, 0xCC, 0x81, 0xC9, 0x44, 0xCF, 0x89, + 0xCC, 0x81, 0xC9, 0x44, 0xD7, 0x90, 0xD6, 0xB7, + 0x31, 0x44, 0xD7, 0x90, 0xD6, 0xB8, 0x35, 0x44, + 0xD7, 0x90, 0xD6, 0xBC, 0x41, 0x44, 0xD7, 0x91, + 0xD6, 0xBC, 0x41, 0x44, 0xD7, 0x91, 0xD6, 0xBF, + 0x49, 0x44, 0xD7, 0x92, 0xD6, 0xBC, 0x41, 0x44, + 0xD7, 0x93, 0xD6, 0xBC, 0x41, 0x44, 0xD7, 0x94, + 0xD6, 0xBC, 0x41, 0x44, 0xD7, 0x95, 0xD6, 0xB9, + // Bytes 4380 - 43bf + 0x39, 0x44, 0xD7, 0x95, 0xD6, 0xBC, 0x41, 0x44, + 0xD7, 0x96, 0xD6, 0xBC, 0x41, 0x44, 0xD7, 0x98, + 0xD6, 0xBC, 0x41, 0x44, 0xD7, 0x99, 0xD6, 0xB4, + 0x25, 0x44, 0xD7, 0x99, 0xD6, 0xBC, 0x41, 0x44, + 0xD7, 0x9A, 0xD6, 0xBC, 0x41, 0x44, 0xD7, 0x9B, + 0xD6, 0xBC, 0x41, 0x44, 0xD7, 0x9B, 0xD6, 0xBF, + 0x49, 0x44, 0xD7, 0x9C, 0xD6, 0xBC, 0x41, 0x44, + 0xD7, 0x9E, 0xD6, 0xBC, 0x41, 0x44, 0xD7, 0xA0, + // Bytes 43c0 - 43ff + 0xD6, 0xBC, 0x41, 0x44, 0xD7, 0xA1, 0xD6, 0xBC, + 0x41, 0x44, 0xD7, 0xA3, 0xD6, 0xBC, 0x41, 0x44, + 0xD7, 0xA4, 0xD6, 0xBC, 0x41, 0x44, 0xD7, 0xA4, + 0xD6, 0xBF, 0x49, 0x44, 0xD7, 0xA6, 0xD6, 0xBC, + 0x41, 0x44, 0xD7, 0xA7, 0xD6, 0xBC, 0x41, 0x44, + 0xD7, 0xA8, 0xD6, 0xBC, 0x41, 0x44, 0xD7, 0xA9, + 0xD6, 0xBC, 0x41, 0x44, 0xD7, 0xA9, 0xD7, 0x81, + 0x4D, 0x44, 0xD7, 0xA9, 0xD7, 0x82, 0x51, 0x44, + // Bytes 4400 - 443f + 0xD7, 0xAA, 0xD6, 0xBC, 0x41, 0x44, 0xD7, 0xB2, + 0xD6, 0xB7, 0x31, 0x44, 0xD8, 0xA7, 0xD9, 0x8B, + 0x59, 0x44, 0xD8, 0xA7, 0xD9, 0x93, 0xC9, 0x44, + 0xD8, 0xA7, 0xD9, 0x94, 0xC9, 0x44, 0xD8, 0xA7, + 0xD9, 0x95, 0xB5, 0x44, 0xD8, 0xB0, 0xD9, 0xB0, + 0x79, 0x44, 0xD8, 0xB1, 0xD9, 0xB0, 0x79, 0x44, + 0xD9, 0x80, 0xD9, 0x8B, 0x59, 0x44, 0xD9, 0x80, + 0xD9, 0x8E, 0x65, 0x44, 0xD9, 0x80, 0xD9, 0x8F, + // Bytes 4440 - 447f + 0x69, 0x44, 0xD9, 0x80, 0xD9, 0x90, 0x6D, 0x44, + 0xD9, 0x80, 0xD9, 0x91, 0x71, 0x44, 0xD9, 0x80, + 0xD9, 0x92, 0x75, 0x44, 0xD9, 0x87, 0xD9, 0xB0, + 0x79, 0x44, 0xD9, 0x88, 0xD9, 0x94, 0xC9, 0x44, + 0xD9, 0x89, 0xD9, 0xB0, 0x79, 0x44, 0xD9, 0x8A, + 0xD9, 0x94, 0xC9, 0x44, 0xDB, 0x92, 0xD9, 0x94, + 0xC9, 0x44, 0xDB, 0x95, 0xD9, 0x94, 0xC9, 0x45, + 0x20, 0xCC, 0x88, 0xCC, 0x80, 0xCA, 0x45, 0x20, + // Bytes 4480 - 44bf + 0xCC, 0x88, 0xCC, 0x81, 0xCA, 0x45, 0x20, 0xCC, + 0x88, 0xCD, 0x82, 0xCA, 0x45, 0x20, 0xCC, 0x93, + 0xCC, 0x80, 0xCA, 0x45, 0x20, 0xCC, 0x93, 0xCC, + 0x81, 0xCA, 0x45, 0x20, 0xCC, 0x93, 0xCD, 0x82, + 0xCA, 0x45, 0x20, 0xCC, 0x94, 0xCC, 0x80, 0xCA, + 0x45, 0x20, 0xCC, 0x94, 0xCC, 0x81, 0xCA, 0x45, + 0x20, 0xCC, 0x94, 0xCD, 0x82, 0xCA, 0x45, 0x20, + 0xD9, 0x8C, 0xD9, 0x91, 0x72, 0x45, 0x20, 0xD9, + // Bytes 44c0 - 44ff + 0x8D, 0xD9, 0x91, 0x72, 0x45, 0x20, 0xD9, 0x8E, + 0xD9, 0x91, 0x72, 0x45, 0x20, 0xD9, 0x8F, 0xD9, + 0x91, 0x72, 0x45, 0x20, 0xD9, 0x90, 0xD9, 0x91, + 0x72, 0x45, 0x20, 0xD9, 0x91, 0xD9, 0xB0, 0x7A, + 0x45, 0xE2, 0xAB, 0x9D, 0xCC, 0xB8, 0x05, 0x46, + 0xCE, 0xB9, 0xCC, 0x88, 0xCC, 0x81, 0xCA, 0x46, + 0xCF, 0x85, 0xCC, 0x88, 0xCC, 0x81, 0xCA, 0x46, + 0xD7, 0xA9, 0xD6, 0xBC, 0xD7, 0x81, 0x4E, 0x46, + // Bytes 4500 - 453f + 0xD7, 0xA9, 0xD6, 0xBC, 0xD7, 0x82, 0x52, 0x46, + 0xD9, 0x80, 0xD9, 0x8E, 0xD9, 0x91, 0x72, 0x46, + 0xD9, 0x80, 0xD9, 0x8F, 0xD9, 0x91, 0x72, 0x46, + 0xD9, 0x80, 0xD9, 0x90, 0xD9, 0x91, 0x72, 0x46, + 0xE0, 0xA4, 0x95, 0xE0, 0xA4, 0xBC, 0x09, 0x46, + 0xE0, 0xA4, 0x96, 0xE0, 0xA4, 0xBC, 0x09, 0x46, + 0xE0, 0xA4, 0x97, 0xE0, 0xA4, 0xBC, 0x09, 0x46, + 0xE0, 0xA4, 0x9C, 0xE0, 0xA4, 0xBC, 0x09, 0x46, + // Bytes 4540 - 457f + 0xE0, 0xA4, 0xA1, 0xE0, 0xA4, 0xBC, 0x09, 0x46, + 0xE0, 0xA4, 0xA2, 0xE0, 0xA4, 0xBC, 0x09, 0x46, + 0xE0, 0xA4, 0xAB, 0xE0, 0xA4, 0xBC, 0x09, 0x46, + 0xE0, 0xA4, 0xAF, 0xE0, 0xA4, 0xBC, 0x09, 0x46, + 0xE0, 0xA6, 0xA1, 0xE0, 0xA6, 0xBC, 0x09, 0x46, + 0xE0, 0xA6, 0xA2, 0xE0, 0xA6, 0xBC, 0x09, 0x46, + 0xE0, 0xA6, 0xAF, 0xE0, 0xA6, 0xBC, 0x09, 0x46, + 0xE0, 0xA8, 0x96, 0xE0, 0xA8, 0xBC, 0x09, 0x46, + // Bytes 4580 - 45bf + 0xE0, 0xA8, 0x97, 0xE0, 0xA8, 0xBC, 0x09, 0x46, + 0xE0, 0xA8, 0x9C, 0xE0, 0xA8, 0xBC, 0x09, 0x46, + 0xE0, 0xA8, 0xAB, 0xE0, 0xA8, 0xBC, 0x09, 0x46, + 0xE0, 0xA8, 0xB2, 0xE0, 0xA8, 0xBC, 0x09, 0x46, + 0xE0, 0xA8, 0xB8, 0xE0, 0xA8, 0xBC, 0x09, 0x46, + 0xE0, 0xAC, 0xA1, 0xE0, 0xAC, 0xBC, 0x09, 0x46, + 0xE0, 0xAC, 0xA2, 0xE0, 0xAC, 0xBC, 0x09, 0x46, + 0xE0, 0xBE, 0xB2, 0xE0, 0xBE, 0x80, 0x9D, 0x46, + // Bytes 45c0 - 45ff + 0xE0, 0xBE, 0xB3, 0xE0, 0xBE, 0x80, 0x9D, 0x46, + 0xE3, 0x83, 0x86, 0xE3, 0x82, 0x99, 0x0D, 0x48, + 0xF0, 0x9D, 0x85, 0x97, 0xF0, 0x9D, 0x85, 0xA5, + 0xAD, 0x48, 0xF0, 0x9D, 0x85, 0x98, 0xF0, 0x9D, + 0x85, 0xA5, 0xAD, 0x48, 0xF0, 0x9D, 0x86, 0xB9, + 0xF0, 0x9D, 0x85, 0xA5, 0xAD, 0x48, 0xF0, 0x9D, + 0x86, 0xBA, 0xF0, 0x9D, 0x85, 0xA5, 0xAD, 0x49, + 0xE0, 0xBE, 0xB2, 0xE0, 0xBD, 0xB1, 0xE0, 0xBE, + // Bytes 4600 - 463f + 0x80, 0x9E, 0x49, 0xE0, 0xBE, 0xB3, 0xE0, 0xBD, + 0xB1, 0xE0, 0xBE, 0x80, 0x9E, 0x4C, 0xF0, 0x9D, + 0x85, 0x98, 0xF0, 0x9D, 0x85, 0xA5, 0xF0, 0x9D, + 0x85, 0xAE, 0xAE, 0x4C, 0xF0, 0x9D, 0x85, 0x98, + 0xF0, 0x9D, 0x85, 0xA5, 0xF0, 0x9D, 0x85, 0xAF, + 0xAE, 0x4C, 0xF0, 0x9D, 0x85, 0x98, 0xF0, 0x9D, + 0x85, 0xA5, 0xF0, 0x9D, 0x85, 0xB0, 0xAE, 0x4C, + 0xF0, 0x9D, 0x85, 0x98, 0xF0, 0x9D, 0x85, 0xA5, + // Bytes 4640 - 467f + 0xF0, 0x9D, 0x85, 0xB1, 0xAE, 0x4C, 0xF0, 0x9D, + 0x85, 0x98, 0xF0, 0x9D, 0x85, 0xA5, 0xF0, 0x9D, + 0x85, 0xB2, 0xAE, 0x4C, 0xF0, 0x9D, 0x86, 0xB9, + 0xF0, 0x9D, 0x85, 0xA5, 0xF0, 0x9D, 0x85, 0xAE, + 0xAE, 0x4C, 0xF0, 0x9D, 0x86, 0xB9, 0xF0, 0x9D, + 0x85, 0xA5, 0xF0, 0x9D, 0x85, 0xAF, 0xAE, 0x4C, + 0xF0, 0x9D, 0x86, 0xBA, 0xF0, 0x9D, 0x85, 0xA5, + 0xF0, 0x9D, 0x85, 0xAE, 0xAE, 0x4C, 0xF0, 0x9D, + // Bytes 4680 - 46bf + 0x86, 0xBA, 0xF0, 0x9D, 0x85, 0xA5, 0xF0, 0x9D, + 0x85, 0xAF, 0xAE, 0x83, 0x41, 0xCC, 0x82, 0xC9, + 0x83, 0x41, 0xCC, 0x86, 0xC9, 0x83, 0x41, 0xCC, + 0x87, 0xC9, 0x83, 0x41, 0xCC, 0x88, 0xC9, 0x83, + 0x41, 0xCC, 0x8A, 0xC9, 0x83, 0x41, 0xCC, 0xA3, + 0xB5, 0x83, 0x43, 0xCC, 0xA7, 0xA5, 0x83, 0x45, + 0xCC, 0x82, 0xC9, 0x83, 0x45, 0xCC, 0x84, 0xC9, + 0x83, 0x45, 0xCC, 0xA3, 0xB5, 0x83, 0x45, 0xCC, + // Bytes 46c0 - 46ff + 0xA7, 0xA5, 0x83, 0x49, 0xCC, 0x88, 0xC9, 0x83, + 0x4C, 0xCC, 0xA3, 0xB5, 0x83, 0x4F, 0xCC, 0x82, + 0xC9, 0x83, 0x4F, 0xCC, 0x83, 0xC9, 0x83, 0x4F, + 0xCC, 0x84, 0xC9, 0x83, 0x4F, 0xCC, 0x87, 0xC9, + 0x83, 0x4F, 0xCC, 0x88, 0xC9, 0x83, 0x4F, 0xCC, + 0x9B, 0xAD, 0x83, 0x4F, 0xCC, 0xA3, 0xB5, 0x83, + 0x4F, 0xCC, 0xA8, 0xA5, 0x83, 0x52, 0xCC, 0xA3, + 0xB5, 0x83, 0x53, 0xCC, 0x81, 0xC9, 0x83, 0x53, + // Bytes 4700 - 473f + 0xCC, 0x8C, 0xC9, 0x83, 0x53, 0xCC, 0xA3, 0xB5, + 0x83, 0x55, 0xCC, 0x83, 0xC9, 0x83, 0x55, 0xCC, + 0x84, 0xC9, 0x83, 0x55, 0xCC, 0x88, 0xC9, 0x83, + 0x55, 0xCC, 0x9B, 0xAD, 0x83, 0x61, 0xCC, 0x82, + 0xC9, 0x83, 0x61, 0xCC, 0x86, 0xC9, 0x83, 0x61, + 0xCC, 0x87, 0xC9, 0x83, 0x61, 0xCC, 0x88, 0xC9, + 0x83, 0x61, 0xCC, 0x8A, 0xC9, 0x83, 0x61, 0xCC, + 0xA3, 0xB5, 0x83, 0x63, 0xCC, 0xA7, 0xA5, 0x83, + // Bytes 4740 - 477f + 0x65, 0xCC, 0x82, 0xC9, 0x83, 0x65, 0xCC, 0x84, + 0xC9, 0x83, 0x65, 0xCC, 0xA3, 0xB5, 0x83, 0x65, + 0xCC, 0xA7, 0xA5, 0x83, 0x69, 0xCC, 0x88, 0xC9, + 0x83, 0x6C, 0xCC, 0xA3, 0xB5, 0x83, 0x6F, 0xCC, + 0x82, 0xC9, 0x83, 0x6F, 0xCC, 0x83, 0xC9, 0x83, + 0x6F, 0xCC, 0x84, 0xC9, 0x83, 0x6F, 0xCC, 0x87, + 0xC9, 0x83, 0x6F, 0xCC, 0x88, 0xC9, 0x83, 0x6F, + 0xCC, 0x9B, 0xAD, 0x83, 0x6F, 0xCC, 0xA3, 0xB5, + // Bytes 4780 - 47bf + 0x83, 0x6F, 0xCC, 0xA8, 0xA5, 0x83, 0x72, 0xCC, + 0xA3, 0xB5, 0x83, 0x73, 0xCC, 0x81, 0xC9, 0x83, + 0x73, 0xCC, 0x8C, 0xC9, 0x83, 0x73, 0xCC, 0xA3, + 0xB5, 0x83, 0x75, 0xCC, 0x83, 0xC9, 0x83, 0x75, + 0xCC, 0x84, 0xC9, 0x83, 0x75, 0xCC, 0x88, 0xC9, + 0x83, 0x75, 0xCC, 0x9B, 0xAD, 0x84, 0xCE, 0x91, + 0xCC, 0x93, 0xC9, 0x84, 0xCE, 0x91, 0xCC, 0x94, + 0xC9, 0x84, 0xCE, 0x95, 0xCC, 0x93, 0xC9, 0x84, + // Bytes 47c0 - 47ff + 0xCE, 0x95, 0xCC, 0x94, 0xC9, 0x84, 0xCE, 0x97, + 0xCC, 0x93, 0xC9, 0x84, 0xCE, 0x97, 0xCC, 0x94, + 0xC9, 0x84, 0xCE, 0x99, 0xCC, 0x93, 0xC9, 0x84, + 0xCE, 0x99, 0xCC, 0x94, 0xC9, 0x84, 0xCE, 0x9F, + 0xCC, 0x93, 0xC9, 0x84, 0xCE, 0x9F, 0xCC, 0x94, + 0xC9, 0x84, 0xCE, 0xA5, 0xCC, 0x94, 0xC9, 0x84, + 0xCE, 0xA9, 0xCC, 0x93, 0xC9, 0x84, 0xCE, 0xA9, + 0xCC, 0x94, 0xC9, 0x84, 0xCE, 0xB1, 0xCC, 0x80, + // Bytes 4800 - 483f + 0xC9, 0x84, 0xCE, 0xB1, 0xCC, 0x81, 0xC9, 0x84, + 0xCE, 0xB1, 0xCC, 0x93, 0xC9, 0x84, 0xCE, 0xB1, + 0xCC, 0x94, 0xC9, 0x84, 0xCE, 0xB1, 0xCD, 0x82, + 0xC9, 0x84, 0xCE, 0xB5, 0xCC, 0x93, 0xC9, 0x84, + 0xCE, 0xB5, 0xCC, 0x94, 0xC9, 0x84, 0xCE, 0xB7, + 0xCC, 0x80, 0xC9, 0x84, 0xCE, 0xB7, 0xCC, 0x81, + 0xC9, 0x84, 0xCE, 0xB7, 0xCC, 0x93, 0xC9, 0x84, + 0xCE, 0xB7, 0xCC, 0x94, 0xC9, 0x84, 0xCE, 0xB7, + // Bytes 4840 - 487f + 0xCD, 0x82, 0xC9, 0x84, 0xCE, 0xB9, 0xCC, 0x88, + 0xC9, 0x84, 0xCE, 0xB9, 0xCC, 0x93, 0xC9, 0x84, + 0xCE, 0xB9, 0xCC, 0x94, 0xC9, 0x84, 0xCE, 0xBF, + 0xCC, 0x93, 0xC9, 0x84, 0xCE, 0xBF, 0xCC, 0x94, + 0xC9, 0x84, 0xCF, 0x85, 0xCC, 0x88, 0xC9, 0x84, + 0xCF, 0x85, 0xCC, 0x93, 0xC9, 0x84, 0xCF, 0x85, + 0xCC, 0x94, 0xC9, 0x84, 0xCF, 0x89, 0xCC, 0x80, + 0xC9, 0x84, 0xCF, 0x89, 0xCC, 0x81, 0xC9, 0x84, + // Bytes 4880 - 48bf + 0xCF, 0x89, 0xCC, 0x93, 0xC9, 0x84, 0xCF, 0x89, + 0xCC, 0x94, 0xC9, 0x84, 0xCF, 0x89, 0xCD, 0x82, + 0xC9, 0x86, 0xCE, 0x91, 0xCC, 0x93, 0xCC, 0x80, + 0xCA, 0x86, 0xCE, 0x91, 0xCC, 0x93, 0xCC, 0x81, + 0xCA, 0x86, 0xCE, 0x91, 0xCC, 0x93, 0xCD, 0x82, + 0xCA, 0x86, 0xCE, 0x91, 0xCC, 0x94, 0xCC, 0x80, + 0xCA, 0x86, 0xCE, 0x91, 0xCC, 0x94, 0xCC, 0x81, + 0xCA, 0x86, 0xCE, 0x91, 0xCC, 0x94, 0xCD, 0x82, + // Bytes 48c0 - 48ff + 0xCA, 0x86, 0xCE, 0x97, 0xCC, 0x93, 0xCC, 0x80, + 0xCA, 0x86, 0xCE, 0x97, 0xCC, 0x93, 0xCC, 0x81, + 0xCA, 0x86, 0xCE, 0x97, 0xCC, 0x93, 0xCD, 0x82, + 0xCA, 0x86, 0xCE, 0x97, 0xCC, 0x94, 0xCC, 0x80, + 0xCA, 0x86, 0xCE, 0x97, 0xCC, 0x94, 0xCC, 0x81, + 0xCA, 0x86, 0xCE, 0x97, 0xCC, 0x94, 0xCD, 0x82, + 0xCA, 0x86, 0xCE, 0xA9, 0xCC, 0x93, 0xCC, 0x80, + 0xCA, 0x86, 0xCE, 0xA9, 0xCC, 0x93, 0xCC, 0x81, + // Bytes 4900 - 493f + 0xCA, 0x86, 0xCE, 0xA9, 0xCC, 0x93, 0xCD, 0x82, + 0xCA, 0x86, 0xCE, 0xA9, 0xCC, 0x94, 0xCC, 0x80, + 0xCA, 0x86, 0xCE, 0xA9, 0xCC, 0x94, 0xCC, 0x81, + 0xCA, 0x86, 0xCE, 0xA9, 0xCC, 0x94, 0xCD, 0x82, + 0xCA, 0x86, 0xCE, 0xB1, 0xCC, 0x93, 0xCC, 0x80, + 0xCA, 0x86, 0xCE, 0xB1, 0xCC, 0x93, 0xCC, 0x81, + 0xCA, 0x86, 0xCE, 0xB1, 0xCC, 0x93, 0xCD, 0x82, + 0xCA, 0x86, 0xCE, 0xB1, 0xCC, 0x94, 0xCC, 0x80, + // Bytes 4940 - 497f + 0xCA, 0x86, 0xCE, 0xB1, 0xCC, 0x94, 0xCC, 0x81, + 0xCA, 0x86, 0xCE, 0xB1, 0xCC, 0x94, 0xCD, 0x82, + 0xCA, 0x86, 0xCE, 0xB7, 0xCC, 0x93, 0xCC, 0x80, + 0xCA, 0x86, 0xCE, 0xB7, 0xCC, 0x93, 0xCC, 0x81, + 0xCA, 0x86, 0xCE, 0xB7, 0xCC, 0x93, 0xCD, 0x82, + 0xCA, 0x86, 0xCE, 0xB7, 0xCC, 0x94, 0xCC, 0x80, + 0xCA, 0x86, 0xCE, 0xB7, 0xCC, 0x94, 0xCC, 0x81, + 0xCA, 0x86, 0xCE, 0xB7, 0xCC, 0x94, 0xCD, 0x82, + // Bytes 4980 - 49bf + 0xCA, 0x86, 0xCF, 0x89, 0xCC, 0x93, 0xCC, 0x80, + 0xCA, 0x86, 0xCF, 0x89, 0xCC, 0x93, 0xCC, 0x81, + 0xCA, 0x86, 0xCF, 0x89, 0xCC, 0x93, 0xCD, 0x82, + 0xCA, 0x86, 0xCF, 0x89, 0xCC, 0x94, 0xCC, 0x80, + 0xCA, 0x86, 0xCF, 0x89, 0xCC, 0x94, 0xCC, 0x81, + 0xCA, 0x86, 0xCF, 0x89, 0xCC, 0x94, 0xCD, 0x82, + 0xCA, 0x42, 0xCC, 0x80, 0xC9, 0x32, 0x42, 0xCC, + 0x81, 0xC9, 0x32, 0x42, 0xCC, 0x93, 0xC9, 0x32, + // Bytes 49c0 - 49ff + 0x43, 0xE1, 0x85, 0xA1, 0x01, 0x00, 0x43, 0xE1, + 0x85, 0xA2, 0x01, 0x00, 0x43, 0xE1, 0x85, 0xA3, + 0x01, 0x00, 0x43, 0xE1, 0x85, 0xA4, 0x01, 0x00, + 0x43, 0xE1, 0x85, 0xA5, 0x01, 0x00, 0x43, 0xE1, + 0x85, 0xA6, 0x01, 0x00, 0x43, 0xE1, 0x85, 0xA7, + 0x01, 0x00, 0x43, 0xE1, 0x85, 0xA8, 0x01, 0x00, + 0x43, 0xE1, 0x85, 0xA9, 0x01, 0x00, 0x43, 0xE1, + 0x85, 0xAA, 0x01, 0x00, 0x43, 0xE1, 0x85, 0xAB, + // Bytes 4a00 - 4a3f + 0x01, 0x00, 0x43, 0xE1, 0x85, 0xAC, 0x01, 0x00, + 0x43, 0xE1, 0x85, 0xAD, 0x01, 0x00, 0x43, 0xE1, + 0x85, 0xAE, 0x01, 0x00, 0x43, 0xE1, 0x85, 0xAF, + 0x01, 0x00, 0x43, 0xE1, 0x85, 0xB0, 0x01, 0x00, + 0x43, 0xE1, 0x85, 0xB1, 0x01, 0x00, 0x43, 0xE1, + 0x85, 0xB2, 0x01, 0x00, 0x43, 0xE1, 0x85, 0xB3, + 0x01, 0x00, 0x43, 0xE1, 0x85, 0xB4, 0x01, 0x00, + 0x43, 0xE1, 0x85, 0xB5, 0x01, 0x00, 0x43, 0xE1, + // Bytes 4a40 - 4a7f + 0x86, 0xAA, 0x01, 0x00, 0x43, 0xE1, 0x86, 0xAC, + 0x01, 0x00, 0x43, 0xE1, 0x86, 0xAD, 0x01, 0x00, + 0x43, 0xE1, 0x86, 0xB0, 0x01, 0x00, 0x43, 0xE1, + 0x86, 0xB1, 0x01, 0x00, 0x43, 0xE1, 0x86, 0xB2, + 0x01, 0x00, 0x43, 0xE1, 0x86, 0xB3, 0x01, 0x00, + 0x43, 0xE1, 0x86, 0xB4, 0x01, 0x00, 0x43, 0xE1, + 0x86, 0xB5, 0x01, 0x00, 0x44, 0xCC, 0x88, 0xCC, + 0x81, 0xCA, 0x32, 0x43, 0xE3, 0x82, 0x99, 0x0D, + // Bytes 4a80 - 4abf + 0x03, 0x43, 0xE3, 0x82, 0x9A, 0x0D, 0x03, 0x46, + 0xE0, 0xBD, 0xB1, 0xE0, 0xBD, 0xB2, 0x9E, 0x26, + 0x46, 0xE0, 0xBD, 0xB1, 0xE0, 0xBD, 0xB4, 0xA2, + 0x26, 0x46, 0xE0, 0xBD, 0xB1, 0xE0, 0xBE, 0x80, + 0x9E, 0x26, 0x00, 0x01, +} + +// lookup returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *nfcTrie) lookup(s []byte) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return nfcValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := nfcIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := nfcIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = nfcIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := nfcIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = nfcIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = nfcIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *nfcTrie) lookupUnsafe(s []byte) uint16 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return nfcValues[c0] + } + i := nfcIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = nfcIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = nfcIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// lookupString returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *nfcTrie) lookupString(s string) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return nfcValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := nfcIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := nfcIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = nfcIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := nfcIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = nfcIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = nfcIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *nfcTrie) lookupStringUnsafe(s string) uint16 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return nfcValues[c0] + } + i := nfcIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = nfcIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = nfcIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// nfcTrie. Total size: 10610 bytes (10.36 KiB). Checksum: 95e8869a9f81e5e6. +type nfcTrie struct{} + +func newNfcTrie(i int) *nfcTrie { + return &nfcTrie{} +} + +// lookupValue determines the type of block n and looks up the value for b. +func (t *nfcTrie) lookupValue(n uint32, b byte) uint16 { + switch { + case n < 46: + return uint16(nfcValues[n<<6+uint32(b)]) + default: + n -= 46 + return uint16(nfcSparse.lookup(n, b)) + } +} + +// nfcValues: 48 blocks, 3072 entries, 6144 bytes +// The third block is the zero block. +var nfcValues = [3072]uint16{ + // Block 0x0, offset 0x0 + 0x3c: 0xa000, 0x3d: 0xa000, 0x3e: 0xa000, + // Block 0x1, offset 0x40 + 0x41: 0xa000, 0x42: 0xa000, 0x43: 0xa000, 0x44: 0xa000, 0x45: 0xa000, + 0x46: 0xa000, 0x47: 0xa000, 0x48: 0xa000, 0x49: 0xa000, 0x4a: 0xa000, 0x4b: 0xa000, + 0x4c: 0xa000, 0x4d: 0xa000, 0x4e: 0xa000, 0x4f: 0xa000, 0x50: 0xa000, + 0x52: 0xa000, 0x53: 0xa000, 0x54: 0xa000, 0x55: 0xa000, 0x56: 0xa000, 0x57: 0xa000, + 0x58: 0xa000, 0x59: 0xa000, 0x5a: 0xa000, + 0x61: 0xa000, 0x62: 0xa000, 0x63: 0xa000, + 0x64: 0xa000, 0x65: 0xa000, 0x66: 0xa000, 0x67: 0xa000, 0x68: 0xa000, 0x69: 0xa000, + 0x6a: 0xa000, 0x6b: 0xa000, 0x6c: 0xa000, 0x6d: 0xa000, 0x6e: 0xa000, 0x6f: 0xa000, + 0x70: 0xa000, 0x72: 0xa000, 0x73: 0xa000, 0x74: 0xa000, 0x75: 0xa000, + 0x76: 0xa000, 0x77: 0xa000, 0x78: 0xa000, 0x79: 0xa000, 0x7a: 0xa000, + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xc0: 0x2f72, 0xc1: 0x2f77, 0xc2: 0x468b, 0xc3: 0x2f7c, 0xc4: 0x469a, 0xc5: 0x469f, + 0xc6: 0xa000, 0xc7: 0x46a9, 0xc8: 0x2fe5, 0xc9: 0x2fea, 0xca: 0x46ae, 0xcb: 0x2ffe, + 0xcc: 0x3071, 0xcd: 0x3076, 0xce: 0x307b, 0xcf: 0x46c2, 0xd1: 0x3107, + 0xd2: 0x312a, 0xd3: 0x312f, 0xd4: 0x46cc, 0xd5: 0x46d1, 0xd6: 0x46e0, + 0xd8: 0xa000, 0xd9: 0x31b6, 0xda: 0x31bb, 0xdb: 0x31c0, 0xdc: 0x4712, 0xdd: 0x3238, + 0xe0: 0x327e, 0xe1: 0x3283, 0xe2: 0x471c, 0xe3: 0x3288, + 0xe4: 0x472b, 0xe5: 0x4730, 0xe6: 0xa000, 0xe7: 0x473a, 0xe8: 0x32f1, 0xe9: 0x32f6, + 0xea: 0x473f, 0xeb: 0x330a, 0xec: 0x3382, 0xed: 0x3387, 0xee: 0x338c, 0xef: 0x4753, + 0xf1: 0x3418, 0xf2: 0x343b, 0xf3: 0x3440, 0xf4: 0x475d, 0xf5: 0x4762, + 0xf6: 0x4771, 0xf8: 0xa000, 0xf9: 0x34cc, 0xfa: 0x34d1, 0xfb: 0x34d6, + 0xfc: 0x47a3, 0xfd: 0x3553, 0xff: 0x356c, + // Block 0x4, offset 0x100 + 0x100: 0x2f81, 0x101: 0x328d, 0x102: 0x4690, 0x103: 0x4721, 0x104: 0x2f9f, 0x105: 0x32ab, + 0x106: 0x2fb3, 0x107: 0x32bf, 0x108: 0x2fb8, 0x109: 0x32c4, 0x10a: 0x2fbd, 0x10b: 0x32c9, + 0x10c: 0x2fc2, 0x10d: 0x32ce, 0x10e: 0x2fcc, 0x10f: 0x32d8, + 0x112: 0x46b3, 0x113: 0x4744, 0x114: 0x2ff4, 0x115: 0x3300, 0x116: 0x2ff9, 0x117: 0x3305, + 0x118: 0x3017, 0x119: 0x3323, 0x11a: 0x3008, 0x11b: 0x3314, 0x11c: 0x3030, 0x11d: 0x333c, + 0x11e: 0x303a, 0x11f: 0x3346, 0x120: 0x303f, 0x121: 0x334b, 0x122: 0x3049, 0x123: 0x3355, + 0x124: 0x304e, 0x125: 0x335a, 0x128: 0x3080, 0x129: 0x3391, + 0x12a: 0x3085, 0x12b: 0x3396, 0x12c: 0x308a, 0x12d: 0x339b, 0x12e: 0x30ad, 0x12f: 0x33b9, + 0x130: 0x308f, 0x134: 0x30b7, 0x135: 0x33c3, + 0x136: 0x30cb, 0x137: 0x33dc, 0x139: 0x30d5, 0x13a: 0x33e6, 0x13b: 0x30df, + 0x13c: 0x33f0, 0x13d: 0x30da, 0x13e: 0x33eb, + // Block 0x5, offset 0x140 + 0x143: 0x3102, 0x144: 0x3413, 0x145: 0x311b, + 0x146: 0x342c, 0x147: 0x3111, 0x148: 0x3422, + 0x14c: 0x46d6, 0x14d: 0x4767, 0x14e: 0x3134, 0x14f: 0x3445, 0x150: 0x313e, 0x151: 0x344f, + 0x154: 0x315c, 0x155: 0x346d, 0x156: 0x3175, 0x157: 0x3486, + 0x158: 0x3166, 0x159: 0x3477, 0x15a: 0x46f9, 0x15b: 0x478a, 0x15c: 0x317f, 0x15d: 0x3490, + 0x15e: 0x318e, 0x15f: 0x349f, 0x160: 0x46fe, 0x161: 0x478f, 0x162: 0x31a7, 0x163: 0x34bd, + 0x164: 0x3198, 0x165: 0x34ae, 0x168: 0x4708, 0x169: 0x4799, + 0x16a: 0x470d, 0x16b: 0x479e, 0x16c: 0x31c5, 0x16d: 0x34db, 0x16e: 0x31cf, 0x16f: 0x34e5, + 0x170: 0x31d4, 0x171: 0x34ea, 0x172: 0x31f2, 0x173: 0x3508, 0x174: 0x3215, 0x175: 0x352b, + 0x176: 0x323d, 0x177: 0x3558, 0x178: 0x3251, 0x179: 0x3260, 0x17a: 0x3580, 0x17b: 0x326a, + 0x17c: 0x358a, 0x17d: 0x326f, 0x17e: 0x358f, 0x17f: 0xa000, + // Block 0x6, offset 0x180 + 0x184: 0x8100, 0x185: 0x8100, + 0x186: 0x8100, + 0x18d: 0x2f8b, 0x18e: 0x3297, 0x18f: 0x3099, 0x190: 0x33a5, 0x191: 0x3143, + 0x192: 0x3454, 0x193: 0x31d9, 0x194: 0x34ef, 0x195: 0x39d2, 0x196: 0x3b61, 0x197: 0x39cb, + 0x198: 0x3b5a, 0x199: 0x39d9, 0x19a: 0x3b68, 0x19b: 0x39c4, 0x19c: 0x3b53, + 0x19e: 0x38b3, 0x19f: 0x3a42, 0x1a0: 0x38ac, 0x1a1: 0x3a3b, 0x1a2: 0x35b6, 0x1a3: 0x35c8, + 0x1a6: 0x3044, 0x1a7: 0x3350, 0x1a8: 0x30c1, 0x1a9: 0x33d2, + 0x1aa: 0x46ef, 0x1ab: 0x4780, 0x1ac: 0x3993, 0x1ad: 0x3b22, 0x1ae: 0x35da, 0x1af: 0x35e0, + 0x1b0: 0x33c8, 0x1b4: 0x302b, 0x1b5: 0x3337, + 0x1b8: 0x30fd, 0x1b9: 0x340e, 0x1ba: 0x38ba, 0x1bb: 0x3a49, + 0x1bc: 0x35b0, 0x1bd: 0x35c2, 0x1be: 0x35bc, 0x1bf: 0x35ce, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x2f90, 0x1c1: 0x329c, 0x1c2: 0x2f95, 0x1c3: 0x32a1, 0x1c4: 0x300d, 0x1c5: 0x3319, + 0x1c6: 0x3012, 0x1c7: 0x331e, 0x1c8: 0x309e, 0x1c9: 0x33aa, 0x1ca: 0x30a3, 0x1cb: 0x33af, + 0x1cc: 0x3148, 0x1cd: 0x3459, 0x1ce: 0x314d, 0x1cf: 0x345e, 0x1d0: 0x316b, 0x1d1: 0x347c, + 0x1d2: 0x3170, 0x1d3: 0x3481, 0x1d4: 0x31de, 0x1d5: 0x34f4, 0x1d6: 0x31e3, 0x1d7: 0x34f9, + 0x1d8: 0x3189, 0x1d9: 0x349a, 0x1da: 0x31a2, 0x1db: 0x34b8, + 0x1de: 0x305d, 0x1df: 0x3369, + 0x1e6: 0x4695, 0x1e7: 0x4726, 0x1e8: 0x46bd, 0x1e9: 0x474e, + 0x1ea: 0x3962, 0x1eb: 0x3af1, 0x1ec: 0x393f, 0x1ed: 0x3ace, 0x1ee: 0x46db, 0x1ef: 0x476c, + 0x1f0: 0x395b, 0x1f1: 0x3aea, 0x1f2: 0x3247, 0x1f3: 0x3562, + // Block 0x8, offset 0x200 + 0x200: 0x9932, 0x201: 0x9932, 0x202: 0x9932, 0x203: 0x9932, 0x204: 0x9932, 0x205: 0x8132, + 0x206: 0x9932, 0x207: 0x9932, 0x208: 0x9932, 0x209: 0x9932, 0x20a: 0x9932, 0x20b: 0x9932, + 0x20c: 0x9932, 0x20d: 0x8132, 0x20e: 0x8132, 0x20f: 0x9932, 0x210: 0x8132, 0x211: 0x9932, + 0x212: 0x8132, 0x213: 0x9932, 0x214: 0x9932, 0x215: 0x8133, 0x216: 0x812d, 0x217: 0x812d, + 0x218: 0x812d, 0x219: 0x812d, 0x21a: 0x8133, 0x21b: 0x992b, 0x21c: 0x812d, 0x21d: 0x812d, + 0x21e: 0x812d, 0x21f: 0x812d, 0x220: 0x812d, 0x221: 0x8129, 0x222: 0x8129, 0x223: 0x992d, + 0x224: 0x992d, 0x225: 0x992d, 0x226: 0x992d, 0x227: 0x9929, 0x228: 0x9929, 0x229: 0x812d, + 0x22a: 0x812d, 0x22b: 0x812d, 0x22c: 0x812d, 0x22d: 0x992d, 0x22e: 0x992d, 0x22f: 0x812d, + 0x230: 0x992d, 0x231: 0x992d, 0x232: 0x812d, 0x233: 0x812d, 0x234: 0x8101, 0x235: 0x8101, + 0x236: 0x8101, 0x237: 0x8101, 0x238: 0x9901, 0x239: 0x812d, 0x23a: 0x812d, 0x23b: 0x812d, + 0x23c: 0x812d, 0x23d: 0x8132, 0x23e: 0x8132, 0x23f: 0x8132, + // Block 0x9, offset 0x240 + 0x240: 0x49b1, 0x241: 0x49b6, 0x242: 0x9932, 0x243: 0x49bb, 0x244: 0x4a74, 0x245: 0x9936, + 0x246: 0x8132, 0x247: 0x812d, 0x248: 0x812d, 0x249: 0x812d, 0x24a: 0x8132, 0x24b: 0x8132, + 0x24c: 0x8132, 0x24d: 0x812d, 0x24e: 0x812d, 0x250: 0x8132, 0x251: 0x8132, + 0x252: 0x8132, 0x253: 0x812d, 0x254: 0x812d, 0x255: 0x812d, 0x256: 0x812d, 0x257: 0x8132, + 0x258: 0x8133, 0x259: 0x812d, 0x25a: 0x812d, 0x25b: 0x8132, 0x25c: 0x8134, 0x25d: 0x8135, + 0x25e: 0x8135, 0x25f: 0x8134, 0x260: 0x8135, 0x261: 0x8135, 0x262: 0x8134, 0x263: 0x8132, + 0x264: 0x8132, 0x265: 0x8132, 0x266: 0x8132, 0x267: 0x8132, 0x268: 0x8132, 0x269: 0x8132, + 0x26a: 0x8132, 0x26b: 0x8132, 0x26c: 0x8132, 0x26d: 0x8132, 0x26e: 0x8132, 0x26f: 0x8132, + 0x274: 0x0170, + 0x27a: 0x8100, + 0x27e: 0x0037, + // Block 0xa, offset 0x280 + 0x284: 0x8100, 0x285: 0x35a4, + 0x286: 0x35ec, 0x287: 0x00ce, 0x288: 0x360a, 0x289: 0x3616, 0x28a: 0x3628, + 0x28c: 0x3646, 0x28e: 0x3658, 0x28f: 0x3676, 0x290: 0x3e0b, 0x291: 0xa000, + 0x295: 0xa000, 0x297: 0xa000, + 0x299: 0xa000, + 0x29f: 0xa000, 0x2a1: 0xa000, + 0x2a5: 0xa000, 0x2a9: 0xa000, + 0x2aa: 0x363a, 0x2ab: 0x366a, 0x2ac: 0x4801, 0x2ad: 0x369a, 0x2ae: 0x482b, 0x2af: 0x36ac, + 0x2b0: 0x3e73, 0x2b1: 0xa000, 0x2b5: 0xa000, + 0x2b7: 0xa000, 0x2b9: 0xa000, + 0x2bf: 0xa000, + // Block 0xb, offset 0x2c0 + 0x2c0: 0x3724, 0x2c1: 0x3730, 0x2c3: 0x371e, + 0x2c6: 0xa000, 0x2c7: 0x370c, + 0x2cc: 0x3760, 0x2cd: 0x3748, 0x2ce: 0x3772, 0x2d0: 0xa000, + 0x2d3: 0xa000, 0x2d5: 0xa000, 0x2d6: 0xa000, 0x2d7: 0xa000, + 0x2d8: 0xa000, 0x2d9: 0x3754, 0x2da: 0xa000, + 0x2de: 0xa000, 0x2e3: 0xa000, + 0x2e7: 0xa000, + 0x2eb: 0xa000, 0x2ed: 0xa000, + 0x2f0: 0xa000, 0x2f3: 0xa000, 0x2f5: 0xa000, + 0x2f6: 0xa000, 0x2f7: 0xa000, 0x2f8: 0xa000, 0x2f9: 0x37d8, 0x2fa: 0xa000, + 0x2fe: 0xa000, + // Block 0xc, offset 0x300 + 0x301: 0x3736, 0x302: 0x37ba, + 0x310: 0x3712, 0x311: 0x3796, + 0x312: 0x3718, 0x313: 0x379c, 0x316: 0x372a, 0x317: 0x37ae, + 0x318: 0xa000, 0x319: 0xa000, 0x31a: 0x382c, 0x31b: 0x3832, 0x31c: 0x373c, 0x31d: 0x37c0, + 0x31e: 0x3742, 0x31f: 0x37c6, 0x322: 0x374e, 0x323: 0x37d2, + 0x324: 0x375a, 0x325: 0x37de, 0x326: 0x3766, 0x327: 0x37ea, 0x328: 0xa000, 0x329: 0xa000, + 0x32a: 0x3838, 0x32b: 0x383e, 0x32c: 0x3790, 0x32d: 0x3814, 0x32e: 0x376c, 0x32f: 0x37f0, + 0x330: 0x3778, 0x331: 0x37fc, 0x332: 0x377e, 0x333: 0x3802, 0x334: 0x3784, 0x335: 0x3808, + 0x338: 0x378a, 0x339: 0x380e, + // Block 0xd, offset 0x340 + 0x351: 0x812d, + 0x352: 0x8132, 0x353: 0x8132, 0x354: 0x8132, 0x355: 0x8132, 0x356: 0x812d, 0x357: 0x8132, + 0x358: 0x8132, 0x359: 0x8132, 0x35a: 0x812e, 0x35b: 0x812d, 0x35c: 0x8132, 0x35d: 0x8132, + 0x35e: 0x8132, 0x35f: 0x8132, 0x360: 0x8132, 0x361: 0x8132, 0x362: 0x812d, 0x363: 0x812d, + 0x364: 0x812d, 0x365: 0x812d, 0x366: 0x812d, 0x367: 0x812d, 0x368: 0x8132, 0x369: 0x8132, + 0x36a: 0x812d, 0x36b: 0x8132, 0x36c: 0x8132, 0x36d: 0x812e, 0x36e: 0x8131, 0x36f: 0x8132, + 0x370: 0x8105, 0x371: 0x8106, 0x372: 0x8107, 0x373: 0x8108, 0x374: 0x8109, 0x375: 0x810a, + 0x376: 0x810b, 0x377: 0x810c, 0x378: 0x810d, 0x379: 0x810e, 0x37a: 0x810e, 0x37b: 0x810f, + 0x37c: 0x8110, 0x37d: 0x8111, 0x37f: 0x8112, + // Block 0xe, offset 0x380 + 0x388: 0xa000, 0x38a: 0xa000, 0x38b: 0x8116, + 0x38c: 0x8117, 0x38d: 0x8118, 0x38e: 0x8119, 0x38f: 0x811a, 0x390: 0x811b, 0x391: 0x811c, + 0x392: 0x811d, 0x393: 0x9932, 0x394: 0x9932, 0x395: 0x992d, 0x396: 0x812d, 0x397: 0x8132, + 0x398: 0x8132, 0x399: 0x8132, 0x39a: 0x8132, 0x39b: 0x8132, 0x39c: 0x812d, 0x39d: 0x8132, + 0x39e: 0x8132, 0x39f: 0x812d, + 0x3b0: 0x811e, + // Block 0xf, offset 0x3c0 + 0x3d3: 0x812d, 0x3d4: 0x8132, 0x3d5: 0x8132, 0x3d6: 0x8132, 0x3d7: 0x8132, + 0x3d8: 0x8132, 0x3d9: 0x8132, 0x3da: 0x8132, 0x3db: 0x8132, 0x3dc: 0x8132, 0x3dd: 0x8132, + 0x3de: 0x8132, 0x3df: 0x8132, 0x3e0: 0x8132, 0x3e1: 0x8132, 0x3e3: 0x812d, + 0x3e4: 0x8132, 0x3e5: 0x8132, 0x3e6: 0x812d, 0x3e7: 0x8132, 0x3e8: 0x8132, 0x3e9: 0x812d, + 0x3ea: 0x8132, 0x3eb: 0x8132, 0x3ec: 0x8132, 0x3ed: 0x812d, 0x3ee: 0x812d, 0x3ef: 0x812d, + 0x3f0: 0x8116, 0x3f1: 0x8117, 0x3f2: 0x8118, 0x3f3: 0x8132, 0x3f4: 0x8132, 0x3f5: 0x8132, + 0x3f6: 0x812d, 0x3f7: 0x8132, 0x3f8: 0x8132, 0x3f9: 0x812d, 0x3fa: 0x812d, 0x3fb: 0x8132, + 0x3fc: 0x8132, 0x3fd: 0x8132, 0x3fe: 0x8132, 0x3ff: 0x8132, + // Block 0x10, offset 0x400 + 0x405: 0xa000, + 0x406: 0x2d29, 0x407: 0xa000, 0x408: 0x2d31, 0x409: 0xa000, 0x40a: 0x2d39, 0x40b: 0xa000, + 0x40c: 0x2d41, 0x40d: 0xa000, 0x40e: 0x2d49, 0x411: 0xa000, + 0x412: 0x2d51, + 0x434: 0x8102, 0x435: 0x9900, + 0x43a: 0xa000, 0x43b: 0x2d59, + 0x43c: 0xa000, 0x43d: 0x2d61, 0x43e: 0xa000, 0x43f: 0xa000, + // Block 0x11, offset 0x440 + 0x440: 0x8132, 0x441: 0x8132, 0x442: 0x812d, 0x443: 0x8132, 0x444: 0x8132, 0x445: 0x8132, + 0x446: 0x8132, 0x447: 0x8132, 0x448: 0x8132, 0x449: 0x8132, 0x44a: 0x812d, 0x44b: 0x8132, + 0x44c: 0x8132, 0x44d: 0x8135, 0x44e: 0x812a, 0x44f: 0x812d, 0x450: 0x8129, 0x451: 0x8132, + 0x452: 0x8132, 0x453: 0x8132, 0x454: 0x8132, 0x455: 0x8132, 0x456: 0x8132, 0x457: 0x8132, + 0x458: 0x8132, 0x459: 0x8132, 0x45a: 0x8132, 0x45b: 0x8132, 0x45c: 0x8132, 0x45d: 0x8132, + 0x45e: 0x8132, 0x45f: 0x8132, 0x460: 0x8132, 0x461: 0x8132, 0x462: 0x8132, 0x463: 0x8132, + 0x464: 0x8132, 0x465: 0x8132, 0x466: 0x8132, 0x467: 0x8132, 0x468: 0x8132, 0x469: 0x8132, + 0x46a: 0x8132, 0x46b: 0x8132, 0x46c: 0x8132, 0x46d: 0x8132, 0x46e: 0x8132, 0x46f: 0x8132, + 0x470: 0x8132, 0x471: 0x8132, 0x472: 0x8132, 0x473: 0x8132, 0x474: 0x8132, 0x475: 0x8132, + 0x476: 0x8133, 0x477: 0x8131, 0x478: 0x8131, 0x479: 0x812d, 0x47b: 0x8132, + 0x47c: 0x8134, 0x47d: 0x812d, 0x47e: 0x8132, 0x47f: 0x812d, + // Block 0x12, offset 0x480 + 0x480: 0x2f9a, 0x481: 0x32a6, 0x482: 0x2fa4, 0x483: 0x32b0, 0x484: 0x2fa9, 0x485: 0x32b5, + 0x486: 0x2fae, 0x487: 0x32ba, 0x488: 0x38cf, 0x489: 0x3a5e, 0x48a: 0x2fc7, 0x48b: 0x32d3, + 0x48c: 0x2fd1, 0x48d: 0x32dd, 0x48e: 0x2fe0, 0x48f: 0x32ec, 0x490: 0x2fd6, 0x491: 0x32e2, + 0x492: 0x2fdb, 0x493: 0x32e7, 0x494: 0x38f2, 0x495: 0x3a81, 0x496: 0x38f9, 0x497: 0x3a88, + 0x498: 0x301c, 0x499: 0x3328, 0x49a: 0x3021, 0x49b: 0x332d, 0x49c: 0x3907, 0x49d: 0x3a96, + 0x49e: 0x3026, 0x49f: 0x3332, 0x4a0: 0x3035, 0x4a1: 0x3341, 0x4a2: 0x3053, 0x4a3: 0x335f, + 0x4a4: 0x3062, 0x4a5: 0x336e, 0x4a6: 0x3058, 0x4a7: 0x3364, 0x4a8: 0x3067, 0x4a9: 0x3373, + 0x4aa: 0x306c, 0x4ab: 0x3378, 0x4ac: 0x30b2, 0x4ad: 0x33be, 0x4ae: 0x390e, 0x4af: 0x3a9d, + 0x4b0: 0x30bc, 0x4b1: 0x33cd, 0x4b2: 0x30c6, 0x4b3: 0x33d7, 0x4b4: 0x30d0, 0x4b5: 0x33e1, + 0x4b6: 0x46c7, 0x4b7: 0x4758, 0x4b8: 0x3915, 0x4b9: 0x3aa4, 0x4ba: 0x30e9, 0x4bb: 0x33fa, + 0x4bc: 0x30e4, 0x4bd: 0x33f5, 0x4be: 0x30ee, 0x4bf: 0x33ff, + // Block 0x13, offset 0x4c0 + 0x4c0: 0x30f3, 0x4c1: 0x3404, 0x4c2: 0x30f8, 0x4c3: 0x3409, 0x4c4: 0x310c, 0x4c5: 0x341d, + 0x4c6: 0x3116, 0x4c7: 0x3427, 0x4c8: 0x3125, 0x4c9: 0x3436, 0x4ca: 0x3120, 0x4cb: 0x3431, + 0x4cc: 0x3938, 0x4cd: 0x3ac7, 0x4ce: 0x3946, 0x4cf: 0x3ad5, 0x4d0: 0x394d, 0x4d1: 0x3adc, + 0x4d2: 0x3954, 0x4d3: 0x3ae3, 0x4d4: 0x3152, 0x4d5: 0x3463, 0x4d6: 0x3157, 0x4d7: 0x3468, + 0x4d8: 0x3161, 0x4d9: 0x3472, 0x4da: 0x46f4, 0x4db: 0x4785, 0x4dc: 0x399a, 0x4dd: 0x3b29, + 0x4de: 0x317a, 0x4df: 0x348b, 0x4e0: 0x3184, 0x4e1: 0x3495, 0x4e2: 0x4703, 0x4e3: 0x4794, + 0x4e4: 0x39a1, 0x4e5: 0x3b30, 0x4e6: 0x39a8, 0x4e7: 0x3b37, 0x4e8: 0x39af, 0x4e9: 0x3b3e, + 0x4ea: 0x3193, 0x4eb: 0x34a4, 0x4ec: 0x319d, 0x4ed: 0x34b3, 0x4ee: 0x31b1, 0x4ef: 0x34c7, + 0x4f0: 0x31ac, 0x4f1: 0x34c2, 0x4f2: 0x31ed, 0x4f3: 0x3503, 0x4f4: 0x31fc, 0x4f5: 0x3512, + 0x4f6: 0x31f7, 0x4f7: 0x350d, 0x4f8: 0x39b6, 0x4f9: 0x3b45, 0x4fa: 0x39bd, 0x4fb: 0x3b4c, + 0x4fc: 0x3201, 0x4fd: 0x3517, 0x4fe: 0x3206, 0x4ff: 0x351c, + // Block 0x14, offset 0x500 + 0x500: 0x320b, 0x501: 0x3521, 0x502: 0x3210, 0x503: 0x3526, 0x504: 0x321f, 0x505: 0x3535, + 0x506: 0x321a, 0x507: 0x3530, 0x508: 0x3224, 0x509: 0x353f, 0x50a: 0x3229, 0x50b: 0x3544, + 0x50c: 0x322e, 0x50d: 0x3549, 0x50e: 0x324c, 0x50f: 0x3567, 0x510: 0x3265, 0x511: 0x3585, + 0x512: 0x3274, 0x513: 0x3594, 0x514: 0x3279, 0x515: 0x3599, 0x516: 0x337d, 0x517: 0x34a9, + 0x518: 0x353a, 0x519: 0x3576, 0x51b: 0x35d4, + 0x520: 0x46a4, 0x521: 0x4735, 0x522: 0x2f86, 0x523: 0x3292, + 0x524: 0x387b, 0x525: 0x3a0a, 0x526: 0x3874, 0x527: 0x3a03, 0x528: 0x3889, 0x529: 0x3a18, + 0x52a: 0x3882, 0x52b: 0x3a11, 0x52c: 0x38c1, 0x52d: 0x3a50, 0x52e: 0x3897, 0x52f: 0x3a26, + 0x530: 0x3890, 0x531: 0x3a1f, 0x532: 0x38a5, 0x533: 0x3a34, 0x534: 0x389e, 0x535: 0x3a2d, + 0x536: 0x38c8, 0x537: 0x3a57, 0x538: 0x46b8, 0x539: 0x4749, 0x53a: 0x3003, 0x53b: 0x330f, + 0x53c: 0x2fef, 0x53d: 0x32fb, 0x53e: 0x38dd, 0x53f: 0x3a6c, + // Block 0x15, offset 0x540 + 0x540: 0x38d6, 0x541: 0x3a65, 0x542: 0x38eb, 0x543: 0x3a7a, 0x544: 0x38e4, 0x545: 0x3a73, + 0x546: 0x3900, 0x547: 0x3a8f, 0x548: 0x3094, 0x549: 0x33a0, 0x54a: 0x30a8, 0x54b: 0x33b4, + 0x54c: 0x46ea, 0x54d: 0x477b, 0x54e: 0x3139, 0x54f: 0x344a, 0x550: 0x3923, 0x551: 0x3ab2, + 0x552: 0x391c, 0x553: 0x3aab, 0x554: 0x3931, 0x555: 0x3ac0, 0x556: 0x392a, 0x557: 0x3ab9, + 0x558: 0x398c, 0x559: 0x3b1b, 0x55a: 0x3970, 0x55b: 0x3aff, 0x55c: 0x3969, 0x55d: 0x3af8, + 0x55e: 0x397e, 0x55f: 0x3b0d, 0x560: 0x3977, 0x561: 0x3b06, 0x562: 0x3985, 0x563: 0x3b14, + 0x564: 0x31e8, 0x565: 0x34fe, 0x566: 0x31ca, 0x567: 0x34e0, 0x568: 0x39e7, 0x569: 0x3b76, + 0x56a: 0x39e0, 0x56b: 0x3b6f, 0x56c: 0x39f5, 0x56d: 0x3b84, 0x56e: 0x39ee, 0x56f: 0x3b7d, + 0x570: 0x39fc, 0x571: 0x3b8b, 0x572: 0x3233, 0x573: 0x354e, 0x574: 0x325b, 0x575: 0x357b, + 0x576: 0x3256, 0x577: 0x3571, 0x578: 0x3242, 0x579: 0x355d, + // Block 0x16, offset 0x580 + 0x580: 0x4807, 0x581: 0x480d, 0x582: 0x4921, 0x583: 0x4939, 0x584: 0x4929, 0x585: 0x4941, + 0x586: 0x4931, 0x587: 0x4949, 0x588: 0x47ad, 0x589: 0x47b3, 0x58a: 0x4891, 0x58b: 0x48a9, + 0x58c: 0x4899, 0x58d: 0x48b1, 0x58e: 0x48a1, 0x58f: 0x48b9, 0x590: 0x4819, 0x591: 0x481f, + 0x592: 0x3dbb, 0x593: 0x3dcb, 0x594: 0x3dc3, 0x595: 0x3dd3, + 0x598: 0x47b9, 0x599: 0x47bf, 0x59a: 0x3ceb, 0x59b: 0x3cfb, 0x59c: 0x3cf3, 0x59d: 0x3d03, + 0x5a0: 0x4831, 0x5a1: 0x4837, 0x5a2: 0x4951, 0x5a3: 0x4969, + 0x5a4: 0x4959, 0x5a5: 0x4971, 0x5a6: 0x4961, 0x5a7: 0x4979, 0x5a8: 0x47c5, 0x5a9: 0x47cb, + 0x5aa: 0x48c1, 0x5ab: 0x48d9, 0x5ac: 0x48c9, 0x5ad: 0x48e1, 0x5ae: 0x48d1, 0x5af: 0x48e9, + 0x5b0: 0x4849, 0x5b1: 0x484f, 0x5b2: 0x3e1b, 0x5b3: 0x3e33, 0x5b4: 0x3e23, 0x5b5: 0x3e3b, + 0x5b6: 0x3e2b, 0x5b7: 0x3e43, 0x5b8: 0x47d1, 0x5b9: 0x47d7, 0x5ba: 0x3d1b, 0x5bb: 0x3d33, + 0x5bc: 0x3d23, 0x5bd: 0x3d3b, 0x5be: 0x3d2b, 0x5bf: 0x3d43, + // Block 0x17, offset 0x5c0 + 0x5c0: 0x4855, 0x5c1: 0x485b, 0x5c2: 0x3e4b, 0x5c3: 0x3e5b, 0x5c4: 0x3e53, 0x5c5: 0x3e63, + 0x5c8: 0x47dd, 0x5c9: 0x47e3, 0x5ca: 0x3d4b, 0x5cb: 0x3d5b, + 0x5cc: 0x3d53, 0x5cd: 0x3d63, 0x5d0: 0x4867, 0x5d1: 0x486d, + 0x5d2: 0x3e83, 0x5d3: 0x3e9b, 0x5d4: 0x3e8b, 0x5d5: 0x3ea3, 0x5d6: 0x3e93, 0x5d7: 0x3eab, + 0x5d9: 0x47e9, 0x5db: 0x3d6b, 0x5dd: 0x3d73, + 0x5df: 0x3d7b, 0x5e0: 0x487f, 0x5e1: 0x4885, 0x5e2: 0x4981, 0x5e3: 0x4999, + 0x5e4: 0x4989, 0x5e5: 0x49a1, 0x5e6: 0x4991, 0x5e7: 0x49a9, 0x5e8: 0x47ef, 0x5e9: 0x47f5, + 0x5ea: 0x48f1, 0x5eb: 0x4909, 0x5ec: 0x48f9, 0x5ed: 0x4911, 0x5ee: 0x4901, 0x5ef: 0x4919, + 0x5f0: 0x47fb, 0x5f1: 0x4321, 0x5f2: 0x3694, 0x5f3: 0x4327, 0x5f4: 0x4825, 0x5f5: 0x432d, + 0x5f6: 0x36a6, 0x5f7: 0x4333, 0x5f8: 0x36c4, 0x5f9: 0x4339, 0x5fa: 0x36dc, 0x5fb: 0x433f, + 0x5fc: 0x4873, 0x5fd: 0x4345, + // Block 0x18, offset 0x600 + 0x600: 0x3da3, 0x601: 0x3dab, 0x602: 0x4187, 0x603: 0x41a5, 0x604: 0x4191, 0x605: 0x41af, + 0x606: 0x419b, 0x607: 0x41b9, 0x608: 0x3cdb, 0x609: 0x3ce3, 0x60a: 0x40d3, 0x60b: 0x40f1, + 0x60c: 0x40dd, 0x60d: 0x40fb, 0x60e: 0x40e7, 0x60f: 0x4105, 0x610: 0x3deb, 0x611: 0x3df3, + 0x612: 0x41c3, 0x613: 0x41e1, 0x614: 0x41cd, 0x615: 0x41eb, 0x616: 0x41d7, 0x617: 0x41f5, + 0x618: 0x3d0b, 0x619: 0x3d13, 0x61a: 0x410f, 0x61b: 0x412d, 0x61c: 0x4119, 0x61d: 0x4137, + 0x61e: 0x4123, 0x61f: 0x4141, 0x620: 0x3ec3, 0x621: 0x3ecb, 0x622: 0x41ff, 0x623: 0x421d, + 0x624: 0x4209, 0x625: 0x4227, 0x626: 0x4213, 0x627: 0x4231, 0x628: 0x3d83, 0x629: 0x3d8b, + 0x62a: 0x414b, 0x62b: 0x4169, 0x62c: 0x4155, 0x62d: 0x4173, 0x62e: 0x415f, 0x62f: 0x417d, + 0x630: 0x3688, 0x631: 0x3682, 0x632: 0x3d93, 0x633: 0x368e, 0x634: 0x3d9b, + 0x636: 0x4813, 0x637: 0x3db3, 0x638: 0x35f8, 0x639: 0x35f2, 0x63a: 0x35e6, 0x63b: 0x42f1, + 0x63c: 0x35fe, 0x63d: 0x8100, 0x63e: 0x01d3, 0x63f: 0xa100, + // Block 0x19, offset 0x640 + 0x640: 0x8100, 0x641: 0x35aa, 0x642: 0x3ddb, 0x643: 0x36a0, 0x644: 0x3de3, + 0x646: 0x483d, 0x647: 0x3dfb, 0x648: 0x3604, 0x649: 0x42f7, 0x64a: 0x3610, 0x64b: 0x42fd, + 0x64c: 0x361c, 0x64d: 0x3b92, 0x64e: 0x3b99, 0x64f: 0x3ba0, 0x650: 0x36b8, 0x651: 0x36b2, + 0x652: 0x3e03, 0x653: 0x44e7, 0x656: 0x36be, 0x657: 0x3e13, + 0x658: 0x3634, 0x659: 0x362e, 0x65a: 0x3622, 0x65b: 0x4303, 0x65d: 0x3ba7, + 0x65e: 0x3bae, 0x65f: 0x3bb5, 0x660: 0x36ee, 0x661: 0x36e8, 0x662: 0x3e6b, 0x663: 0x44ef, + 0x664: 0x36d0, 0x665: 0x36d6, 0x666: 0x36f4, 0x667: 0x3e7b, 0x668: 0x3664, 0x669: 0x365e, + 0x66a: 0x3652, 0x66b: 0x430f, 0x66c: 0x364c, 0x66d: 0x359e, 0x66e: 0x42eb, 0x66f: 0x0081, + 0x672: 0x3eb3, 0x673: 0x36fa, 0x674: 0x3ebb, + 0x676: 0x488b, 0x677: 0x3ed3, 0x678: 0x3640, 0x679: 0x4309, 0x67a: 0x3670, 0x67b: 0x431b, + 0x67c: 0x367c, 0x67d: 0x4259, 0x67e: 0xa100, + // Block 0x1a, offset 0x680 + 0x681: 0x3c09, 0x683: 0xa000, 0x684: 0x3c10, 0x685: 0xa000, + 0x687: 0x3c17, 0x688: 0xa000, 0x689: 0x3c1e, + 0x68d: 0xa000, + 0x6a0: 0x2f68, 0x6a1: 0xa000, 0x6a2: 0x3c2c, + 0x6a4: 0xa000, 0x6a5: 0xa000, + 0x6ad: 0x3c25, 0x6ae: 0x2f63, 0x6af: 0x2f6d, + 0x6b0: 0x3c33, 0x6b1: 0x3c3a, 0x6b2: 0xa000, 0x6b3: 0xa000, 0x6b4: 0x3c41, 0x6b5: 0x3c48, + 0x6b6: 0xa000, 0x6b7: 0xa000, 0x6b8: 0x3c4f, 0x6b9: 0x3c56, 0x6ba: 0xa000, 0x6bb: 0xa000, + 0x6bc: 0xa000, 0x6bd: 0xa000, + // Block 0x1b, offset 0x6c0 + 0x6c0: 0x3c5d, 0x6c1: 0x3c64, 0x6c2: 0xa000, 0x6c3: 0xa000, 0x6c4: 0x3c79, 0x6c5: 0x3c80, + 0x6c6: 0xa000, 0x6c7: 0xa000, 0x6c8: 0x3c87, 0x6c9: 0x3c8e, + 0x6d1: 0xa000, + 0x6d2: 0xa000, + 0x6e2: 0xa000, + 0x6e8: 0xa000, 0x6e9: 0xa000, + 0x6eb: 0xa000, 0x6ec: 0x3ca3, 0x6ed: 0x3caa, 0x6ee: 0x3cb1, 0x6ef: 0x3cb8, + 0x6f2: 0xa000, 0x6f3: 0xa000, 0x6f4: 0xa000, 0x6f5: 0xa000, + // Block 0x1c, offset 0x700 + 0x706: 0xa000, 0x70b: 0xa000, + 0x70c: 0x3f0b, 0x70d: 0xa000, 0x70e: 0x3f13, 0x70f: 0xa000, 0x710: 0x3f1b, 0x711: 0xa000, + 0x712: 0x3f23, 0x713: 0xa000, 0x714: 0x3f2b, 0x715: 0xa000, 0x716: 0x3f33, 0x717: 0xa000, + 0x718: 0x3f3b, 0x719: 0xa000, 0x71a: 0x3f43, 0x71b: 0xa000, 0x71c: 0x3f4b, 0x71d: 0xa000, + 0x71e: 0x3f53, 0x71f: 0xa000, 0x720: 0x3f5b, 0x721: 0xa000, 0x722: 0x3f63, + 0x724: 0xa000, 0x725: 0x3f6b, 0x726: 0xa000, 0x727: 0x3f73, 0x728: 0xa000, 0x729: 0x3f7b, + 0x72f: 0xa000, + 0x730: 0x3f83, 0x731: 0x3f8b, 0x732: 0xa000, 0x733: 0x3f93, 0x734: 0x3f9b, 0x735: 0xa000, + 0x736: 0x3fa3, 0x737: 0x3fab, 0x738: 0xa000, 0x739: 0x3fb3, 0x73a: 0x3fbb, 0x73b: 0xa000, + 0x73c: 0x3fc3, 0x73d: 0x3fcb, + // Block 0x1d, offset 0x740 + 0x754: 0x3f03, + 0x759: 0x9903, 0x75a: 0x9903, 0x75b: 0x8100, 0x75c: 0x8100, 0x75d: 0xa000, + 0x75e: 0x3fd3, + 0x766: 0xa000, + 0x76b: 0xa000, 0x76c: 0x3fe3, 0x76d: 0xa000, 0x76e: 0x3feb, 0x76f: 0xa000, + 0x770: 0x3ff3, 0x771: 0xa000, 0x772: 0x3ffb, 0x773: 0xa000, 0x774: 0x4003, 0x775: 0xa000, + 0x776: 0x400b, 0x777: 0xa000, 0x778: 0x4013, 0x779: 0xa000, 0x77a: 0x401b, 0x77b: 0xa000, + 0x77c: 0x4023, 0x77d: 0xa000, 0x77e: 0x402b, 0x77f: 0xa000, + // Block 0x1e, offset 0x780 + 0x780: 0x4033, 0x781: 0xa000, 0x782: 0x403b, 0x784: 0xa000, 0x785: 0x4043, + 0x786: 0xa000, 0x787: 0x404b, 0x788: 0xa000, 0x789: 0x4053, + 0x78f: 0xa000, 0x790: 0x405b, 0x791: 0x4063, + 0x792: 0xa000, 0x793: 0x406b, 0x794: 0x4073, 0x795: 0xa000, 0x796: 0x407b, 0x797: 0x4083, + 0x798: 0xa000, 0x799: 0x408b, 0x79a: 0x4093, 0x79b: 0xa000, 0x79c: 0x409b, 0x79d: 0x40a3, + 0x7af: 0xa000, + 0x7b0: 0xa000, 0x7b1: 0xa000, 0x7b2: 0xa000, 0x7b4: 0x3fdb, + 0x7b7: 0x40ab, 0x7b8: 0x40b3, 0x7b9: 0x40bb, 0x7ba: 0x40c3, + 0x7bd: 0xa000, 0x7be: 0x40cb, + // Block 0x1f, offset 0x7c0 + 0x7c0: 0x1377, 0x7c1: 0x0cfb, 0x7c2: 0x13d3, 0x7c3: 0x139f, 0x7c4: 0x0e57, 0x7c5: 0x06eb, + 0x7c6: 0x08df, 0x7c7: 0x162b, 0x7c8: 0x162b, 0x7c9: 0x0a0b, 0x7ca: 0x145f, 0x7cb: 0x0943, + 0x7cc: 0x0a07, 0x7cd: 0x0bef, 0x7ce: 0x0fcf, 0x7cf: 0x115f, 0x7d0: 0x1297, 0x7d1: 0x12d3, + 0x7d2: 0x1307, 0x7d3: 0x141b, 0x7d4: 0x0d73, 0x7d5: 0x0dff, 0x7d6: 0x0eab, 0x7d7: 0x0f43, + 0x7d8: 0x125f, 0x7d9: 0x1447, 0x7da: 0x1573, 0x7db: 0x070f, 0x7dc: 0x08b3, 0x7dd: 0x0d87, + 0x7de: 0x0ecf, 0x7df: 0x1293, 0x7e0: 0x15c3, 0x7e1: 0x0ab3, 0x7e2: 0x0e77, 0x7e3: 0x1283, + 0x7e4: 0x1317, 0x7e5: 0x0c23, 0x7e6: 0x11bb, 0x7e7: 0x12df, 0x7e8: 0x0b1f, 0x7e9: 0x0d0f, + 0x7ea: 0x0e17, 0x7eb: 0x0f1b, 0x7ec: 0x1427, 0x7ed: 0x074f, 0x7ee: 0x07e7, 0x7ef: 0x0853, + 0x7f0: 0x0c8b, 0x7f1: 0x0d7f, 0x7f2: 0x0ecb, 0x7f3: 0x0fef, 0x7f4: 0x1177, 0x7f5: 0x128b, + 0x7f6: 0x12a3, 0x7f7: 0x13c7, 0x7f8: 0x14ef, 0x7f9: 0x15a3, 0x7fa: 0x15bf, 0x7fb: 0x102b, + 0x7fc: 0x106b, 0x7fd: 0x1123, 0x7fe: 0x1243, 0x7ff: 0x147b, + // Block 0x20, offset 0x800 + 0x800: 0x15cb, 0x801: 0x134b, 0x802: 0x09c7, 0x803: 0x0b3b, 0x804: 0x10db, 0x805: 0x119b, + 0x806: 0x0eff, 0x807: 0x1033, 0x808: 0x1397, 0x809: 0x14e7, 0x80a: 0x09c3, 0x80b: 0x0a8f, + 0x80c: 0x0d77, 0x80d: 0x0e2b, 0x80e: 0x0e5f, 0x80f: 0x1113, 0x810: 0x113b, 0x811: 0x14a7, + 0x812: 0x084f, 0x813: 0x11a7, 0x814: 0x07f3, 0x815: 0x07ef, 0x816: 0x1097, 0x817: 0x1127, + 0x818: 0x125b, 0x819: 0x14af, 0x81a: 0x1367, 0x81b: 0x0c27, 0x81c: 0x0d73, 0x81d: 0x1357, + 0x81e: 0x06f7, 0x81f: 0x0a63, 0x820: 0x0b93, 0x821: 0x0f2f, 0x822: 0x0faf, 0x823: 0x0873, + 0x824: 0x103b, 0x825: 0x075f, 0x826: 0x0b77, 0x827: 0x06d7, 0x828: 0x0deb, 0x829: 0x0ca3, + 0x82a: 0x110f, 0x82b: 0x08c7, 0x82c: 0x09b3, 0x82d: 0x0ffb, 0x82e: 0x1263, 0x82f: 0x133b, + 0x830: 0x0db7, 0x831: 0x13f7, 0x832: 0x0de3, 0x833: 0x0c37, 0x834: 0x121b, 0x835: 0x0c57, + 0x836: 0x0fab, 0x837: 0x072b, 0x838: 0x07a7, 0x839: 0x07eb, 0x83a: 0x0d53, 0x83b: 0x10fb, + 0x83c: 0x11f3, 0x83d: 0x1347, 0x83e: 0x145b, 0x83f: 0x085b, + // Block 0x21, offset 0x840 + 0x840: 0x090f, 0x841: 0x0a17, 0x842: 0x0b2f, 0x843: 0x0cbf, 0x844: 0x0e7b, 0x845: 0x103f, + 0x846: 0x1497, 0x847: 0x157b, 0x848: 0x15cf, 0x849: 0x15e7, 0x84a: 0x0837, 0x84b: 0x0cf3, + 0x84c: 0x0da3, 0x84d: 0x13eb, 0x84e: 0x0afb, 0x84f: 0x0bd7, 0x850: 0x0bf3, 0x851: 0x0c83, + 0x852: 0x0e6b, 0x853: 0x0eb7, 0x854: 0x0f67, 0x855: 0x108b, 0x856: 0x112f, 0x857: 0x1193, + 0x858: 0x13db, 0x859: 0x126b, 0x85a: 0x1403, 0x85b: 0x147f, 0x85c: 0x080f, 0x85d: 0x083b, + 0x85e: 0x0923, 0x85f: 0x0ea7, 0x860: 0x12f3, 0x861: 0x133b, 0x862: 0x0b1b, 0x863: 0x0b8b, + 0x864: 0x0c4f, 0x865: 0x0daf, 0x866: 0x10d7, 0x867: 0x0f23, 0x868: 0x073b, 0x869: 0x097f, + 0x86a: 0x0a63, 0x86b: 0x0ac7, 0x86c: 0x0b97, 0x86d: 0x0f3f, 0x86e: 0x0f5b, 0x86f: 0x116b, + 0x870: 0x118b, 0x871: 0x1463, 0x872: 0x14e3, 0x873: 0x14f3, 0x874: 0x152f, 0x875: 0x0753, + 0x876: 0x107f, 0x877: 0x144f, 0x878: 0x14cb, 0x879: 0x0baf, 0x87a: 0x0717, 0x87b: 0x0777, + 0x87c: 0x0a67, 0x87d: 0x0a87, 0x87e: 0x0caf, 0x87f: 0x0d73, + // Block 0x22, offset 0x880 + 0x880: 0x0ec3, 0x881: 0x0fcb, 0x882: 0x1277, 0x883: 0x1417, 0x884: 0x1623, 0x885: 0x0ce3, + 0x886: 0x14a3, 0x887: 0x0833, 0x888: 0x0d2f, 0x889: 0x0d3b, 0x88a: 0x0e0f, 0x88b: 0x0e47, + 0x88c: 0x0f4b, 0x88d: 0x0fa7, 0x88e: 0x1027, 0x88f: 0x110b, 0x890: 0x153b, 0x891: 0x07af, + 0x892: 0x0c03, 0x893: 0x14b3, 0x894: 0x0767, 0x895: 0x0aab, 0x896: 0x0e2f, 0x897: 0x13df, + 0x898: 0x0b67, 0x899: 0x0bb7, 0x89a: 0x0d43, 0x89b: 0x0f2f, 0x89c: 0x14bb, 0x89d: 0x0817, + 0x89e: 0x08ff, 0x89f: 0x0a97, 0x8a0: 0x0cd3, 0x8a1: 0x0d1f, 0x8a2: 0x0d5f, 0x8a3: 0x0df3, + 0x8a4: 0x0f47, 0x8a5: 0x0fbb, 0x8a6: 0x1157, 0x8a7: 0x12f7, 0x8a8: 0x1303, 0x8a9: 0x1457, + 0x8aa: 0x14d7, 0x8ab: 0x0883, 0x8ac: 0x0e4b, 0x8ad: 0x0903, 0x8ae: 0x0ec7, 0x8af: 0x0f6b, + 0x8b0: 0x1287, 0x8b1: 0x14bf, 0x8b2: 0x15ab, 0x8b3: 0x15d3, 0x8b4: 0x0d37, 0x8b5: 0x0e27, + 0x8b6: 0x11c3, 0x8b7: 0x10b7, 0x8b8: 0x10c3, 0x8b9: 0x10e7, 0x8ba: 0x0f17, 0x8bb: 0x0e9f, + 0x8bc: 0x1363, 0x8bd: 0x0733, 0x8be: 0x122b, 0x8bf: 0x081b, + // Block 0x23, offset 0x8c0 + 0x8c0: 0x080b, 0x8c1: 0x0b0b, 0x8c2: 0x0c2b, 0x8c3: 0x10f3, 0x8c4: 0x0a53, 0x8c5: 0x0e03, + 0x8c6: 0x0cef, 0x8c7: 0x13e7, 0x8c8: 0x12e7, 0x8c9: 0x14ab, 0x8ca: 0x1323, 0x8cb: 0x0b27, + 0x8cc: 0x0787, 0x8cd: 0x095b, 0x8d0: 0x09af, + 0x8d2: 0x0cdf, 0x8d5: 0x07f7, 0x8d6: 0x0f1f, 0x8d7: 0x0fe3, + 0x8d8: 0x1047, 0x8d9: 0x1063, 0x8da: 0x1067, 0x8db: 0x107b, 0x8dc: 0x14fb, 0x8dd: 0x10eb, + 0x8de: 0x116f, 0x8e0: 0x128f, 0x8e2: 0x1353, + 0x8e5: 0x1407, 0x8e6: 0x1433, + 0x8ea: 0x154f, 0x8eb: 0x1553, 0x8ec: 0x1557, 0x8ed: 0x15bb, 0x8ee: 0x142b, 0x8ef: 0x14c7, + 0x8f0: 0x0757, 0x8f1: 0x077b, 0x8f2: 0x078f, 0x8f3: 0x084b, 0x8f4: 0x0857, 0x8f5: 0x0897, + 0x8f6: 0x094b, 0x8f7: 0x0967, 0x8f8: 0x096f, 0x8f9: 0x09ab, 0x8fa: 0x09b7, 0x8fb: 0x0a93, + 0x8fc: 0x0a9b, 0x8fd: 0x0ba3, 0x8fe: 0x0bcb, 0x8ff: 0x0bd3, + // Block 0x24, offset 0x900 + 0x900: 0x0beb, 0x901: 0x0c97, 0x902: 0x0cc7, 0x903: 0x0ce7, 0x904: 0x0d57, 0x905: 0x0e1b, + 0x906: 0x0e37, 0x907: 0x0e67, 0x908: 0x0ebb, 0x909: 0x0edb, 0x90a: 0x0f4f, 0x90b: 0x102f, + 0x90c: 0x104b, 0x90d: 0x1053, 0x90e: 0x104f, 0x90f: 0x1057, 0x910: 0x105b, 0x911: 0x105f, + 0x912: 0x1073, 0x913: 0x1077, 0x914: 0x109b, 0x915: 0x10af, 0x916: 0x10cb, 0x917: 0x112f, + 0x918: 0x1137, 0x919: 0x113f, 0x91a: 0x1153, 0x91b: 0x117b, 0x91c: 0x11cb, 0x91d: 0x11ff, + 0x91e: 0x11ff, 0x91f: 0x1267, 0x920: 0x130f, 0x921: 0x1327, 0x922: 0x135b, 0x923: 0x135f, + 0x924: 0x13a3, 0x925: 0x13a7, 0x926: 0x13ff, 0x927: 0x1407, 0x928: 0x14db, 0x929: 0x151f, + 0x92a: 0x1537, 0x92b: 0x0b9b, 0x92c: 0x171e, 0x92d: 0x11e3, + 0x930: 0x06df, 0x931: 0x07e3, 0x932: 0x07a3, 0x933: 0x074b, 0x934: 0x078b, 0x935: 0x07b7, + 0x936: 0x0847, 0x937: 0x0863, 0x938: 0x094b, 0x939: 0x0937, 0x93a: 0x0947, 0x93b: 0x0963, + 0x93c: 0x09af, 0x93d: 0x09bf, 0x93e: 0x0a03, 0x93f: 0x0a0f, + // Block 0x25, offset 0x940 + 0x940: 0x0a2b, 0x941: 0x0a3b, 0x942: 0x0b23, 0x943: 0x0b2b, 0x944: 0x0b5b, 0x945: 0x0b7b, + 0x946: 0x0bab, 0x947: 0x0bc3, 0x948: 0x0bb3, 0x949: 0x0bd3, 0x94a: 0x0bc7, 0x94b: 0x0beb, + 0x94c: 0x0c07, 0x94d: 0x0c5f, 0x94e: 0x0c6b, 0x94f: 0x0c73, 0x950: 0x0c9b, 0x951: 0x0cdf, + 0x952: 0x0d0f, 0x953: 0x0d13, 0x954: 0x0d27, 0x955: 0x0da7, 0x956: 0x0db7, 0x957: 0x0e0f, + 0x958: 0x0e5b, 0x959: 0x0e53, 0x95a: 0x0e67, 0x95b: 0x0e83, 0x95c: 0x0ebb, 0x95d: 0x1013, + 0x95e: 0x0edf, 0x95f: 0x0f13, 0x960: 0x0f1f, 0x961: 0x0f5f, 0x962: 0x0f7b, 0x963: 0x0f9f, + 0x964: 0x0fc3, 0x965: 0x0fc7, 0x966: 0x0fe3, 0x967: 0x0fe7, 0x968: 0x0ff7, 0x969: 0x100b, + 0x96a: 0x1007, 0x96b: 0x1037, 0x96c: 0x10b3, 0x96d: 0x10cb, 0x96e: 0x10e3, 0x96f: 0x111b, + 0x970: 0x112f, 0x971: 0x114b, 0x972: 0x117b, 0x973: 0x122f, 0x974: 0x1257, 0x975: 0x12cb, + 0x976: 0x1313, 0x977: 0x131f, 0x978: 0x1327, 0x979: 0x133f, 0x97a: 0x1353, 0x97b: 0x1343, + 0x97c: 0x135b, 0x97d: 0x1357, 0x97e: 0x134f, 0x97f: 0x135f, + // Block 0x26, offset 0x980 + 0x980: 0x136b, 0x981: 0x13a7, 0x982: 0x13e3, 0x983: 0x1413, 0x984: 0x144b, 0x985: 0x146b, + 0x986: 0x14b7, 0x987: 0x14db, 0x988: 0x14fb, 0x989: 0x150f, 0x98a: 0x151f, 0x98b: 0x152b, + 0x98c: 0x1537, 0x98d: 0x158b, 0x98e: 0x162b, 0x98f: 0x16b5, 0x990: 0x16b0, 0x991: 0x16e2, + 0x992: 0x0607, 0x993: 0x062f, 0x994: 0x0633, 0x995: 0x1764, 0x996: 0x1791, 0x997: 0x1809, + 0x998: 0x1617, 0x999: 0x1627, + // Block 0x27, offset 0x9c0 + 0x9c0: 0x06fb, 0x9c1: 0x06f3, 0x9c2: 0x0703, 0x9c3: 0x1647, 0x9c4: 0x0747, 0x9c5: 0x0757, + 0x9c6: 0x075b, 0x9c7: 0x0763, 0x9c8: 0x076b, 0x9c9: 0x076f, 0x9ca: 0x077b, 0x9cb: 0x0773, + 0x9cc: 0x05b3, 0x9cd: 0x165b, 0x9ce: 0x078f, 0x9cf: 0x0793, 0x9d0: 0x0797, 0x9d1: 0x07b3, + 0x9d2: 0x164c, 0x9d3: 0x05b7, 0x9d4: 0x079f, 0x9d5: 0x07bf, 0x9d6: 0x1656, 0x9d7: 0x07cf, + 0x9d8: 0x07d7, 0x9d9: 0x0737, 0x9da: 0x07df, 0x9db: 0x07e3, 0x9dc: 0x1831, 0x9dd: 0x07ff, + 0x9de: 0x0807, 0x9df: 0x05bf, 0x9e0: 0x081f, 0x9e1: 0x0823, 0x9e2: 0x082b, 0x9e3: 0x082f, + 0x9e4: 0x05c3, 0x9e5: 0x0847, 0x9e6: 0x084b, 0x9e7: 0x0857, 0x9e8: 0x0863, 0x9e9: 0x0867, + 0x9ea: 0x086b, 0x9eb: 0x0873, 0x9ec: 0x0893, 0x9ed: 0x0897, 0x9ee: 0x089f, 0x9ef: 0x08af, + 0x9f0: 0x08b7, 0x9f1: 0x08bb, 0x9f2: 0x08bb, 0x9f3: 0x08bb, 0x9f4: 0x166a, 0x9f5: 0x0e93, + 0x9f6: 0x08cf, 0x9f7: 0x08d7, 0x9f8: 0x166f, 0x9f9: 0x08e3, 0x9fa: 0x08eb, 0x9fb: 0x08f3, + 0x9fc: 0x091b, 0x9fd: 0x0907, 0x9fe: 0x0913, 0x9ff: 0x0917, + // Block 0x28, offset 0xa00 + 0xa00: 0x091f, 0xa01: 0x0927, 0xa02: 0x092b, 0xa03: 0x0933, 0xa04: 0x093b, 0xa05: 0x093f, + 0xa06: 0x093f, 0xa07: 0x0947, 0xa08: 0x094f, 0xa09: 0x0953, 0xa0a: 0x095f, 0xa0b: 0x0983, + 0xa0c: 0x0967, 0xa0d: 0x0987, 0xa0e: 0x096b, 0xa0f: 0x0973, 0xa10: 0x080b, 0xa11: 0x09cf, + 0xa12: 0x0997, 0xa13: 0x099b, 0xa14: 0x099f, 0xa15: 0x0993, 0xa16: 0x09a7, 0xa17: 0x09a3, + 0xa18: 0x09bb, 0xa19: 0x1674, 0xa1a: 0x09d7, 0xa1b: 0x09db, 0xa1c: 0x09e3, 0xa1d: 0x09ef, + 0xa1e: 0x09f7, 0xa1f: 0x0a13, 0xa20: 0x1679, 0xa21: 0x167e, 0xa22: 0x0a1f, 0xa23: 0x0a23, + 0xa24: 0x0a27, 0xa25: 0x0a1b, 0xa26: 0x0a2f, 0xa27: 0x05c7, 0xa28: 0x05cb, 0xa29: 0x0a37, + 0xa2a: 0x0a3f, 0xa2b: 0x0a3f, 0xa2c: 0x1683, 0xa2d: 0x0a5b, 0xa2e: 0x0a5f, 0xa2f: 0x0a63, + 0xa30: 0x0a6b, 0xa31: 0x1688, 0xa32: 0x0a73, 0xa33: 0x0a77, 0xa34: 0x0b4f, 0xa35: 0x0a7f, + 0xa36: 0x05cf, 0xa37: 0x0a8b, 0xa38: 0x0a9b, 0xa39: 0x0aa7, 0xa3a: 0x0aa3, 0xa3b: 0x1692, + 0xa3c: 0x0aaf, 0xa3d: 0x1697, 0xa3e: 0x0abb, 0xa3f: 0x0ab7, + // Block 0x29, offset 0xa40 + 0xa40: 0x0abf, 0xa41: 0x0acf, 0xa42: 0x0ad3, 0xa43: 0x05d3, 0xa44: 0x0ae3, 0xa45: 0x0aeb, + 0xa46: 0x0aef, 0xa47: 0x0af3, 0xa48: 0x05d7, 0xa49: 0x169c, 0xa4a: 0x05db, 0xa4b: 0x0b0f, + 0xa4c: 0x0b13, 0xa4d: 0x0b17, 0xa4e: 0x0b1f, 0xa4f: 0x1863, 0xa50: 0x0b37, 0xa51: 0x16a6, + 0xa52: 0x16a6, 0xa53: 0x11d7, 0xa54: 0x0b47, 0xa55: 0x0b47, 0xa56: 0x05df, 0xa57: 0x16c9, + 0xa58: 0x179b, 0xa59: 0x0b57, 0xa5a: 0x0b5f, 0xa5b: 0x05e3, 0xa5c: 0x0b73, 0xa5d: 0x0b83, + 0xa5e: 0x0b87, 0xa5f: 0x0b8f, 0xa60: 0x0b9f, 0xa61: 0x05eb, 0xa62: 0x05e7, 0xa63: 0x0ba3, + 0xa64: 0x16ab, 0xa65: 0x0ba7, 0xa66: 0x0bbb, 0xa67: 0x0bbf, 0xa68: 0x0bc3, 0xa69: 0x0bbf, + 0xa6a: 0x0bcf, 0xa6b: 0x0bd3, 0xa6c: 0x0be3, 0xa6d: 0x0bdb, 0xa6e: 0x0bdf, 0xa6f: 0x0be7, + 0xa70: 0x0beb, 0xa71: 0x0bef, 0xa72: 0x0bfb, 0xa73: 0x0bff, 0xa74: 0x0c17, 0xa75: 0x0c1f, + 0xa76: 0x0c2f, 0xa77: 0x0c43, 0xa78: 0x16ba, 0xa79: 0x0c3f, 0xa7a: 0x0c33, 0xa7b: 0x0c4b, + 0xa7c: 0x0c53, 0xa7d: 0x0c67, 0xa7e: 0x16bf, 0xa7f: 0x0c6f, + // Block 0x2a, offset 0xa80 + 0xa80: 0x0c63, 0xa81: 0x0c5b, 0xa82: 0x05ef, 0xa83: 0x0c77, 0xa84: 0x0c7f, 0xa85: 0x0c87, + 0xa86: 0x0c7b, 0xa87: 0x05f3, 0xa88: 0x0c97, 0xa89: 0x0c9f, 0xa8a: 0x16c4, 0xa8b: 0x0ccb, + 0xa8c: 0x0cff, 0xa8d: 0x0cdb, 0xa8e: 0x05ff, 0xa8f: 0x0ce7, 0xa90: 0x05fb, 0xa91: 0x05f7, + 0xa92: 0x07c3, 0xa93: 0x07c7, 0xa94: 0x0d03, 0xa95: 0x0ceb, 0xa96: 0x11ab, 0xa97: 0x0663, + 0xa98: 0x0d0f, 0xa99: 0x0d13, 0xa9a: 0x0d17, 0xa9b: 0x0d2b, 0xa9c: 0x0d23, 0xa9d: 0x16dd, + 0xa9e: 0x0603, 0xa9f: 0x0d3f, 0xaa0: 0x0d33, 0xaa1: 0x0d4f, 0xaa2: 0x0d57, 0xaa3: 0x16e7, + 0xaa4: 0x0d5b, 0xaa5: 0x0d47, 0xaa6: 0x0d63, 0xaa7: 0x0607, 0xaa8: 0x0d67, 0xaa9: 0x0d6b, + 0xaaa: 0x0d6f, 0xaab: 0x0d7b, 0xaac: 0x16ec, 0xaad: 0x0d83, 0xaae: 0x060b, 0xaaf: 0x0d8f, + 0xab0: 0x16f1, 0xab1: 0x0d93, 0xab2: 0x060f, 0xab3: 0x0d9f, 0xab4: 0x0dab, 0xab5: 0x0db7, + 0xab6: 0x0dbb, 0xab7: 0x16f6, 0xab8: 0x168d, 0xab9: 0x16fb, 0xaba: 0x0ddb, 0xabb: 0x1700, + 0xabc: 0x0de7, 0xabd: 0x0def, 0xabe: 0x0ddf, 0xabf: 0x0dfb, + // Block 0x2b, offset 0xac0 + 0xac0: 0x0e0b, 0xac1: 0x0e1b, 0xac2: 0x0e0f, 0xac3: 0x0e13, 0xac4: 0x0e1f, 0xac5: 0x0e23, + 0xac6: 0x1705, 0xac7: 0x0e07, 0xac8: 0x0e3b, 0xac9: 0x0e3f, 0xaca: 0x0613, 0xacb: 0x0e53, + 0xacc: 0x0e4f, 0xacd: 0x170a, 0xace: 0x0e33, 0xacf: 0x0e6f, 0xad0: 0x170f, 0xad1: 0x1714, + 0xad2: 0x0e73, 0xad3: 0x0e87, 0xad4: 0x0e83, 0xad5: 0x0e7f, 0xad6: 0x0617, 0xad7: 0x0e8b, + 0xad8: 0x0e9b, 0xad9: 0x0e97, 0xada: 0x0ea3, 0xadb: 0x1651, 0xadc: 0x0eb3, 0xadd: 0x1719, + 0xade: 0x0ebf, 0xadf: 0x1723, 0xae0: 0x0ed3, 0xae1: 0x0edf, 0xae2: 0x0ef3, 0xae3: 0x1728, + 0xae4: 0x0f07, 0xae5: 0x0f0b, 0xae6: 0x172d, 0xae7: 0x1732, 0xae8: 0x0f27, 0xae9: 0x0f37, + 0xaea: 0x061b, 0xaeb: 0x0f3b, 0xaec: 0x061f, 0xaed: 0x061f, 0xaee: 0x0f53, 0xaef: 0x0f57, + 0xaf0: 0x0f5f, 0xaf1: 0x0f63, 0xaf2: 0x0f6f, 0xaf3: 0x0623, 0xaf4: 0x0f87, 0xaf5: 0x1737, + 0xaf6: 0x0fa3, 0xaf7: 0x173c, 0xaf8: 0x0faf, 0xaf9: 0x16a1, 0xafa: 0x0fbf, 0xafb: 0x1741, + 0xafc: 0x1746, 0xafd: 0x174b, 0xafe: 0x0627, 0xaff: 0x062b, + // Block 0x2c, offset 0xb00 + 0xb00: 0x0ff7, 0xb01: 0x1755, 0xb02: 0x1750, 0xb03: 0x175a, 0xb04: 0x175f, 0xb05: 0x0fff, + 0xb06: 0x1003, 0xb07: 0x1003, 0xb08: 0x100b, 0xb09: 0x0633, 0xb0a: 0x100f, 0xb0b: 0x0637, + 0xb0c: 0x063b, 0xb0d: 0x1769, 0xb0e: 0x1023, 0xb0f: 0x102b, 0xb10: 0x1037, 0xb11: 0x063f, + 0xb12: 0x176e, 0xb13: 0x105b, 0xb14: 0x1773, 0xb15: 0x1778, 0xb16: 0x107b, 0xb17: 0x1093, + 0xb18: 0x0643, 0xb19: 0x109b, 0xb1a: 0x109f, 0xb1b: 0x10a3, 0xb1c: 0x177d, 0xb1d: 0x1782, + 0xb1e: 0x1782, 0xb1f: 0x10bb, 0xb20: 0x0647, 0xb21: 0x1787, 0xb22: 0x10cf, 0xb23: 0x10d3, + 0xb24: 0x064b, 0xb25: 0x178c, 0xb26: 0x10ef, 0xb27: 0x064f, 0xb28: 0x10ff, 0xb29: 0x10f7, + 0xb2a: 0x1107, 0xb2b: 0x1796, 0xb2c: 0x111f, 0xb2d: 0x0653, 0xb2e: 0x112b, 0xb2f: 0x1133, + 0xb30: 0x1143, 0xb31: 0x0657, 0xb32: 0x17a0, 0xb33: 0x17a5, 0xb34: 0x065b, 0xb35: 0x17aa, + 0xb36: 0x115b, 0xb37: 0x17af, 0xb38: 0x1167, 0xb39: 0x1173, 0xb3a: 0x117b, 0xb3b: 0x17b4, + 0xb3c: 0x17b9, 0xb3d: 0x118f, 0xb3e: 0x17be, 0xb3f: 0x1197, + // Block 0x2d, offset 0xb40 + 0xb40: 0x16ce, 0xb41: 0x065f, 0xb42: 0x11af, 0xb43: 0x11b3, 0xb44: 0x0667, 0xb45: 0x11b7, + 0xb46: 0x0a33, 0xb47: 0x17c3, 0xb48: 0x17c8, 0xb49: 0x16d3, 0xb4a: 0x16d8, 0xb4b: 0x11d7, + 0xb4c: 0x11db, 0xb4d: 0x13f3, 0xb4e: 0x066b, 0xb4f: 0x1207, 0xb50: 0x1203, 0xb51: 0x120b, + 0xb52: 0x083f, 0xb53: 0x120f, 0xb54: 0x1213, 0xb55: 0x1217, 0xb56: 0x121f, 0xb57: 0x17cd, + 0xb58: 0x121b, 0xb59: 0x1223, 0xb5a: 0x1237, 0xb5b: 0x123b, 0xb5c: 0x1227, 0xb5d: 0x123f, + 0xb5e: 0x1253, 0xb5f: 0x1267, 0xb60: 0x1233, 0xb61: 0x1247, 0xb62: 0x124b, 0xb63: 0x124f, + 0xb64: 0x17d2, 0xb65: 0x17dc, 0xb66: 0x17d7, 0xb67: 0x066f, 0xb68: 0x126f, 0xb69: 0x1273, + 0xb6a: 0x127b, 0xb6b: 0x17f0, 0xb6c: 0x127f, 0xb6d: 0x17e1, 0xb6e: 0x0673, 0xb6f: 0x0677, + 0xb70: 0x17e6, 0xb71: 0x17eb, 0xb72: 0x067b, 0xb73: 0x129f, 0xb74: 0x12a3, 0xb75: 0x12a7, + 0xb76: 0x12ab, 0xb77: 0x12b7, 0xb78: 0x12b3, 0xb79: 0x12bf, 0xb7a: 0x12bb, 0xb7b: 0x12cb, + 0xb7c: 0x12c3, 0xb7d: 0x12c7, 0xb7e: 0x12cf, 0xb7f: 0x067f, + // Block 0x2e, offset 0xb80 + 0xb80: 0x12d7, 0xb81: 0x12db, 0xb82: 0x0683, 0xb83: 0x12eb, 0xb84: 0x12ef, 0xb85: 0x17f5, + 0xb86: 0x12fb, 0xb87: 0x12ff, 0xb88: 0x0687, 0xb89: 0x130b, 0xb8a: 0x05bb, 0xb8b: 0x17fa, + 0xb8c: 0x17ff, 0xb8d: 0x068b, 0xb8e: 0x068f, 0xb8f: 0x1337, 0xb90: 0x134f, 0xb91: 0x136b, + 0xb92: 0x137b, 0xb93: 0x1804, 0xb94: 0x138f, 0xb95: 0x1393, 0xb96: 0x13ab, 0xb97: 0x13b7, + 0xb98: 0x180e, 0xb99: 0x1660, 0xb9a: 0x13c3, 0xb9b: 0x13bf, 0xb9c: 0x13cb, 0xb9d: 0x1665, + 0xb9e: 0x13d7, 0xb9f: 0x13e3, 0xba0: 0x1813, 0xba1: 0x1818, 0xba2: 0x1423, 0xba3: 0x142f, + 0xba4: 0x1437, 0xba5: 0x181d, 0xba6: 0x143b, 0xba7: 0x1467, 0xba8: 0x1473, 0xba9: 0x1477, + 0xbaa: 0x146f, 0xbab: 0x1483, 0xbac: 0x1487, 0xbad: 0x1822, 0xbae: 0x1493, 0xbaf: 0x0693, + 0xbb0: 0x149b, 0xbb1: 0x1827, 0xbb2: 0x0697, 0xbb3: 0x14d3, 0xbb4: 0x0ac3, 0xbb5: 0x14eb, + 0xbb6: 0x182c, 0xbb7: 0x1836, 0xbb8: 0x069b, 0xbb9: 0x069f, 0xbba: 0x1513, 0xbbb: 0x183b, + 0xbbc: 0x06a3, 0xbbd: 0x1840, 0xbbe: 0x152b, 0xbbf: 0x152b, + // Block 0x2f, offset 0xbc0 + 0xbc0: 0x1533, 0xbc1: 0x1845, 0xbc2: 0x154b, 0xbc3: 0x06a7, 0xbc4: 0x155b, 0xbc5: 0x1567, + 0xbc6: 0x156f, 0xbc7: 0x1577, 0xbc8: 0x06ab, 0xbc9: 0x184a, 0xbca: 0x158b, 0xbcb: 0x15a7, + 0xbcc: 0x15b3, 0xbcd: 0x06af, 0xbce: 0x06b3, 0xbcf: 0x15b7, 0xbd0: 0x184f, 0xbd1: 0x06b7, + 0xbd2: 0x1854, 0xbd3: 0x1859, 0xbd4: 0x185e, 0xbd5: 0x15db, 0xbd6: 0x06bb, 0xbd7: 0x15ef, + 0xbd8: 0x15f7, 0xbd9: 0x15fb, 0xbda: 0x1603, 0xbdb: 0x160b, 0xbdc: 0x1613, 0xbdd: 0x1868, +} + +// nfcIndex: 22 blocks, 1408 entries, 1408 bytes +// Block 0 is the zero block. +var nfcIndex = [1408]uint8{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xc2: 0x2e, 0xc3: 0x01, 0xc4: 0x02, 0xc5: 0x03, 0xc6: 0x2f, 0xc7: 0x04, + 0xc8: 0x05, 0xca: 0x30, 0xcb: 0x31, 0xcc: 0x06, 0xcd: 0x07, 0xce: 0x08, 0xcf: 0x32, + 0xd0: 0x09, 0xd1: 0x33, 0xd2: 0x34, 0xd3: 0x0a, 0xd6: 0x0b, 0xd7: 0x35, + 0xd8: 0x36, 0xd9: 0x0c, 0xdb: 0x37, 0xdc: 0x38, 0xdd: 0x39, 0xdf: 0x3a, + 0xe0: 0x02, 0xe1: 0x03, 0xe2: 0x04, 0xe3: 0x05, + 0xea: 0x06, 0xeb: 0x07, 0xec: 0x08, 0xed: 0x09, 0xef: 0x0a, + 0xf0: 0x13, + // Block 0x4, offset 0x100 + 0x120: 0x3b, 0x121: 0x3c, 0x123: 0x0d, 0x124: 0x3d, 0x125: 0x3e, 0x126: 0x3f, 0x127: 0x40, + 0x128: 0x41, 0x129: 0x42, 0x12a: 0x43, 0x12b: 0x44, 0x12c: 0x3f, 0x12d: 0x45, 0x12e: 0x46, 0x12f: 0x47, + 0x131: 0x48, 0x132: 0x49, 0x133: 0x4a, 0x134: 0x4b, 0x135: 0x4c, 0x137: 0x4d, + 0x138: 0x4e, 0x139: 0x4f, 0x13a: 0x50, 0x13b: 0x51, 0x13c: 0x52, 0x13d: 0x53, 0x13e: 0x54, 0x13f: 0x55, + // Block 0x5, offset 0x140 + 0x140: 0x56, 0x142: 0x57, 0x144: 0x58, 0x145: 0x59, 0x146: 0x5a, 0x147: 0x5b, + 0x14d: 0x5c, + 0x15c: 0x5d, 0x15f: 0x5e, + 0x162: 0x5f, 0x164: 0x60, + 0x168: 0x61, 0x169: 0x62, 0x16a: 0x63, 0x16c: 0x0e, 0x16d: 0x64, 0x16e: 0x65, 0x16f: 0x66, + 0x170: 0x67, 0x173: 0x68, 0x177: 0x0f, + 0x178: 0x10, 0x179: 0x11, 0x17a: 0x12, 0x17b: 0x13, 0x17c: 0x14, 0x17d: 0x15, 0x17e: 0x16, 0x17f: 0x17, + // Block 0x6, offset 0x180 + 0x180: 0x69, 0x183: 0x6a, 0x184: 0x6b, 0x186: 0x6c, 0x187: 0x6d, + 0x188: 0x6e, 0x189: 0x18, 0x18a: 0x19, 0x18b: 0x6f, 0x18c: 0x70, + 0x1ab: 0x71, + 0x1b3: 0x72, 0x1b5: 0x73, 0x1b7: 0x74, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x75, 0x1c1: 0x1a, 0x1c2: 0x1b, 0x1c3: 0x1c, 0x1c4: 0x76, 0x1c5: 0x77, + 0x1c9: 0x78, 0x1cc: 0x79, 0x1cd: 0x7a, + // Block 0x8, offset 0x200 + 0x219: 0x7b, 0x21a: 0x7c, 0x21b: 0x7d, + 0x220: 0x7e, 0x223: 0x7f, 0x224: 0x80, 0x225: 0x81, 0x226: 0x82, 0x227: 0x83, + 0x22a: 0x84, 0x22b: 0x85, 0x22f: 0x86, + 0x230: 0x87, 0x231: 0x88, 0x232: 0x89, 0x233: 0x8a, 0x234: 0x8b, 0x235: 0x8c, 0x236: 0x8d, 0x237: 0x87, + 0x238: 0x88, 0x239: 0x89, 0x23a: 0x8a, 0x23b: 0x8b, 0x23c: 0x8c, 0x23d: 0x8d, 0x23e: 0x87, 0x23f: 0x88, + // Block 0x9, offset 0x240 + 0x240: 0x89, 0x241: 0x8a, 0x242: 0x8b, 0x243: 0x8c, 0x244: 0x8d, 0x245: 0x87, 0x246: 0x88, 0x247: 0x89, + 0x248: 0x8a, 0x249: 0x8b, 0x24a: 0x8c, 0x24b: 0x8d, 0x24c: 0x87, 0x24d: 0x88, 0x24e: 0x89, 0x24f: 0x8a, + 0x250: 0x8b, 0x251: 0x8c, 0x252: 0x8d, 0x253: 0x87, 0x254: 0x88, 0x255: 0x89, 0x256: 0x8a, 0x257: 0x8b, + 0x258: 0x8c, 0x259: 0x8d, 0x25a: 0x87, 0x25b: 0x88, 0x25c: 0x89, 0x25d: 0x8a, 0x25e: 0x8b, 0x25f: 0x8c, + 0x260: 0x8d, 0x261: 0x87, 0x262: 0x88, 0x263: 0x89, 0x264: 0x8a, 0x265: 0x8b, 0x266: 0x8c, 0x267: 0x8d, + 0x268: 0x87, 0x269: 0x88, 0x26a: 0x89, 0x26b: 0x8a, 0x26c: 0x8b, 0x26d: 0x8c, 0x26e: 0x8d, 0x26f: 0x87, + 0x270: 0x88, 0x271: 0x89, 0x272: 0x8a, 0x273: 0x8b, 0x274: 0x8c, 0x275: 0x8d, 0x276: 0x87, 0x277: 0x88, + 0x278: 0x89, 0x279: 0x8a, 0x27a: 0x8b, 0x27b: 0x8c, 0x27c: 0x8d, 0x27d: 0x87, 0x27e: 0x88, 0x27f: 0x89, + // Block 0xa, offset 0x280 + 0x280: 0x8a, 0x281: 0x8b, 0x282: 0x8c, 0x283: 0x8d, 0x284: 0x87, 0x285: 0x88, 0x286: 0x89, 0x287: 0x8a, + 0x288: 0x8b, 0x289: 0x8c, 0x28a: 0x8d, 0x28b: 0x87, 0x28c: 0x88, 0x28d: 0x89, 0x28e: 0x8a, 0x28f: 0x8b, + 0x290: 0x8c, 0x291: 0x8d, 0x292: 0x87, 0x293: 0x88, 0x294: 0x89, 0x295: 0x8a, 0x296: 0x8b, 0x297: 0x8c, + 0x298: 0x8d, 0x299: 0x87, 0x29a: 0x88, 0x29b: 0x89, 0x29c: 0x8a, 0x29d: 0x8b, 0x29e: 0x8c, 0x29f: 0x8d, + 0x2a0: 0x87, 0x2a1: 0x88, 0x2a2: 0x89, 0x2a3: 0x8a, 0x2a4: 0x8b, 0x2a5: 0x8c, 0x2a6: 0x8d, 0x2a7: 0x87, + 0x2a8: 0x88, 0x2a9: 0x89, 0x2aa: 0x8a, 0x2ab: 0x8b, 0x2ac: 0x8c, 0x2ad: 0x8d, 0x2ae: 0x87, 0x2af: 0x88, + 0x2b0: 0x89, 0x2b1: 0x8a, 0x2b2: 0x8b, 0x2b3: 0x8c, 0x2b4: 0x8d, 0x2b5: 0x87, 0x2b6: 0x88, 0x2b7: 0x89, + 0x2b8: 0x8a, 0x2b9: 0x8b, 0x2ba: 0x8c, 0x2bb: 0x8d, 0x2bc: 0x87, 0x2bd: 0x88, 0x2be: 0x89, 0x2bf: 0x8a, + // Block 0xb, offset 0x2c0 + 0x2c0: 0x8b, 0x2c1: 0x8c, 0x2c2: 0x8d, 0x2c3: 0x87, 0x2c4: 0x88, 0x2c5: 0x89, 0x2c6: 0x8a, 0x2c7: 0x8b, + 0x2c8: 0x8c, 0x2c9: 0x8d, 0x2ca: 0x87, 0x2cb: 0x88, 0x2cc: 0x89, 0x2cd: 0x8a, 0x2ce: 0x8b, 0x2cf: 0x8c, + 0x2d0: 0x8d, 0x2d1: 0x87, 0x2d2: 0x88, 0x2d3: 0x89, 0x2d4: 0x8a, 0x2d5: 0x8b, 0x2d6: 0x8c, 0x2d7: 0x8d, + 0x2d8: 0x87, 0x2d9: 0x88, 0x2da: 0x89, 0x2db: 0x8a, 0x2dc: 0x8b, 0x2dd: 0x8c, 0x2de: 0x8e, + // Block 0xc, offset 0x300 + 0x324: 0x1d, 0x325: 0x1e, 0x326: 0x1f, 0x327: 0x20, + 0x328: 0x21, 0x329: 0x22, 0x32a: 0x23, 0x32b: 0x24, 0x32c: 0x8f, 0x32d: 0x90, 0x32e: 0x91, + 0x331: 0x92, 0x332: 0x93, 0x333: 0x94, 0x334: 0x95, + 0x338: 0x96, 0x339: 0x97, 0x33a: 0x98, 0x33b: 0x99, 0x33e: 0x9a, 0x33f: 0x9b, + // Block 0xd, offset 0x340 + 0x347: 0x9c, + 0x34b: 0x9d, 0x34d: 0x9e, + 0x368: 0x9f, 0x36b: 0xa0, + 0x374: 0xa1, + 0x37d: 0xa2, + // Block 0xe, offset 0x380 + 0x381: 0xa3, 0x382: 0xa4, 0x384: 0xa5, 0x385: 0x82, 0x387: 0xa6, + 0x388: 0xa7, 0x38b: 0xa8, 0x38c: 0xa9, 0x38d: 0xaa, + 0x391: 0xab, 0x392: 0xac, 0x393: 0xad, 0x396: 0xae, 0x397: 0xaf, + 0x398: 0x73, 0x39a: 0xb0, 0x39c: 0xb1, + 0x3a0: 0xb2, 0x3a7: 0xb3, + 0x3a8: 0xb4, 0x3a9: 0xb5, 0x3aa: 0xb6, + 0x3b0: 0x73, 0x3b5: 0xb7, 0x3b6: 0xb8, + // Block 0xf, offset 0x3c0 + 0x3eb: 0xb9, 0x3ec: 0xba, + // Block 0x10, offset 0x400 + 0x432: 0xbb, + // Block 0x11, offset 0x440 + 0x445: 0xbc, 0x446: 0xbd, 0x447: 0xbe, + 0x449: 0xbf, + // Block 0x12, offset 0x480 + 0x480: 0xc0, 0x484: 0xba, + 0x48b: 0xc1, + 0x4a3: 0xc2, 0x4a5: 0xc3, + // Block 0x13, offset 0x4c0 + 0x4c8: 0xc4, + // Block 0x14, offset 0x500 + 0x520: 0x25, 0x521: 0x26, 0x522: 0x27, 0x523: 0x28, 0x524: 0x29, 0x525: 0x2a, 0x526: 0x2b, 0x527: 0x2c, + 0x528: 0x2d, + // Block 0x15, offset 0x540 + 0x550: 0x0b, 0x551: 0x0c, 0x556: 0x0d, + 0x55b: 0x0e, 0x55d: 0x0f, 0x55e: 0x10, 0x55f: 0x11, + 0x56f: 0x12, +} + +// nfcSparseOffset: 151 entries, 302 bytes +var nfcSparseOffset = []uint16{0x0, 0x5, 0x9, 0xb, 0xd, 0x18, 0x28, 0x2a, 0x2f, 0x3a, 0x49, 0x56, 0x5e, 0x63, 0x68, 0x6a, 0x72, 0x79, 0x7c, 0x84, 0x88, 0x8c, 0x8e, 0x90, 0x99, 0x9d, 0xa4, 0xa9, 0xac, 0xb6, 0xb9, 0xc0, 0xc8, 0xcb, 0xcd, 0xd0, 0xd2, 0xd7, 0xe8, 0xf4, 0xf6, 0xfc, 0xfe, 0x100, 0x102, 0x104, 0x106, 0x108, 0x10b, 0x10e, 0x110, 0x113, 0x116, 0x11a, 0x11f, 0x128, 0x12a, 0x12d, 0x12f, 0x13a, 0x13e, 0x14c, 0x14f, 0x155, 0x15b, 0x166, 0x16a, 0x16c, 0x16e, 0x170, 0x172, 0x174, 0x17a, 0x17e, 0x180, 0x182, 0x18a, 0x18e, 0x191, 0x193, 0x195, 0x197, 0x19a, 0x19c, 0x19e, 0x1a0, 0x1a2, 0x1a8, 0x1ab, 0x1ad, 0x1b4, 0x1ba, 0x1c0, 0x1c8, 0x1ce, 0x1d4, 0x1da, 0x1de, 0x1ec, 0x1f5, 0x1f8, 0x1fb, 0x1fd, 0x200, 0x202, 0x206, 0x20b, 0x20d, 0x20f, 0x214, 0x21a, 0x21c, 0x21e, 0x220, 0x226, 0x229, 0x22b, 0x231, 0x234, 0x23c, 0x243, 0x246, 0x249, 0x24b, 0x24e, 0x256, 0x25a, 0x261, 0x264, 0x26a, 0x26c, 0x26f, 0x271, 0x274, 0x276, 0x278, 0x27a, 0x27c, 0x27f, 0x281, 0x283, 0x285, 0x287, 0x294, 0x29e, 0x2a0, 0x2a2, 0x2a8, 0x2aa, 0x2ac, 0x2af} + +// nfcSparseValues: 689 entries, 2756 bytes +var nfcSparseValues = [689]valueRange{ + // Block 0x0, offset 0x0 + {value: 0x0000, lo: 0x04}, + {value: 0xa100, lo: 0xa8, hi: 0xa8}, + {value: 0x8100, lo: 0xaf, hi: 0xaf}, + {value: 0x8100, lo: 0xb4, hi: 0xb4}, + {value: 0x8100, lo: 0xb8, hi: 0xb8}, + // Block 0x1, offset 0x5 + {value: 0x0091, lo: 0x03}, + {value: 0x46e5, lo: 0xa0, hi: 0xa1}, + {value: 0x4717, lo: 0xaf, hi: 0xb0}, + {value: 0xa000, lo: 0xb7, hi: 0xb7}, + // Block 0x2, offset 0x9 + {value: 0x0000, lo: 0x01}, + {value: 0xa000, lo: 0x92, hi: 0x92}, + // Block 0x3, offset 0xb + {value: 0x0000, lo: 0x01}, + {value: 0x8100, lo: 0x98, hi: 0x9d}, + // Block 0x4, offset 0xd + {value: 0x0006, lo: 0x0a}, + {value: 0xa000, lo: 0x81, hi: 0x81}, + {value: 0xa000, lo: 0x85, hi: 0x85}, + {value: 0xa000, lo: 0x89, hi: 0x89}, + {value: 0x4843, lo: 0x8a, hi: 0x8a}, + {value: 0x4861, lo: 0x8b, hi: 0x8b}, + {value: 0x36ca, lo: 0x8c, hi: 0x8c}, + {value: 0x36e2, lo: 0x8d, hi: 0x8d}, + {value: 0x4879, lo: 0x8e, hi: 0x8e}, + {value: 0xa000, lo: 0x92, hi: 0x92}, + {value: 0x3700, lo: 0x93, hi: 0x94}, + // Block 0x5, offset 0x18 + {value: 0x0000, lo: 0x0f}, + {value: 0xa000, lo: 0x83, hi: 0x83}, + {value: 0xa000, lo: 0x87, hi: 0x87}, + {value: 0xa000, lo: 0x8b, hi: 0x8b}, + {value: 0xa000, lo: 0x8d, hi: 0x8d}, + {value: 0x37a8, lo: 0x90, hi: 0x90}, + {value: 0x37b4, lo: 0x91, hi: 0x91}, + {value: 0x37a2, lo: 0x93, hi: 0x93}, + {value: 0xa000, lo: 0x96, hi: 0x96}, + {value: 0x381a, lo: 0x97, hi: 0x97}, + {value: 0x37e4, lo: 0x9c, hi: 0x9c}, + {value: 0x37cc, lo: 0x9d, hi: 0x9d}, + {value: 0x37f6, lo: 0x9e, hi: 0x9e}, + {value: 0xa000, lo: 0xb4, hi: 0xb5}, + {value: 0x3820, lo: 0xb6, hi: 0xb6}, + {value: 0x3826, lo: 0xb7, hi: 0xb7}, + // Block 0x6, offset 0x28 + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0x83, hi: 0x87}, + // Block 0x7, offset 0x2a + {value: 0x0001, lo: 0x04}, + {value: 0x8113, lo: 0x81, hi: 0x82}, + {value: 0x8132, lo: 0x84, hi: 0x84}, + {value: 0x812d, lo: 0x85, hi: 0x85}, + {value: 0x810d, lo: 0x87, hi: 0x87}, + // Block 0x8, offset 0x2f + {value: 0x0000, lo: 0x0a}, + {value: 0x8132, lo: 0x90, hi: 0x97}, + {value: 0x8119, lo: 0x98, hi: 0x98}, + {value: 0x811a, lo: 0x99, hi: 0x99}, + {value: 0x811b, lo: 0x9a, hi: 0x9a}, + {value: 0x3844, lo: 0xa2, hi: 0xa2}, + {value: 0x384a, lo: 0xa3, hi: 0xa3}, + {value: 0x3856, lo: 0xa4, hi: 0xa4}, + {value: 0x3850, lo: 0xa5, hi: 0xa5}, + {value: 0x385c, lo: 0xa6, hi: 0xa6}, + {value: 0xa000, lo: 0xa7, hi: 0xa7}, + // Block 0x9, offset 0x3a + {value: 0x0000, lo: 0x0e}, + {value: 0x386e, lo: 0x80, hi: 0x80}, + {value: 0xa000, lo: 0x81, hi: 0x81}, + {value: 0x3862, lo: 0x82, hi: 0x82}, + {value: 0xa000, lo: 0x92, hi: 0x92}, + {value: 0x3868, lo: 0x93, hi: 0x93}, + {value: 0xa000, lo: 0x95, hi: 0x95}, + {value: 0x8132, lo: 0x96, hi: 0x9c}, + {value: 0x8132, lo: 0x9f, hi: 0xa2}, + {value: 0x812d, lo: 0xa3, hi: 0xa3}, + {value: 0x8132, lo: 0xa4, hi: 0xa4}, + {value: 0x8132, lo: 0xa7, hi: 0xa8}, + {value: 0x812d, lo: 0xaa, hi: 0xaa}, + {value: 0x8132, lo: 0xab, hi: 0xac}, + {value: 0x812d, lo: 0xad, hi: 0xad}, + // Block 0xa, offset 0x49 + {value: 0x0000, lo: 0x0c}, + {value: 0x811f, lo: 0x91, hi: 0x91}, + {value: 0x8132, lo: 0xb0, hi: 0xb0}, + {value: 0x812d, lo: 0xb1, hi: 0xb1}, + {value: 0x8132, lo: 0xb2, hi: 0xb3}, + {value: 0x812d, lo: 0xb4, hi: 0xb4}, + {value: 0x8132, lo: 0xb5, hi: 0xb6}, + {value: 0x812d, lo: 0xb7, hi: 0xb9}, + {value: 0x8132, lo: 0xba, hi: 0xba}, + {value: 0x812d, lo: 0xbb, hi: 0xbc}, + {value: 0x8132, lo: 0xbd, hi: 0xbd}, + {value: 0x812d, lo: 0xbe, hi: 0xbe}, + {value: 0x8132, lo: 0xbf, hi: 0xbf}, + // Block 0xb, offset 0x56 + {value: 0x0005, lo: 0x07}, + {value: 0x8132, lo: 0x80, hi: 0x80}, + {value: 0x8132, lo: 0x81, hi: 0x81}, + {value: 0x812d, lo: 0x82, hi: 0x83}, + {value: 0x812d, lo: 0x84, hi: 0x85}, + {value: 0x812d, lo: 0x86, hi: 0x87}, + {value: 0x812d, lo: 0x88, hi: 0x89}, + {value: 0x8132, lo: 0x8a, hi: 0x8a}, + // Block 0xc, offset 0x5e + {value: 0x0000, lo: 0x04}, + {value: 0x8132, lo: 0xab, hi: 0xb1}, + {value: 0x812d, lo: 0xb2, hi: 0xb2}, + {value: 0x8132, lo: 0xb3, hi: 0xb3}, + {value: 0x812d, lo: 0xbd, hi: 0xbd}, + // Block 0xd, offset 0x63 + {value: 0x0000, lo: 0x04}, + {value: 0x8132, lo: 0x96, hi: 0x99}, + {value: 0x8132, lo: 0x9b, hi: 0xa3}, + {value: 0x8132, lo: 0xa5, hi: 0xa7}, + {value: 0x8132, lo: 0xa9, hi: 0xad}, + // Block 0xe, offset 0x68 + {value: 0x0000, lo: 0x01}, + {value: 0x812d, lo: 0x99, hi: 0x9b}, + // Block 0xf, offset 0x6a + {value: 0x0000, lo: 0x07}, + {value: 0xa000, lo: 0xa8, hi: 0xa8}, + {value: 0x3edb, lo: 0xa9, hi: 0xa9}, + {value: 0xa000, lo: 0xb0, hi: 0xb0}, + {value: 0x3ee3, lo: 0xb1, hi: 0xb1}, + {value: 0xa000, lo: 0xb3, hi: 0xb3}, + {value: 0x3eeb, lo: 0xb4, hi: 0xb4}, + {value: 0x9902, lo: 0xbc, hi: 0xbc}, + // Block 0x10, offset 0x72 + {value: 0x0008, lo: 0x06}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x8132, lo: 0x91, hi: 0x91}, + {value: 0x812d, lo: 0x92, hi: 0x92}, + {value: 0x8132, lo: 0x93, hi: 0x93}, + {value: 0x8132, lo: 0x94, hi: 0x94}, + {value: 0x451f, lo: 0x98, hi: 0x9f}, + // Block 0x11, offset 0x79 + {value: 0x0000, lo: 0x02}, + {value: 0x8102, lo: 0xbc, hi: 0xbc}, + {value: 0x9900, lo: 0xbe, hi: 0xbe}, + // Block 0x12, offset 0x7c + {value: 0x0008, lo: 0x07}, + {value: 0xa000, lo: 0x87, hi: 0x87}, + {value: 0x2ca1, lo: 0x8b, hi: 0x8c}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x9900, lo: 0x97, hi: 0x97}, + {value: 0x455f, lo: 0x9c, hi: 0x9d}, + {value: 0x456f, lo: 0x9f, hi: 0x9f}, + {value: 0x8132, lo: 0xbe, hi: 0xbe}, + // Block 0x13, offset 0x84 + {value: 0x0000, lo: 0x03}, + {value: 0x4597, lo: 0xb3, hi: 0xb3}, + {value: 0x459f, lo: 0xb6, hi: 0xb6}, + {value: 0x8102, lo: 0xbc, hi: 0xbc}, + // Block 0x14, offset 0x88 + {value: 0x0008, lo: 0x03}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x4577, lo: 0x99, hi: 0x9b}, + {value: 0x458f, lo: 0x9e, hi: 0x9e}, + // Block 0x15, offset 0x8c + {value: 0x0000, lo: 0x01}, + {value: 0x8102, lo: 0xbc, hi: 0xbc}, + // Block 0x16, offset 0x8e + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + // Block 0x17, offset 0x90 + {value: 0x0000, lo: 0x08}, + {value: 0xa000, lo: 0x87, hi: 0x87}, + {value: 0x2cb9, lo: 0x88, hi: 0x88}, + {value: 0x2cb1, lo: 0x8b, hi: 0x8b}, + {value: 0x2cc1, lo: 0x8c, hi: 0x8c}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x9900, lo: 0x96, hi: 0x97}, + {value: 0x45a7, lo: 0x9c, hi: 0x9c}, + {value: 0x45af, lo: 0x9d, hi: 0x9d}, + // Block 0x18, offset 0x99 + {value: 0x0000, lo: 0x03}, + {value: 0xa000, lo: 0x92, hi: 0x92}, + {value: 0x2cc9, lo: 0x94, hi: 0x94}, + {value: 0x9900, lo: 0xbe, hi: 0xbe}, + // Block 0x19, offset 0x9d + {value: 0x0000, lo: 0x06}, + {value: 0xa000, lo: 0x86, hi: 0x87}, + {value: 0x2cd1, lo: 0x8a, hi: 0x8a}, + {value: 0x2ce1, lo: 0x8b, hi: 0x8b}, + {value: 0x2cd9, lo: 0x8c, hi: 0x8c}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x9900, lo: 0x97, hi: 0x97}, + // Block 0x1a, offset 0xa4 + {value: 0x1801, lo: 0x04}, + {value: 0xa000, lo: 0x86, hi: 0x86}, + {value: 0x3ef3, lo: 0x88, hi: 0x88}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x8120, lo: 0x95, hi: 0x96}, + // Block 0x1b, offset 0xa9 + {value: 0x0000, lo: 0x02}, + {value: 0x8102, lo: 0xbc, hi: 0xbc}, + {value: 0xa000, lo: 0xbf, hi: 0xbf}, + // Block 0x1c, offset 0xac + {value: 0x0000, lo: 0x09}, + {value: 0x2ce9, lo: 0x80, hi: 0x80}, + {value: 0x9900, lo: 0x82, hi: 0x82}, + {value: 0xa000, lo: 0x86, hi: 0x86}, + {value: 0x2cf1, lo: 0x87, hi: 0x87}, + {value: 0x2cf9, lo: 0x88, hi: 0x88}, + {value: 0x2f53, lo: 0x8a, hi: 0x8a}, + {value: 0x2ddb, lo: 0x8b, hi: 0x8b}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x9900, lo: 0x95, hi: 0x96}, + // Block 0x1d, offset 0xb6 + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0xbb, hi: 0xbc}, + {value: 0x9900, lo: 0xbe, hi: 0xbe}, + // Block 0x1e, offset 0xb9 + {value: 0x0000, lo: 0x06}, + {value: 0xa000, lo: 0x86, hi: 0x87}, + {value: 0x2d01, lo: 0x8a, hi: 0x8a}, + {value: 0x2d11, lo: 0x8b, hi: 0x8b}, + {value: 0x2d09, lo: 0x8c, hi: 0x8c}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x9900, lo: 0x97, hi: 0x97}, + // Block 0x1f, offset 0xc0 + {value: 0x6be7, lo: 0x07}, + {value: 0x9904, lo: 0x8a, hi: 0x8a}, + {value: 0x9900, lo: 0x8f, hi: 0x8f}, + {value: 0xa000, lo: 0x99, hi: 0x99}, + {value: 0x3efb, lo: 0x9a, hi: 0x9a}, + {value: 0x2f5b, lo: 0x9c, hi: 0x9c}, + {value: 0x2de6, lo: 0x9d, hi: 0x9d}, + {value: 0x2d19, lo: 0x9e, hi: 0x9f}, + // Block 0x20, offset 0xc8 + {value: 0x0000, lo: 0x02}, + {value: 0x8122, lo: 0xb8, hi: 0xb9}, + {value: 0x8104, lo: 0xba, hi: 0xba}, + // Block 0x21, offset 0xcb + {value: 0x0000, lo: 0x01}, + {value: 0x8123, lo: 0x88, hi: 0x8b}, + // Block 0x22, offset 0xcd + {value: 0x0000, lo: 0x02}, + {value: 0x8124, lo: 0xb8, hi: 0xb9}, + {value: 0x8104, lo: 0xba, hi: 0xba}, + // Block 0x23, offset 0xd0 + {value: 0x0000, lo: 0x01}, + {value: 0x8125, lo: 0x88, hi: 0x8b}, + // Block 0x24, offset 0xd2 + {value: 0x0000, lo: 0x04}, + {value: 0x812d, lo: 0x98, hi: 0x99}, + {value: 0x812d, lo: 0xb5, hi: 0xb5}, + {value: 0x812d, lo: 0xb7, hi: 0xb7}, + {value: 0x812b, lo: 0xb9, hi: 0xb9}, + // Block 0x25, offset 0xd7 + {value: 0x0000, lo: 0x10}, + {value: 0x2647, lo: 0x83, hi: 0x83}, + {value: 0x264e, lo: 0x8d, hi: 0x8d}, + {value: 0x2655, lo: 0x92, hi: 0x92}, + {value: 0x265c, lo: 0x97, hi: 0x97}, + {value: 0x2663, lo: 0x9c, hi: 0x9c}, + {value: 0x2640, lo: 0xa9, hi: 0xa9}, + {value: 0x8126, lo: 0xb1, hi: 0xb1}, + {value: 0x8127, lo: 0xb2, hi: 0xb2}, + {value: 0x4a87, lo: 0xb3, hi: 0xb3}, + {value: 0x8128, lo: 0xb4, hi: 0xb4}, + {value: 0x4a90, lo: 0xb5, hi: 0xb5}, + {value: 0x45b7, lo: 0xb6, hi: 0xb6}, + {value: 0x8200, lo: 0xb7, hi: 0xb7}, + {value: 0x45bf, lo: 0xb8, hi: 0xb8}, + {value: 0x8200, lo: 0xb9, hi: 0xb9}, + {value: 0x8127, lo: 0xba, hi: 0xbd}, + // Block 0x26, offset 0xe8 + {value: 0x0000, lo: 0x0b}, + {value: 0x8127, lo: 0x80, hi: 0x80}, + {value: 0x4a99, lo: 0x81, hi: 0x81}, + {value: 0x8132, lo: 0x82, hi: 0x83}, + {value: 0x8104, lo: 0x84, hi: 0x84}, + {value: 0x8132, lo: 0x86, hi: 0x87}, + {value: 0x2671, lo: 0x93, hi: 0x93}, + {value: 0x2678, lo: 0x9d, hi: 0x9d}, + {value: 0x267f, lo: 0xa2, hi: 0xa2}, + {value: 0x2686, lo: 0xa7, hi: 0xa7}, + {value: 0x268d, lo: 0xac, hi: 0xac}, + {value: 0x266a, lo: 0xb9, hi: 0xb9}, + // Block 0x27, offset 0xf4 + {value: 0x0000, lo: 0x01}, + {value: 0x812d, lo: 0x86, hi: 0x86}, + // Block 0x28, offset 0xf6 + {value: 0x0000, lo: 0x05}, + {value: 0xa000, lo: 0xa5, hi: 0xa5}, + {value: 0x2d21, lo: 0xa6, hi: 0xa6}, + {value: 0x9900, lo: 0xae, hi: 0xae}, + {value: 0x8102, lo: 0xb7, hi: 0xb7}, + {value: 0x8104, lo: 0xb9, hi: 0xba}, + // Block 0x29, offset 0xfc + {value: 0x0000, lo: 0x01}, + {value: 0x812d, lo: 0x8d, hi: 0x8d}, + // Block 0x2a, offset 0xfe + {value: 0x0000, lo: 0x01}, + {value: 0xa000, lo: 0x80, hi: 0x92}, + // Block 0x2b, offset 0x100 + {value: 0x0000, lo: 0x01}, + {value: 0xb900, lo: 0xa1, hi: 0xb5}, + // Block 0x2c, offset 0x102 + {value: 0x0000, lo: 0x01}, + {value: 0x9900, lo: 0xa8, hi: 0xbf}, + // Block 0x2d, offset 0x104 + {value: 0x0000, lo: 0x01}, + {value: 0x9900, lo: 0x80, hi: 0x82}, + // Block 0x2e, offset 0x106 + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0x9d, hi: 0x9f}, + // Block 0x2f, offset 0x108 + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0x94, hi: 0x94}, + {value: 0x8104, lo: 0xb4, hi: 0xb4}, + // Block 0x30, offset 0x10b + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0x92, hi: 0x92}, + {value: 0x8132, lo: 0x9d, hi: 0x9d}, + // Block 0x31, offset 0x10e + {value: 0x0000, lo: 0x01}, + {value: 0x8131, lo: 0xa9, hi: 0xa9}, + // Block 0x32, offset 0x110 + {value: 0x0004, lo: 0x02}, + {value: 0x812e, lo: 0xb9, hi: 0xba}, + {value: 0x812d, lo: 0xbb, hi: 0xbb}, + // Block 0x33, offset 0x113 + {value: 0x0000, lo: 0x02}, + {value: 0x8132, lo: 0x97, hi: 0x97}, + {value: 0x812d, lo: 0x98, hi: 0x98}, + // Block 0x34, offset 0x116 + {value: 0x0000, lo: 0x03}, + {value: 0x8104, lo: 0xa0, hi: 0xa0}, + {value: 0x8132, lo: 0xb5, hi: 0xbc}, + {value: 0x812d, lo: 0xbf, hi: 0xbf}, + // Block 0x35, offset 0x11a + {value: 0x0000, lo: 0x04}, + {value: 0x8132, lo: 0xb0, hi: 0xb4}, + {value: 0x812d, lo: 0xb5, hi: 0xba}, + {value: 0x8132, lo: 0xbb, hi: 0xbc}, + {value: 0x812d, lo: 0xbd, hi: 0xbd}, + // Block 0x36, offset 0x11f + {value: 0x0000, lo: 0x08}, + {value: 0x2d69, lo: 0x80, hi: 0x80}, + {value: 0x2d71, lo: 0x81, hi: 0x81}, + {value: 0xa000, lo: 0x82, hi: 0x82}, + {value: 0x2d79, lo: 0x83, hi: 0x83}, + {value: 0x8104, lo: 0x84, hi: 0x84}, + {value: 0x8132, lo: 0xab, hi: 0xab}, + {value: 0x812d, lo: 0xac, hi: 0xac}, + {value: 0x8132, lo: 0xad, hi: 0xb3}, + // Block 0x37, offset 0x128 + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0xaa, hi: 0xab}, + // Block 0x38, offset 0x12a + {value: 0x0000, lo: 0x02}, + {value: 0x8102, lo: 0xa6, hi: 0xa6}, + {value: 0x8104, lo: 0xb2, hi: 0xb3}, + // Block 0x39, offset 0x12d + {value: 0x0000, lo: 0x01}, + {value: 0x8102, lo: 0xb7, hi: 0xb7}, + // Block 0x3a, offset 0x12f + {value: 0x0000, lo: 0x0a}, + {value: 0x8132, lo: 0x90, hi: 0x92}, + {value: 0x8101, lo: 0x94, hi: 0x94}, + {value: 0x812d, lo: 0x95, hi: 0x99}, + {value: 0x8132, lo: 0x9a, hi: 0x9b}, + {value: 0x812d, lo: 0x9c, hi: 0x9f}, + {value: 0x8132, lo: 0xa0, hi: 0xa0}, + {value: 0x8101, lo: 0xa2, hi: 0xa8}, + {value: 0x812d, lo: 0xad, hi: 0xad}, + {value: 0x8132, lo: 0xb4, hi: 0xb4}, + {value: 0x8132, lo: 0xb8, hi: 0xb9}, + // Block 0x3b, offset 0x13a + {value: 0x0004, lo: 0x03}, + {value: 0x0433, lo: 0x80, hi: 0x81}, + {value: 0x8100, lo: 0x97, hi: 0x97}, + {value: 0x8100, lo: 0xbe, hi: 0xbe}, + // Block 0x3c, offset 0x13e + {value: 0x0000, lo: 0x0d}, + {value: 0x8132, lo: 0x90, hi: 0x91}, + {value: 0x8101, lo: 0x92, hi: 0x93}, + {value: 0x8132, lo: 0x94, hi: 0x97}, + {value: 0x8101, lo: 0x98, hi: 0x9a}, + {value: 0x8132, lo: 0x9b, hi: 0x9c}, + {value: 0x8132, lo: 0xa1, hi: 0xa1}, + {value: 0x8101, lo: 0xa5, hi: 0xa6}, + {value: 0x8132, lo: 0xa7, hi: 0xa7}, + {value: 0x812d, lo: 0xa8, hi: 0xa8}, + {value: 0x8132, lo: 0xa9, hi: 0xa9}, + {value: 0x8101, lo: 0xaa, hi: 0xab}, + {value: 0x812d, lo: 0xac, hi: 0xaf}, + {value: 0x8132, lo: 0xb0, hi: 0xb0}, + // Block 0x3d, offset 0x14c + {value: 0x427e, lo: 0x02}, + {value: 0x01b8, lo: 0xa6, hi: 0xa6}, + {value: 0x0057, lo: 0xaa, hi: 0xab}, + // Block 0x3e, offset 0x14f + {value: 0x0007, lo: 0x05}, + {value: 0xa000, lo: 0x90, hi: 0x90}, + {value: 0xa000, lo: 0x92, hi: 0x92}, + {value: 0xa000, lo: 0x94, hi: 0x94}, + {value: 0x3bbc, lo: 0x9a, hi: 0x9b}, + {value: 0x3bca, lo: 0xae, hi: 0xae}, + // Block 0x3f, offset 0x155 + {value: 0x000e, lo: 0x05}, + {value: 0x3bd1, lo: 0x8d, hi: 0x8e}, + {value: 0x3bd8, lo: 0x8f, hi: 0x8f}, + {value: 0xa000, lo: 0x90, hi: 0x90}, + {value: 0xa000, lo: 0x92, hi: 0x92}, + {value: 0xa000, lo: 0x94, hi: 0x94}, + // Block 0x40, offset 0x15b + {value: 0x6405, lo: 0x0a}, + {value: 0xa000, lo: 0x83, hi: 0x83}, + {value: 0x3be6, lo: 0x84, hi: 0x84}, + {value: 0xa000, lo: 0x88, hi: 0x88}, + {value: 0x3bed, lo: 0x89, hi: 0x89}, + {value: 0xa000, lo: 0x8b, hi: 0x8b}, + {value: 0x3bf4, lo: 0x8c, hi: 0x8c}, + {value: 0xa000, lo: 0xa3, hi: 0xa3}, + {value: 0x3bfb, lo: 0xa4, hi: 0xa5}, + {value: 0x3c02, lo: 0xa6, hi: 0xa6}, + {value: 0xa000, lo: 0xbc, hi: 0xbc}, + // Block 0x41, offset 0x166 + {value: 0x0007, lo: 0x03}, + {value: 0x3c6b, lo: 0xa0, hi: 0xa1}, + {value: 0x3c95, lo: 0xa2, hi: 0xa3}, + {value: 0x3cbf, lo: 0xaa, hi: 0xad}, + // Block 0x42, offset 0x16a + {value: 0x0004, lo: 0x01}, + {value: 0x048b, lo: 0xa9, hi: 0xaa}, + // Block 0x43, offset 0x16c + {value: 0x0000, lo: 0x01}, + {value: 0x44e0, lo: 0x9c, hi: 0x9c}, + // Block 0x44, offset 0x16e + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0xaf, hi: 0xb1}, + // Block 0x45, offset 0x170 + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0xbf, hi: 0xbf}, + // Block 0x46, offset 0x172 + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0xa0, hi: 0xbf}, + // Block 0x47, offset 0x174 + {value: 0x0000, lo: 0x05}, + {value: 0x812c, lo: 0xaa, hi: 0xaa}, + {value: 0x8131, lo: 0xab, hi: 0xab}, + {value: 0x8133, lo: 0xac, hi: 0xac}, + {value: 0x812e, lo: 0xad, hi: 0xad}, + {value: 0x812f, lo: 0xae, hi: 0xaf}, + // Block 0x48, offset 0x17a + {value: 0x0000, lo: 0x03}, + {value: 0x4aa2, lo: 0xb3, hi: 0xb3}, + {value: 0x4aa2, lo: 0xb5, hi: 0xb6}, + {value: 0x4aa2, lo: 0xba, hi: 0xbf}, + // Block 0x49, offset 0x17e + {value: 0x0000, lo: 0x01}, + {value: 0x4aa2, lo: 0x8f, hi: 0xa3}, + // Block 0x4a, offset 0x180 + {value: 0x0000, lo: 0x01}, + {value: 0x8100, lo: 0xae, hi: 0xbe}, + // Block 0x4b, offset 0x182 + {value: 0x0000, lo: 0x07}, + {value: 0x8100, lo: 0x84, hi: 0x84}, + {value: 0x8100, lo: 0x87, hi: 0x87}, + {value: 0x8100, lo: 0x90, hi: 0x90}, + {value: 0x8100, lo: 0x9e, hi: 0x9e}, + {value: 0x8100, lo: 0xa1, hi: 0xa1}, + {value: 0x8100, lo: 0xb2, hi: 0xb2}, + {value: 0x8100, lo: 0xbb, hi: 0xbb}, + // Block 0x4c, offset 0x18a + {value: 0x0000, lo: 0x03}, + {value: 0x8100, lo: 0x80, hi: 0x80}, + {value: 0x8100, lo: 0x8b, hi: 0x8b}, + {value: 0x8100, lo: 0x8e, hi: 0x8e}, + // Block 0x4d, offset 0x18e + {value: 0x0000, lo: 0x02}, + {value: 0x8132, lo: 0xaf, hi: 0xaf}, + {value: 0x8132, lo: 0xb4, hi: 0xbd}, + // Block 0x4e, offset 0x191 + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0x9e, hi: 0x9f}, + // Block 0x4f, offset 0x193 + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0xb0, hi: 0xb1}, + // Block 0x50, offset 0x195 + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0x86, hi: 0x86}, + // Block 0x51, offset 0x197 + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0x84, hi: 0x84}, + {value: 0x8132, lo: 0xa0, hi: 0xb1}, + // Block 0x52, offset 0x19a + {value: 0x0000, lo: 0x01}, + {value: 0x812d, lo: 0xab, hi: 0xad}, + // Block 0x53, offset 0x19c + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0x93, hi: 0x93}, + // Block 0x54, offset 0x19e + {value: 0x0000, lo: 0x01}, + {value: 0x8102, lo: 0xb3, hi: 0xb3}, + // Block 0x55, offset 0x1a0 + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0x80, hi: 0x80}, + // Block 0x56, offset 0x1a2 + {value: 0x0000, lo: 0x05}, + {value: 0x8132, lo: 0xb0, hi: 0xb0}, + {value: 0x8132, lo: 0xb2, hi: 0xb3}, + {value: 0x812d, lo: 0xb4, hi: 0xb4}, + {value: 0x8132, lo: 0xb7, hi: 0xb8}, + {value: 0x8132, lo: 0xbe, hi: 0xbf}, + // Block 0x57, offset 0x1a8 + {value: 0x0000, lo: 0x02}, + {value: 0x8132, lo: 0x81, hi: 0x81}, + {value: 0x8104, lo: 0xb6, hi: 0xb6}, + // Block 0x58, offset 0x1ab + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0xad, hi: 0xad}, + // Block 0x59, offset 0x1ad + {value: 0x0000, lo: 0x06}, + {value: 0xe500, lo: 0x80, hi: 0x80}, + {value: 0xc600, lo: 0x81, hi: 0x9b}, + {value: 0xe500, lo: 0x9c, hi: 0x9c}, + {value: 0xc600, lo: 0x9d, hi: 0xb7}, + {value: 0xe500, lo: 0xb8, hi: 0xb8}, + {value: 0xc600, lo: 0xb9, hi: 0xbf}, + // Block 0x5a, offset 0x1b4 + {value: 0x0000, lo: 0x05}, + {value: 0xc600, lo: 0x80, hi: 0x93}, + {value: 0xe500, lo: 0x94, hi: 0x94}, + {value: 0xc600, lo: 0x95, hi: 0xaf}, + {value: 0xe500, lo: 0xb0, hi: 0xb0}, + {value: 0xc600, lo: 0xb1, hi: 0xbf}, + // Block 0x5b, offset 0x1ba + {value: 0x0000, lo: 0x05}, + {value: 0xc600, lo: 0x80, hi: 0x8b}, + {value: 0xe500, lo: 0x8c, hi: 0x8c}, + {value: 0xc600, lo: 0x8d, hi: 0xa7}, + {value: 0xe500, lo: 0xa8, hi: 0xa8}, + {value: 0xc600, lo: 0xa9, hi: 0xbf}, + // Block 0x5c, offset 0x1c0 + {value: 0x0000, lo: 0x07}, + {value: 0xc600, lo: 0x80, hi: 0x83}, + {value: 0xe500, lo: 0x84, hi: 0x84}, + {value: 0xc600, lo: 0x85, hi: 0x9f}, + {value: 0xe500, lo: 0xa0, hi: 0xa0}, + {value: 0xc600, lo: 0xa1, hi: 0xbb}, + {value: 0xe500, lo: 0xbc, hi: 0xbc}, + {value: 0xc600, lo: 0xbd, hi: 0xbf}, + // Block 0x5d, offset 0x1c8 + {value: 0x0000, lo: 0x05}, + {value: 0xc600, lo: 0x80, hi: 0x97}, + {value: 0xe500, lo: 0x98, hi: 0x98}, + {value: 0xc600, lo: 0x99, hi: 0xb3}, + {value: 0xe500, lo: 0xb4, hi: 0xb4}, + {value: 0xc600, lo: 0xb5, hi: 0xbf}, + // Block 0x5e, offset 0x1ce + {value: 0x0000, lo: 0x05}, + {value: 0xc600, lo: 0x80, hi: 0x8f}, + {value: 0xe500, lo: 0x90, hi: 0x90}, + {value: 0xc600, lo: 0x91, hi: 0xab}, + {value: 0xe500, lo: 0xac, hi: 0xac}, + {value: 0xc600, lo: 0xad, hi: 0xbf}, + // Block 0x5f, offset 0x1d4 + {value: 0x0000, lo: 0x05}, + {value: 0xc600, lo: 0x80, hi: 0x87}, + {value: 0xe500, lo: 0x88, hi: 0x88}, + {value: 0xc600, lo: 0x89, hi: 0xa3}, + {value: 0xe500, lo: 0xa4, hi: 0xa4}, + {value: 0xc600, lo: 0xa5, hi: 0xbf}, + // Block 0x60, offset 0x1da + {value: 0x0000, lo: 0x03}, + {value: 0xc600, lo: 0x80, hi: 0x87}, + {value: 0xe500, lo: 0x88, hi: 0x88}, + {value: 0xc600, lo: 0x89, hi: 0xa3}, + // Block 0x61, offset 0x1de + {value: 0x0006, lo: 0x0d}, + {value: 0x4393, lo: 0x9d, hi: 0x9d}, + {value: 0x8115, lo: 0x9e, hi: 0x9e}, + {value: 0x4405, lo: 0x9f, hi: 0x9f}, + {value: 0x43f3, lo: 0xaa, hi: 0xab}, + {value: 0x44f7, lo: 0xac, hi: 0xac}, + {value: 0x44ff, lo: 0xad, hi: 0xad}, + {value: 0x434b, lo: 0xae, hi: 0xb1}, + {value: 0x4369, lo: 0xb2, hi: 0xb4}, + {value: 0x4381, lo: 0xb5, hi: 0xb6}, + {value: 0x438d, lo: 0xb8, hi: 0xb8}, + {value: 0x4399, lo: 0xb9, hi: 0xbb}, + {value: 0x43b1, lo: 0xbc, hi: 0xbc}, + {value: 0x43b7, lo: 0xbe, hi: 0xbe}, + // Block 0x62, offset 0x1ec + {value: 0x0006, lo: 0x08}, + {value: 0x43bd, lo: 0x80, hi: 0x81}, + {value: 0x43c9, lo: 0x83, hi: 0x84}, + {value: 0x43db, lo: 0x86, hi: 0x89}, + {value: 0x43ff, lo: 0x8a, hi: 0x8a}, + {value: 0x437b, lo: 0x8b, hi: 0x8b}, + {value: 0x4363, lo: 0x8c, hi: 0x8c}, + {value: 0x43ab, lo: 0x8d, hi: 0x8d}, + {value: 0x43d5, lo: 0x8e, hi: 0x8e}, + // Block 0x63, offset 0x1f5 + {value: 0x0000, lo: 0x02}, + {value: 0x8100, lo: 0xa4, hi: 0xa5}, + {value: 0x8100, lo: 0xb0, hi: 0xb1}, + // Block 0x64, offset 0x1f8 + {value: 0x0000, lo: 0x02}, + {value: 0x8100, lo: 0x9b, hi: 0x9d}, + {value: 0x8200, lo: 0x9e, hi: 0xa3}, + // Block 0x65, offset 0x1fb + {value: 0x0000, lo: 0x01}, + {value: 0x8100, lo: 0x90, hi: 0x90}, + // Block 0x66, offset 0x1fd + {value: 0x0000, lo: 0x02}, + {value: 0x8100, lo: 0x99, hi: 0x99}, + {value: 0x8200, lo: 0xb2, hi: 0xb4}, + // Block 0x67, offset 0x200 + {value: 0x0000, lo: 0x01}, + {value: 0x8100, lo: 0xbc, hi: 0xbd}, + // Block 0x68, offset 0x202 + {value: 0x0000, lo: 0x03}, + {value: 0x8132, lo: 0xa0, hi: 0xa6}, + {value: 0x812d, lo: 0xa7, hi: 0xad}, + {value: 0x8132, lo: 0xae, hi: 0xaf}, + // Block 0x69, offset 0x206 + {value: 0x0000, lo: 0x04}, + {value: 0x8100, lo: 0x89, hi: 0x8c}, + {value: 0x8100, lo: 0xb0, hi: 0xb2}, + {value: 0x8100, lo: 0xb4, hi: 0xb4}, + {value: 0x8100, lo: 0xb6, hi: 0xbf}, + // Block 0x6a, offset 0x20b + {value: 0x0000, lo: 0x01}, + {value: 0x8100, lo: 0x81, hi: 0x8c}, + // Block 0x6b, offset 0x20d + {value: 0x0000, lo: 0x01}, + {value: 0x8100, lo: 0xb5, hi: 0xba}, + // Block 0x6c, offset 0x20f + {value: 0x0000, lo: 0x04}, + {value: 0x4aa2, lo: 0x9e, hi: 0x9f}, + {value: 0x4aa2, lo: 0xa3, hi: 0xa3}, + {value: 0x4aa2, lo: 0xa5, hi: 0xa6}, + {value: 0x4aa2, lo: 0xaa, hi: 0xaf}, + // Block 0x6d, offset 0x214 + {value: 0x0000, lo: 0x05}, + {value: 0x4aa2, lo: 0x82, hi: 0x87}, + {value: 0x4aa2, lo: 0x8a, hi: 0x8f}, + {value: 0x4aa2, lo: 0x92, hi: 0x97}, + {value: 0x4aa2, lo: 0x9a, hi: 0x9c}, + {value: 0x8100, lo: 0xa3, hi: 0xa3}, + // Block 0x6e, offset 0x21a + {value: 0x0000, lo: 0x01}, + {value: 0x812d, lo: 0xbd, hi: 0xbd}, + // Block 0x6f, offset 0x21c + {value: 0x0000, lo: 0x01}, + {value: 0x812d, lo: 0xa0, hi: 0xa0}, + // Block 0x70, offset 0x21e + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0xb6, hi: 0xba}, + // Block 0x71, offset 0x220 + {value: 0x002c, lo: 0x05}, + {value: 0x812d, lo: 0x8d, hi: 0x8d}, + {value: 0x8132, lo: 0x8f, hi: 0x8f}, + {value: 0x8132, lo: 0xb8, hi: 0xb8}, + {value: 0x8101, lo: 0xb9, hi: 0xba}, + {value: 0x8104, lo: 0xbf, hi: 0xbf}, + // Block 0x72, offset 0x226 + {value: 0x0000, lo: 0x02}, + {value: 0x8132, lo: 0xa5, hi: 0xa5}, + {value: 0x812d, lo: 0xa6, hi: 0xa6}, + // Block 0x73, offset 0x229 + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0xa4, hi: 0xa7}, + // Block 0x74, offset 0x22b + {value: 0x0000, lo: 0x05}, + {value: 0x812d, lo: 0x86, hi: 0x87}, + {value: 0x8132, lo: 0x88, hi: 0x8a}, + {value: 0x812d, lo: 0x8b, hi: 0x8b}, + {value: 0x8132, lo: 0x8c, hi: 0x8c}, + {value: 0x812d, lo: 0x8d, hi: 0x90}, + // Block 0x75, offset 0x231 + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0x86, hi: 0x86}, + {value: 0x8104, lo: 0xbf, hi: 0xbf}, + // Block 0x76, offset 0x234 + {value: 0x17fe, lo: 0x07}, + {value: 0xa000, lo: 0x99, hi: 0x99}, + {value: 0x423b, lo: 0x9a, hi: 0x9a}, + {value: 0xa000, lo: 0x9b, hi: 0x9b}, + {value: 0x4245, lo: 0x9c, hi: 0x9c}, + {value: 0xa000, lo: 0xa5, hi: 0xa5}, + {value: 0x424f, lo: 0xab, hi: 0xab}, + {value: 0x8104, lo: 0xb9, hi: 0xba}, + // Block 0x77, offset 0x23c + {value: 0x0000, lo: 0x06}, + {value: 0x8132, lo: 0x80, hi: 0x82}, + {value: 0x9900, lo: 0xa7, hi: 0xa7}, + {value: 0x2d81, lo: 0xae, hi: 0xae}, + {value: 0x2d8b, lo: 0xaf, hi: 0xaf}, + {value: 0xa000, lo: 0xb1, hi: 0xb2}, + {value: 0x8104, lo: 0xb3, hi: 0xb4}, + // Block 0x78, offset 0x243 + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0x80, hi: 0x80}, + {value: 0x8102, lo: 0x8a, hi: 0x8a}, + // Block 0x79, offset 0x246 + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0xb5, hi: 0xb5}, + {value: 0x8102, lo: 0xb6, hi: 0xb6}, + // Block 0x7a, offset 0x249 + {value: 0x0002, lo: 0x01}, + {value: 0x8102, lo: 0xa9, hi: 0xaa}, + // Block 0x7b, offset 0x24b + {value: 0x0000, lo: 0x02}, + {value: 0x8102, lo: 0xbb, hi: 0xbc}, + {value: 0x9900, lo: 0xbe, hi: 0xbe}, + // Block 0x7c, offset 0x24e + {value: 0x0000, lo: 0x07}, + {value: 0xa000, lo: 0x87, hi: 0x87}, + {value: 0x2d95, lo: 0x8b, hi: 0x8b}, + {value: 0x2d9f, lo: 0x8c, hi: 0x8c}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x9900, lo: 0x97, hi: 0x97}, + {value: 0x8132, lo: 0xa6, hi: 0xac}, + {value: 0x8132, lo: 0xb0, hi: 0xb4}, + // Block 0x7d, offset 0x256 + {value: 0x0000, lo: 0x03}, + {value: 0x8104, lo: 0x82, hi: 0x82}, + {value: 0x8102, lo: 0x86, hi: 0x86}, + {value: 0x8132, lo: 0x9e, hi: 0x9e}, + // Block 0x7e, offset 0x25a + {value: 0x6b57, lo: 0x06}, + {value: 0x9900, lo: 0xb0, hi: 0xb0}, + {value: 0xa000, lo: 0xb9, hi: 0xb9}, + {value: 0x9900, lo: 0xba, hi: 0xba}, + {value: 0x2db3, lo: 0xbb, hi: 0xbb}, + {value: 0x2da9, lo: 0xbc, hi: 0xbd}, + {value: 0x2dbd, lo: 0xbe, hi: 0xbe}, + // Block 0x7f, offset 0x261 + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0x82, hi: 0x82}, + {value: 0x8102, lo: 0x83, hi: 0x83}, + // Block 0x80, offset 0x264 + {value: 0x0000, lo: 0x05}, + {value: 0x9900, lo: 0xaf, hi: 0xaf}, + {value: 0xa000, lo: 0xb8, hi: 0xb9}, + {value: 0x2dc7, lo: 0xba, hi: 0xba}, + {value: 0x2dd1, lo: 0xbb, hi: 0xbb}, + {value: 0x8104, lo: 0xbf, hi: 0xbf}, + // Block 0x81, offset 0x26a + {value: 0x0000, lo: 0x01}, + {value: 0x8102, lo: 0x80, hi: 0x80}, + // Block 0x82, offset 0x26c + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0xb6, hi: 0xb6}, + {value: 0x8102, lo: 0xb7, hi: 0xb7}, + // Block 0x83, offset 0x26f + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0xab, hi: 0xab}, + // Block 0x84, offset 0x271 + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0xb9, hi: 0xb9}, + {value: 0x8102, lo: 0xba, hi: 0xba}, + // Block 0x85, offset 0x274 + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0xa0, hi: 0xa0}, + // Block 0x86, offset 0x276 + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0xb4, hi: 0xb4}, + // Block 0x87, offset 0x278 + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0x87, hi: 0x87}, + // Block 0x88, offset 0x27a + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0x99, hi: 0x99}, + // Block 0x89, offset 0x27c + {value: 0x0000, lo: 0x02}, + {value: 0x8102, lo: 0x82, hi: 0x82}, + {value: 0x8104, lo: 0x84, hi: 0x85}, + // Block 0x8a, offset 0x27f + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0x97, hi: 0x97}, + // Block 0x8b, offset 0x281 + {value: 0x0000, lo: 0x01}, + {value: 0x8101, lo: 0xb0, hi: 0xb4}, + // Block 0x8c, offset 0x283 + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0xb0, hi: 0xb6}, + // Block 0x8d, offset 0x285 + {value: 0x0000, lo: 0x01}, + {value: 0x8101, lo: 0x9e, hi: 0x9e}, + // Block 0x8e, offset 0x287 + {value: 0x0000, lo: 0x0c}, + {value: 0x45cf, lo: 0x9e, hi: 0x9e}, + {value: 0x45d9, lo: 0x9f, hi: 0x9f}, + {value: 0x460d, lo: 0xa0, hi: 0xa0}, + {value: 0x461b, lo: 0xa1, hi: 0xa1}, + {value: 0x4629, lo: 0xa2, hi: 0xa2}, + {value: 0x4637, lo: 0xa3, hi: 0xa3}, + {value: 0x4645, lo: 0xa4, hi: 0xa4}, + {value: 0x812b, lo: 0xa5, hi: 0xa6}, + {value: 0x8101, lo: 0xa7, hi: 0xa9}, + {value: 0x8130, lo: 0xad, hi: 0xad}, + {value: 0x812b, lo: 0xae, hi: 0xb2}, + {value: 0x812d, lo: 0xbb, hi: 0xbf}, + // Block 0x8f, offset 0x294 + {value: 0x0000, lo: 0x09}, + {value: 0x812d, lo: 0x80, hi: 0x82}, + {value: 0x8132, lo: 0x85, hi: 0x89}, + {value: 0x812d, lo: 0x8a, hi: 0x8b}, + {value: 0x8132, lo: 0xaa, hi: 0xad}, + {value: 0x45e3, lo: 0xbb, hi: 0xbb}, + {value: 0x45ed, lo: 0xbc, hi: 0xbc}, + {value: 0x4653, lo: 0xbd, hi: 0xbd}, + {value: 0x466f, lo: 0xbe, hi: 0xbe}, + {value: 0x4661, lo: 0xbf, hi: 0xbf}, + // Block 0x90, offset 0x29e + {value: 0x0000, lo: 0x01}, + {value: 0x467d, lo: 0x80, hi: 0x80}, + // Block 0x91, offset 0x2a0 + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0x82, hi: 0x84}, + // Block 0x92, offset 0x2a2 + {value: 0x0000, lo: 0x05}, + {value: 0x8132, lo: 0x80, hi: 0x86}, + {value: 0x8132, lo: 0x88, hi: 0x98}, + {value: 0x8132, lo: 0x9b, hi: 0xa1}, + {value: 0x8132, lo: 0xa3, hi: 0xa4}, + {value: 0x8132, lo: 0xa6, hi: 0xaa}, + // Block 0x93, offset 0x2a8 + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0xac, hi: 0xaf}, + // Block 0x94, offset 0x2aa + {value: 0x0000, lo: 0x01}, + {value: 0x812d, lo: 0x90, hi: 0x96}, + // Block 0x95, offset 0x2ac + {value: 0x0000, lo: 0x02}, + {value: 0x8132, lo: 0x84, hi: 0x89}, + {value: 0x8102, lo: 0x8a, hi: 0x8a}, + // Block 0x96, offset 0x2af + {value: 0x0000, lo: 0x01}, + {value: 0x8100, lo: 0x93, hi: 0x93}, +} + +// lookup returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *nfkcTrie) lookup(s []byte) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return nfkcValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := nfkcIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := nfkcIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = nfkcIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := nfkcIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = nfkcIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = nfkcIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *nfkcTrie) lookupUnsafe(s []byte) uint16 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return nfkcValues[c0] + } + i := nfkcIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = nfkcIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = nfkcIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// lookupString returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *nfkcTrie) lookupString(s string) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return nfkcValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := nfkcIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := nfkcIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = nfkcIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := nfkcIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = nfkcIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = nfkcIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *nfkcTrie) lookupStringUnsafe(s string) uint16 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return nfkcValues[c0] + } + i := nfkcIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = nfkcIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = nfkcIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// nfkcTrie. Total size: 18684 bytes (18.25 KiB). Checksum: 113e23c477adfabd. +type nfkcTrie struct{} + +func newNfkcTrie(i int) *nfkcTrie { + return &nfkcTrie{} +} + +// lookupValue determines the type of block n and looks up the value for b. +func (t *nfkcTrie) lookupValue(n uint32, b byte) uint16 { + switch { + case n < 92: + return uint16(nfkcValues[n<<6+uint32(b)]) + default: + n -= 92 + return uint16(nfkcSparse.lookup(n, b)) + } +} + +// nfkcValues: 94 blocks, 6016 entries, 12032 bytes +// The third block is the zero block. +var nfkcValues = [6016]uint16{ + // Block 0x0, offset 0x0 + 0x3c: 0xa000, 0x3d: 0xa000, 0x3e: 0xa000, + // Block 0x1, offset 0x40 + 0x41: 0xa000, 0x42: 0xa000, 0x43: 0xa000, 0x44: 0xa000, 0x45: 0xa000, + 0x46: 0xa000, 0x47: 0xa000, 0x48: 0xa000, 0x49: 0xa000, 0x4a: 0xa000, 0x4b: 0xa000, + 0x4c: 0xa000, 0x4d: 0xa000, 0x4e: 0xa000, 0x4f: 0xa000, 0x50: 0xa000, + 0x52: 0xa000, 0x53: 0xa000, 0x54: 0xa000, 0x55: 0xa000, 0x56: 0xa000, 0x57: 0xa000, + 0x58: 0xa000, 0x59: 0xa000, 0x5a: 0xa000, + 0x61: 0xa000, 0x62: 0xa000, 0x63: 0xa000, + 0x64: 0xa000, 0x65: 0xa000, 0x66: 0xa000, 0x67: 0xa000, 0x68: 0xa000, 0x69: 0xa000, + 0x6a: 0xa000, 0x6b: 0xa000, 0x6c: 0xa000, 0x6d: 0xa000, 0x6e: 0xa000, 0x6f: 0xa000, + 0x70: 0xa000, 0x72: 0xa000, 0x73: 0xa000, 0x74: 0xa000, 0x75: 0xa000, + 0x76: 0xa000, 0x77: 0xa000, 0x78: 0xa000, 0x79: 0xa000, 0x7a: 0xa000, + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xc0: 0x2f72, 0xc1: 0x2f77, 0xc2: 0x468b, 0xc3: 0x2f7c, 0xc4: 0x469a, 0xc5: 0x469f, + 0xc6: 0xa000, 0xc7: 0x46a9, 0xc8: 0x2fe5, 0xc9: 0x2fea, 0xca: 0x46ae, 0xcb: 0x2ffe, + 0xcc: 0x3071, 0xcd: 0x3076, 0xce: 0x307b, 0xcf: 0x46c2, 0xd1: 0x3107, + 0xd2: 0x312a, 0xd3: 0x312f, 0xd4: 0x46cc, 0xd5: 0x46d1, 0xd6: 0x46e0, + 0xd8: 0xa000, 0xd9: 0x31b6, 0xda: 0x31bb, 0xdb: 0x31c0, 0xdc: 0x4712, 0xdd: 0x3238, + 0xe0: 0x327e, 0xe1: 0x3283, 0xe2: 0x471c, 0xe3: 0x3288, + 0xe4: 0x472b, 0xe5: 0x4730, 0xe6: 0xa000, 0xe7: 0x473a, 0xe8: 0x32f1, 0xe9: 0x32f6, + 0xea: 0x473f, 0xeb: 0x330a, 0xec: 0x3382, 0xed: 0x3387, 0xee: 0x338c, 0xef: 0x4753, + 0xf1: 0x3418, 0xf2: 0x343b, 0xf3: 0x3440, 0xf4: 0x475d, 0xf5: 0x4762, + 0xf6: 0x4771, 0xf8: 0xa000, 0xf9: 0x34cc, 0xfa: 0x34d1, 0xfb: 0x34d6, + 0xfc: 0x47a3, 0xfd: 0x3553, 0xff: 0x356c, + // Block 0x4, offset 0x100 + 0x100: 0x2f81, 0x101: 0x328d, 0x102: 0x4690, 0x103: 0x4721, 0x104: 0x2f9f, 0x105: 0x32ab, + 0x106: 0x2fb3, 0x107: 0x32bf, 0x108: 0x2fb8, 0x109: 0x32c4, 0x10a: 0x2fbd, 0x10b: 0x32c9, + 0x10c: 0x2fc2, 0x10d: 0x32ce, 0x10e: 0x2fcc, 0x10f: 0x32d8, + 0x112: 0x46b3, 0x113: 0x4744, 0x114: 0x2ff4, 0x115: 0x3300, 0x116: 0x2ff9, 0x117: 0x3305, + 0x118: 0x3017, 0x119: 0x3323, 0x11a: 0x3008, 0x11b: 0x3314, 0x11c: 0x3030, 0x11d: 0x333c, + 0x11e: 0x303a, 0x11f: 0x3346, 0x120: 0x303f, 0x121: 0x334b, 0x122: 0x3049, 0x123: 0x3355, + 0x124: 0x304e, 0x125: 0x335a, 0x128: 0x3080, 0x129: 0x3391, + 0x12a: 0x3085, 0x12b: 0x3396, 0x12c: 0x308a, 0x12d: 0x339b, 0x12e: 0x30ad, 0x12f: 0x33b9, + 0x130: 0x308f, 0x132: 0x195d, 0x133: 0x19ea, 0x134: 0x30b7, 0x135: 0x33c3, + 0x136: 0x30cb, 0x137: 0x33dc, 0x139: 0x30d5, 0x13a: 0x33e6, 0x13b: 0x30df, + 0x13c: 0x33f0, 0x13d: 0x30da, 0x13e: 0x33eb, 0x13f: 0x1baf, + // Block 0x5, offset 0x140 + 0x140: 0x1c37, 0x143: 0x3102, 0x144: 0x3413, 0x145: 0x311b, + 0x146: 0x342c, 0x147: 0x3111, 0x148: 0x3422, 0x149: 0x1c5f, + 0x14c: 0x46d6, 0x14d: 0x4767, 0x14e: 0x3134, 0x14f: 0x3445, 0x150: 0x313e, 0x151: 0x344f, + 0x154: 0x315c, 0x155: 0x346d, 0x156: 0x3175, 0x157: 0x3486, + 0x158: 0x3166, 0x159: 0x3477, 0x15a: 0x46f9, 0x15b: 0x478a, 0x15c: 0x317f, 0x15d: 0x3490, + 0x15e: 0x318e, 0x15f: 0x349f, 0x160: 0x46fe, 0x161: 0x478f, 0x162: 0x31a7, 0x163: 0x34bd, + 0x164: 0x3198, 0x165: 0x34ae, 0x168: 0x4708, 0x169: 0x4799, + 0x16a: 0x470d, 0x16b: 0x479e, 0x16c: 0x31c5, 0x16d: 0x34db, 0x16e: 0x31cf, 0x16f: 0x34e5, + 0x170: 0x31d4, 0x171: 0x34ea, 0x172: 0x31f2, 0x173: 0x3508, 0x174: 0x3215, 0x175: 0x352b, + 0x176: 0x323d, 0x177: 0x3558, 0x178: 0x3251, 0x179: 0x3260, 0x17a: 0x3580, 0x17b: 0x326a, + 0x17c: 0x358a, 0x17d: 0x326f, 0x17e: 0x358f, 0x17f: 0x00a7, + // Block 0x6, offset 0x180 + 0x184: 0x2df1, 0x185: 0x2df7, + 0x186: 0x2dfd, 0x187: 0x1972, 0x188: 0x1975, 0x189: 0x1a0b, 0x18a: 0x198a, 0x18b: 0x198d, + 0x18c: 0x1a41, 0x18d: 0x2f8b, 0x18e: 0x3297, 0x18f: 0x3099, 0x190: 0x33a5, 0x191: 0x3143, + 0x192: 0x3454, 0x193: 0x31d9, 0x194: 0x34ef, 0x195: 0x39d2, 0x196: 0x3b61, 0x197: 0x39cb, + 0x198: 0x3b5a, 0x199: 0x39d9, 0x19a: 0x3b68, 0x19b: 0x39c4, 0x19c: 0x3b53, + 0x19e: 0x38b3, 0x19f: 0x3a42, 0x1a0: 0x38ac, 0x1a1: 0x3a3b, 0x1a2: 0x35b6, 0x1a3: 0x35c8, + 0x1a6: 0x3044, 0x1a7: 0x3350, 0x1a8: 0x30c1, 0x1a9: 0x33d2, + 0x1aa: 0x46ef, 0x1ab: 0x4780, 0x1ac: 0x3993, 0x1ad: 0x3b22, 0x1ae: 0x35da, 0x1af: 0x35e0, + 0x1b0: 0x33c8, 0x1b1: 0x1942, 0x1b2: 0x1945, 0x1b3: 0x19d2, 0x1b4: 0x302b, 0x1b5: 0x3337, + 0x1b8: 0x30fd, 0x1b9: 0x340e, 0x1ba: 0x38ba, 0x1bb: 0x3a49, + 0x1bc: 0x35b0, 0x1bd: 0x35c2, 0x1be: 0x35bc, 0x1bf: 0x35ce, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x2f90, 0x1c1: 0x329c, 0x1c2: 0x2f95, 0x1c3: 0x32a1, 0x1c4: 0x300d, 0x1c5: 0x3319, + 0x1c6: 0x3012, 0x1c7: 0x331e, 0x1c8: 0x309e, 0x1c9: 0x33aa, 0x1ca: 0x30a3, 0x1cb: 0x33af, + 0x1cc: 0x3148, 0x1cd: 0x3459, 0x1ce: 0x314d, 0x1cf: 0x345e, 0x1d0: 0x316b, 0x1d1: 0x347c, + 0x1d2: 0x3170, 0x1d3: 0x3481, 0x1d4: 0x31de, 0x1d5: 0x34f4, 0x1d6: 0x31e3, 0x1d7: 0x34f9, + 0x1d8: 0x3189, 0x1d9: 0x349a, 0x1da: 0x31a2, 0x1db: 0x34b8, + 0x1de: 0x305d, 0x1df: 0x3369, + 0x1e6: 0x4695, 0x1e7: 0x4726, 0x1e8: 0x46bd, 0x1e9: 0x474e, + 0x1ea: 0x3962, 0x1eb: 0x3af1, 0x1ec: 0x393f, 0x1ed: 0x3ace, 0x1ee: 0x46db, 0x1ef: 0x476c, + 0x1f0: 0x395b, 0x1f1: 0x3aea, 0x1f2: 0x3247, 0x1f3: 0x3562, + // Block 0x8, offset 0x200 + 0x200: 0x9932, 0x201: 0x9932, 0x202: 0x9932, 0x203: 0x9932, 0x204: 0x9932, 0x205: 0x8132, + 0x206: 0x9932, 0x207: 0x9932, 0x208: 0x9932, 0x209: 0x9932, 0x20a: 0x9932, 0x20b: 0x9932, + 0x20c: 0x9932, 0x20d: 0x8132, 0x20e: 0x8132, 0x20f: 0x9932, 0x210: 0x8132, 0x211: 0x9932, + 0x212: 0x8132, 0x213: 0x9932, 0x214: 0x9932, 0x215: 0x8133, 0x216: 0x812d, 0x217: 0x812d, + 0x218: 0x812d, 0x219: 0x812d, 0x21a: 0x8133, 0x21b: 0x992b, 0x21c: 0x812d, 0x21d: 0x812d, + 0x21e: 0x812d, 0x21f: 0x812d, 0x220: 0x812d, 0x221: 0x8129, 0x222: 0x8129, 0x223: 0x992d, + 0x224: 0x992d, 0x225: 0x992d, 0x226: 0x992d, 0x227: 0x9929, 0x228: 0x9929, 0x229: 0x812d, + 0x22a: 0x812d, 0x22b: 0x812d, 0x22c: 0x812d, 0x22d: 0x992d, 0x22e: 0x992d, 0x22f: 0x812d, + 0x230: 0x992d, 0x231: 0x992d, 0x232: 0x812d, 0x233: 0x812d, 0x234: 0x8101, 0x235: 0x8101, + 0x236: 0x8101, 0x237: 0x8101, 0x238: 0x9901, 0x239: 0x812d, 0x23a: 0x812d, 0x23b: 0x812d, + 0x23c: 0x812d, 0x23d: 0x8132, 0x23e: 0x8132, 0x23f: 0x8132, + // Block 0x9, offset 0x240 + 0x240: 0x49b1, 0x241: 0x49b6, 0x242: 0x9932, 0x243: 0x49bb, 0x244: 0x4a74, 0x245: 0x9936, + 0x246: 0x8132, 0x247: 0x812d, 0x248: 0x812d, 0x249: 0x812d, 0x24a: 0x8132, 0x24b: 0x8132, + 0x24c: 0x8132, 0x24d: 0x812d, 0x24e: 0x812d, 0x250: 0x8132, 0x251: 0x8132, + 0x252: 0x8132, 0x253: 0x812d, 0x254: 0x812d, 0x255: 0x812d, 0x256: 0x812d, 0x257: 0x8132, + 0x258: 0x8133, 0x259: 0x812d, 0x25a: 0x812d, 0x25b: 0x8132, 0x25c: 0x8134, 0x25d: 0x8135, + 0x25e: 0x8135, 0x25f: 0x8134, 0x260: 0x8135, 0x261: 0x8135, 0x262: 0x8134, 0x263: 0x8132, + 0x264: 0x8132, 0x265: 0x8132, 0x266: 0x8132, 0x267: 0x8132, 0x268: 0x8132, 0x269: 0x8132, + 0x26a: 0x8132, 0x26b: 0x8132, 0x26c: 0x8132, 0x26d: 0x8132, 0x26e: 0x8132, 0x26f: 0x8132, + 0x274: 0x0170, + 0x27a: 0x42a8, + 0x27e: 0x0037, + // Block 0xa, offset 0x280 + 0x284: 0x425d, 0x285: 0x447e, + 0x286: 0x35ec, 0x287: 0x00ce, 0x288: 0x360a, 0x289: 0x3616, 0x28a: 0x3628, + 0x28c: 0x3646, 0x28e: 0x3658, 0x28f: 0x3676, 0x290: 0x3e0b, 0x291: 0xa000, + 0x295: 0xa000, 0x297: 0xa000, + 0x299: 0xa000, + 0x29f: 0xa000, 0x2a1: 0xa000, + 0x2a5: 0xa000, 0x2a9: 0xa000, + 0x2aa: 0x363a, 0x2ab: 0x366a, 0x2ac: 0x4801, 0x2ad: 0x369a, 0x2ae: 0x482b, 0x2af: 0x36ac, + 0x2b0: 0x3e73, 0x2b1: 0xa000, 0x2b5: 0xa000, + 0x2b7: 0xa000, 0x2b9: 0xa000, + 0x2bf: 0xa000, + // Block 0xb, offset 0x2c0 + 0x2c1: 0xa000, 0x2c5: 0xa000, + 0x2c9: 0xa000, 0x2ca: 0x4843, 0x2cb: 0x4861, + 0x2cc: 0x36ca, 0x2cd: 0x36e2, 0x2ce: 0x4879, 0x2d0: 0x01be, 0x2d1: 0x01d0, + 0x2d2: 0x01ac, 0x2d3: 0x430f, 0x2d4: 0x4315, 0x2d5: 0x01fa, 0x2d6: 0x01e8, + 0x2f0: 0x01d6, 0x2f1: 0x01eb, 0x2f2: 0x01ee, 0x2f4: 0x0188, 0x2f5: 0x01c7, + 0x2f9: 0x01a6, + // Block 0xc, offset 0x300 + 0x300: 0x3724, 0x301: 0x3730, 0x303: 0x371e, + 0x306: 0xa000, 0x307: 0x370c, + 0x30c: 0x3760, 0x30d: 0x3748, 0x30e: 0x3772, 0x310: 0xa000, + 0x313: 0xa000, 0x315: 0xa000, 0x316: 0xa000, 0x317: 0xa000, + 0x318: 0xa000, 0x319: 0x3754, 0x31a: 0xa000, + 0x31e: 0xa000, 0x323: 0xa000, + 0x327: 0xa000, + 0x32b: 0xa000, 0x32d: 0xa000, + 0x330: 0xa000, 0x333: 0xa000, 0x335: 0xa000, + 0x336: 0xa000, 0x337: 0xa000, 0x338: 0xa000, 0x339: 0x37d8, 0x33a: 0xa000, + 0x33e: 0xa000, + // Block 0xd, offset 0x340 + 0x341: 0x3736, 0x342: 0x37ba, + 0x350: 0x3712, 0x351: 0x3796, + 0x352: 0x3718, 0x353: 0x379c, 0x356: 0x372a, 0x357: 0x37ae, + 0x358: 0xa000, 0x359: 0xa000, 0x35a: 0x382c, 0x35b: 0x3832, 0x35c: 0x373c, 0x35d: 0x37c0, + 0x35e: 0x3742, 0x35f: 0x37c6, 0x362: 0x374e, 0x363: 0x37d2, + 0x364: 0x375a, 0x365: 0x37de, 0x366: 0x3766, 0x367: 0x37ea, 0x368: 0xa000, 0x369: 0xa000, + 0x36a: 0x3838, 0x36b: 0x383e, 0x36c: 0x3790, 0x36d: 0x3814, 0x36e: 0x376c, 0x36f: 0x37f0, + 0x370: 0x3778, 0x371: 0x37fc, 0x372: 0x377e, 0x373: 0x3802, 0x374: 0x3784, 0x375: 0x3808, + 0x378: 0x378a, 0x379: 0x380e, + // Block 0xe, offset 0x380 + 0x387: 0x1d64, + 0x391: 0x812d, + 0x392: 0x8132, 0x393: 0x8132, 0x394: 0x8132, 0x395: 0x8132, 0x396: 0x812d, 0x397: 0x8132, + 0x398: 0x8132, 0x399: 0x8132, 0x39a: 0x812e, 0x39b: 0x812d, 0x39c: 0x8132, 0x39d: 0x8132, + 0x39e: 0x8132, 0x39f: 0x8132, 0x3a0: 0x8132, 0x3a1: 0x8132, 0x3a2: 0x812d, 0x3a3: 0x812d, + 0x3a4: 0x812d, 0x3a5: 0x812d, 0x3a6: 0x812d, 0x3a7: 0x812d, 0x3a8: 0x8132, 0x3a9: 0x8132, + 0x3aa: 0x812d, 0x3ab: 0x8132, 0x3ac: 0x8132, 0x3ad: 0x812e, 0x3ae: 0x8131, 0x3af: 0x8132, + 0x3b0: 0x8105, 0x3b1: 0x8106, 0x3b2: 0x8107, 0x3b3: 0x8108, 0x3b4: 0x8109, 0x3b5: 0x810a, + 0x3b6: 0x810b, 0x3b7: 0x810c, 0x3b8: 0x810d, 0x3b9: 0x810e, 0x3ba: 0x810e, 0x3bb: 0x810f, + 0x3bc: 0x8110, 0x3bd: 0x8111, 0x3bf: 0x8112, + // Block 0xf, offset 0x3c0 + 0x3c8: 0xa000, 0x3ca: 0xa000, 0x3cb: 0x8116, + 0x3cc: 0x8117, 0x3cd: 0x8118, 0x3ce: 0x8119, 0x3cf: 0x811a, 0x3d0: 0x811b, 0x3d1: 0x811c, + 0x3d2: 0x811d, 0x3d3: 0x9932, 0x3d4: 0x9932, 0x3d5: 0x992d, 0x3d6: 0x812d, 0x3d7: 0x8132, + 0x3d8: 0x8132, 0x3d9: 0x8132, 0x3da: 0x8132, 0x3db: 0x8132, 0x3dc: 0x812d, 0x3dd: 0x8132, + 0x3de: 0x8132, 0x3df: 0x812d, + 0x3f0: 0x811e, 0x3f5: 0x1d87, + 0x3f6: 0x2016, 0x3f7: 0x2052, 0x3f8: 0x204d, + // Block 0x10, offset 0x400 + 0x413: 0x812d, 0x414: 0x8132, 0x415: 0x8132, 0x416: 0x8132, 0x417: 0x8132, + 0x418: 0x8132, 0x419: 0x8132, 0x41a: 0x8132, 0x41b: 0x8132, 0x41c: 0x8132, 0x41d: 0x8132, + 0x41e: 0x8132, 0x41f: 0x8132, 0x420: 0x8132, 0x421: 0x8132, 0x423: 0x812d, + 0x424: 0x8132, 0x425: 0x8132, 0x426: 0x812d, 0x427: 0x8132, 0x428: 0x8132, 0x429: 0x812d, + 0x42a: 0x8132, 0x42b: 0x8132, 0x42c: 0x8132, 0x42d: 0x812d, 0x42e: 0x812d, 0x42f: 0x812d, + 0x430: 0x8116, 0x431: 0x8117, 0x432: 0x8118, 0x433: 0x8132, 0x434: 0x8132, 0x435: 0x8132, + 0x436: 0x812d, 0x437: 0x8132, 0x438: 0x8132, 0x439: 0x812d, 0x43a: 0x812d, 0x43b: 0x8132, + 0x43c: 0x8132, 0x43d: 0x8132, 0x43e: 0x8132, 0x43f: 0x8132, + // Block 0x11, offset 0x440 + 0x445: 0xa000, + 0x446: 0x2d29, 0x447: 0xa000, 0x448: 0x2d31, 0x449: 0xa000, 0x44a: 0x2d39, 0x44b: 0xa000, + 0x44c: 0x2d41, 0x44d: 0xa000, 0x44e: 0x2d49, 0x451: 0xa000, + 0x452: 0x2d51, + 0x474: 0x8102, 0x475: 0x9900, + 0x47a: 0xa000, 0x47b: 0x2d59, + 0x47c: 0xa000, 0x47d: 0x2d61, 0x47e: 0xa000, 0x47f: 0xa000, + // Block 0x12, offset 0x480 + 0x480: 0x0069, 0x481: 0x006b, 0x482: 0x006f, 0x483: 0x0083, 0x484: 0x00f5, 0x485: 0x00f8, + 0x486: 0x0413, 0x487: 0x0085, 0x488: 0x0089, 0x489: 0x008b, 0x48a: 0x0104, 0x48b: 0x0107, + 0x48c: 0x010a, 0x48d: 0x008f, 0x48f: 0x0097, 0x490: 0x009b, 0x491: 0x00e0, + 0x492: 0x009f, 0x493: 0x00fe, 0x494: 0x0417, 0x495: 0x041b, 0x496: 0x00a1, 0x497: 0x00a9, + 0x498: 0x00ab, 0x499: 0x0423, 0x49a: 0x012b, 0x49b: 0x00ad, 0x49c: 0x0427, 0x49d: 0x01be, + 0x49e: 0x01c1, 0x49f: 0x01c4, 0x4a0: 0x01fa, 0x4a1: 0x01fd, 0x4a2: 0x0093, 0x4a3: 0x00a5, + 0x4a4: 0x00ab, 0x4a5: 0x00ad, 0x4a6: 0x01be, 0x4a7: 0x01c1, 0x4a8: 0x01eb, 0x4a9: 0x01fa, + 0x4aa: 0x01fd, + 0x4b8: 0x020c, + // Block 0x13, offset 0x4c0 + 0x4db: 0x00fb, 0x4dc: 0x0087, 0x4dd: 0x0101, + 0x4de: 0x00d4, 0x4df: 0x010a, 0x4e0: 0x008d, 0x4e1: 0x010d, 0x4e2: 0x0110, 0x4e3: 0x0116, + 0x4e4: 0x011c, 0x4e5: 0x011f, 0x4e6: 0x0122, 0x4e7: 0x042b, 0x4e8: 0x016a, 0x4e9: 0x0128, + 0x4ea: 0x042f, 0x4eb: 0x016d, 0x4ec: 0x0131, 0x4ed: 0x012e, 0x4ee: 0x0134, 0x4ef: 0x0137, + 0x4f0: 0x013a, 0x4f1: 0x013d, 0x4f2: 0x0140, 0x4f3: 0x014c, 0x4f4: 0x014f, 0x4f5: 0x00ec, + 0x4f6: 0x0152, 0x4f7: 0x0155, 0x4f8: 0x041f, 0x4f9: 0x0158, 0x4fa: 0x015b, 0x4fb: 0x00b5, + 0x4fc: 0x015e, 0x4fd: 0x0161, 0x4fe: 0x0164, 0x4ff: 0x01d0, + // Block 0x14, offset 0x500 + 0x500: 0x8132, 0x501: 0x8132, 0x502: 0x812d, 0x503: 0x8132, 0x504: 0x8132, 0x505: 0x8132, + 0x506: 0x8132, 0x507: 0x8132, 0x508: 0x8132, 0x509: 0x8132, 0x50a: 0x812d, 0x50b: 0x8132, + 0x50c: 0x8132, 0x50d: 0x8135, 0x50e: 0x812a, 0x50f: 0x812d, 0x510: 0x8129, 0x511: 0x8132, + 0x512: 0x8132, 0x513: 0x8132, 0x514: 0x8132, 0x515: 0x8132, 0x516: 0x8132, 0x517: 0x8132, + 0x518: 0x8132, 0x519: 0x8132, 0x51a: 0x8132, 0x51b: 0x8132, 0x51c: 0x8132, 0x51d: 0x8132, + 0x51e: 0x8132, 0x51f: 0x8132, 0x520: 0x8132, 0x521: 0x8132, 0x522: 0x8132, 0x523: 0x8132, + 0x524: 0x8132, 0x525: 0x8132, 0x526: 0x8132, 0x527: 0x8132, 0x528: 0x8132, 0x529: 0x8132, + 0x52a: 0x8132, 0x52b: 0x8132, 0x52c: 0x8132, 0x52d: 0x8132, 0x52e: 0x8132, 0x52f: 0x8132, + 0x530: 0x8132, 0x531: 0x8132, 0x532: 0x8132, 0x533: 0x8132, 0x534: 0x8132, 0x535: 0x8132, + 0x536: 0x8133, 0x537: 0x8131, 0x538: 0x8131, 0x539: 0x812d, 0x53b: 0x8132, + 0x53c: 0x8134, 0x53d: 0x812d, 0x53e: 0x8132, 0x53f: 0x812d, + // Block 0x15, offset 0x540 + 0x540: 0x2f9a, 0x541: 0x32a6, 0x542: 0x2fa4, 0x543: 0x32b0, 0x544: 0x2fa9, 0x545: 0x32b5, + 0x546: 0x2fae, 0x547: 0x32ba, 0x548: 0x38cf, 0x549: 0x3a5e, 0x54a: 0x2fc7, 0x54b: 0x32d3, + 0x54c: 0x2fd1, 0x54d: 0x32dd, 0x54e: 0x2fe0, 0x54f: 0x32ec, 0x550: 0x2fd6, 0x551: 0x32e2, + 0x552: 0x2fdb, 0x553: 0x32e7, 0x554: 0x38f2, 0x555: 0x3a81, 0x556: 0x38f9, 0x557: 0x3a88, + 0x558: 0x301c, 0x559: 0x3328, 0x55a: 0x3021, 0x55b: 0x332d, 0x55c: 0x3907, 0x55d: 0x3a96, + 0x55e: 0x3026, 0x55f: 0x3332, 0x560: 0x3035, 0x561: 0x3341, 0x562: 0x3053, 0x563: 0x335f, + 0x564: 0x3062, 0x565: 0x336e, 0x566: 0x3058, 0x567: 0x3364, 0x568: 0x3067, 0x569: 0x3373, + 0x56a: 0x306c, 0x56b: 0x3378, 0x56c: 0x30b2, 0x56d: 0x33be, 0x56e: 0x390e, 0x56f: 0x3a9d, + 0x570: 0x30bc, 0x571: 0x33cd, 0x572: 0x30c6, 0x573: 0x33d7, 0x574: 0x30d0, 0x575: 0x33e1, + 0x576: 0x46c7, 0x577: 0x4758, 0x578: 0x3915, 0x579: 0x3aa4, 0x57a: 0x30e9, 0x57b: 0x33fa, + 0x57c: 0x30e4, 0x57d: 0x33f5, 0x57e: 0x30ee, 0x57f: 0x33ff, + // Block 0x16, offset 0x580 + 0x580: 0x30f3, 0x581: 0x3404, 0x582: 0x30f8, 0x583: 0x3409, 0x584: 0x310c, 0x585: 0x341d, + 0x586: 0x3116, 0x587: 0x3427, 0x588: 0x3125, 0x589: 0x3436, 0x58a: 0x3120, 0x58b: 0x3431, + 0x58c: 0x3938, 0x58d: 0x3ac7, 0x58e: 0x3946, 0x58f: 0x3ad5, 0x590: 0x394d, 0x591: 0x3adc, + 0x592: 0x3954, 0x593: 0x3ae3, 0x594: 0x3152, 0x595: 0x3463, 0x596: 0x3157, 0x597: 0x3468, + 0x598: 0x3161, 0x599: 0x3472, 0x59a: 0x46f4, 0x59b: 0x4785, 0x59c: 0x399a, 0x59d: 0x3b29, + 0x59e: 0x317a, 0x59f: 0x348b, 0x5a0: 0x3184, 0x5a1: 0x3495, 0x5a2: 0x4703, 0x5a3: 0x4794, + 0x5a4: 0x39a1, 0x5a5: 0x3b30, 0x5a6: 0x39a8, 0x5a7: 0x3b37, 0x5a8: 0x39af, 0x5a9: 0x3b3e, + 0x5aa: 0x3193, 0x5ab: 0x34a4, 0x5ac: 0x319d, 0x5ad: 0x34b3, 0x5ae: 0x31b1, 0x5af: 0x34c7, + 0x5b0: 0x31ac, 0x5b1: 0x34c2, 0x5b2: 0x31ed, 0x5b3: 0x3503, 0x5b4: 0x31fc, 0x5b5: 0x3512, + 0x5b6: 0x31f7, 0x5b7: 0x350d, 0x5b8: 0x39b6, 0x5b9: 0x3b45, 0x5ba: 0x39bd, 0x5bb: 0x3b4c, + 0x5bc: 0x3201, 0x5bd: 0x3517, 0x5be: 0x3206, 0x5bf: 0x351c, + // Block 0x17, offset 0x5c0 + 0x5c0: 0x320b, 0x5c1: 0x3521, 0x5c2: 0x3210, 0x5c3: 0x3526, 0x5c4: 0x321f, 0x5c5: 0x3535, + 0x5c6: 0x321a, 0x5c7: 0x3530, 0x5c8: 0x3224, 0x5c9: 0x353f, 0x5ca: 0x3229, 0x5cb: 0x3544, + 0x5cc: 0x322e, 0x5cd: 0x3549, 0x5ce: 0x324c, 0x5cf: 0x3567, 0x5d0: 0x3265, 0x5d1: 0x3585, + 0x5d2: 0x3274, 0x5d3: 0x3594, 0x5d4: 0x3279, 0x5d5: 0x3599, 0x5d6: 0x337d, 0x5d7: 0x34a9, + 0x5d8: 0x353a, 0x5d9: 0x3576, 0x5da: 0x1be3, 0x5db: 0x42da, + 0x5e0: 0x46a4, 0x5e1: 0x4735, 0x5e2: 0x2f86, 0x5e3: 0x3292, + 0x5e4: 0x387b, 0x5e5: 0x3a0a, 0x5e6: 0x3874, 0x5e7: 0x3a03, 0x5e8: 0x3889, 0x5e9: 0x3a18, + 0x5ea: 0x3882, 0x5eb: 0x3a11, 0x5ec: 0x38c1, 0x5ed: 0x3a50, 0x5ee: 0x3897, 0x5ef: 0x3a26, + 0x5f0: 0x3890, 0x5f1: 0x3a1f, 0x5f2: 0x38a5, 0x5f3: 0x3a34, 0x5f4: 0x389e, 0x5f5: 0x3a2d, + 0x5f6: 0x38c8, 0x5f7: 0x3a57, 0x5f8: 0x46b8, 0x5f9: 0x4749, 0x5fa: 0x3003, 0x5fb: 0x330f, + 0x5fc: 0x2fef, 0x5fd: 0x32fb, 0x5fe: 0x38dd, 0x5ff: 0x3a6c, + // Block 0x18, offset 0x600 + 0x600: 0x38d6, 0x601: 0x3a65, 0x602: 0x38eb, 0x603: 0x3a7a, 0x604: 0x38e4, 0x605: 0x3a73, + 0x606: 0x3900, 0x607: 0x3a8f, 0x608: 0x3094, 0x609: 0x33a0, 0x60a: 0x30a8, 0x60b: 0x33b4, + 0x60c: 0x46ea, 0x60d: 0x477b, 0x60e: 0x3139, 0x60f: 0x344a, 0x610: 0x3923, 0x611: 0x3ab2, + 0x612: 0x391c, 0x613: 0x3aab, 0x614: 0x3931, 0x615: 0x3ac0, 0x616: 0x392a, 0x617: 0x3ab9, + 0x618: 0x398c, 0x619: 0x3b1b, 0x61a: 0x3970, 0x61b: 0x3aff, 0x61c: 0x3969, 0x61d: 0x3af8, + 0x61e: 0x397e, 0x61f: 0x3b0d, 0x620: 0x3977, 0x621: 0x3b06, 0x622: 0x3985, 0x623: 0x3b14, + 0x624: 0x31e8, 0x625: 0x34fe, 0x626: 0x31ca, 0x627: 0x34e0, 0x628: 0x39e7, 0x629: 0x3b76, + 0x62a: 0x39e0, 0x62b: 0x3b6f, 0x62c: 0x39f5, 0x62d: 0x3b84, 0x62e: 0x39ee, 0x62f: 0x3b7d, + 0x630: 0x39fc, 0x631: 0x3b8b, 0x632: 0x3233, 0x633: 0x354e, 0x634: 0x325b, 0x635: 0x357b, + 0x636: 0x3256, 0x637: 0x3571, 0x638: 0x3242, 0x639: 0x355d, + // Block 0x19, offset 0x640 + 0x640: 0x4807, 0x641: 0x480d, 0x642: 0x4921, 0x643: 0x4939, 0x644: 0x4929, 0x645: 0x4941, + 0x646: 0x4931, 0x647: 0x4949, 0x648: 0x47ad, 0x649: 0x47b3, 0x64a: 0x4891, 0x64b: 0x48a9, + 0x64c: 0x4899, 0x64d: 0x48b1, 0x64e: 0x48a1, 0x64f: 0x48b9, 0x650: 0x4819, 0x651: 0x481f, + 0x652: 0x3dbb, 0x653: 0x3dcb, 0x654: 0x3dc3, 0x655: 0x3dd3, + 0x658: 0x47b9, 0x659: 0x47bf, 0x65a: 0x3ceb, 0x65b: 0x3cfb, 0x65c: 0x3cf3, 0x65d: 0x3d03, + 0x660: 0x4831, 0x661: 0x4837, 0x662: 0x4951, 0x663: 0x4969, + 0x664: 0x4959, 0x665: 0x4971, 0x666: 0x4961, 0x667: 0x4979, 0x668: 0x47c5, 0x669: 0x47cb, + 0x66a: 0x48c1, 0x66b: 0x48d9, 0x66c: 0x48c9, 0x66d: 0x48e1, 0x66e: 0x48d1, 0x66f: 0x48e9, + 0x670: 0x4849, 0x671: 0x484f, 0x672: 0x3e1b, 0x673: 0x3e33, 0x674: 0x3e23, 0x675: 0x3e3b, + 0x676: 0x3e2b, 0x677: 0x3e43, 0x678: 0x47d1, 0x679: 0x47d7, 0x67a: 0x3d1b, 0x67b: 0x3d33, + 0x67c: 0x3d23, 0x67d: 0x3d3b, 0x67e: 0x3d2b, 0x67f: 0x3d43, + // Block 0x1a, offset 0x680 + 0x680: 0x4855, 0x681: 0x485b, 0x682: 0x3e4b, 0x683: 0x3e5b, 0x684: 0x3e53, 0x685: 0x3e63, + 0x688: 0x47dd, 0x689: 0x47e3, 0x68a: 0x3d4b, 0x68b: 0x3d5b, + 0x68c: 0x3d53, 0x68d: 0x3d63, 0x690: 0x4867, 0x691: 0x486d, + 0x692: 0x3e83, 0x693: 0x3e9b, 0x694: 0x3e8b, 0x695: 0x3ea3, 0x696: 0x3e93, 0x697: 0x3eab, + 0x699: 0x47e9, 0x69b: 0x3d6b, 0x69d: 0x3d73, + 0x69f: 0x3d7b, 0x6a0: 0x487f, 0x6a1: 0x4885, 0x6a2: 0x4981, 0x6a3: 0x4999, + 0x6a4: 0x4989, 0x6a5: 0x49a1, 0x6a6: 0x4991, 0x6a7: 0x49a9, 0x6a8: 0x47ef, 0x6a9: 0x47f5, + 0x6aa: 0x48f1, 0x6ab: 0x4909, 0x6ac: 0x48f9, 0x6ad: 0x4911, 0x6ae: 0x4901, 0x6af: 0x4919, + 0x6b0: 0x47fb, 0x6b1: 0x4321, 0x6b2: 0x3694, 0x6b3: 0x4327, 0x6b4: 0x4825, 0x6b5: 0x432d, + 0x6b6: 0x36a6, 0x6b7: 0x4333, 0x6b8: 0x36c4, 0x6b9: 0x4339, 0x6ba: 0x36dc, 0x6bb: 0x433f, + 0x6bc: 0x4873, 0x6bd: 0x4345, + // Block 0x1b, offset 0x6c0 + 0x6c0: 0x3da3, 0x6c1: 0x3dab, 0x6c2: 0x4187, 0x6c3: 0x41a5, 0x6c4: 0x4191, 0x6c5: 0x41af, + 0x6c6: 0x419b, 0x6c7: 0x41b9, 0x6c8: 0x3cdb, 0x6c9: 0x3ce3, 0x6ca: 0x40d3, 0x6cb: 0x40f1, + 0x6cc: 0x40dd, 0x6cd: 0x40fb, 0x6ce: 0x40e7, 0x6cf: 0x4105, 0x6d0: 0x3deb, 0x6d1: 0x3df3, + 0x6d2: 0x41c3, 0x6d3: 0x41e1, 0x6d4: 0x41cd, 0x6d5: 0x41eb, 0x6d6: 0x41d7, 0x6d7: 0x41f5, + 0x6d8: 0x3d0b, 0x6d9: 0x3d13, 0x6da: 0x410f, 0x6db: 0x412d, 0x6dc: 0x4119, 0x6dd: 0x4137, + 0x6de: 0x4123, 0x6df: 0x4141, 0x6e0: 0x3ec3, 0x6e1: 0x3ecb, 0x6e2: 0x41ff, 0x6e3: 0x421d, + 0x6e4: 0x4209, 0x6e5: 0x4227, 0x6e6: 0x4213, 0x6e7: 0x4231, 0x6e8: 0x3d83, 0x6e9: 0x3d8b, + 0x6ea: 0x414b, 0x6eb: 0x4169, 0x6ec: 0x4155, 0x6ed: 0x4173, 0x6ee: 0x415f, 0x6ef: 0x417d, + 0x6f0: 0x3688, 0x6f1: 0x3682, 0x6f2: 0x3d93, 0x6f3: 0x368e, 0x6f4: 0x3d9b, + 0x6f6: 0x4813, 0x6f7: 0x3db3, 0x6f8: 0x35f8, 0x6f9: 0x35f2, 0x6fa: 0x35e6, 0x6fb: 0x42f1, + 0x6fc: 0x35fe, 0x6fd: 0x428a, 0x6fe: 0x01d3, 0x6ff: 0x428a, + // Block 0x1c, offset 0x700 + 0x700: 0x42a3, 0x701: 0x4485, 0x702: 0x3ddb, 0x703: 0x36a0, 0x704: 0x3de3, + 0x706: 0x483d, 0x707: 0x3dfb, 0x708: 0x3604, 0x709: 0x42f7, 0x70a: 0x3610, 0x70b: 0x42fd, + 0x70c: 0x361c, 0x70d: 0x448c, 0x70e: 0x4493, 0x70f: 0x449a, 0x710: 0x36b8, 0x711: 0x36b2, + 0x712: 0x3e03, 0x713: 0x44e7, 0x716: 0x36be, 0x717: 0x3e13, + 0x718: 0x3634, 0x719: 0x362e, 0x71a: 0x3622, 0x71b: 0x4303, 0x71d: 0x44a1, + 0x71e: 0x44a8, 0x71f: 0x44af, 0x720: 0x36ee, 0x721: 0x36e8, 0x722: 0x3e6b, 0x723: 0x44ef, + 0x724: 0x36d0, 0x725: 0x36d6, 0x726: 0x36f4, 0x727: 0x3e7b, 0x728: 0x3664, 0x729: 0x365e, + 0x72a: 0x3652, 0x72b: 0x430f, 0x72c: 0x364c, 0x72d: 0x4477, 0x72e: 0x447e, 0x72f: 0x0081, + 0x732: 0x3eb3, 0x733: 0x36fa, 0x734: 0x3ebb, + 0x736: 0x488b, 0x737: 0x3ed3, 0x738: 0x3640, 0x739: 0x4309, 0x73a: 0x3670, 0x73b: 0x431b, + 0x73c: 0x367c, 0x73d: 0x425d, 0x73e: 0x428f, + // Block 0x1d, offset 0x740 + 0x740: 0x1bdb, 0x741: 0x1bdf, 0x742: 0x0047, 0x743: 0x1c57, 0x745: 0x1beb, + 0x746: 0x1bef, 0x747: 0x00e9, 0x749: 0x1c5b, 0x74a: 0x008f, 0x74b: 0x0051, + 0x74c: 0x0051, 0x74d: 0x0051, 0x74e: 0x0091, 0x74f: 0x00da, 0x750: 0x0053, 0x751: 0x0053, + 0x752: 0x0059, 0x753: 0x0099, 0x755: 0x005d, 0x756: 0x1990, + 0x759: 0x0061, 0x75a: 0x0063, 0x75b: 0x0065, 0x75c: 0x0065, 0x75d: 0x0065, + 0x760: 0x19a2, 0x761: 0x1bcb, 0x762: 0x19ab, + 0x764: 0x0075, 0x766: 0x01b8, 0x768: 0x0075, + 0x76a: 0x0057, 0x76b: 0x42d5, 0x76c: 0x0045, 0x76d: 0x0047, 0x76f: 0x008b, + 0x770: 0x004b, 0x771: 0x004d, 0x773: 0x005b, 0x774: 0x009f, 0x775: 0x0215, + 0x776: 0x0218, 0x777: 0x021b, 0x778: 0x021e, 0x779: 0x0093, 0x77b: 0x1b9b, + 0x77c: 0x01e8, 0x77d: 0x01c1, 0x77e: 0x0179, 0x77f: 0x01a0, + // Block 0x1e, offset 0x780 + 0x780: 0x0463, 0x785: 0x0049, + 0x786: 0x0089, 0x787: 0x008b, 0x788: 0x0093, 0x789: 0x0095, + 0x790: 0x2231, 0x791: 0x223d, + 0x792: 0x22f1, 0x793: 0x2219, 0x794: 0x229d, 0x795: 0x2225, 0x796: 0x22a3, 0x797: 0x22bb, + 0x798: 0x22c7, 0x799: 0x222b, 0x79a: 0x22cd, 0x79b: 0x2237, 0x79c: 0x22c1, 0x79d: 0x22d3, + 0x79e: 0x22d9, 0x79f: 0x1cbf, 0x7a0: 0x0053, 0x7a1: 0x195a, 0x7a2: 0x1ba7, 0x7a3: 0x1963, + 0x7a4: 0x006d, 0x7a5: 0x19ae, 0x7a6: 0x1bd3, 0x7a7: 0x1d4b, 0x7a8: 0x1966, 0x7a9: 0x0071, + 0x7aa: 0x19ba, 0x7ab: 0x1bd7, 0x7ac: 0x0059, 0x7ad: 0x0047, 0x7ae: 0x0049, 0x7af: 0x005b, + 0x7b0: 0x0093, 0x7b1: 0x19e7, 0x7b2: 0x1c1b, 0x7b3: 0x19f0, 0x7b4: 0x00ad, 0x7b5: 0x1a65, + 0x7b6: 0x1c4f, 0x7b7: 0x1d5f, 0x7b8: 0x19f3, 0x7b9: 0x00b1, 0x7ba: 0x1a68, 0x7bb: 0x1c53, + 0x7bc: 0x0099, 0x7bd: 0x0087, 0x7be: 0x0089, 0x7bf: 0x009b, + // Block 0x1f, offset 0x7c0 + 0x7c1: 0x3c09, 0x7c3: 0xa000, 0x7c4: 0x3c10, 0x7c5: 0xa000, + 0x7c7: 0x3c17, 0x7c8: 0xa000, 0x7c9: 0x3c1e, + 0x7cd: 0xa000, + 0x7e0: 0x2f68, 0x7e1: 0xa000, 0x7e2: 0x3c2c, + 0x7e4: 0xa000, 0x7e5: 0xa000, + 0x7ed: 0x3c25, 0x7ee: 0x2f63, 0x7ef: 0x2f6d, + 0x7f0: 0x3c33, 0x7f1: 0x3c3a, 0x7f2: 0xa000, 0x7f3: 0xa000, 0x7f4: 0x3c41, 0x7f5: 0x3c48, + 0x7f6: 0xa000, 0x7f7: 0xa000, 0x7f8: 0x3c4f, 0x7f9: 0x3c56, 0x7fa: 0xa000, 0x7fb: 0xa000, + 0x7fc: 0xa000, 0x7fd: 0xa000, + // Block 0x20, offset 0x800 + 0x800: 0x3c5d, 0x801: 0x3c64, 0x802: 0xa000, 0x803: 0xa000, 0x804: 0x3c79, 0x805: 0x3c80, + 0x806: 0xa000, 0x807: 0xa000, 0x808: 0x3c87, 0x809: 0x3c8e, + 0x811: 0xa000, + 0x812: 0xa000, + 0x822: 0xa000, + 0x828: 0xa000, 0x829: 0xa000, + 0x82b: 0xa000, 0x82c: 0x3ca3, 0x82d: 0x3caa, 0x82e: 0x3cb1, 0x82f: 0x3cb8, + 0x832: 0xa000, 0x833: 0xa000, 0x834: 0xa000, 0x835: 0xa000, + // Block 0x21, offset 0x840 + 0x860: 0x0023, 0x861: 0x0025, 0x862: 0x0027, 0x863: 0x0029, + 0x864: 0x002b, 0x865: 0x002d, 0x866: 0x002f, 0x867: 0x0031, 0x868: 0x0033, 0x869: 0x1882, + 0x86a: 0x1885, 0x86b: 0x1888, 0x86c: 0x188b, 0x86d: 0x188e, 0x86e: 0x1891, 0x86f: 0x1894, + 0x870: 0x1897, 0x871: 0x189a, 0x872: 0x189d, 0x873: 0x18a6, 0x874: 0x1a6b, 0x875: 0x1a6f, + 0x876: 0x1a73, 0x877: 0x1a77, 0x878: 0x1a7b, 0x879: 0x1a7f, 0x87a: 0x1a83, 0x87b: 0x1a87, + 0x87c: 0x1a8b, 0x87d: 0x1c83, 0x87e: 0x1c88, 0x87f: 0x1c8d, + // Block 0x22, offset 0x880 + 0x880: 0x1c92, 0x881: 0x1c97, 0x882: 0x1c9c, 0x883: 0x1ca1, 0x884: 0x1ca6, 0x885: 0x1cab, + 0x886: 0x1cb0, 0x887: 0x1cb5, 0x888: 0x187f, 0x889: 0x18a3, 0x88a: 0x18c7, 0x88b: 0x18eb, + 0x88c: 0x190f, 0x88d: 0x1918, 0x88e: 0x191e, 0x88f: 0x1924, 0x890: 0x192a, 0x891: 0x1b63, + 0x892: 0x1b67, 0x893: 0x1b6b, 0x894: 0x1b6f, 0x895: 0x1b73, 0x896: 0x1b77, 0x897: 0x1b7b, + 0x898: 0x1b7f, 0x899: 0x1b83, 0x89a: 0x1b87, 0x89b: 0x1b8b, 0x89c: 0x1af7, 0x89d: 0x1afb, + 0x89e: 0x1aff, 0x89f: 0x1b03, 0x8a0: 0x1b07, 0x8a1: 0x1b0b, 0x8a2: 0x1b0f, 0x8a3: 0x1b13, + 0x8a4: 0x1b17, 0x8a5: 0x1b1b, 0x8a6: 0x1b1f, 0x8a7: 0x1b23, 0x8a8: 0x1b27, 0x8a9: 0x1b2b, + 0x8aa: 0x1b2f, 0x8ab: 0x1b33, 0x8ac: 0x1b37, 0x8ad: 0x1b3b, 0x8ae: 0x1b3f, 0x8af: 0x1b43, + 0x8b0: 0x1b47, 0x8b1: 0x1b4b, 0x8b2: 0x1b4f, 0x8b3: 0x1b53, 0x8b4: 0x1b57, 0x8b5: 0x1b5b, + 0x8b6: 0x0043, 0x8b7: 0x0045, 0x8b8: 0x0047, 0x8b9: 0x0049, 0x8ba: 0x004b, 0x8bb: 0x004d, + 0x8bc: 0x004f, 0x8bd: 0x0051, 0x8be: 0x0053, 0x8bf: 0x0055, + // Block 0x23, offset 0x8c0 + 0x8c0: 0x06bf, 0x8c1: 0x06e3, 0x8c2: 0x06ef, 0x8c3: 0x06ff, 0x8c4: 0x0707, 0x8c5: 0x0713, + 0x8c6: 0x071b, 0x8c7: 0x0723, 0x8c8: 0x072f, 0x8c9: 0x0783, 0x8ca: 0x079b, 0x8cb: 0x07ab, + 0x8cc: 0x07bb, 0x8cd: 0x07cb, 0x8ce: 0x07db, 0x8cf: 0x07fb, 0x8d0: 0x07ff, 0x8d1: 0x0803, + 0x8d2: 0x0837, 0x8d3: 0x085f, 0x8d4: 0x086f, 0x8d5: 0x0877, 0x8d6: 0x087b, 0x8d7: 0x0887, + 0x8d8: 0x08a3, 0x8d9: 0x08a7, 0x8da: 0x08bf, 0x8db: 0x08c3, 0x8dc: 0x08cb, 0x8dd: 0x08db, + 0x8de: 0x0977, 0x8df: 0x098b, 0x8e0: 0x09cb, 0x8e1: 0x09df, 0x8e2: 0x09e7, 0x8e3: 0x09eb, + 0x8e4: 0x09fb, 0x8e5: 0x0a17, 0x8e6: 0x0a43, 0x8e7: 0x0a4f, 0x8e8: 0x0a6f, 0x8e9: 0x0a7b, + 0x8ea: 0x0a7f, 0x8eb: 0x0a83, 0x8ec: 0x0a9b, 0x8ed: 0x0a9f, 0x8ee: 0x0acb, 0x8ef: 0x0ad7, + 0x8f0: 0x0adf, 0x8f1: 0x0ae7, 0x8f2: 0x0af7, 0x8f3: 0x0aff, 0x8f4: 0x0b07, 0x8f5: 0x0b33, + 0x8f6: 0x0b37, 0x8f7: 0x0b3f, 0x8f8: 0x0b43, 0x8f9: 0x0b4b, 0x8fa: 0x0b53, 0x8fb: 0x0b63, + 0x8fc: 0x0b7f, 0x8fd: 0x0bf7, 0x8fe: 0x0c0b, 0x8ff: 0x0c0f, + // Block 0x24, offset 0x900 + 0x900: 0x0c8f, 0x901: 0x0c93, 0x902: 0x0ca7, 0x903: 0x0cab, 0x904: 0x0cb3, 0x905: 0x0cbb, + 0x906: 0x0cc3, 0x907: 0x0ccf, 0x908: 0x0cf7, 0x909: 0x0d07, 0x90a: 0x0d1b, 0x90b: 0x0d8b, + 0x90c: 0x0d97, 0x90d: 0x0da7, 0x90e: 0x0db3, 0x90f: 0x0dbf, 0x910: 0x0dc7, 0x911: 0x0dcb, + 0x912: 0x0dcf, 0x913: 0x0dd3, 0x914: 0x0dd7, 0x915: 0x0e8f, 0x916: 0x0ed7, 0x917: 0x0ee3, + 0x918: 0x0ee7, 0x919: 0x0eeb, 0x91a: 0x0eef, 0x91b: 0x0ef7, 0x91c: 0x0efb, 0x91d: 0x0f0f, + 0x91e: 0x0f2b, 0x91f: 0x0f33, 0x920: 0x0f73, 0x921: 0x0f77, 0x922: 0x0f7f, 0x923: 0x0f83, + 0x924: 0x0f8b, 0x925: 0x0f8f, 0x926: 0x0fb3, 0x927: 0x0fb7, 0x928: 0x0fd3, 0x929: 0x0fd7, + 0x92a: 0x0fdb, 0x92b: 0x0fdf, 0x92c: 0x0ff3, 0x92d: 0x1017, 0x92e: 0x101b, 0x92f: 0x101f, + 0x930: 0x1043, 0x931: 0x1083, 0x932: 0x1087, 0x933: 0x10a7, 0x934: 0x10b7, 0x935: 0x10bf, + 0x936: 0x10df, 0x937: 0x1103, 0x938: 0x1147, 0x939: 0x114f, 0x93a: 0x1163, 0x93b: 0x116f, + 0x93c: 0x1177, 0x93d: 0x117f, 0x93e: 0x1183, 0x93f: 0x1187, + // Block 0x25, offset 0x940 + 0x940: 0x119f, 0x941: 0x11a3, 0x942: 0x11bf, 0x943: 0x11c7, 0x944: 0x11cf, 0x945: 0x11d3, + 0x946: 0x11df, 0x947: 0x11e7, 0x948: 0x11eb, 0x949: 0x11ef, 0x94a: 0x11f7, 0x94b: 0x11fb, + 0x94c: 0x129b, 0x94d: 0x12af, 0x94e: 0x12e3, 0x94f: 0x12e7, 0x950: 0x12ef, 0x951: 0x131b, + 0x952: 0x1323, 0x953: 0x132b, 0x954: 0x1333, 0x955: 0x136f, 0x956: 0x1373, 0x957: 0x137b, + 0x958: 0x137f, 0x959: 0x1383, 0x95a: 0x13af, 0x95b: 0x13b3, 0x95c: 0x13bb, 0x95d: 0x13cf, + 0x95e: 0x13d3, 0x95f: 0x13ef, 0x960: 0x13f7, 0x961: 0x13fb, 0x962: 0x141f, 0x963: 0x143f, + 0x964: 0x1453, 0x965: 0x1457, 0x966: 0x145f, 0x967: 0x148b, 0x968: 0x148f, 0x969: 0x149f, + 0x96a: 0x14c3, 0x96b: 0x14cf, 0x96c: 0x14df, 0x96d: 0x14f7, 0x96e: 0x14ff, 0x96f: 0x1503, + 0x970: 0x1507, 0x971: 0x150b, 0x972: 0x1517, 0x973: 0x151b, 0x974: 0x1523, 0x975: 0x153f, + 0x976: 0x1543, 0x977: 0x1547, 0x978: 0x155f, 0x979: 0x1563, 0x97a: 0x156b, 0x97b: 0x157f, + 0x97c: 0x1583, 0x97d: 0x1587, 0x97e: 0x158f, 0x97f: 0x1593, + // Block 0x26, offset 0x980 + 0x986: 0xa000, 0x98b: 0xa000, + 0x98c: 0x3f0b, 0x98d: 0xa000, 0x98e: 0x3f13, 0x98f: 0xa000, 0x990: 0x3f1b, 0x991: 0xa000, + 0x992: 0x3f23, 0x993: 0xa000, 0x994: 0x3f2b, 0x995: 0xa000, 0x996: 0x3f33, 0x997: 0xa000, + 0x998: 0x3f3b, 0x999: 0xa000, 0x99a: 0x3f43, 0x99b: 0xa000, 0x99c: 0x3f4b, 0x99d: 0xa000, + 0x99e: 0x3f53, 0x99f: 0xa000, 0x9a0: 0x3f5b, 0x9a1: 0xa000, 0x9a2: 0x3f63, + 0x9a4: 0xa000, 0x9a5: 0x3f6b, 0x9a6: 0xa000, 0x9a7: 0x3f73, 0x9a8: 0xa000, 0x9a9: 0x3f7b, + 0x9af: 0xa000, + 0x9b0: 0x3f83, 0x9b1: 0x3f8b, 0x9b2: 0xa000, 0x9b3: 0x3f93, 0x9b4: 0x3f9b, 0x9b5: 0xa000, + 0x9b6: 0x3fa3, 0x9b7: 0x3fab, 0x9b8: 0xa000, 0x9b9: 0x3fb3, 0x9ba: 0x3fbb, 0x9bb: 0xa000, + 0x9bc: 0x3fc3, 0x9bd: 0x3fcb, + // Block 0x27, offset 0x9c0 + 0x9d4: 0x3f03, + 0x9d9: 0x9903, 0x9da: 0x9903, 0x9db: 0x42df, 0x9dc: 0x42e5, 0x9dd: 0xa000, + 0x9de: 0x3fd3, 0x9df: 0x26b7, + 0x9e6: 0xa000, + 0x9eb: 0xa000, 0x9ec: 0x3fe3, 0x9ed: 0xa000, 0x9ee: 0x3feb, 0x9ef: 0xa000, + 0x9f0: 0x3ff3, 0x9f1: 0xa000, 0x9f2: 0x3ffb, 0x9f3: 0xa000, 0x9f4: 0x4003, 0x9f5: 0xa000, + 0x9f6: 0x400b, 0x9f7: 0xa000, 0x9f8: 0x4013, 0x9f9: 0xa000, 0x9fa: 0x401b, 0x9fb: 0xa000, + 0x9fc: 0x4023, 0x9fd: 0xa000, 0x9fe: 0x402b, 0x9ff: 0xa000, + // Block 0x28, offset 0xa00 + 0xa00: 0x4033, 0xa01: 0xa000, 0xa02: 0x403b, 0xa04: 0xa000, 0xa05: 0x4043, + 0xa06: 0xa000, 0xa07: 0x404b, 0xa08: 0xa000, 0xa09: 0x4053, + 0xa0f: 0xa000, 0xa10: 0x405b, 0xa11: 0x4063, + 0xa12: 0xa000, 0xa13: 0x406b, 0xa14: 0x4073, 0xa15: 0xa000, 0xa16: 0x407b, 0xa17: 0x4083, + 0xa18: 0xa000, 0xa19: 0x408b, 0xa1a: 0x4093, 0xa1b: 0xa000, 0xa1c: 0x409b, 0xa1d: 0x40a3, + 0xa2f: 0xa000, + 0xa30: 0xa000, 0xa31: 0xa000, 0xa32: 0xa000, 0xa34: 0x3fdb, + 0xa37: 0x40ab, 0xa38: 0x40b3, 0xa39: 0x40bb, 0xa3a: 0x40c3, + 0xa3d: 0xa000, 0xa3e: 0x40cb, 0xa3f: 0x26cc, + // Block 0x29, offset 0xa40 + 0xa40: 0x0367, 0xa41: 0x032b, 0xa42: 0x032f, 0xa43: 0x0333, 0xa44: 0x037b, 0xa45: 0x0337, + 0xa46: 0x033b, 0xa47: 0x033f, 0xa48: 0x0343, 0xa49: 0x0347, 0xa4a: 0x034b, 0xa4b: 0x034f, + 0xa4c: 0x0353, 0xa4d: 0x0357, 0xa4e: 0x035b, 0xa4f: 0x49c0, 0xa50: 0x49c6, 0xa51: 0x49cc, + 0xa52: 0x49d2, 0xa53: 0x49d8, 0xa54: 0x49de, 0xa55: 0x49e4, 0xa56: 0x49ea, 0xa57: 0x49f0, + 0xa58: 0x49f6, 0xa59: 0x49fc, 0xa5a: 0x4a02, 0xa5b: 0x4a08, 0xa5c: 0x4a0e, 0xa5d: 0x4a14, + 0xa5e: 0x4a1a, 0xa5f: 0x4a20, 0xa60: 0x4a26, 0xa61: 0x4a2c, 0xa62: 0x4a32, 0xa63: 0x4a38, + 0xa64: 0x03c3, 0xa65: 0x035f, 0xa66: 0x0363, 0xa67: 0x03e7, 0xa68: 0x03eb, 0xa69: 0x03ef, + 0xa6a: 0x03f3, 0xa6b: 0x03f7, 0xa6c: 0x03fb, 0xa6d: 0x03ff, 0xa6e: 0x036b, 0xa6f: 0x0403, + 0xa70: 0x0407, 0xa71: 0x036f, 0xa72: 0x0373, 0xa73: 0x0377, 0xa74: 0x037f, 0xa75: 0x0383, + 0xa76: 0x0387, 0xa77: 0x038b, 0xa78: 0x038f, 0xa79: 0x0393, 0xa7a: 0x0397, 0xa7b: 0x039b, + 0xa7c: 0x039f, 0xa7d: 0x03a3, 0xa7e: 0x03a7, 0xa7f: 0x03ab, + // Block 0x2a, offset 0xa80 + 0xa80: 0x03af, 0xa81: 0x03b3, 0xa82: 0x040b, 0xa83: 0x040f, 0xa84: 0x03b7, 0xa85: 0x03bb, + 0xa86: 0x03bf, 0xa87: 0x03c7, 0xa88: 0x03cb, 0xa89: 0x03cf, 0xa8a: 0x03d3, 0xa8b: 0x03d7, + 0xa8c: 0x03db, 0xa8d: 0x03df, 0xa8e: 0x03e3, + 0xa92: 0x06bf, 0xa93: 0x071b, 0xa94: 0x06cb, 0xa95: 0x097b, 0xa96: 0x06cf, 0xa97: 0x06e7, + 0xa98: 0x06d3, 0xa99: 0x0f93, 0xa9a: 0x0707, 0xa9b: 0x06db, 0xa9c: 0x06c3, 0xa9d: 0x09ff, + 0xa9e: 0x098f, 0xa9f: 0x072f, + // Block 0x2b, offset 0xac0 + 0xac0: 0x2057, 0xac1: 0x205d, 0xac2: 0x2063, 0xac3: 0x2069, 0xac4: 0x206f, 0xac5: 0x2075, + 0xac6: 0x207b, 0xac7: 0x2081, 0xac8: 0x2087, 0xac9: 0x208d, 0xaca: 0x2093, 0xacb: 0x2099, + 0xacc: 0x209f, 0xacd: 0x20a5, 0xace: 0x2729, 0xacf: 0x2732, 0xad0: 0x273b, 0xad1: 0x2744, + 0xad2: 0x274d, 0xad3: 0x2756, 0xad4: 0x275f, 0xad5: 0x2768, 0xad6: 0x2771, 0xad7: 0x2783, + 0xad8: 0x278c, 0xad9: 0x2795, 0xada: 0x279e, 0xadb: 0x27a7, 0xadc: 0x277a, 0xadd: 0x2baf, + 0xade: 0x2af0, 0xae0: 0x20ab, 0xae1: 0x20c3, 0xae2: 0x20b7, 0xae3: 0x210b, + 0xae4: 0x20c9, 0xae5: 0x20e7, 0xae6: 0x20b1, 0xae7: 0x20e1, 0xae8: 0x20bd, 0xae9: 0x20f3, + 0xaea: 0x2123, 0xaeb: 0x2141, 0xaec: 0x213b, 0xaed: 0x212f, 0xaee: 0x217d, 0xaef: 0x2111, + 0xaf0: 0x211d, 0xaf1: 0x2135, 0xaf2: 0x2129, 0xaf3: 0x2153, 0xaf4: 0x20ff, 0xaf5: 0x2147, + 0xaf6: 0x2171, 0xaf7: 0x2159, 0xaf8: 0x20ed, 0xaf9: 0x20cf, 0xafa: 0x2105, 0xafb: 0x2117, + 0xafc: 0x214d, 0xafd: 0x20d5, 0xafe: 0x2177, 0xaff: 0x20f9, + // Block 0x2c, offset 0xb00 + 0xb00: 0x215f, 0xb01: 0x20db, 0xb02: 0x2165, 0xb03: 0x216b, 0xb04: 0x092f, 0xb05: 0x0b03, + 0xb06: 0x0ca7, 0xb07: 0x10c7, + 0xb10: 0x1bc7, 0xb11: 0x18a9, + 0xb12: 0x18ac, 0xb13: 0x18af, 0xb14: 0x18b2, 0xb15: 0x18b5, 0xb16: 0x18b8, 0xb17: 0x18bb, + 0xb18: 0x18be, 0xb19: 0x18c1, 0xb1a: 0x18ca, 0xb1b: 0x18cd, 0xb1c: 0x18d0, 0xb1d: 0x18d3, + 0xb1e: 0x18d6, 0xb1f: 0x18d9, 0xb20: 0x0313, 0xb21: 0x031b, 0xb22: 0x031f, 0xb23: 0x0327, + 0xb24: 0x032b, 0xb25: 0x032f, 0xb26: 0x0337, 0xb27: 0x033f, 0xb28: 0x0343, 0xb29: 0x034b, + 0xb2a: 0x034f, 0xb2b: 0x0353, 0xb2c: 0x0357, 0xb2d: 0x035b, 0xb2e: 0x2e1b, 0xb2f: 0x2e23, + 0xb30: 0x2e2b, 0xb31: 0x2e33, 0xb32: 0x2e3b, 0xb33: 0x2e43, 0xb34: 0x2e4b, 0xb35: 0x2e53, + 0xb36: 0x2e63, 0xb37: 0x2e6b, 0xb38: 0x2e73, 0xb39: 0x2e7b, 0xb3a: 0x2e83, 0xb3b: 0x2e8b, + 0xb3c: 0x2ed6, 0xb3d: 0x2e9e, 0xb3e: 0x2e5b, + // Block 0x2d, offset 0xb40 + 0xb40: 0x06bf, 0xb41: 0x071b, 0xb42: 0x06cb, 0xb43: 0x097b, 0xb44: 0x071f, 0xb45: 0x07af, + 0xb46: 0x06c7, 0xb47: 0x07ab, 0xb48: 0x070b, 0xb49: 0x0887, 0xb4a: 0x0d07, 0xb4b: 0x0e8f, + 0xb4c: 0x0dd7, 0xb4d: 0x0d1b, 0xb4e: 0x145f, 0xb4f: 0x098b, 0xb50: 0x0ccf, 0xb51: 0x0d4b, + 0xb52: 0x0d0b, 0xb53: 0x104b, 0xb54: 0x08fb, 0xb55: 0x0f03, 0xb56: 0x1387, 0xb57: 0x105f, + 0xb58: 0x0843, 0xb59: 0x108f, 0xb5a: 0x0f9b, 0xb5b: 0x0a17, 0xb5c: 0x140f, 0xb5d: 0x077f, + 0xb5e: 0x08ab, 0xb5f: 0x0df7, 0xb60: 0x1527, 0xb61: 0x0743, 0xb62: 0x07d3, 0xb63: 0x0d9b, + 0xb64: 0x06cf, 0xb65: 0x06e7, 0xb66: 0x06d3, 0xb67: 0x0adb, 0xb68: 0x08ef, 0xb69: 0x087f, + 0xb6a: 0x0a57, 0xb6b: 0x0a4b, 0xb6c: 0x0feb, 0xb6d: 0x073f, 0xb6e: 0x139b, 0xb6f: 0x089b, + 0xb70: 0x09f3, 0xb71: 0x18dc, 0xb72: 0x18df, 0xb73: 0x18e2, 0xb74: 0x18e5, 0xb75: 0x18ee, + 0xb76: 0x18f1, 0xb77: 0x18f4, 0xb78: 0x18f7, 0xb79: 0x18fa, 0xb7a: 0x18fd, 0xb7b: 0x1900, + 0xb7c: 0x1903, 0xb7d: 0x1906, 0xb7e: 0x1909, 0xb7f: 0x1912, + // Block 0x2e, offset 0xb80 + 0xb80: 0x1cc9, 0xb81: 0x1cd8, 0xb82: 0x1ce7, 0xb83: 0x1cf6, 0xb84: 0x1d05, 0xb85: 0x1d14, + 0xb86: 0x1d23, 0xb87: 0x1d32, 0xb88: 0x1d41, 0xb89: 0x218f, 0xb8a: 0x21a1, 0xb8b: 0x21b3, + 0xb8c: 0x1954, 0xb8d: 0x1c07, 0xb8e: 0x19d5, 0xb8f: 0x1bab, 0xb90: 0x04cb, 0xb91: 0x04d3, + 0xb92: 0x04db, 0xb93: 0x04e3, 0xb94: 0x04eb, 0xb95: 0x04ef, 0xb96: 0x04f3, 0xb97: 0x04f7, + 0xb98: 0x04fb, 0xb99: 0x04ff, 0xb9a: 0x0503, 0xb9b: 0x0507, 0xb9c: 0x050b, 0xb9d: 0x050f, + 0xb9e: 0x0513, 0xb9f: 0x0517, 0xba0: 0x051b, 0xba1: 0x0523, 0xba2: 0x0527, 0xba3: 0x052b, + 0xba4: 0x052f, 0xba5: 0x0533, 0xba6: 0x0537, 0xba7: 0x053b, 0xba8: 0x053f, 0xba9: 0x0543, + 0xbaa: 0x0547, 0xbab: 0x054b, 0xbac: 0x054f, 0xbad: 0x0553, 0xbae: 0x0557, 0xbaf: 0x055b, + 0xbb0: 0x055f, 0xbb1: 0x0563, 0xbb2: 0x0567, 0xbb3: 0x056f, 0xbb4: 0x0577, 0xbb5: 0x057f, + 0xbb6: 0x0583, 0xbb7: 0x0587, 0xbb8: 0x058b, 0xbb9: 0x058f, 0xbba: 0x0593, 0xbbb: 0x0597, + 0xbbc: 0x059b, 0xbbd: 0x059f, 0xbbe: 0x05a3, + // Block 0x2f, offset 0xbc0 + 0xbc0: 0x2b0f, 0xbc1: 0x29ab, 0xbc2: 0x2b1f, 0xbc3: 0x2883, 0xbc4: 0x2ee7, 0xbc5: 0x288d, + 0xbc6: 0x2897, 0xbc7: 0x2f2b, 0xbc8: 0x29b8, 0xbc9: 0x28a1, 0xbca: 0x28ab, 0xbcb: 0x28b5, + 0xbcc: 0x29df, 0xbcd: 0x29ec, 0xbce: 0x29c5, 0xbcf: 0x29d2, 0xbd0: 0x2eac, 0xbd1: 0x29f9, + 0xbd2: 0x2a06, 0xbd3: 0x2bc1, 0xbd4: 0x26be, 0xbd5: 0x2bd4, 0xbd6: 0x2be7, 0xbd7: 0x2b2f, + 0xbd8: 0x2a13, 0xbd9: 0x2bfa, 0xbda: 0x2c0d, 0xbdb: 0x2a20, 0xbdc: 0x28bf, 0xbdd: 0x28c9, + 0xbde: 0x2eba, 0xbdf: 0x2a2d, 0xbe0: 0x2b3f, 0xbe1: 0x2ef8, 0xbe2: 0x28d3, 0xbe3: 0x28dd, + 0xbe4: 0x2a3a, 0xbe5: 0x28e7, 0xbe6: 0x28f1, 0xbe7: 0x26d3, 0xbe8: 0x26da, 0xbe9: 0x28fb, + 0xbea: 0x2905, 0xbeb: 0x2c20, 0xbec: 0x2a47, 0xbed: 0x2b4f, 0xbee: 0x2c33, 0xbef: 0x2a54, + 0xbf0: 0x2919, 0xbf1: 0x290f, 0xbf2: 0x2f3f, 0xbf3: 0x2a61, 0xbf4: 0x2c46, 0xbf5: 0x2923, + 0xbf6: 0x2b5f, 0xbf7: 0x292d, 0xbf8: 0x2a7b, 0xbf9: 0x2937, 0xbfa: 0x2a88, 0xbfb: 0x2f09, + 0xbfc: 0x2a6e, 0xbfd: 0x2b6f, 0xbfe: 0x2a95, 0xbff: 0x26e1, + // Block 0x30, offset 0xc00 + 0xc00: 0x2f1a, 0xc01: 0x2941, 0xc02: 0x294b, 0xc03: 0x2aa2, 0xc04: 0x2955, 0xc05: 0x295f, + 0xc06: 0x2969, 0xc07: 0x2b7f, 0xc08: 0x2aaf, 0xc09: 0x26e8, 0xc0a: 0x2c59, 0xc0b: 0x2e93, + 0xc0c: 0x2b8f, 0xc0d: 0x2abc, 0xc0e: 0x2ec8, 0xc0f: 0x2973, 0xc10: 0x297d, 0xc11: 0x2ac9, + 0xc12: 0x26ef, 0xc13: 0x2ad6, 0xc14: 0x2b9f, 0xc15: 0x26f6, 0xc16: 0x2c6c, 0xc17: 0x2987, + 0xc18: 0x1cba, 0xc19: 0x1cce, 0xc1a: 0x1cdd, 0xc1b: 0x1cec, 0xc1c: 0x1cfb, 0xc1d: 0x1d0a, + 0xc1e: 0x1d19, 0xc1f: 0x1d28, 0xc20: 0x1d37, 0xc21: 0x1d46, 0xc22: 0x2195, 0xc23: 0x21a7, + 0xc24: 0x21b9, 0xc25: 0x21c5, 0xc26: 0x21d1, 0xc27: 0x21dd, 0xc28: 0x21e9, 0xc29: 0x21f5, + 0xc2a: 0x2201, 0xc2b: 0x220d, 0xc2c: 0x2249, 0xc2d: 0x2255, 0xc2e: 0x2261, 0xc2f: 0x226d, + 0xc30: 0x2279, 0xc31: 0x1c17, 0xc32: 0x19c9, 0xc33: 0x1936, 0xc34: 0x1be7, 0xc35: 0x1a4a, + 0xc36: 0x1a59, 0xc37: 0x19cf, 0xc38: 0x1bff, 0xc39: 0x1c03, 0xc3a: 0x1960, 0xc3b: 0x2704, + 0xc3c: 0x2712, 0xc3d: 0x26fd, 0xc3e: 0x270b, 0xc3f: 0x2ae3, + // Block 0x31, offset 0xc40 + 0xc40: 0x1a4d, 0xc41: 0x1a35, 0xc42: 0x1c63, 0xc43: 0x1a1d, 0xc44: 0x19f6, 0xc45: 0x1969, + 0xc46: 0x1978, 0xc47: 0x1948, 0xc48: 0x1bf3, 0xc49: 0x1d55, 0xc4a: 0x1a50, 0xc4b: 0x1a38, + 0xc4c: 0x1c67, 0xc4d: 0x1c73, 0xc4e: 0x1a29, 0xc4f: 0x19ff, 0xc50: 0x1957, 0xc51: 0x1c1f, + 0xc52: 0x1bb3, 0xc53: 0x1b9f, 0xc54: 0x1bcf, 0xc55: 0x1c77, 0xc56: 0x1a2c, 0xc57: 0x19cc, + 0xc58: 0x1a02, 0xc59: 0x19e1, 0xc5a: 0x1a44, 0xc5b: 0x1c7b, 0xc5c: 0x1a2f, 0xc5d: 0x19c3, + 0xc5e: 0x1a05, 0xc5f: 0x1c3f, 0xc60: 0x1bf7, 0xc61: 0x1a17, 0xc62: 0x1c27, 0xc63: 0x1c43, + 0xc64: 0x1bfb, 0xc65: 0x1a1a, 0xc66: 0x1c2b, 0xc67: 0x22eb, 0xc68: 0x22ff, 0xc69: 0x1999, + 0xc6a: 0x1c23, 0xc6b: 0x1bb7, 0xc6c: 0x1ba3, 0xc6d: 0x1c4b, 0xc6e: 0x2719, 0xc6f: 0x27b0, + 0xc70: 0x1a5c, 0xc71: 0x1a47, 0xc72: 0x1c7f, 0xc73: 0x1a32, 0xc74: 0x1a53, 0xc75: 0x1a3b, + 0xc76: 0x1c6b, 0xc77: 0x1a20, 0xc78: 0x19f9, 0xc79: 0x1984, 0xc7a: 0x1a56, 0xc7b: 0x1a3e, + 0xc7c: 0x1c6f, 0xc7d: 0x1a23, 0xc7e: 0x19fc, 0xc7f: 0x1987, + // Block 0x32, offset 0xc80 + 0xc80: 0x1c2f, 0xc81: 0x1bbb, 0xc82: 0x1d50, 0xc83: 0x1939, 0xc84: 0x19bd, 0xc85: 0x19c0, + 0xc86: 0x22f8, 0xc87: 0x1b97, 0xc88: 0x19c6, 0xc89: 0x194b, 0xc8a: 0x19e4, 0xc8b: 0x194e, + 0xc8c: 0x19ed, 0xc8d: 0x196c, 0xc8e: 0x196f, 0xc8f: 0x1a08, 0xc90: 0x1a0e, 0xc91: 0x1a11, + 0xc92: 0x1c33, 0xc93: 0x1a14, 0xc94: 0x1a26, 0xc95: 0x1c3b, 0xc96: 0x1c47, 0xc97: 0x1993, + 0xc98: 0x1d5a, 0xc99: 0x1bbf, 0xc9a: 0x1996, 0xc9b: 0x1a5f, 0xc9c: 0x19a8, 0xc9d: 0x19b7, + 0xc9e: 0x22e5, 0xc9f: 0x22df, 0xca0: 0x1cc4, 0xca1: 0x1cd3, 0xca2: 0x1ce2, 0xca3: 0x1cf1, + 0xca4: 0x1d00, 0xca5: 0x1d0f, 0xca6: 0x1d1e, 0xca7: 0x1d2d, 0xca8: 0x1d3c, 0xca9: 0x2189, + 0xcaa: 0x219b, 0xcab: 0x21ad, 0xcac: 0x21bf, 0xcad: 0x21cb, 0xcae: 0x21d7, 0xcaf: 0x21e3, + 0xcb0: 0x21ef, 0xcb1: 0x21fb, 0xcb2: 0x2207, 0xcb3: 0x2243, 0xcb4: 0x224f, 0xcb5: 0x225b, + 0xcb6: 0x2267, 0xcb7: 0x2273, 0xcb8: 0x227f, 0xcb9: 0x2285, 0xcba: 0x228b, 0xcbb: 0x2291, + 0xcbc: 0x2297, 0xcbd: 0x22a9, 0xcbe: 0x22af, 0xcbf: 0x1c13, + // Block 0x33, offset 0xcc0 + 0xcc0: 0x1377, 0xcc1: 0x0cfb, 0xcc2: 0x13d3, 0xcc3: 0x139f, 0xcc4: 0x0e57, 0xcc5: 0x06eb, + 0xcc6: 0x08df, 0xcc7: 0x162b, 0xcc8: 0x162b, 0xcc9: 0x0a0b, 0xcca: 0x145f, 0xccb: 0x0943, + 0xccc: 0x0a07, 0xccd: 0x0bef, 0xcce: 0x0fcf, 0xccf: 0x115f, 0xcd0: 0x1297, 0xcd1: 0x12d3, + 0xcd2: 0x1307, 0xcd3: 0x141b, 0xcd4: 0x0d73, 0xcd5: 0x0dff, 0xcd6: 0x0eab, 0xcd7: 0x0f43, + 0xcd8: 0x125f, 0xcd9: 0x1447, 0xcda: 0x1573, 0xcdb: 0x070f, 0xcdc: 0x08b3, 0xcdd: 0x0d87, + 0xcde: 0x0ecf, 0xcdf: 0x1293, 0xce0: 0x15c3, 0xce1: 0x0ab3, 0xce2: 0x0e77, 0xce3: 0x1283, + 0xce4: 0x1317, 0xce5: 0x0c23, 0xce6: 0x11bb, 0xce7: 0x12df, 0xce8: 0x0b1f, 0xce9: 0x0d0f, + 0xcea: 0x0e17, 0xceb: 0x0f1b, 0xcec: 0x1427, 0xced: 0x074f, 0xcee: 0x07e7, 0xcef: 0x0853, + 0xcf0: 0x0c8b, 0xcf1: 0x0d7f, 0xcf2: 0x0ecb, 0xcf3: 0x0fef, 0xcf4: 0x1177, 0xcf5: 0x128b, + 0xcf6: 0x12a3, 0xcf7: 0x13c7, 0xcf8: 0x14ef, 0xcf9: 0x15a3, 0xcfa: 0x15bf, 0xcfb: 0x102b, + 0xcfc: 0x106b, 0xcfd: 0x1123, 0xcfe: 0x1243, 0xcff: 0x147b, + // Block 0x34, offset 0xd00 + 0xd00: 0x15cb, 0xd01: 0x134b, 0xd02: 0x09c7, 0xd03: 0x0b3b, 0xd04: 0x10db, 0xd05: 0x119b, + 0xd06: 0x0eff, 0xd07: 0x1033, 0xd08: 0x1397, 0xd09: 0x14e7, 0xd0a: 0x09c3, 0xd0b: 0x0a8f, + 0xd0c: 0x0d77, 0xd0d: 0x0e2b, 0xd0e: 0x0e5f, 0xd0f: 0x1113, 0xd10: 0x113b, 0xd11: 0x14a7, + 0xd12: 0x084f, 0xd13: 0x11a7, 0xd14: 0x07f3, 0xd15: 0x07ef, 0xd16: 0x1097, 0xd17: 0x1127, + 0xd18: 0x125b, 0xd19: 0x14af, 0xd1a: 0x1367, 0xd1b: 0x0c27, 0xd1c: 0x0d73, 0xd1d: 0x1357, + 0xd1e: 0x06f7, 0xd1f: 0x0a63, 0xd20: 0x0b93, 0xd21: 0x0f2f, 0xd22: 0x0faf, 0xd23: 0x0873, + 0xd24: 0x103b, 0xd25: 0x075f, 0xd26: 0x0b77, 0xd27: 0x06d7, 0xd28: 0x0deb, 0xd29: 0x0ca3, + 0xd2a: 0x110f, 0xd2b: 0x08c7, 0xd2c: 0x09b3, 0xd2d: 0x0ffb, 0xd2e: 0x1263, 0xd2f: 0x133b, + 0xd30: 0x0db7, 0xd31: 0x13f7, 0xd32: 0x0de3, 0xd33: 0x0c37, 0xd34: 0x121b, 0xd35: 0x0c57, + 0xd36: 0x0fab, 0xd37: 0x072b, 0xd38: 0x07a7, 0xd39: 0x07eb, 0xd3a: 0x0d53, 0xd3b: 0x10fb, + 0xd3c: 0x11f3, 0xd3d: 0x1347, 0xd3e: 0x145b, 0xd3f: 0x085b, + // Block 0x35, offset 0xd40 + 0xd40: 0x090f, 0xd41: 0x0a17, 0xd42: 0x0b2f, 0xd43: 0x0cbf, 0xd44: 0x0e7b, 0xd45: 0x103f, + 0xd46: 0x1497, 0xd47: 0x157b, 0xd48: 0x15cf, 0xd49: 0x15e7, 0xd4a: 0x0837, 0xd4b: 0x0cf3, + 0xd4c: 0x0da3, 0xd4d: 0x13eb, 0xd4e: 0x0afb, 0xd4f: 0x0bd7, 0xd50: 0x0bf3, 0xd51: 0x0c83, + 0xd52: 0x0e6b, 0xd53: 0x0eb7, 0xd54: 0x0f67, 0xd55: 0x108b, 0xd56: 0x112f, 0xd57: 0x1193, + 0xd58: 0x13db, 0xd59: 0x126b, 0xd5a: 0x1403, 0xd5b: 0x147f, 0xd5c: 0x080f, 0xd5d: 0x083b, + 0xd5e: 0x0923, 0xd5f: 0x0ea7, 0xd60: 0x12f3, 0xd61: 0x133b, 0xd62: 0x0b1b, 0xd63: 0x0b8b, + 0xd64: 0x0c4f, 0xd65: 0x0daf, 0xd66: 0x10d7, 0xd67: 0x0f23, 0xd68: 0x073b, 0xd69: 0x097f, + 0xd6a: 0x0a63, 0xd6b: 0x0ac7, 0xd6c: 0x0b97, 0xd6d: 0x0f3f, 0xd6e: 0x0f5b, 0xd6f: 0x116b, + 0xd70: 0x118b, 0xd71: 0x1463, 0xd72: 0x14e3, 0xd73: 0x14f3, 0xd74: 0x152f, 0xd75: 0x0753, + 0xd76: 0x107f, 0xd77: 0x144f, 0xd78: 0x14cb, 0xd79: 0x0baf, 0xd7a: 0x0717, 0xd7b: 0x0777, + 0xd7c: 0x0a67, 0xd7d: 0x0a87, 0xd7e: 0x0caf, 0xd7f: 0x0d73, + // Block 0x36, offset 0xd80 + 0xd80: 0x0ec3, 0xd81: 0x0fcb, 0xd82: 0x1277, 0xd83: 0x1417, 0xd84: 0x1623, 0xd85: 0x0ce3, + 0xd86: 0x14a3, 0xd87: 0x0833, 0xd88: 0x0d2f, 0xd89: 0x0d3b, 0xd8a: 0x0e0f, 0xd8b: 0x0e47, + 0xd8c: 0x0f4b, 0xd8d: 0x0fa7, 0xd8e: 0x1027, 0xd8f: 0x110b, 0xd90: 0x153b, 0xd91: 0x07af, + 0xd92: 0x0c03, 0xd93: 0x14b3, 0xd94: 0x0767, 0xd95: 0x0aab, 0xd96: 0x0e2f, 0xd97: 0x13df, + 0xd98: 0x0b67, 0xd99: 0x0bb7, 0xd9a: 0x0d43, 0xd9b: 0x0f2f, 0xd9c: 0x14bb, 0xd9d: 0x0817, + 0xd9e: 0x08ff, 0xd9f: 0x0a97, 0xda0: 0x0cd3, 0xda1: 0x0d1f, 0xda2: 0x0d5f, 0xda3: 0x0df3, + 0xda4: 0x0f47, 0xda5: 0x0fbb, 0xda6: 0x1157, 0xda7: 0x12f7, 0xda8: 0x1303, 0xda9: 0x1457, + 0xdaa: 0x14d7, 0xdab: 0x0883, 0xdac: 0x0e4b, 0xdad: 0x0903, 0xdae: 0x0ec7, 0xdaf: 0x0f6b, + 0xdb0: 0x1287, 0xdb1: 0x14bf, 0xdb2: 0x15ab, 0xdb3: 0x15d3, 0xdb4: 0x0d37, 0xdb5: 0x0e27, + 0xdb6: 0x11c3, 0xdb7: 0x10b7, 0xdb8: 0x10c3, 0xdb9: 0x10e7, 0xdba: 0x0f17, 0xdbb: 0x0e9f, + 0xdbc: 0x1363, 0xdbd: 0x0733, 0xdbe: 0x122b, 0xdbf: 0x081b, + // Block 0x37, offset 0xdc0 + 0xdc0: 0x080b, 0xdc1: 0x0b0b, 0xdc2: 0x0c2b, 0xdc3: 0x10f3, 0xdc4: 0x0a53, 0xdc5: 0x0e03, + 0xdc6: 0x0cef, 0xdc7: 0x13e7, 0xdc8: 0x12e7, 0xdc9: 0x14ab, 0xdca: 0x1323, 0xdcb: 0x0b27, + 0xdcc: 0x0787, 0xdcd: 0x095b, 0xdd0: 0x09af, + 0xdd2: 0x0cdf, 0xdd5: 0x07f7, 0xdd6: 0x0f1f, 0xdd7: 0x0fe3, + 0xdd8: 0x1047, 0xdd9: 0x1063, 0xdda: 0x1067, 0xddb: 0x107b, 0xddc: 0x14fb, 0xddd: 0x10eb, + 0xdde: 0x116f, 0xde0: 0x128f, 0xde2: 0x1353, + 0xde5: 0x1407, 0xde6: 0x1433, + 0xdea: 0x154f, 0xdeb: 0x1553, 0xdec: 0x1557, 0xded: 0x15bb, 0xdee: 0x142b, 0xdef: 0x14c7, + 0xdf0: 0x0757, 0xdf1: 0x077b, 0xdf2: 0x078f, 0xdf3: 0x084b, 0xdf4: 0x0857, 0xdf5: 0x0897, + 0xdf6: 0x094b, 0xdf7: 0x0967, 0xdf8: 0x096f, 0xdf9: 0x09ab, 0xdfa: 0x09b7, 0xdfb: 0x0a93, + 0xdfc: 0x0a9b, 0xdfd: 0x0ba3, 0xdfe: 0x0bcb, 0xdff: 0x0bd3, + // Block 0x38, offset 0xe00 + 0xe00: 0x0beb, 0xe01: 0x0c97, 0xe02: 0x0cc7, 0xe03: 0x0ce7, 0xe04: 0x0d57, 0xe05: 0x0e1b, + 0xe06: 0x0e37, 0xe07: 0x0e67, 0xe08: 0x0ebb, 0xe09: 0x0edb, 0xe0a: 0x0f4f, 0xe0b: 0x102f, + 0xe0c: 0x104b, 0xe0d: 0x1053, 0xe0e: 0x104f, 0xe0f: 0x1057, 0xe10: 0x105b, 0xe11: 0x105f, + 0xe12: 0x1073, 0xe13: 0x1077, 0xe14: 0x109b, 0xe15: 0x10af, 0xe16: 0x10cb, 0xe17: 0x112f, + 0xe18: 0x1137, 0xe19: 0x113f, 0xe1a: 0x1153, 0xe1b: 0x117b, 0xe1c: 0x11cb, 0xe1d: 0x11ff, + 0xe1e: 0x11ff, 0xe1f: 0x1267, 0xe20: 0x130f, 0xe21: 0x1327, 0xe22: 0x135b, 0xe23: 0x135f, + 0xe24: 0x13a3, 0xe25: 0x13a7, 0xe26: 0x13ff, 0xe27: 0x1407, 0xe28: 0x14db, 0xe29: 0x151f, + 0xe2a: 0x1537, 0xe2b: 0x0b9b, 0xe2c: 0x171e, 0xe2d: 0x11e3, + 0xe30: 0x06df, 0xe31: 0x07e3, 0xe32: 0x07a3, 0xe33: 0x074b, 0xe34: 0x078b, 0xe35: 0x07b7, + 0xe36: 0x0847, 0xe37: 0x0863, 0xe38: 0x094b, 0xe39: 0x0937, 0xe3a: 0x0947, 0xe3b: 0x0963, + 0xe3c: 0x09af, 0xe3d: 0x09bf, 0xe3e: 0x0a03, 0xe3f: 0x0a0f, + // Block 0x39, offset 0xe40 + 0xe40: 0x0a2b, 0xe41: 0x0a3b, 0xe42: 0x0b23, 0xe43: 0x0b2b, 0xe44: 0x0b5b, 0xe45: 0x0b7b, + 0xe46: 0x0bab, 0xe47: 0x0bc3, 0xe48: 0x0bb3, 0xe49: 0x0bd3, 0xe4a: 0x0bc7, 0xe4b: 0x0beb, + 0xe4c: 0x0c07, 0xe4d: 0x0c5f, 0xe4e: 0x0c6b, 0xe4f: 0x0c73, 0xe50: 0x0c9b, 0xe51: 0x0cdf, + 0xe52: 0x0d0f, 0xe53: 0x0d13, 0xe54: 0x0d27, 0xe55: 0x0da7, 0xe56: 0x0db7, 0xe57: 0x0e0f, + 0xe58: 0x0e5b, 0xe59: 0x0e53, 0xe5a: 0x0e67, 0xe5b: 0x0e83, 0xe5c: 0x0ebb, 0xe5d: 0x1013, + 0xe5e: 0x0edf, 0xe5f: 0x0f13, 0xe60: 0x0f1f, 0xe61: 0x0f5f, 0xe62: 0x0f7b, 0xe63: 0x0f9f, + 0xe64: 0x0fc3, 0xe65: 0x0fc7, 0xe66: 0x0fe3, 0xe67: 0x0fe7, 0xe68: 0x0ff7, 0xe69: 0x100b, + 0xe6a: 0x1007, 0xe6b: 0x1037, 0xe6c: 0x10b3, 0xe6d: 0x10cb, 0xe6e: 0x10e3, 0xe6f: 0x111b, + 0xe70: 0x112f, 0xe71: 0x114b, 0xe72: 0x117b, 0xe73: 0x122f, 0xe74: 0x1257, 0xe75: 0x12cb, + 0xe76: 0x1313, 0xe77: 0x131f, 0xe78: 0x1327, 0xe79: 0x133f, 0xe7a: 0x1353, 0xe7b: 0x1343, + 0xe7c: 0x135b, 0xe7d: 0x1357, 0xe7e: 0x134f, 0xe7f: 0x135f, + // Block 0x3a, offset 0xe80 + 0xe80: 0x136b, 0xe81: 0x13a7, 0xe82: 0x13e3, 0xe83: 0x1413, 0xe84: 0x144b, 0xe85: 0x146b, + 0xe86: 0x14b7, 0xe87: 0x14db, 0xe88: 0x14fb, 0xe89: 0x150f, 0xe8a: 0x151f, 0xe8b: 0x152b, + 0xe8c: 0x1537, 0xe8d: 0x158b, 0xe8e: 0x162b, 0xe8f: 0x16b5, 0xe90: 0x16b0, 0xe91: 0x16e2, + 0xe92: 0x0607, 0xe93: 0x062f, 0xe94: 0x0633, 0xe95: 0x1764, 0xe96: 0x1791, 0xe97: 0x1809, + 0xe98: 0x1617, 0xe99: 0x1627, + // Block 0x3b, offset 0xec0 + 0xec0: 0x19d8, 0xec1: 0x19db, 0xec2: 0x19de, 0xec3: 0x1c0b, 0xec4: 0x1c0f, 0xec5: 0x1a62, + 0xec6: 0x1a62, + 0xed3: 0x1d78, 0xed4: 0x1d69, 0xed5: 0x1d6e, 0xed6: 0x1d7d, 0xed7: 0x1d73, + 0xedd: 0x4393, + 0xede: 0x8115, 0xedf: 0x4405, 0xee0: 0x022d, 0xee1: 0x0215, 0xee2: 0x021e, 0xee3: 0x0221, + 0xee4: 0x0224, 0xee5: 0x0227, 0xee6: 0x022a, 0xee7: 0x0230, 0xee8: 0x0233, 0xee9: 0x0017, + 0xeea: 0x43f3, 0xeeb: 0x43f9, 0xeec: 0x44f7, 0xeed: 0x44ff, 0xeee: 0x434b, 0xeef: 0x4351, + 0xef0: 0x4357, 0xef1: 0x435d, 0xef2: 0x4369, 0xef3: 0x436f, 0xef4: 0x4375, 0xef5: 0x4381, + 0xef6: 0x4387, 0xef8: 0x438d, 0xef9: 0x4399, 0xefa: 0x439f, 0xefb: 0x43a5, + 0xefc: 0x43b1, 0xefe: 0x43b7, + // Block 0x3c, offset 0xf00 + 0xf00: 0x43bd, 0xf01: 0x43c3, 0xf03: 0x43c9, 0xf04: 0x43cf, + 0xf06: 0x43db, 0xf07: 0x43e1, 0xf08: 0x43e7, 0xf09: 0x43ed, 0xf0a: 0x43ff, 0xf0b: 0x437b, + 0xf0c: 0x4363, 0xf0d: 0x43ab, 0xf0e: 0x43d5, 0xf0f: 0x1d82, 0xf10: 0x0299, 0xf11: 0x0299, + 0xf12: 0x02a2, 0xf13: 0x02a2, 0xf14: 0x02a2, 0xf15: 0x02a2, 0xf16: 0x02a5, 0xf17: 0x02a5, + 0xf18: 0x02a5, 0xf19: 0x02a5, 0xf1a: 0x02ab, 0xf1b: 0x02ab, 0xf1c: 0x02ab, 0xf1d: 0x02ab, + 0xf1e: 0x029f, 0xf1f: 0x029f, 0xf20: 0x029f, 0xf21: 0x029f, 0xf22: 0x02a8, 0xf23: 0x02a8, + 0xf24: 0x02a8, 0xf25: 0x02a8, 0xf26: 0x029c, 0xf27: 0x029c, 0xf28: 0x029c, 0xf29: 0x029c, + 0xf2a: 0x02cf, 0xf2b: 0x02cf, 0xf2c: 0x02cf, 0xf2d: 0x02cf, 0xf2e: 0x02d2, 0xf2f: 0x02d2, + 0xf30: 0x02d2, 0xf31: 0x02d2, 0xf32: 0x02b1, 0xf33: 0x02b1, 0xf34: 0x02b1, 0xf35: 0x02b1, + 0xf36: 0x02ae, 0xf37: 0x02ae, 0xf38: 0x02ae, 0xf39: 0x02ae, 0xf3a: 0x02b4, 0xf3b: 0x02b4, + 0xf3c: 0x02b4, 0xf3d: 0x02b4, 0xf3e: 0x02b7, 0xf3f: 0x02b7, + // Block 0x3d, offset 0xf40 + 0xf40: 0x02b7, 0xf41: 0x02b7, 0xf42: 0x02c0, 0xf43: 0x02c0, 0xf44: 0x02bd, 0xf45: 0x02bd, + 0xf46: 0x02c3, 0xf47: 0x02c3, 0xf48: 0x02ba, 0xf49: 0x02ba, 0xf4a: 0x02c9, 0xf4b: 0x02c9, + 0xf4c: 0x02c6, 0xf4d: 0x02c6, 0xf4e: 0x02d5, 0xf4f: 0x02d5, 0xf50: 0x02d5, 0xf51: 0x02d5, + 0xf52: 0x02db, 0xf53: 0x02db, 0xf54: 0x02db, 0xf55: 0x02db, 0xf56: 0x02e1, 0xf57: 0x02e1, + 0xf58: 0x02e1, 0xf59: 0x02e1, 0xf5a: 0x02de, 0xf5b: 0x02de, 0xf5c: 0x02de, 0xf5d: 0x02de, + 0xf5e: 0x02e4, 0xf5f: 0x02e4, 0xf60: 0x02e7, 0xf61: 0x02e7, 0xf62: 0x02e7, 0xf63: 0x02e7, + 0xf64: 0x4471, 0xf65: 0x4471, 0xf66: 0x02ed, 0xf67: 0x02ed, 0xf68: 0x02ed, 0xf69: 0x02ed, + 0xf6a: 0x02ea, 0xf6b: 0x02ea, 0xf6c: 0x02ea, 0xf6d: 0x02ea, 0xf6e: 0x0308, 0xf6f: 0x0308, + 0xf70: 0x446b, 0xf71: 0x446b, + // Block 0x3e, offset 0xf80 + 0xf93: 0x02d8, 0xf94: 0x02d8, 0xf95: 0x02d8, 0xf96: 0x02d8, 0xf97: 0x02f6, + 0xf98: 0x02f6, 0xf99: 0x02f3, 0xf9a: 0x02f3, 0xf9b: 0x02f9, 0xf9c: 0x02f9, 0xf9d: 0x2052, + 0xf9e: 0x02ff, 0xf9f: 0x02ff, 0xfa0: 0x02f0, 0xfa1: 0x02f0, 0xfa2: 0x02fc, 0xfa3: 0x02fc, + 0xfa4: 0x0305, 0xfa5: 0x0305, 0xfa6: 0x0305, 0xfa7: 0x0305, 0xfa8: 0x028d, 0xfa9: 0x028d, + 0xfaa: 0x25ad, 0xfab: 0x25ad, 0xfac: 0x261d, 0xfad: 0x261d, 0xfae: 0x25ec, 0xfaf: 0x25ec, + 0xfb0: 0x2608, 0xfb1: 0x2608, 0xfb2: 0x2601, 0xfb3: 0x2601, 0xfb4: 0x260f, 0xfb5: 0x260f, + 0xfb6: 0x2616, 0xfb7: 0x2616, 0xfb8: 0x2616, 0xfb9: 0x25f3, 0xfba: 0x25f3, 0xfbb: 0x25f3, + 0xfbc: 0x0302, 0xfbd: 0x0302, 0xfbe: 0x0302, 0xfbf: 0x0302, + // Block 0x3f, offset 0xfc0 + 0xfc0: 0x25b4, 0xfc1: 0x25bb, 0xfc2: 0x25d7, 0xfc3: 0x25f3, 0xfc4: 0x25fa, 0xfc5: 0x1d8c, + 0xfc6: 0x1d91, 0xfc7: 0x1d96, 0xfc8: 0x1da5, 0xfc9: 0x1db4, 0xfca: 0x1db9, 0xfcb: 0x1dbe, + 0xfcc: 0x1dc3, 0xfcd: 0x1dc8, 0xfce: 0x1dd7, 0xfcf: 0x1de6, 0xfd0: 0x1deb, 0xfd1: 0x1df0, + 0xfd2: 0x1dff, 0xfd3: 0x1e0e, 0xfd4: 0x1e13, 0xfd5: 0x1e18, 0xfd6: 0x1e1d, 0xfd7: 0x1e2c, + 0xfd8: 0x1e31, 0xfd9: 0x1e40, 0xfda: 0x1e45, 0xfdb: 0x1e4a, 0xfdc: 0x1e59, 0xfdd: 0x1e5e, + 0xfde: 0x1e63, 0xfdf: 0x1e6d, 0xfe0: 0x1ea9, 0xfe1: 0x1eb8, 0xfe2: 0x1ec7, 0xfe3: 0x1ecc, + 0xfe4: 0x1ed1, 0xfe5: 0x1edb, 0xfe6: 0x1eea, 0xfe7: 0x1eef, 0xfe8: 0x1efe, 0xfe9: 0x1f03, + 0xfea: 0x1f08, 0xfeb: 0x1f17, 0xfec: 0x1f1c, 0xfed: 0x1f2b, 0xfee: 0x1f30, 0xfef: 0x1f35, + 0xff0: 0x1f3a, 0xff1: 0x1f3f, 0xff2: 0x1f44, 0xff3: 0x1f49, 0xff4: 0x1f4e, 0xff5: 0x1f53, + 0xff6: 0x1f58, 0xff7: 0x1f5d, 0xff8: 0x1f62, 0xff9: 0x1f67, 0xffa: 0x1f6c, 0xffb: 0x1f71, + 0xffc: 0x1f76, 0xffd: 0x1f7b, 0xffe: 0x1f80, 0xfff: 0x1f8a, + // Block 0x40, offset 0x1000 + 0x1000: 0x1f8f, 0x1001: 0x1f94, 0x1002: 0x1f99, 0x1003: 0x1fa3, 0x1004: 0x1fa8, 0x1005: 0x1fb2, + 0x1006: 0x1fb7, 0x1007: 0x1fbc, 0x1008: 0x1fc1, 0x1009: 0x1fc6, 0x100a: 0x1fcb, 0x100b: 0x1fd0, + 0x100c: 0x1fd5, 0x100d: 0x1fda, 0x100e: 0x1fe9, 0x100f: 0x1ff8, 0x1010: 0x1ffd, 0x1011: 0x2002, + 0x1012: 0x2007, 0x1013: 0x200c, 0x1014: 0x2011, 0x1015: 0x201b, 0x1016: 0x2020, 0x1017: 0x2025, + 0x1018: 0x2034, 0x1019: 0x2043, 0x101a: 0x2048, 0x101b: 0x4423, 0x101c: 0x4429, 0x101d: 0x445f, + 0x101e: 0x44b6, 0x101f: 0x44bd, 0x1020: 0x44c4, 0x1021: 0x44cb, 0x1022: 0x44d2, 0x1023: 0x44d9, + 0x1024: 0x25c9, 0x1025: 0x25d0, 0x1026: 0x25d7, 0x1027: 0x25de, 0x1028: 0x25f3, 0x1029: 0x25fa, + 0x102a: 0x1d9b, 0x102b: 0x1da0, 0x102c: 0x1da5, 0x102d: 0x1daa, 0x102e: 0x1db4, 0x102f: 0x1db9, + 0x1030: 0x1dcd, 0x1031: 0x1dd2, 0x1032: 0x1dd7, 0x1033: 0x1ddc, 0x1034: 0x1de6, 0x1035: 0x1deb, + 0x1036: 0x1df5, 0x1037: 0x1dfa, 0x1038: 0x1dff, 0x1039: 0x1e04, 0x103a: 0x1e0e, 0x103b: 0x1e13, + 0x103c: 0x1f3f, 0x103d: 0x1f44, 0x103e: 0x1f53, 0x103f: 0x1f58, + // Block 0x41, offset 0x1040 + 0x1040: 0x1f5d, 0x1041: 0x1f71, 0x1042: 0x1f76, 0x1043: 0x1f7b, 0x1044: 0x1f80, 0x1045: 0x1f99, + 0x1046: 0x1fa3, 0x1047: 0x1fa8, 0x1048: 0x1fad, 0x1049: 0x1fc1, 0x104a: 0x1fdf, 0x104b: 0x1fe4, + 0x104c: 0x1fe9, 0x104d: 0x1fee, 0x104e: 0x1ff8, 0x104f: 0x1ffd, 0x1050: 0x445f, 0x1051: 0x202a, + 0x1052: 0x202f, 0x1053: 0x2034, 0x1054: 0x2039, 0x1055: 0x2043, 0x1056: 0x2048, 0x1057: 0x25b4, + 0x1058: 0x25bb, 0x1059: 0x25c2, 0x105a: 0x25d7, 0x105b: 0x25e5, 0x105c: 0x1d8c, 0x105d: 0x1d91, + 0x105e: 0x1d96, 0x105f: 0x1da5, 0x1060: 0x1daf, 0x1061: 0x1dbe, 0x1062: 0x1dc3, 0x1063: 0x1dc8, + 0x1064: 0x1dd7, 0x1065: 0x1de1, 0x1066: 0x1dff, 0x1067: 0x1e18, 0x1068: 0x1e1d, 0x1069: 0x1e2c, + 0x106a: 0x1e31, 0x106b: 0x1e40, 0x106c: 0x1e4a, 0x106d: 0x1e59, 0x106e: 0x1e5e, 0x106f: 0x1e63, + 0x1070: 0x1e6d, 0x1071: 0x1ea9, 0x1072: 0x1eae, 0x1073: 0x1eb8, 0x1074: 0x1ec7, 0x1075: 0x1ecc, + 0x1076: 0x1ed1, 0x1077: 0x1edb, 0x1078: 0x1eea, 0x1079: 0x1efe, 0x107a: 0x1f03, 0x107b: 0x1f08, + 0x107c: 0x1f17, 0x107d: 0x1f1c, 0x107e: 0x1f2b, 0x107f: 0x1f30, + // Block 0x42, offset 0x1080 + 0x1080: 0x1f35, 0x1081: 0x1f3a, 0x1082: 0x1f49, 0x1083: 0x1f4e, 0x1084: 0x1f62, 0x1085: 0x1f67, + 0x1086: 0x1f6c, 0x1087: 0x1f71, 0x1088: 0x1f76, 0x1089: 0x1f8a, 0x108a: 0x1f8f, 0x108b: 0x1f94, + 0x108c: 0x1f99, 0x108d: 0x1f9e, 0x108e: 0x1fb2, 0x108f: 0x1fb7, 0x1090: 0x1fbc, 0x1091: 0x1fc1, + 0x1092: 0x1fd0, 0x1093: 0x1fd5, 0x1094: 0x1fda, 0x1095: 0x1fe9, 0x1096: 0x1ff3, 0x1097: 0x2002, + 0x1098: 0x2007, 0x1099: 0x4453, 0x109a: 0x201b, 0x109b: 0x2020, 0x109c: 0x2025, 0x109d: 0x2034, + 0x109e: 0x203e, 0x109f: 0x25d7, 0x10a0: 0x25e5, 0x10a1: 0x1da5, 0x10a2: 0x1daf, 0x10a3: 0x1dd7, + 0x10a4: 0x1de1, 0x10a5: 0x1dff, 0x10a6: 0x1e09, 0x10a7: 0x1e6d, 0x10a8: 0x1e72, 0x10a9: 0x1e95, + 0x10aa: 0x1e9a, 0x10ab: 0x1f71, 0x10ac: 0x1f76, 0x10ad: 0x1f99, 0x10ae: 0x1fe9, 0x10af: 0x1ff3, + 0x10b0: 0x2034, 0x10b1: 0x203e, 0x10b2: 0x4507, 0x10b3: 0x450f, 0x10b4: 0x4517, 0x10b5: 0x1ef4, + 0x10b6: 0x1ef9, 0x10b7: 0x1f0d, 0x10b8: 0x1f12, 0x10b9: 0x1f21, 0x10ba: 0x1f26, 0x10bb: 0x1e77, + 0x10bc: 0x1e7c, 0x10bd: 0x1e9f, 0x10be: 0x1ea4, 0x10bf: 0x1e36, + // Block 0x43, offset 0x10c0 + 0x10c0: 0x1e3b, 0x10c1: 0x1e22, 0x10c2: 0x1e27, 0x10c3: 0x1e4f, 0x10c4: 0x1e54, 0x10c5: 0x1ebd, + 0x10c6: 0x1ec2, 0x10c7: 0x1ee0, 0x10c8: 0x1ee5, 0x10c9: 0x1e81, 0x10ca: 0x1e86, 0x10cb: 0x1e8b, + 0x10cc: 0x1e95, 0x10cd: 0x1e90, 0x10ce: 0x1e68, 0x10cf: 0x1eb3, 0x10d0: 0x1ed6, 0x10d1: 0x1ef4, + 0x10d2: 0x1ef9, 0x10d3: 0x1f0d, 0x10d4: 0x1f12, 0x10d5: 0x1f21, 0x10d6: 0x1f26, 0x10d7: 0x1e77, + 0x10d8: 0x1e7c, 0x10d9: 0x1e9f, 0x10da: 0x1ea4, 0x10db: 0x1e36, 0x10dc: 0x1e3b, 0x10dd: 0x1e22, + 0x10de: 0x1e27, 0x10df: 0x1e4f, 0x10e0: 0x1e54, 0x10e1: 0x1ebd, 0x10e2: 0x1ec2, 0x10e3: 0x1ee0, + 0x10e4: 0x1ee5, 0x10e5: 0x1e81, 0x10e6: 0x1e86, 0x10e7: 0x1e8b, 0x10e8: 0x1e95, 0x10e9: 0x1e90, + 0x10ea: 0x1e68, 0x10eb: 0x1eb3, 0x10ec: 0x1ed6, 0x10ed: 0x1e81, 0x10ee: 0x1e86, 0x10ef: 0x1e8b, + 0x10f0: 0x1e95, 0x10f1: 0x1e72, 0x10f2: 0x1e9a, 0x10f3: 0x1eef, 0x10f4: 0x1e59, 0x10f5: 0x1e5e, + 0x10f6: 0x1e63, 0x10f7: 0x1e81, 0x10f8: 0x1e86, 0x10f9: 0x1e8b, 0x10fa: 0x1eef, 0x10fb: 0x1efe, + 0x10fc: 0x440b, 0x10fd: 0x440b, + // Block 0x44, offset 0x1100 + 0x1110: 0x2314, 0x1111: 0x2329, + 0x1112: 0x2329, 0x1113: 0x2330, 0x1114: 0x2337, 0x1115: 0x234c, 0x1116: 0x2353, 0x1117: 0x235a, + 0x1118: 0x237d, 0x1119: 0x237d, 0x111a: 0x23a0, 0x111b: 0x2399, 0x111c: 0x23b5, 0x111d: 0x23a7, + 0x111e: 0x23ae, 0x111f: 0x23d1, 0x1120: 0x23d1, 0x1121: 0x23ca, 0x1122: 0x23d8, 0x1123: 0x23d8, + 0x1124: 0x2402, 0x1125: 0x2402, 0x1126: 0x241e, 0x1127: 0x23e6, 0x1128: 0x23e6, 0x1129: 0x23df, + 0x112a: 0x23f4, 0x112b: 0x23f4, 0x112c: 0x23fb, 0x112d: 0x23fb, 0x112e: 0x2425, 0x112f: 0x2433, + 0x1130: 0x2433, 0x1131: 0x243a, 0x1132: 0x243a, 0x1133: 0x2441, 0x1134: 0x2448, 0x1135: 0x244f, + 0x1136: 0x2456, 0x1137: 0x2456, 0x1138: 0x245d, 0x1139: 0x246b, 0x113a: 0x2479, 0x113b: 0x2472, + 0x113c: 0x2480, 0x113d: 0x2480, 0x113e: 0x2495, 0x113f: 0x249c, + // Block 0x45, offset 0x1140 + 0x1140: 0x24cd, 0x1141: 0x24db, 0x1142: 0x24d4, 0x1143: 0x24b8, 0x1144: 0x24b8, 0x1145: 0x24e2, + 0x1146: 0x24e2, 0x1147: 0x24e9, 0x1148: 0x24e9, 0x1149: 0x2513, 0x114a: 0x251a, 0x114b: 0x2521, + 0x114c: 0x24f7, 0x114d: 0x2505, 0x114e: 0x2528, 0x114f: 0x252f, + 0x1152: 0x24fe, 0x1153: 0x2583, 0x1154: 0x258a, 0x1155: 0x2560, 0x1156: 0x2567, 0x1157: 0x254b, + 0x1158: 0x254b, 0x1159: 0x2552, 0x115a: 0x257c, 0x115b: 0x2575, 0x115c: 0x259f, 0x115d: 0x259f, + 0x115e: 0x230d, 0x115f: 0x2322, 0x1160: 0x231b, 0x1161: 0x2345, 0x1162: 0x233e, 0x1163: 0x2368, + 0x1164: 0x2361, 0x1165: 0x238b, 0x1166: 0x236f, 0x1167: 0x2384, 0x1168: 0x23bc, 0x1169: 0x2409, + 0x116a: 0x23ed, 0x116b: 0x242c, 0x116c: 0x24c6, 0x116d: 0x24f0, 0x116e: 0x2598, 0x116f: 0x2591, + 0x1170: 0x25a6, 0x1171: 0x253d, 0x1172: 0x24a3, 0x1173: 0x256e, 0x1174: 0x2495, 0x1175: 0x24cd, + 0x1176: 0x2464, 0x1177: 0x24b1, 0x1178: 0x2544, 0x1179: 0x2536, 0x117a: 0x24bf, 0x117b: 0x24aa, + 0x117c: 0x24bf, 0x117d: 0x2544, 0x117e: 0x2376, 0x117f: 0x2392, + // Block 0x46, offset 0x1180 + 0x1180: 0x250c, 0x1181: 0x2487, 0x1182: 0x2306, 0x1183: 0x24aa, 0x1184: 0x244f, 0x1185: 0x241e, + 0x1186: 0x23c3, 0x1187: 0x2559, + 0x11b0: 0x2417, 0x11b1: 0x248e, 0x11b2: 0x27c2, 0x11b3: 0x27b9, 0x11b4: 0x27ef, 0x11b5: 0x27dd, + 0x11b6: 0x27cb, 0x11b7: 0x27e6, 0x11b8: 0x27f8, 0x11b9: 0x2410, 0x11ba: 0x2c7f, 0x11bb: 0x2aff, + 0x11bc: 0x27d4, + // Block 0x47, offset 0x11c0 + 0x11d0: 0x0019, 0x11d1: 0x0483, + 0x11d2: 0x0487, 0x11d3: 0x0035, 0x11d4: 0x0037, 0x11d5: 0x0003, 0x11d6: 0x003f, 0x11d7: 0x04bf, + 0x11d8: 0x04c3, 0x11d9: 0x1b5f, + 0x11e0: 0x8132, 0x11e1: 0x8132, 0x11e2: 0x8132, 0x11e3: 0x8132, + 0x11e4: 0x8132, 0x11e5: 0x8132, 0x11e6: 0x8132, 0x11e7: 0x812d, 0x11e8: 0x812d, 0x11e9: 0x812d, + 0x11ea: 0x812d, 0x11eb: 0x812d, 0x11ec: 0x812d, 0x11ed: 0x812d, 0x11ee: 0x8132, 0x11ef: 0x8132, + 0x11f0: 0x1873, 0x11f1: 0x0443, 0x11f2: 0x043f, 0x11f3: 0x007f, 0x11f4: 0x007f, 0x11f5: 0x0011, + 0x11f6: 0x0013, 0x11f7: 0x00b7, 0x11f8: 0x00bb, 0x11f9: 0x04b7, 0x11fa: 0x04bb, 0x11fb: 0x04ab, + 0x11fc: 0x04af, 0x11fd: 0x0493, 0x11fe: 0x0497, 0x11ff: 0x048b, + // Block 0x48, offset 0x1200 + 0x1200: 0x048f, 0x1201: 0x049b, 0x1202: 0x049f, 0x1203: 0x04a3, 0x1204: 0x04a7, + 0x1207: 0x0077, 0x1208: 0x007b, 0x1209: 0x426c, 0x120a: 0x426c, 0x120b: 0x426c, + 0x120c: 0x426c, 0x120d: 0x007f, 0x120e: 0x007f, 0x120f: 0x007f, 0x1210: 0x0019, 0x1211: 0x0483, + 0x1212: 0x001d, 0x1214: 0x0037, 0x1215: 0x0035, 0x1216: 0x003f, 0x1217: 0x0003, + 0x1218: 0x0443, 0x1219: 0x0011, 0x121a: 0x0013, 0x121b: 0x00b7, 0x121c: 0x00bb, 0x121d: 0x04b7, + 0x121e: 0x04bb, 0x121f: 0x0007, 0x1220: 0x000d, 0x1221: 0x0015, 0x1222: 0x0017, 0x1223: 0x001b, + 0x1224: 0x0039, 0x1225: 0x003d, 0x1226: 0x003b, 0x1228: 0x0079, 0x1229: 0x0009, + 0x122a: 0x000b, 0x122b: 0x0041, + 0x1230: 0x42ad, 0x1231: 0x442f, 0x1232: 0x42b2, 0x1234: 0x42b7, + 0x1236: 0x42bc, 0x1237: 0x4435, 0x1238: 0x42c1, 0x1239: 0x443b, 0x123a: 0x42c6, 0x123b: 0x4441, + 0x123c: 0x42cb, 0x123d: 0x4447, 0x123e: 0x42d0, 0x123f: 0x444d, + // Block 0x49, offset 0x1240 + 0x1240: 0x0236, 0x1241: 0x4411, 0x1242: 0x4411, 0x1243: 0x4417, 0x1244: 0x4417, 0x1245: 0x4459, + 0x1246: 0x4459, 0x1247: 0x441d, 0x1248: 0x441d, 0x1249: 0x4465, 0x124a: 0x4465, 0x124b: 0x4465, + 0x124c: 0x4465, 0x124d: 0x0239, 0x124e: 0x0239, 0x124f: 0x023c, 0x1250: 0x023c, 0x1251: 0x023c, + 0x1252: 0x023c, 0x1253: 0x023f, 0x1254: 0x023f, 0x1255: 0x0242, 0x1256: 0x0242, 0x1257: 0x0242, + 0x1258: 0x0242, 0x1259: 0x0245, 0x125a: 0x0245, 0x125b: 0x0245, 0x125c: 0x0245, 0x125d: 0x0248, + 0x125e: 0x0248, 0x125f: 0x0248, 0x1260: 0x0248, 0x1261: 0x024b, 0x1262: 0x024b, 0x1263: 0x024b, + 0x1264: 0x024b, 0x1265: 0x024e, 0x1266: 0x024e, 0x1267: 0x024e, 0x1268: 0x024e, 0x1269: 0x0251, + 0x126a: 0x0251, 0x126b: 0x0254, 0x126c: 0x0254, 0x126d: 0x0257, 0x126e: 0x0257, 0x126f: 0x025a, + 0x1270: 0x025a, 0x1271: 0x025d, 0x1272: 0x025d, 0x1273: 0x025d, 0x1274: 0x025d, 0x1275: 0x0260, + 0x1276: 0x0260, 0x1277: 0x0260, 0x1278: 0x0260, 0x1279: 0x0263, 0x127a: 0x0263, 0x127b: 0x0263, + 0x127c: 0x0263, 0x127d: 0x0266, 0x127e: 0x0266, 0x127f: 0x0266, + // Block 0x4a, offset 0x1280 + 0x1280: 0x0266, 0x1281: 0x0269, 0x1282: 0x0269, 0x1283: 0x0269, 0x1284: 0x0269, 0x1285: 0x026c, + 0x1286: 0x026c, 0x1287: 0x026c, 0x1288: 0x026c, 0x1289: 0x026f, 0x128a: 0x026f, 0x128b: 0x026f, + 0x128c: 0x026f, 0x128d: 0x0272, 0x128e: 0x0272, 0x128f: 0x0272, 0x1290: 0x0272, 0x1291: 0x0275, + 0x1292: 0x0275, 0x1293: 0x0275, 0x1294: 0x0275, 0x1295: 0x0278, 0x1296: 0x0278, 0x1297: 0x0278, + 0x1298: 0x0278, 0x1299: 0x027b, 0x129a: 0x027b, 0x129b: 0x027b, 0x129c: 0x027b, 0x129d: 0x027e, + 0x129e: 0x027e, 0x129f: 0x027e, 0x12a0: 0x027e, 0x12a1: 0x0281, 0x12a2: 0x0281, 0x12a3: 0x0281, + 0x12a4: 0x0281, 0x12a5: 0x0284, 0x12a6: 0x0284, 0x12a7: 0x0284, 0x12a8: 0x0284, 0x12a9: 0x0287, + 0x12aa: 0x0287, 0x12ab: 0x0287, 0x12ac: 0x0287, 0x12ad: 0x028a, 0x12ae: 0x028a, 0x12af: 0x028d, + 0x12b0: 0x028d, 0x12b1: 0x0290, 0x12b2: 0x0290, 0x12b3: 0x0290, 0x12b4: 0x0290, 0x12b5: 0x2e03, + 0x12b6: 0x2e03, 0x12b7: 0x2e0b, 0x12b8: 0x2e0b, 0x12b9: 0x2e13, 0x12ba: 0x2e13, 0x12bb: 0x1f85, + 0x12bc: 0x1f85, + // Block 0x4b, offset 0x12c0 + 0x12c0: 0x0081, 0x12c1: 0x0083, 0x12c2: 0x0085, 0x12c3: 0x0087, 0x12c4: 0x0089, 0x12c5: 0x008b, + 0x12c6: 0x008d, 0x12c7: 0x008f, 0x12c8: 0x0091, 0x12c9: 0x0093, 0x12ca: 0x0095, 0x12cb: 0x0097, + 0x12cc: 0x0099, 0x12cd: 0x009b, 0x12ce: 0x009d, 0x12cf: 0x009f, 0x12d0: 0x00a1, 0x12d1: 0x00a3, + 0x12d2: 0x00a5, 0x12d3: 0x00a7, 0x12d4: 0x00a9, 0x12d5: 0x00ab, 0x12d6: 0x00ad, 0x12d7: 0x00af, + 0x12d8: 0x00b1, 0x12d9: 0x00b3, 0x12da: 0x00b5, 0x12db: 0x00b7, 0x12dc: 0x00b9, 0x12dd: 0x00bb, + 0x12de: 0x00bd, 0x12df: 0x0477, 0x12e0: 0x047b, 0x12e1: 0x0487, 0x12e2: 0x049b, 0x12e3: 0x049f, + 0x12e4: 0x0483, 0x12e5: 0x05ab, 0x12e6: 0x05a3, 0x12e7: 0x04c7, 0x12e8: 0x04cf, 0x12e9: 0x04d7, + 0x12ea: 0x04df, 0x12eb: 0x04e7, 0x12ec: 0x056b, 0x12ed: 0x0573, 0x12ee: 0x057b, 0x12ef: 0x051f, + 0x12f0: 0x05af, 0x12f1: 0x04cb, 0x12f2: 0x04d3, 0x12f3: 0x04db, 0x12f4: 0x04e3, 0x12f5: 0x04eb, + 0x12f6: 0x04ef, 0x12f7: 0x04f3, 0x12f8: 0x04f7, 0x12f9: 0x04fb, 0x12fa: 0x04ff, 0x12fb: 0x0503, + 0x12fc: 0x0507, 0x12fd: 0x050b, 0x12fe: 0x050f, 0x12ff: 0x0513, + // Block 0x4c, offset 0x1300 + 0x1300: 0x0517, 0x1301: 0x051b, 0x1302: 0x0523, 0x1303: 0x0527, 0x1304: 0x052b, 0x1305: 0x052f, + 0x1306: 0x0533, 0x1307: 0x0537, 0x1308: 0x053b, 0x1309: 0x053f, 0x130a: 0x0543, 0x130b: 0x0547, + 0x130c: 0x054b, 0x130d: 0x054f, 0x130e: 0x0553, 0x130f: 0x0557, 0x1310: 0x055b, 0x1311: 0x055f, + 0x1312: 0x0563, 0x1313: 0x0567, 0x1314: 0x056f, 0x1315: 0x0577, 0x1316: 0x057f, 0x1317: 0x0583, + 0x1318: 0x0587, 0x1319: 0x058b, 0x131a: 0x058f, 0x131b: 0x0593, 0x131c: 0x0597, 0x131d: 0x05a7, + 0x131e: 0x4a7b, 0x131f: 0x4a81, 0x1320: 0x03c3, 0x1321: 0x0313, 0x1322: 0x0317, 0x1323: 0x4a3e, + 0x1324: 0x031b, 0x1325: 0x4a44, 0x1326: 0x4a4a, 0x1327: 0x031f, 0x1328: 0x0323, 0x1329: 0x0327, + 0x132a: 0x4a50, 0x132b: 0x4a56, 0x132c: 0x4a5c, 0x132d: 0x4a62, 0x132e: 0x4a68, 0x132f: 0x4a6e, + 0x1330: 0x0367, 0x1331: 0x032b, 0x1332: 0x032f, 0x1333: 0x0333, 0x1334: 0x037b, 0x1335: 0x0337, + 0x1336: 0x033b, 0x1337: 0x033f, 0x1338: 0x0343, 0x1339: 0x0347, 0x133a: 0x034b, 0x133b: 0x034f, + 0x133c: 0x0353, 0x133d: 0x0357, 0x133e: 0x035b, + // Block 0x4d, offset 0x1340 + 0x1342: 0x49c0, 0x1343: 0x49c6, 0x1344: 0x49cc, 0x1345: 0x49d2, + 0x1346: 0x49d8, 0x1347: 0x49de, 0x134a: 0x49e4, 0x134b: 0x49ea, + 0x134c: 0x49f0, 0x134d: 0x49f6, 0x134e: 0x49fc, 0x134f: 0x4a02, + 0x1352: 0x4a08, 0x1353: 0x4a0e, 0x1354: 0x4a14, 0x1355: 0x4a1a, 0x1356: 0x4a20, 0x1357: 0x4a26, + 0x135a: 0x4a2c, 0x135b: 0x4a32, 0x135c: 0x4a38, + 0x1360: 0x00bf, 0x1361: 0x00c2, 0x1362: 0x00cb, 0x1363: 0x4267, + 0x1364: 0x00c8, 0x1365: 0x00c5, 0x1366: 0x0447, 0x1368: 0x046b, 0x1369: 0x044b, + 0x136a: 0x044f, 0x136b: 0x0453, 0x136c: 0x0457, 0x136d: 0x046f, 0x136e: 0x0473, + // Block 0x4e, offset 0x1380 + 0x1380: 0x0063, 0x1381: 0x0065, 0x1382: 0x0067, 0x1383: 0x0069, 0x1384: 0x006b, 0x1385: 0x006d, + 0x1386: 0x006f, 0x1387: 0x0071, 0x1388: 0x0073, 0x1389: 0x0075, 0x138a: 0x0083, 0x138b: 0x0085, + 0x138c: 0x0087, 0x138d: 0x0089, 0x138e: 0x008b, 0x138f: 0x008d, 0x1390: 0x008f, 0x1391: 0x0091, + 0x1392: 0x0093, 0x1393: 0x0095, 0x1394: 0x0097, 0x1395: 0x0099, 0x1396: 0x009b, 0x1397: 0x009d, + 0x1398: 0x009f, 0x1399: 0x00a1, 0x139a: 0x00a3, 0x139b: 0x00a5, 0x139c: 0x00a7, 0x139d: 0x00a9, + 0x139e: 0x00ab, 0x139f: 0x00ad, 0x13a0: 0x00af, 0x13a1: 0x00b1, 0x13a2: 0x00b3, 0x13a3: 0x00b5, + 0x13a4: 0x00dd, 0x13a5: 0x00f2, 0x13a8: 0x0173, 0x13a9: 0x0176, + 0x13aa: 0x0179, 0x13ab: 0x017c, 0x13ac: 0x017f, 0x13ad: 0x0182, 0x13ae: 0x0185, 0x13af: 0x0188, + 0x13b0: 0x018b, 0x13b1: 0x018e, 0x13b2: 0x0191, 0x13b3: 0x0194, 0x13b4: 0x0197, 0x13b5: 0x019a, + 0x13b6: 0x019d, 0x13b7: 0x01a0, 0x13b8: 0x01a3, 0x13b9: 0x0188, 0x13ba: 0x01a6, 0x13bb: 0x01a9, + 0x13bc: 0x01ac, 0x13bd: 0x01af, 0x13be: 0x01b2, 0x13bf: 0x01b5, + // Block 0x4f, offset 0x13c0 + 0x13c0: 0x01fd, 0x13c1: 0x0200, 0x13c2: 0x0203, 0x13c3: 0x045b, 0x13c4: 0x01c7, 0x13c5: 0x01d0, + 0x13c6: 0x01d6, 0x13c7: 0x01fa, 0x13c8: 0x01eb, 0x13c9: 0x01e8, 0x13ca: 0x0206, 0x13cb: 0x0209, + 0x13ce: 0x0021, 0x13cf: 0x0023, 0x13d0: 0x0025, 0x13d1: 0x0027, + 0x13d2: 0x0029, 0x13d3: 0x002b, 0x13d4: 0x002d, 0x13d5: 0x002f, 0x13d6: 0x0031, 0x13d7: 0x0033, + 0x13d8: 0x0021, 0x13d9: 0x0023, 0x13da: 0x0025, 0x13db: 0x0027, 0x13dc: 0x0029, 0x13dd: 0x002b, + 0x13de: 0x002d, 0x13df: 0x002f, 0x13e0: 0x0031, 0x13e1: 0x0033, 0x13e2: 0x0021, 0x13e3: 0x0023, + 0x13e4: 0x0025, 0x13e5: 0x0027, 0x13e6: 0x0029, 0x13e7: 0x002b, 0x13e8: 0x002d, 0x13e9: 0x002f, + 0x13ea: 0x0031, 0x13eb: 0x0033, 0x13ec: 0x0021, 0x13ed: 0x0023, 0x13ee: 0x0025, 0x13ef: 0x0027, + 0x13f0: 0x0029, 0x13f1: 0x002b, 0x13f2: 0x002d, 0x13f3: 0x002f, 0x13f4: 0x0031, 0x13f5: 0x0033, + 0x13f6: 0x0021, 0x13f7: 0x0023, 0x13f8: 0x0025, 0x13f9: 0x0027, 0x13fa: 0x0029, 0x13fb: 0x002b, + 0x13fc: 0x002d, 0x13fd: 0x002f, 0x13fe: 0x0031, 0x13ff: 0x0033, + // Block 0x50, offset 0x1400 + 0x1400: 0x0239, 0x1401: 0x023c, 0x1402: 0x0248, 0x1403: 0x0251, 0x1405: 0x028a, + 0x1406: 0x025a, 0x1407: 0x024b, 0x1408: 0x0269, 0x1409: 0x0290, 0x140a: 0x027b, 0x140b: 0x027e, + 0x140c: 0x0281, 0x140d: 0x0284, 0x140e: 0x025d, 0x140f: 0x026f, 0x1410: 0x0275, 0x1411: 0x0263, + 0x1412: 0x0278, 0x1413: 0x0257, 0x1414: 0x0260, 0x1415: 0x0242, 0x1416: 0x0245, 0x1417: 0x024e, + 0x1418: 0x0254, 0x1419: 0x0266, 0x141a: 0x026c, 0x141b: 0x0272, 0x141c: 0x0293, 0x141d: 0x02e4, + 0x141e: 0x02cc, 0x141f: 0x0296, 0x1421: 0x023c, 0x1422: 0x0248, + 0x1424: 0x0287, 0x1427: 0x024b, 0x1429: 0x0290, + 0x142a: 0x027b, 0x142b: 0x027e, 0x142c: 0x0281, 0x142d: 0x0284, 0x142e: 0x025d, 0x142f: 0x026f, + 0x1430: 0x0275, 0x1431: 0x0263, 0x1432: 0x0278, 0x1434: 0x0260, 0x1435: 0x0242, + 0x1436: 0x0245, 0x1437: 0x024e, 0x1439: 0x0266, 0x143b: 0x0272, + // Block 0x51, offset 0x1440 + 0x1442: 0x0248, + 0x1447: 0x024b, 0x1449: 0x0290, 0x144b: 0x027e, + 0x144d: 0x0284, 0x144e: 0x025d, 0x144f: 0x026f, 0x1451: 0x0263, + 0x1452: 0x0278, 0x1454: 0x0260, 0x1457: 0x024e, + 0x1459: 0x0266, 0x145b: 0x0272, 0x145d: 0x02e4, + 0x145f: 0x0296, 0x1461: 0x023c, 0x1462: 0x0248, + 0x1464: 0x0287, 0x1467: 0x024b, 0x1468: 0x0269, 0x1469: 0x0290, + 0x146a: 0x027b, 0x146c: 0x0281, 0x146d: 0x0284, 0x146e: 0x025d, 0x146f: 0x026f, + 0x1470: 0x0275, 0x1471: 0x0263, 0x1472: 0x0278, 0x1474: 0x0260, 0x1475: 0x0242, + 0x1476: 0x0245, 0x1477: 0x024e, 0x1479: 0x0266, 0x147a: 0x026c, 0x147b: 0x0272, + 0x147c: 0x0293, 0x147e: 0x02cc, + // Block 0x52, offset 0x1480 + 0x1480: 0x0239, 0x1481: 0x023c, 0x1482: 0x0248, 0x1483: 0x0251, 0x1484: 0x0287, 0x1485: 0x028a, + 0x1486: 0x025a, 0x1487: 0x024b, 0x1488: 0x0269, 0x1489: 0x0290, 0x148b: 0x027e, + 0x148c: 0x0281, 0x148d: 0x0284, 0x148e: 0x025d, 0x148f: 0x026f, 0x1490: 0x0275, 0x1491: 0x0263, + 0x1492: 0x0278, 0x1493: 0x0257, 0x1494: 0x0260, 0x1495: 0x0242, 0x1496: 0x0245, 0x1497: 0x024e, + 0x1498: 0x0254, 0x1499: 0x0266, 0x149a: 0x026c, 0x149b: 0x0272, + 0x14a1: 0x023c, 0x14a2: 0x0248, 0x14a3: 0x0251, + 0x14a5: 0x028a, 0x14a6: 0x025a, 0x14a7: 0x024b, 0x14a8: 0x0269, 0x14a9: 0x0290, + 0x14ab: 0x027e, 0x14ac: 0x0281, 0x14ad: 0x0284, 0x14ae: 0x025d, 0x14af: 0x026f, + 0x14b0: 0x0275, 0x14b1: 0x0263, 0x14b2: 0x0278, 0x14b3: 0x0257, 0x14b4: 0x0260, 0x14b5: 0x0242, + 0x14b6: 0x0245, 0x14b7: 0x024e, 0x14b8: 0x0254, 0x14b9: 0x0266, 0x14ba: 0x026c, 0x14bb: 0x0272, + // Block 0x53, offset 0x14c0 + 0x14c0: 0x1879, 0x14c1: 0x1876, 0x14c2: 0x187c, 0x14c3: 0x18a0, 0x14c4: 0x18c4, 0x14c5: 0x18e8, + 0x14c6: 0x190c, 0x14c7: 0x1915, 0x14c8: 0x191b, 0x14c9: 0x1921, 0x14ca: 0x1927, + 0x14d0: 0x1a8f, 0x14d1: 0x1a93, + 0x14d2: 0x1a97, 0x14d3: 0x1a9b, 0x14d4: 0x1a9f, 0x14d5: 0x1aa3, 0x14d6: 0x1aa7, 0x14d7: 0x1aab, + 0x14d8: 0x1aaf, 0x14d9: 0x1ab3, 0x14da: 0x1ab7, 0x14db: 0x1abb, 0x14dc: 0x1abf, 0x14dd: 0x1ac3, + 0x14de: 0x1ac7, 0x14df: 0x1acb, 0x14e0: 0x1acf, 0x14e1: 0x1ad3, 0x14e2: 0x1ad7, 0x14e3: 0x1adb, + 0x14e4: 0x1adf, 0x14e5: 0x1ae3, 0x14e6: 0x1ae7, 0x14e7: 0x1aeb, 0x14e8: 0x1aef, 0x14e9: 0x1af3, + 0x14ea: 0x2721, 0x14eb: 0x0047, 0x14ec: 0x0065, 0x14ed: 0x193c, 0x14ee: 0x19b4, + 0x14f0: 0x0043, 0x14f1: 0x0045, 0x14f2: 0x0047, 0x14f3: 0x0049, 0x14f4: 0x004b, 0x14f5: 0x004d, + 0x14f6: 0x004f, 0x14f7: 0x0051, 0x14f8: 0x0053, 0x14f9: 0x0055, 0x14fa: 0x0057, 0x14fb: 0x0059, + 0x14fc: 0x005b, 0x14fd: 0x005d, 0x14fe: 0x005f, 0x14ff: 0x0061, + // Block 0x54, offset 0x1500 + 0x1500: 0x26b0, 0x1501: 0x26c5, 0x1502: 0x0503, + 0x1510: 0x0c0f, 0x1511: 0x0a47, + 0x1512: 0x08d3, 0x1513: 0x45c7, 0x1514: 0x071b, 0x1515: 0x09ef, 0x1516: 0x132f, 0x1517: 0x09ff, + 0x1518: 0x0727, 0x1519: 0x0cd7, 0x151a: 0x0eaf, 0x151b: 0x0caf, 0x151c: 0x0827, 0x151d: 0x0b6b, + 0x151e: 0x07bf, 0x151f: 0x0cb7, 0x1520: 0x0813, 0x1521: 0x1117, 0x1522: 0x0f83, 0x1523: 0x138b, + 0x1524: 0x09d3, 0x1525: 0x090b, 0x1526: 0x0e63, 0x1527: 0x0c1b, 0x1528: 0x0c47, 0x1529: 0x06bf, + 0x152a: 0x06cb, 0x152b: 0x140b, 0x152c: 0x0adb, 0x152d: 0x06e7, 0x152e: 0x08ef, 0x152f: 0x0c3b, + 0x1530: 0x13b3, 0x1531: 0x0c13, 0x1532: 0x106f, 0x1533: 0x10ab, 0x1534: 0x08f7, 0x1535: 0x0e43, + 0x1536: 0x0d0b, 0x1537: 0x0d07, 0x1538: 0x0f97, 0x1539: 0x082b, 0x153a: 0x0957, 0x153b: 0x1443, + // Block 0x55, offset 0x1540 + 0x1540: 0x06fb, 0x1541: 0x06f3, 0x1542: 0x0703, 0x1543: 0x1647, 0x1544: 0x0747, 0x1545: 0x0757, + 0x1546: 0x075b, 0x1547: 0x0763, 0x1548: 0x076b, 0x1549: 0x076f, 0x154a: 0x077b, 0x154b: 0x0773, + 0x154c: 0x05b3, 0x154d: 0x165b, 0x154e: 0x078f, 0x154f: 0x0793, 0x1550: 0x0797, 0x1551: 0x07b3, + 0x1552: 0x164c, 0x1553: 0x05b7, 0x1554: 0x079f, 0x1555: 0x07bf, 0x1556: 0x1656, 0x1557: 0x07cf, + 0x1558: 0x07d7, 0x1559: 0x0737, 0x155a: 0x07df, 0x155b: 0x07e3, 0x155c: 0x1831, 0x155d: 0x07ff, + 0x155e: 0x0807, 0x155f: 0x05bf, 0x1560: 0x081f, 0x1561: 0x0823, 0x1562: 0x082b, 0x1563: 0x082f, + 0x1564: 0x05c3, 0x1565: 0x0847, 0x1566: 0x084b, 0x1567: 0x0857, 0x1568: 0x0863, 0x1569: 0x0867, + 0x156a: 0x086b, 0x156b: 0x0873, 0x156c: 0x0893, 0x156d: 0x0897, 0x156e: 0x089f, 0x156f: 0x08af, + 0x1570: 0x08b7, 0x1571: 0x08bb, 0x1572: 0x08bb, 0x1573: 0x08bb, 0x1574: 0x166a, 0x1575: 0x0e93, + 0x1576: 0x08cf, 0x1577: 0x08d7, 0x1578: 0x166f, 0x1579: 0x08e3, 0x157a: 0x08eb, 0x157b: 0x08f3, + 0x157c: 0x091b, 0x157d: 0x0907, 0x157e: 0x0913, 0x157f: 0x0917, + // Block 0x56, offset 0x1580 + 0x1580: 0x091f, 0x1581: 0x0927, 0x1582: 0x092b, 0x1583: 0x0933, 0x1584: 0x093b, 0x1585: 0x093f, + 0x1586: 0x093f, 0x1587: 0x0947, 0x1588: 0x094f, 0x1589: 0x0953, 0x158a: 0x095f, 0x158b: 0x0983, + 0x158c: 0x0967, 0x158d: 0x0987, 0x158e: 0x096b, 0x158f: 0x0973, 0x1590: 0x080b, 0x1591: 0x09cf, + 0x1592: 0x0997, 0x1593: 0x099b, 0x1594: 0x099f, 0x1595: 0x0993, 0x1596: 0x09a7, 0x1597: 0x09a3, + 0x1598: 0x09bb, 0x1599: 0x1674, 0x159a: 0x09d7, 0x159b: 0x09db, 0x159c: 0x09e3, 0x159d: 0x09ef, + 0x159e: 0x09f7, 0x159f: 0x0a13, 0x15a0: 0x1679, 0x15a1: 0x167e, 0x15a2: 0x0a1f, 0x15a3: 0x0a23, + 0x15a4: 0x0a27, 0x15a5: 0x0a1b, 0x15a6: 0x0a2f, 0x15a7: 0x05c7, 0x15a8: 0x05cb, 0x15a9: 0x0a37, + 0x15aa: 0x0a3f, 0x15ab: 0x0a3f, 0x15ac: 0x1683, 0x15ad: 0x0a5b, 0x15ae: 0x0a5f, 0x15af: 0x0a63, + 0x15b0: 0x0a6b, 0x15b1: 0x1688, 0x15b2: 0x0a73, 0x15b3: 0x0a77, 0x15b4: 0x0b4f, 0x15b5: 0x0a7f, + 0x15b6: 0x05cf, 0x15b7: 0x0a8b, 0x15b8: 0x0a9b, 0x15b9: 0x0aa7, 0x15ba: 0x0aa3, 0x15bb: 0x1692, + 0x15bc: 0x0aaf, 0x15bd: 0x1697, 0x15be: 0x0abb, 0x15bf: 0x0ab7, + // Block 0x57, offset 0x15c0 + 0x15c0: 0x0abf, 0x15c1: 0x0acf, 0x15c2: 0x0ad3, 0x15c3: 0x05d3, 0x15c4: 0x0ae3, 0x15c5: 0x0aeb, + 0x15c6: 0x0aef, 0x15c7: 0x0af3, 0x15c8: 0x05d7, 0x15c9: 0x169c, 0x15ca: 0x05db, 0x15cb: 0x0b0f, + 0x15cc: 0x0b13, 0x15cd: 0x0b17, 0x15ce: 0x0b1f, 0x15cf: 0x1863, 0x15d0: 0x0b37, 0x15d1: 0x16a6, + 0x15d2: 0x16a6, 0x15d3: 0x11d7, 0x15d4: 0x0b47, 0x15d5: 0x0b47, 0x15d6: 0x05df, 0x15d7: 0x16c9, + 0x15d8: 0x179b, 0x15d9: 0x0b57, 0x15da: 0x0b5f, 0x15db: 0x05e3, 0x15dc: 0x0b73, 0x15dd: 0x0b83, + 0x15de: 0x0b87, 0x15df: 0x0b8f, 0x15e0: 0x0b9f, 0x15e1: 0x05eb, 0x15e2: 0x05e7, 0x15e3: 0x0ba3, + 0x15e4: 0x16ab, 0x15e5: 0x0ba7, 0x15e6: 0x0bbb, 0x15e7: 0x0bbf, 0x15e8: 0x0bc3, 0x15e9: 0x0bbf, + 0x15ea: 0x0bcf, 0x15eb: 0x0bd3, 0x15ec: 0x0be3, 0x15ed: 0x0bdb, 0x15ee: 0x0bdf, 0x15ef: 0x0be7, + 0x15f0: 0x0beb, 0x15f1: 0x0bef, 0x15f2: 0x0bfb, 0x15f3: 0x0bff, 0x15f4: 0x0c17, 0x15f5: 0x0c1f, + 0x15f6: 0x0c2f, 0x15f7: 0x0c43, 0x15f8: 0x16ba, 0x15f9: 0x0c3f, 0x15fa: 0x0c33, 0x15fb: 0x0c4b, + 0x15fc: 0x0c53, 0x15fd: 0x0c67, 0x15fe: 0x16bf, 0x15ff: 0x0c6f, + // Block 0x58, offset 0x1600 + 0x1600: 0x0c63, 0x1601: 0x0c5b, 0x1602: 0x05ef, 0x1603: 0x0c77, 0x1604: 0x0c7f, 0x1605: 0x0c87, + 0x1606: 0x0c7b, 0x1607: 0x05f3, 0x1608: 0x0c97, 0x1609: 0x0c9f, 0x160a: 0x16c4, 0x160b: 0x0ccb, + 0x160c: 0x0cff, 0x160d: 0x0cdb, 0x160e: 0x05ff, 0x160f: 0x0ce7, 0x1610: 0x05fb, 0x1611: 0x05f7, + 0x1612: 0x07c3, 0x1613: 0x07c7, 0x1614: 0x0d03, 0x1615: 0x0ceb, 0x1616: 0x11ab, 0x1617: 0x0663, + 0x1618: 0x0d0f, 0x1619: 0x0d13, 0x161a: 0x0d17, 0x161b: 0x0d2b, 0x161c: 0x0d23, 0x161d: 0x16dd, + 0x161e: 0x0603, 0x161f: 0x0d3f, 0x1620: 0x0d33, 0x1621: 0x0d4f, 0x1622: 0x0d57, 0x1623: 0x16e7, + 0x1624: 0x0d5b, 0x1625: 0x0d47, 0x1626: 0x0d63, 0x1627: 0x0607, 0x1628: 0x0d67, 0x1629: 0x0d6b, + 0x162a: 0x0d6f, 0x162b: 0x0d7b, 0x162c: 0x16ec, 0x162d: 0x0d83, 0x162e: 0x060b, 0x162f: 0x0d8f, + 0x1630: 0x16f1, 0x1631: 0x0d93, 0x1632: 0x060f, 0x1633: 0x0d9f, 0x1634: 0x0dab, 0x1635: 0x0db7, + 0x1636: 0x0dbb, 0x1637: 0x16f6, 0x1638: 0x168d, 0x1639: 0x16fb, 0x163a: 0x0ddb, 0x163b: 0x1700, + 0x163c: 0x0de7, 0x163d: 0x0def, 0x163e: 0x0ddf, 0x163f: 0x0dfb, + // Block 0x59, offset 0x1640 + 0x1640: 0x0e0b, 0x1641: 0x0e1b, 0x1642: 0x0e0f, 0x1643: 0x0e13, 0x1644: 0x0e1f, 0x1645: 0x0e23, + 0x1646: 0x1705, 0x1647: 0x0e07, 0x1648: 0x0e3b, 0x1649: 0x0e3f, 0x164a: 0x0613, 0x164b: 0x0e53, + 0x164c: 0x0e4f, 0x164d: 0x170a, 0x164e: 0x0e33, 0x164f: 0x0e6f, 0x1650: 0x170f, 0x1651: 0x1714, + 0x1652: 0x0e73, 0x1653: 0x0e87, 0x1654: 0x0e83, 0x1655: 0x0e7f, 0x1656: 0x0617, 0x1657: 0x0e8b, + 0x1658: 0x0e9b, 0x1659: 0x0e97, 0x165a: 0x0ea3, 0x165b: 0x1651, 0x165c: 0x0eb3, 0x165d: 0x1719, + 0x165e: 0x0ebf, 0x165f: 0x1723, 0x1660: 0x0ed3, 0x1661: 0x0edf, 0x1662: 0x0ef3, 0x1663: 0x1728, + 0x1664: 0x0f07, 0x1665: 0x0f0b, 0x1666: 0x172d, 0x1667: 0x1732, 0x1668: 0x0f27, 0x1669: 0x0f37, + 0x166a: 0x061b, 0x166b: 0x0f3b, 0x166c: 0x061f, 0x166d: 0x061f, 0x166e: 0x0f53, 0x166f: 0x0f57, + 0x1670: 0x0f5f, 0x1671: 0x0f63, 0x1672: 0x0f6f, 0x1673: 0x0623, 0x1674: 0x0f87, 0x1675: 0x1737, + 0x1676: 0x0fa3, 0x1677: 0x173c, 0x1678: 0x0faf, 0x1679: 0x16a1, 0x167a: 0x0fbf, 0x167b: 0x1741, + 0x167c: 0x1746, 0x167d: 0x174b, 0x167e: 0x0627, 0x167f: 0x062b, + // Block 0x5a, offset 0x1680 + 0x1680: 0x0ff7, 0x1681: 0x1755, 0x1682: 0x1750, 0x1683: 0x175a, 0x1684: 0x175f, 0x1685: 0x0fff, + 0x1686: 0x1003, 0x1687: 0x1003, 0x1688: 0x100b, 0x1689: 0x0633, 0x168a: 0x100f, 0x168b: 0x0637, + 0x168c: 0x063b, 0x168d: 0x1769, 0x168e: 0x1023, 0x168f: 0x102b, 0x1690: 0x1037, 0x1691: 0x063f, + 0x1692: 0x176e, 0x1693: 0x105b, 0x1694: 0x1773, 0x1695: 0x1778, 0x1696: 0x107b, 0x1697: 0x1093, + 0x1698: 0x0643, 0x1699: 0x109b, 0x169a: 0x109f, 0x169b: 0x10a3, 0x169c: 0x177d, 0x169d: 0x1782, + 0x169e: 0x1782, 0x169f: 0x10bb, 0x16a0: 0x0647, 0x16a1: 0x1787, 0x16a2: 0x10cf, 0x16a3: 0x10d3, + 0x16a4: 0x064b, 0x16a5: 0x178c, 0x16a6: 0x10ef, 0x16a7: 0x064f, 0x16a8: 0x10ff, 0x16a9: 0x10f7, + 0x16aa: 0x1107, 0x16ab: 0x1796, 0x16ac: 0x111f, 0x16ad: 0x0653, 0x16ae: 0x112b, 0x16af: 0x1133, + 0x16b0: 0x1143, 0x16b1: 0x0657, 0x16b2: 0x17a0, 0x16b3: 0x17a5, 0x16b4: 0x065b, 0x16b5: 0x17aa, + 0x16b6: 0x115b, 0x16b7: 0x17af, 0x16b8: 0x1167, 0x16b9: 0x1173, 0x16ba: 0x117b, 0x16bb: 0x17b4, + 0x16bc: 0x17b9, 0x16bd: 0x118f, 0x16be: 0x17be, 0x16bf: 0x1197, + // Block 0x5b, offset 0x16c0 + 0x16c0: 0x16ce, 0x16c1: 0x065f, 0x16c2: 0x11af, 0x16c3: 0x11b3, 0x16c4: 0x0667, 0x16c5: 0x11b7, + 0x16c6: 0x0a33, 0x16c7: 0x17c3, 0x16c8: 0x17c8, 0x16c9: 0x16d3, 0x16ca: 0x16d8, 0x16cb: 0x11d7, + 0x16cc: 0x11db, 0x16cd: 0x13f3, 0x16ce: 0x066b, 0x16cf: 0x1207, 0x16d0: 0x1203, 0x16d1: 0x120b, + 0x16d2: 0x083f, 0x16d3: 0x120f, 0x16d4: 0x1213, 0x16d5: 0x1217, 0x16d6: 0x121f, 0x16d7: 0x17cd, + 0x16d8: 0x121b, 0x16d9: 0x1223, 0x16da: 0x1237, 0x16db: 0x123b, 0x16dc: 0x1227, 0x16dd: 0x123f, + 0x16de: 0x1253, 0x16df: 0x1267, 0x16e0: 0x1233, 0x16e1: 0x1247, 0x16e2: 0x124b, 0x16e3: 0x124f, + 0x16e4: 0x17d2, 0x16e5: 0x17dc, 0x16e6: 0x17d7, 0x16e7: 0x066f, 0x16e8: 0x126f, 0x16e9: 0x1273, + 0x16ea: 0x127b, 0x16eb: 0x17f0, 0x16ec: 0x127f, 0x16ed: 0x17e1, 0x16ee: 0x0673, 0x16ef: 0x0677, + 0x16f0: 0x17e6, 0x16f1: 0x17eb, 0x16f2: 0x067b, 0x16f3: 0x129f, 0x16f4: 0x12a3, 0x16f5: 0x12a7, + 0x16f6: 0x12ab, 0x16f7: 0x12b7, 0x16f8: 0x12b3, 0x16f9: 0x12bf, 0x16fa: 0x12bb, 0x16fb: 0x12cb, + 0x16fc: 0x12c3, 0x16fd: 0x12c7, 0x16fe: 0x12cf, 0x16ff: 0x067f, + // Block 0x5c, offset 0x1700 + 0x1700: 0x12d7, 0x1701: 0x12db, 0x1702: 0x0683, 0x1703: 0x12eb, 0x1704: 0x12ef, 0x1705: 0x17f5, + 0x1706: 0x12fb, 0x1707: 0x12ff, 0x1708: 0x0687, 0x1709: 0x130b, 0x170a: 0x05bb, 0x170b: 0x17fa, + 0x170c: 0x17ff, 0x170d: 0x068b, 0x170e: 0x068f, 0x170f: 0x1337, 0x1710: 0x134f, 0x1711: 0x136b, + 0x1712: 0x137b, 0x1713: 0x1804, 0x1714: 0x138f, 0x1715: 0x1393, 0x1716: 0x13ab, 0x1717: 0x13b7, + 0x1718: 0x180e, 0x1719: 0x1660, 0x171a: 0x13c3, 0x171b: 0x13bf, 0x171c: 0x13cb, 0x171d: 0x1665, + 0x171e: 0x13d7, 0x171f: 0x13e3, 0x1720: 0x1813, 0x1721: 0x1818, 0x1722: 0x1423, 0x1723: 0x142f, + 0x1724: 0x1437, 0x1725: 0x181d, 0x1726: 0x143b, 0x1727: 0x1467, 0x1728: 0x1473, 0x1729: 0x1477, + 0x172a: 0x146f, 0x172b: 0x1483, 0x172c: 0x1487, 0x172d: 0x1822, 0x172e: 0x1493, 0x172f: 0x0693, + 0x1730: 0x149b, 0x1731: 0x1827, 0x1732: 0x0697, 0x1733: 0x14d3, 0x1734: 0x0ac3, 0x1735: 0x14eb, + 0x1736: 0x182c, 0x1737: 0x1836, 0x1738: 0x069b, 0x1739: 0x069f, 0x173a: 0x1513, 0x173b: 0x183b, + 0x173c: 0x06a3, 0x173d: 0x1840, 0x173e: 0x152b, 0x173f: 0x152b, + // Block 0x5d, offset 0x1740 + 0x1740: 0x1533, 0x1741: 0x1845, 0x1742: 0x154b, 0x1743: 0x06a7, 0x1744: 0x155b, 0x1745: 0x1567, + 0x1746: 0x156f, 0x1747: 0x1577, 0x1748: 0x06ab, 0x1749: 0x184a, 0x174a: 0x158b, 0x174b: 0x15a7, + 0x174c: 0x15b3, 0x174d: 0x06af, 0x174e: 0x06b3, 0x174f: 0x15b7, 0x1750: 0x184f, 0x1751: 0x06b7, + 0x1752: 0x1854, 0x1753: 0x1859, 0x1754: 0x185e, 0x1755: 0x15db, 0x1756: 0x06bb, 0x1757: 0x15ef, + 0x1758: 0x15f7, 0x1759: 0x15fb, 0x175a: 0x1603, 0x175b: 0x160b, 0x175c: 0x1613, 0x175d: 0x1868, +} + +// nfkcIndex: 22 blocks, 1408 entries, 2816 bytes +// Block 0 is the zero block. +var nfkcIndex = [1408]uint16{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xc2: 0x5c, 0xc3: 0x01, 0xc4: 0x02, 0xc5: 0x03, 0xc6: 0x5d, 0xc7: 0x04, + 0xc8: 0x05, 0xca: 0x5e, 0xcb: 0x5f, 0xcc: 0x06, 0xcd: 0x07, 0xce: 0x08, 0xcf: 0x09, + 0xd0: 0x0a, 0xd1: 0x60, 0xd2: 0x61, 0xd3: 0x0b, 0xd6: 0x0c, 0xd7: 0x62, + 0xd8: 0x63, 0xd9: 0x0d, 0xdb: 0x64, 0xdc: 0x65, 0xdd: 0x66, 0xdf: 0x67, + 0xe0: 0x02, 0xe1: 0x03, 0xe2: 0x04, 0xe3: 0x05, + 0xea: 0x06, 0xeb: 0x07, 0xec: 0x08, 0xed: 0x09, 0xef: 0x0a, + 0xf0: 0x13, + // Block 0x4, offset 0x100 + 0x120: 0x68, 0x121: 0x69, 0x123: 0x0e, 0x124: 0x6a, 0x125: 0x6b, 0x126: 0x6c, 0x127: 0x6d, + 0x128: 0x6e, 0x129: 0x6f, 0x12a: 0x70, 0x12b: 0x71, 0x12c: 0x6c, 0x12d: 0x72, 0x12e: 0x73, 0x12f: 0x74, + 0x131: 0x75, 0x132: 0x76, 0x133: 0x77, 0x134: 0x78, 0x135: 0x79, 0x137: 0x7a, + 0x138: 0x7b, 0x139: 0x7c, 0x13a: 0x7d, 0x13b: 0x7e, 0x13c: 0x7f, 0x13d: 0x80, 0x13e: 0x81, 0x13f: 0x82, + // Block 0x5, offset 0x140 + 0x140: 0x83, 0x142: 0x84, 0x143: 0x85, 0x144: 0x86, 0x145: 0x87, 0x146: 0x88, 0x147: 0x89, + 0x14d: 0x8a, + 0x15c: 0x8b, 0x15f: 0x8c, + 0x162: 0x8d, 0x164: 0x8e, + 0x168: 0x8f, 0x169: 0x90, 0x16a: 0x91, 0x16c: 0x0f, 0x16d: 0x92, 0x16e: 0x93, 0x16f: 0x94, + 0x170: 0x95, 0x173: 0x96, 0x174: 0x97, 0x175: 0x10, 0x176: 0x11, 0x177: 0x12, + 0x178: 0x13, 0x179: 0x14, 0x17a: 0x15, 0x17b: 0x16, 0x17c: 0x17, 0x17d: 0x18, 0x17e: 0x19, 0x17f: 0x1a, + // Block 0x6, offset 0x180 + 0x180: 0x98, 0x181: 0x99, 0x182: 0x9a, 0x183: 0x9b, 0x184: 0x1b, 0x185: 0x1c, 0x186: 0x9c, 0x187: 0x9d, + 0x188: 0x9e, 0x189: 0x1d, 0x18a: 0x1e, 0x18b: 0x9f, 0x18c: 0xa0, + 0x191: 0x1f, 0x192: 0x20, 0x193: 0xa1, + 0x1a8: 0xa2, 0x1a9: 0xa3, 0x1ab: 0xa4, + 0x1b1: 0xa5, 0x1b3: 0xa6, 0x1b5: 0xa7, 0x1b7: 0xa8, + 0x1ba: 0xa9, 0x1bb: 0xaa, 0x1bc: 0x21, 0x1bd: 0x22, 0x1be: 0x23, 0x1bf: 0xab, + // Block 0x7, offset 0x1c0 + 0x1c0: 0xac, 0x1c1: 0x24, 0x1c2: 0x25, 0x1c3: 0x26, 0x1c4: 0xad, 0x1c5: 0x27, 0x1c6: 0x28, + 0x1c8: 0x29, 0x1c9: 0x2a, 0x1ca: 0x2b, 0x1cb: 0x2c, 0x1cc: 0x2d, 0x1cd: 0x2e, 0x1ce: 0x2f, 0x1cf: 0x30, + // Block 0x8, offset 0x200 + 0x219: 0xae, 0x21a: 0xaf, 0x21b: 0xb0, 0x21d: 0xb1, 0x21f: 0xb2, + 0x220: 0xb3, 0x223: 0xb4, 0x224: 0xb5, 0x225: 0xb6, 0x226: 0xb7, 0x227: 0xb8, + 0x22a: 0xb9, 0x22b: 0xba, 0x22d: 0xbb, 0x22f: 0xbc, + 0x230: 0xbd, 0x231: 0xbe, 0x232: 0xbf, 0x233: 0xc0, 0x234: 0xc1, 0x235: 0xc2, 0x236: 0xc3, 0x237: 0xbd, + 0x238: 0xbe, 0x239: 0xbf, 0x23a: 0xc0, 0x23b: 0xc1, 0x23c: 0xc2, 0x23d: 0xc3, 0x23e: 0xbd, 0x23f: 0xbe, + // Block 0x9, offset 0x240 + 0x240: 0xbf, 0x241: 0xc0, 0x242: 0xc1, 0x243: 0xc2, 0x244: 0xc3, 0x245: 0xbd, 0x246: 0xbe, 0x247: 0xbf, + 0x248: 0xc0, 0x249: 0xc1, 0x24a: 0xc2, 0x24b: 0xc3, 0x24c: 0xbd, 0x24d: 0xbe, 0x24e: 0xbf, 0x24f: 0xc0, + 0x250: 0xc1, 0x251: 0xc2, 0x252: 0xc3, 0x253: 0xbd, 0x254: 0xbe, 0x255: 0xbf, 0x256: 0xc0, 0x257: 0xc1, + 0x258: 0xc2, 0x259: 0xc3, 0x25a: 0xbd, 0x25b: 0xbe, 0x25c: 0xbf, 0x25d: 0xc0, 0x25e: 0xc1, 0x25f: 0xc2, + 0x260: 0xc3, 0x261: 0xbd, 0x262: 0xbe, 0x263: 0xbf, 0x264: 0xc0, 0x265: 0xc1, 0x266: 0xc2, 0x267: 0xc3, + 0x268: 0xbd, 0x269: 0xbe, 0x26a: 0xbf, 0x26b: 0xc0, 0x26c: 0xc1, 0x26d: 0xc2, 0x26e: 0xc3, 0x26f: 0xbd, + 0x270: 0xbe, 0x271: 0xbf, 0x272: 0xc0, 0x273: 0xc1, 0x274: 0xc2, 0x275: 0xc3, 0x276: 0xbd, 0x277: 0xbe, + 0x278: 0xbf, 0x279: 0xc0, 0x27a: 0xc1, 0x27b: 0xc2, 0x27c: 0xc3, 0x27d: 0xbd, 0x27e: 0xbe, 0x27f: 0xbf, + // Block 0xa, offset 0x280 + 0x280: 0xc0, 0x281: 0xc1, 0x282: 0xc2, 0x283: 0xc3, 0x284: 0xbd, 0x285: 0xbe, 0x286: 0xbf, 0x287: 0xc0, + 0x288: 0xc1, 0x289: 0xc2, 0x28a: 0xc3, 0x28b: 0xbd, 0x28c: 0xbe, 0x28d: 0xbf, 0x28e: 0xc0, 0x28f: 0xc1, + 0x290: 0xc2, 0x291: 0xc3, 0x292: 0xbd, 0x293: 0xbe, 0x294: 0xbf, 0x295: 0xc0, 0x296: 0xc1, 0x297: 0xc2, + 0x298: 0xc3, 0x299: 0xbd, 0x29a: 0xbe, 0x29b: 0xbf, 0x29c: 0xc0, 0x29d: 0xc1, 0x29e: 0xc2, 0x29f: 0xc3, + 0x2a0: 0xbd, 0x2a1: 0xbe, 0x2a2: 0xbf, 0x2a3: 0xc0, 0x2a4: 0xc1, 0x2a5: 0xc2, 0x2a6: 0xc3, 0x2a7: 0xbd, + 0x2a8: 0xbe, 0x2a9: 0xbf, 0x2aa: 0xc0, 0x2ab: 0xc1, 0x2ac: 0xc2, 0x2ad: 0xc3, 0x2ae: 0xbd, 0x2af: 0xbe, + 0x2b0: 0xbf, 0x2b1: 0xc0, 0x2b2: 0xc1, 0x2b3: 0xc2, 0x2b4: 0xc3, 0x2b5: 0xbd, 0x2b6: 0xbe, 0x2b7: 0xbf, + 0x2b8: 0xc0, 0x2b9: 0xc1, 0x2ba: 0xc2, 0x2bb: 0xc3, 0x2bc: 0xbd, 0x2bd: 0xbe, 0x2be: 0xbf, 0x2bf: 0xc0, + // Block 0xb, offset 0x2c0 + 0x2c0: 0xc1, 0x2c1: 0xc2, 0x2c2: 0xc3, 0x2c3: 0xbd, 0x2c4: 0xbe, 0x2c5: 0xbf, 0x2c6: 0xc0, 0x2c7: 0xc1, + 0x2c8: 0xc2, 0x2c9: 0xc3, 0x2ca: 0xbd, 0x2cb: 0xbe, 0x2cc: 0xbf, 0x2cd: 0xc0, 0x2ce: 0xc1, 0x2cf: 0xc2, + 0x2d0: 0xc3, 0x2d1: 0xbd, 0x2d2: 0xbe, 0x2d3: 0xbf, 0x2d4: 0xc0, 0x2d5: 0xc1, 0x2d6: 0xc2, 0x2d7: 0xc3, + 0x2d8: 0xbd, 0x2d9: 0xbe, 0x2da: 0xbf, 0x2db: 0xc0, 0x2dc: 0xc1, 0x2dd: 0xc2, 0x2de: 0xc4, + // Block 0xc, offset 0x300 + 0x324: 0x31, 0x325: 0x32, 0x326: 0x33, 0x327: 0x34, + 0x328: 0x35, 0x329: 0x36, 0x32a: 0x37, 0x32b: 0x38, 0x32c: 0x39, 0x32d: 0x3a, 0x32e: 0x3b, 0x32f: 0x3c, + 0x330: 0x3d, 0x331: 0x3e, 0x332: 0x3f, 0x333: 0x40, 0x334: 0x41, 0x335: 0x42, 0x336: 0x43, 0x337: 0x44, + 0x338: 0x45, 0x339: 0x46, 0x33a: 0x47, 0x33b: 0x48, 0x33c: 0xc5, 0x33d: 0x49, 0x33e: 0x4a, 0x33f: 0x4b, + // Block 0xd, offset 0x340 + 0x347: 0xc6, + 0x34b: 0xc7, 0x34d: 0xc8, + 0x368: 0xc9, 0x36b: 0xca, + 0x374: 0xcb, + 0x37d: 0xcc, + // Block 0xe, offset 0x380 + 0x381: 0xcd, 0x382: 0xce, 0x384: 0xcf, 0x385: 0xb7, 0x387: 0xd0, + 0x388: 0xd1, 0x38b: 0xd2, 0x38c: 0xd3, 0x38d: 0xd4, + 0x391: 0xd5, 0x392: 0xd6, 0x393: 0xd7, 0x396: 0xd8, 0x397: 0xd9, + 0x398: 0xda, 0x39a: 0xdb, 0x39c: 0xdc, + 0x3a0: 0xdd, 0x3a7: 0xde, + 0x3a8: 0xdf, 0x3a9: 0xe0, 0x3aa: 0xe1, + 0x3b0: 0xda, 0x3b5: 0xe2, 0x3b6: 0xe3, + // Block 0xf, offset 0x3c0 + 0x3eb: 0xe4, 0x3ec: 0xe5, + // Block 0x10, offset 0x400 + 0x432: 0xe6, + // Block 0x11, offset 0x440 + 0x445: 0xe7, 0x446: 0xe8, 0x447: 0xe9, + 0x449: 0xea, + 0x450: 0xeb, 0x451: 0xec, 0x452: 0xed, 0x453: 0xee, 0x454: 0xef, 0x455: 0xf0, 0x456: 0xf1, 0x457: 0xf2, + 0x458: 0xf3, 0x459: 0xf4, 0x45a: 0x4c, 0x45b: 0xf5, 0x45c: 0xf6, 0x45d: 0xf7, 0x45e: 0xf8, 0x45f: 0x4d, + // Block 0x12, offset 0x480 + 0x480: 0xf9, 0x484: 0xe5, + 0x48b: 0xfa, + 0x4a3: 0xfb, 0x4a5: 0xfc, + 0x4b8: 0x4e, 0x4b9: 0x4f, 0x4ba: 0x50, + // Block 0x13, offset 0x4c0 + 0x4c4: 0x51, 0x4c5: 0xfd, 0x4c6: 0xfe, + 0x4c8: 0x52, 0x4c9: 0xff, + // Block 0x14, offset 0x500 + 0x520: 0x53, 0x521: 0x54, 0x522: 0x55, 0x523: 0x56, 0x524: 0x57, 0x525: 0x58, 0x526: 0x59, 0x527: 0x5a, + 0x528: 0x5b, + // Block 0x15, offset 0x540 + 0x550: 0x0b, 0x551: 0x0c, 0x556: 0x0d, + 0x55b: 0x0e, 0x55d: 0x0f, 0x55e: 0x10, 0x55f: 0x11, + 0x56f: 0x12, +} + +// nfkcSparseOffset: 164 entries, 328 bytes +var nfkcSparseOffset = []uint16{0x0, 0xe, 0x12, 0x1b, 0x25, 0x35, 0x37, 0x3c, 0x47, 0x56, 0x63, 0x6b, 0x70, 0x75, 0x77, 0x7f, 0x86, 0x89, 0x91, 0x95, 0x99, 0x9b, 0x9d, 0xa6, 0xaa, 0xb1, 0xb6, 0xb9, 0xc3, 0xc6, 0xcd, 0xd5, 0xd9, 0xdb, 0xdf, 0xe3, 0xe9, 0xfa, 0x106, 0x108, 0x10e, 0x110, 0x112, 0x114, 0x116, 0x118, 0x11a, 0x11c, 0x11f, 0x122, 0x124, 0x127, 0x12a, 0x12e, 0x133, 0x13c, 0x13e, 0x141, 0x143, 0x14e, 0x159, 0x167, 0x175, 0x185, 0x193, 0x19a, 0x1a0, 0x1af, 0x1b3, 0x1b5, 0x1b9, 0x1bb, 0x1be, 0x1c0, 0x1c3, 0x1c5, 0x1c8, 0x1ca, 0x1cc, 0x1ce, 0x1da, 0x1e4, 0x1ee, 0x1f1, 0x1f5, 0x1f7, 0x1f9, 0x1fb, 0x1fd, 0x200, 0x202, 0x204, 0x206, 0x208, 0x20e, 0x211, 0x215, 0x217, 0x21e, 0x224, 0x22a, 0x232, 0x238, 0x23e, 0x244, 0x248, 0x24a, 0x24c, 0x24e, 0x250, 0x256, 0x259, 0x25b, 0x261, 0x264, 0x26c, 0x273, 0x276, 0x279, 0x27b, 0x27e, 0x286, 0x28a, 0x291, 0x294, 0x29a, 0x29c, 0x29e, 0x2a1, 0x2a3, 0x2a6, 0x2a8, 0x2aa, 0x2ac, 0x2ae, 0x2b1, 0x2b3, 0x2b5, 0x2b7, 0x2b9, 0x2c6, 0x2d0, 0x2d2, 0x2d4, 0x2d8, 0x2dd, 0x2e9, 0x2ee, 0x2f7, 0x2fd, 0x302, 0x306, 0x30b, 0x30f, 0x31f, 0x32d, 0x33b, 0x349, 0x34f, 0x351, 0x353, 0x356, 0x361, 0x363} + +// nfkcSparseValues: 877 entries, 3508 bytes +var nfkcSparseValues = [877]valueRange{ + // Block 0x0, offset 0x0 + {value: 0x0002, lo: 0x0d}, + {value: 0x0001, lo: 0xa0, hi: 0xa0}, + {value: 0x427b, lo: 0xa8, hi: 0xa8}, + {value: 0x0083, lo: 0xaa, hi: 0xaa}, + {value: 0x4267, lo: 0xaf, hi: 0xaf}, + {value: 0x0025, lo: 0xb2, hi: 0xb3}, + {value: 0x425d, lo: 0xb4, hi: 0xb4}, + {value: 0x01dc, lo: 0xb5, hi: 0xb5}, + {value: 0x4294, lo: 0xb8, hi: 0xb8}, + {value: 0x0023, lo: 0xb9, hi: 0xb9}, + {value: 0x009f, lo: 0xba, hi: 0xba}, + {value: 0x221f, lo: 0xbc, hi: 0xbc}, + {value: 0x2213, lo: 0xbd, hi: 0xbd}, + {value: 0x22b5, lo: 0xbe, hi: 0xbe}, + // Block 0x1, offset 0xe + {value: 0x0091, lo: 0x03}, + {value: 0x46e5, lo: 0xa0, hi: 0xa1}, + {value: 0x4717, lo: 0xaf, hi: 0xb0}, + {value: 0xa000, lo: 0xb7, hi: 0xb7}, + // Block 0x2, offset 0x12 + {value: 0x0003, lo: 0x08}, + {value: 0xa000, lo: 0x92, hi: 0x92}, + {value: 0x0091, lo: 0xb0, hi: 0xb0}, + {value: 0x0119, lo: 0xb1, hi: 0xb1}, + {value: 0x0095, lo: 0xb2, hi: 0xb2}, + {value: 0x00a5, lo: 0xb3, hi: 0xb3}, + {value: 0x0143, lo: 0xb4, hi: 0xb6}, + {value: 0x00af, lo: 0xb7, hi: 0xb7}, + {value: 0x00b3, lo: 0xb8, hi: 0xb8}, + // Block 0x3, offset 0x1b + {value: 0x000a, lo: 0x09}, + {value: 0x4271, lo: 0x98, hi: 0x98}, + {value: 0x4276, lo: 0x99, hi: 0x9a}, + {value: 0x4299, lo: 0x9b, hi: 0x9b}, + {value: 0x4262, lo: 0x9c, hi: 0x9c}, + {value: 0x4285, lo: 0x9d, hi: 0x9d}, + {value: 0x0113, lo: 0xa0, hi: 0xa0}, + {value: 0x0099, lo: 0xa1, hi: 0xa1}, + {value: 0x00a7, lo: 0xa2, hi: 0xa3}, + {value: 0x0167, lo: 0xa4, hi: 0xa4}, + // Block 0x4, offset 0x25 + {value: 0x0000, lo: 0x0f}, + {value: 0xa000, lo: 0x83, hi: 0x83}, + {value: 0xa000, lo: 0x87, hi: 0x87}, + {value: 0xa000, lo: 0x8b, hi: 0x8b}, + {value: 0xa000, lo: 0x8d, hi: 0x8d}, + {value: 0x37a8, lo: 0x90, hi: 0x90}, + {value: 0x37b4, lo: 0x91, hi: 0x91}, + {value: 0x37a2, lo: 0x93, hi: 0x93}, + {value: 0xa000, lo: 0x96, hi: 0x96}, + {value: 0x381a, lo: 0x97, hi: 0x97}, + {value: 0x37e4, lo: 0x9c, hi: 0x9c}, + {value: 0x37cc, lo: 0x9d, hi: 0x9d}, + {value: 0x37f6, lo: 0x9e, hi: 0x9e}, + {value: 0xa000, lo: 0xb4, hi: 0xb5}, + {value: 0x3820, lo: 0xb6, hi: 0xb6}, + {value: 0x3826, lo: 0xb7, hi: 0xb7}, + // Block 0x5, offset 0x35 + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0x83, hi: 0x87}, + // Block 0x6, offset 0x37 + {value: 0x0001, lo: 0x04}, + {value: 0x8113, lo: 0x81, hi: 0x82}, + {value: 0x8132, lo: 0x84, hi: 0x84}, + {value: 0x812d, lo: 0x85, hi: 0x85}, + {value: 0x810d, lo: 0x87, hi: 0x87}, + // Block 0x7, offset 0x3c + {value: 0x0000, lo: 0x0a}, + {value: 0x8132, lo: 0x90, hi: 0x97}, + {value: 0x8119, lo: 0x98, hi: 0x98}, + {value: 0x811a, lo: 0x99, hi: 0x99}, + {value: 0x811b, lo: 0x9a, hi: 0x9a}, + {value: 0x3844, lo: 0xa2, hi: 0xa2}, + {value: 0x384a, lo: 0xa3, hi: 0xa3}, + {value: 0x3856, lo: 0xa4, hi: 0xa4}, + {value: 0x3850, lo: 0xa5, hi: 0xa5}, + {value: 0x385c, lo: 0xa6, hi: 0xa6}, + {value: 0xa000, lo: 0xa7, hi: 0xa7}, + // Block 0x8, offset 0x47 + {value: 0x0000, lo: 0x0e}, + {value: 0x386e, lo: 0x80, hi: 0x80}, + {value: 0xa000, lo: 0x81, hi: 0x81}, + {value: 0x3862, lo: 0x82, hi: 0x82}, + {value: 0xa000, lo: 0x92, hi: 0x92}, + {value: 0x3868, lo: 0x93, hi: 0x93}, + {value: 0xa000, lo: 0x95, hi: 0x95}, + {value: 0x8132, lo: 0x96, hi: 0x9c}, + {value: 0x8132, lo: 0x9f, hi: 0xa2}, + {value: 0x812d, lo: 0xa3, hi: 0xa3}, + {value: 0x8132, lo: 0xa4, hi: 0xa4}, + {value: 0x8132, lo: 0xa7, hi: 0xa8}, + {value: 0x812d, lo: 0xaa, hi: 0xaa}, + {value: 0x8132, lo: 0xab, hi: 0xac}, + {value: 0x812d, lo: 0xad, hi: 0xad}, + // Block 0x9, offset 0x56 + {value: 0x0000, lo: 0x0c}, + {value: 0x811f, lo: 0x91, hi: 0x91}, + {value: 0x8132, lo: 0xb0, hi: 0xb0}, + {value: 0x812d, lo: 0xb1, hi: 0xb1}, + {value: 0x8132, lo: 0xb2, hi: 0xb3}, + {value: 0x812d, lo: 0xb4, hi: 0xb4}, + {value: 0x8132, lo: 0xb5, hi: 0xb6}, + {value: 0x812d, lo: 0xb7, hi: 0xb9}, + {value: 0x8132, lo: 0xba, hi: 0xba}, + {value: 0x812d, lo: 0xbb, hi: 0xbc}, + {value: 0x8132, lo: 0xbd, hi: 0xbd}, + {value: 0x812d, lo: 0xbe, hi: 0xbe}, + {value: 0x8132, lo: 0xbf, hi: 0xbf}, + // Block 0xa, offset 0x63 + {value: 0x0005, lo: 0x07}, + {value: 0x8132, lo: 0x80, hi: 0x80}, + {value: 0x8132, lo: 0x81, hi: 0x81}, + {value: 0x812d, lo: 0x82, hi: 0x83}, + {value: 0x812d, lo: 0x84, hi: 0x85}, + {value: 0x812d, lo: 0x86, hi: 0x87}, + {value: 0x812d, lo: 0x88, hi: 0x89}, + {value: 0x8132, lo: 0x8a, hi: 0x8a}, + // Block 0xb, offset 0x6b + {value: 0x0000, lo: 0x04}, + {value: 0x8132, lo: 0xab, hi: 0xb1}, + {value: 0x812d, lo: 0xb2, hi: 0xb2}, + {value: 0x8132, lo: 0xb3, hi: 0xb3}, + {value: 0x812d, lo: 0xbd, hi: 0xbd}, + // Block 0xc, offset 0x70 + {value: 0x0000, lo: 0x04}, + {value: 0x8132, lo: 0x96, hi: 0x99}, + {value: 0x8132, lo: 0x9b, hi: 0xa3}, + {value: 0x8132, lo: 0xa5, hi: 0xa7}, + {value: 0x8132, lo: 0xa9, hi: 0xad}, + // Block 0xd, offset 0x75 + {value: 0x0000, lo: 0x01}, + {value: 0x812d, lo: 0x99, hi: 0x9b}, + // Block 0xe, offset 0x77 + {value: 0x0000, lo: 0x07}, + {value: 0xa000, lo: 0xa8, hi: 0xa8}, + {value: 0x3edb, lo: 0xa9, hi: 0xa9}, + {value: 0xa000, lo: 0xb0, hi: 0xb0}, + {value: 0x3ee3, lo: 0xb1, hi: 0xb1}, + {value: 0xa000, lo: 0xb3, hi: 0xb3}, + {value: 0x3eeb, lo: 0xb4, hi: 0xb4}, + {value: 0x9902, lo: 0xbc, hi: 0xbc}, + // Block 0xf, offset 0x7f + {value: 0x0008, lo: 0x06}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x8132, lo: 0x91, hi: 0x91}, + {value: 0x812d, lo: 0x92, hi: 0x92}, + {value: 0x8132, lo: 0x93, hi: 0x93}, + {value: 0x8132, lo: 0x94, hi: 0x94}, + {value: 0x451f, lo: 0x98, hi: 0x9f}, + // Block 0x10, offset 0x86 + {value: 0x0000, lo: 0x02}, + {value: 0x8102, lo: 0xbc, hi: 0xbc}, + {value: 0x9900, lo: 0xbe, hi: 0xbe}, + // Block 0x11, offset 0x89 + {value: 0x0008, lo: 0x07}, + {value: 0xa000, lo: 0x87, hi: 0x87}, + {value: 0x2ca1, lo: 0x8b, hi: 0x8c}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x9900, lo: 0x97, hi: 0x97}, + {value: 0x455f, lo: 0x9c, hi: 0x9d}, + {value: 0x456f, lo: 0x9f, hi: 0x9f}, + {value: 0x8132, lo: 0xbe, hi: 0xbe}, + // Block 0x12, offset 0x91 + {value: 0x0000, lo: 0x03}, + {value: 0x4597, lo: 0xb3, hi: 0xb3}, + {value: 0x459f, lo: 0xb6, hi: 0xb6}, + {value: 0x8102, lo: 0xbc, hi: 0xbc}, + // Block 0x13, offset 0x95 + {value: 0x0008, lo: 0x03}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x4577, lo: 0x99, hi: 0x9b}, + {value: 0x458f, lo: 0x9e, hi: 0x9e}, + // Block 0x14, offset 0x99 + {value: 0x0000, lo: 0x01}, + {value: 0x8102, lo: 0xbc, hi: 0xbc}, + // Block 0x15, offset 0x9b + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + // Block 0x16, offset 0x9d + {value: 0x0000, lo: 0x08}, + {value: 0xa000, lo: 0x87, hi: 0x87}, + {value: 0x2cb9, lo: 0x88, hi: 0x88}, + {value: 0x2cb1, lo: 0x8b, hi: 0x8b}, + {value: 0x2cc1, lo: 0x8c, hi: 0x8c}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x9900, lo: 0x96, hi: 0x97}, + {value: 0x45a7, lo: 0x9c, hi: 0x9c}, + {value: 0x45af, lo: 0x9d, hi: 0x9d}, + // Block 0x17, offset 0xa6 + {value: 0x0000, lo: 0x03}, + {value: 0xa000, lo: 0x92, hi: 0x92}, + {value: 0x2cc9, lo: 0x94, hi: 0x94}, + {value: 0x9900, lo: 0xbe, hi: 0xbe}, + // Block 0x18, offset 0xaa + {value: 0x0000, lo: 0x06}, + {value: 0xa000, lo: 0x86, hi: 0x87}, + {value: 0x2cd1, lo: 0x8a, hi: 0x8a}, + {value: 0x2ce1, lo: 0x8b, hi: 0x8b}, + {value: 0x2cd9, lo: 0x8c, hi: 0x8c}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x9900, lo: 0x97, hi: 0x97}, + // Block 0x19, offset 0xb1 + {value: 0x1801, lo: 0x04}, + {value: 0xa000, lo: 0x86, hi: 0x86}, + {value: 0x3ef3, lo: 0x88, hi: 0x88}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x8120, lo: 0x95, hi: 0x96}, + // Block 0x1a, offset 0xb6 + {value: 0x0000, lo: 0x02}, + {value: 0x8102, lo: 0xbc, hi: 0xbc}, + {value: 0xa000, lo: 0xbf, hi: 0xbf}, + // Block 0x1b, offset 0xb9 + {value: 0x0000, lo: 0x09}, + {value: 0x2ce9, lo: 0x80, hi: 0x80}, + {value: 0x9900, lo: 0x82, hi: 0x82}, + {value: 0xa000, lo: 0x86, hi: 0x86}, + {value: 0x2cf1, lo: 0x87, hi: 0x87}, + {value: 0x2cf9, lo: 0x88, hi: 0x88}, + {value: 0x2f53, lo: 0x8a, hi: 0x8a}, + {value: 0x2ddb, lo: 0x8b, hi: 0x8b}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x9900, lo: 0x95, hi: 0x96}, + // Block 0x1c, offset 0xc3 + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0xbb, hi: 0xbc}, + {value: 0x9900, lo: 0xbe, hi: 0xbe}, + // Block 0x1d, offset 0xc6 + {value: 0x0000, lo: 0x06}, + {value: 0xa000, lo: 0x86, hi: 0x87}, + {value: 0x2d01, lo: 0x8a, hi: 0x8a}, + {value: 0x2d11, lo: 0x8b, hi: 0x8b}, + {value: 0x2d09, lo: 0x8c, hi: 0x8c}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x9900, lo: 0x97, hi: 0x97}, + // Block 0x1e, offset 0xcd + {value: 0x6be7, lo: 0x07}, + {value: 0x9904, lo: 0x8a, hi: 0x8a}, + {value: 0x9900, lo: 0x8f, hi: 0x8f}, + {value: 0xa000, lo: 0x99, hi: 0x99}, + {value: 0x3efb, lo: 0x9a, hi: 0x9a}, + {value: 0x2f5b, lo: 0x9c, hi: 0x9c}, + {value: 0x2de6, lo: 0x9d, hi: 0x9d}, + {value: 0x2d19, lo: 0x9e, hi: 0x9f}, + // Block 0x1f, offset 0xd5 + {value: 0x0000, lo: 0x03}, + {value: 0x2624, lo: 0xb3, hi: 0xb3}, + {value: 0x8122, lo: 0xb8, hi: 0xb9}, + {value: 0x8104, lo: 0xba, hi: 0xba}, + // Block 0x20, offset 0xd9 + {value: 0x0000, lo: 0x01}, + {value: 0x8123, lo: 0x88, hi: 0x8b}, + // Block 0x21, offset 0xdb + {value: 0x0000, lo: 0x03}, + {value: 0x2639, lo: 0xb3, hi: 0xb3}, + {value: 0x8124, lo: 0xb8, hi: 0xb9}, + {value: 0x8104, lo: 0xba, hi: 0xba}, + // Block 0x22, offset 0xdf + {value: 0x0000, lo: 0x03}, + {value: 0x8125, lo: 0x88, hi: 0x8b}, + {value: 0x262b, lo: 0x9c, hi: 0x9c}, + {value: 0x2632, lo: 0x9d, hi: 0x9d}, + // Block 0x23, offset 0xe3 + {value: 0x0000, lo: 0x05}, + {value: 0x030b, lo: 0x8c, hi: 0x8c}, + {value: 0x812d, lo: 0x98, hi: 0x99}, + {value: 0x812d, lo: 0xb5, hi: 0xb5}, + {value: 0x812d, lo: 0xb7, hi: 0xb7}, + {value: 0x812b, lo: 0xb9, hi: 0xb9}, + // Block 0x24, offset 0xe9 + {value: 0x0000, lo: 0x10}, + {value: 0x2647, lo: 0x83, hi: 0x83}, + {value: 0x264e, lo: 0x8d, hi: 0x8d}, + {value: 0x2655, lo: 0x92, hi: 0x92}, + {value: 0x265c, lo: 0x97, hi: 0x97}, + {value: 0x2663, lo: 0x9c, hi: 0x9c}, + {value: 0x2640, lo: 0xa9, hi: 0xa9}, + {value: 0x8126, lo: 0xb1, hi: 0xb1}, + {value: 0x8127, lo: 0xb2, hi: 0xb2}, + {value: 0x4a87, lo: 0xb3, hi: 0xb3}, + {value: 0x8128, lo: 0xb4, hi: 0xb4}, + {value: 0x4a90, lo: 0xb5, hi: 0xb5}, + {value: 0x45b7, lo: 0xb6, hi: 0xb6}, + {value: 0x45f7, lo: 0xb7, hi: 0xb7}, + {value: 0x45bf, lo: 0xb8, hi: 0xb8}, + {value: 0x4602, lo: 0xb9, hi: 0xb9}, + {value: 0x8127, lo: 0xba, hi: 0xbd}, + // Block 0x25, offset 0xfa + {value: 0x0000, lo: 0x0b}, + {value: 0x8127, lo: 0x80, hi: 0x80}, + {value: 0x4a99, lo: 0x81, hi: 0x81}, + {value: 0x8132, lo: 0x82, hi: 0x83}, + {value: 0x8104, lo: 0x84, hi: 0x84}, + {value: 0x8132, lo: 0x86, hi: 0x87}, + {value: 0x2671, lo: 0x93, hi: 0x93}, + {value: 0x2678, lo: 0x9d, hi: 0x9d}, + {value: 0x267f, lo: 0xa2, hi: 0xa2}, + {value: 0x2686, lo: 0xa7, hi: 0xa7}, + {value: 0x268d, lo: 0xac, hi: 0xac}, + {value: 0x266a, lo: 0xb9, hi: 0xb9}, + // Block 0x26, offset 0x106 + {value: 0x0000, lo: 0x01}, + {value: 0x812d, lo: 0x86, hi: 0x86}, + // Block 0x27, offset 0x108 + {value: 0x0000, lo: 0x05}, + {value: 0xa000, lo: 0xa5, hi: 0xa5}, + {value: 0x2d21, lo: 0xa6, hi: 0xa6}, + {value: 0x9900, lo: 0xae, hi: 0xae}, + {value: 0x8102, lo: 0xb7, hi: 0xb7}, + {value: 0x8104, lo: 0xb9, hi: 0xba}, + // Block 0x28, offset 0x10e + {value: 0x0000, lo: 0x01}, + {value: 0x812d, lo: 0x8d, hi: 0x8d}, + // Block 0x29, offset 0x110 + {value: 0x0000, lo: 0x01}, + {value: 0x030f, lo: 0xbc, hi: 0xbc}, + // Block 0x2a, offset 0x112 + {value: 0x0000, lo: 0x01}, + {value: 0xa000, lo: 0x80, hi: 0x92}, + // Block 0x2b, offset 0x114 + {value: 0x0000, lo: 0x01}, + {value: 0xb900, lo: 0xa1, hi: 0xb5}, + // Block 0x2c, offset 0x116 + {value: 0x0000, lo: 0x01}, + {value: 0x9900, lo: 0xa8, hi: 0xbf}, + // Block 0x2d, offset 0x118 + {value: 0x0000, lo: 0x01}, + {value: 0x9900, lo: 0x80, hi: 0x82}, + // Block 0x2e, offset 0x11a + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0x9d, hi: 0x9f}, + // Block 0x2f, offset 0x11c + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0x94, hi: 0x94}, + {value: 0x8104, lo: 0xb4, hi: 0xb4}, + // Block 0x30, offset 0x11f + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0x92, hi: 0x92}, + {value: 0x8132, lo: 0x9d, hi: 0x9d}, + // Block 0x31, offset 0x122 + {value: 0x0000, lo: 0x01}, + {value: 0x8131, lo: 0xa9, hi: 0xa9}, + // Block 0x32, offset 0x124 + {value: 0x0004, lo: 0x02}, + {value: 0x812e, lo: 0xb9, hi: 0xba}, + {value: 0x812d, lo: 0xbb, hi: 0xbb}, + // Block 0x33, offset 0x127 + {value: 0x0000, lo: 0x02}, + {value: 0x8132, lo: 0x97, hi: 0x97}, + {value: 0x812d, lo: 0x98, hi: 0x98}, + // Block 0x34, offset 0x12a + {value: 0x0000, lo: 0x03}, + {value: 0x8104, lo: 0xa0, hi: 0xa0}, + {value: 0x8132, lo: 0xb5, hi: 0xbc}, + {value: 0x812d, lo: 0xbf, hi: 0xbf}, + // Block 0x35, offset 0x12e + {value: 0x0000, lo: 0x04}, + {value: 0x8132, lo: 0xb0, hi: 0xb4}, + {value: 0x812d, lo: 0xb5, hi: 0xba}, + {value: 0x8132, lo: 0xbb, hi: 0xbc}, + {value: 0x812d, lo: 0xbd, hi: 0xbd}, + // Block 0x36, offset 0x133 + {value: 0x0000, lo: 0x08}, + {value: 0x2d69, lo: 0x80, hi: 0x80}, + {value: 0x2d71, lo: 0x81, hi: 0x81}, + {value: 0xa000, lo: 0x82, hi: 0x82}, + {value: 0x2d79, lo: 0x83, hi: 0x83}, + {value: 0x8104, lo: 0x84, hi: 0x84}, + {value: 0x8132, lo: 0xab, hi: 0xab}, + {value: 0x812d, lo: 0xac, hi: 0xac}, + {value: 0x8132, lo: 0xad, hi: 0xb3}, + // Block 0x37, offset 0x13c + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0xaa, hi: 0xab}, + // Block 0x38, offset 0x13e + {value: 0x0000, lo: 0x02}, + {value: 0x8102, lo: 0xa6, hi: 0xa6}, + {value: 0x8104, lo: 0xb2, hi: 0xb3}, + // Block 0x39, offset 0x141 + {value: 0x0000, lo: 0x01}, + {value: 0x8102, lo: 0xb7, hi: 0xb7}, + // Block 0x3a, offset 0x143 + {value: 0x0000, lo: 0x0a}, + {value: 0x8132, lo: 0x90, hi: 0x92}, + {value: 0x8101, lo: 0x94, hi: 0x94}, + {value: 0x812d, lo: 0x95, hi: 0x99}, + {value: 0x8132, lo: 0x9a, hi: 0x9b}, + {value: 0x812d, lo: 0x9c, hi: 0x9f}, + {value: 0x8132, lo: 0xa0, hi: 0xa0}, + {value: 0x8101, lo: 0xa2, hi: 0xa8}, + {value: 0x812d, lo: 0xad, hi: 0xad}, + {value: 0x8132, lo: 0xb4, hi: 0xb4}, + {value: 0x8132, lo: 0xb8, hi: 0xb9}, + // Block 0x3b, offset 0x14e + {value: 0x0002, lo: 0x0a}, + {value: 0x0043, lo: 0xac, hi: 0xac}, + {value: 0x00d1, lo: 0xad, hi: 0xad}, + {value: 0x0045, lo: 0xae, hi: 0xae}, + {value: 0x0049, lo: 0xb0, hi: 0xb1}, + {value: 0x00e6, lo: 0xb2, hi: 0xb2}, + {value: 0x004f, lo: 0xb3, hi: 0xba}, + {value: 0x005f, lo: 0xbc, hi: 0xbc}, + {value: 0x00ef, lo: 0xbd, hi: 0xbd}, + {value: 0x0061, lo: 0xbe, hi: 0xbe}, + {value: 0x0065, lo: 0xbf, hi: 0xbf}, + // Block 0x3c, offset 0x159 + {value: 0x0000, lo: 0x0d}, + {value: 0x0001, lo: 0x80, hi: 0x8a}, + {value: 0x043b, lo: 0x91, hi: 0x91}, + {value: 0x429e, lo: 0x97, hi: 0x97}, + {value: 0x001d, lo: 0xa4, hi: 0xa4}, + {value: 0x1873, lo: 0xa5, hi: 0xa5}, + {value: 0x1b5f, lo: 0xa6, hi: 0xa6}, + {value: 0x0001, lo: 0xaf, hi: 0xaf}, + {value: 0x2694, lo: 0xb3, hi: 0xb3}, + {value: 0x2801, lo: 0xb4, hi: 0xb4}, + {value: 0x269b, lo: 0xb6, hi: 0xb6}, + {value: 0x280b, lo: 0xb7, hi: 0xb7}, + {value: 0x186d, lo: 0xbc, hi: 0xbc}, + {value: 0x426c, lo: 0xbe, hi: 0xbe}, + // Block 0x3d, offset 0x167 + {value: 0x0002, lo: 0x0d}, + {value: 0x1933, lo: 0x87, hi: 0x87}, + {value: 0x1930, lo: 0x88, hi: 0x88}, + {value: 0x1870, lo: 0x89, hi: 0x89}, + {value: 0x2991, lo: 0x97, hi: 0x97}, + {value: 0x0001, lo: 0x9f, hi: 0x9f}, + {value: 0x0021, lo: 0xb0, hi: 0xb0}, + {value: 0x0093, lo: 0xb1, hi: 0xb1}, + {value: 0x0029, lo: 0xb4, hi: 0xb9}, + {value: 0x0017, lo: 0xba, hi: 0xba}, + {value: 0x0467, lo: 0xbb, hi: 0xbb}, + {value: 0x003b, lo: 0xbc, hi: 0xbc}, + {value: 0x0011, lo: 0xbd, hi: 0xbe}, + {value: 0x009d, lo: 0xbf, hi: 0xbf}, + // Block 0x3e, offset 0x175 + {value: 0x0002, lo: 0x0f}, + {value: 0x0021, lo: 0x80, hi: 0x89}, + {value: 0x0017, lo: 0x8a, hi: 0x8a}, + {value: 0x0467, lo: 0x8b, hi: 0x8b}, + {value: 0x003b, lo: 0x8c, hi: 0x8c}, + {value: 0x0011, lo: 0x8d, hi: 0x8e}, + {value: 0x0083, lo: 0x90, hi: 0x90}, + {value: 0x008b, lo: 0x91, hi: 0x91}, + {value: 0x009f, lo: 0x92, hi: 0x92}, + {value: 0x00b1, lo: 0x93, hi: 0x93}, + {value: 0x0104, lo: 0x94, hi: 0x94}, + {value: 0x0091, lo: 0x95, hi: 0x95}, + {value: 0x0097, lo: 0x96, hi: 0x99}, + {value: 0x00a1, lo: 0x9a, hi: 0x9a}, + {value: 0x00a7, lo: 0x9b, hi: 0x9c}, + {value: 0x199c, lo: 0xa8, hi: 0xa8}, + // Block 0x3f, offset 0x185 + {value: 0x0000, lo: 0x0d}, + {value: 0x8132, lo: 0x90, hi: 0x91}, + {value: 0x8101, lo: 0x92, hi: 0x93}, + {value: 0x8132, lo: 0x94, hi: 0x97}, + {value: 0x8101, lo: 0x98, hi: 0x9a}, + {value: 0x8132, lo: 0x9b, hi: 0x9c}, + {value: 0x8132, lo: 0xa1, hi: 0xa1}, + {value: 0x8101, lo: 0xa5, hi: 0xa6}, + {value: 0x8132, lo: 0xa7, hi: 0xa7}, + {value: 0x812d, lo: 0xa8, hi: 0xa8}, + {value: 0x8132, lo: 0xa9, hi: 0xa9}, + {value: 0x8101, lo: 0xaa, hi: 0xab}, + {value: 0x812d, lo: 0xac, hi: 0xaf}, + {value: 0x8132, lo: 0xb0, hi: 0xb0}, + // Block 0x40, offset 0x193 + {value: 0x0007, lo: 0x06}, + {value: 0x2183, lo: 0x89, hi: 0x89}, + {value: 0xa000, lo: 0x90, hi: 0x90}, + {value: 0xa000, lo: 0x92, hi: 0x92}, + {value: 0xa000, lo: 0x94, hi: 0x94}, + {value: 0x3bbc, lo: 0x9a, hi: 0x9b}, + {value: 0x3bca, lo: 0xae, hi: 0xae}, + // Block 0x41, offset 0x19a + {value: 0x000e, lo: 0x05}, + {value: 0x3bd1, lo: 0x8d, hi: 0x8e}, + {value: 0x3bd8, lo: 0x8f, hi: 0x8f}, + {value: 0xa000, lo: 0x90, hi: 0x90}, + {value: 0xa000, lo: 0x92, hi: 0x92}, + {value: 0xa000, lo: 0x94, hi: 0x94}, + // Block 0x42, offset 0x1a0 + {value: 0x0173, lo: 0x0e}, + {value: 0xa000, lo: 0x83, hi: 0x83}, + {value: 0x3be6, lo: 0x84, hi: 0x84}, + {value: 0xa000, lo: 0x88, hi: 0x88}, + {value: 0x3bed, lo: 0x89, hi: 0x89}, + {value: 0xa000, lo: 0x8b, hi: 0x8b}, + {value: 0x3bf4, lo: 0x8c, hi: 0x8c}, + {value: 0xa000, lo: 0xa3, hi: 0xa3}, + {value: 0x3bfb, lo: 0xa4, hi: 0xa4}, + {value: 0xa000, lo: 0xa5, hi: 0xa5}, + {value: 0x3c02, lo: 0xa6, hi: 0xa6}, + {value: 0x26a2, lo: 0xac, hi: 0xad}, + {value: 0x26a9, lo: 0xaf, hi: 0xaf}, + {value: 0x281f, lo: 0xb0, hi: 0xb0}, + {value: 0xa000, lo: 0xbc, hi: 0xbc}, + // Block 0x43, offset 0x1af + {value: 0x0007, lo: 0x03}, + {value: 0x3c6b, lo: 0xa0, hi: 0xa1}, + {value: 0x3c95, lo: 0xa2, hi: 0xa3}, + {value: 0x3cbf, lo: 0xaa, hi: 0xad}, + // Block 0x44, offset 0x1b3 + {value: 0x0004, lo: 0x01}, + {value: 0x048b, lo: 0xa9, hi: 0xaa}, + // Block 0x45, offset 0x1b5 + {value: 0x0002, lo: 0x03}, + {value: 0x0057, lo: 0x80, hi: 0x8f}, + {value: 0x0083, lo: 0x90, hi: 0xa9}, + {value: 0x0021, lo: 0xaa, hi: 0xaa}, + // Block 0x46, offset 0x1b9 + {value: 0x0000, lo: 0x01}, + {value: 0x299e, lo: 0x8c, hi: 0x8c}, + // Block 0x47, offset 0x1bb + {value: 0x0266, lo: 0x02}, + {value: 0x1b8f, lo: 0xb4, hi: 0xb4}, + {value: 0x192d, lo: 0xb5, hi: 0xb6}, + // Block 0x48, offset 0x1be + {value: 0x0000, lo: 0x01}, + {value: 0x44e0, lo: 0x9c, hi: 0x9c}, + // Block 0x49, offset 0x1c0 + {value: 0x0000, lo: 0x02}, + {value: 0x0095, lo: 0xbc, hi: 0xbc}, + {value: 0x006d, lo: 0xbd, hi: 0xbd}, + // Block 0x4a, offset 0x1c3 + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0xaf, hi: 0xb1}, + // Block 0x4b, offset 0x1c5 + {value: 0x0000, lo: 0x02}, + {value: 0x047f, lo: 0xaf, hi: 0xaf}, + {value: 0x8104, lo: 0xbf, hi: 0xbf}, + // Block 0x4c, offset 0x1c8 + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0xa0, hi: 0xbf}, + // Block 0x4d, offset 0x1ca + {value: 0x0000, lo: 0x01}, + {value: 0x0dc3, lo: 0x9f, hi: 0x9f}, + // Block 0x4e, offset 0x1cc + {value: 0x0000, lo: 0x01}, + {value: 0x162f, lo: 0xb3, hi: 0xb3}, + // Block 0x4f, offset 0x1ce + {value: 0x0004, lo: 0x0b}, + {value: 0x1597, lo: 0x80, hi: 0x82}, + {value: 0x15af, lo: 0x83, hi: 0x83}, + {value: 0x15c7, lo: 0x84, hi: 0x85}, + {value: 0x15d7, lo: 0x86, hi: 0x89}, + {value: 0x15eb, lo: 0x8a, hi: 0x8c}, + {value: 0x15ff, lo: 0x8d, hi: 0x8d}, + {value: 0x1607, lo: 0x8e, hi: 0x8e}, + {value: 0x160f, lo: 0x8f, hi: 0x90}, + {value: 0x161b, lo: 0x91, hi: 0x93}, + {value: 0x162b, lo: 0x94, hi: 0x94}, + {value: 0x1633, lo: 0x95, hi: 0x95}, + // Block 0x50, offset 0x1da + {value: 0x0004, lo: 0x09}, + {value: 0x0001, lo: 0x80, hi: 0x80}, + {value: 0x812c, lo: 0xaa, hi: 0xaa}, + {value: 0x8131, lo: 0xab, hi: 0xab}, + {value: 0x8133, lo: 0xac, hi: 0xac}, + {value: 0x812e, lo: 0xad, hi: 0xad}, + {value: 0x812f, lo: 0xae, hi: 0xae}, + {value: 0x812f, lo: 0xaf, hi: 0xaf}, + {value: 0x04b3, lo: 0xb6, hi: 0xb6}, + {value: 0x0887, lo: 0xb8, hi: 0xba}, + // Block 0x51, offset 0x1e4 + {value: 0x0006, lo: 0x09}, + {value: 0x0313, lo: 0xb1, hi: 0xb1}, + {value: 0x0317, lo: 0xb2, hi: 0xb2}, + {value: 0x4a3e, lo: 0xb3, hi: 0xb3}, + {value: 0x031b, lo: 0xb4, hi: 0xb4}, + {value: 0x4a44, lo: 0xb5, hi: 0xb6}, + {value: 0x031f, lo: 0xb7, hi: 0xb7}, + {value: 0x0323, lo: 0xb8, hi: 0xb8}, + {value: 0x0327, lo: 0xb9, hi: 0xb9}, + {value: 0x4a50, lo: 0xba, hi: 0xbf}, + // Block 0x52, offset 0x1ee + {value: 0x0000, lo: 0x02}, + {value: 0x8132, lo: 0xaf, hi: 0xaf}, + {value: 0x8132, lo: 0xb4, hi: 0xbd}, + // Block 0x53, offset 0x1f1 + {value: 0x0000, lo: 0x03}, + {value: 0x020f, lo: 0x9c, hi: 0x9c}, + {value: 0x0212, lo: 0x9d, hi: 0x9d}, + {value: 0x8132, lo: 0x9e, hi: 0x9f}, + // Block 0x54, offset 0x1f5 + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0xb0, hi: 0xb1}, + // Block 0x55, offset 0x1f7 + {value: 0x0000, lo: 0x01}, + {value: 0x163b, lo: 0xb0, hi: 0xb0}, + // Block 0x56, offset 0x1f9 + {value: 0x000c, lo: 0x01}, + {value: 0x00d7, lo: 0xb8, hi: 0xb9}, + // Block 0x57, offset 0x1fb + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0x86, hi: 0x86}, + // Block 0x58, offset 0x1fd + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0x84, hi: 0x84}, + {value: 0x8132, lo: 0xa0, hi: 0xb1}, + // Block 0x59, offset 0x200 + {value: 0x0000, lo: 0x01}, + {value: 0x812d, lo: 0xab, hi: 0xad}, + // Block 0x5a, offset 0x202 + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0x93, hi: 0x93}, + // Block 0x5b, offset 0x204 + {value: 0x0000, lo: 0x01}, + {value: 0x8102, lo: 0xb3, hi: 0xb3}, + // Block 0x5c, offset 0x206 + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0x80, hi: 0x80}, + // Block 0x5d, offset 0x208 + {value: 0x0000, lo: 0x05}, + {value: 0x8132, lo: 0xb0, hi: 0xb0}, + {value: 0x8132, lo: 0xb2, hi: 0xb3}, + {value: 0x812d, lo: 0xb4, hi: 0xb4}, + {value: 0x8132, lo: 0xb7, hi: 0xb8}, + {value: 0x8132, lo: 0xbe, hi: 0xbf}, + // Block 0x5e, offset 0x20e + {value: 0x0000, lo: 0x02}, + {value: 0x8132, lo: 0x81, hi: 0x81}, + {value: 0x8104, lo: 0xb6, hi: 0xb6}, + // Block 0x5f, offset 0x211 + {value: 0x0008, lo: 0x03}, + {value: 0x1637, lo: 0x9c, hi: 0x9d}, + {value: 0x0125, lo: 0x9e, hi: 0x9e}, + {value: 0x1643, lo: 0x9f, hi: 0x9f}, + // Block 0x60, offset 0x215 + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0xad, hi: 0xad}, + // Block 0x61, offset 0x217 + {value: 0x0000, lo: 0x06}, + {value: 0xe500, lo: 0x80, hi: 0x80}, + {value: 0xc600, lo: 0x81, hi: 0x9b}, + {value: 0xe500, lo: 0x9c, hi: 0x9c}, + {value: 0xc600, lo: 0x9d, hi: 0xb7}, + {value: 0xe500, lo: 0xb8, hi: 0xb8}, + {value: 0xc600, lo: 0xb9, hi: 0xbf}, + // Block 0x62, offset 0x21e + {value: 0x0000, lo: 0x05}, + {value: 0xc600, lo: 0x80, hi: 0x93}, + {value: 0xe500, lo: 0x94, hi: 0x94}, + {value: 0xc600, lo: 0x95, hi: 0xaf}, + {value: 0xe500, lo: 0xb0, hi: 0xb0}, + {value: 0xc600, lo: 0xb1, hi: 0xbf}, + // Block 0x63, offset 0x224 + {value: 0x0000, lo: 0x05}, + {value: 0xc600, lo: 0x80, hi: 0x8b}, + {value: 0xe500, lo: 0x8c, hi: 0x8c}, + {value: 0xc600, lo: 0x8d, hi: 0xa7}, + {value: 0xe500, lo: 0xa8, hi: 0xa8}, + {value: 0xc600, lo: 0xa9, hi: 0xbf}, + // Block 0x64, offset 0x22a + {value: 0x0000, lo: 0x07}, + {value: 0xc600, lo: 0x80, hi: 0x83}, + {value: 0xe500, lo: 0x84, hi: 0x84}, + {value: 0xc600, lo: 0x85, hi: 0x9f}, + {value: 0xe500, lo: 0xa0, hi: 0xa0}, + {value: 0xc600, lo: 0xa1, hi: 0xbb}, + {value: 0xe500, lo: 0xbc, hi: 0xbc}, + {value: 0xc600, lo: 0xbd, hi: 0xbf}, + // Block 0x65, offset 0x232 + {value: 0x0000, lo: 0x05}, + {value: 0xc600, lo: 0x80, hi: 0x97}, + {value: 0xe500, lo: 0x98, hi: 0x98}, + {value: 0xc600, lo: 0x99, hi: 0xb3}, + {value: 0xe500, lo: 0xb4, hi: 0xb4}, + {value: 0xc600, lo: 0xb5, hi: 0xbf}, + // Block 0x66, offset 0x238 + {value: 0x0000, lo: 0x05}, + {value: 0xc600, lo: 0x80, hi: 0x8f}, + {value: 0xe500, lo: 0x90, hi: 0x90}, + {value: 0xc600, lo: 0x91, hi: 0xab}, + {value: 0xe500, lo: 0xac, hi: 0xac}, + {value: 0xc600, lo: 0xad, hi: 0xbf}, + // Block 0x67, offset 0x23e + {value: 0x0000, lo: 0x05}, + {value: 0xc600, lo: 0x80, hi: 0x87}, + {value: 0xe500, lo: 0x88, hi: 0x88}, + {value: 0xc600, lo: 0x89, hi: 0xa3}, + {value: 0xe500, lo: 0xa4, hi: 0xa4}, + {value: 0xc600, lo: 0xa5, hi: 0xbf}, + // Block 0x68, offset 0x244 + {value: 0x0000, lo: 0x03}, + {value: 0xc600, lo: 0x80, hi: 0x87}, + {value: 0xe500, lo: 0x88, hi: 0x88}, + {value: 0xc600, lo: 0x89, hi: 0xa3}, + // Block 0x69, offset 0x248 + {value: 0x0002, lo: 0x01}, + {value: 0x0003, lo: 0x81, hi: 0xbf}, + // Block 0x6a, offset 0x24a + {value: 0x0000, lo: 0x01}, + {value: 0x812d, lo: 0xbd, hi: 0xbd}, + // Block 0x6b, offset 0x24c + {value: 0x0000, lo: 0x01}, + {value: 0x812d, lo: 0xa0, hi: 0xa0}, + // Block 0x6c, offset 0x24e + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0xb6, hi: 0xba}, + // Block 0x6d, offset 0x250 + {value: 0x002c, lo: 0x05}, + {value: 0x812d, lo: 0x8d, hi: 0x8d}, + {value: 0x8132, lo: 0x8f, hi: 0x8f}, + {value: 0x8132, lo: 0xb8, hi: 0xb8}, + {value: 0x8101, lo: 0xb9, hi: 0xba}, + {value: 0x8104, lo: 0xbf, hi: 0xbf}, + // Block 0x6e, offset 0x256 + {value: 0x0000, lo: 0x02}, + {value: 0x8132, lo: 0xa5, hi: 0xa5}, + {value: 0x812d, lo: 0xa6, hi: 0xa6}, + // Block 0x6f, offset 0x259 + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0xa4, hi: 0xa7}, + // Block 0x70, offset 0x25b + {value: 0x0000, lo: 0x05}, + {value: 0x812d, lo: 0x86, hi: 0x87}, + {value: 0x8132, lo: 0x88, hi: 0x8a}, + {value: 0x812d, lo: 0x8b, hi: 0x8b}, + {value: 0x8132, lo: 0x8c, hi: 0x8c}, + {value: 0x812d, lo: 0x8d, hi: 0x90}, + // Block 0x71, offset 0x261 + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0x86, hi: 0x86}, + {value: 0x8104, lo: 0xbf, hi: 0xbf}, + // Block 0x72, offset 0x264 + {value: 0x17fe, lo: 0x07}, + {value: 0xa000, lo: 0x99, hi: 0x99}, + {value: 0x423b, lo: 0x9a, hi: 0x9a}, + {value: 0xa000, lo: 0x9b, hi: 0x9b}, + {value: 0x4245, lo: 0x9c, hi: 0x9c}, + {value: 0xa000, lo: 0xa5, hi: 0xa5}, + {value: 0x424f, lo: 0xab, hi: 0xab}, + {value: 0x8104, lo: 0xb9, hi: 0xba}, + // Block 0x73, offset 0x26c + {value: 0x0000, lo: 0x06}, + {value: 0x8132, lo: 0x80, hi: 0x82}, + {value: 0x9900, lo: 0xa7, hi: 0xa7}, + {value: 0x2d81, lo: 0xae, hi: 0xae}, + {value: 0x2d8b, lo: 0xaf, hi: 0xaf}, + {value: 0xa000, lo: 0xb1, hi: 0xb2}, + {value: 0x8104, lo: 0xb3, hi: 0xb4}, + // Block 0x74, offset 0x273 + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0x80, hi: 0x80}, + {value: 0x8102, lo: 0x8a, hi: 0x8a}, + // Block 0x75, offset 0x276 + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0xb5, hi: 0xb5}, + {value: 0x8102, lo: 0xb6, hi: 0xb6}, + // Block 0x76, offset 0x279 + {value: 0x0002, lo: 0x01}, + {value: 0x8102, lo: 0xa9, hi: 0xaa}, + // Block 0x77, offset 0x27b + {value: 0x0000, lo: 0x02}, + {value: 0x8102, lo: 0xbb, hi: 0xbc}, + {value: 0x9900, lo: 0xbe, hi: 0xbe}, + // Block 0x78, offset 0x27e + {value: 0x0000, lo: 0x07}, + {value: 0xa000, lo: 0x87, hi: 0x87}, + {value: 0x2d95, lo: 0x8b, hi: 0x8b}, + {value: 0x2d9f, lo: 0x8c, hi: 0x8c}, + {value: 0x8104, lo: 0x8d, hi: 0x8d}, + {value: 0x9900, lo: 0x97, hi: 0x97}, + {value: 0x8132, lo: 0xa6, hi: 0xac}, + {value: 0x8132, lo: 0xb0, hi: 0xb4}, + // Block 0x79, offset 0x286 + {value: 0x0000, lo: 0x03}, + {value: 0x8104, lo: 0x82, hi: 0x82}, + {value: 0x8102, lo: 0x86, hi: 0x86}, + {value: 0x8132, lo: 0x9e, hi: 0x9e}, + // Block 0x7a, offset 0x28a + {value: 0x6b57, lo: 0x06}, + {value: 0x9900, lo: 0xb0, hi: 0xb0}, + {value: 0xa000, lo: 0xb9, hi: 0xb9}, + {value: 0x9900, lo: 0xba, hi: 0xba}, + {value: 0x2db3, lo: 0xbb, hi: 0xbb}, + {value: 0x2da9, lo: 0xbc, hi: 0xbd}, + {value: 0x2dbd, lo: 0xbe, hi: 0xbe}, + // Block 0x7b, offset 0x291 + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0x82, hi: 0x82}, + {value: 0x8102, lo: 0x83, hi: 0x83}, + // Block 0x7c, offset 0x294 + {value: 0x0000, lo: 0x05}, + {value: 0x9900, lo: 0xaf, hi: 0xaf}, + {value: 0xa000, lo: 0xb8, hi: 0xb9}, + {value: 0x2dc7, lo: 0xba, hi: 0xba}, + {value: 0x2dd1, lo: 0xbb, hi: 0xbb}, + {value: 0x8104, lo: 0xbf, hi: 0xbf}, + // Block 0x7d, offset 0x29a + {value: 0x0000, lo: 0x01}, + {value: 0x8102, lo: 0x80, hi: 0x80}, + // Block 0x7e, offset 0x29c + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0xbf, hi: 0xbf}, + // Block 0x7f, offset 0x29e + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0xb6, hi: 0xb6}, + {value: 0x8102, lo: 0xb7, hi: 0xb7}, + // Block 0x80, offset 0x2a1 + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0xab, hi: 0xab}, + // Block 0x81, offset 0x2a3 + {value: 0x0000, lo: 0x02}, + {value: 0x8104, lo: 0xb9, hi: 0xb9}, + {value: 0x8102, lo: 0xba, hi: 0xba}, + // Block 0x82, offset 0x2a6 + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0xa0, hi: 0xa0}, + // Block 0x83, offset 0x2a8 + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0xb4, hi: 0xb4}, + // Block 0x84, offset 0x2aa + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0x87, hi: 0x87}, + // Block 0x85, offset 0x2ac + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0x99, hi: 0x99}, + // Block 0x86, offset 0x2ae + {value: 0x0000, lo: 0x02}, + {value: 0x8102, lo: 0x82, hi: 0x82}, + {value: 0x8104, lo: 0x84, hi: 0x85}, + // Block 0x87, offset 0x2b1 + {value: 0x0000, lo: 0x01}, + {value: 0x8104, lo: 0x97, hi: 0x97}, + // Block 0x88, offset 0x2b3 + {value: 0x0000, lo: 0x01}, + {value: 0x8101, lo: 0xb0, hi: 0xb4}, + // Block 0x89, offset 0x2b5 + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0xb0, hi: 0xb6}, + // Block 0x8a, offset 0x2b7 + {value: 0x0000, lo: 0x01}, + {value: 0x8101, lo: 0x9e, hi: 0x9e}, + // Block 0x8b, offset 0x2b9 + {value: 0x0000, lo: 0x0c}, + {value: 0x45cf, lo: 0x9e, hi: 0x9e}, + {value: 0x45d9, lo: 0x9f, hi: 0x9f}, + {value: 0x460d, lo: 0xa0, hi: 0xa0}, + {value: 0x461b, lo: 0xa1, hi: 0xa1}, + {value: 0x4629, lo: 0xa2, hi: 0xa2}, + {value: 0x4637, lo: 0xa3, hi: 0xa3}, + {value: 0x4645, lo: 0xa4, hi: 0xa4}, + {value: 0x812b, lo: 0xa5, hi: 0xa6}, + {value: 0x8101, lo: 0xa7, hi: 0xa9}, + {value: 0x8130, lo: 0xad, hi: 0xad}, + {value: 0x812b, lo: 0xae, hi: 0xb2}, + {value: 0x812d, lo: 0xbb, hi: 0xbf}, + // Block 0x8c, offset 0x2c6 + {value: 0x0000, lo: 0x09}, + {value: 0x812d, lo: 0x80, hi: 0x82}, + {value: 0x8132, lo: 0x85, hi: 0x89}, + {value: 0x812d, lo: 0x8a, hi: 0x8b}, + {value: 0x8132, lo: 0xaa, hi: 0xad}, + {value: 0x45e3, lo: 0xbb, hi: 0xbb}, + {value: 0x45ed, lo: 0xbc, hi: 0xbc}, + {value: 0x4653, lo: 0xbd, hi: 0xbd}, + {value: 0x466f, lo: 0xbe, hi: 0xbe}, + {value: 0x4661, lo: 0xbf, hi: 0xbf}, + // Block 0x8d, offset 0x2d0 + {value: 0x0000, lo: 0x01}, + {value: 0x467d, lo: 0x80, hi: 0x80}, + // Block 0x8e, offset 0x2d2 + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0x82, hi: 0x84}, + // Block 0x8f, offset 0x2d4 + {value: 0x0002, lo: 0x03}, + {value: 0x0043, lo: 0x80, hi: 0x99}, + {value: 0x0083, lo: 0x9a, hi: 0xb3}, + {value: 0x0043, lo: 0xb4, hi: 0xbf}, + // Block 0x90, offset 0x2d8 + {value: 0x0002, lo: 0x04}, + {value: 0x005b, lo: 0x80, hi: 0x8d}, + {value: 0x0083, lo: 0x8e, hi: 0x94}, + {value: 0x0093, lo: 0x96, hi: 0xa7}, + {value: 0x0043, lo: 0xa8, hi: 0xbf}, + // Block 0x91, offset 0x2dd + {value: 0x0002, lo: 0x0b}, + {value: 0x0073, lo: 0x80, hi: 0x81}, + {value: 0x0083, lo: 0x82, hi: 0x9b}, + {value: 0x0043, lo: 0x9c, hi: 0x9c}, + {value: 0x0047, lo: 0x9e, hi: 0x9f}, + {value: 0x004f, lo: 0xa2, hi: 0xa2}, + {value: 0x0055, lo: 0xa5, hi: 0xa6}, + {value: 0x005d, lo: 0xa9, hi: 0xac}, + {value: 0x0067, lo: 0xae, hi: 0xb5}, + {value: 0x0083, lo: 0xb6, hi: 0xb9}, + {value: 0x008d, lo: 0xbb, hi: 0xbb}, + {value: 0x0091, lo: 0xbd, hi: 0xbf}, + // Block 0x92, offset 0x2e9 + {value: 0x0002, lo: 0x04}, + {value: 0x0097, lo: 0x80, hi: 0x83}, + {value: 0x00a1, lo: 0x85, hi: 0x8f}, + {value: 0x0043, lo: 0x90, hi: 0xa9}, + {value: 0x0083, lo: 0xaa, hi: 0xbf}, + // Block 0x93, offset 0x2ee + {value: 0x0002, lo: 0x08}, + {value: 0x00af, lo: 0x80, hi: 0x83}, + {value: 0x0043, lo: 0x84, hi: 0x85}, + {value: 0x0049, lo: 0x87, hi: 0x8a}, + {value: 0x0055, lo: 0x8d, hi: 0x94}, + {value: 0x0067, lo: 0x96, hi: 0x9c}, + {value: 0x0083, lo: 0x9e, hi: 0xb7}, + {value: 0x0043, lo: 0xb8, hi: 0xb9}, + {value: 0x0049, lo: 0xbb, hi: 0xbe}, + // Block 0x94, offset 0x2f7 + {value: 0x0002, lo: 0x05}, + {value: 0x0053, lo: 0x80, hi: 0x84}, + {value: 0x005f, lo: 0x86, hi: 0x86}, + {value: 0x0067, lo: 0x8a, hi: 0x90}, + {value: 0x0083, lo: 0x92, hi: 0xab}, + {value: 0x0043, lo: 0xac, hi: 0xbf}, + // Block 0x95, offset 0x2fd + {value: 0x0002, lo: 0x04}, + {value: 0x006b, lo: 0x80, hi: 0x85}, + {value: 0x0083, lo: 0x86, hi: 0x9f}, + {value: 0x0043, lo: 0xa0, hi: 0xb9}, + {value: 0x0083, lo: 0xba, hi: 0xbf}, + // Block 0x96, offset 0x302 + {value: 0x0002, lo: 0x03}, + {value: 0x008f, lo: 0x80, hi: 0x93}, + {value: 0x0043, lo: 0x94, hi: 0xad}, + {value: 0x0083, lo: 0xae, hi: 0xbf}, + // Block 0x97, offset 0x306 + {value: 0x0002, lo: 0x04}, + {value: 0x00a7, lo: 0x80, hi: 0x87}, + {value: 0x0043, lo: 0x88, hi: 0xa1}, + {value: 0x0083, lo: 0xa2, hi: 0xbb}, + {value: 0x0043, lo: 0xbc, hi: 0xbf}, + // Block 0x98, offset 0x30b + {value: 0x0002, lo: 0x03}, + {value: 0x004b, lo: 0x80, hi: 0x95}, + {value: 0x0083, lo: 0x96, hi: 0xaf}, + {value: 0x0043, lo: 0xb0, hi: 0xbf}, + // Block 0x99, offset 0x30f + {value: 0x0003, lo: 0x0f}, + {value: 0x01b8, lo: 0x80, hi: 0x80}, + {value: 0x045f, lo: 0x81, hi: 0x81}, + {value: 0x01bb, lo: 0x82, hi: 0x9a}, + {value: 0x045b, lo: 0x9b, hi: 0x9b}, + {value: 0x01c7, lo: 0x9c, hi: 0x9c}, + {value: 0x01d0, lo: 0x9d, hi: 0x9d}, + {value: 0x01d6, lo: 0x9e, hi: 0x9e}, + {value: 0x01fa, lo: 0x9f, hi: 0x9f}, + {value: 0x01eb, lo: 0xa0, hi: 0xa0}, + {value: 0x01e8, lo: 0xa1, hi: 0xa1}, + {value: 0x0173, lo: 0xa2, hi: 0xb2}, + {value: 0x0188, lo: 0xb3, hi: 0xb3}, + {value: 0x01a6, lo: 0xb4, hi: 0xba}, + {value: 0x045f, lo: 0xbb, hi: 0xbb}, + {value: 0x01bb, lo: 0xbc, hi: 0xbf}, + // Block 0x9a, offset 0x31f + {value: 0x0003, lo: 0x0d}, + {value: 0x01c7, lo: 0x80, hi: 0x94}, + {value: 0x045b, lo: 0x95, hi: 0x95}, + {value: 0x01c7, lo: 0x96, hi: 0x96}, + {value: 0x01d0, lo: 0x97, hi: 0x97}, + {value: 0x01d6, lo: 0x98, hi: 0x98}, + {value: 0x01fa, lo: 0x99, hi: 0x99}, + {value: 0x01eb, lo: 0x9a, hi: 0x9a}, + {value: 0x01e8, lo: 0x9b, hi: 0x9b}, + {value: 0x0173, lo: 0x9c, hi: 0xac}, + {value: 0x0188, lo: 0xad, hi: 0xad}, + {value: 0x01a6, lo: 0xae, hi: 0xb4}, + {value: 0x045f, lo: 0xb5, hi: 0xb5}, + {value: 0x01bb, lo: 0xb6, hi: 0xbf}, + // Block 0x9b, offset 0x32d + {value: 0x0003, lo: 0x0d}, + {value: 0x01d9, lo: 0x80, hi: 0x8e}, + {value: 0x045b, lo: 0x8f, hi: 0x8f}, + {value: 0x01c7, lo: 0x90, hi: 0x90}, + {value: 0x01d0, lo: 0x91, hi: 0x91}, + {value: 0x01d6, lo: 0x92, hi: 0x92}, + {value: 0x01fa, lo: 0x93, hi: 0x93}, + {value: 0x01eb, lo: 0x94, hi: 0x94}, + {value: 0x01e8, lo: 0x95, hi: 0x95}, + {value: 0x0173, lo: 0x96, hi: 0xa6}, + {value: 0x0188, lo: 0xa7, hi: 0xa7}, + {value: 0x01a6, lo: 0xa8, hi: 0xae}, + {value: 0x045f, lo: 0xaf, hi: 0xaf}, + {value: 0x01bb, lo: 0xb0, hi: 0xbf}, + // Block 0x9c, offset 0x33b + {value: 0x0003, lo: 0x0d}, + {value: 0x01eb, lo: 0x80, hi: 0x88}, + {value: 0x045b, lo: 0x89, hi: 0x89}, + {value: 0x01c7, lo: 0x8a, hi: 0x8a}, + {value: 0x01d0, lo: 0x8b, hi: 0x8b}, + {value: 0x01d6, lo: 0x8c, hi: 0x8c}, + {value: 0x01fa, lo: 0x8d, hi: 0x8d}, + {value: 0x01eb, lo: 0x8e, hi: 0x8e}, + {value: 0x01e8, lo: 0x8f, hi: 0x8f}, + {value: 0x0173, lo: 0x90, hi: 0xa0}, + {value: 0x0188, lo: 0xa1, hi: 0xa1}, + {value: 0x01a6, lo: 0xa2, hi: 0xa8}, + {value: 0x045f, lo: 0xa9, hi: 0xa9}, + {value: 0x01bb, lo: 0xaa, hi: 0xbf}, + // Block 0x9d, offset 0x349 + {value: 0x0000, lo: 0x05}, + {value: 0x8132, lo: 0x80, hi: 0x86}, + {value: 0x8132, lo: 0x88, hi: 0x98}, + {value: 0x8132, lo: 0x9b, hi: 0xa1}, + {value: 0x8132, lo: 0xa3, hi: 0xa4}, + {value: 0x8132, lo: 0xa6, hi: 0xaa}, + // Block 0x9e, offset 0x34f + {value: 0x0000, lo: 0x01}, + {value: 0x8132, lo: 0xac, hi: 0xaf}, + // Block 0x9f, offset 0x351 + {value: 0x0000, lo: 0x01}, + {value: 0x812d, lo: 0x90, hi: 0x96}, + // Block 0xa0, offset 0x353 + {value: 0x0000, lo: 0x02}, + {value: 0x8132, lo: 0x84, hi: 0x89}, + {value: 0x8102, lo: 0x8a, hi: 0x8a}, + // Block 0xa1, offset 0x356 + {value: 0x0002, lo: 0x0a}, + {value: 0x0063, lo: 0x80, hi: 0x89}, + {value: 0x1951, lo: 0x8a, hi: 0x8a}, + {value: 0x1984, lo: 0x8b, hi: 0x8b}, + {value: 0x199f, lo: 0x8c, hi: 0x8c}, + {value: 0x19a5, lo: 0x8d, hi: 0x8d}, + {value: 0x1bc3, lo: 0x8e, hi: 0x8e}, + {value: 0x19b1, lo: 0x8f, hi: 0x8f}, + {value: 0x197b, lo: 0xaa, hi: 0xaa}, + {value: 0x197e, lo: 0xab, hi: 0xab}, + {value: 0x1981, lo: 0xac, hi: 0xac}, + // Block 0xa2, offset 0x361 + {value: 0x0000, lo: 0x01}, + {value: 0x193f, lo: 0x90, hi: 0x90}, + // Block 0xa3, offset 0x363 + {value: 0x0028, lo: 0x09}, + {value: 0x2865, lo: 0x80, hi: 0x80}, + {value: 0x2829, lo: 0x81, hi: 0x81}, + {value: 0x2833, lo: 0x82, hi: 0x82}, + {value: 0x2847, lo: 0x83, hi: 0x84}, + {value: 0x2851, lo: 0x85, hi: 0x86}, + {value: 0x283d, lo: 0x87, hi: 0x87}, + {value: 0x285b, lo: 0x88, hi: 0x88}, + {value: 0x0b6f, lo: 0x90, hi: 0x90}, + {value: 0x08e7, lo: 0x91, hi: 0x91}, +} + +// recompMap: 7520 bytes (entries only) +var recompMap map[uint32]rune +var recompMapOnce sync.Once + +const recompMapPacked = "" + + "\x00A\x03\x00\x00\x00\x00\xc0" + // 0x00410300: 0x000000C0 + "\x00A\x03\x01\x00\x00\x00\xc1" + // 0x00410301: 0x000000C1 + "\x00A\x03\x02\x00\x00\x00\xc2" + // 0x00410302: 0x000000C2 + "\x00A\x03\x03\x00\x00\x00\xc3" + // 0x00410303: 0x000000C3 + "\x00A\x03\b\x00\x00\x00\xc4" + // 0x00410308: 0x000000C4 + "\x00A\x03\n\x00\x00\x00\xc5" + // 0x0041030A: 0x000000C5 + "\x00C\x03'\x00\x00\x00\xc7" + // 0x00430327: 0x000000C7 + "\x00E\x03\x00\x00\x00\x00\xc8" + // 0x00450300: 0x000000C8 + "\x00E\x03\x01\x00\x00\x00\xc9" + // 0x00450301: 0x000000C9 + "\x00E\x03\x02\x00\x00\x00\xca" + // 0x00450302: 0x000000CA + "\x00E\x03\b\x00\x00\x00\xcb" + // 0x00450308: 0x000000CB + "\x00I\x03\x00\x00\x00\x00\xcc" + // 0x00490300: 0x000000CC + "\x00I\x03\x01\x00\x00\x00\xcd" + // 0x00490301: 0x000000CD + "\x00I\x03\x02\x00\x00\x00\xce" + // 0x00490302: 0x000000CE + "\x00I\x03\b\x00\x00\x00\xcf" + // 0x00490308: 0x000000CF + "\x00N\x03\x03\x00\x00\x00\xd1" + // 0x004E0303: 0x000000D1 + "\x00O\x03\x00\x00\x00\x00\xd2" + // 0x004F0300: 0x000000D2 + "\x00O\x03\x01\x00\x00\x00\xd3" + // 0x004F0301: 0x000000D3 + "\x00O\x03\x02\x00\x00\x00\xd4" + // 0x004F0302: 0x000000D4 + "\x00O\x03\x03\x00\x00\x00\xd5" + // 0x004F0303: 0x000000D5 + "\x00O\x03\b\x00\x00\x00\xd6" + // 0x004F0308: 0x000000D6 + "\x00U\x03\x00\x00\x00\x00\xd9" + // 0x00550300: 0x000000D9 + "\x00U\x03\x01\x00\x00\x00\xda" + // 0x00550301: 0x000000DA + "\x00U\x03\x02\x00\x00\x00\xdb" + // 0x00550302: 0x000000DB + "\x00U\x03\b\x00\x00\x00\xdc" + // 0x00550308: 0x000000DC + "\x00Y\x03\x01\x00\x00\x00\xdd" + // 0x00590301: 0x000000DD + "\x00a\x03\x00\x00\x00\x00\xe0" + // 0x00610300: 0x000000E0 + "\x00a\x03\x01\x00\x00\x00\xe1" + // 0x00610301: 0x000000E1 + "\x00a\x03\x02\x00\x00\x00\xe2" + // 0x00610302: 0x000000E2 + "\x00a\x03\x03\x00\x00\x00\xe3" + // 0x00610303: 0x000000E3 + "\x00a\x03\b\x00\x00\x00\xe4" + // 0x00610308: 0x000000E4 + "\x00a\x03\n\x00\x00\x00\xe5" + // 0x0061030A: 0x000000E5 + "\x00c\x03'\x00\x00\x00\xe7" + // 0x00630327: 0x000000E7 + "\x00e\x03\x00\x00\x00\x00\xe8" + // 0x00650300: 0x000000E8 + "\x00e\x03\x01\x00\x00\x00\xe9" + // 0x00650301: 0x000000E9 + "\x00e\x03\x02\x00\x00\x00\xea" + // 0x00650302: 0x000000EA + "\x00e\x03\b\x00\x00\x00\xeb" + // 0x00650308: 0x000000EB + "\x00i\x03\x00\x00\x00\x00\xec" + // 0x00690300: 0x000000EC + "\x00i\x03\x01\x00\x00\x00\xed" + // 0x00690301: 0x000000ED + "\x00i\x03\x02\x00\x00\x00\xee" + // 0x00690302: 0x000000EE + "\x00i\x03\b\x00\x00\x00\xef" + // 0x00690308: 0x000000EF + "\x00n\x03\x03\x00\x00\x00\xf1" + // 0x006E0303: 0x000000F1 + "\x00o\x03\x00\x00\x00\x00\xf2" + // 0x006F0300: 0x000000F2 + "\x00o\x03\x01\x00\x00\x00\xf3" + // 0x006F0301: 0x000000F3 + "\x00o\x03\x02\x00\x00\x00\xf4" + // 0x006F0302: 0x000000F4 + "\x00o\x03\x03\x00\x00\x00\xf5" + // 0x006F0303: 0x000000F5 + "\x00o\x03\b\x00\x00\x00\xf6" + // 0x006F0308: 0x000000F6 + "\x00u\x03\x00\x00\x00\x00\xf9" + // 0x00750300: 0x000000F9 + "\x00u\x03\x01\x00\x00\x00\xfa" + // 0x00750301: 0x000000FA + "\x00u\x03\x02\x00\x00\x00\xfb" + // 0x00750302: 0x000000FB + "\x00u\x03\b\x00\x00\x00\xfc" + // 0x00750308: 0x000000FC + "\x00y\x03\x01\x00\x00\x00\xfd" + // 0x00790301: 0x000000FD + "\x00y\x03\b\x00\x00\x00\xff" + // 0x00790308: 0x000000FF + "\x00A\x03\x04\x00\x00\x01\x00" + // 0x00410304: 0x00000100 + "\x00a\x03\x04\x00\x00\x01\x01" + // 0x00610304: 0x00000101 + "\x00A\x03\x06\x00\x00\x01\x02" + // 0x00410306: 0x00000102 + "\x00a\x03\x06\x00\x00\x01\x03" + // 0x00610306: 0x00000103 + "\x00A\x03(\x00\x00\x01\x04" + // 0x00410328: 0x00000104 + "\x00a\x03(\x00\x00\x01\x05" + // 0x00610328: 0x00000105 + "\x00C\x03\x01\x00\x00\x01\x06" + // 0x00430301: 0x00000106 + "\x00c\x03\x01\x00\x00\x01\a" + // 0x00630301: 0x00000107 + "\x00C\x03\x02\x00\x00\x01\b" + // 0x00430302: 0x00000108 + "\x00c\x03\x02\x00\x00\x01\t" + // 0x00630302: 0x00000109 + "\x00C\x03\a\x00\x00\x01\n" + // 0x00430307: 0x0000010A + "\x00c\x03\a\x00\x00\x01\v" + // 0x00630307: 0x0000010B + "\x00C\x03\f\x00\x00\x01\f" + // 0x0043030C: 0x0000010C + "\x00c\x03\f\x00\x00\x01\r" + // 0x0063030C: 0x0000010D + "\x00D\x03\f\x00\x00\x01\x0e" + // 0x0044030C: 0x0000010E + "\x00d\x03\f\x00\x00\x01\x0f" + // 0x0064030C: 0x0000010F + "\x00E\x03\x04\x00\x00\x01\x12" + // 0x00450304: 0x00000112 + "\x00e\x03\x04\x00\x00\x01\x13" + // 0x00650304: 0x00000113 + "\x00E\x03\x06\x00\x00\x01\x14" + // 0x00450306: 0x00000114 + "\x00e\x03\x06\x00\x00\x01\x15" + // 0x00650306: 0x00000115 + "\x00E\x03\a\x00\x00\x01\x16" + // 0x00450307: 0x00000116 + "\x00e\x03\a\x00\x00\x01\x17" + // 0x00650307: 0x00000117 + "\x00E\x03(\x00\x00\x01\x18" + // 0x00450328: 0x00000118 + "\x00e\x03(\x00\x00\x01\x19" + // 0x00650328: 0x00000119 + "\x00E\x03\f\x00\x00\x01\x1a" + // 0x0045030C: 0x0000011A + "\x00e\x03\f\x00\x00\x01\x1b" + // 0x0065030C: 0x0000011B + "\x00G\x03\x02\x00\x00\x01\x1c" + // 0x00470302: 0x0000011C + "\x00g\x03\x02\x00\x00\x01\x1d" + // 0x00670302: 0x0000011D + "\x00G\x03\x06\x00\x00\x01\x1e" + // 0x00470306: 0x0000011E + "\x00g\x03\x06\x00\x00\x01\x1f" + // 0x00670306: 0x0000011F + "\x00G\x03\a\x00\x00\x01 " + // 0x00470307: 0x00000120 + "\x00g\x03\a\x00\x00\x01!" + // 0x00670307: 0x00000121 + "\x00G\x03'\x00\x00\x01\"" + // 0x00470327: 0x00000122 + "\x00g\x03'\x00\x00\x01#" + // 0x00670327: 0x00000123 + "\x00H\x03\x02\x00\x00\x01$" + // 0x00480302: 0x00000124 + "\x00h\x03\x02\x00\x00\x01%" + // 0x00680302: 0x00000125 + "\x00I\x03\x03\x00\x00\x01(" + // 0x00490303: 0x00000128 + "\x00i\x03\x03\x00\x00\x01)" + // 0x00690303: 0x00000129 + "\x00I\x03\x04\x00\x00\x01*" + // 0x00490304: 0x0000012A + "\x00i\x03\x04\x00\x00\x01+" + // 0x00690304: 0x0000012B + "\x00I\x03\x06\x00\x00\x01," + // 0x00490306: 0x0000012C + "\x00i\x03\x06\x00\x00\x01-" + // 0x00690306: 0x0000012D + "\x00I\x03(\x00\x00\x01." + // 0x00490328: 0x0000012E + "\x00i\x03(\x00\x00\x01/" + // 0x00690328: 0x0000012F + "\x00I\x03\a\x00\x00\x010" + // 0x00490307: 0x00000130 + "\x00J\x03\x02\x00\x00\x014" + // 0x004A0302: 0x00000134 + "\x00j\x03\x02\x00\x00\x015" + // 0x006A0302: 0x00000135 + "\x00K\x03'\x00\x00\x016" + // 0x004B0327: 0x00000136 + "\x00k\x03'\x00\x00\x017" + // 0x006B0327: 0x00000137 + "\x00L\x03\x01\x00\x00\x019" + // 0x004C0301: 0x00000139 + "\x00l\x03\x01\x00\x00\x01:" + // 0x006C0301: 0x0000013A + "\x00L\x03'\x00\x00\x01;" + // 0x004C0327: 0x0000013B + "\x00l\x03'\x00\x00\x01<" + // 0x006C0327: 0x0000013C + "\x00L\x03\f\x00\x00\x01=" + // 0x004C030C: 0x0000013D + "\x00l\x03\f\x00\x00\x01>" + // 0x006C030C: 0x0000013E + "\x00N\x03\x01\x00\x00\x01C" + // 0x004E0301: 0x00000143 + "\x00n\x03\x01\x00\x00\x01D" + // 0x006E0301: 0x00000144 + "\x00N\x03'\x00\x00\x01E" + // 0x004E0327: 0x00000145 + "\x00n\x03'\x00\x00\x01F" + // 0x006E0327: 0x00000146 + "\x00N\x03\f\x00\x00\x01G" + // 0x004E030C: 0x00000147 + "\x00n\x03\f\x00\x00\x01H" + // 0x006E030C: 0x00000148 + "\x00O\x03\x04\x00\x00\x01L" + // 0x004F0304: 0x0000014C + "\x00o\x03\x04\x00\x00\x01M" + // 0x006F0304: 0x0000014D + "\x00O\x03\x06\x00\x00\x01N" + // 0x004F0306: 0x0000014E + "\x00o\x03\x06\x00\x00\x01O" + // 0x006F0306: 0x0000014F + "\x00O\x03\v\x00\x00\x01P" + // 0x004F030B: 0x00000150 + "\x00o\x03\v\x00\x00\x01Q" + // 0x006F030B: 0x00000151 + "\x00R\x03\x01\x00\x00\x01T" + // 0x00520301: 0x00000154 + "\x00r\x03\x01\x00\x00\x01U" + // 0x00720301: 0x00000155 + "\x00R\x03'\x00\x00\x01V" + // 0x00520327: 0x00000156 + "\x00r\x03'\x00\x00\x01W" + // 0x00720327: 0x00000157 + "\x00R\x03\f\x00\x00\x01X" + // 0x0052030C: 0x00000158 + "\x00r\x03\f\x00\x00\x01Y" + // 0x0072030C: 0x00000159 + "\x00S\x03\x01\x00\x00\x01Z" + // 0x00530301: 0x0000015A + "\x00s\x03\x01\x00\x00\x01[" + // 0x00730301: 0x0000015B + "\x00S\x03\x02\x00\x00\x01\\" + // 0x00530302: 0x0000015C + "\x00s\x03\x02\x00\x00\x01]" + // 0x00730302: 0x0000015D + "\x00S\x03'\x00\x00\x01^" + // 0x00530327: 0x0000015E + "\x00s\x03'\x00\x00\x01_" + // 0x00730327: 0x0000015F + "\x00S\x03\f\x00\x00\x01`" + // 0x0053030C: 0x00000160 + "\x00s\x03\f\x00\x00\x01a" + // 0x0073030C: 0x00000161 + "\x00T\x03'\x00\x00\x01b" + // 0x00540327: 0x00000162 + "\x00t\x03'\x00\x00\x01c" + // 0x00740327: 0x00000163 + "\x00T\x03\f\x00\x00\x01d" + // 0x0054030C: 0x00000164 + "\x00t\x03\f\x00\x00\x01e" + // 0x0074030C: 0x00000165 + "\x00U\x03\x03\x00\x00\x01h" + // 0x00550303: 0x00000168 + "\x00u\x03\x03\x00\x00\x01i" + // 0x00750303: 0x00000169 + "\x00U\x03\x04\x00\x00\x01j" + // 0x00550304: 0x0000016A + "\x00u\x03\x04\x00\x00\x01k" + // 0x00750304: 0x0000016B + "\x00U\x03\x06\x00\x00\x01l" + // 0x00550306: 0x0000016C + "\x00u\x03\x06\x00\x00\x01m" + // 0x00750306: 0x0000016D + "\x00U\x03\n\x00\x00\x01n" + // 0x0055030A: 0x0000016E + "\x00u\x03\n\x00\x00\x01o" + // 0x0075030A: 0x0000016F + "\x00U\x03\v\x00\x00\x01p" + // 0x0055030B: 0x00000170 + "\x00u\x03\v\x00\x00\x01q" + // 0x0075030B: 0x00000171 + "\x00U\x03(\x00\x00\x01r" + // 0x00550328: 0x00000172 + "\x00u\x03(\x00\x00\x01s" + // 0x00750328: 0x00000173 + "\x00W\x03\x02\x00\x00\x01t" + // 0x00570302: 0x00000174 + "\x00w\x03\x02\x00\x00\x01u" + // 0x00770302: 0x00000175 + "\x00Y\x03\x02\x00\x00\x01v" + // 0x00590302: 0x00000176 + "\x00y\x03\x02\x00\x00\x01w" + // 0x00790302: 0x00000177 + "\x00Y\x03\b\x00\x00\x01x" + // 0x00590308: 0x00000178 + "\x00Z\x03\x01\x00\x00\x01y" + // 0x005A0301: 0x00000179 + "\x00z\x03\x01\x00\x00\x01z" + // 0x007A0301: 0x0000017A + "\x00Z\x03\a\x00\x00\x01{" + // 0x005A0307: 0x0000017B + "\x00z\x03\a\x00\x00\x01|" + // 0x007A0307: 0x0000017C + "\x00Z\x03\f\x00\x00\x01}" + // 0x005A030C: 0x0000017D + "\x00z\x03\f\x00\x00\x01~" + // 0x007A030C: 0x0000017E + "\x00O\x03\x1b\x00\x00\x01\xa0" + // 0x004F031B: 0x000001A0 + "\x00o\x03\x1b\x00\x00\x01\xa1" + // 0x006F031B: 0x000001A1 + "\x00U\x03\x1b\x00\x00\x01\xaf" + // 0x0055031B: 0x000001AF + "\x00u\x03\x1b\x00\x00\x01\xb0" + // 0x0075031B: 0x000001B0 + "\x00A\x03\f\x00\x00\x01\xcd" + // 0x0041030C: 0x000001CD + "\x00a\x03\f\x00\x00\x01\xce" + // 0x0061030C: 0x000001CE + "\x00I\x03\f\x00\x00\x01\xcf" + // 0x0049030C: 0x000001CF + "\x00i\x03\f\x00\x00\x01\xd0" + // 0x0069030C: 0x000001D0 + "\x00O\x03\f\x00\x00\x01\xd1" + // 0x004F030C: 0x000001D1 + "\x00o\x03\f\x00\x00\x01\xd2" + // 0x006F030C: 0x000001D2 + "\x00U\x03\f\x00\x00\x01\xd3" + // 0x0055030C: 0x000001D3 + "\x00u\x03\f\x00\x00\x01\xd4" + // 0x0075030C: 0x000001D4 + "\x00\xdc\x03\x04\x00\x00\x01\xd5" + // 0x00DC0304: 0x000001D5 + "\x00\xfc\x03\x04\x00\x00\x01\xd6" + // 0x00FC0304: 0x000001D6 + "\x00\xdc\x03\x01\x00\x00\x01\xd7" + // 0x00DC0301: 0x000001D7 + "\x00\xfc\x03\x01\x00\x00\x01\xd8" + // 0x00FC0301: 0x000001D8 + "\x00\xdc\x03\f\x00\x00\x01\xd9" + // 0x00DC030C: 0x000001D9 + "\x00\xfc\x03\f\x00\x00\x01\xda" + // 0x00FC030C: 0x000001DA + "\x00\xdc\x03\x00\x00\x00\x01\xdb" + // 0x00DC0300: 0x000001DB + "\x00\xfc\x03\x00\x00\x00\x01\xdc" + // 0x00FC0300: 0x000001DC + "\x00\xc4\x03\x04\x00\x00\x01\xde" + // 0x00C40304: 0x000001DE + "\x00\xe4\x03\x04\x00\x00\x01\xdf" + // 0x00E40304: 0x000001DF + "\x02&\x03\x04\x00\x00\x01\xe0" + // 0x02260304: 0x000001E0 + "\x02'\x03\x04\x00\x00\x01\xe1" + // 0x02270304: 0x000001E1 + "\x00\xc6\x03\x04\x00\x00\x01\xe2" + // 0x00C60304: 0x000001E2 + "\x00\xe6\x03\x04\x00\x00\x01\xe3" + // 0x00E60304: 0x000001E3 + "\x00G\x03\f\x00\x00\x01\xe6" + // 0x0047030C: 0x000001E6 + "\x00g\x03\f\x00\x00\x01\xe7" + // 0x0067030C: 0x000001E7 + "\x00K\x03\f\x00\x00\x01\xe8" + // 0x004B030C: 0x000001E8 + "\x00k\x03\f\x00\x00\x01\xe9" + // 0x006B030C: 0x000001E9 + "\x00O\x03(\x00\x00\x01\xea" + // 0x004F0328: 0x000001EA + "\x00o\x03(\x00\x00\x01\xeb" + // 0x006F0328: 0x000001EB + "\x01\xea\x03\x04\x00\x00\x01\xec" + // 0x01EA0304: 0x000001EC + "\x01\xeb\x03\x04\x00\x00\x01\xed" + // 0x01EB0304: 0x000001ED + "\x01\xb7\x03\f\x00\x00\x01\xee" + // 0x01B7030C: 0x000001EE + "\x02\x92\x03\f\x00\x00\x01\xef" + // 0x0292030C: 0x000001EF + "\x00j\x03\f\x00\x00\x01\xf0" + // 0x006A030C: 0x000001F0 + "\x00G\x03\x01\x00\x00\x01\xf4" + // 0x00470301: 0x000001F4 + "\x00g\x03\x01\x00\x00\x01\xf5" + // 0x00670301: 0x000001F5 + "\x00N\x03\x00\x00\x00\x01\xf8" + // 0x004E0300: 0x000001F8 + "\x00n\x03\x00\x00\x00\x01\xf9" + // 0x006E0300: 0x000001F9 + "\x00\xc5\x03\x01\x00\x00\x01\xfa" + // 0x00C50301: 0x000001FA + "\x00\xe5\x03\x01\x00\x00\x01\xfb" + // 0x00E50301: 0x000001FB + "\x00\xc6\x03\x01\x00\x00\x01\xfc" + // 0x00C60301: 0x000001FC + "\x00\xe6\x03\x01\x00\x00\x01\xfd" + // 0x00E60301: 0x000001FD + "\x00\xd8\x03\x01\x00\x00\x01\xfe" + // 0x00D80301: 0x000001FE + "\x00\xf8\x03\x01\x00\x00\x01\xff" + // 0x00F80301: 0x000001FF + "\x00A\x03\x0f\x00\x00\x02\x00" + // 0x0041030F: 0x00000200 + "\x00a\x03\x0f\x00\x00\x02\x01" + // 0x0061030F: 0x00000201 + "\x00A\x03\x11\x00\x00\x02\x02" + // 0x00410311: 0x00000202 + "\x00a\x03\x11\x00\x00\x02\x03" + // 0x00610311: 0x00000203 + "\x00E\x03\x0f\x00\x00\x02\x04" + // 0x0045030F: 0x00000204 + "\x00e\x03\x0f\x00\x00\x02\x05" + // 0x0065030F: 0x00000205 + "\x00E\x03\x11\x00\x00\x02\x06" + // 0x00450311: 0x00000206 + "\x00e\x03\x11\x00\x00\x02\a" + // 0x00650311: 0x00000207 + "\x00I\x03\x0f\x00\x00\x02\b" + // 0x0049030F: 0x00000208 + "\x00i\x03\x0f\x00\x00\x02\t" + // 0x0069030F: 0x00000209 + "\x00I\x03\x11\x00\x00\x02\n" + // 0x00490311: 0x0000020A + "\x00i\x03\x11\x00\x00\x02\v" + // 0x00690311: 0x0000020B + "\x00O\x03\x0f\x00\x00\x02\f" + // 0x004F030F: 0x0000020C + "\x00o\x03\x0f\x00\x00\x02\r" + // 0x006F030F: 0x0000020D + "\x00O\x03\x11\x00\x00\x02\x0e" + // 0x004F0311: 0x0000020E + "\x00o\x03\x11\x00\x00\x02\x0f" + // 0x006F0311: 0x0000020F + "\x00R\x03\x0f\x00\x00\x02\x10" + // 0x0052030F: 0x00000210 + "\x00r\x03\x0f\x00\x00\x02\x11" + // 0x0072030F: 0x00000211 + "\x00R\x03\x11\x00\x00\x02\x12" + // 0x00520311: 0x00000212 + "\x00r\x03\x11\x00\x00\x02\x13" + // 0x00720311: 0x00000213 + "\x00U\x03\x0f\x00\x00\x02\x14" + // 0x0055030F: 0x00000214 + "\x00u\x03\x0f\x00\x00\x02\x15" + // 0x0075030F: 0x00000215 + "\x00U\x03\x11\x00\x00\x02\x16" + // 0x00550311: 0x00000216 + "\x00u\x03\x11\x00\x00\x02\x17" + // 0x00750311: 0x00000217 + "\x00S\x03&\x00\x00\x02\x18" + // 0x00530326: 0x00000218 + "\x00s\x03&\x00\x00\x02\x19" + // 0x00730326: 0x00000219 + "\x00T\x03&\x00\x00\x02\x1a" + // 0x00540326: 0x0000021A + "\x00t\x03&\x00\x00\x02\x1b" + // 0x00740326: 0x0000021B + "\x00H\x03\f\x00\x00\x02\x1e" + // 0x0048030C: 0x0000021E + "\x00h\x03\f\x00\x00\x02\x1f" + // 0x0068030C: 0x0000021F + "\x00A\x03\a\x00\x00\x02&" + // 0x00410307: 0x00000226 + "\x00a\x03\a\x00\x00\x02'" + // 0x00610307: 0x00000227 + "\x00E\x03'\x00\x00\x02(" + // 0x00450327: 0x00000228 + "\x00e\x03'\x00\x00\x02)" + // 0x00650327: 0x00000229 + "\x00\xd6\x03\x04\x00\x00\x02*" + // 0x00D60304: 0x0000022A + "\x00\xf6\x03\x04\x00\x00\x02+" + // 0x00F60304: 0x0000022B + "\x00\xd5\x03\x04\x00\x00\x02," + // 0x00D50304: 0x0000022C + "\x00\xf5\x03\x04\x00\x00\x02-" + // 0x00F50304: 0x0000022D + "\x00O\x03\a\x00\x00\x02." + // 0x004F0307: 0x0000022E + "\x00o\x03\a\x00\x00\x02/" + // 0x006F0307: 0x0000022F + "\x02.\x03\x04\x00\x00\x020" + // 0x022E0304: 0x00000230 + "\x02/\x03\x04\x00\x00\x021" + // 0x022F0304: 0x00000231 + "\x00Y\x03\x04\x00\x00\x022" + // 0x00590304: 0x00000232 + "\x00y\x03\x04\x00\x00\x023" + // 0x00790304: 0x00000233 + "\x00\xa8\x03\x01\x00\x00\x03\x85" + // 0x00A80301: 0x00000385 + "\x03\x91\x03\x01\x00\x00\x03\x86" + // 0x03910301: 0x00000386 + "\x03\x95\x03\x01\x00\x00\x03\x88" + // 0x03950301: 0x00000388 + "\x03\x97\x03\x01\x00\x00\x03\x89" + // 0x03970301: 0x00000389 + "\x03\x99\x03\x01\x00\x00\x03\x8a" + // 0x03990301: 0x0000038A + "\x03\x9f\x03\x01\x00\x00\x03\x8c" + // 0x039F0301: 0x0000038C + "\x03\xa5\x03\x01\x00\x00\x03\x8e" + // 0x03A50301: 0x0000038E + "\x03\xa9\x03\x01\x00\x00\x03\x8f" + // 0x03A90301: 0x0000038F + "\x03\xca\x03\x01\x00\x00\x03\x90" + // 0x03CA0301: 0x00000390 + "\x03\x99\x03\b\x00\x00\x03\xaa" + // 0x03990308: 0x000003AA + "\x03\xa5\x03\b\x00\x00\x03\xab" + // 0x03A50308: 0x000003AB + "\x03\xb1\x03\x01\x00\x00\x03\xac" + // 0x03B10301: 0x000003AC + "\x03\xb5\x03\x01\x00\x00\x03\xad" + // 0x03B50301: 0x000003AD + "\x03\xb7\x03\x01\x00\x00\x03\xae" + // 0x03B70301: 0x000003AE + "\x03\xb9\x03\x01\x00\x00\x03\xaf" + // 0x03B90301: 0x000003AF + "\x03\xcb\x03\x01\x00\x00\x03\xb0" + // 0x03CB0301: 0x000003B0 + "\x03\xb9\x03\b\x00\x00\x03\xca" + // 0x03B90308: 0x000003CA + "\x03\xc5\x03\b\x00\x00\x03\xcb" + // 0x03C50308: 0x000003CB + "\x03\xbf\x03\x01\x00\x00\x03\xcc" + // 0x03BF0301: 0x000003CC + "\x03\xc5\x03\x01\x00\x00\x03\xcd" + // 0x03C50301: 0x000003CD + "\x03\xc9\x03\x01\x00\x00\x03\xce" + // 0x03C90301: 0x000003CE + "\x03\xd2\x03\x01\x00\x00\x03\xd3" + // 0x03D20301: 0x000003D3 + "\x03\xd2\x03\b\x00\x00\x03\xd4" + // 0x03D20308: 0x000003D4 + "\x04\x15\x03\x00\x00\x00\x04\x00" + // 0x04150300: 0x00000400 + "\x04\x15\x03\b\x00\x00\x04\x01" + // 0x04150308: 0x00000401 + "\x04\x13\x03\x01\x00\x00\x04\x03" + // 0x04130301: 0x00000403 + "\x04\x06\x03\b\x00\x00\x04\a" + // 0x04060308: 0x00000407 + "\x04\x1a\x03\x01\x00\x00\x04\f" + // 0x041A0301: 0x0000040C + "\x04\x18\x03\x00\x00\x00\x04\r" + // 0x04180300: 0x0000040D + "\x04#\x03\x06\x00\x00\x04\x0e" + // 0x04230306: 0x0000040E + "\x04\x18\x03\x06\x00\x00\x04\x19" + // 0x04180306: 0x00000419 + "\x048\x03\x06\x00\x00\x049" + // 0x04380306: 0x00000439 + "\x045\x03\x00\x00\x00\x04P" + // 0x04350300: 0x00000450 + "\x045\x03\b\x00\x00\x04Q" + // 0x04350308: 0x00000451 + "\x043\x03\x01\x00\x00\x04S" + // 0x04330301: 0x00000453 + "\x04V\x03\b\x00\x00\x04W" + // 0x04560308: 0x00000457 + "\x04:\x03\x01\x00\x00\x04\\" + // 0x043A0301: 0x0000045C + "\x048\x03\x00\x00\x00\x04]" + // 0x04380300: 0x0000045D + "\x04C\x03\x06\x00\x00\x04^" + // 0x04430306: 0x0000045E + "\x04t\x03\x0f\x00\x00\x04v" + // 0x0474030F: 0x00000476 + "\x04u\x03\x0f\x00\x00\x04w" + // 0x0475030F: 0x00000477 + "\x04\x16\x03\x06\x00\x00\x04\xc1" + // 0x04160306: 0x000004C1 + "\x046\x03\x06\x00\x00\x04\xc2" + // 0x04360306: 0x000004C2 + "\x04\x10\x03\x06\x00\x00\x04\xd0" + // 0x04100306: 0x000004D0 + "\x040\x03\x06\x00\x00\x04\xd1" + // 0x04300306: 0x000004D1 + "\x04\x10\x03\b\x00\x00\x04\xd2" + // 0x04100308: 0x000004D2 + "\x040\x03\b\x00\x00\x04\xd3" + // 0x04300308: 0x000004D3 + "\x04\x15\x03\x06\x00\x00\x04\xd6" + // 0x04150306: 0x000004D6 + "\x045\x03\x06\x00\x00\x04\xd7" + // 0x04350306: 0x000004D7 + "\x04\xd8\x03\b\x00\x00\x04\xda" + // 0x04D80308: 0x000004DA + "\x04\xd9\x03\b\x00\x00\x04\xdb" + // 0x04D90308: 0x000004DB + "\x04\x16\x03\b\x00\x00\x04\xdc" + // 0x04160308: 0x000004DC + "\x046\x03\b\x00\x00\x04\xdd" + // 0x04360308: 0x000004DD + "\x04\x17\x03\b\x00\x00\x04\xde" + // 0x04170308: 0x000004DE + "\x047\x03\b\x00\x00\x04\xdf" + // 0x04370308: 0x000004DF + "\x04\x18\x03\x04\x00\x00\x04\xe2" + // 0x04180304: 0x000004E2 + "\x048\x03\x04\x00\x00\x04\xe3" + // 0x04380304: 0x000004E3 + "\x04\x18\x03\b\x00\x00\x04\xe4" + // 0x04180308: 0x000004E4 + "\x048\x03\b\x00\x00\x04\xe5" + // 0x04380308: 0x000004E5 + "\x04\x1e\x03\b\x00\x00\x04\xe6" + // 0x041E0308: 0x000004E6 + "\x04>\x03\b\x00\x00\x04\xe7" + // 0x043E0308: 0x000004E7 + "\x04\xe8\x03\b\x00\x00\x04\xea" + // 0x04E80308: 0x000004EA + "\x04\xe9\x03\b\x00\x00\x04\xeb" + // 0x04E90308: 0x000004EB + "\x04-\x03\b\x00\x00\x04\xec" + // 0x042D0308: 0x000004EC + "\x04M\x03\b\x00\x00\x04\xed" + // 0x044D0308: 0x000004ED + "\x04#\x03\x04\x00\x00\x04\xee" + // 0x04230304: 0x000004EE + "\x04C\x03\x04\x00\x00\x04\xef" + // 0x04430304: 0x000004EF + "\x04#\x03\b\x00\x00\x04\xf0" + // 0x04230308: 0x000004F0 + "\x04C\x03\b\x00\x00\x04\xf1" + // 0x04430308: 0x000004F1 + "\x04#\x03\v\x00\x00\x04\xf2" + // 0x0423030B: 0x000004F2 + "\x04C\x03\v\x00\x00\x04\xf3" + // 0x0443030B: 0x000004F3 + "\x04'\x03\b\x00\x00\x04\xf4" + // 0x04270308: 0x000004F4 + "\x04G\x03\b\x00\x00\x04\xf5" + // 0x04470308: 0x000004F5 + "\x04+\x03\b\x00\x00\x04\xf8" + // 0x042B0308: 0x000004F8 + "\x04K\x03\b\x00\x00\x04\xf9" + // 0x044B0308: 0x000004F9 + "\x06'\x06S\x00\x00\x06\"" + // 0x06270653: 0x00000622 + "\x06'\x06T\x00\x00\x06#" + // 0x06270654: 0x00000623 + "\x06H\x06T\x00\x00\x06$" + // 0x06480654: 0x00000624 + "\x06'\x06U\x00\x00\x06%" + // 0x06270655: 0x00000625 + "\x06J\x06T\x00\x00\x06&" + // 0x064A0654: 0x00000626 + "\x06\xd5\x06T\x00\x00\x06\xc0" + // 0x06D50654: 0x000006C0 + "\x06\xc1\x06T\x00\x00\x06\xc2" + // 0x06C10654: 0x000006C2 + "\x06\xd2\x06T\x00\x00\x06\xd3" + // 0x06D20654: 0x000006D3 + "\t(\t<\x00\x00\t)" + // 0x0928093C: 0x00000929 + "\t0\t<\x00\x00\t1" + // 0x0930093C: 0x00000931 + "\t3\t<\x00\x00\t4" + // 0x0933093C: 0x00000934 + "\t\xc7\t\xbe\x00\x00\t\xcb" + // 0x09C709BE: 0x000009CB + "\t\xc7\t\xd7\x00\x00\t\xcc" + // 0x09C709D7: 0x000009CC + "\vG\vV\x00\x00\vH" + // 0x0B470B56: 0x00000B48 + "\vG\v>\x00\x00\vK" + // 0x0B470B3E: 0x00000B4B + "\vG\vW\x00\x00\vL" + // 0x0B470B57: 0x00000B4C + "\v\x92\v\xd7\x00\x00\v\x94" + // 0x0B920BD7: 0x00000B94 + "\v\xc6\v\xbe\x00\x00\v\xca" + // 0x0BC60BBE: 0x00000BCA + "\v\xc7\v\xbe\x00\x00\v\xcb" + // 0x0BC70BBE: 0x00000BCB + "\v\xc6\v\xd7\x00\x00\v\xcc" + // 0x0BC60BD7: 0x00000BCC + "\fF\fV\x00\x00\fH" + // 0x0C460C56: 0x00000C48 + "\f\xbf\f\xd5\x00\x00\f\xc0" + // 0x0CBF0CD5: 0x00000CC0 + "\f\xc6\f\xd5\x00\x00\f\xc7" + // 0x0CC60CD5: 0x00000CC7 + "\f\xc6\f\xd6\x00\x00\f\xc8" + // 0x0CC60CD6: 0x00000CC8 + "\f\xc6\f\xc2\x00\x00\f\xca" + // 0x0CC60CC2: 0x00000CCA + "\f\xca\f\xd5\x00\x00\f\xcb" + // 0x0CCA0CD5: 0x00000CCB + "\rF\r>\x00\x00\rJ" + // 0x0D460D3E: 0x00000D4A + "\rG\r>\x00\x00\rK" + // 0x0D470D3E: 0x00000D4B + "\rF\rW\x00\x00\rL" + // 0x0D460D57: 0x00000D4C + "\r\xd9\r\xca\x00\x00\r\xda" + // 0x0DD90DCA: 0x00000DDA + "\r\xd9\r\xcf\x00\x00\r\xdc" + // 0x0DD90DCF: 0x00000DDC + "\r\xdc\r\xca\x00\x00\r\xdd" + // 0x0DDC0DCA: 0x00000DDD + "\r\xd9\r\xdf\x00\x00\r\xde" + // 0x0DD90DDF: 0x00000DDE + "\x10%\x10.\x00\x00\x10&" + // 0x1025102E: 0x00001026 + "\x1b\x05\x1b5\x00\x00\x1b\x06" + // 0x1B051B35: 0x00001B06 + "\x1b\a\x1b5\x00\x00\x1b\b" + // 0x1B071B35: 0x00001B08 + "\x1b\t\x1b5\x00\x00\x1b\n" + // 0x1B091B35: 0x00001B0A + "\x1b\v\x1b5\x00\x00\x1b\f" + // 0x1B0B1B35: 0x00001B0C + "\x1b\r\x1b5\x00\x00\x1b\x0e" + // 0x1B0D1B35: 0x00001B0E + "\x1b\x11\x1b5\x00\x00\x1b\x12" + // 0x1B111B35: 0x00001B12 + "\x1b:\x1b5\x00\x00\x1b;" + // 0x1B3A1B35: 0x00001B3B + "\x1b<\x1b5\x00\x00\x1b=" + // 0x1B3C1B35: 0x00001B3D + "\x1b>\x1b5\x00\x00\x1b@" + // 0x1B3E1B35: 0x00001B40 + "\x1b?\x1b5\x00\x00\x1bA" + // 0x1B3F1B35: 0x00001B41 + "\x1bB\x1b5\x00\x00\x1bC" + // 0x1B421B35: 0x00001B43 + "\x00A\x03%\x00\x00\x1e\x00" + // 0x00410325: 0x00001E00 + "\x00a\x03%\x00\x00\x1e\x01" + // 0x00610325: 0x00001E01 + "\x00B\x03\a\x00\x00\x1e\x02" + // 0x00420307: 0x00001E02 + "\x00b\x03\a\x00\x00\x1e\x03" + // 0x00620307: 0x00001E03 + "\x00B\x03#\x00\x00\x1e\x04" + // 0x00420323: 0x00001E04 + "\x00b\x03#\x00\x00\x1e\x05" + // 0x00620323: 0x00001E05 + "\x00B\x031\x00\x00\x1e\x06" + // 0x00420331: 0x00001E06 + "\x00b\x031\x00\x00\x1e\a" + // 0x00620331: 0x00001E07 + "\x00\xc7\x03\x01\x00\x00\x1e\b" + // 0x00C70301: 0x00001E08 + "\x00\xe7\x03\x01\x00\x00\x1e\t" + // 0x00E70301: 0x00001E09 + "\x00D\x03\a\x00\x00\x1e\n" + // 0x00440307: 0x00001E0A + "\x00d\x03\a\x00\x00\x1e\v" + // 0x00640307: 0x00001E0B + "\x00D\x03#\x00\x00\x1e\f" + // 0x00440323: 0x00001E0C + "\x00d\x03#\x00\x00\x1e\r" + // 0x00640323: 0x00001E0D + "\x00D\x031\x00\x00\x1e\x0e" + // 0x00440331: 0x00001E0E + "\x00d\x031\x00\x00\x1e\x0f" + // 0x00640331: 0x00001E0F + "\x00D\x03'\x00\x00\x1e\x10" + // 0x00440327: 0x00001E10 + "\x00d\x03'\x00\x00\x1e\x11" + // 0x00640327: 0x00001E11 + "\x00D\x03-\x00\x00\x1e\x12" + // 0x0044032D: 0x00001E12 + "\x00d\x03-\x00\x00\x1e\x13" + // 0x0064032D: 0x00001E13 + "\x01\x12\x03\x00\x00\x00\x1e\x14" + // 0x01120300: 0x00001E14 + "\x01\x13\x03\x00\x00\x00\x1e\x15" + // 0x01130300: 0x00001E15 + "\x01\x12\x03\x01\x00\x00\x1e\x16" + // 0x01120301: 0x00001E16 + "\x01\x13\x03\x01\x00\x00\x1e\x17" + // 0x01130301: 0x00001E17 + "\x00E\x03-\x00\x00\x1e\x18" + // 0x0045032D: 0x00001E18 + "\x00e\x03-\x00\x00\x1e\x19" + // 0x0065032D: 0x00001E19 + "\x00E\x030\x00\x00\x1e\x1a" + // 0x00450330: 0x00001E1A + "\x00e\x030\x00\x00\x1e\x1b" + // 0x00650330: 0x00001E1B + "\x02(\x03\x06\x00\x00\x1e\x1c" + // 0x02280306: 0x00001E1C + "\x02)\x03\x06\x00\x00\x1e\x1d" + // 0x02290306: 0x00001E1D + "\x00F\x03\a\x00\x00\x1e\x1e" + // 0x00460307: 0x00001E1E + "\x00f\x03\a\x00\x00\x1e\x1f" + // 0x00660307: 0x00001E1F + "\x00G\x03\x04\x00\x00\x1e " + // 0x00470304: 0x00001E20 + "\x00g\x03\x04\x00\x00\x1e!" + // 0x00670304: 0x00001E21 + "\x00H\x03\a\x00\x00\x1e\"" + // 0x00480307: 0x00001E22 + "\x00h\x03\a\x00\x00\x1e#" + // 0x00680307: 0x00001E23 + "\x00H\x03#\x00\x00\x1e$" + // 0x00480323: 0x00001E24 + "\x00h\x03#\x00\x00\x1e%" + // 0x00680323: 0x00001E25 + "\x00H\x03\b\x00\x00\x1e&" + // 0x00480308: 0x00001E26 + "\x00h\x03\b\x00\x00\x1e'" + // 0x00680308: 0x00001E27 + "\x00H\x03'\x00\x00\x1e(" + // 0x00480327: 0x00001E28 + "\x00h\x03'\x00\x00\x1e)" + // 0x00680327: 0x00001E29 + "\x00H\x03.\x00\x00\x1e*" + // 0x0048032E: 0x00001E2A + "\x00h\x03.\x00\x00\x1e+" + // 0x0068032E: 0x00001E2B + "\x00I\x030\x00\x00\x1e," + // 0x00490330: 0x00001E2C + "\x00i\x030\x00\x00\x1e-" + // 0x00690330: 0x00001E2D + "\x00\xcf\x03\x01\x00\x00\x1e." + // 0x00CF0301: 0x00001E2E + "\x00\xef\x03\x01\x00\x00\x1e/" + // 0x00EF0301: 0x00001E2F + "\x00K\x03\x01\x00\x00\x1e0" + // 0x004B0301: 0x00001E30 + "\x00k\x03\x01\x00\x00\x1e1" + // 0x006B0301: 0x00001E31 + "\x00K\x03#\x00\x00\x1e2" + // 0x004B0323: 0x00001E32 + "\x00k\x03#\x00\x00\x1e3" + // 0x006B0323: 0x00001E33 + "\x00K\x031\x00\x00\x1e4" + // 0x004B0331: 0x00001E34 + "\x00k\x031\x00\x00\x1e5" + // 0x006B0331: 0x00001E35 + "\x00L\x03#\x00\x00\x1e6" + // 0x004C0323: 0x00001E36 + "\x00l\x03#\x00\x00\x1e7" + // 0x006C0323: 0x00001E37 + "\x1e6\x03\x04\x00\x00\x1e8" + // 0x1E360304: 0x00001E38 + "\x1e7\x03\x04\x00\x00\x1e9" + // 0x1E370304: 0x00001E39 + "\x00L\x031\x00\x00\x1e:" + // 0x004C0331: 0x00001E3A + "\x00l\x031\x00\x00\x1e;" + // 0x006C0331: 0x00001E3B + "\x00L\x03-\x00\x00\x1e<" + // 0x004C032D: 0x00001E3C + "\x00l\x03-\x00\x00\x1e=" + // 0x006C032D: 0x00001E3D + "\x00M\x03\x01\x00\x00\x1e>" + // 0x004D0301: 0x00001E3E + "\x00m\x03\x01\x00\x00\x1e?" + // 0x006D0301: 0x00001E3F + "\x00M\x03\a\x00\x00\x1e@" + // 0x004D0307: 0x00001E40 + "\x00m\x03\a\x00\x00\x1eA" + // 0x006D0307: 0x00001E41 + "\x00M\x03#\x00\x00\x1eB" + // 0x004D0323: 0x00001E42 + "\x00m\x03#\x00\x00\x1eC" + // 0x006D0323: 0x00001E43 + "\x00N\x03\a\x00\x00\x1eD" + // 0x004E0307: 0x00001E44 + "\x00n\x03\a\x00\x00\x1eE" + // 0x006E0307: 0x00001E45 + "\x00N\x03#\x00\x00\x1eF" + // 0x004E0323: 0x00001E46 + "\x00n\x03#\x00\x00\x1eG" + // 0x006E0323: 0x00001E47 + "\x00N\x031\x00\x00\x1eH" + // 0x004E0331: 0x00001E48 + "\x00n\x031\x00\x00\x1eI" + // 0x006E0331: 0x00001E49 + "\x00N\x03-\x00\x00\x1eJ" + // 0x004E032D: 0x00001E4A + "\x00n\x03-\x00\x00\x1eK" + // 0x006E032D: 0x00001E4B + "\x00\xd5\x03\x01\x00\x00\x1eL" + // 0x00D50301: 0x00001E4C + "\x00\xf5\x03\x01\x00\x00\x1eM" + // 0x00F50301: 0x00001E4D + "\x00\xd5\x03\b\x00\x00\x1eN" + // 0x00D50308: 0x00001E4E + "\x00\xf5\x03\b\x00\x00\x1eO" + // 0x00F50308: 0x00001E4F + "\x01L\x03\x00\x00\x00\x1eP" + // 0x014C0300: 0x00001E50 + "\x01M\x03\x00\x00\x00\x1eQ" + // 0x014D0300: 0x00001E51 + "\x01L\x03\x01\x00\x00\x1eR" + // 0x014C0301: 0x00001E52 + "\x01M\x03\x01\x00\x00\x1eS" + // 0x014D0301: 0x00001E53 + "\x00P\x03\x01\x00\x00\x1eT" + // 0x00500301: 0x00001E54 + "\x00p\x03\x01\x00\x00\x1eU" + // 0x00700301: 0x00001E55 + "\x00P\x03\a\x00\x00\x1eV" + // 0x00500307: 0x00001E56 + "\x00p\x03\a\x00\x00\x1eW" + // 0x00700307: 0x00001E57 + "\x00R\x03\a\x00\x00\x1eX" + // 0x00520307: 0x00001E58 + "\x00r\x03\a\x00\x00\x1eY" + // 0x00720307: 0x00001E59 + "\x00R\x03#\x00\x00\x1eZ" + // 0x00520323: 0x00001E5A + "\x00r\x03#\x00\x00\x1e[" + // 0x00720323: 0x00001E5B + "\x1eZ\x03\x04\x00\x00\x1e\\" + // 0x1E5A0304: 0x00001E5C + "\x1e[\x03\x04\x00\x00\x1e]" + // 0x1E5B0304: 0x00001E5D + "\x00R\x031\x00\x00\x1e^" + // 0x00520331: 0x00001E5E + "\x00r\x031\x00\x00\x1e_" + // 0x00720331: 0x00001E5F + "\x00S\x03\a\x00\x00\x1e`" + // 0x00530307: 0x00001E60 + "\x00s\x03\a\x00\x00\x1ea" + // 0x00730307: 0x00001E61 + "\x00S\x03#\x00\x00\x1eb" + // 0x00530323: 0x00001E62 + "\x00s\x03#\x00\x00\x1ec" + // 0x00730323: 0x00001E63 + "\x01Z\x03\a\x00\x00\x1ed" + // 0x015A0307: 0x00001E64 + "\x01[\x03\a\x00\x00\x1ee" + // 0x015B0307: 0x00001E65 + "\x01`\x03\a\x00\x00\x1ef" + // 0x01600307: 0x00001E66 + "\x01a\x03\a\x00\x00\x1eg" + // 0x01610307: 0x00001E67 + "\x1eb\x03\a\x00\x00\x1eh" + // 0x1E620307: 0x00001E68 + "\x1ec\x03\a\x00\x00\x1ei" + // 0x1E630307: 0x00001E69 + "\x00T\x03\a\x00\x00\x1ej" + // 0x00540307: 0x00001E6A + "\x00t\x03\a\x00\x00\x1ek" + // 0x00740307: 0x00001E6B + "\x00T\x03#\x00\x00\x1el" + // 0x00540323: 0x00001E6C + "\x00t\x03#\x00\x00\x1em" + // 0x00740323: 0x00001E6D + "\x00T\x031\x00\x00\x1en" + // 0x00540331: 0x00001E6E + "\x00t\x031\x00\x00\x1eo" + // 0x00740331: 0x00001E6F + "\x00T\x03-\x00\x00\x1ep" + // 0x0054032D: 0x00001E70 + "\x00t\x03-\x00\x00\x1eq" + // 0x0074032D: 0x00001E71 + "\x00U\x03$\x00\x00\x1er" + // 0x00550324: 0x00001E72 + "\x00u\x03$\x00\x00\x1es" + // 0x00750324: 0x00001E73 + "\x00U\x030\x00\x00\x1et" + // 0x00550330: 0x00001E74 + "\x00u\x030\x00\x00\x1eu" + // 0x00750330: 0x00001E75 + "\x00U\x03-\x00\x00\x1ev" + // 0x0055032D: 0x00001E76 + "\x00u\x03-\x00\x00\x1ew" + // 0x0075032D: 0x00001E77 + "\x01h\x03\x01\x00\x00\x1ex" + // 0x01680301: 0x00001E78 + "\x01i\x03\x01\x00\x00\x1ey" + // 0x01690301: 0x00001E79 + "\x01j\x03\b\x00\x00\x1ez" + // 0x016A0308: 0x00001E7A + "\x01k\x03\b\x00\x00\x1e{" + // 0x016B0308: 0x00001E7B + "\x00V\x03\x03\x00\x00\x1e|" + // 0x00560303: 0x00001E7C + "\x00v\x03\x03\x00\x00\x1e}" + // 0x00760303: 0x00001E7D + "\x00V\x03#\x00\x00\x1e~" + // 0x00560323: 0x00001E7E + "\x00v\x03#\x00\x00\x1e\u007f" + // 0x00760323: 0x00001E7F + "\x00W\x03\x00\x00\x00\x1e\x80" + // 0x00570300: 0x00001E80 + "\x00w\x03\x00\x00\x00\x1e\x81" + // 0x00770300: 0x00001E81 + "\x00W\x03\x01\x00\x00\x1e\x82" + // 0x00570301: 0x00001E82 + "\x00w\x03\x01\x00\x00\x1e\x83" + // 0x00770301: 0x00001E83 + "\x00W\x03\b\x00\x00\x1e\x84" + // 0x00570308: 0x00001E84 + "\x00w\x03\b\x00\x00\x1e\x85" + // 0x00770308: 0x00001E85 + "\x00W\x03\a\x00\x00\x1e\x86" + // 0x00570307: 0x00001E86 + "\x00w\x03\a\x00\x00\x1e\x87" + // 0x00770307: 0x00001E87 + "\x00W\x03#\x00\x00\x1e\x88" + // 0x00570323: 0x00001E88 + "\x00w\x03#\x00\x00\x1e\x89" + // 0x00770323: 0x00001E89 + "\x00X\x03\a\x00\x00\x1e\x8a" + // 0x00580307: 0x00001E8A + "\x00x\x03\a\x00\x00\x1e\x8b" + // 0x00780307: 0x00001E8B + "\x00X\x03\b\x00\x00\x1e\x8c" + // 0x00580308: 0x00001E8C + "\x00x\x03\b\x00\x00\x1e\x8d" + // 0x00780308: 0x00001E8D + "\x00Y\x03\a\x00\x00\x1e\x8e" + // 0x00590307: 0x00001E8E + "\x00y\x03\a\x00\x00\x1e\x8f" + // 0x00790307: 0x00001E8F + "\x00Z\x03\x02\x00\x00\x1e\x90" + // 0x005A0302: 0x00001E90 + "\x00z\x03\x02\x00\x00\x1e\x91" + // 0x007A0302: 0x00001E91 + "\x00Z\x03#\x00\x00\x1e\x92" + // 0x005A0323: 0x00001E92 + "\x00z\x03#\x00\x00\x1e\x93" + // 0x007A0323: 0x00001E93 + "\x00Z\x031\x00\x00\x1e\x94" + // 0x005A0331: 0x00001E94 + "\x00z\x031\x00\x00\x1e\x95" + // 0x007A0331: 0x00001E95 + "\x00h\x031\x00\x00\x1e\x96" + // 0x00680331: 0x00001E96 + "\x00t\x03\b\x00\x00\x1e\x97" + // 0x00740308: 0x00001E97 + "\x00w\x03\n\x00\x00\x1e\x98" + // 0x0077030A: 0x00001E98 + "\x00y\x03\n\x00\x00\x1e\x99" + // 0x0079030A: 0x00001E99 + "\x01\u007f\x03\a\x00\x00\x1e\x9b" + // 0x017F0307: 0x00001E9B + "\x00A\x03#\x00\x00\x1e\xa0" + // 0x00410323: 0x00001EA0 + "\x00a\x03#\x00\x00\x1e\xa1" + // 0x00610323: 0x00001EA1 + "\x00A\x03\t\x00\x00\x1e\xa2" + // 0x00410309: 0x00001EA2 + "\x00a\x03\t\x00\x00\x1e\xa3" + // 0x00610309: 0x00001EA3 + "\x00\xc2\x03\x01\x00\x00\x1e\xa4" + // 0x00C20301: 0x00001EA4 + "\x00\xe2\x03\x01\x00\x00\x1e\xa5" + // 0x00E20301: 0x00001EA5 + "\x00\xc2\x03\x00\x00\x00\x1e\xa6" + // 0x00C20300: 0x00001EA6 + "\x00\xe2\x03\x00\x00\x00\x1e\xa7" + // 0x00E20300: 0x00001EA7 + "\x00\xc2\x03\t\x00\x00\x1e\xa8" + // 0x00C20309: 0x00001EA8 + "\x00\xe2\x03\t\x00\x00\x1e\xa9" + // 0x00E20309: 0x00001EA9 + "\x00\xc2\x03\x03\x00\x00\x1e\xaa" + // 0x00C20303: 0x00001EAA + "\x00\xe2\x03\x03\x00\x00\x1e\xab" + // 0x00E20303: 0x00001EAB + "\x1e\xa0\x03\x02\x00\x00\x1e\xac" + // 0x1EA00302: 0x00001EAC + "\x1e\xa1\x03\x02\x00\x00\x1e\xad" + // 0x1EA10302: 0x00001EAD + "\x01\x02\x03\x01\x00\x00\x1e\xae" + // 0x01020301: 0x00001EAE + "\x01\x03\x03\x01\x00\x00\x1e\xaf" + // 0x01030301: 0x00001EAF + "\x01\x02\x03\x00\x00\x00\x1e\xb0" + // 0x01020300: 0x00001EB0 + "\x01\x03\x03\x00\x00\x00\x1e\xb1" + // 0x01030300: 0x00001EB1 + "\x01\x02\x03\t\x00\x00\x1e\xb2" + // 0x01020309: 0x00001EB2 + "\x01\x03\x03\t\x00\x00\x1e\xb3" + // 0x01030309: 0x00001EB3 + "\x01\x02\x03\x03\x00\x00\x1e\xb4" + // 0x01020303: 0x00001EB4 + "\x01\x03\x03\x03\x00\x00\x1e\xb5" + // 0x01030303: 0x00001EB5 + "\x1e\xa0\x03\x06\x00\x00\x1e\xb6" + // 0x1EA00306: 0x00001EB6 + "\x1e\xa1\x03\x06\x00\x00\x1e\xb7" + // 0x1EA10306: 0x00001EB7 + "\x00E\x03#\x00\x00\x1e\xb8" + // 0x00450323: 0x00001EB8 + "\x00e\x03#\x00\x00\x1e\xb9" + // 0x00650323: 0x00001EB9 + "\x00E\x03\t\x00\x00\x1e\xba" + // 0x00450309: 0x00001EBA + "\x00e\x03\t\x00\x00\x1e\xbb" + // 0x00650309: 0x00001EBB + "\x00E\x03\x03\x00\x00\x1e\xbc" + // 0x00450303: 0x00001EBC + "\x00e\x03\x03\x00\x00\x1e\xbd" + // 0x00650303: 0x00001EBD + "\x00\xca\x03\x01\x00\x00\x1e\xbe" + // 0x00CA0301: 0x00001EBE + "\x00\xea\x03\x01\x00\x00\x1e\xbf" + // 0x00EA0301: 0x00001EBF + "\x00\xca\x03\x00\x00\x00\x1e\xc0" + // 0x00CA0300: 0x00001EC0 + "\x00\xea\x03\x00\x00\x00\x1e\xc1" + // 0x00EA0300: 0x00001EC1 + "\x00\xca\x03\t\x00\x00\x1e\xc2" + // 0x00CA0309: 0x00001EC2 + "\x00\xea\x03\t\x00\x00\x1e\xc3" + // 0x00EA0309: 0x00001EC3 + "\x00\xca\x03\x03\x00\x00\x1e\xc4" + // 0x00CA0303: 0x00001EC4 + "\x00\xea\x03\x03\x00\x00\x1e\xc5" + // 0x00EA0303: 0x00001EC5 + "\x1e\xb8\x03\x02\x00\x00\x1e\xc6" + // 0x1EB80302: 0x00001EC6 + "\x1e\xb9\x03\x02\x00\x00\x1e\xc7" + // 0x1EB90302: 0x00001EC7 + "\x00I\x03\t\x00\x00\x1e\xc8" + // 0x00490309: 0x00001EC8 + "\x00i\x03\t\x00\x00\x1e\xc9" + // 0x00690309: 0x00001EC9 + "\x00I\x03#\x00\x00\x1e\xca" + // 0x00490323: 0x00001ECA + "\x00i\x03#\x00\x00\x1e\xcb" + // 0x00690323: 0x00001ECB + "\x00O\x03#\x00\x00\x1e\xcc" + // 0x004F0323: 0x00001ECC + "\x00o\x03#\x00\x00\x1e\xcd" + // 0x006F0323: 0x00001ECD + "\x00O\x03\t\x00\x00\x1e\xce" + // 0x004F0309: 0x00001ECE + "\x00o\x03\t\x00\x00\x1e\xcf" + // 0x006F0309: 0x00001ECF + "\x00\xd4\x03\x01\x00\x00\x1e\xd0" + // 0x00D40301: 0x00001ED0 + "\x00\xf4\x03\x01\x00\x00\x1e\xd1" + // 0x00F40301: 0x00001ED1 + "\x00\xd4\x03\x00\x00\x00\x1e\xd2" + // 0x00D40300: 0x00001ED2 + "\x00\xf4\x03\x00\x00\x00\x1e\xd3" + // 0x00F40300: 0x00001ED3 + "\x00\xd4\x03\t\x00\x00\x1e\xd4" + // 0x00D40309: 0x00001ED4 + "\x00\xf4\x03\t\x00\x00\x1e\xd5" + // 0x00F40309: 0x00001ED5 + "\x00\xd4\x03\x03\x00\x00\x1e\xd6" + // 0x00D40303: 0x00001ED6 + "\x00\xf4\x03\x03\x00\x00\x1e\xd7" + // 0x00F40303: 0x00001ED7 + "\x1e\xcc\x03\x02\x00\x00\x1e\xd8" + // 0x1ECC0302: 0x00001ED8 + "\x1e\xcd\x03\x02\x00\x00\x1e\xd9" + // 0x1ECD0302: 0x00001ED9 + "\x01\xa0\x03\x01\x00\x00\x1e\xda" + // 0x01A00301: 0x00001EDA + "\x01\xa1\x03\x01\x00\x00\x1e\xdb" + // 0x01A10301: 0x00001EDB + "\x01\xa0\x03\x00\x00\x00\x1e\xdc" + // 0x01A00300: 0x00001EDC + "\x01\xa1\x03\x00\x00\x00\x1e\xdd" + // 0x01A10300: 0x00001EDD + "\x01\xa0\x03\t\x00\x00\x1e\xde" + // 0x01A00309: 0x00001EDE + "\x01\xa1\x03\t\x00\x00\x1e\xdf" + // 0x01A10309: 0x00001EDF + "\x01\xa0\x03\x03\x00\x00\x1e\xe0" + // 0x01A00303: 0x00001EE0 + "\x01\xa1\x03\x03\x00\x00\x1e\xe1" + // 0x01A10303: 0x00001EE1 + "\x01\xa0\x03#\x00\x00\x1e\xe2" + // 0x01A00323: 0x00001EE2 + "\x01\xa1\x03#\x00\x00\x1e\xe3" + // 0x01A10323: 0x00001EE3 + "\x00U\x03#\x00\x00\x1e\xe4" + // 0x00550323: 0x00001EE4 + "\x00u\x03#\x00\x00\x1e\xe5" + // 0x00750323: 0x00001EE5 + "\x00U\x03\t\x00\x00\x1e\xe6" + // 0x00550309: 0x00001EE6 + "\x00u\x03\t\x00\x00\x1e\xe7" + // 0x00750309: 0x00001EE7 + "\x01\xaf\x03\x01\x00\x00\x1e\xe8" + // 0x01AF0301: 0x00001EE8 + "\x01\xb0\x03\x01\x00\x00\x1e\xe9" + // 0x01B00301: 0x00001EE9 + "\x01\xaf\x03\x00\x00\x00\x1e\xea" + // 0x01AF0300: 0x00001EEA + "\x01\xb0\x03\x00\x00\x00\x1e\xeb" + // 0x01B00300: 0x00001EEB + "\x01\xaf\x03\t\x00\x00\x1e\xec" + // 0x01AF0309: 0x00001EEC + "\x01\xb0\x03\t\x00\x00\x1e\xed" + // 0x01B00309: 0x00001EED + "\x01\xaf\x03\x03\x00\x00\x1e\xee" + // 0x01AF0303: 0x00001EEE + "\x01\xb0\x03\x03\x00\x00\x1e\xef" + // 0x01B00303: 0x00001EEF + "\x01\xaf\x03#\x00\x00\x1e\xf0" + // 0x01AF0323: 0x00001EF0 + "\x01\xb0\x03#\x00\x00\x1e\xf1" + // 0x01B00323: 0x00001EF1 + "\x00Y\x03\x00\x00\x00\x1e\xf2" + // 0x00590300: 0x00001EF2 + "\x00y\x03\x00\x00\x00\x1e\xf3" + // 0x00790300: 0x00001EF3 + "\x00Y\x03#\x00\x00\x1e\xf4" + // 0x00590323: 0x00001EF4 + "\x00y\x03#\x00\x00\x1e\xf5" + // 0x00790323: 0x00001EF5 + "\x00Y\x03\t\x00\x00\x1e\xf6" + // 0x00590309: 0x00001EF6 + "\x00y\x03\t\x00\x00\x1e\xf7" + // 0x00790309: 0x00001EF7 + "\x00Y\x03\x03\x00\x00\x1e\xf8" + // 0x00590303: 0x00001EF8 + "\x00y\x03\x03\x00\x00\x1e\xf9" + // 0x00790303: 0x00001EF9 + "\x03\xb1\x03\x13\x00\x00\x1f\x00" + // 0x03B10313: 0x00001F00 + "\x03\xb1\x03\x14\x00\x00\x1f\x01" + // 0x03B10314: 0x00001F01 + "\x1f\x00\x03\x00\x00\x00\x1f\x02" + // 0x1F000300: 0x00001F02 + "\x1f\x01\x03\x00\x00\x00\x1f\x03" + // 0x1F010300: 0x00001F03 + "\x1f\x00\x03\x01\x00\x00\x1f\x04" + // 0x1F000301: 0x00001F04 + "\x1f\x01\x03\x01\x00\x00\x1f\x05" + // 0x1F010301: 0x00001F05 + "\x1f\x00\x03B\x00\x00\x1f\x06" + // 0x1F000342: 0x00001F06 + "\x1f\x01\x03B\x00\x00\x1f\a" + // 0x1F010342: 0x00001F07 + "\x03\x91\x03\x13\x00\x00\x1f\b" + // 0x03910313: 0x00001F08 + "\x03\x91\x03\x14\x00\x00\x1f\t" + // 0x03910314: 0x00001F09 + "\x1f\b\x03\x00\x00\x00\x1f\n" + // 0x1F080300: 0x00001F0A + "\x1f\t\x03\x00\x00\x00\x1f\v" + // 0x1F090300: 0x00001F0B + "\x1f\b\x03\x01\x00\x00\x1f\f" + // 0x1F080301: 0x00001F0C + "\x1f\t\x03\x01\x00\x00\x1f\r" + // 0x1F090301: 0x00001F0D + "\x1f\b\x03B\x00\x00\x1f\x0e" + // 0x1F080342: 0x00001F0E + "\x1f\t\x03B\x00\x00\x1f\x0f" + // 0x1F090342: 0x00001F0F + "\x03\xb5\x03\x13\x00\x00\x1f\x10" + // 0x03B50313: 0x00001F10 + "\x03\xb5\x03\x14\x00\x00\x1f\x11" + // 0x03B50314: 0x00001F11 + "\x1f\x10\x03\x00\x00\x00\x1f\x12" + // 0x1F100300: 0x00001F12 + "\x1f\x11\x03\x00\x00\x00\x1f\x13" + // 0x1F110300: 0x00001F13 + "\x1f\x10\x03\x01\x00\x00\x1f\x14" + // 0x1F100301: 0x00001F14 + "\x1f\x11\x03\x01\x00\x00\x1f\x15" + // 0x1F110301: 0x00001F15 + "\x03\x95\x03\x13\x00\x00\x1f\x18" + // 0x03950313: 0x00001F18 + "\x03\x95\x03\x14\x00\x00\x1f\x19" + // 0x03950314: 0x00001F19 + "\x1f\x18\x03\x00\x00\x00\x1f\x1a" + // 0x1F180300: 0x00001F1A + "\x1f\x19\x03\x00\x00\x00\x1f\x1b" + // 0x1F190300: 0x00001F1B + "\x1f\x18\x03\x01\x00\x00\x1f\x1c" + // 0x1F180301: 0x00001F1C + "\x1f\x19\x03\x01\x00\x00\x1f\x1d" + // 0x1F190301: 0x00001F1D + "\x03\xb7\x03\x13\x00\x00\x1f " + // 0x03B70313: 0x00001F20 + "\x03\xb7\x03\x14\x00\x00\x1f!" + // 0x03B70314: 0x00001F21 + "\x1f \x03\x00\x00\x00\x1f\"" + // 0x1F200300: 0x00001F22 + "\x1f!\x03\x00\x00\x00\x1f#" + // 0x1F210300: 0x00001F23 + "\x1f \x03\x01\x00\x00\x1f$" + // 0x1F200301: 0x00001F24 + "\x1f!\x03\x01\x00\x00\x1f%" + // 0x1F210301: 0x00001F25 + "\x1f \x03B\x00\x00\x1f&" + // 0x1F200342: 0x00001F26 + "\x1f!\x03B\x00\x00\x1f'" + // 0x1F210342: 0x00001F27 + "\x03\x97\x03\x13\x00\x00\x1f(" + // 0x03970313: 0x00001F28 + "\x03\x97\x03\x14\x00\x00\x1f)" + // 0x03970314: 0x00001F29 + "\x1f(\x03\x00\x00\x00\x1f*" + // 0x1F280300: 0x00001F2A + "\x1f)\x03\x00\x00\x00\x1f+" + // 0x1F290300: 0x00001F2B + "\x1f(\x03\x01\x00\x00\x1f," + // 0x1F280301: 0x00001F2C + "\x1f)\x03\x01\x00\x00\x1f-" + // 0x1F290301: 0x00001F2D + "\x1f(\x03B\x00\x00\x1f." + // 0x1F280342: 0x00001F2E + "\x1f)\x03B\x00\x00\x1f/" + // 0x1F290342: 0x00001F2F + "\x03\xb9\x03\x13\x00\x00\x1f0" + // 0x03B90313: 0x00001F30 + "\x03\xb9\x03\x14\x00\x00\x1f1" + // 0x03B90314: 0x00001F31 + "\x1f0\x03\x00\x00\x00\x1f2" + // 0x1F300300: 0x00001F32 + "\x1f1\x03\x00\x00\x00\x1f3" + // 0x1F310300: 0x00001F33 + "\x1f0\x03\x01\x00\x00\x1f4" + // 0x1F300301: 0x00001F34 + "\x1f1\x03\x01\x00\x00\x1f5" + // 0x1F310301: 0x00001F35 + "\x1f0\x03B\x00\x00\x1f6" + // 0x1F300342: 0x00001F36 + "\x1f1\x03B\x00\x00\x1f7" + // 0x1F310342: 0x00001F37 + "\x03\x99\x03\x13\x00\x00\x1f8" + // 0x03990313: 0x00001F38 + "\x03\x99\x03\x14\x00\x00\x1f9" + // 0x03990314: 0x00001F39 + "\x1f8\x03\x00\x00\x00\x1f:" + // 0x1F380300: 0x00001F3A + "\x1f9\x03\x00\x00\x00\x1f;" + // 0x1F390300: 0x00001F3B + "\x1f8\x03\x01\x00\x00\x1f<" + // 0x1F380301: 0x00001F3C + "\x1f9\x03\x01\x00\x00\x1f=" + // 0x1F390301: 0x00001F3D + "\x1f8\x03B\x00\x00\x1f>" + // 0x1F380342: 0x00001F3E + "\x1f9\x03B\x00\x00\x1f?" + // 0x1F390342: 0x00001F3F + "\x03\xbf\x03\x13\x00\x00\x1f@" + // 0x03BF0313: 0x00001F40 + "\x03\xbf\x03\x14\x00\x00\x1fA" + // 0x03BF0314: 0x00001F41 + "\x1f@\x03\x00\x00\x00\x1fB" + // 0x1F400300: 0x00001F42 + "\x1fA\x03\x00\x00\x00\x1fC" + // 0x1F410300: 0x00001F43 + "\x1f@\x03\x01\x00\x00\x1fD" + // 0x1F400301: 0x00001F44 + "\x1fA\x03\x01\x00\x00\x1fE" + // 0x1F410301: 0x00001F45 + "\x03\x9f\x03\x13\x00\x00\x1fH" + // 0x039F0313: 0x00001F48 + "\x03\x9f\x03\x14\x00\x00\x1fI" + // 0x039F0314: 0x00001F49 + "\x1fH\x03\x00\x00\x00\x1fJ" + // 0x1F480300: 0x00001F4A + "\x1fI\x03\x00\x00\x00\x1fK" + // 0x1F490300: 0x00001F4B + "\x1fH\x03\x01\x00\x00\x1fL" + // 0x1F480301: 0x00001F4C + "\x1fI\x03\x01\x00\x00\x1fM" + // 0x1F490301: 0x00001F4D + "\x03\xc5\x03\x13\x00\x00\x1fP" + // 0x03C50313: 0x00001F50 + "\x03\xc5\x03\x14\x00\x00\x1fQ" + // 0x03C50314: 0x00001F51 + "\x1fP\x03\x00\x00\x00\x1fR" + // 0x1F500300: 0x00001F52 + "\x1fQ\x03\x00\x00\x00\x1fS" + // 0x1F510300: 0x00001F53 + "\x1fP\x03\x01\x00\x00\x1fT" + // 0x1F500301: 0x00001F54 + "\x1fQ\x03\x01\x00\x00\x1fU" + // 0x1F510301: 0x00001F55 + "\x1fP\x03B\x00\x00\x1fV" + // 0x1F500342: 0x00001F56 + "\x1fQ\x03B\x00\x00\x1fW" + // 0x1F510342: 0x00001F57 + "\x03\xa5\x03\x14\x00\x00\x1fY" + // 0x03A50314: 0x00001F59 + "\x1fY\x03\x00\x00\x00\x1f[" + // 0x1F590300: 0x00001F5B + "\x1fY\x03\x01\x00\x00\x1f]" + // 0x1F590301: 0x00001F5D + "\x1fY\x03B\x00\x00\x1f_" + // 0x1F590342: 0x00001F5F + "\x03\xc9\x03\x13\x00\x00\x1f`" + // 0x03C90313: 0x00001F60 + "\x03\xc9\x03\x14\x00\x00\x1fa" + // 0x03C90314: 0x00001F61 + "\x1f`\x03\x00\x00\x00\x1fb" + // 0x1F600300: 0x00001F62 + "\x1fa\x03\x00\x00\x00\x1fc" + // 0x1F610300: 0x00001F63 + "\x1f`\x03\x01\x00\x00\x1fd" + // 0x1F600301: 0x00001F64 + "\x1fa\x03\x01\x00\x00\x1fe" + // 0x1F610301: 0x00001F65 + "\x1f`\x03B\x00\x00\x1ff" + // 0x1F600342: 0x00001F66 + "\x1fa\x03B\x00\x00\x1fg" + // 0x1F610342: 0x00001F67 + "\x03\xa9\x03\x13\x00\x00\x1fh" + // 0x03A90313: 0x00001F68 + "\x03\xa9\x03\x14\x00\x00\x1fi" + // 0x03A90314: 0x00001F69 + "\x1fh\x03\x00\x00\x00\x1fj" + // 0x1F680300: 0x00001F6A + "\x1fi\x03\x00\x00\x00\x1fk" + // 0x1F690300: 0x00001F6B + "\x1fh\x03\x01\x00\x00\x1fl" + // 0x1F680301: 0x00001F6C + "\x1fi\x03\x01\x00\x00\x1fm" + // 0x1F690301: 0x00001F6D + "\x1fh\x03B\x00\x00\x1fn" + // 0x1F680342: 0x00001F6E + "\x1fi\x03B\x00\x00\x1fo" + // 0x1F690342: 0x00001F6F + "\x03\xb1\x03\x00\x00\x00\x1fp" + // 0x03B10300: 0x00001F70 + "\x03\xb5\x03\x00\x00\x00\x1fr" + // 0x03B50300: 0x00001F72 + "\x03\xb7\x03\x00\x00\x00\x1ft" + // 0x03B70300: 0x00001F74 + "\x03\xb9\x03\x00\x00\x00\x1fv" + // 0x03B90300: 0x00001F76 + "\x03\xbf\x03\x00\x00\x00\x1fx" + // 0x03BF0300: 0x00001F78 + "\x03\xc5\x03\x00\x00\x00\x1fz" + // 0x03C50300: 0x00001F7A + "\x03\xc9\x03\x00\x00\x00\x1f|" + // 0x03C90300: 0x00001F7C + "\x1f\x00\x03E\x00\x00\x1f\x80" + // 0x1F000345: 0x00001F80 + "\x1f\x01\x03E\x00\x00\x1f\x81" + // 0x1F010345: 0x00001F81 + "\x1f\x02\x03E\x00\x00\x1f\x82" + // 0x1F020345: 0x00001F82 + "\x1f\x03\x03E\x00\x00\x1f\x83" + // 0x1F030345: 0x00001F83 + "\x1f\x04\x03E\x00\x00\x1f\x84" + // 0x1F040345: 0x00001F84 + "\x1f\x05\x03E\x00\x00\x1f\x85" + // 0x1F050345: 0x00001F85 + "\x1f\x06\x03E\x00\x00\x1f\x86" + // 0x1F060345: 0x00001F86 + "\x1f\a\x03E\x00\x00\x1f\x87" + // 0x1F070345: 0x00001F87 + "\x1f\b\x03E\x00\x00\x1f\x88" + // 0x1F080345: 0x00001F88 + "\x1f\t\x03E\x00\x00\x1f\x89" + // 0x1F090345: 0x00001F89 + "\x1f\n\x03E\x00\x00\x1f\x8a" + // 0x1F0A0345: 0x00001F8A + "\x1f\v\x03E\x00\x00\x1f\x8b" + // 0x1F0B0345: 0x00001F8B + "\x1f\f\x03E\x00\x00\x1f\x8c" + // 0x1F0C0345: 0x00001F8C + "\x1f\r\x03E\x00\x00\x1f\x8d" + // 0x1F0D0345: 0x00001F8D + "\x1f\x0e\x03E\x00\x00\x1f\x8e" + // 0x1F0E0345: 0x00001F8E + "\x1f\x0f\x03E\x00\x00\x1f\x8f" + // 0x1F0F0345: 0x00001F8F + "\x1f \x03E\x00\x00\x1f\x90" + // 0x1F200345: 0x00001F90 + "\x1f!\x03E\x00\x00\x1f\x91" + // 0x1F210345: 0x00001F91 + "\x1f\"\x03E\x00\x00\x1f\x92" + // 0x1F220345: 0x00001F92 + "\x1f#\x03E\x00\x00\x1f\x93" + // 0x1F230345: 0x00001F93 + "\x1f$\x03E\x00\x00\x1f\x94" + // 0x1F240345: 0x00001F94 + "\x1f%\x03E\x00\x00\x1f\x95" + // 0x1F250345: 0x00001F95 + "\x1f&\x03E\x00\x00\x1f\x96" + // 0x1F260345: 0x00001F96 + "\x1f'\x03E\x00\x00\x1f\x97" + // 0x1F270345: 0x00001F97 + "\x1f(\x03E\x00\x00\x1f\x98" + // 0x1F280345: 0x00001F98 + "\x1f)\x03E\x00\x00\x1f\x99" + // 0x1F290345: 0x00001F99 + "\x1f*\x03E\x00\x00\x1f\x9a" + // 0x1F2A0345: 0x00001F9A + "\x1f+\x03E\x00\x00\x1f\x9b" + // 0x1F2B0345: 0x00001F9B + "\x1f,\x03E\x00\x00\x1f\x9c" + // 0x1F2C0345: 0x00001F9C + "\x1f-\x03E\x00\x00\x1f\x9d" + // 0x1F2D0345: 0x00001F9D + "\x1f.\x03E\x00\x00\x1f\x9e" + // 0x1F2E0345: 0x00001F9E + "\x1f/\x03E\x00\x00\x1f\x9f" + // 0x1F2F0345: 0x00001F9F + "\x1f`\x03E\x00\x00\x1f\xa0" + // 0x1F600345: 0x00001FA0 + "\x1fa\x03E\x00\x00\x1f\xa1" + // 0x1F610345: 0x00001FA1 + "\x1fb\x03E\x00\x00\x1f\xa2" + // 0x1F620345: 0x00001FA2 + "\x1fc\x03E\x00\x00\x1f\xa3" + // 0x1F630345: 0x00001FA3 + "\x1fd\x03E\x00\x00\x1f\xa4" + // 0x1F640345: 0x00001FA4 + "\x1fe\x03E\x00\x00\x1f\xa5" + // 0x1F650345: 0x00001FA5 + "\x1ff\x03E\x00\x00\x1f\xa6" + // 0x1F660345: 0x00001FA6 + "\x1fg\x03E\x00\x00\x1f\xa7" + // 0x1F670345: 0x00001FA7 + "\x1fh\x03E\x00\x00\x1f\xa8" + // 0x1F680345: 0x00001FA8 + "\x1fi\x03E\x00\x00\x1f\xa9" + // 0x1F690345: 0x00001FA9 + "\x1fj\x03E\x00\x00\x1f\xaa" + // 0x1F6A0345: 0x00001FAA + "\x1fk\x03E\x00\x00\x1f\xab" + // 0x1F6B0345: 0x00001FAB + "\x1fl\x03E\x00\x00\x1f\xac" + // 0x1F6C0345: 0x00001FAC + "\x1fm\x03E\x00\x00\x1f\xad" + // 0x1F6D0345: 0x00001FAD + "\x1fn\x03E\x00\x00\x1f\xae" + // 0x1F6E0345: 0x00001FAE + "\x1fo\x03E\x00\x00\x1f\xaf" + // 0x1F6F0345: 0x00001FAF + "\x03\xb1\x03\x06\x00\x00\x1f\xb0" + // 0x03B10306: 0x00001FB0 + "\x03\xb1\x03\x04\x00\x00\x1f\xb1" + // 0x03B10304: 0x00001FB1 + "\x1fp\x03E\x00\x00\x1f\xb2" + // 0x1F700345: 0x00001FB2 + "\x03\xb1\x03E\x00\x00\x1f\xb3" + // 0x03B10345: 0x00001FB3 + "\x03\xac\x03E\x00\x00\x1f\xb4" + // 0x03AC0345: 0x00001FB4 + "\x03\xb1\x03B\x00\x00\x1f\xb6" + // 0x03B10342: 0x00001FB6 + "\x1f\xb6\x03E\x00\x00\x1f\xb7" + // 0x1FB60345: 0x00001FB7 + "\x03\x91\x03\x06\x00\x00\x1f\xb8" + // 0x03910306: 0x00001FB8 + "\x03\x91\x03\x04\x00\x00\x1f\xb9" + // 0x03910304: 0x00001FB9 + "\x03\x91\x03\x00\x00\x00\x1f\xba" + // 0x03910300: 0x00001FBA + "\x03\x91\x03E\x00\x00\x1f\xbc" + // 0x03910345: 0x00001FBC + "\x00\xa8\x03B\x00\x00\x1f\xc1" + // 0x00A80342: 0x00001FC1 + "\x1ft\x03E\x00\x00\x1f\xc2" + // 0x1F740345: 0x00001FC2 + "\x03\xb7\x03E\x00\x00\x1f\xc3" + // 0x03B70345: 0x00001FC3 + "\x03\xae\x03E\x00\x00\x1f\xc4" + // 0x03AE0345: 0x00001FC4 + "\x03\xb7\x03B\x00\x00\x1f\xc6" + // 0x03B70342: 0x00001FC6 + "\x1f\xc6\x03E\x00\x00\x1f\xc7" + // 0x1FC60345: 0x00001FC7 + "\x03\x95\x03\x00\x00\x00\x1f\xc8" + // 0x03950300: 0x00001FC8 + "\x03\x97\x03\x00\x00\x00\x1f\xca" + // 0x03970300: 0x00001FCA + "\x03\x97\x03E\x00\x00\x1f\xcc" + // 0x03970345: 0x00001FCC + "\x1f\xbf\x03\x00\x00\x00\x1f\xcd" + // 0x1FBF0300: 0x00001FCD + "\x1f\xbf\x03\x01\x00\x00\x1f\xce" + // 0x1FBF0301: 0x00001FCE + "\x1f\xbf\x03B\x00\x00\x1f\xcf" + // 0x1FBF0342: 0x00001FCF + "\x03\xb9\x03\x06\x00\x00\x1f\xd0" + // 0x03B90306: 0x00001FD0 + "\x03\xb9\x03\x04\x00\x00\x1f\xd1" + // 0x03B90304: 0x00001FD1 + "\x03\xca\x03\x00\x00\x00\x1f\xd2" + // 0x03CA0300: 0x00001FD2 + "\x03\xb9\x03B\x00\x00\x1f\xd6" + // 0x03B90342: 0x00001FD6 + "\x03\xca\x03B\x00\x00\x1f\xd7" + // 0x03CA0342: 0x00001FD7 + "\x03\x99\x03\x06\x00\x00\x1f\xd8" + // 0x03990306: 0x00001FD8 + "\x03\x99\x03\x04\x00\x00\x1f\xd9" + // 0x03990304: 0x00001FD9 + "\x03\x99\x03\x00\x00\x00\x1f\xda" + // 0x03990300: 0x00001FDA + "\x1f\xfe\x03\x00\x00\x00\x1f\xdd" + // 0x1FFE0300: 0x00001FDD + "\x1f\xfe\x03\x01\x00\x00\x1f\xde" + // 0x1FFE0301: 0x00001FDE + "\x1f\xfe\x03B\x00\x00\x1f\xdf" + // 0x1FFE0342: 0x00001FDF + "\x03\xc5\x03\x06\x00\x00\x1f\xe0" + // 0x03C50306: 0x00001FE0 + "\x03\xc5\x03\x04\x00\x00\x1f\xe1" + // 0x03C50304: 0x00001FE1 + "\x03\xcb\x03\x00\x00\x00\x1f\xe2" + // 0x03CB0300: 0x00001FE2 + "\x03\xc1\x03\x13\x00\x00\x1f\xe4" + // 0x03C10313: 0x00001FE4 + "\x03\xc1\x03\x14\x00\x00\x1f\xe5" + // 0x03C10314: 0x00001FE5 + "\x03\xc5\x03B\x00\x00\x1f\xe6" + // 0x03C50342: 0x00001FE6 + "\x03\xcb\x03B\x00\x00\x1f\xe7" + // 0x03CB0342: 0x00001FE7 + "\x03\xa5\x03\x06\x00\x00\x1f\xe8" + // 0x03A50306: 0x00001FE8 + "\x03\xa5\x03\x04\x00\x00\x1f\xe9" + // 0x03A50304: 0x00001FE9 + "\x03\xa5\x03\x00\x00\x00\x1f\xea" + // 0x03A50300: 0x00001FEA + "\x03\xa1\x03\x14\x00\x00\x1f\xec" + // 0x03A10314: 0x00001FEC + "\x00\xa8\x03\x00\x00\x00\x1f\xed" + // 0x00A80300: 0x00001FED + "\x1f|\x03E\x00\x00\x1f\xf2" + // 0x1F7C0345: 0x00001FF2 + "\x03\xc9\x03E\x00\x00\x1f\xf3" + // 0x03C90345: 0x00001FF3 + "\x03\xce\x03E\x00\x00\x1f\xf4" + // 0x03CE0345: 0x00001FF4 + "\x03\xc9\x03B\x00\x00\x1f\xf6" + // 0x03C90342: 0x00001FF6 + "\x1f\xf6\x03E\x00\x00\x1f\xf7" + // 0x1FF60345: 0x00001FF7 + "\x03\x9f\x03\x00\x00\x00\x1f\xf8" + // 0x039F0300: 0x00001FF8 + "\x03\xa9\x03\x00\x00\x00\x1f\xfa" + // 0x03A90300: 0x00001FFA + "\x03\xa9\x03E\x00\x00\x1f\xfc" + // 0x03A90345: 0x00001FFC + "!\x90\x038\x00\x00!\x9a" + // 0x21900338: 0x0000219A + "!\x92\x038\x00\x00!\x9b" + // 0x21920338: 0x0000219B + "!\x94\x038\x00\x00!\xae" + // 0x21940338: 0x000021AE + "!\xd0\x038\x00\x00!\xcd" + // 0x21D00338: 0x000021CD + "!\xd4\x038\x00\x00!\xce" + // 0x21D40338: 0x000021CE + "!\xd2\x038\x00\x00!\xcf" + // 0x21D20338: 0x000021CF + "\"\x03\x038\x00\x00\"\x04" + // 0x22030338: 0x00002204 + "\"\b\x038\x00\x00\"\t" + // 0x22080338: 0x00002209 + "\"\v\x038\x00\x00\"\f" + // 0x220B0338: 0x0000220C + "\"#\x038\x00\x00\"$" + // 0x22230338: 0x00002224 + "\"%\x038\x00\x00\"&" + // 0x22250338: 0x00002226 + "\"<\x038\x00\x00\"A" + // 0x223C0338: 0x00002241 + "\"C\x038\x00\x00\"D" + // 0x22430338: 0x00002244 + "\"E\x038\x00\x00\"G" + // 0x22450338: 0x00002247 + "\"H\x038\x00\x00\"I" + // 0x22480338: 0x00002249 + "\x00=\x038\x00\x00\"`" + // 0x003D0338: 0x00002260 + "\"a\x038\x00\x00\"b" + // 0x22610338: 0x00002262 + "\"M\x038\x00\x00\"m" + // 0x224D0338: 0x0000226D + "\x00<\x038\x00\x00\"n" + // 0x003C0338: 0x0000226E + "\x00>\x038\x00\x00\"o" + // 0x003E0338: 0x0000226F + "\"d\x038\x00\x00\"p" + // 0x22640338: 0x00002270 + "\"e\x038\x00\x00\"q" + // 0x22650338: 0x00002271 + "\"r\x038\x00\x00\"t" + // 0x22720338: 0x00002274 + "\"s\x038\x00\x00\"u" + // 0x22730338: 0x00002275 + "\"v\x038\x00\x00\"x" + // 0x22760338: 0x00002278 + "\"w\x038\x00\x00\"y" + // 0x22770338: 0x00002279 + "\"z\x038\x00\x00\"\x80" + // 0x227A0338: 0x00002280 + "\"{\x038\x00\x00\"\x81" + // 0x227B0338: 0x00002281 + "\"\x82\x038\x00\x00\"\x84" + // 0x22820338: 0x00002284 + "\"\x83\x038\x00\x00\"\x85" + // 0x22830338: 0x00002285 + "\"\x86\x038\x00\x00\"\x88" + // 0x22860338: 0x00002288 + "\"\x87\x038\x00\x00\"\x89" + // 0x22870338: 0x00002289 + "\"\xa2\x038\x00\x00\"\xac" + // 0x22A20338: 0x000022AC + "\"\xa8\x038\x00\x00\"\xad" + // 0x22A80338: 0x000022AD + "\"\xa9\x038\x00\x00\"\xae" + // 0x22A90338: 0x000022AE + "\"\xab\x038\x00\x00\"\xaf" + // 0x22AB0338: 0x000022AF + "\"|\x038\x00\x00\"\xe0" + // 0x227C0338: 0x000022E0 + "\"}\x038\x00\x00\"\xe1" + // 0x227D0338: 0x000022E1 + "\"\x91\x038\x00\x00\"\xe2" + // 0x22910338: 0x000022E2 + "\"\x92\x038\x00\x00\"\xe3" + // 0x22920338: 0x000022E3 + "\"\xb2\x038\x00\x00\"\xea" + // 0x22B20338: 0x000022EA + "\"\xb3\x038\x00\x00\"\xeb" + // 0x22B30338: 0x000022EB + "\"\xb4\x038\x00\x00\"\xec" + // 0x22B40338: 0x000022EC + "\"\xb5\x038\x00\x00\"\xed" + // 0x22B50338: 0x000022ED + "0K0\x99\x00\x000L" + // 0x304B3099: 0x0000304C + "0M0\x99\x00\x000N" + // 0x304D3099: 0x0000304E + "0O0\x99\x00\x000P" + // 0x304F3099: 0x00003050 + "0Q0\x99\x00\x000R" + // 0x30513099: 0x00003052 + "0S0\x99\x00\x000T" + // 0x30533099: 0x00003054 + "0U0\x99\x00\x000V" + // 0x30553099: 0x00003056 + "0W0\x99\x00\x000X" + // 0x30573099: 0x00003058 + "0Y0\x99\x00\x000Z" + // 0x30593099: 0x0000305A + "0[0\x99\x00\x000\\" + // 0x305B3099: 0x0000305C + "0]0\x99\x00\x000^" + // 0x305D3099: 0x0000305E + "0_0\x99\x00\x000`" + // 0x305F3099: 0x00003060 + "0a0\x99\x00\x000b" + // 0x30613099: 0x00003062 + "0d0\x99\x00\x000e" + // 0x30643099: 0x00003065 + "0f0\x99\x00\x000g" + // 0x30663099: 0x00003067 + "0h0\x99\x00\x000i" + // 0x30683099: 0x00003069 + "0o0\x99\x00\x000p" + // 0x306F3099: 0x00003070 + "0o0\x9a\x00\x000q" + // 0x306F309A: 0x00003071 + "0r0\x99\x00\x000s" + // 0x30723099: 0x00003073 + "0r0\x9a\x00\x000t" + // 0x3072309A: 0x00003074 + "0u0\x99\x00\x000v" + // 0x30753099: 0x00003076 + "0u0\x9a\x00\x000w" + // 0x3075309A: 0x00003077 + "0x0\x99\x00\x000y" + // 0x30783099: 0x00003079 + "0x0\x9a\x00\x000z" + // 0x3078309A: 0x0000307A + "0{0\x99\x00\x000|" + // 0x307B3099: 0x0000307C + "0{0\x9a\x00\x000}" + // 0x307B309A: 0x0000307D + "0F0\x99\x00\x000\x94" + // 0x30463099: 0x00003094 + "0\x9d0\x99\x00\x000\x9e" + // 0x309D3099: 0x0000309E + "0\xab0\x99\x00\x000\xac" + // 0x30AB3099: 0x000030AC + "0\xad0\x99\x00\x000\xae" + // 0x30AD3099: 0x000030AE + "0\xaf0\x99\x00\x000\xb0" + // 0x30AF3099: 0x000030B0 + "0\xb10\x99\x00\x000\xb2" + // 0x30B13099: 0x000030B2 + "0\xb30\x99\x00\x000\xb4" + // 0x30B33099: 0x000030B4 + "0\xb50\x99\x00\x000\xb6" + // 0x30B53099: 0x000030B6 + "0\xb70\x99\x00\x000\xb8" + // 0x30B73099: 0x000030B8 + "0\xb90\x99\x00\x000\xba" + // 0x30B93099: 0x000030BA + "0\xbb0\x99\x00\x000\xbc" + // 0x30BB3099: 0x000030BC + "0\xbd0\x99\x00\x000\xbe" + // 0x30BD3099: 0x000030BE + "0\xbf0\x99\x00\x000\xc0" + // 0x30BF3099: 0x000030C0 + "0\xc10\x99\x00\x000\xc2" + // 0x30C13099: 0x000030C2 + "0\xc40\x99\x00\x000\xc5" + // 0x30C43099: 0x000030C5 + "0\xc60\x99\x00\x000\xc7" + // 0x30C63099: 0x000030C7 + "0\xc80\x99\x00\x000\xc9" + // 0x30C83099: 0x000030C9 + "0\xcf0\x99\x00\x000\xd0" + // 0x30CF3099: 0x000030D0 + "0\xcf0\x9a\x00\x000\xd1" + // 0x30CF309A: 0x000030D1 + "0\xd20\x99\x00\x000\xd3" + // 0x30D23099: 0x000030D3 + "0\xd20\x9a\x00\x000\xd4" + // 0x30D2309A: 0x000030D4 + "0\xd50\x99\x00\x000\xd6" + // 0x30D53099: 0x000030D6 + "0\xd50\x9a\x00\x000\xd7" + // 0x30D5309A: 0x000030D7 + "0\xd80\x99\x00\x000\xd9" + // 0x30D83099: 0x000030D9 + "0\xd80\x9a\x00\x000\xda" + // 0x30D8309A: 0x000030DA + "0\xdb0\x99\x00\x000\xdc" + // 0x30DB3099: 0x000030DC + "0\xdb0\x9a\x00\x000\xdd" + // 0x30DB309A: 0x000030DD + "0\xa60\x99\x00\x000\xf4" + // 0x30A63099: 0x000030F4 + "0\xef0\x99\x00\x000\xf7" + // 0x30EF3099: 0x000030F7 + "0\xf00\x99\x00\x000\xf8" + // 0x30F03099: 0x000030F8 + "0\xf10\x99\x00\x000\xf9" + // 0x30F13099: 0x000030F9 + "0\xf20\x99\x00\x000\xfa" + // 0x30F23099: 0x000030FA + "0\xfd0\x99\x00\x000\xfe" + // 0x30FD3099: 0x000030FE + "\x10\x99\x10\xba\x00\x01\x10\x9a" + // 0x109910BA: 0x0001109A + "\x10\x9b\x10\xba\x00\x01\x10\x9c" + // 0x109B10BA: 0x0001109C + "\x10\xa5\x10\xba\x00\x01\x10\xab" + // 0x10A510BA: 0x000110AB + "\x111\x11'\x00\x01\x11." + // 0x11311127: 0x0001112E + "\x112\x11'\x00\x01\x11/" + // 0x11321127: 0x0001112F + "\x13G\x13>\x00\x01\x13K" + // 0x1347133E: 0x0001134B + "\x13G\x13W\x00\x01\x13L" + // 0x13471357: 0x0001134C + "\x14\xb9\x14\xba\x00\x01\x14\xbb" + // 0x14B914BA: 0x000114BB + "\x14\xb9\x14\xb0\x00\x01\x14\xbc" + // 0x14B914B0: 0x000114BC + "\x14\xb9\x14\xbd\x00\x01\x14\xbe" + // 0x14B914BD: 0x000114BE + "\x15\xb8\x15\xaf\x00\x01\x15\xba" + // 0x15B815AF: 0x000115BA + "\x15\xb9\x15\xaf\x00\x01\x15\xbb" + // 0x15B915AF: 0x000115BB + "" + // Total size of tables: 55KB (55977 bytes) diff --git a/golang.org/x/text/width/tables11.0.0.go b/golang.org/x/text/width/tables11.0.0.go index d6def0e7be..3c75e428fd 100644 --- a/golang.org/x/text/width/tables11.0.0.go +++ b/golang.org/x/text/width/tables11.0.0.go @@ -1,6 +1,6 @@ // Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. -// +build go1.13 +// +build go1.13,!go1.14 package width diff --git a/golang.org/x/text/width/tables12.0.0.go b/golang.org/x/text/width/tables12.0.0.go new file mode 100644 index 0000000000..5c859677a7 --- /dev/null +++ b/golang.org/x/text/width/tables12.0.0.go @@ -0,0 +1,1350 @@ +// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT. + +// +build go1.14 + +package width + +// UnicodeVersion is the Unicode version from which the tables in this package are derived. +const UnicodeVersion = "12.0.0" + +// lookup returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *widthTrie) lookup(s []byte) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return widthValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = widthIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = widthIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = widthIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *widthTrie) lookupUnsafe(s []byte) uint16 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return widthValues[c0] + } + i := widthIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = widthIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = widthIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// lookupString returns the trie value for the first UTF-8 encoding in s and +// the width in bytes of this encoding. The size will be 0 if s does not +// hold enough bytes to complete the encoding. len(s) must be greater than 0. +func (t *widthTrie) lookupString(s string) (v uint16, sz int) { + c0 := s[0] + switch { + case c0 < 0x80: // is ASCII + return widthValues[c0], 1 + case c0 < 0xC2: + return 0, 1 // Illegal UTF-8: not a starter, not ASCII. + case c0 < 0xE0: // 2-byte UTF-8 + if len(s) < 2 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c1), 2 + case c0 < 0xF0: // 3-byte UTF-8 + if len(s) < 3 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = widthIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c2), 3 + case c0 < 0xF8: // 4-byte UTF-8 + if len(s) < 4 { + return 0, 0 + } + i := widthIndex[c0] + c1 := s[1] + if c1 < 0x80 || 0xC0 <= c1 { + return 0, 1 // Illegal UTF-8: not a continuation byte. + } + o := uint32(i)<<6 + uint32(c1) + i = widthIndex[o] + c2 := s[2] + if c2 < 0x80 || 0xC0 <= c2 { + return 0, 2 // Illegal UTF-8: not a continuation byte. + } + o = uint32(i)<<6 + uint32(c2) + i = widthIndex[o] + c3 := s[3] + if c3 < 0x80 || 0xC0 <= c3 { + return 0, 3 // Illegal UTF-8: not a continuation byte. + } + return t.lookupValue(uint32(i), c3), 4 + } + // Illegal rune + return 0, 1 +} + +// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s. +// s must start with a full and valid UTF-8 encoded rune. +func (t *widthTrie) lookupStringUnsafe(s string) uint16 { + c0 := s[0] + if c0 < 0x80 { // is ASCII + return widthValues[c0] + } + i := widthIndex[c0] + if c0 < 0xE0 { // 2-byte UTF-8 + return t.lookupValue(uint32(i), s[1]) + } + i = widthIndex[uint32(i)<<6+uint32(s[1])] + if c0 < 0xF0 { // 3-byte UTF-8 + return t.lookupValue(uint32(i), s[2]) + } + i = widthIndex[uint32(i)<<6+uint32(s[2])] + if c0 < 0xF8 { // 4-byte UTF-8 + return t.lookupValue(uint32(i), s[3]) + } + return 0 +} + +// widthTrie. Total size: 14720 bytes (14.38 KiB). Checksum: 3f4f2516ded5489b. +type widthTrie struct{} + +func newWidthTrie(i int) *widthTrie { + return &widthTrie{} +} + +// lookupValue determines the type of block n and looks up the value for b. +func (t *widthTrie) lookupValue(n uint32, b byte) uint16 { + switch { + default: + return uint16(widthValues[n<<6+uint32(b)]) + } +} + +// widthValues: 104 blocks, 6656 entries, 13312 bytes +// The third block is the zero block. +var widthValues = [6656]uint16{ + // Block 0x0, offset 0x0 + 0x20: 0x6001, 0x21: 0x6002, 0x22: 0x6002, 0x23: 0x6002, + 0x24: 0x6002, 0x25: 0x6002, 0x26: 0x6002, 0x27: 0x6002, 0x28: 0x6002, 0x29: 0x6002, + 0x2a: 0x6002, 0x2b: 0x6002, 0x2c: 0x6002, 0x2d: 0x6002, 0x2e: 0x6002, 0x2f: 0x6002, + 0x30: 0x6002, 0x31: 0x6002, 0x32: 0x6002, 0x33: 0x6002, 0x34: 0x6002, 0x35: 0x6002, + 0x36: 0x6002, 0x37: 0x6002, 0x38: 0x6002, 0x39: 0x6002, 0x3a: 0x6002, 0x3b: 0x6002, + 0x3c: 0x6002, 0x3d: 0x6002, 0x3e: 0x6002, 0x3f: 0x6002, + // Block 0x1, offset 0x40 + 0x40: 0x6003, 0x41: 0x6003, 0x42: 0x6003, 0x43: 0x6003, 0x44: 0x6003, 0x45: 0x6003, + 0x46: 0x6003, 0x47: 0x6003, 0x48: 0x6003, 0x49: 0x6003, 0x4a: 0x6003, 0x4b: 0x6003, + 0x4c: 0x6003, 0x4d: 0x6003, 0x4e: 0x6003, 0x4f: 0x6003, 0x50: 0x6003, 0x51: 0x6003, + 0x52: 0x6003, 0x53: 0x6003, 0x54: 0x6003, 0x55: 0x6003, 0x56: 0x6003, 0x57: 0x6003, + 0x58: 0x6003, 0x59: 0x6003, 0x5a: 0x6003, 0x5b: 0x6003, 0x5c: 0x6003, 0x5d: 0x6003, + 0x5e: 0x6003, 0x5f: 0x6003, 0x60: 0x6004, 0x61: 0x6004, 0x62: 0x6004, 0x63: 0x6004, + 0x64: 0x6004, 0x65: 0x6004, 0x66: 0x6004, 0x67: 0x6004, 0x68: 0x6004, 0x69: 0x6004, + 0x6a: 0x6004, 0x6b: 0x6004, 0x6c: 0x6004, 0x6d: 0x6004, 0x6e: 0x6004, 0x6f: 0x6004, + 0x70: 0x6004, 0x71: 0x6004, 0x72: 0x6004, 0x73: 0x6004, 0x74: 0x6004, 0x75: 0x6004, + 0x76: 0x6004, 0x77: 0x6004, 0x78: 0x6004, 0x79: 0x6004, 0x7a: 0x6004, 0x7b: 0x6004, + 0x7c: 0x6004, 0x7d: 0x6004, 0x7e: 0x6004, + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xe1: 0x2000, 0xe2: 0x6005, 0xe3: 0x6005, + 0xe4: 0x2000, 0xe5: 0x6006, 0xe6: 0x6005, 0xe7: 0x2000, 0xe8: 0x2000, + 0xea: 0x2000, 0xec: 0x6007, 0xed: 0x2000, 0xee: 0x2000, 0xef: 0x6008, + 0xf0: 0x2000, 0xf1: 0x2000, 0xf2: 0x2000, 0xf3: 0x2000, 0xf4: 0x2000, + 0xf6: 0x2000, 0xf7: 0x2000, 0xf8: 0x2000, 0xf9: 0x2000, 0xfa: 0x2000, + 0xfc: 0x2000, 0xfd: 0x2000, 0xfe: 0x2000, 0xff: 0x2000, + // Block 0x4, offset 0x100 + 0x106: 0x2000, + 0x110: 0x2000, + 0x117: 0x2000, + 0x118: 0x2000, + 0x11e: 0x2000, 0x11f: 0x2000, 0x120: 0x2000, 0x121: 0x2000, + 0x126: 0x2000, 0x128: 0x2000, 0x129: 0x2000, + 0x12a: 0x2000, 0x12c: 0x2000, 0x12d: 0x2000, + 0x130: 0x2000, 0x132: 0x2000, 0x133: 0x2000, + 0x137: 0x2000, 0x138: 0x2000, 0x139: 0x2000, 0x13a: 0x2000, + 0x13c: 0x2000, 0x13e: 0x2000, + // Block 0x5, offset 0x140 + 0x141: 0x2000, + 0x151: 0x2000, + 0x153: 0x2000, + 0x15b: 0x2000, + 0x166: 0x2000, 0x167: 0x2000, + 0x16b: 0x2000, + 0x171: 0x2000, 0x172: 0x2000, 0x173: 0x2000, + 0x178: 0x2000, + 0x17f: 0x2000, + // Block 0x6, offset 0x180 + 0x180: 0x2000, 0x181: 0x2000, 0x182: 0x2000, 0x184: 0x2000, + 0x188: 0x2000, 0x189: 0x2000, 0x18a: 0x2000, 0x18b: 0x2000, + 0x18d: 0x2000, + 0x192: 0x2000, 0x193: 0x2000, + 0x1a6: 0x2000, 0x1a7: 0x2000, + 0x1ab: 0x2000, + // Block 0x7, offset 0x1c0 + 0x1ce: 0x2000, 0x1d0: 0x2000, + 0x1d2: 0x2000, 0x1d4: 0x2000, 0x1d6: 0x2000, + 0x1d8: 0x2000, 0x1da: 0x2000, 0x1dc: 0x2000, + // Block 0x8, offset 0x200 + 0x211: 0x2000, + 0x221: 0x2000, + // Block 0x9, offset 0x240 + 0x244: 0x2000, + 0x247: 0x2000, 0x249: 0x2000, 0x24a: 0x2000, 0x24b: 0x2000, + 0x24d: 0x2000, 0x250: 0x2000, + 0x258: 0x2000, 0x259: 0x2000, 0x25a: 0x2000, 0x25b: 0x2000, 0x25d: 0x2000, + 0x25f: 0x2000, + // Block 0xa, offset 0x280 + 0x280: 0x2000, 0x281: 0x2000, 0x282: 0x2000, 0x283: 0x2000, 0x284: 0x2000, 0x285: 0x2000, + 0x286: 0x2000, 0x287: 0x2000, 0x288: 0x2000, 0x289: 0x2000, 0x28a: 0x2000, 0x28b: 0x2000, + 0x28c: 0x2000, 0x28d: 0x2000, 0x28e: 0x2000, 0x28f: 0x2000, 0x290: 0x2000, 0x291: 0x2000, + 0x292: 0x2000, 0x293: 0x2000, 0x294: 0x2000, 0x295: 0x2000, 0x296: 0x2000, 0x297: 0x2000, + 0x298: 0x2000, 0x299: 0x2000, 0x29a: 0x2000, 0x29b: 0x2000, 0x29c: 0x2000, 0x29d: 0x2000, + 0x29e: 0x2000, 0x29f: 0x2000, 0x2a0: 0x2000, 0x2a1: 0x2000, 0x2a2: 0x2000, 0x2a3: 0x2000, + 0x2a4: 0x2000, 0x2a5: 0x2000, 0x2a6: 0x2000, 0x2a7: 0x2000, 0x2a8: 0x2000, 0x2a9: 0x2000, + 0x2aa: 0x2000, 0x2ab: 0x2000, 0x2ac: 0x2000, 0x2ad: 0x2000, 0x2ae: 0x2000, 0x2af: 0x2000, + 0x2b0: 0x2000, 0x2b1: 0x2000, 0x2b2: 0x2000, 0x2b3: 0x2000, 0x2b4: 0x2000, 0x2b5: 0x2000, + 0x2b6: 0x2000, 0x2b7: 0x2000, 0x2b8: 0x2000, 0x2b9: 0x2000, 0x2ba: 0x2000, 0x2bb: 0x2000, + 0x2bc: 0x2000, 0x2bd: 0x2000, 0x2be: 0x2000, 0x2bf: 0x2000, + // Block 0xb, offset 0x2c0 + 0x2c0: 0x2000, 0x2c1: 0x2000, 0x2c2: 0x2000, 0x2c3: 0x2000, 0x2c4: 0x2000, 0x2c5: 0x2000, + 0x2c6: 0x2000, 0x2c7: 0x2000, 0x2c8: 0x2000, 0x2c9: 0x2000, 0x2ca: 0x2000, 0x2cb: 0x2000, + 0x2cc: 0x2000, 0x2cd: 0x2000, 0x2ce: 0x2000, 0x2cf: 0x2000, 0x2d0: 0x2000, 0x2d1: 0x2000, + 0x2d2: 0x2000, 0x2d3: 0x2000, 0x2d4: 0x2000, 0x2d5: 0x2000, 0x2d6: 0x2000, 0x2d7: 0x2000, + 0x2d8: 0x2000, 0x2d9: 0x2000, 0x2da: 0x2000, 0x2db: 0x2000, 0x2dc: 0x2000, 0x2dd: 0x2000, + 0x2de: 0x2000, 0x2df: 0x2000, 0x2e0: 0x2000, 0x2e1: 0x2000, 0x2e2: 0x2000, 0x2e3: 0x2000, + 0x2e4: 0x2000, 0x2e5: 0x2000, 0x2e6: 0x2000, 0x2e7: 0x2000, 0x2e8: 0x2000, 0x2e9: 0x2000, + 0x2ea: 0x2000, 0x2eb: 0x2000, 0x2ec: 0x2000, 0x2ed: 0x2000, 0x2ee: 0x2000, 0x2ef: 0x2000, + // Block 0xc, offset 0x300 + 0x311: 0x2000, + 0x312: 0x2000, 0x313: 0x2000, 0x314: 0x2000, 0x315: 0x2000, 0x316: 0x2000, 0x317: 0x2000, + 0x318: 0x2000, 0x319: 0x2000, 0x31a: 0x2000, 0x31b: 0x2000, 0x31c: 0x2000, 0x31d: 0x2000, + 0x31e: 0x2000, 0x31f: 0x2000, 0x320: 0x2000, 0x321: 0x2000, 0x323: 0x2000, + 0x324: 0x2000, 0x325: 0x2000, 0x326: 0x2000, 0x327: 0x2000, 0x328: 0x2000, 0x329: 0x2000, + 0x331: 0x2000, 0x332: 0x2000, 0x333: 0x2000, 0x334: 0x2000, 0x335: 0x2000, + 0x336: 0x2000, 0x337: 0x2000, 0x338: 0x2000, 0x339: 0x2000, 0x33a: 0x2000, 0x33b: 0x2000, + 0x33c: 0x2000, 0x33d: 0x2000, 0x33e: 0x2000, 0x33f: 0x2000, + // Block 0xd, offset 0x340 + 0x340: 0x2000, 0x341: 0x2000, 0x343: 0x2000, 0x344: 0x2000, 0x345: 0x2000, + 0x346: 0x2000, 0x347: 0x2000, 0x348: 0x2000, 0x349: 0x2000, + // Block 0xe, offset 0x380 + 0x381: 0x2000, + 0x390: 0x2000, 0x391: 0x2000, + 0x392: 0x2000, 0x393: 0x2000, 0x394: 0x2000, 0x395: 0x2000, 0x396: 0x2000, 0x397: 0x2000, + 0x398: 0x2000, 0x399: 0x2000, 0x39a: 0x2000, 0x39b: 0x2000, 0x39c: 0x2000, 0x39d: 0x2000, + 0x39e: 0x2000, 0x39f: 0x2000, 0x3a0: 0x2000, 0x3a1: 0x2000, 0x3a2: 0x2000, 0x3a3: 0x2000, + 0x3a4: 0x2000, 0x3a5: 0x2000, 0x3a6: 0x2000, 0x3a7: 0x2000, 0x3a8: 0x2000, 0x3a9: 0x2000, + 0x3aa: 0x2000, 0x3ab: 0x2000, 0x3ac: 0x2000, 0x3ad: 0x2000, 0x3ae: 0x2000, 0x3af: 0x2000, + 0x3b0: 0x2000, 0x3b1: 0x2000, 0x3b2: 0x2000, 0x3b3: 0x2000, 0x3b4: 0x2000, 0x3b5: 0x2000, + 0x3b6: 0x2000, 0x3b7: 0x2000, 0x3b8: 0x2000, 0x3b9: 0x2000, 0x3ba: 0x2000, 0x3bb: 0x2000, + 0x3bc: 0x2000, 0x3bd: 0x2000, 0x3be: 0x2000, 0x3bf: 0x2000, + // Block 0xf, offset 0x3c0 + 0x3c0: 0x2000, 0x3c1: 0x2000, 0x3c2: 0x2000, 0x3c3: 0x2000, 0x3c4: 0x2000, 0x3c5: 0x2000, + 0x3c6: 0x2000, 0x3c7: 0x2000, 0x3c8: 0x2000, 0x3c9: 0x2000, 0x3ca: 0x2000, 0x3cb: 0x2000, + 0x3cc: 0x2000, 0x3cd: 0x2000, 0x3ce: 0x2000, 0x3cf: 0x2000, 0x3d1: 0x2000, + // Block 0x10, offset 0x400 + 0x400: 0x4000, 0x401: 0x4000, 0x402: 0x4000, 0x403: 0x4000, 0x404: 0x4000, 0x405: 0x4000, + 0x406: 0x4000, 0x407: 0x4000, 0x408: 0x4000, 0x409: 0x4000, 0x40a: 0x4000, 0x40b: 0x4000, + 0x40c: 0x4000, 0x40d: 0x4000, 0x40e: 0x4000, 0x40f: 0x4000, 0x410: 0x4000, 0x411: 0x4000, + 0x412: 0x4000, 0x413: 0x4000, 0x414: 0x4000, 0x415: 0x4000, 0x416: 0x4000, 0x417: 0x4000, + 0x418: 0x4000, 0x419: 0x4000, 0x41a: 0x4000, 0x41b: 0x4000, 0x41c: 0x4000, 0x41d: 0x4000, + 0x41e: 0x4000, 0x41f: 0x4000, 0x420: 0x4000, 0x421: 0x4000, 0x422: 0x4000, 0x423: 0x4000, + 0x424: 0x4000, 0x425: 0x4000, 0x426: 0x4000, 0x427: 0x4000, 0x428: 0x4000, 0x429: 0x4000, + 0x42a: 0x4000, 0x42b: 0x4000, 0x42c: 0x4000, 0x42d: 0x4000, 0x42e: 0x4000, 0x42f: 0x4000, + 0x430: 0x4000, 0x431: 0x4000, 0x432: 0x4000, 0x433: 0x4000, 0x434: 0x4000, 0x435: 0x4000, + 0x436: 0x4000, 0x437: 0x4000, 0x438: 0x4000, 0x439: 0x4000, 0x43a: 0x4000, 0x43b: 0x4000, + 0x43c: 0x4000, 0x43d: 0x4000, 0x43e: 0x4000, 0x43f: 0x4000, + // Block 0x11, offset 0x440 + 0x440: 0x4000, 0x441: 0x4000, 0x442: 0x4000, 0x443: 0x4000, 0x444: 0x4000, 0x445: 0x4000, + 0x446: 0x4000, 0x447: 0x4000, 0x448: 0x4000, 0x449: 0x4000, 0x44a: 0x4000, 0x44b: 0x4000, + 0x44c: 0x4000, 0x44d: 0x4000, 0x44e: 0x4000, 0x44f: 0x4000, 0x450: 0x4000, 0x451: 0x4000, + 0x452: 0x4000, 0x453: 0x4000, 0x454: 0x4000, 0x455: 0x4000, 0x456: 0x4000, 0x457: 0x4000, + 0x458: 0x4000, 0x459: 0x4000, 0x45a: 0x4000, 0x45b: 0x4000, 0x45c: 0x4000, 0x45d: 0x4000, + 0x45e: 0x4000, 0x45f: 0x4000, + // Block 0x12, offset 0x480 + 0x490: 0x2000, + 0x493: 0x2000, 0x494: 0x2000, 0x495: 0x2000, 0x496: 0x2000, + 0x498: 0x2000, 0x499: 0x2000, 0x49c: 0x2000, 0x49d: 0x2000, + 0x4a0: 0x2000, 0x4a1: 0x2000, 0x4a2: 0x2000, + 0x4a4: 0x2000, 0x4a5: 0x2000, 0x4a6: 0x2000, 0x4a7: 0x2000, + 0x4b0: 0x2000, 0x4b2: 0x2000, 0x4b3: 0x2000, 0x4b5: 0x2000, + 0x4bb: 0x2000, + 0x4be: 0x2000, + // Block 0x13, offset 0x4c0 + 0x4f4: 0x2000, + 0x4ff: 0x2000, + // Block 0x14, offset 0x500 + 0x501: 0x2000, 0x502: 0x2000, 0x503: 0x2000, 0x504: 0x2000, + 0x529: 0xa009, + 0x52c: 0x2000, + // Block 0x15, offset 0x540 + 0x543: 0x2000, 0x545: 0x2000, + 0x549: 0x2000, + 0x553: 0x2000, 0x556: 0x2000, + 0x561: 0x2000, 0x562: 0x2000, + 0x566: 0x2000, + 0x56b: 0x2000, + // Block 0x16, offset 0x580 + 0x593: 0x2000, 0x594: 0x2000, + 0x59b: 0x2000, 0x59c: 0x2000, 0x59d: 0x2000, + 0x59e: 0x2000, 0x5a0: 0x2000, 0x5a1: 0x2000, 0x5a2: 0x2000, 0x5a3: 0x2000, + 0x5a4: 0x2000, 0x5a5: 0x2000, 0x5a6: 0x2000, 0x5a7: 0x2000, 0x5a8: 0x2000, 0x5a9: 0x2000, + 0x5aa: 0x2000, 0x5ab: 0x2000, + 0x5b0: 0x2000, 0x5b1: 0x2000, 0x5b2: 0x2000, 0x5b3: 0x2000, 0x5b4: 0x2000, 0x5b5: 0x2000, + 0x5b6: 0x2000, 0x5b7: 0x2000, 0x5b8: 0x2000, 0x5b9: 0x2000, + // Block 0x17, offset 0x5c0 + 0x5c9: 0x2000, + 0x5d0: 0x200a, 0x5d1: 0x200b, + 0x5d2: 0x200a, 0x5d3: 0x200c, 0x5d4: 0x2000, 0x5d5: 0x2000, 0x5d6: 0x2000, 0x5d7: 0x2000, + 0x5d8: 0x2000, 0x5d9: 0x2000, + 0x5f8: 0x2000, 0x5f9: 0x2000, + // Block 0x18, offset 0x600 + 0x612: 0x2000, 0x614: 0x2000, + 0x627: 0x2000, + // Block 0x19, offset 0x640 + 0x640: 0x2000, 0x642: 0x2000, 0x643: 0x2000, + 0x647: 0x2000, 0x648: 0x2000, 0x64b: 0x2000, + 0x64f: 0x2000, 0x651: 0x2000, + 0x655: 0x2000, + 0x65a: 0x2000, 0x65d: 0x2000, + 0x65e: 0x2000, 0x65f: 0x2000, 0x660: 0x2000, 0x663: 0x2000, + 0x665: 0x2000, 0x667: 0x2000, 0x668: 0x2000, 0x669: 0x2000, + 0x66a: 0x2000, 0x66b: 0x2000, 0x66c: 0x2000, 0x66e: 0x2000, + 0x674: 0x2000, 0x675: 0x2000, + 0x676: 0x2000, 0x677: 0x2000, + 0x67c: 0x2000, 0x67d: 0x2000, + // Block 0x1a, offset 0x680 + 0x688: 0x2000, + 0x68c: 0x2000, + 0x692: 0x2000, + 0x6a0: 0x2000, 0x6a1: 0x2000, + 0x6a4: 0x2000, 0x6a5: 0x2000, 0x6a6: 0x2000, 0x6a7: 0x2000, + 0x6aa: 0x2000, 0x6ab: 0x2000, 0x6ae: 0x2000, 0x6af: 0x2000, + // Block 0x1b, offset 0x6c0 + 0x6c2: 0x2000, 0x6c3: 0x2000, + 0x6c6: 0x2000, 0x6c7: 0x2000, + 0x6d5: 0x2000, + 0x6d9: 0x2000, + 0x6e5: 0x2000, + 0x6ff: 0x2000, + // Block 0x1c, offset 0x700 + 0x712: 0x2000, + 0x71a: 0x4000, 0x71b: 0x4000, + 0x729: 0x4000, + 0x72a: 0x4000, + // Block 0x1d, offset 0x740 + 0x769: 0x4000, + 0x76a: 0x4000, 0x76b: 0x4000, 0x76c: 0x4000, + 0x770: 0x4000, 0x773: 0x4000, + // Block 0x1e, offset 0x780 + 0x7a0: 0x2000, 0x7a1: 0x2000, 0x7a2: 0x2000, 0x7a3: 0x2000, + 0x7a4: 0x2000, 0x7a5: 0x2000, 0x7a6: 0x2000, 0x7a7: 0x2000, 0x7a8: 0x2000, 0x7a9: 0x2000, + 0x7aa: 0x2000, 0x7ab: 0x2000, 0x7ac: 0x2000, 0x7ad: 0x2000, 0x7ae: 0x2000, 0x7af: 0x2000, + 0x7b0: 0x2000, 0x7b1: 0x2000, 0x7b2: 0x2000, 0x7b3: 0x2000, 0x7b4: 0x2000, 0x7b5: 0x2000, + 0x7b6: 0x2000, 0x7b7: 0x2000, 0x7b8: 0x2000, 0x7b9: 0x2000, 0x7ba: 0x2000, 0x7bb: 0x2000, + 0x7bc: 0x2000, 0x7bd: 0x2000, 0x7be: 0x2000, 0x7bf: 0x2000, + // Block 0x1f, offset 0x7c0 + 0x7c0: 0x2000, 0x7c1: 0x2000, 0x7c2: 0x2000, 0x7c3: 0x2000, 0x7c4: 0x2000, 0x7c5: 0x2000, + 0x7c6: 0x2000, 0x7c7: 0x2000, 0x7c8: 0x2000, 0x7c9: 0x2000, 0x7ca: 0x2000, 0x7cb: 0x2000, + 0x7cc: 0x2000, 0x7cd: 0x2000, 0x7ce: 0x2000, 0x7cf: 0x2000, 0x7d0: 0x2000, 0x7d1: 0x2000, + 0x7d2: 0x2000, 0x7d3: 0x2000, 0x7d4: 0x2000, 0x7d5: 0x2000, 0x7d6: 0x2000, 0x7d7: 0x2000, + 0x7d8: 0x2000, 0x7d9: 0x2000, 0x7da: 0x2000, 0x7db: 0x2000, 0x7dc: 0x2000, 0x7dd: 0x2000, + 0x7de: 0x2000, 0x7df: 0x2000, 0x7e0: 0x2000, 0x7e1: 0x2000, 0x7e2: 0x2000, 0x7e3: 0x2000, + 0x7e4: 0x2000, 0x7e5: 0x2000, 0x7e6: 0x2000, 0x7e7: 0x2000, 0x7e8: 0x2000, 0x7e9: 0x2000, + 0x7eb: 0x2000, 0x7ec: 0x2000, 0x7ed: 0x2000, 0x7ee: 0x2000, 0x7ef: 0x2000, + 0x7f0: 0x2000, 0x7f1: 0x2000, 0x7f2: 0x2000, 0x7f3: 0x2000, 0x7f4: 0x2000, 0x7f5: 0x2000, + 0x7f6: 0x2000, 0x7f7: 0x2000, 0x7f8: 0x2000, 0x7f9: 0x2000, 0x7fa: 0x2000, 0x7fb: 0x2000, + 0x7fc: 0x2000, 0x7fd: 0x2000, 0x7fe: 0x2000, 0x7ff: 0x2000, + // Block 0x20, offset 0x800 + 0x800: 0x2000, 0x801: 0x2000, 0x802: 0x200d, 0x803: 0x2000, 0x804: 0x2000, 0x805: 0x2000, + 0x806: 0x2000, 0x807: 0x2000, 0x808: 0x2000, 0x809: 0x2000, 0x80a: 0x2000, 0x80b: 0x2000, + 0x80c: 0x2000, 0x80d: 0x2000, 0x80e: 0x2000, 0x80f: 0x2000, 0x810: 0x2000, 0x811: 0x2000, + 0x812: 0x2000, 0x813: 0x2000, 0x814: 0x2000, 0x815: 0x2000, 0x816: 0x2000, 0x817: 0x2000, + 0x818: 0x2000, 0x819: 0x2000, 0x81a: 0x2000, 0x81b: 0x2000, 0x81c: 0x2000, 0x81d: 0x2000, + 0x81e: 0x2000, 0x81f: 0x2000, 0x820: 0x2000, 0x821: 0x2000, 0x822: 0x2000, 0x823: 0x2000, + 0x824: 0x2000, 0x825: 0x2000, 0x826: 0x2000, 0x827: 0x2000, 0x828: 0x2000, 0x829: 0x2000, + 0x82a: 0x2000, 0x82b: 0x2000, 0x82c: 0x2000, 0x82d: 0x2000, 0x82e: 0x2000, 0x82f: 0x2000, + 0x830: 0x2000, 0x831: 0x2000, 0x832: 0x2000, 0x833: 0x2000, 0x834: 0x2000, 0x835: 0x2000, + 0x836: 0x2000, 0x837: 0x2000, 0x838: 0x2000, 0x839: 0x2000, 0x83a: 0x2000, 0x83b: 0x2000, + 0x83c: 0x2000, 0x83d: 0x2000, 0x83e: 0x2000, 0x83f: 0x2000, + // Block 0x21, offset 0x840 + 0x840: 0x2000, 0x841: 0x2000, 0x842: 0x2000, 0x843: 0x2000, 0x844: 0x2000, 0x845: 0x2000, + 0x846: 0x2000, 0x847: 0x2000, 0x848: 0x2000, 0x849: 0x2000, 0x84a: 0x2000, 0x84b: 0x2000, + 0x850: 0x2000, 0x851: 0x2000, + 0x852: 0x2000, 0x853: 0x2000, 0x854: 0x2000, 0x855: 0x2000, 0x856: 0x2000, 0x857: 0x2000, + 0x858: 0x2000, 0x859: 0x2000, 0x85a: 0x2000, 0x85b: 0x2000, 0x85c: 0x2000, 0x85d: 0x2000, + 0x85e: 0x2000, 0x85f: 0x2000, 0x860: 0x2000, 0x861: 0x2000, 0x862: 0x2000, 0x863: 0x2000, + 0x864: 0x2000, 0x865: 0x2000, 0x866: 0x2000, 0x867: 0x2000, 0x868: 0x2000, 0x869: 0x2000, + 0x86a: 0x2000, 0x86b: 0x2000, 0x86c: 0x2000, 0x86d: 0x2000, 0x86e: 0x2000, 0x86f: 0x2000, + 0x870: 0x2000, 0x871: 0x2000, 0x872: 0x2000, 0x873: 0x2000, + // Block 0x22, offset 0x880 + 0x880: 0x2000, 0x881: 0x2000, 0x882: 0x2000, 0x883: 0x2000, 0x884: 0x2000, 0x885: 0x2000, + 0x886: 0x2000, 0x887: 0x2000, 0x888: 0x2000, 0x889: 0x2000, 0x88a: 0x2000, 0x88b: 0x2000, + 0x88c: 0x2000, 0x88d: 0x2000, 0x88e: 0x2000, 0x88f: 0x2000, + 0x892: 0x2000, 0x893: 0x2000, 0x894: 0x2000, 0x895: 0x2000, + 0x8a0: 0x200e, 0x8a1: 0x2000, 0x8a3: 0x2000, + 0x8a4: 0x2000, 0x8a5: 0x2000, 0x8a6: 0x2000, 0x8a7: 0x2000, 0x8a8: 0x2000, 0x8a9: 0x2000, + 0x8b2: 0x2000, 0x8b3: 0x2000, + 0x8b6: 0x2000, 0x8b7: 0x2000, + 0x8bc: 0x2000, 0x8bd: 0x2000, + // Block 0x23, offset 0x8c0 + 0x8c0: 0x2000, 0x8c1: 0x2000, + 0x8c6: 0x2000, 0x8c7: 0x2000, 0x8c8: 0x2000, 0x8cb: 0x200f, + 0x8ce: 0x2000, 0x8cf: 0x2000, 0x8d0: 0x2000, 0x8d1: 0x2000, + 0x8e2: 0x2000, 0x8e3: 0x2000, + 0x8e4: 0x2000, 0x8e5: 0x2000, + 0x8ef: 0x2000, + 0x8fd: 0x4000, 0x8fe: 0x4000, + // Block 0x24, offset 0x900 + 0x905: 0x2000, + 0x906: 0x2000, 0x909: 0x2000, + 0x90e: 0x2000, 0x90f: 0x2000, + 0x914: 0x4000, 0x915: 0x4000, + 0x91c: 0x2000, + 0x91e: 0x2000, + // Block 0x25, offset 0x940 + 0x940: 0x2000, 0x942: 0x2000, + 0x948: 0x4000, 0x949: 0x4000, 0x94a: 0x4000, 0x94b: 0x4000, + 0x94c: 0x4000, 0x94d: 0x4000, 0x94e: 0x4000, 0x94f: 0x4000, 0x950: 0x4000, 0x951: 0x4000, + 0x952: 0x4000, 0x953: 0x4000, + 0x960: 0x2000, 0x961: 0x2000, 0x963: 0x2000, + 0x964: 0x2000, 0x965: 0x2000, 0x967: 0x2000, 0x968: 0x2000, 0x969: 0x2000, + 0x96a: 0x2000, 0x96c: 0x2000, 0x96d: 0x2000, 0x96f: 0x2000, + 0x97f: 0x4000, + // Block 0x26, offset 0x980 + 0x993: 0x4000, + 0x99e: 0x2000, 0x99f: 0x2000, 0x9a1: 0x4000, + 0x9aa: 0x4000, 0x9ab: 0x4000, + 0x9bd: 0x4000, 0x9be: 0x4000, 0x9bf: 0x2000, + // Block 0x27, offset 0x9c0 + 0x9c4: 0x4000, 0x9c5: 0x4000, + 0x9c6: 0x2000, 0x9c7: 0x2000, 0x9c8: 0x2000, 0x9c9: 0x2000, 0x9ca: 0x2000, 0x9cb: 0x2000, + 0x9cc: 0x2000, 0x9cd: 0x2000, 0x9ce: 0x4000, 0x9cf: 0x2000, 0x9d0: 0x2000, 0x9d1: 0x2000, + 0x9d2: 0x2000, 0x9d3: 0x2000, 0x9d4: 0x4000, 0x9d5: 0x2000, 0x9d6: 0x2000, 0x9d7: 0x2000, + 0x9d8: 0x2000, 0x9d9: 0x2000, 0x9da: 0x2000, 0x9db: 0x2000, 0x9dc: 0x2000, 0x9dd: 0x2000, + 0x9de: 0x2000, 0x9df: 0x2000, 0x9e0: 0x2000, 0x9e1: 0x2000, 0x9e3: 0x2000, + 0x9e8: 0x2000, 0x9e9: 0x2000, + 0x9ea: 0x4000, 0x9eb: 0x2000, 0x9ec: 0x2000, 0x9ed: 0x2000, 0x9ee: 0x2000, 0x9ef: 0x2000, + 0x9f0: 0x2000, 0x9f1: 0x2000, 0x9f2: 0x4000, 0x9f3: 0x4000, 0x9f4: 0x2000, 0x9f5: 0x4000, + 0x9f6: 0x2000, 0x9f7: 0x2000, 0x9f8: 0x2000, 0x9f9: 0x2000, 0x9fa: 0x4000, 0x9fb: 0x2000, + 0x9fc: 0x2000, 0x9fd: 0x4000, 0x9fe: 0x2000, 0x9ff: 0x2000, + // Block 0x28, offset 0xa00 + 0xa05: 0x4000, + 0xa0a: 0x4000, 0xa0b: 0x4000, + 0xa28: 0x4000, + 0xa3d: 0x2000, + // Block 0x29, offset 0xa40 + 0xa4c: 0x4000, 0xa4e: 0x4000, + 0xa53: 0x4000, 0xa54: 0x4000, 0xa55: 0x4000, 0xa57: 0x4000, + 0xa76: 0x2000, 0xa77: 0x2000, 0xa78: 0x2000, 0xa79: 0x2000, 0xa7a: 0x2000, 0xa7b: 0x2000, + 0xa7c: 0x2000, 0xa7d: 0x2000, 0xa7e: 0x2000, 0xa7f: 0x2000, + // Block 0x2a, offset 0xa80 + 0xa95: 0x4000, 0xa96: 0x4000, 0xa97: 0x4000, + 0xab0: 0x4000, + 0xabf: 0x4000, + // Block 0x2b, offset 0xac0 + 0xae6: 0x6000, 0xae7: 0x6000, 0xae8: 0x6000, 0xae9: 0x6000, + 0xaea: 0x6000, 0xaeb: 0x6000, 0xaec: 0x6000, 0xaed: 0x6000, + // Block 0x2c, offset 0xb00 + 0xb05: 0x6010, + 0xb06: 0x6011, + // Block 0x2d, offset 0xb40 + 0xb5b: 0x4000, 0xb5c: 0x4000, + // Block 0x2e, offset 0xb80 + 0xb90: 0x4000, + 0xb95: 0x4000, 0xb96: 0x2000, 0xb97: 0x2000, + 0xb98: 0x2000, 0xb99: 0x2000, + // Block 0x2f, offset 0xbc0 + 0xbc0: 0x4000, 0xbc1: 0x4000, 0xbc2: 0x4000, 0xbc3: 0x4000, 0xbc4: 0x4000, 0xbc5: 0x4000, + 0xbc6: 0x4000, 0xbc7: 0x4000, 0xbc8: 0x4000, 0xbc9: 0x4000, 0xbca: 0x4000, 0xbcb: 0x4000, + 0xbcc: 0x4000, 0xbcd: 0x4000, 0xbce: 0x4000, 0xbcf: 0x4000, 0xbd0: 0x4000, 0xbd1: 0x4000, + 0xbd2: 0x4000, 0xbd3: 0x4000, 0xbd4: 0x4000, 0xbd5: 0x4000, 0xbd6: 0x4000, 0xbd7: 0x4000, + 0xbd8: 0x4000, 0xbd9: 0x4000, 0xbdb: 0x4000, 0xbdc: 0x4000, 0xbdd: 0x4000, + 0xbde: 0x4000, 0xbdf: 0x4000, 0xbe0: 0x4000, 0xbe1: 0x4000, 0xbe2: 0x4000, 0xbe3: 0x4000, + 0xbe4: 0x4000, 0xbe5: 0x4000, 0xbe6: 0x4000, 0xbe7: 0x4000, 0xbe8: 0x4000, 0xbe9: 0x4000, + 0xbea: 0x4000, 0xbeb: 0x4000, 0xbec: 0x4000, 0xbed: 0x4000, 0xbee: 0x4000, 0xbef: 0x4000, + 0xbf0: 0x4000, 0xbf1: 0x4000, 0xbf2: 0x4000, 0xbf3: 0x4000, 0xbf4: 0x4000, 0xbf5: 0x4000, + 0xbf6: 0x4000, 0xbf7: 0x4000, 0xbf8: 0x4000, 0xbf9: 0x4000, 0xbfa: 0x4000, 0xbfb: 0x4000, + 0xbfc: 0x4000, 0xbfd: 0x4000, 0xbfe: 0x4000, 0xbff: 0x4000, + // Block 0x30, offset 0xc00 + 0xc00: 0x4000, 0xc01: 0x4000, 0xc02: 0x4000, 0xc03: 0x4000, 0xc04: 0x4000, 0xc05: 0x4000, + 0xc06: 0x4000, 0xc07: 0x4000, 0xc08: 0x4000, 0xc09: 0x4000, 0xc0a: 0x4000, 0xc0b: 0x4000, + 0xc0c: 0x4000, 0xc0d: 0x4000, 0xc0e: 0x4000, 0xc0f: 0x4000, 0xc10: 0x4000, 0xc11: 0x4000, + 0xc12: 0x4000, 0xc13: 0x4000, 0xc14: 0x4000, 0xc15: 0x4000, 0xc16: 0x4000, 0xc17: 0x4000, + 0xc18: 0x4000, 0xc19: 0x4000, 0xc1a: 0x4000, 0xc1b: 0x4000, 0xc1c: 0x4000, 0xc1d: 0x4000, + 0xc1e: 0x4000, 0xc1f: 0x4000, 0xc20: 0x4000, 0xc21: 0x4000, 0xc22: 0x4000, 0xc23: 0x4000, + 0xc24: 0x4000, 0xc25: 0x4000, 0xc26: 0x4000, 0xc27: 0x4000, 0xc28: 0x4000, 0xc29: 0x4000, + 0xc2a: 0x4000, 0xc2b: 0x4000, 0xc2c: 0x4000, 0xc2d: 0x4000, 0xc2e: 0x4000, 0xc2f: 0x4000, + 0xc30: 0x4000, 0xc31: 0x4000, 0xc32: 0x4000, 0xc33: 0x4000, + // Block 0x31, offset 0xc40 + 0xc40: 0x4000, 0xc41: 0x4000, 0xc42: 0x4000, 0xc43: 0x4000, 0xc44: 0x4000, 0xc45: 0x4000, + 0xc46: 0x4000, 0xc47: 0x4000, 0xc48: 0x4000, 0xc49: 0x4000, 0xc4a: 0x4000, 0xc4b: 0x4000, + 0xc4c: 0x4000, 0xc4d: 0x4000, 0xc4e: 0x4000, 0xc4f: 0x4000, 0xc50: 0x4000, 0xc51: 0x4000, + 0xc52: 0x4000, 0xc53: 0x4000, 0xc54: 0x4000, 0xc55: 0x4000, + 0xc70: 0x4000, 0xc71: 0x4000, 0xc72: 0x4000, 0xc73: 0x4000, 0xc74: 0x4000, 0xc75: 0x4000, + 0xc76: 0x4000, 0xc77: 0x4000, 0xc78: 0x4000, 0xc79: 0x4000, 0xc7a: 0x4000, 0xc7b: 0x4000, + // Block 0x32, offset 0xc80 + 0xc80: 0x9012, 0xc81: 0x4013, 0xc82: 0x4014, 0xc83: 0x4000, 0xc84: 0x4000, 0xc85: 0x4000, + 0xc86: 0x4000, 0xc87: 0x4000, 0xc88: 0x4000, 0xc89: 0x4000, 0xc8a: 0x4000, 0xc8b: 0x4000, + 0xc8c: 0x4015, 0xc8d: 0x4015, 0xc8e: 0x4000, 0xc8f: 0x4000, 0xc90: 0x4000, 0xc91: 0x4000, + 0xc92: 0x4000, 0xc93: 0x4000, 0xc94: 0x4000, 0xc95: 0x4000, 0xc96: 0x4000, 0xc97: 0x4000, + 0xc98: 0x4000, 0xc99: 0x4000, 0xc9a: 0x4000, 0xc9b: 0x4000, 0xc9c: 0x4000, 0xc9d: 0x4000, + 0xc9e: 0x4000, 0xc9f: 0x4000, 0xca0: 0x4000, 0xca1: 0x4000, 0xca2: 0x4000, 0xca3: 0x4000, + 0xca4: 0x4000, 0xca5: 0x4000, 0xca6: 0x4000, 0xca7: 0x4000, 0xca8: 0x4000, 0xca9: 0x4000, + 0xcaa: 0x4000, 0xcab: 0x4000, 0xcac: 0x4000, 0xcad: 0x4000, 0xcae: 0x4000, 0xcaf: 0x4000, + 0xcb0: 0x4000, 0xcb1: 0x4000, 0xcb2: 0x4000, 0xcb3: 0x4000, 0xcb4: 0x4000, 0xcb5: 0x4000, + 0xcb6: 0x4000, 0xcb7: 0x4000, 0xcb8: 0x4000, 0xcb9: 0x4000, 0xcba: 0x4000, 0xcbb: 0x4000, + 0xcbc: 0x4000, 0xcbd: 0x4000, 0xcbe: 0x4000, + // Block 0x33, offset 0xcc0 + 0xcc1: 0x4000, 0xcc2: 0x4000, 0xcc3: 0x4000, 0xcc4: 0x4000, 0xcc5: 0x4000, + 0xcc6: 0x4000, 0xcc7: 0x4000, 0xcc8: 0x4000, 0xcc9: 0x4000, 0xcca: 0x4000, 0xccb: 0x4000, + 0xccc: 0x4000, 0xccd: 0x4000, 0xcce: 0x4000, 0xccf: 0x4000, 0xcd0: 0x4000, 0xcd1: 0x4000, + 0xcd2: 0x4000, 0xcd3: 0x4000, 0xcd4: 0x4000, 0xcd5: 0x4000, 0xcd6: 0x4000, 0xcd7: 0x4000, + 0xcd8: 0x4000, 0xcd9: 0x4000, 0xcda: 0x4000, 0xcdb: 0x4000, 0xcdc: 0x4000, 0xcdd: 0x4000, + 0xcde: 0x4000, 0xcdf: 0x4000, 0xce0: 0x4000, 0xce1: 0x4000, 0xce2: 0x4000, 0xce3: 0x4000, + 0xce4: 0x4000, 0xce5: 0x4000, 0xce6: 0x4000, 0xce7: 0x4000, 0xce8: 0x4000, 0xce9: 0x4000, + 0xcea: 0x4000, 0xceb: 0x4000, 0xcec: 0x4000, 0xced: 0x4000, 0xcee: 0x4000, 0xcef: 0x4000, + 0xcf0: 0x4000, 0xcf1: 0x4000, 0xcf2: 0x4000, 0xcf3: 0x4000, 0xcf4: 0x4000, 0xcf5: 0x4000, + 0xcf6: 0x4000, 0xcf7: 0x4000, 0xcf8: 0x4000, 0xcf9: 0x4000, 0xcfa: 0x4000, 0xcfb: 0x4000, + 0xcfc: 0x4000, 0xcfd: 0x4000, 0xcfe: 0x4000, 0xcff: 0x4000, + // Block 0x34, offset 0xd00 + 0xd00: 0x4000, 0xd01: 0x4000, 0xd02: 0x4000, 0xd03: 0x4000, 0xd04: 0x4000, 0xd05: 0x4000, + 0xd06: 0x4000, 0xd07: 0x4000, 0xd08: 0x4000, 0xd09: 0x4000, 0xd0a: 0x4000, 0xd0b: 0x4000, + 0xd0c: 0x4000, 0xd0d: 0x4000, 0xd0e: 0x4000, 0xd0f: 0x4000, 0xd10: 0x4000, 0xd11: 0x4000, + 0xd12: 0x4000, 0xd13: 0x4000, 0xd14: 0x4000, 0xd15: 0x4000, 0xd16: 0x4000, + 0xd19: 0x4016, 0xd1a: 0x4017, 0xd1b: 0x4000, 0xd1c: 0x4000, 0xd1d: 0x4000, + 0xd1e: 0x4000, 0xd1f: 0x4000, 0xd20: 0x4000, 0xd21: 0x4018, 0xd22: 0x4019, 0xd23: 0x401a, + 0xd24: 0x401b, 0xd25: 0x401c, 0xd26: 0x401d, 0xd27: 0x401e, 0xd28: 0x401f, 0xd29: 0x4020, + 0xd2a: 0x4021, 0xd2b: 0x4022, 0xd2c: 0x4000, 0xd2d: 0x4010, 0xd2e: 0x4000, 0xd2f: 0x4023, + 0xd30: 0x4000, 0xd31: 0x4024, 0xd32: 0x4000, 0xd33: 0x4025, 0xd34: 0x4000, 0xd35: 0x4026, + 0xd36: 0x4000, 0xd37: 0x401a, 0xd38: 0x4000, 0xd39: 0x4027, 0xd3a: 0x4000, 0xd3b: 0x4028, + 0xd3c: 0x4000, 0xd3d: 0x4020, 0xd3e: 0x4000, 0xd3f: 0x4029, + // Block 0x35, offset 0xd40 + 0xd40: 0x4000, 0xd41: 0x402a, 0xd42: 0x4000, 0xd43: 0x402b, 0xd44: 0x402c, 0xd45: 0x4000, + 0xd46: 0x4017, 0xd47: 0x4000, 0xd48: 0x402d, 0xd49: 0x4000, 0xd4a: 0x402e, 0xd4b: 0x402f, + 0xd4c: 0x4030, 0xd4d: 0x4017, 0xd4e: 0x4016, 0xd4f: 0x4017, 0xd50: 0x4000, 0xd51: 0x4000, + 0xd52: 0x4031, 0xd53: 0x4000, 0xd54: 0x4000, 0xd55: 0x4031, 0xd56: 0x4000, 0xd57: 0x4000, + 0xd58: 0x4032, 0xd59: 0x4000, 0xd5a: 0x4000, 0xd5b: 0x4032, 0xd5c: 0x4000, 0xd5d: 0x4000, + 0xd5e: 0x4033, 0xd5f: 0x402e, 0xd60: 0x4034, 0xd61: 0x4035, 0xd62: 0x4034, 0xd63: 0x4036, + 0xd64: 0x4037, 0xd65: 0x4024, 0xd66: 0x4035, 0xd67: 0x4025, 0xd68: 0x4038, 0xd69: 0x4038, + 0xd6a: 0x4039, 0xd6b: 0x4039, 0xd6c: 0x403a, 0xd6d: 0x403a, 0xd6e: 0x4000, 0xd6f: 0x4035, + 0xd70: 0x4000, 0xd71: 0x4000, 0xd72: 0x403b, 0xd73: 0x403c, 0xd74: 0x4000, 0xd75: 0x4000, + 0xd76: 0x4000, 0xd77: 0x4000, 0xd78: 0x4000, 0xd79: 0x4000, 0xd7a: 0x4000, 0xd7b: 0x403d, + 0xd7c: 0x401c, 0xd7d: 0x4000, 0xd7e: 0x4000, 0xd7f: 0x4000, + // Block 0x36, offset 0xd80 + 0xd85: 0x4000, + 0xd86: 0x4000, 0xd87: 0x4000, 0xd88: 0x4000, 0xd89: 0x4000, 0xd8a: 0x4000, 0xd8b: 0x4000, + 0xd8c: 0x4000, 0xd8d: 0x4000, 0xd8e: 0x4000, 0xd8f: 0x4000, 0xd90: 0x4000, 0xd91: 0x4000, + 0xd92: 0x4000, 0xd93: 0x4000, 0xd94: 0x4000, 0xd95: 0x4000, 0xd96: 0x4000, 0xd97: 0x4000, + 0xd98: 0x4000, 0xd99: 0x4000, 0xd9a: 0x4000, 0xd9b: 0x4000, 0xd9c: 0x4000, 0xd9d: 0x4000, + 0xd9e: 0x4000, 0xd9f: 0x4000, 0xda0: 0x4000, 0xda1: 0x4000, 0xda2: 0x4000, 0xda3: 0x4000, + 0xda4: 0x4000, 0xda5: 0x4000, 0xda6: 0x4000, 0xda7: 0x4000, 0xda8: 0x4000, 0xda9: 0x4000, + 0xdaa: 0x4000, 0xdab: 0x4000, 0xdac: 0x4000, 0xdad: 0x4000, 0xdae: 0x4000, 0xdaf: 0x4000, + 0xdb1: 0x403e, 0xdb2: 0x403e, 0xdb3: 0x403e, 0xdb4: 0x403e, 0xdb5: 0x403e, + 0xdb6: 0x403e, 0xdb7: 0x403e, 0xdb8: 0x403e, 0xdb9: 0x403e, 0xdba: 0x403e, 0xdbb: 0x403e, + 0xdbc: 0x403e, 0xdbd: 0x403e, 0xdbe: 0x403e, 0xdbf: 0x403e, + // Block 0x37, offset 0xdc0 + 0xdc0: 0x4037, 0xdc1: 0x4037, 0xdc2: 0x4037, 0xdc3: 0x4037, 0xdc4: 0x4037, 0xdc5: 0x4037, + 0xdc6: 0x4037, 0xdc7: 0x4037, 0xdc8: 0x4037, 0xdc9: 0x4037, 0xdca: 0x4037, 0xdcb: 0x4037, + 0xdcc: 0x4037, 0xdcd: 0x4037, 0xdce: 0x4037, 0xdcf: 0x400e, 0xdd0: 0x403f, 0xdd1: 0x4040, + 0xdd2: 0x4041, 0xdd3: 0x4040, 0xdd4: 0x403f, 0xdd5: 0x4042, 0xdd6: 0x4043, 0xdd7: 0x4044, + 0xdd8: 0x4040, 0xdd9: 0x4041, 0xdda: 0x4040, 0xddb: 0x4045, 0xddc: 0x4009, 0xddd: 0x4045, + 0xdde: 0x4046, 0xddf: 0x4045, 0xde0: 0x4047, 0xde1: 0x400b, 0xde2: 0x400a, 0xde3: 0x400c, + 0xde4: 0x4048, 0xde5: 0x4000, 0xde6: 0x4000, 0xde7: 0x4000, 0xde8: 0x4000, 0xde9: 0x4000, + 0xdea: 0x4000, 0xdeb: 0x4000, 0xdec: 0x4000, 0xded: 0x4000, 0xdee: 0x4000, 0xdef: 0x4000, + 0xdf0: 0x4000, 0xdf1: 0x4000, 0xdf2: 0x4000, 0xdf3: 0x4000, 0xdf4: 0x4000, 0xdf5: 0x4000, + 0xdf6: 0x4000, 0xdf7: 0x4000, 0xdf8: 0x4000, 0xdf9: 0x4000, 0xdfa: 0x4000, 0xdfb: 0x4000, + 0xdfc: 0x4000, 0xdfd: 0x4000, 0xdfe: 0x4000, 0xdff: 0x4000, + // Block 0x38, offset 0xe00 + 0xe00: 0x4000, 0xe01: 0x4000, 0xe02: 0x4000, 0xe03: 0x4000, 0xe04: 0x4000, 0xe05: 0x4000, + 0xe06: 0x4000, 0xe07: 0x4000, 0xe08: 0x4000, 0xe09: 0x4000, 0xe0a: 0x4000, 0xe0b: 0x4000, + 0xe0c: 0x4000, 0xe0d: 0x4000, 0xe0e: 0x4000, 0xe10: 0x4000, 0xe11: 0x4000, + 0xe12: 0x4000, 0xe13: 0x4000, 0xe14: 0x4000, 0xe15: 0x4000, 0xe16: 0x4000, 0xe17: 0x4000, + 0xe18: 0x4000, 0xe19: 0x4000, 0xe1a: 0x4000, 0xe1b: 0x4000, 0xe1c: 0x4000, 0xe1d: 0x4000, + 0xe1e: 0x4000, 0xe1f: 0x4000, 0xe20: 0x4000, 0xe21: 0x4000, 0xe22: 0x4000, 0xe23: 0x4000, + 0xe24: 0x4000, 0xe25: 0x4000, 0xe26: 0x4000, 0xe27: 0x4000, 0xe28: 0x4000, 0xe29: 0x4000, + 0xe2a: 0x4000, 0xe2b: 0x4000, 0xe2c: 0x4000, 0xe2d: 0x4000, 0xe2e: 0x4000, 0xe2f: 0x4000, + 0xe30: 0x4000, 0xe31: 0x4000, 0xe32: 0x4000, 0xe33: 0x4000, 0xe34: 0x4000, 0xe35: 0x4000, + 0xe36: 0x4000, 0xe37: 0x4000, 0xe38: 0x4000, 0xe39: 0x4000, 0xe3a: 0x4000, + // Block 0x39, offset 0xe40 + 0xe40: 0x4000, 0xe41: 0x4000, 0xe42: 0x4000, 0xe43: 0x4000, 0xe44: 0x4000, 0xe45: 0x4000, + 0xe46: 0x4000, 0xe47: 0x4000, 0xe48: 0x4000, 0xe49: 0x4000, 0xe4a: 0x4000, 0xe4b: 0x4000, + 0xe4c: 0x4000, 0xe4d: 0x4000, 0xe4e: 0x4000, 0xe4f: 0x4000, 0xe50: 0x4000, 0xe51: 0x4000, + 0xe52: 0x4000, 0xe53: 0x4000, 0xe54: 0x4000, 0xe55: 0x4000, 0xe56: 0x4000, 0xe57: 0x4000, + 0xe58: 0x4000, 0xe59: 0x4000, 0xe5a: 0x4000, 0xe5b: 0x4000, 0xe5c: 0x4000, 0xe5d: 0x4000, + 0xe5e: 0x4000, 0xe5f: 0x4000, 0xe60: 0x4000, 0xe61: 0x4000, 0xe62: 0x4000, 0xe63: 0x4000, + 0xe70: 0x4000, 0xe71: 0x4000, 0xe72: 0x4000, 0xe73: 0x4000, 0xe74: 0x4000, 0xe75: 0x4000, + 0xe76: 0x4000, 0xe77: 0x4000, 0xe78: 0x4000, 0xe79: 0x4000, 0xe7a: 0x4000, 0xe7b: 0x4000, + 0xe7c: 0x4000, 0xe7d: 0x4000, 0xe7e: 0x4000, 0xe7f: 0x4000, + // Block 0x3a, offset 0xe80 + 0xe80: 0x4000, 0xe81: 0x4000, 0xe82: 0x4000, 0xe83: 0x4000, 0xe84: 0x4000, 0xe85: 0x4000, + 0xe86: 0x4000, 0xe87: 0x4000, 0xe88: 0x4000, 0xe89: 0x4000, 0xe8a: 0x4000, 0xe8b: 0x4000, + 0xe8c: 0x4000, 0xe8d: 0x4000, 0xe8e: 0x4000, 0xe8f: 0x4000, 0xe90: 0x4000, 0xe91: 0x4000, + 0xe92: 0x4000, 0xe93: 0x4000, 0xe94: 0x4000, 0xe95: 0x4000, 0xe96: 0x4000, 0xe97: 0x4000, + 0xe98: 0x4000, 0xe99: 0x4000, 0xe9a: 0x4000, 0xe9b: 0x4000, 0xe9c: 0x4000, 0xe9d: 0x4000, + 0xe9e: 0x4000, 0xea0: 0x4000, 0xea1: 0x4000, 0xea2: 0x4000, 0xea3: 0x4000, + 0xea4: 0x4000, 0xea5: 0x4000, 0xea6: 0x4000, 0xea7: 0x4000, 0xea8: 0x4000, 0xea9: 0x4000, + 0xeaa: 0x4000, 0xeab: 0x4000, 0xeac: 0x4000, 0xead: 0x4000, 0xeae: 0x4000, 0xeaf: 0x4000, + 0xeb0: 0x4000, 0xeb1: 0x4000, 0xeb2: 0x4000, 0xeb3: 0x4000, 0xeb4: 0x4000, 0xeb5: 0x4000, + 0xeb6: 0x4000, 0xeb7: 0x4000, 0xeb8: 0x4000, 0xeb9: 0x4000, 0xeba: 0x4000, 0xebb: 0x4000, + 0xebc: 0x4000, 0xebd: 0x4000, 0xebe: 0x4000, 0xebf: 0x4000, + // Block 0x3b, offset 0xec0 + 0xec0: 0x4000, 0xec1: 0x4000, 0xec2: 0x4000, 0xec3: 0x4000, 0xec4: 0x4000, 0xec5: 0x4000, + 0xec6: 0x4000, 0xec7: 0x4000, 0xec8: 0x2000, 0xec9: 0x2000, 0xeca: 0x2000, 0xecb: 0x2000, + 0xecc: 0x2000, 0xecd: 0x2000, 0xece: 0x2000, 0xecf: 0x2000, 0xed0: 0x4000, 0xed1: 0x4000, + 0xed2: 0x4000, 0xed3: 0x4000, 0xed4: 0x4000, 0xed5: 0x4000, 0xed6: 0x4000, 0xed7: 0x4000, + 0xed8: 0x4000, 0xed9: 0x4000, 0xeda: 0x4000, 0xedb: 0x4000, 0xedc: 0x4000, 0xedd: 0x4000, + 0xede: 0x4000, 0xedf: 0x4000, 0xee0: 0x4000, 0xee1: 0x4000, 0xee2: 0x4000, 0xee3: 0x4000, + 0xee4: 0x4000, 0xee5: 0x4000, 0xee6: 0x4000, 0xee7: 0x4000, 0xee8: 0x4000, 0xee9: 0x4000, + 0xeea: 0x4000, 0xeeb: 0x4000, 0xeec: 0x4000, 0xeed: 0x4000, 0xeee: 0x4000, 0xeef: 0x4000, + 0xef0: 0x4000, 0xef1: 0x4000, 0xef2: 0x4000, 0xef3: 0x4000, 0xef4: 0x4000, 0xef5: 0x4000, + 0xef6: 0x4000, 0xef7: 0x4000, 0xef8: 0x4000, 0xef9: 0x4000, 0xefa: 0x4000, 0xefb: 0x4000, + 0xefc: 0x4000, 0xefd: 0x4000, 0xefe: 0x4000, 0xeff: 0x4000, + // Block 0x3c, offset 0xf00 + 0xf00: 0x4000, 0xf01: 0x4000, 0xf02: 0x4000, 0xf03: 0x4000, 0xf04: 0x4000, 0xf05: 0x4000, + 0xf06: 0x4000, 0xf07: 0x4000, 0xf08: 0x4000, 0xf09: 0x4000, 0xf0a: 0x4000, 0xf0b: 0x4000, + 0xf0c: 0x4000, 0xf0d: 0x4000, 0xf0e: 0x4000, 0xf0f: 0x4000, 0xf10: 0x4000, 0xf11: 0x4000, + 0xf12: 0x4000, 0xf13: 0x4000, 0xf14: 0x4000, 0xf15: 0x4000, 0xf16: 0x4000, 0xf17: 0x4000, + 0xf18: 0x4000, 0xf19: 0x4000, 0xf1a: 0x4000, 0xf1b: 0x4000, 0xf1c: 0x4000, 0xf1d: 0x4000, + 0xf1e: 0x4000, 0xf1f: 0x4000, 0xf20: 0x4000, 0xf21: 0x4000, 0xf22: 0x4000, 0xf23: 0x4000, + 0xf24: 0x4000, 0xf25: 0x4000, 0xf26: 0x4000, 0xf27: 0x4000, 0xf28: 0x4000, 0xf29: 0x4000, + 0xf2a: 0x4000, 0xf2b: 0x4000, 0xf2c: 0x4000, 0xf2d: 0x4000, 0xf2e: 0x4000, 0xf2f: 0x4000, + 0xf30: 0x4000, 0xf31: 0x4000, 0xf32: 0x4000, 0xf33: 0x4000, 0xf34: 0x4000, 0xf35: 0x4000, + 0xf36: 0x4000, 0xf37: 0x4000, 0xf38: 0x4000, 0xf39: 0x4000, 0xf3a: 0x4000, 0xf3b: 0x4000, + 0xf3c: 0x4000, 0xf3d: 0x4000, 0xf3e: 0x4000, + // Block 0x3d, offset 0xf40 + 0xf40: 0x4000, 0xf41: 0x4000, 0xf42: 0x4000, 0xf43: 0x4000, 0xf44: 0x4000, 0xf45: 0x4000, + 0xf46: 0x4000, 0xf47: 0x4000, 0xf48: 0x4000, 0xf49: 0x4000, 0xf4a: 0x4000, 0xf4b: 0x4000, + 0xf4c: 0x4000, 0xf50: 0x4000, 0xf51: 0x4000, + 0xf52: 0x4000, 0xf53: 0x4000, 0xf54: 0x4000, 0xf55: 0x4000, 0xf56: 0x4000, 0xf57: 0x4000, + 0xf58: 0x4000, 0xf59: 0x4000, 0xf5a: 0x4000, 0xf5b: 0x4000, 0xf5c: 0x4000, 0xf5d: 0x4000, + 0xf5e: 0x4000, 0xf5f: 0x4000, 0xf60: 0x4000, 0xf61: 0x4000, 0xf62: 0x4000, 0xf63: 0x4000, + 0xf64: 0x4000, 0xf65: 0x4000, 0xf66: 0x4000, 0xf67: 0x4000, 0xf68: 0x4000, 0xf69: 0x4000, + 0xf6a: 0x4000, 0xf6b: 0x4000, 0xf6c: 0x4000, 0xf6d: 0x4000, 0xf6e: 0x4000, 0xf6f: 0x4000, + 0xf70: 0x4000, 0xf71: 0x4000, 0xf72: 0x4000, 0xf73: 0x4000, 0xf74: 0x4000, 0xf75: 0x4000, + 0xf76: 0x4000, 0xf77: 0x4000, 0xf78: 0x4000, 0xf79: 0x4000, 0xf7a: 0x4000, 0xf7b: 0x4000, + 0xf7c: 0x4000, 0xf7d: 0x4000, 0xf7e: 0x4000, 0xf7f: 0x4000, + // Block 0x3e, offset 0xf80 + 0xf80: 0x4000, 0xf81: 0x4000, 0xf82: 0x4000, 0xf83: 0x4000, 0xf84: 0x4000, 0xf85: 0x4000, + 0xf86: 0x4000, + // Block 0x3f, offset 0xfc0 + 0xfe0: 0x4000, 0xfe1: 0x4000, 0xfe2: 0x4000, 0xfe3: 0x4000, + 0xfe4: 0x4000, 0xfe5: 0x4000, 0xfe6: 0x4000, 0xfe7: 0x4000, 0xfe8: 0x4000, 0xfe9: 0x4000, + 0xfea: 0x4000, 0xfeb: 0x4000, 0xfec: 0x4000, 0xfed: 0x4000, 0xfee: 0x4000, 0xfef: 0x4000, + 0xff0: 0x4000, 0xff1: 0x4000, 0xff2: 0x4000, 0xff3: 0x4000, 0xff4: 0x4000, 0xff5: 0x4000, + 0xff6: 0x4000, 0xff7: 0x4000, 0xff8: 0x4000, 0xff9: 0x4000, 0xffa: 0x4000, 0xffb: 0x4000, + 0xffc: 0x4000, + // Block 0x40, offset 0x1000 + 0x1000: 0x4000, 0x1001: 0x4000, 0x1002: 0x4000, 0x1003: 0x4000, 0x1004: 0x4000, 0x1005: 0x4000, + 0x1006: 0x4000, 0x1007: 0x4000, 0x1008: 0x4000, 0x1009: 0x4000, 0x100a: 0x4000, 0x100b: 0x4000, + 0x100c: 0x4000, 0x100d: 0x4000, 0x100e: 0x4000, 0x100f: 0x4000, 0x1010: 0x4000, 0x1011: 0x4000, + 0x1012: 0x4000, 0x1013: 0x4000, 0x1014: 0x4000, 0x1015: 0x4000, 0x1016: 0x4000, 0x1017: 0x4000, + 0x1018: 0x4000, 0x1019: 0x4000, 0x101a: 0x4000, 0x101b: 0x4000, 0x101c: 0x4000, 0x101d: 0x4000, + 0x101e: 0x4000, 0x101f: 0x4000, 0x1020: 0x4000, 0x1021: 0x4000, 0x1022: 0x4000, 0x1023: 0x4000, + // Block 0x41, offset 0x1040 + 0x1040: 0x2000, 0x1041: 0x2000, 0x1042: 0x2000, 0x1043: 0x2000, 0x1044: 0x2000, 0x1045: 0x2000, + 0x1046: 0x2000, 0x1047: 0x2000, 0x1048: 0x2000, 0x1049: 0x2000, 0x104a: 0x2000, 0x104b: 0x2000, + 0x104c: 0x2000, 0x104d: 0x2000, 0x104e: 0x2000, 0x104f: 0x2000, 0x1050: 0x4000, 0x1051: 0x4000, + 0x1052: 0x4000, 0x1053: 0x4000, 0x1054: 0x4000, 0x1055: 0x4000, 0x1056: 0x4000, 0x1057: 0x4000, + 0x1058: 0x4000, 0x1059: 0x4000, + 0x1070: 0x4000, 0x1071: 0x4000, 0x1072: 0x4000, 0x1073: 0x4000, 0x1074: 0x4000, 0x1075: 0x4000, + 0x1076: 0x4000, 0x1077: 0x4000, 0x1078: 0x4000, 0x1079: 0x4000, 0x107a: 0x4000, 0x107b: 0x4000, + 0x107c: 0x4000, 0x107d: 0x4000, 0x107e: 0x4000, 0x107f: 0x4000, + // Block 0x42, offset 0x1080 + 0x1080: 0x4000, 0x1081: 0x4000, 0x1082: 0x4000, 0x1083: 0x4000, 0x1084: 0x4000, 0x1085: 0x4000, + 0x1086: 0x4000, 0x1087: 0x4000, 0x1088: 0x4000, 0x1089: 0x4000, 0x108a: 0x4000, 0x108b: 0x4000, + 0x108c: 0x4000, 0x108d: 0x4000, 0x108e: 0x4000, 0x108f: 0x4000, 0x1090: 0x4000, 0x1091: 0x4000, + 0x1092: 0x4000, 0x1094: 0x4000, 0x1095: 0x4000, 0x1096: 0x4000, 0x1097: 0x4000, + 0x1098: 0x4000, 0x1099: 0x4000, 0x109a: 0x4000, 0x109b: 0x4000, 0x109c: 0x4000, 0x109d: 0x4000, + 0x109e: 0x4000, 0x109f: 0x4000, 0x10a0: 0x4000, 0x10a1: 0x4000, 0x10a2: 0x4000, 0x10a3: 0x4000, + 0x10a4: 0x4000, 0x10a5: 0x4000, 0x10a6: 0x4000, 0x10a8: 0x4000, 0x10a9: 0x4000, + 0x10aa: 0x4000, 0x10ab: 0x4000, + // Block 0x43, offset 0x10c0 + 0x10c1: 0x9012, 0x10c2: 0x9012, 0x10c3: 0x9012, 0x10c4: 0x9012, 0x10c5: 0x9012, + 0x10c6: 0x9012, 0x10c7: 0x9012, 0x10c8: 0x9012, 0x10c9: 0x9012, 0x10ca: 0x9012, 0x10cb: 0x9012, + 0x10cc: 0x9012, 0x10cd: 0x9012, 0x10ce: 0x9012, 0x10cf: 0x9012, 0x10d0: 0x9012, 0x10d1: 0x9012, + 0x10d2: 0x9012, 0x10d3: 0x9012, 0x10d4: 0x9012, 0x10d5: 0x9012, 0x10d6: 0x9012, 0x10d7: 0x9012, + 0x10d8: 0x9012, 0x10d9: 0x9012, 0x10da: 0x9012, 0x10db: 0x9012, 0x10dc: 0x9012, 0x10dd: 0x9012, + 0x10de: 0x9012, 0x10df: 0x9012, 0x10e0: 0x9049, 0x10e1: 0x9049, 0x10e2: 0x9049, 0x10e3: 0x9049, + 0x10e4: 0x9049, 0x10e5: 0x9049, 0x10e6: 0x9049, 0x10e7: 0x9049, 0x10e8: 0x9049, 0x10e9: 0x9049, + 0x10ea: 0x9049, 0x10eb: 0x9049, 0x10ec: 0x9049, 0x10ed: 0x9049, 0x10ee: 0x9049, 0x10ef: 0x9049, + 0x10f0: 0x9049, 0x10f1: 0x9049, 0x10f2: 0x9049, 0x10f3: 0x9049, 0x10f4: 0x9049, 0x10f5: 0x9049, + 0x10f6: 0x9049, 0x10f7: 0x9049, 0x10f8: 0x9049, 0x10f9: 0x9049, 0x10fa: 0x9049, 0x10fb: 0x9049, + 0x10fc: 0x9049, 0x10fd: 0x9049, 0x10fe: 0x9049, 0x10ff: 0x9049, + // Block 0x44, offset 0x1100 + 0x1100: 0x9049, 0x1101: 0x9049, 0x1102: 0x9049, 0x1103: 0x9049, 0x1104: 0x9049, 0x1105: 0x9049, + 0x1106: 0x9049, 0x1107: 0x9049, 0x1108: 0x9049, 0x1109: 0x9049, 0x110a: 0x9049, 0x110b: 0x9049, + 0x110c: 0x9049, 0x110d: 0x9049, 0x110e: 0x9049, 0x110f: 0x9049, 0x1110: 0x9049, 0x1111: 0x9049, + 0x1112: 0x9049, 0x1113: 0x9049, 0x1114: 0x9049, 0x1115: 0x9049, 0x1116: 0x9049, 0x1117: 0x9049, + 0x1118: 0x9049, 0x1119: 0x9049, 0x111a: 0x9049, 0x111b: 0x9049, 0x111c: 0x9049, 0x111d: 0x9049, + 0x111e: 0x9049, 0x111f: 0x904a, 0x1120: 0x904b, 0x1121: 0xb04c, 0x1122: 0xb04d, 0x1123: 0xb04d, + 0x1124: 0xb04e, 0x1125: 0xb04f, 0x1126: 0xb050, 0x1127: 0xb051, 0x1128: 0xb052, 0x1129: 0xb053, + 0x112a: 0xb054, 0x112b: 0xb055, 0x112c: 0xb056, 0x112d: 0xb057, 0x112e: 0xb058, 0x112f: 0xb059, + 0x1130: 0xb05a, 0x1131: 0xb05b, 0x1132: 0xb05c, 0x1133: 0xb05d, 0x1134: 0xb05e, 0x1135: 0xb05f, + 0x1136: 0xb060, 0x1137: 0xb061, 0x1138: 0xb062, 0x1139: 0xb063, 0x113a: 0xb064, 0x113b: 0xb065, + 0x113c: 0xb052, 0x113d: 0xb066, 0x113e: 0xb067, 0x113f: 0xb055, + // Block 0x45, offset 0x1140 + 0x1140: 0xb068, 0x1141: 0xb069, 0x1142: 0xb06a, 0x1143: 0xb06b, 0x1144: 0xb05a, 0x1145: 0xb056, + 0x1146: 0xb06c, 0x1147: 0xb06d, 0x1148: 0xb06b, 0x1149: 0xb06e, 0x114a: 0xb06b, 0x114b: 0xb06f, + 0x114c: 0xb06f, 0x114d: 0xb070, 0x114e: 0xb070, 0x114f: 0xb071, 0x1150: 0xb056, 0x1151: 0xb072, + 0x1152: 0xb073, 0x1153: 0xb072, 0x1154: 0xb074, 0x1155: 0xb073, 0x1156: 0xb075, 0x1157: 0xb075, + 0x1158: 0xb076, 0x1159: 0xb076, 0x115a: 0xb077, 0x115b: 0xb077, 0x115c: 0xb073, 0x115d: 0xb078, + 0x115e: 0xb079, 0x115f: 0xb067, 0x1160: 0xb07a, 0x1161: 0xb07b, 0x1162: 0xb07b, 0x1163: 0xb07b, + 0x1164: 0xb07b, 0x1165: 0xb07b, 0x1166: 0xb07b, 0x1167: 0xb07b, 0x1168: 0xb07b, 0x1169: 0xb07b, + 0x116a: 0xb07b, 0x116b: 0xb07b, 0x116c: 0xb07b, 0x116d: 0xb07b, 0x116e: 0xb07b, 0x116f: 0xb07b, + 0x1170: 0xb07c, 0x1171: 0xb07c, 0x1172: 0xb07c, 0x1173: 0xb07c, 0x1174: 0xb07c, 0x1175: 0xb07c, + 0x1176: 0xb07c, 0x1177: 0xb07c, 0x1178: 0xb07c, 0x1179: 0xb07c, 0x117a: 0xb07c, 0x117b: 0xb07c, + 0x117c: 0xb07c, 0x117d: 0xb07c, 0x117e: 0xb07c, + // Block 0x46, offset 0x1180 + 0x1182: 0xb07d, 0x1183: 0xb07e, 0x1184: 0xb07f, 0x1185: 0xb080, + 0x1186: 0xb07f, 0x1187: 0xb07e, 0x118a: 0xb081, 0x118b: 0xb082, + 0x118c: 0xb083, 0x118d: 0xb07f, 0x118e: 0xb080, 0x118f: 0xb07f, + 0x1192: 0xb084, 0x1193: 0xb085, 0x1194: 0xb084, 0x1195: 0xb086, 0x1196: 0xb084, 0x1197: 0xb087, + 0x119a: 0xb088, 0x119b: 0xb089, 0x119c: 0xb08a, + 0x11a0: 0x908b, 0x11a1: 0x908b, 0x11a2: 0x908c, 0x11a3: 0x908d, + 0x11a4: 0x908b, 0x11a5: 0x908e, 0x11a6: 0x908f, 0x11a8: 0xb090, 0x11a9: 0xb091, + 0x11aa: 0xb092, 0x11ab: 0xb091, 0x11ac: 0xb093, 0x11ad: 0xb094, 0x11ae: 0xb095, + 0x11bd: 0x2000, + // Block 0x47, offset 0x11c0 + 0x11e0: 0x4000, 0x11e1: 0x4000, 0x11e2: 0x4000, 0x11e3: 0x4000, + // Block 0x48, offset 0x1200 + 0x1200: 0x4000, 0x1201: 0x4000, 0x1202: 0x4000, 0x1203: 0x4000, 0x1204: 0x4000, 0x1205: 0x4000, + 0x1206: 0x4000, 0x1207: 0x4000, 0x1208: 0x4000, 0x1209: 0x4000, 0x120a: 0x4000, 0x120b: 0x4000, + 0x120c: 0x4000, 0x120d: 0x4000, 0x120e: 0x4000, 0x120f: 0x4000, 0x1210: 0x4000, 0x1211: 0x4000, + 0x1212: 0x4000, 0x1213: 0x4000, 0x1214: 0x4000, 0x1215: 0x4000, 0x1216: 0x4000, 0x1217: 0x4000, + 0x1218: 0x4000, 0x1219: 0x4000, 0x121a: 0x4000, 0x121b: 0x4000, 0x121c: 0x4000, 0x121d: 0x4000, + 0x121e: 0x4000, 0x121f: 0x4000, 0x1220: 0x4000, 0x1221: 0x4000, 0x1222: 0x4000, 0x1223: 0x4000, + 0x1224: 0x4000, 0x1225: 0x4000, 0x1226: 0x4000, 0x1227: 0x4000, 0x1228: 0x4000, 0x1229: 0x4000, + 0x122a: 0x4000, 0x122b: 0x4000, 0x122c: 0x4000, 0x122d: 0x4000, 0x122e: 0x4000, 0x122f: 0x4000, + 0x1230: 0x4000, 0x1231: 0x4000, 0x1232: 0x4000, 0x1233: 0x4000, 0x1234: 0x4000, 0x1235: 0x4000, + 0x1236: 0x4000, 0x1237: 0x4000, + // Block 0x49, offset 0x1240 + 0x1240: 0x4000, 0x1241: 0x4000, 0x1242: 0x4000, 0x1243: 0x4000, 0x1244: 0x4000, 0x1245: 0x4000, + 0x1246: 0x4000, 0x1247: 0x4000, 0x1248: 0x4000, 0x1249: 0x4000, 0x124a: 0x4000, 0x124b: 0x4000, + 0x124c: 0x4000, 0x124d: 0x4000, 0x124e: 0x4000, 0x124f: 0x4000, 0x1250: 0x4000, 0x1251: 0x4000, + 0x1252: 0x4000, 0x1253: 0x4000, 0x1254: 0x4000, 0x1255: 0x4000, 0x1256: 0x4000, 0x1257: 0x4000, + 0x1258: 0x4000, 0x1259: 0x4000, 0x125a: 0x4000, 0x125b: 0x4000, 0x125c: 0x4000, 0x125d: 0x4000, + 0x125e: 0x4000, 0x125f: 0x4000, 0x1260: 0x4000, 0x1261: 0x4000, 0x1262: 0x4000, 0x1263: 0x4000, + 0x1264: 0x4000, 0x1265: 0x4000, 0x1266: 0x4000, 0x1267: 0x4000, 0x1268: 0x4000, 0x1269: 0x4000, + 0x126a: 0x4000, 0x126b: 0x4000, 0x126c: 0x4000, 0x126d: 0x4000, 0x126e: 0x4000, 0x126f: 0x4000, + 0x1270: 0x4000, 0x1271: 0x4000, 0x1272: 0x4000, + // Block 0x4a, offset 0x1280 + 0x1280: 0x4000, 0x1281: 0x4000, 0x1282: 0x4000, 0x1283: 0x4000, 0x1284: 0x4000, 0x1285: 0x4000, + 0x1286: 0x4000, 0x1287: 0x4000, 0x1288: 0x4000, 0x1289: 0x4000, 0x128a: 0x4000, 0x128b: 0x4000, + 0x128c: 0x4000, 0x128d: 0x4000, 0x128e: 0x4000, 0x128f: 0x4000, 0x1290: 0x4000, 0x1291: 0x4000, + 0x1292: 0x4000, 0x1293: 0x4000, 0x1294: 0x4000, 0x1295: 0x4000, 0x1296: 0x4000, 0x1297: 0x4000, + 0x1298: 0x4000, 0x1299: 0x4000, 0x129a: 0x4000, 0x129b: 0x4000, 0x129c: 0x4000, 0x129d: 0x4000, + 0x129e: 0x4000, + // Block 0x4b, offset 0x12c0 + 0x12d0: 0x4000, 0x12d1: 0x4000, + 0x12d2: 0x4000, + 0x12e4: 0x4000, 0x12e5: 0x4000, 0x12e6: 0x4000, 0x12e7: 0x4000, + 0x12f0: 0x4000, 0x12f1: 0x4000, 0x12f2: 0x4000, 0x12f3: 0x4000, 0x12f4: 0x4000, 0x12f5: 0x4000, + 0x12f6: 0x4000, 0x12f7: 0x4000, 0x12f8: 0x4000, 0x12f9: 0x4000, 0x12fa: 0x4000, 0x12fb: 0x4000, + 0x12fc: 0x4000, 0x12fd: 0x4000, 0x12fe: 0x4000, 0x12ff: 0x4000, + // Block 0x4c, offset 0x1300 + 0x1300: 0x4000, 0x1301: 0x4000, 0x1302: 0x4000, 0x1303: 0x4000, 0x1304: 0x4000, 0x1305: 0x4000, + 0x1306: 0x4000, 0x1307: 0x4000, 0x1308: 0x4000, 0x1309: 0x4000, 0x130a: 0x4000, 0x130b: 0x4000, + 0x130c: 0x4000, 0x130d: 0x4000, 0x130e: 0x4000, 0x130f: 0x4000, 0x1310: 0x4000, 0x1311: 0x4000, + 0x1312: 0x4000, 0x1313: 0x4000, 0x1314: 0x4000, 0x1315: 0x4000, 0x1316: 0x4000, 0x1317: 0x4000, + 0x1318: 0x4000, 0x1319: 0x4000, 0x131a: 0x4000, 0x131b: 0x4000, 0x131c: 0x4000, 0x131d: 0x4000, + 0x131e: 0x4000, 0x131f: 0x4000, 0x1320: 0x4000, 0x1321: 0x4000, 0x1322: 0x4000, 0x1323: 0x4000, + 0x1324: 0x4000, 0x1325: 0x4000, 0x1326: 0x4000, 0x1327: 0x4000, 0x1328: 0x4000, 0x1329: 0x4000, + 0x132a: 0x4000, 0x132b: 0x4000, 0x132c: 0x4000, 0x132d: 0x4000, 0x132e: 0x4000, 0x132f: 0x4000, + 0x1330: 0x4000, 0x1331: 0x4000, 0x1332: 0x4000, 0x1333: 0x4000, 0x1334: 0x4000, 0x1335: 0x4000, + 0x1336: 0x4000, 0x1337: 0x4000, 0x1338: 0x4000, 0x1339: 0x4000, 0x133a: 0x4000, 0x133b: 0x4000, + // Block 0x4d, offset 0x1340 + 0x1344: 0x4000, + // Block 0x4e, offset 0x1380 + 0x138f: 0x4000, + // Block 0x4f, offset 0x13c0 + 0x13c0: 0x2000, 0x13c1: 0x2000, 0x13c2: 0x2000, 0x13c3: 0x2000, 0x13c4: 0x2000, 0x13c5: 0x2000, + 0x13c6: 0x2000, 0x13c7: 0x2000, 0x13c8: 0x2000, 0x13c9: 0x2000, 0x13ca: 0x2000, + 0x13d0: 0x2000, 0x13d1: 0x2000, + 0x13d2: 0x2000, 0x13d3: 0x2000, 0x13d4: 0x2000, 0x13d5: 0x2000, 0x13d6: 0x2000, 0x13d7: 0x2000, + 0x13d8: 0x2000, 0x13d9: 0x2000, 0x13da: 0x2000, 0x13db: 0x2000, 0x13dc: 0x2000, 0x13dd: 0x2000, + 0x13de: 0x2000, 0x13df: 0x2000, 0x13e0: 0x2000, 0x13e1: 0x2000, 0x13e2: 0x2000, 0x13e3: 0x2000, + 0x13e4: 0x2000, 0x13e5: 0x2000, 0x13e6: 0x2000, 0x13e7: 0x2000, 0x13e8: 0x2000, 0x13e9: 0x2000, + 0x13ea: 0x2000, 0x13eb: 0x2000, 0x13ec: 0x2000, 0x13ed: 0x2000, + 0x13f0: 0x2000, 0x13f1: 0x2000, 0x13f2: 0x2000, 0x13f3: 0x2000, 0x13f4: 0x2000, 0x13f5: 0x2000, + 0x13f6: 0x2000, 0x13f7: 0x2000, 0x13f8: 0x2000, 0x13f9: 0x2000, 0x13fa: 0x2000, 0x13fb: 0x2000, + 0x13fc: 0x2000, 0x13fd: 0x2000, 0x13fe: 0x2000, 0x13ff: 0x2000, + // Block 0x50, offset 0x1400 + 0x1400: 0x2000, 0x1401: 0x2000, 0x1402: 0x2000, 0x1403: 0x2000, 0x1404: 0x2000, 0x1405: 0x2000, + 0x1406: 0x2000, 0x1407: 0x2000, 0x1408: 0x2000, 0x1409: 0x2000, 0x140a: 0x2000, 0x140b: 0x2000, + 0x140c: 0x2000, 0x140d: 0x2000, 0x140e: 0x2000, 0x140f: 0x2000, 0x1410: 0x2000, 0x1411: 0x2000, + 0x1412: 0x2000, 0x1413: 0x2000, 0x1414: 0x2000, 0x1415: 0x2000, 0x1416: 0x2000, 0x1417: 0x2000, + 0x1418: 0x2000, 0x1419: 0x2000, 0x141a: 0x2000, 0x141b: 0x2000, 0x141c: 0x2000, 0x141d: 0x2000, + 0x141e: 0x2000, 0x141f: 0x2000, 0x1420: 0x2000, 0x1421: 0x2000, 0x1422: 0x2000, 0x1423: 0x2000, + 0x1424: 0x2000, 0x1425: 0x2000, 0x1426: 0x2000, 0x1427: 0x2000, 0x1428: 0x2000, 0x1429: 0x2000, + 0x1430: 0x2000, 0x1431: 0x2000, 0x1432: 0x2000, 0x1433: 0x2000, 0x1434: 0x2000, 0x1435: 0x2000, + 0x1436: 0x2000, 0x1437: 0x2000, 0x1438: 0x2000, 0x1439: 0x2000, 0x143a: 0x2000, 0x143b: 0x2000, + 0x143c: 0x2000, 0x143d: 0x2000, 0x143e: 0x2000, 0x143f: 0x2000, + // Block 0x51, offset 0x1440 + 0x1440: 0x2000, 0x1441: 0x2000, 0x1442: 0x2000, 0x1443: 0x2000, 0x1444: 0x2000, 0x1445: 0x2000, + 0x1446: 0x2000, 0x1447: 0x2000, 0x1448: 0x2000, 0x1449: 0x2000, 0x144a: 0x2000, 0x144b: 0x2000, + 0x144c: 0x2000, 0x144d: 0x2000, 0x144e: 0x4000, 0x144f: 0x2000, 0x1450: 0x2000, 0x1451: 0x4000, + 0x1452: 0x4000, 0x1453: 0x4000, 0x1454: 0x4000, 0x1455: 0x4000, 0x1456: 0x4000, 0x1457: 0x4000, + 0x1458: 0x4000, 0x1459: 0x4000, 0x145a: 0x4000, 0x145b: 0x2000, 0x145c: 0x2000, 0x145d: 0x2000, + 0x145e: 0x2000, 0x145f: 0x2000, 0x1460: 0x2000, 0x1461: 0x2000, 0x1462: 0x2000, 0x1463: 0x2000, + 0x1464: 0x2000, 0x1465: 0x2000, 0x1466: 0x2000, 0x1467: 0x2000, 0x1468: 0x2000, 0x1469: 0x2000, + 0x146a: 0x2000, 0x146b: 0x2000, 0x146c: 0x2000, + // Block 0x52, offset 0x1480 + 0x1480: 0x4000, 0x1481: 0x4000, 0x1482: 0x4000, + 0x1490: 0x4000, 0x1491: 0x4000, + 0x1492: 0x4000, 0x1493: 0x4000, 0x1494: 0x4000, 0x1495: 0x4000, 0x1496: 0x4000, 0x1497: 0x4000, + 0x1498: 0x4000, 0x1499: 0x4000, 0x149a: 0x4000, 0x149b: 0x4000, 0x149c: 0x4000, 0x149d: 0x4000, + 0x149e: 0x4000, 0x149f: 0x4000, 0x14a0: 0x4000, 0x14a1: 0x4000, 0x14a2: 0x4000, 0x14a3: 0x4000, + 0x14a4: 0x4000, 0x14a5: 0x4000, 0x14a6: 0x4000, 0x14a7: 0x4000, 0x14a8: 0x4000, 0x14a9: 0x4000, + 0x14aa: 0x4000, 0x14ab: 0x4000, 0x14ac: 0x4000, 0x14ad: 0x4000, 0x14ae: 0x4000, 0x14af: 0x4000, + 0x14b0: 0x4000, 0x14b1: 0x4000, 0x14b2: 0x4000, 0x14b3: 0x4000, 0x14b4: 0x4000, 0x14b5: 0x4000, + 0x14b6: 0x4000, 0x14b7: 0x4000, 0x14b8: 0x4000, 0x14b9: 0x4000, 0x14ba: 0x4000, 0x14bb: 0x4000, + // Block 0x53, offset 0x14c0 + 0x14c0: 0x4000, 0x14c1: 0x4000, 0x14c2: 0x4000, 0x14c3: 0x4000, 0x14c4: 0x4000, 0x14c5: 0x4000, + 0x14c6: 0x4000, 0x14c7: 0x4000, 0x14c8: 0x4000, + 0x14d0: 0x4000, 0x14d1: 0x4000, + 0x14e0: 0x4000, 0x14e1: 0x4000, 0x14e2: 0x4000, 0x14e3: 0x4000, + 0x14e4: 0x4000, 0x14e5: 0x4000, + // Block 0x54, offset 0x1500 + 0x1500: 0x4000, 0x1501: 0x4000, 0x1502: 0x4000, 0x1503: 0x4000, 0x1504: 0x4000, 0x1505: 0x4000, + 0x1506: 0x4000, 0x1507: 0x4000, 0x1508: 0x4000, 0x1509: 0x4000, 0x150a: 0x4000, 0x150b: 0x4000, + 0x150c: 0x4000, 0x150d: 0x4000, 0x150e: 0x4000, 0x150f: 0x4000, 0x1510: 0x4000, 0x1511: 0x4000, + 0x1512: 0x4000, 0x1513: 0x4000, 0x1514: 0x4000, 0x1515: 0x4000, 0x1516: 0x4000, 0x1517: 0x4000, + 0x1518: 0x4000, 0x1519: 0x4000, 0x151a: 0x4000, 0x151b: 0x4000, 0x151c: 0x4000, 0x151d: 0x4000, + 0x151e: 0x4000, 0x151f: 0x4000, 0x1520: 0x4000, + 0x152d: 0x4000, 0x152e: 0x4000, 0x152f: 0x4000, + 0x1530: 0x4000, 0x1531: 0x4000, 0x1532: 0x4000, 0x1533: 0x4000, 0x1534: 0x4000, 0x1535: 0x4000, + 0x1537: 0x4000, 0x1538: 0x4000, 0x1539: 0x4000, 0x153a: 0x4000, 0x153b: 0x4000, + 0x153c: 0x4000, 0x153d: 0x4000, 0x153e: 0x4000, 0x153f: 0x4000, + // Block 0x55, offset 0x1540 + 0x1540: 0x4000, 0x1541: 0x4000, 0x1542: 0x4000, 0x1543: 0x4000, 0x1544: 0x4000, 0x1545: 0x4000, + 0x1546: 0x4000, 0x1547: 0x4000, 0x1548: 0x4000, 0x1549: 0x4000, 0x154a: 0x4000, 0x154b: 0x4000, + 0x154c: 0x4000, 0x154d: 0x4000, 0x154e: 0x4000, 0x154f: 0x4000, 0x1550: 0x4000, 0x1551: 0x4000, + 0x1552: 0x4000, 0x1553: 0x4000, 0x1554: 0x4000, 0x1555: 0x4000, 0x1556: 0x4000, 0x1557: 0x4000, + 0x1558: 0x4000, 0x1559: 0x4000, 0x155a: 0x4000, 0x155b: 0x4000, 0x155c: 0x4000, 0x155d: 0x4000, + 0x155e: 0x4000, 0x155f: 0x4000, 0x1560: 0x4000, 0x1561: 0x4000, 0x1562: 0x4000, 0x1563: 0x4000, + 0x1564: 0x4000, 0x1565: 0x4000, 0x1566: 0x4000, 0x1567: 0x4000, 0x1568: 0x4000, 0x1569: 0x4000, + 0x156a: 0x4000, 0x156b: 0x4000, 0x156c: 0x4000, 0x156d: 0x4000, 0x156e: 0x4000, 0x156f: 0x4000, + 0x1570: 0x4000, 0x1571: 0x4000, 0x1572: 0x4000, 0x1573: 0x4000, 0x1574: 0x4000, 0x1575: 0x4000, + 0x1576: 0x4000, 0x1577: 0x4000, 0x1578: 0x4000, 0x1579: 0x4000, 0x157a: 0x4000, 0x157b: 0x4000, + 0x157c: 0x4000, 0x157e: 0x4000, 0x157f: 0x4000, + // Block 0x56, offset 0x1580 + 0x1580: 0x4000, 0x1581: 0x4000, 0x1582: 0x4000, 0x1583: 0x4000, 0x1584: 0x4000, 0x1585: 0x4000, + 0x1586: 0x4000, 0x1587: 0x4000, 0x1588: 0x4000, 0x1589: 0x4000, 0x158a: 0x4000, 0x158b: 0x4000, + 0x158c: 0x4000, 0x158d: 0x4000, 0x158e: 0x4000, 0x158f: 0x4000, 0x1590: 0x4000, 0x1591: 0x4000, + 0x1592: 0x4000, 0x1593: 0x4000, + 0x15a0: 0x4000, 0x15a1: 0x4000, 0x15a2: 0x4000, 0x15a3: 0x4000, + 0x15a4: 0x4000, 0x15a5: 0x4000, 0x15a6: 0x4000, 0x15a7: 0x4000, 0x15a8: 0x4000, 0x15a9: 0x4000, + 0x15aa: 0x4000, 0x15ab: 0x4000, 0x15ac: 0x4000, 0x15ad: 0x4000, 0x15ae: 0x4000, 0x15af: 0x4000, + 0x15b0: 0x4000, 0x15b1: 0x4000, 0x15b2: 0x4000, 0x15b3: 0x4000, 0x15b4: 0x4000, 0x15b5: 0x4000, + 0x15b6: 0x4000, 0x15b7: 0x4000, 0x15b8: 0x4000, 0x15b9: 0x4000, 0x15ba: 0x4000, 0x15bb: 0x4000, + 0x15bc: 0x4000, 0x15bd: 0x4000, 0x15be: 0x4000, 0x15bf: 0x4000, + // Block 0x57, offset 0x15c0 + 0x15c0: 0x4000, 0x15c1: 0x4000, 0x15c2: 0x4000, 0x15c3: 0x4000, 0x15c4: 0x4000, 0x15c5: 0x4000, + 0x15c6: 0x4000, 0x15c7: 0x4000, 0x15c8: 0x4000, 0x15c9: 0x4000, 0x15ca: 0x4000, + 0x15cf: 0x4000, 0x15d0: 0x4000, 0x15d1: 0x4000, + 0x15d2: 0x4000, 0x15d3: 0x4000, + 0x15e0: 0x4000, 0x15e1: 0x4000, 0x15e2: 0x4000, 0x15e3: 0x4000, + 0x15e4: 0x4000, 0x15e5: 0x4000, 0x15e6: 0x4000, 0x15e7: 0x4000, 0x15e8: 0x4000, 0x15e9: 0x4000, + 0x15ea: 0x4000, 0x15eb: 0x4000, 0x15ec: 0x4000, 0x15ed: 0x4000, 0x15ee: 0x4000, 0x15ef: 0x4000, + 0x15f0: 0x4000, 0x15f4: 0x4000, + 0x15f8: 0x4000, 0x15f9: 0x4000, 0x15fa: 0x4000, 0x15fb: 0x4000, + 0x15fc: 0x4000, 0x15fd: 0x4000, 0x15fe: 0x4000, 0x15ff: 0x4000, + // Block 0x58, offset 0x1600 + 0x1600: 0x4000, 0x1602: 0x4000, 0x1603: 0x4000, 0x1604: 0x4000, 0x1605: 0x4000, + 0x1606: 0x4000, 0x1607: 0x4000, 0x1608: 0x4000, 0x1609: 0x4000, 0x160a: 0x4000, 0x160b: 0x4000, + 0x160c: 0x4000, 0x160d: 0x4000, 0x160e: 0x4000, 0x160f: 0x4000, 0x1610: 0x4000, 0x1611: 0x4000, + 0x1612: 0x4000, 0x1613: 0x4000, 0x1614: 0x4000, 0x1615: 0x4000, 0x1616: 0x4000, 0x1617: 0x4000, + 0x1618: 0x4000, 0x1619: 0x4000, 0x161a: 0x4000, 0x161b: 0x4000, 0x161c: 0x4000, 0x161d: 0x4000, + 0x161e: 0x4000, 0x161f: 0x4000, 0x1620: 0x4000, 0x1621: 0x4000, 0x1622: 0x4000, 0x1623: 0x4000, + 0x1624: 0x4000, 0x1625: 0x4000, 0x1626: 0x4000, 0x1627: 0x4000, 0x1628: 0x4000, 0x1629: 0x4000, + 0x162a: 0x4000, 0x162b: 0x4000, 0x162c: 0x4000, 0x162d: 0x4000, 0x162e: 0x4000, 0x162f: 0x4000, + 0x1630: 0x4000, 0x1631: 0x4000, 0x1632: 0x4000, 0x1633: 0x4000, 0x1634: 0x4000, 0x1635: 0x4000, + 0x1636: 0x4000, 0x1637: 0x4000, 0x1638: 0x4000, 0x1639: 0x4000, 0x163a: 0x4000, 0x163b: 0x4000, + 0x163c: 0x4000, 0x163d: 0x4000, 0x163e: 0x4000, 0x163f: 0x4000, + // Block 0x59, offset 0x1640 + 0x1640: 0x4000, 0x1641: 0x4000, 0x1642: 0x4000, 0x1643: 0x4000, 0x1644: 0x4000, 0x1645: 0x4000, + 0x1646: 0x4000, 0x1647: 0x4000, 0x1648: 0x4000, 0x1649: 0x4000, 0x164a: 0x4000, 0x164b: 0x4000, + 0x164c: 0x4000, 0x164d: 0x4000, 0x164e: 0x4000, 0x164f: 0x4000, 0x1650: 0x4000, 0x1651: 0x4000, + 0x1652: 0x4000, 0x1653: 0x4000, 0x1654: 0x4000, 0x1655: 0x4000, 0x1656: 0x4000, 0x1657: 0x4000, + 0x1658: 0x4000, 0x1659: 0x4000, 0x165a: 0x4000, 0x165b: 0x4000, 0x165c: 0x4000, 0x165d: 0x4000, + 0x165e: 0x4000, 0x165f: 0x4000, 0x1660: 0x4000, 0x1661: 0x4000, 0x1662: 0x4000, 0x1663: 0x4000, + 0x1664: 0x4000, 0x1665: 0x4000, 0x1666: 0x4000, 0x1667: 0x4000, 0x1668: 0x4000, 0x1669: 0x4000, + 0x166a: 0x4000, 0x166b: 0x4000, 0x166c: 0x4000, 0x166d: 0x4000, 0x166e: 0x4000, 0x166f: 0x4000, + 0x1670: 0x4000, 0x1671: 0x4000, 0x1672: 0x4000, 0x1673: 0x4000, 0x1674: 0x4000, 0x1675: 0x4000, + 0x1676: 0x4000, 0x1677: 0x4000, 0x1678: 0x4000, 0x1679: 0x4000, 0x167a: 0x4000, 0x167b: 0x4000, + 0x167c: 0x4000, 0x167f: 0x4000, + // Block 0x5a, offset 0x1680 + 0x1680: 0x4000, 0x1681: 0x4000, 0x1682: 0x4000, 0x1683: 0x4000, 0x1684: 0x4000, 0x1685: 0x4000, + 0x1686: 0x4000, 0x1687: 0x4000, 0x1688: 0x4000, 0x1689: 0x4000, 0x168a: 0x4000, 0x168b: 0x4000, + 0x168c: 0x4000, 0x168d: 0x4000, 0x168e: 0x4000, 0x168f: 0x4000, 0x1690: 0x4000, 0x1691: 0x4000, + 0x1692: 0x4000, 0x1693: 0x4000, 0x1694: 0x4000, 0x1695: 0x4000, 0x1696: 0x4000, 0x1697: 0x4000, + 0x1698: 0x4000, 0x1699: 0x4000, 0x169a: 0x4000, 0x169b: 0x4000, 0x169c: 0x4000, 0x169d: 0x4000, + 0x169e: 0x4000, 0x169f: 0x4000, 0x16a0: 0x4000, 0x16a1: 0x4000, 0x16a2: 0x4000, 0x16a3: 0x4000, + 0x16a4: 0x4000, 0x16a5: 0x4000, 0x16a6: 0x4000, 0x16a7: 0x4000, 0x16a8: 0x4000, 0x16a9: 0x4000, + 0x16aa: 0x4000, 0x16ab: 0x4000, 0x16ac: 0x4000, 0x16ad: 0x4000, 0x16ae: 0x4000, 0x16af: 0x4000, + 0x16b0: 0x4000, 0x16b1: 0x4000, 0x16b2: 0x4000, 0x16b3: 0x4000, 0x16b4: 0x4000, 0x16b5: 0x4000, + 0x16b6: 0x4000, 0x16b7: 0x4000, 0x16b8: 0x4000, 0x16b9: 0x4000, 0x16ba: 0x4000, 0x16bb: 0x4000, + 0x16bc: 0x4000, 0x16bd: 0x4000, + // Block 0x5b, offset 0x16c0 + 0x16cb: 0x4000, + 0x16cc: 0x4000, 0x16cd: 0x4000, 0x16ce: 0x4000, 0x16d0: 0x4000, 0x16d1: 0x4000, + 0x16d2: 0x4000, 0x16d3: 0x4000, 0x16d4: 0x4000, 0x16d5: 0x4000, 0x16d6: 0x4000, 0x16d7: 0x4000, + 0x16d8: 0x4000, 0x16d9: 0x4000, 0x16da: 0x4000, 0x16db: 0x4000, 0x16dc: 0x4000, 0x16dd: 0x4000, + 0x16de: 0x4000, 0x16df: 0x4000, 0x16e0: 0x4000, 0x16e1: 0x4000, 0x16e2: 0x4000, 0x16e3: 0x4000, + 0x16e4: 0x4000, 0x16e5: 0x4000, 0x16e6: 0x4000, 0x16e7: 0x4000, + 0x16fa: 0x4000, + // Block 0x5c, offset 0x1700 + 0x1715: 0x4000, 0x1716: 0x4000, + 0x1724: 0x4000, + // Block 0x5d, offset 0x1740 + 0x177b: 0x4000, + 0x177c: 0x4000, 0x177d: 0x4000, 0x177e: 0x4000, 0x177f: 0x4000, + // Block 0x5e, offset 0x1780 + 0x1780: 0x4000, 0x1781: 0x4000, 0x1782: 0x4000, 0x1783: 0x4000, 0x1784: 0x4000, 0x1785: 0x4000, + 0x1786: 0x4000, 0x1787: 0x4000, 0x1788: 0x4000, 0x1789: 0x4000, 0x178a: 0x4000, 0x178b: 0x4000, + 0x178c: 0x4000, 0x178d: 0x4000, 0x178e: 0x4000, 0x178f: 0x4000, + // Block 0x5f, offset 0x17c0 + 0x17c0: 0x4000, 0x17c1: 0x4000, 0x17c2: 0x4000, 0x17c3: 0x4000, 0x17c4: 0x4000, 0x17c5: 0x4000, + 0x17cc: 0x4000, 0x17d0: 0x4000, 0x17d1: 0x4000, + 0x17d2: 0x4000, 0x17d5: 0x4000, + 0x17eb: 0x4000, 0x17ec: 0x4000, + 0x17f4: 0x4000, 0x17f5: 0x4000, + 0x17f6: 0x4000, 0x17f7: 0x4000, 0x17f8: 0x4000, 0x17f9: 0x4000, 0x17fa: 0x4000, + // Block 0x60, offset 0x1800 + 0x1820: 0x4000, 0x1821: 0x4000, 0x1822: 0x4000, 0x1823: 0x4000, + 0x1824: 0x4000, 0x1825: 0x4000, 0x1826: 0x4000, 0x1827: 0x4000, 0x1828: 0x4000, 0x1829: 0x4000, + 0x182a: 0x4000, 0x182b: 0x4000, + // Block 0x61, offset 0x1840 + 0x184d: 0x4000, 0x184e: 0x4000, 0x184f: 0x4000, 0x1850: 0x4000, 0x1851: 0x4000, + 0x1852: 0x4000, 0x1853: 0x4000, 0x1854: 0x4000, 0x1855: 0x4000, 0x1856: 0x4000, 0x1857: 0x4000, + 0x1858: 0x4000, 0x1859: 0x4000, 0x185a: 0x4000, 0x185b: 0x4000, 0x185c: 0x4000, 0x185d: 0x4000, + 0x185e: 0x4000, 0x185f: 0x4000, 0x1860: 0x4000, 0x1861: 0x4000, 0x1862: 0x4000, 0x1863: 0x4000, + 0x1864: 0x4000, 0x1865: 0x4000, 0x1866: 0x4000, 0x1867: 0x4000, 0x1868: 0x4000, 0x1869: 0x4000, + 0x186a: 0x4000, 0x186b: 0x4000, 0x186c: 0x4000, 0x186d: 0x4000, 0x186e: 0x4000, 0x186f: 0x4000, + 0x1870: 0x4000, 0x1871: 0x4000, 0x1872: 0x4000, 0x1873: 0x4000, 0x1874: 0x4000, 0x1875: 0x4000, + 0x1876: 0x4000, 0x1877: 0x4000, 0x1878: 0x4000, 0x1879: 0x4000, 0x187a: 0x4000, 0x187b: 0x4000, + 0x187c: 0x4000, 0x187d: 0x4000, 0x187e: 0x4000, 0x187f: 0x4000, + // Block 0x62, offset 0x1880 + 0x1880: 0x4000, 0x1881: 0x4000, 0x1882: 0x4000, 0x1883: 0x4000, 0x1884: 0x4000, 0x1885: 0x4000, + 0x1886: 0x4000, 0x1887: 0x4000, 0x1888: 0x4000, 0x1889: 0x4000, 0x188a: 0x4000, 0x188b: 0x4000, + 0x188c: 0x4000, 0x188d: 0x4000, 0x188e: 0x4000, 0x188f: 0x4000, 0x1890: 0x4000, 0x1891: 0x4000, + 0x1892: 0x4000, 0x1893: 0x4000, 0x1894: 0x4000, 0x1895: 0x4000, 0x1896: 0x4000, 0x1897: 0x4000, + 0x1898: 0x4000, 0x1899: 0x4000, 0x189a: 0x4000, 0x189b: 0x4000, 0x189c: 0x4000, 0x189d: 0x4000, + 0x189e: 0x4000, 0x189f: 0x4000, 0x18a0: 0x4000, 0x18a1: 0x4000, 0x18a2: 0x4000, 0x18a3: 0x4000, + 0x18a4: 0x4000, 0x18a5: 0x4000, 0x18a6: 0x4000, 0x18a7: 0x4000, 0x18a8: 0x4000, 0x18a9: 0x4000, + 0x18aa: 0x4000, 0x18ab: 0x4000, 0x18ac: 0x4000, 0x18ad: 0x4000, 0x18ae: 0x4000, 0x18af: 0x4000, + 0x18b0: 0x4000, 0x18b1: 0x4000, 0x18b3: 0x4000, 0x18b4: 0x4000, 0x18b5: 0x4000, + 0x18b6: 0x4000, 0x18ba: 0x4000, 0x18bb: 0x4000, + 0x18bc: 0x4000, 0x18bd: 0x4000, 0x18be: 0x4000, 0x18bf: 0x4000, + // Block 0x63, offset 0x18c0 + 0x18c0: 0x4000, 0x18c1: 0x4000, 0x18c2: 0x4000, 0x18c3: 0x4000, 0x18c4: 0x4000, 0x18c5: 0x4000, + 0x18c6: 0x4000, 0x18c7: 0x4000, 0x18c8: 0x4000, 0x18c9: 0x4000, 0x18ca: 0x4000, 0x18cb: 0x4000, + 0x18cc: 0x4000, 0x18cd: 0x4000, 0x18ce: 0x4000, 0x18cf: 0x4000, 0x18d0: 0x4000, 0x18d1: 0x4000, + 0x18d2: 0x4000, 0x18d3: 0x4000, 0x18d4: 0x4000, 0x18d5: 0x4000, 0x18d6: 0x4000, 0x18d7: 0x4000, + 0x18d8: 0x4000, 0x18d9: 0x4000, 0x18da: 0x4000, 0x18db: 0x4000, 0x18dc: 0x4000, 0x18dd: 0x4000, + 0x18de: 0x4000, 0x18df: 0x4000, 0x18e0: 0x4000, 0x18e1: 0x4000, 0x18e2: 0x4000, + 0x18e5: 0x4000, 0x18e6: 0x4000, 0x18e7: 0x4000, 0x18e8: 0x4000, 0x18e9: 0x4000, + 0x18ea: 0x4000, 0x18ee: 0x4000, 0x18ef: 0x4000, + 0x18f0: 0x4000, 0x18f1: 0x4000, 0x18f2: 0x4000, 0x18f3: 0x4000, 0x18f4: 0x4000, 0x18f5: 0x4000, + 0x18f6: 0x4000, 0x18f7: 0x4000, 0x18f8: 0x4000, 0x18f9: 0x4000, 0x18fa: 0x4000, 0x18fb: 0x4000, + 0x18fc: 0x4000, 0x18fd: 0x4000, 0x18fe: 0x4000, 0x18ff: 0x4000, + // Block 0x64, offset 0x1900 + 0x1900: 0x4000, 0x1901: 0x4000, 0x1902: 0x4000, 0x1903: 0x4000, 0x1904: 0x4000, 0x1905: 0x4000, + 0x1906: 0x4000, 0x1907: 0x4000, 0x1908: 0x4000, 0x1909: 0x4000, 0x190a: 0x4000, + 0x190d: 0x4000, 0x190e: 0x4000, 0x190f: 0x4000, 0x1910: 0x4000, 0x1911: 0x4000, + 0x1912: 0x4000, 0x1913: 0x4000, 0x1914: 0x4000, 0x1915: 0x4000, 0x1916: 0x4000, 0x1917: 0x4000, + 0x1918: 0x4000, 0x1919: 0x4000, 0x191a: 0x4000, 0x191b: 0x4000, 0x191c: 0x4000, 0x191d: 0x4000, + 0x191e: 0x4000, 0x191f: 0x4000, 0x1920: 0x4000, 0x1921: 0x4000, 0x1922: 0x4000, 0x1923: 0x4000, + 0x1924: 0x4000, 0x1925: 0x4000, 0x1926: 0x4000, 0x1927: 0x4000, 0x1928: 0x4000, 0x1929: 0x4000, + 0x192a: 0x4000, 0x192b: 0x4000, 0x192c: 0x4000, 0x192d: 0x4000, 0x192e: 0x4000, 0x192f: 0x4000, + 0x1930: 0x4000, 0x1931: 0x4000, 0x1932: 0x4000, 0x1933: 0x4000, 0x1934: 0x4000, 0x1935: 0x4000, + 0x1936: 0x4000, 0x1937: 0x4000, 0x1938: 0x4000, 0x1939: 0x4000, 0x193a: 0x4000, 0x193b: 0x4000, + 0x193c: 0x4000, 0x193d: 0x4000, 0x193e: 0x4000, 0x193f: 0x4000, + // Block 0x65, offset 0x1940 + 0x1970: 0x4000, 0x1971: 0x4000, 0x1972: 0x4000, 0x1973: 0x4000, + 0x1978: 0x4000, 0x1979: 0x4000, 0x197a: 0x4000, + // Block 0x66, offset 0x1980 + 0x1980: 0x4000, 0x1981: 0x4000, 0x1982: 0x4000, + 0x1990: 0x4000, 0x1991: 0x4000, + 0x1992: 0x4000, 0x1993: 0x4000, 0x1994: 0x4000, 0x1995: 0x4000, + // Block 0x67, offset 0x19c0 + 0x19c0: 0x2000, 0x19c1: 0x2000, 0x19c2: 0x2000, 0x19c3: 0x2000, 0x19c4: 0x2000, 0x19c5: 0x2000, + 0x19c6: 0x2000, 0x19c7: 0x2000, 0x19c8: 0x2000, 0x19c9: 0x2000, 0x19ca: 0x2000, 0x19cb: 0x2000, + 0x19cc: 0x2000, 0x19cd: 0x2000, 0x19ce: 0x2000, 0x19cf: 0x2000, 0x19d0: 0x2000, 0x19d1: 0x2000, + 0x19d2: 0x2000, 0x19d3: 0x2000, 0x19d4: 0x2000, 0x19d5: 0x2000, 0x19d6: 0x2000, 0x19d7: 0x2000, + 0x19d8: 0x2000, 0x19d9: 0x2000, 0x19da: 0x2000, 0x19db: 0x2000, 0x19dc: 0x2000, 0x19dd: 0x2000, + 0x19de: 0x2000, 0x19df: 0x2000, 0x19e0: 0x2000, 0x19e1: 0x2000, 0x19e2: 0x2000, 0x19e3: 0x2000, + 0x19e4: 0x2000, 0x19e5: 0x2000, 0x19e6: 0x2000, 0x19e7: 0x2000, 0x19e8: 0x2000, 0x19e9: 0x2000, + 0x19ea: 0x2000, 0x19eb: 0x2000, 0x19ec: 0x2000, 0x19ed: 0x2000, 0x19ee: 0x2000, 0x19ef: 0x2000, + 0x19f0: 0x2000, 0x19f1: 0x2000, 0x19f2: 0x2000, 0x19f3: 0x2000, 0x19f4: 0x2000, 0x19f5: 0x2000, + 0x19f6: 0x2000, 0x19f7: 0x2000, 0x19f8: 0x2000, 0x19f9: 0x2000, 0x19fa: 0x2000, 0x19fb: 0x2000, + 0x19fc: 0x2000, 0x19fd: 0x2000, +} + +// widthIndex: 22 blocks, 1408 entries, 1408 bytes +// Block 0 is the zero block. +var widthIndex = [1408]uint8{ + // Block 0x0, offset 0x0 + // Block 0x1, offset 0x40 + // Block 0x2, offset 0x80 + // Block 0x3, offset 0xc0 + 0xc2: 0x01, 0xc3: 0x02, 0xc4: 0x03, 0xc5: 0x04, 0xc7: 0x05, + 0xc9: 0x06, 0xcb: 0x07, 0xcc: 0x08, 0xcd: 0x09, 0xce: 0x0a, 0xcf: 0x0b, + 0xd0: 0x0c, 0xd1: 0x0d, + 0xe1: 0x02, 0xe2: 0x03, 0xe3: 0x04, 0xe4: 0x05, 0xe5: 0x06, 0xe6: 0x06, 0xe7: 0x06, + 0xe8: 0x06, 0xe9: 0x06, 0xea: 0x07, 0xeb: 0x06, 0xec: 0x06, 0xed: 0x08, 0xee: 0x09, 0xef: 0x0a, + 0xf0: 0x0f, 0xf3: 0x12, 0xf4: 0x13, + // Block 0x4, offset 0x100 + 0x104: 0x0e, 0x105: 0x0f, + // Block 0x5, offset 0x140 + 0x140: 0x10, 0x141: 0x11, 0x142: 0x12, 0x144: 0x13, 0x145: 0x14, 0x146: 0x15, 0x147: 0x16, + 0x148: 0x17, 0x149: 0x18, 0x14a: 0x19, 0x14c: 0x1a, 0x14f: 0x1b, + 0x151: 0x1c, 0x152: 0x08, 0x153: 0x1d, 0x154: 0x1e, 0x155: 0x1f, 0x156: 0x20, 0x157: 0x21, + 0x158: 0x22, 0x159: 0x23, 0x15a: 0x24, 0x15b: 0x25, 0x15c: 0x26, 0x15d: 0x27, 0x15e: 0x28, 0x15f: 0x29, + 0x166: 0x2a, + 0x16c: 0x2b, 0x16d: 0x2c, + 0x17a: 0x2d, 0x17b: 0x2e, 0x17c: 0x0e, 0x17d: 0x0e, 0x17e: 0x0e, 0x17f: 0x2f, + // Block 0x6, offset 0x180 + 0x180: 0x30, 0x181: 0x31, 0x182: 0x32, 0x183: 0x33, 0x184: 0x34, 0x185: 0x35, 0x186: 0x36, 0x187: 0x37, + 0x188: 0x38, 0x189: 0x39, 0x18a: 0x0e, 0x18b: 0x3a, 0x18c: 0x0e, 0x18d: 0x0e, 0x18e: 0x0e, 0x18f: 0x0e, + 0x190: 0x0e, 0x191: 0x0e, 0x192: 0x0e, 0x193: 0x0e, 0x194: 0x0e, 0x195: 0x0e, 0x196: 0x0e, 0x197: 0x0e, + 0x198: 0x0e, 0x199: 0x0e, 0x19a: 0x0e, 0x19b: 0x0e, 0x19c: 0x0e, 0x19d: 0x0e, 0x19e: 0x0e, 0x19f: 0x0e, + 0x1a0: 0x0e, 0x1a1: 0x0e, 0x1a2: 0x0e, 0x1a3: 0x0e, 0x1a4: 0x0e, 0x1a5: 0x0e, 0x1a6: 0x0e, 0x1a7: 0x0e, + 0x1a8: 0x0e, 0x1a9: 0x0e, 0x1aa: 0x0e, 0x1ab: 0x0e, 0x1ac: 0x0e, 0x1ad: 0x0e, 0x1ae: 0x0e, 0x1af: 0x0e, + 0x1b0: 0x0e, 0x1b1: 0x0e, 0x1b2: 0x0e, 0x1b3: 0x0e, 0x1b4: 0x0e, 0x1b5: 0x0e, 0x1b6: 0x0e, 0x1b7: 0x0e, + 0x1b8: 0x0e, 0x1b9: 0x0e, 0x1ba: 0x0e, 0x1bb: 0x0e, 0x1bc: 0x0e, 0x1bd: 0x0e, 0x1be: 0x0e, 0x1bf: 0x0e, + // Block 0x7, offset 0x1c0 + 0x1c0: 0x0e, 0x1c1: 0x0e, 0x1c2: 0x0e, 0x1c3: 0x0e, 0x1c4: 0x0e, 0x1c5: 0x0e, 0x1c6: 0x0e, 0x1c7: 0x0e, + 0x1c8: 0x0e, 0x1c9: 0x0e, 0x1ca: 0x0e, 0x1cb: 0x0e, 0x1cc: 0x0e, 0x1cd: 0x0e, 0x1ce: 0x0e, 0x1cf: 0x0e, + 0x1d0: 0x0e, 0x1d1: 0x0e, 0x1d2: 0x0e, 0x1d3: 0x0e, 0x1d4: 0x0e, 0x1d5: 0x0e, 0x1d6: 0x0e, 0x1d7: 0x0e, + 0x1d8: 0x0e, 0x1d9: 0x0e, 0x1da: 0x0e, 0x1db: 0x0e, 0x1dc: 0x0e, 0x1dd: 0x0e, 0x1de: 0x0e, 0x1df: 0x0e, + 0x1e0: 0x0e, 0x1e1: 0x0e, 0x1e2: 0x0e, 0x1e3: 0x0e, 0x1e4: 0x0e, 0x1e5: 0x0e, 0x1e6: 0x0e, 0x1e7: 0x0e, + 0x1e8: 0x0e, 0x1e9: 0x0e, 0x1ea: 0x0e, 0x1eb: 0x0e, 0x1ec: 0x0e, 0x1ed: 0x0e, 0x1ee: 0x0e, 0x1ef: 0x0e, + 0x1f0: 0x0e, 0x1f1: 0x0e, 0x1f2: 0x0e, 0x1f3: 0x0e, 0x1f4: 0x0e, 0x1f5: 0x0e, 0x1f6: 0x0e, + 0x1f8: 0x0e, 0x1f9: 0x0e, 0x1fa: 0x0e, 0x1fb: 0x0e, 0x1fc: 0x0e, 0x1fd: 0x0e, 0x1fe: 0x0e, 0x1ff: 0x0e, + // Block 0x8, offset 0x200 + 0x200: 0x0e, 0x201: 0x0e, 0x202: 0x0e, 0x203: 0x0e, 0x204: 0x0e, 0x205: 0x0e, 0x206: 0x0e, 0x207: 0x0e, + 0x208: 0x0e, 0x209: 0x0e, 0x20a: 0x0e, 0x20b: 0x0e, 0x20c: 0x0e, 0x20d: 0x0e, 0x20e: 0x0e, 0x20f: 0x0e, + 0x210: 0x0e, 0x211: 0x0e, 0x212: 0x0e, 0x213: 0x0e, 0x214: 0x0e, 0x215: 0x0e, 0x216: 0x0e, 0x217: 0x0e, + 0x218: 0x0e, 0x219: 0x0e, 0x21a: 0x0e, 0x21b: 0x0e, 0x21c: 0x0e, 0x21d: 0x0e, 0x21e: 0x0e, 0x21f: 0x0e, + 0x220: 0x0e, 0x221: 0x0e, 0x222: 0x0e, 0x223: 0x0e, 0x224: 0x0e, 0x225: 0x0e, 0x226: 0x0e, 0x227: 0x0e, + 0x228: 0x0e, 0x229: 0x0e, 0x22a: 0x0e, 0x22b: 0x0e, 0x22c: 0x0e, 0x22d: 0x0e, 0x22e: 0x0e, 0x22f: 0x0e, + 0x230: 0x0e, 0x231: 0x0e, 0x232: 0x0e, 0x233: 0x0e, 0x234: 0x0e, 0x235: 0x0e, 0x236: 0x0e, 0x237: 0x0e, + 0x238: 0x0e, 0x239: 0x0e, 0x23a: 0x0e, 0x23b: 0x0e, 0x23c: 0x0e, 0x23d: 0x0e, 0x23e: 0x0e, 0x23f: 0x0e, + // Block 0x9, offset 0x240 + 0x240: 0x0e, 0x241: 0x0e, 0x242: 0x0e, 0x243: 0x0e, 0x244: 0x0e, 0x245: 0x0e, 0x246: 0x0e, 0x247: 0x0e, + 0x248: 0x0e, 0x249: 0x0e, 0x24a: 0x0e, 0x24b: 0x0e, 0x24c: 0x0e, 0x24d: 0x0e, 0x24e: 0x0e, 0x24f: 0x0e, + 0x250: 0x0e, 0x251: 0x0e, 0x252: 0x3b, 0x253: 0x3c, + 0x265: 0x3d, + 0x270: 0x0e, 0x271: 0x0e, 0x272: 0x0e, 0x273: 0x0e, 0x274: 0x0e, 0x275: 0x0e, 0x276: 0x0e, 0x277: 0x0e, + 0x278: 0x0e, 0x279: 0x0e, 0x27a: 0x0e, 0x27b: 0x0e, 0x27c: 0x0e, 0x27d: 0x0e, 0x27e: 0x0e, 0x27f: 0x0e, + // Block 0xa, offset 0x280 + 0x280: 0x0e, 0x281: 0x0e, 0x282: 0x0e, 0x283: 0x0e, 0x284: 0x0e, 0x285: 0x0e, 0x286: 0x0e, 0x287: 0x0e, + 0x288: 0x0e, 0x289: 0x0e, 0x28a: 0x0e, 0x28b: 0x0e, 0x28c: 0x0e, 0x28d: 0x0e, 0x28e: 0x0e, 0x28f: 0x0e, + 0x290: 0x0e, 0x291: 0x0e, 0x292: 0x0e, 0x293: 0x0e, 0x294: 0x0e, 0x295: 0x0e, 0x296: 0x0e, 0x297: 0x0e, + 0x298: 0x0e, 0x299: 0x0e, 0x29a: 0x0e, 0x29b: 0x0e, 0x29c: 0x0e, 0x29d: 0x0e, 0x29e: 0x3e, + // Block 0xb, offset 0x2c0 + 0x2c0: 0x08, 0x2c1: 0x08, 0x2c2: 0x08, 0x2c3: 0x08, 0x2c4: 0x08, 0x2c5: 0x08, 0x2c6: 0x08, 0x2c7: 0x08, + 0x2c8: 0x08, 0x2c9: 0x08, 0x2ca: 0x08, 0x2cb: 0x08, 0x2cc: 0x08, 0x2cd: 0x08, 0x2ce: 0x08, 0x2cf: 0x08, + 0x2d0: 0x08, 0x2d1: 0x08, 0x2d2: 0x08, 0x2d3: 0x08, 0x2d4: 0x08, 0x2d5: 0x08, 0x2d6: 0x08, 0x2d7: 0x08, + 0x2d8: 0x08, 0x2d9: 0x08, 0x2da: 0x08, 0x2db: 0x08, 0x2dc: 0x08, 0x2dd: 0x08, 0x2de: 0x08, 0x2df: 0x08, + 0x2e0: 0x08, 0x2e1: 0x08, 0x2e2: 0x08, 0x2e3: 0x08, 0x2e4: 0x08, 0x2e5: 0x08, 0x2e6: 0x08, 0x2e7: 0x08, + 0x2e8: 0x08, 0x2e9: 0x08, 0x2ea: 0x08, 0x2eb: 0x08, 0x2ec: 0x08, 0x2ed: 0x08, 0x2ee: 0x08, 0x2ef: 0x08, + 0x2f0: 0x08, 0x2f1: 0x08, 0x2f2: 0x08, 0x2f3: 0x08, 0x2f4: 0x08, 0x2f5: 0x08, 0x2f6: 0x08, 0x2f7: 0x08, + 0x2f8: 0x08, 0x2f9: 0x08, 0x2fa: 0x08, 0x2fb: 0x08, 0x2fc: 0x08, 0x2fd: 0x08, 0x2fe: 0x08, 0x2ff: 0x08, + // Block 0xc, offset 0x300 + 0x300: 0x08, 0x301: 0x08, 0x302: 0x08, 0x303: 0x08, 0x304: 0x08, 0x305: 0x08, 0x306: 0x08, 0x307: 0x08, + 0x308: 0x08, 0x309: 0x08, 0x30a: 0x08, 0x30b: 0x08, 0x30c: 0x08, 0x30d: 0x08, 0x30e: 0x08, 0x30f: 0x08, + 0x310: 0x08, 0x311: 0x08, 0x312: 0x08, 0x313: 0x08, 0x314: 0x08, 0x315: 0x08, 0x316: 0x08, 0x317: 0x08, + 0x318: 0x08, 0x319: 0x08, 0x31a: 0x08, 0x31b: 0x08, 0x31c: 0x08, 0x31d: 0x08, 0x31e: 0x08, 0x31f: 0x08, + 0x320: 0x08, 0x321: 0x08, 0x322: 0x08, 0x323: 0x08, 0x324: 0x0e, 0x325: 0x0e, 0x326: 0x0e, 0x327: 0x0e, + 0x328: 0x0e, 0x329: 0x0e, 0x32a: 0x0e, 0x32b: 0x0e, + 0x338: 0x3f, 0x339: 0x40, 0x33c: 0x41, 0x33d: 0x42, 0x33e: 0x43, 0x33f: 0x44, + // Block 0xd, offset 0x340 + 0x37f: 0x45, + // Block 0xe, offset 0x380 + 0x380: 0x0e, 0x381: 0x0e, 0x382: 0x0e, 0x383: 0x0e, 0x384: 0x0e, 0x385: 0x0e, 0x386: 0x0e, 0x387: 0x0e, + 0x388: 0x0e, 0x389: 0x0e, 0x38a: 0x0e, 0x38b: 0x0e, 0x38c: 0x0e, 0x38d: 0x0e, 0x38e: 0x0e, 0x38f: 0x0e, + 0x390: 0x0e, 0x391: 0x0e, 0x392: 0x0e, 0x393: 0x0e, 0x394: 0x0e, 0x395: 0x0e, 0x396: 0x0e, 0x397: 0x0e, + 0x398: 0x0e, 0x399: 0x0e, 0x39a: 0x0e, 0x39b: 0x0e, 0x39c: 0x0e, 0x39d: 0x0e, 0x39e: 0x0e, 0x39f: 0x46, + 0x3a0: 0x0e, 0x3a1: 0x0e, 0x3a2: 0x0e, 0x3a3: 0x0e, 0x3a4: 0x0e, 0x3a5: 0x0e, 0x3a6: 0x0e, 0x3a7: 0x0e, + 0x3a8: 0x0e, 0x3a9: 0x0e, 0x3aa: 0x0e, 0x3ab: 0x47, + // Block 0xf, offset 0x3c0 + 0x3c0: 0x0e, 0x3c1: 0x0e, 0x3c2: 0x0e, 0x3c3: 0x0e, 0x3c4: 0x48, 0x3c5: 0x49, 0x3c6: 0x0e, 0x3c7: 0x0e, + 0x3c8: 0x0e, 0x3c9: 0x0e, 0x3ca: 0x0e, 0x3cb: 0x4a, + // Block 0x10, offset 0x400 + 0x400: 0x4b, 0x403: 0x4c, 0x404: 0x4d, 0x405: 0x4e, 0x406: 0x4f, + 0x408: 0x50, 0x409: 0x51, 0x40c: 0x52, 0x40d: 0x53, 0x40e: 0x54, 0x40f: 0x55, + 0x410: 0x3a, 0x411: 0x56, 0x412: 0x0e, 0x413: 0x57, 0x414: 0x58, 0x415: 0x59, 0x416: 0x5a, 0x417: 0x5b, + 0x418: 0x0e, 0x419: 0x5c, 0x41a: 0x0e, 0x41b: 0x5d, 0x41f: 0x5e, + 0x424: 0x5f, 0x425: 0x60, 0x426: 0x61, 0x427: 0x62, + 0x429: 0x63, 0x42a: 0x64, + // Block 0x11, offset 0x440 + 0x456: 0x0b, 0x457: 0x06, + 0x458: 0x0c, 0x45b: 0x0d, 0x45f: 0x0e, + 0x460: 0x06, 0x461: 0x06, 0x462: 0x06, 0x463: 0x06, 0x464: 0x06, 0x465: 0x06, 0x466: 0x06, 0x467: 0x06, + 0x468: 0x06, 0x469: 0x06, 0x46a: 0x06, 0x46b: 0x06, 0x46c: 0x06, 0x46d: 0x06, 0x46e: 0x06, 0x46f: 0x06, + 0x470: 0x06, 0x471: 0x06, 0x472: 0x06, 0x473: 0x06, 0x474: 0x06, 0x475: 0x06, 0x476: 0x06, 0x477: 0x06, + 0x478: 0x06, 0x479: 0x06, 0x47a: 0x06, 0x47b: 0x06, 0x47c: 0x06, 0x47d: 0x06, 0x47e: 0x06, 0x47f: 0x06, + // Block 0x12, offset 0x480 + 0x484: 0x08, 0x485: 0x08, 0x486: 0x08, 0x487: 0x09, + // Block 0x13, offset 0x4c0 + 0x4c0: 0x08, 0x4c1: 0x08, 0x4c2: 0x08, 0x4c3: 0x08, 0x4c4: 0x08, 0x4c5: 0x08, 0x4c6: 0x08, 0x4c7: 0x08, + 0x4c8: 0x08, 0x4c9: 0x08, 0x4ca: 0x08, 0x4cb: 0x08, 0x4cc: 0x08, 0x4cd: 0x08, 0x4ce: 0x08, 0x4cf: 0x08, + 0x4d0: 0x08, 0x4d1: 0x08, 0x4d2: 0x08, 0x4d3: 0x08, 0x4d4: 0x08, 0x4d5: 0x08, 0x4d6: 0x08, 0x4d7: 0x08, + 0x4d8: 0x08, 0x4d9: 0x08, 0x4da: 0x08, 0x4db: 0x08, 0x4dc: 0x08, 0x4dd: 0x08, 0x4de: 0x08, 0x4df: 0x08, + 0x4e0: 0x08, 0x4e1: 0x08, 0x4e2: 0x08, 0x4e3: 0x08, 0x4e4: 0x08, 0x4e5: 0x08, 0x4e6: 0x08, 0x4e7: 0x08, + 0x4e8: 0x08, 0x4e9: 0x08, 0x4ea: 0x08, 0x4eb: 0x08, 0x4ec: 0x08, 0x4ed: 0x08, 0x4ee: 0x08, 0x4ef: 0x08, + 0x4f0: 0x08, 0x4f1: 0x08, 0x4f2: 0x08, 0x4f3: 0x08, 0x4f4: 0x08, 0x4f5: 0x08, 0x4f6: 0x08, 0x4f7: 0x08, + 0x4f8: 0x08, 0x4f9: 0x08, 0x4fa: 0x08, 0x4fb: 0x08, 0x4fc: 0x08, 0x4fd: 0x08, 0x4fe: 0x08, 0x4ff: 0x65, + // Block 0x14, offset 0x500 + 0x520: 0x10, + 0x530: 0x09, 0x531: 0x09, 0x532: 0x09, 0x533: 0x09, 0x534: 0x09, 0x535: 0x09, 0x536: 0x09, 0x537: 0x09, + 0x538: 0x09, 0x539: 0x09, 0x53a: 0x09, 0x53b: 0x09, 0x53c: 0x09, 0x53d: 0x09, 0x53e: 0x09, 0x53f: 0x11, + // Block 0x15, offset 0x540 + 0x540: 0x09, 0x541: 0x09, 0x542: 0x09, 0x543: 0x09, 0x544: 0x09, 0x545: 0x09, 0x546: 0x09, 0x547: 0x09, + 0x548: 0x09, 0x549: 0x09, 0x54a: 0x09, 0x54b: 0x09, 0x54c: 0x09, 0x54d: 0x09, 0x54e: 0x09, 0x54f: 0x11, +} + +// inverseData contains 4-byte entries of the following format: +// <0 padding> +// The last byte of the UTF-8-encoded rune is xor-ed with the last byte of the +// UTF-8 encoding of the original rune. Mappings often have the following +// pattern: +// Ａ -> A (U+FF21 -> U+0041) +// ï¼¢ -> B (U+FF22 -> U+0042) +// ... +// By xor-ing the last byte the same entry can be shared by many mappings. This +// reduces the total number of distinct entries by about two thirds. +// The resulting entry for the aforementioned mappings is +// { 0x01, 0xE0, 0x00, 0x00 } +// Using this entry to map U+FF21 (UTF-8 [EF BC A1]), we get +// E0 ^ A1 = 41. +// Similarly, for U+FF22 (UTF-8 [EF BC A2]), we get +// E0 ^ A2 = 42. +// Note that because of the xor-ing, the byte sequence stored in the entry is +// not valid UTF-8. +var inverseData = [150][4]byte{ + {0x00, 0x00, 0x00, 0x00}, + {0x03, 0xe3, 0x80, 0xa0}, + {0x03, 0xef, 0xbc, 0xa0}, + {0x03, 0xef, 0xbc, 0xe0}, + {0x03, 0xef, 0xbd, 0xe0}, + {0x03, 0xef, 0xbf, 0x02}, + {0x03, 0xef, 0xbf, 0x00}, + {0x03, 0xef, 0xbf, 0x0e}, + {0x03, 0xef, 0xbf, 0x0c}, + {0x03, 0xef, 0xbf, 0x0f}, + {0x03, 0xef, 0xbf, 0x39}, + {0x03, 0xef, 0xbf, 0x3b}, + {0x03, 0xef, 0xbf, 0x3f}, + {0x03, 0xef, 0xbf, 0x2a}, + {0x03, 0xef, 0xbf, 0x0d}, + {0x03, 0xef, 0xbf, 0x25}, + {0x03, 0xef, 0xbd, 0x1a}, + {0x03, 0xef, 0xbd, 0x26}, + {0x01, 0xa0, 0x00, 0x00}, + {0x03, 0xef, 0xbd, 0x25}, + {0x03, 0xef, 0xbd, 0x23}, + {0x03, 0xef, 0xbd, 0x2e}, + {0x03, 0xef, 0xbe, 0x07}, + {0x03, 0xef, 0xbe, 0x05}, + {0x03, 0xef, 0xbd, 0x06}, + {0x03, 0xef, 0xbd, 0x13}, + {0x03, 0xef, 0xbd, 0x0b}, + {0x03, 0xef, 0xbd, 0x16}, + {0x03, 0xef, 0xbd, 0x0c}, + {0x03, 0xef, 0xbd, 0x15}, + {0x03, 0xef, 0xbd, 0x0d}, + {0x03, 0xef, 0xbd, 0x1c}, + {0x03, 0xef, 0xbd, 0x02}, + {0x03, 0xef, 0xbd, 0x1f}, + {0x03, 0xef, 0xbd, 0x1d}, + {0x03, 0xef, 0xbd, 0x17}, + {0x03, 0xef, 0xbd, 0x08}, + {0x03, 0xef, 0xbd, 0x09}, + {0x03, 0xef, 0xbd, 0x0e}, + {0x03, 0xef, 0xbd, 0x04}, + {0x03, 0xef, 0xbd, 0x05}, + {0x03, 0xef, 0xbe, 0x3f}, + {0x03, 0xef, 0xbe, 0x00}, + {0x03, 0xef, 0xbd, 0x2c}, + {0x03, 0xef, 0xbe, 0x06}, + {0x03, 0xef, 0xbe, 0x0c}, + {0x03, 0xef, 0xbe, 0x0f}, + {0x03, 0xef, 0xbe, 0x0d}, + {0x03, 0xef, 0xbe, 0x0b}, + {0x03, 0xef, 0xbe, 0x19}, + {0x03, 0xef, 0xbe, 0x15}, + {0x03, 0xef, 0xbe, 0x11}, + {0x03, 0xef, 0xbe, 0x31}, + {0x03, 0xef, 0xbe, 0x33}, + {0x03, 0xef, 0xbd, 0x0f}, + {0x03, 0xef, 0xbe, 0x30}, + {0x03, 0xef, 0xbe, 0x3e}, + {0x03, 0xef, 0xbe, 0x32}, + {0x03, 0xef, 0xbe, 0x36}, + {0x03, 0xef, 0xbd, 0x14}, + {0x03, 0xef, 0xbe, 0x2e}, + {0x03, 0xef, 0xbd, 0x1e}, + {0x03, 0xef, 0xbe, 0x10}, + {0x03, 0xef, 0xbf, 0x13}, + {0x03, 0xef, 0xbf, 0x15}, + {0x03, 0xef, 0xbf, 0x17}, + {0x03, 0xef, 0xbf, 0x1f}, + {0x03, 0xef, 0xbf, 0x1d}, + {0x03, 0xef, 0xbf, 0x1b}, + {0x03, 0xef, 0xbf, 0x09}, + {0x03, 0xef, 0xbf, 0x0b}, + {0x03, 0xef, 0xbf, 0x37}, + {0x03, 0xef, 0xbe, 0x04}, + {0x01, 0xe0, 0x00, 0x00}, + {0x03, 0xe2, 0xa6, 0x1a}, + {0x03, 0xe2, 0xa6, 0x26}, + {0x03, 0xe3, 0x80, 0x23}, + {0x03, 0xe3, 0x80, 0x2e}, + {0x03, 0xe3, 0x80, 0x25}, + {0x03, 0xe3, 0x83, 0x1e}, + {0x03, 0xe3, 0x83, 0x14}, + {0x03, 0xe3, 0x82, 0x06}, + {0x03, 0xe3, 0x82, 0x0b}, + {0x03, 0xe3, 0x82, 0x0c}, + {0x03, 0xe3, 0x82, 0x0d}, + {0x03, 0xe3, 0x82, 0x02}, + {0x03, 0xe3, 0x83, 0x0f}, + {0x03, 0xe3, 0x83, 0x08}, + {0x03, 0xe3, 0x83, 0x09}, + {0x03, 0xe3, 0x83, 0x2c}, + {0x03, 0xe3, 0x83, 0x0c}, + {0x03, 0xe3, 0x82, 0x13}, + {0x03, 0xe3, 0x82, 0x16}, + {0x03, 0xe3, 0x82, 0x15}, + {0x03, 0xe3, 0x82, 0x1c}, + {0x03, 0xe3, 0x82, 0x1f}, + {0x03, 0xe3, 0x82, 0x1d}, + {0x03, 0xe3, 0x82, 0x1a}, + {0x03, 0xe3, 0x82, 0x17}, + {0x03, 0xe3, 0x82, 0x08}, + {0x03, 0xe3, 0x82, 0x09}, + {0x03, 0xe3, 0x82, 0x0e}, + {0x03, 0xe3, 0x82, 0x04}, + {0x03, 0xe3, 0x82, 0x05}, + {0x03, 0xe3, 0x82, 0x3f}, + {0x03, 0xe3, 0x83, 0x00}, + {0x03, 0xe3, 0x83, 0x06}, + {0x03, 0xe3, 0x83, 0x05}, + {0x03, 0xe3, 0x83, 0x0d}, + {0x03, 0xe3, 0x83, 0x0b}, + {0x03, 0xe3, 0x83, 0x07}, + {0x03, 0xe3, 0x83, 0x19}, + {0x03, 0xe3, 0x83, 0x15}, + {0x03, 0xe3, 0x83, 0x11}, + {0x03, 0xe3, 0x83, 0x31}, + {0x03, 0xe3, 0x83, 0x33}, + {0x03, 0xe3, 0x83, 0x30}, + {0x03, 0xe3, 0x83, 0x3e}, + {0x03, 0xe3, 0x83, 0x32}, + {0x03, 0xe3, 0x83, 0x36}, + {0x03, 0xe3, 0x83, 0x2e}, + {0x03, 0xe3, 0x82, 0x07}, + {0x03, 0xe3, 0x85, 0x04}, + {0x03, 0xe3, 0x84, 0x10}, + {0x03, 0xe3, 0x85, 0x30}, + {0x03, 0xe3, 0x85, 0x0d}, + {0x03, 0xe3, 0x85, 0x13}, + {0x03, 0xe3, 0x85, 0x15}, + {0x03, 0xe3, 0x85, 0x17}, + {0x03, 0xe3, 0x85, 0x1f}, + {0x03, 0xe3, 0x85, 0x1d}, + {0x03, 0xe3, 0x85, 0x1b}, + {0x03, 0xe3, 0x85, 0x09}, + {0x03, 0xe3, 0x85, 0x0f}, + {0x03, 0xe3, 0x85, 0x0b}, + {0x03, 0xe3, 0x85, 0x37}, + {0x03, 0xe3, 0x85, 0x3b}, + {0x03, 0xe3, 0x85, 0x39}, + {0x03, 0xe3, 0x85, 0x3f}, + {0x02, 0xc2, 0x02, 0x00}, + {0x02, 0xc2, 0x0e, 0x00}, + {0x02, 0xc2, 0x0c, 0x00}, + {0x02, 0xc2, 0x00, 0x00}, + {0x03, 0xe2, 0x82, 0x0f}, + {0x03, 0xe2, 0x94, 0x2a}, + {0x03, 0xe2, 0x86, 0x39}, + {0x03, 0xe2, 0x86, 0x3b}, + {0x03, 0xe2, 0x86, 0x3f}, + {0x03, 0xe2, 0x96, 0x0d}, + {0x03, 0xe2, 0x97, 0x25}, +} + +// Total table size 15320 bytes (14KiB) diff --git a/modules.txt b/modules.txt index f62e9d7c11..cd46bb411f 100644 --- a/modules.txt +++ b/modules.txt @@ -160,8 +160,6 @@ github.com/cenkalti/backoff ## explicit github.com/client9/misspell github.com/client9/misspell/cmd/misspell -# github.com/cockroachdb/apd v1.1.0 -## explicit # github.com/cockroachdb/apd/v2 v2.0.2 ## explicit github.com/cockroachdb/apd/v2 @@ -507,8 +505,26 @@ github.com/ianlancetaylor/cgosymbolizer github.com/ianlancetaylor/demangle # github.com/inconshreveable/mousetrap v1.0.0 github.com/inconshreveable/mousetrap +# github.com/jackc/chunkreader/v2 v2.0.1 +github.com/jackc/chunkreader/v2 # github.com/jackc/fake v0.0.0-20150926172116-812a484cc733 ## explicit +# github.com/jackc/pgconn v1.6.1 +## explicit +github.com/jackc/pgconn +github.com/jackc/pgconn/internal/ctxwatch +github.com/jackc/pgconn/stmtcache +# github.com/jackc/pgio v1.0.0 +github.com/jackc/pgio +# github.com/jackc/pgpassfile v1.0.0 +github.com/jackc/pgpassfile +# github.com/jackc/pgproto3/v2 v2.0.2 +## explicit +github.com/jackc/pgproto3/v2 +# github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8 +github.com/jackc/pgservicefile +# github.com/jackc/pgtype v1.3.0 +github.com/jackc/pgtype # github.com/jackc/pgx v3.6.2+incompatible ## explicit github.com/jackc/pgx @@ -517,6 +533,10 @@ github.com/jackc/pgx/internal/sanitize github.com/jackc/pgx/pgio github.com/jackc/pgx/pgproto3 github.com/jackc/pgx/pgtype +# github.com/jackc/pgx/v4 v4.6.0 +## explicit +github.com/jackc/pgx/v4 +github.com/jackc/pgx/v4/internal/sanitize # github.com/jaegertracing/jaeger v1.17.0 ## explicit github.com/jaegertracing/jaeger/model/json @@ -551,7 +571,6 @@ github.com/knz/go-libedit/unix ## explicit github.com/knz/strtime # github.com/konsorten/go-windows-terminal-sequences v1.0.2 -## explicit github.com/konsorten/go-windows-terminal-sequences # github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 github.com/kr/logfmt @@ -594,7 +613,7 @@ github.com/maruel/panicparse/stack # github.com/marusama/semaphore v0.0.0-20190110074507-6952cef993b2 ## explicit github.com/marusama/semaphore -# github.com/mattn/go-isatty v0.0.9 +# github.com/mattn/go-isatty v0.0.12 ## explicit github.com/mattn/go-isatty # github.com/mattn/go-runewidth v0.0.7 @@ -776,7 +795,7 @@ go.opencensus.io/trace go.opencensus.io/trace/internal go.opencensus.io/trace/propagation go.opencensus.io/trace/tracestate -# golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 +# golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 ## explicit golang.org/x/crypto/bcrypt golang.org/x/crypto/blowfish @@ -842,7 +861,7 @@ golang.org/x/sys/cpu golang.org/x/sys/internal/unsafeheader golang.org/x/sys/unix golang.org/x/sys/windows -# golang.org/x/text v0.3.2 +# golang.org/x/text v0.3.3 ## explicit golang.org/x/text/cases golang.org/x/text/collate From 15dbca9302a83818f017e1e860c666b435bff3a8 Mon Sep 17 00:00:00 2001 From: Bilal Akhtar Date: Mon, 13 Jul 2020 15:13:48 -0400 Subject: [PATCH 11/49] Bump pebble to 36c1b9c7a833 --- github.com/cockroachdb/pebble/Makefile | 2 +- github.com/cockroachdb/pebble/README.md | 1 + github.com/cockroachdb/pebble/batch.go | 9 +- github.com/cockroachdb/pebble/compaction.go | 591 +++++++++++------- .../cockroachdb/pebble/compaction_picker.go | 492 +++++++++++---- github.com/cockroachdb/pebble/db.go | 27 +- github.com/cockroachdb/pebble/event.go | 188 ++++-- github.com/cockroachdb/pebble/go.mod | 1 + github.com/cockroachdb/pebble/go.sum | 2 + github.com/cockroachdb/pebble/ingest.go | 6 +- .../pebble/internal/manifest/l0_sublevels.go | 98 +-- .../internal/manifest/level_iterator.go | 77 +++ .../pebble/internal/manifest/version.go | 32 +- github.com/cockroachdb/pebble/options.go | 16 +- .../cockroachdb/pebble/sstable/reader.go | 8 +- github.com/cockroachdb/pebble/table_cache.go | 2 +- github.com/cockroachdb/pebble/table_stats.go | 141 +++-- github.com/cockroachdb/pebble/tool/find.go | 2 +- github.com/cockroachdb/pebble/tool/lsm.go | 8 +- .../cockroachdb/pebble/tool/lsm_data.go | 5 +- github.com/cockroachdb/pebble/tool/sstable.go | 2 +- modules.txt | 2 +- 22 files changed, 1173 insertions(+), 539 deletions(-) create mode 100644 github.com/cockroachdb/pebble/internal/manifest/level_iterator.go diff --git a/github.com/cockroachdb/pebble/Makefile b/github.com/cockroachdb/pebble/Makefile index 32d3dd4dfc..ede486fdd9 100644 --- a/github.com/cockroachdb/pebble/Makefile +++ b/github.com/cockroachdb/pebble/Makefile @@ -27,7 +27,7 @@ testrace: test .PHONY: stress stressrace stressrace: testflags += -race -stress stressrace: testflags += -exec 'stress ${STRESSFLAGS}' -timeout 0 +stress stressrace: testflags += -exec 'stress ${STRESSFLAGS}' -timeout 0 -test.v stress stressrace: test .PHONY: stressmeta diff --git a/github.com/cockroachdb/pebble/README.md b/github.com/cockroachdb/pebble/README.md index 6b095e00ee..6c975355e1 100644 --- a/github.com/cockroachdb/pebble/README.md +++ b/github.com/cockroachdb/pebble/README.md @@ -1,4 +1,5 @@ # Pebble [![Build Status](https://travis-ci.org/cockroachdb/pebble.svg?branch=master)](https://travis-ci.org/cockroachdb/pebble) +#### [Nightly benchmarks](https://cockroachdb.github.io/pebble/) Pebble is a LevelDB/RocksDB inspired key-value store focused on performance and internal usage by CockroachDB. Pebble inherits the diff --git a/github.com/cockroachdb/pebble/batch.go b/github.com/cockroachdb/pebble/batch.go index 3020ac8b39..299fd83291 100644 --- a/github.com/cockroachdb/pebble/batch.go +++ b/github.com/cockroachdb/pebble/batch.go @@ -755,10 +755,11 @@ func (b *Batch) init(cap int) { b.data = b.data[:batchHeaderLen] } -// Reset clears the underlying byte slice and effectively empties the batch for -// reuse. Used in cases where Batch is only being used to build a batch, and -// where the end result is a Repr() call, not a Commit call or a Close call. -// Commits and Closes take care of releasing resources when appropriate. +// Reset resets the batch for reuse. The underlying byte slice (that is +// returned by Repr()) is not modified. It is only necessary to call this +// method if a batch is explicitly being reused. Close automatically takes are +// of releasing resources when appropriate for batches that are internally +// being reused. func (b *Batch) Reset() { b.count = 0 b.countRangeDels = 0 diff --git a/github.com/cockroachdb/pebble/compaction.go b/github.com/cockroachdb/pebble/compaction.go index 7dadd47233..3068b781bc 100644 --- a/github.com/cockroachdb/pebble/compaction.go +++ b/github.com/cockroachdb/pebble/compaction.go @@ -16,7 +16,6 @@ import ( "strings" "sync/atomic" "time" - "unsafe" "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" @@ -41,8 +40,8 @@ func expandedCompactionByteSizeLimit(opts *Options, level int) uint64 { return uint64(25 * opts.Level(level).TargetFileSize) } -// maxGrandparentOverlapBytes is the maximum bytes of overlap with level+2 -// before we stop building a single file in a level to level+1 compaction. +// maxGrandparentOverlapBytes is the maximum bytes of overlap with level+1 +// before we stop building a single file in a level-1 to level compaction. func maxGrandparentOverlapBytes(opts *Options, level int) uint64 { return uint64(10 * opts.Level(level).TargetFileSize) } @@ -75,9 +74,19 @@ type compactionLevel struct { files []*fileMetadata } +type compactionKind string + +const ( + compactionKindDefault compactionKind = "default" + compactionKindFlush = "flush" + compactionKindMove = "move" + compactionKindDeleteOnly = "delete-only" +) + // compaction is a table compaction from one level to the next, starting from a // given version. type compaction struct { + kind compactionKind cmp Compare formatKey base.FormatKey logger Logger @@ -138,7 +147,7 @@ type compaction struct { // grandparents are the tables in level+2 that overlap with the files being // compacted. Used to determine output table boundaries. Do not assume that the actual files // in the grandparent when this compaction finishes will be the same. - grandparents []*fileMetadata + grandparents manifest.LevelIterator // Boundaries at which flushes to L0 should be split. Determined by // L0Sublevels. If nil, flushes aren't split. @@ -151,47 +160,101 @@ type compaction struct { inuseKeyRanges []userKeyRange elideTombstoneIndex int - // L0-specific compaction info. Set to a non-nil value for all compactions - // where startLevel == 0 that were generated by L0Sublevels. - lcf *manifest.L0CompactionFiles + // allowedZeroSeqNum is true if seqnums can be zeroed if there are no + // snapshots requiring them to be kept. This determination is made by + // looking for an sstable which overlaps the bounds of the compaction at a + // lower level in the LSM during runCompaction. + allowedZeroSeqNum bool metrics map[int]*LevelMetrics } -func newCompaction( - opts *Options, cur *version, startLevel, - baseLevel int, bytesCompacted *uint64, -) *compaction { - if startLevel > 0 && startLevel < baseLevel { - panic(fmt.Sprintf("invalid compaction: start level %d should be empty (base level %d)", - startLevel, baseLevel)) +func (c *compaction) makeInfo(jobID int) CompactionInfo { + info := CompactionInfo{ + JobID: jobID, + Input: make([]LevelInfo, 0, len(c.inputs)), } - - outputLevel := startLevel + 1 - if startLevel == 0 { - outputLevel = baseLevel + for _, cl := range c.inputs { + inputInfo := LevelInfo{Level: cl.level, Tables: make([]TableInfo, 0, len(cl.files))} + for _, m := range cl.files { + inputInfo.Tables = append(inputInfo.Tables, m.TableInfo()) + } + info.Input = append(info.Input, inputInfo) } - if outputLevel >= numLevels-1 { - outputLevel = numLevels - 1 + if c.outputLevel != nil { + info.Output.Level = c.outputLevel.level + // If there are no inputs from the output level (eg, a move + // compaction), add an empty LevelInfo to info.Input. + if len(c.inputs) > 0 && c.inputs[len(c.inputs)-1].level != c.outputLevel.level { + info.Input = append(info.Input, LevelInfo{Level: c.outputLevel.level}) + } + } else { + // For a delete-only compaction, set the output level to L6. The + // output level is not meaningful here, but complicating the + // info.Output interface with a pointer doesn't seem worth the + // semantic distinction. + info.Output.Level = numLevels - 1 } - // Output level is in the range [baseLevel,numLevels]. For the purpose of - // determining the target output file size, overlap bytes, and expanded - // bytes, we want to adjust the range to [1,numLevels]. - adjustedOutputLevel := 1 + outputLevel - baseLevel + return info +} +func newCompaction( + pc *pickedCompaction, opts *Options, bytesCompacted *uint64, +) *compaction { c := &compaction{ - cmp: opts.Comparer.Compare, + kind: compactionKindDefault, + cmp: pc.cmp, formatKey: opts.Comparer.FormatKey, + score: pc.score, + smallest: pc.smallest, + largest: pc.largest, logger: opts.Logger, - version: cur, - inputs: []compactionLevel{{level: startLevel}, {level: outputLevel}}, - maxOutputFileSize: uint64(opts.Level(adjustedOutputLevel).TargetFileSize), - maxOverlapBytes: maxGrandparentOverlapBytes(opts, adjustedOutputLevel), - maxExpandedBytes: expandedCompactionByteSizeLimit(opts, adjustedOutputLevel), + version: pc.version, + inputs: pc.inputs, + maxOutputFileSize: pc.maxOutputFileSize, + maxOverlapBytes: pc.maxOverlapBytes, + maxExpandedBytes: pc.maxOverlapBytes, atomicBytesIterated: bytesCompacted, } c.startLevel = &c.inputs[0] c.outputLevel = &c.inputs[1] + + // Compute the set of outputLevel+1 files that overlap this compaction (these + // are the grandparent sstables). + if c.outputLevel.level+1 < numLevels { + c.grandparents = c.version.Overlaps(c.outputLevel.level+1, c.cmp, c.smallest.UserKey, c.largest.UserKey) + } + c.setupInuseKeyRanges() + + // Check if this compaction can be converted into a trivial move from one + // level to the next. We avoid such a move if there is lots of overlapping + // grandparent data. Otherwise, the move could create a parent file that + // will require a very expensive merge later on. + if len(c.startLevel.files) == 1 && len(c.outputLevel.files) == 0 && + c.grandparents.SizeSum() <= c.maxOverlapBytes { + c.kind = compactionKindMove + } + return c +} + +func newDeleteOnlyCompaction( + opts *Options, cur *version, inputs []compactionLevel, +) *compaction { + c := &compaction{ + kind: compactionKindDeleteOnly, + cmp: opts.Comparer.Compare, + formatKey: opts.Comparer.FormatKey, + logger: opts.Logger, + version: cur, + inputs: inputs, + } + + // Set c.smallest, c.largest. + levelFiles := make([][]*fileMetadata, 0, len(inputs)) + for _, in := range inputs { + levelFiles = append(levelFiles, in.files) + } + c.smallest, c.largest = manifest.KeyRange(opts.Comparer.Compare, levelFiles...) return c } @@ -199,6 +262,7 @@ func newFlush( opts *Options, cur *version, baseLevel int, flushing flushableList, bytesFlushed *uint64, ) *compaction { c := &compaction{ + kind: compactionKindFlush, cmp: opts.Comparer.Compare, formatKey: opts.Comparer.FormatKey, logger: opts.Logger, @@ -274,166 +338,6 @@ func newFlush( return c } -// setupInputs fills in the rest of the compaction inputs, regardless of -// whether the compaction was automatically scheduled or user initiated. -func (c *compaction) setupInputs() { - // Expand the initial inputs to a clean cut. - c.startLevel.files = c.expandInputs(c.startLevel.level, c.startLevel.files) - c.smallest, c.largest = manifest.KeyRange(c.cmp, c.startLevel.files, nil) - - // Determine the sstables in the output level which overlap with the input - // sstables, and then expand those tables to a clean cut. - c.outputLevel.files = c.version.Overlaps(c.outputLevel.level, c.cmp, c.smallest.UserKey, c.largest.UserKey) - c.outputLevel.files = c.expandInputs(c.outputLevel.level, c.outputLevel.files) - c.smallest, c.largest = manifest.KeyRange(c.cmp, c.startLevel.files, c.outputLevel.files) - - // Grow the sstables in c.startLevel.level as long as it doesn't affect the number - // of sstables included from c.outputLevel.level. - if c.lcf != nil && c.startLevel.level == 0 && c.outputLevel.level != 0 { - // Call the L0-specific compaction extension method. Similar logic as - // c.grow. Additional L0 files are optionally added to the compaction at - // this step. - if c.version.L0Sublevels.ExtendL0ForBaseCompactionTo(c.smallest, c.largest, c.lcf) { - c.startLevel.files = c.startLevel.files[:0] - for j := range c.lcf.FilesIncluded { - if c.lcf.FilesIncluded[j] { - c.startLevel.files = append(c.startLevel.files, c.version.Levels[0][j]) - } - } - c.smallest, c.largest = manifest.KeyRange(c.cmp, c.startLevel.files, c.outputLevel.files) - } - } else if c.grow(c.smallest, c.largest) { - c.smallest, c.largest = manifest.KeyRange(c.cmp, c.startLevel.files, c.outputLevel.files) - } - - // Compute the set of outputLevel+1 files that overlap this compaction (these - // are the grandparent sstables). - if c.outputLevel.level+1 < numLevels { - c.grandparents = c.version.Overlaps(c.outputLevel.level+1, c.cmp, c.smallest.UserKey, c.largest.UserKey) - } - - c.setupInuseKeyRanges() -} - -// expandInputs expands the files in inputs in order to maintain the invariant -// that the versions of keys at level+1 are older than the versions of keys at -// level. This is achieved by adding tables to the right of the current input -// tables such that the rightmost table has a "clean cut". A clean cut is -// either a change in user keys, or when the largest key in the left sstable is -// a range tombstone sentinel key (InternalKeyRangeDeleteSentinel). -// -// In addition to maintaining the seqnum invariant, expandInputs is used to -// provide clean boundaries for range tombstone truncation during -// compaction. In order to achieve these clean boundaries, expandInputs needs -// to find a "clean cut" on the left edge of the compaction as well. This is -// necessary in order for "atomic compaction units" to always be compacted as a -// unit. Failure to do this leads to a subtle bug with truncation of range -// tombstones to atomic compaction unit boundaries. Consider the scenario: -// -// L3: -// 12:[a#2,15-b#1,1] -// 13:[b#0,15-d#72057594037927935,15] -// -// These sstables contain a range tombstone [a-d)#2 which spans the two -// sstables. The two sstables need to always be kept together. Compacting -// sstable 13 independently of sstable 12 would result in: -// -// L3: -// 12:[a#2,15-b#1,1] -// L4: -// 14:[b#0,15-d#72057594037927935,15] -// -// This state is still ok, but when sstable 12 is next compacted, its range -// tombstones will be truncated at "b" (the largest key in its atomic -// compaction unit). In the scenario here, that could result in b#1 becoming -// visible when it should be deleted. -func (c *compaction) expandInputs(level int, inputs []*fileMetadata) []*fileMetadata { - if level == 0 { - // We already called version.Overlaps for L0 and that call guarantees that - // we get a "clean cut". - return inputs - } - if len(inputs) == 0 { - // Nothing to expand. - return inputs - } - files := c.version.Levels[level] - // Pointer arithmetic to figure out the index if inputs[0] with - // files[0]. This requires that the inputs slice is a sub-slice of - // files. This is true for non-L0 files returned from version.overlaps. - if uintptr(unsafe.Pointer(&inputs[0])) < uintptr(unsafe.Pointer(&files[0])) { - panic("pebble: invalid input slice") - } - start := int((uintptr(unsafe.Pointer(&inputs[0])) - - uintptr(unsafe.Pointer(&files[0]))) / unsafe.Sizeof(inputs[0])) - if start >= len(files) { - panic("pebble: invalid input slice") - } - end := start + len(inputs) - - for ; start > 0; start-- { - cur := files[start] - prev := files[start-1] - if c.cmp(prev.Largest.UserKey, cur.Smallest.UserKey) < 0 { - break - } - if prev.Largest.Trailer == InternalKeyRangeDeleteSentinel { - // The range deletion sentinel key is set for the largest key in a - // table when a range deletion tombstone straddles a table. It - // isn't necessary to include the prev table in the atomic - // compaction unit as prev.largest.UserKey does not actually exist - // in the prev table. - break - } - // prev.Largest.UserKey == cur.Smallest.UserKey, so we need to include prev - // in the compaction. - } - - for ; end < len(files); end++ { - cur := files[end-1] - next := files[end] - if c.cmp(cur.Largest.UserKey, next.Smallest.UserKey) < 0 { - break - } - if cur.Largest.Trailer == InternalKeyRangeDeleteSentinel { - // The range deletion sentinel key is set for the largest key in a table - // when a range deletion tombstone straddles a table. It isn't necessary - // to include the next table in the compaction as cur.largest.UserKey - // does not actually exist in the table. - break - } - // cur.Largest.UserKey == next.Smallest.UserKey, so we need to include next - // in the compaction. - } - return files[start:end] -} - -// grow grows the number of inputs at c.level without changing the number of -// c.level+1 files in the compaction, and returns whether the inputs grew. sm -// and la are the smallest and largest InternalKeys in all of the inputs. -func (c *compaction) grow(sm, la InternalKey) bool { - if len(c.outputLevel.files) == 0 { - return false - } - grow0 := c.version.Overlaps(c.startLevel.level, c.cmp, sm.UserKey, la.UserKey) - grow0 = c.expandInputs(c.startLevel.level, grow0) - if len(grow0) <= len(c.startLevel.files) { - return false - } - if totalSize(grow0)+totalSize(c.outputLevel.files) >= c.maxExpandedBytes { - return false - } - sm1, la1 := manifest.KeyRange(c.cmp, grow0, nil) - grow1 := c.version.Overlaps(c.outputLevel.level, c.cmp, sm1.UserKey, la1.UserKey) - grow1 = c.expandInputs(c.outputLevel.level, grow1) - if len(grow1) != len(c.outputLevel.files) { - return false - } - c.startLevel.files = grow0 - c.outputLevel.files = grow1 - return true -} - func (c *compaction) setupInuseKeyRanges() { level := c.outputLevel.level + 1 if c.outputLevel.level == 0 { @@ -446,9 +350,8 @@ func (c *compaction) setupInuseKeyRanges() { // levels. var input []userKeyRange for ; level < numLevels; level++ { - overlaps := c.version.Overlaps(level, c.cmp, c.smallest.UserKey, c.largest.UserKey) - for i := range overlaps { - m := overlaps[i] + iter := c.version.Overlaps(level, c.cmp, c.smallest.UserKey, c.largest.UserKey) + for m := iter.First(); m != nil; m = iter.Next() { input = append(input, userKeyRange{m.Smallest.UserKey, m.Largest.UserKey}) } } @@ -479,21 +382,6 @@ func (c *compaction) setupInuseKeyRanges() { } } -func (c *compaction) trivialMove() bool { - if len(c.flushing) != 0 { - return false - } - // Check for a trivial move of one table from one level to the next. We avoid - // such a move if there is lots of overlapping grandparent data. Otherwise, - // the move could create a parent file that will require a very expensive - // merge later on. - if len(c.startLevel.files) == 1 && len(c.outputLevel.files) == 0 && - totalSize(c.grandparents) <= c.maxOverlapBytes { - return true - } - return false -} - // findGrandparentLimit takes the start user key for a table and returns the // user key to which that table can extend without excessively overlapping // the grandparent level. If no limit is needed considering the grandparent @@ -512,17 +400,15 @@ func (c *compaction) trivialMove() bool { // user-key. Perhaps this isn't a problem if the compaction picking heuristics // always pick the right (older) sibling for compaction first. func (c *compaction) findGrandparentLimit(start []byte) []byte { - lower := sort.Search(len(c.grandparents), func(i int) bool { - return c.cmp(start, c.grandparents[i].Largest.UserKey) <= 0 - }) + iter := c.grandparents var overlappedBytes uint64 - for upper := lower; upper < len(c.grandparents); upper++ { - overlappedBytes += c.grandparents[upper].Size + for f := iter.SeekGE(c.cmp, start); f != nil; f = iter.Next() { + overlappedBytes += f.Size // To ensure forward progress we always return a larger user // key than where we started. See comments above clients of // this function for how this is used. - if overlappedBytes > c.maxOverlapBytes && c.cmp(start, c.grandparents[upper].Largest.UserKey) < 0 { - return c.grandparents[upper].Largest.UserKey + if overlappedBytes > c.maxOverlapBytes && c.cmp(start, f.Largest.UserKey) < 0 { + return f.Largest.UserKey } } return nil @@ -587,8 +473,8 @@ func (c *compaction) elideRangeTombstone(start, end []byte) bool { // being flushed in the same compaction. It's possible for a range tombstone // in one memtable to overlap keys in a preceding memtable in c.flushing. // - // This function is also used in allowZeroSeqNum, so disabling elision of - // range tombstones also disables zeroing of SeqNums. + // This function is also used in setting allowZeroSeqNum, so disabling + // elision of range tombstones also disables zeroing of SeqNums. // // TODO(peter): we disable zeroing of seqnums during flushing to match // RocksDB behavior and to avoid generating overlapping sstables during @@ -832,7 +718,7 @@ func (d *DB) addInProgressCompaction(c *compaction) { d.opts.Logger.Fatalf("L%d->L%d: %s already being compacted", c.startLevel.level, c.outputLevel.level, f.FileNum) } f.Compacting = true - if c.startLevel.level == 0 { + if c.startLevel != nil && c.outputLevel != nil && c.startLevel.level == 0 { if c.outputLevel.level == 0 { f.IsIntraL0Compacting = true isIntraL0 = true @@ -1120,6 +1006,7 @@ func (d *DB) flush1() error { } } + d.maybeUpdateDeleteCompactionHints(c) d.removeInProgressCompaction(c) d.mu.versions.incrementFlushes() d.opts.EventListener.FlushEnd(info) @@ -1185,11 +1072,31 @@ func (d *DB) maybeScheduleCompaction() { bytesCompacted: &d.bytesCompacted, earliestUnflushedSeqNum: d.getEarliestUnflushedSeqNumLocked(), } + + // Check for delete-only compactions first, because they're expected to be + // cheap and reduce future compaction work. + if len(d.mu.compact.deletionHints) > 0 && + d.mu.compact.compactingCount < d.opts.MaxConcurrentCompactions && + !d.opts.private.disableAutomaticCompactions { + v := d.mu.versions.currentVersion() + snapshots := d.mu.snapshots.toSlice() + inputs, unresolvedHints := checkDeleteCompactionHints(d.cmp, v, d.mu.compact.deletionHints, snapshots) + d.mu.compact.deletionHints = unresolvedHints + + if len(inputs) > 0 { + c := newDeleteOnlyCompaction(d.opts, v, inputs) + d.mu.compact.compactingCount++ + d.addInProgressCompaction(c) + go d.compact(c, nil) + } + } + for len(d.mu.compact.manual) > 0 && d.mu.compact.compactingCount < d.opts.MaxConcurrentCompactions { manual := d.mu.compact.manual[0] env.inProgressCompactions = d.getInProgressCompactionInfoLocked(nil) - c, retryLater := d.mu.versions.picker.pickManual(env, manual) - if c != nil { + pc, retryLater := d.mu.versions.picker.pickManual(env, manual) + if pc != nil { + c := newCompaction(pc, d.opts, env.bytesCompacted) d.mu.compact.manual = d.mu.compact.manual[1:] d.mu.compact.compactingCount++ d.addInProgressCompaction(c) @@ -1207,16 +1114,213 @@ func (d *DB) maybeScheduleCompaction() { for !d.opts.private.disableAutomaticCompactions && d.mu.compact.compactingCount < d.opts.MaxConcurrentCompactions { env.inProgressCompactions = d.getInProgressCompactionInfoLocked(nil) - c := d.mu.versions.picker.pickAuto(env) - if c == nil { + pc := d.mu.versions.picker.pickAuto(env) + if pc == nil { break } + c := newCompaction(pc, d.opts, env.bytesCompacted) d.mu.compact.compactingCount++ d.addInProgressCompaction(c) go d.compact(c, nil) } } +// A deleteCompactionHint records a user key and sequence number span that has been +// deleted by a range tombstone. A hint is recorded if at least one sstable +// falls completely within both the user key and sequence number spans. +// Once the tombstones and the observed completely-contained sstables fall +// into the same snapshot stripe, a delete-only compaction may delete any +// sstables within the range. +type deleteCompactionHint struct { + start []byte + end []byte + // The level of the file containing the range tombstone(s) when the hint + // was created. Only lower levels need to be searched for files that may + // be deleted. + tombstoneLevel int + // The file containing the range tombstone(s) that created the hint. + tombstoneFile *fileMetadata + // The smallest and largest sequence numbers of the abutting tombstones + // merged to form this hint. All of a tables' keys must be less than the + // tombstone smallest sequence number to be deleted. All of a tables' + // sequence numbers must fall into the same snapshot stripe as the + // tombstone largest sequence number to be deleted. + tombstoneLargestSeqNum uint64 + tombstoneSmallestSeqNum uint64 + // The smallest sequence number of a sstable that was found to be covered + // by this hint. The hint cannot be resolved until this sequence number is + // in the same snapshot stripe as the largest tombstone sequence number. + // This is set when a hint is created, so the LSM may look different and + // notably no longer contian the sstable that contained the key at this + // sequence number. + fileSmallestSeqNum uint64 +} + +func (h deleteCompactionHint) String() string { + return fmt.Sprintf("L%d.%s %s-%s seqnums(tombstone=%d-%d, file-smallest=%d)", + h.tombstoneLevel, h.tombstoneFile.FileNum, h.start, h.end, + h.tombstoneSmallestSeqNum, h.tombstoneLargestSeqNum, h.fileSmallestSeqNum) +} + +func (h *deleteCompactionHint) canDelete(cmp Compare, m *fileMetadata, snapshots []uint64) bool { + // The file can only be deleted if all of its keys are older than the + // earliest tombstone aggregated into the hint. + if m.LargestSeqNum >= h.tombstoneSmallestSeqNum || m.SmallestSeqNum < h.fileSmallestSeqNum { + return false + } + + // The file's oldest key must be in the same snapshot stripe as the + // newest tombstone. NB: We already checked the hint's sequence numbers, + // but this file's oldest sequence number might be lower than the hint's + // smallest sequence number despite the file falling within the key range + // if this file was constructed after the hint by a compaction. + ti, _ := snapshotIndex(h.tombstoneLargestSeqNum, snapshots) + fi, _ := snapshotIndex(m.SmallestSeqNum, snapshots) + if ti != fi { + return false + } + + // The file's keys must be completely contianed within the hint range. + return cmp(h.start, m.Smallest.UserKey) <= 0 && cmp(m.Largest.UserKey, h.end) < 0 +} + +func (d *DB) maybeUpdateDeleteCompactionHints(c *compaction) { + // Compactions that zero sequence numbers can interfere with compaction + // deletion hints. Deletion hints apply to tables containing keys older + // than a threshold. If a key more recent than the threshold is zeroed in + // a compaction, a delete-only compaction may mistake it as meeting the + // threshold and drop a table containing live data. + // + // To avoid this scenario, compactions that zero sequence numbers remove + // any conflicting deletion hints. A deletion hint is conflicting if both + // of the following conditions apply: + // * its key space overlaps with the compaction + // * at least one of its inputs contains a key as recent as one of the + // hint's tombstones. + // + if !c.allowedZeroSeqNum { + return + } + + updatedHints := d.mu.compact.deletionHints[:0] + for _, h := range d.mu.compact.deletionHints { + // If the compaction's key space is disjoint from the hint's key + // space, the zeroing of sequence numbers won't affect the hint. Keep + // the hint. + keysDisjoint := d.cmp(h.end, c.smallest.UserKey) < 0 || d.cmp(h.start, c.largest.UserKey) > 0 + if keysDisjoint { + updatedHints = append(updatedHints, h) + continue + } + + // All of the compaction's inputs must be older than the hint's + // tombstones. + inputsOlder := true + for _, in := range c.inputs { + for _, f := range in.files { + inputsOlder = inputsOlder && f.LargestSeqNum < h.tombstoneSmallestSeqNum + } + } + if inputsOlder { + updatedHints = append(updatedHints, h) + continue + } + + // Drop h, because the compaction c may have zeroed sequence numbers + // of keys more recent than some of h's tombstones. + } + d.mu.compact.deletionHints = updatedHints +} + +func checkDeleteCompactionHints( + cmp Compare, v *version, hints []deleteCompactionHint, snapshots []uint64, +) ([]compactionLevel, []deleteCompactionHint) { + var files map[*fileMetadata]bool + var byLevel [numLevels][]*fileMetadata + + unresolvedHints := hints[:0] + for _, h := range hints { + // Check each compaction hint to see if it's resolvable. Resolvable + // hints are removed and trigger a delete-only compaction if any files + // in the current LSM still meet their criteria. Unresolvable hints + // are saved and don't trigger a delete-only compaction. + // + // When a compaction hint is created, the sequence numbers of the + // range tombstones and the covered file with the oldest key are + // recorded. The largest tombstone sequence number and the smallest + // file sequence number must be in the same snapshot stripe for the + // hint to be resolved. The below graphic models a compaction hint + // covering the keyspace [b, r). The hint completely contains two + // files, 000002 and 000003. The file 000003 contains the lowest + // covered sequence number at #90. The tombstone b.RANGEDEL.230:h has + // the highest tombstone sequence number incorporated into the hint. + // The hint may be resolved only once the snapshots at #100, #180 and + // #210 are all closed. File 000001 is not included within the hint + // because it extends beyond the range tombstones in user key space. + // + // 250 + // + // |-b...230:h-| + // _____________________________________________________ snapshot #210 + // 200 |--h.RANGEDEL.200:r--| + // + // _____________________________________________________ snapshot #180 + // + // 150 +--------+ + // +---------+ | 000003 | + // | 000002 | | | + // +_________+ | | + // 100_____________________|________|___________________ snapshot #100 + // +--------+ + // _____________________________________________________ snapshot #70 + // +---------------+ + // 50 | 000001 | + // | | + // +---------------+ + // ______________________________________________________________ + // a b c d e f g h i j k l m n o p q r s t u v w x y z + + ti, _ := snapshotIndex(h.tombstoneLargestSeqNum, snapshots) + fi, _ := snapshotIndex(h.fileSmallestSeqNum, snapshots) + if ti != fi { + // Cannot resolve yet. + unresolvedHints = append(unresolvedHints, h) + continue + } + + // The hint h will be resolved and dropped, regardless of whether + // there are any tables that can be deleted. + for l := h.tombstoneLevel + 1; l < numLevels; l++ { + overlaps := v.Overlaps(l, cmp, h.start, h.end) + for m := overlaps.First(); m != nil; m = overlaps.Next() { + if m.Compacting || !h.canDelete(cmp, m, snapshots) || files[m] { + continue + } + + if files == nil { + // Construct files lazily, assuming most calls will not + // produce delete-only compactions. + files = make(map[*fileMetadata]bool) + } + files[m] = true + byLevel[l] = append(byLevel[l], m) + } + } + } + + var compactLevels []compactionLevel + for l, files := range byLevel { + if len(files) == 0 { + continue + } + compactLevels = append(compactLevels, compactionLevel{ + level: l, + files: files, + }) + } + return compactLevels, unresolvedHints +} + // compact runs one compaction and maybe schedules another call to compact. func (d *DB) compact(c *compaction, errChannel chan error) { pprof.Do(context.Background(), compactLabels, func(context.Context) { @@ -1247,20 +1351,7 @@ func (d *DB) compact1(c *compaction, errChannel chan error) (err error) { jobID := d.mu.nextJobID d.mu.nextJobID++ - info := CompactionInfo{ - JobID: jobID, - Input: []LevelInfo{ - {Level: c.startLevel.level}, - {Level: c.outputLevel.level}, - }, - Output: LevelInfo{Level: c.outputLevel.level}, - } - for i, cl := range c.inputs { - for _, m := range cl.files { - info.Input[i].Tables = append(info.Input[i].Tables, m.TableInfo()) - } - } - + info := c.makeInfo(jobID) d.opts.EventListener.CompactionBegin(info) startTime := d.timeNow() @@ -1297,6 +1388,7 @@ func (d *DB) compact1(c *compaction, errChannel chan error) (err error) { } } + d.maybeUpdateDeleteCompactionHints(c) d.removeInProgressCompaction(c) d.mu.versions.incrementCompactions() d.opts.EventListener.CompactionEnd(info) @@ -1322,11 +1414,30 @@ func (d *DB) compact1(c *compaction, errChannel chan error) (err error) { func (d *DB) runCompaction( jobID int, c *compaction, pacer pacer, ) (ve *versionEdit, pendingOutputs []FileNum, retErr error) { + // Check for a delete-only compaction. This can occur when wide range + // tombstones completely contain sstables. + if c.kind == compactionKindDeleteOnly { + c.metrics = make(map[int]*LevelMetrics, len(c.inputs)) + ve := &versionEdit{ + DeletedFiles: map[deletedFileEntry]bool{}, + } + for _, cl := range c.inputs { + c.metrics[cl.level] = &LevelMetrics{} + for _, f := range cl.files { + ve.DeletedFiles[deletedFileEntry{ + Level: cl.level, + FileNum: f.FileNum, + }] = true + } + } + return ve, nil, nil + } + // Check for a trivial move of one table from one level to the next. We avoid // such a move if there is lots of overlapping grandparent data. Otherwise, // the move could create a parent file that will require a very expensive // merge later on. - if c.trivialMove() { + if c.kind == compactionKindMove { meta := c.startLevel.files[0] c.metrics = map[int]*LevelMetrics{ c.outputLevel.level: &LevelMetrics{ @@ -1362,9 +1473,9 @@ func (d *DB) runCompaction( if err != nil { return nil, pendingOutputs, err } - allowZeroSeqNum := c.allowZeroSeqNum(iiter) + c.allowedZeroSeqNum = c.allowZeroSeqNum(iiter) iter := newCompactionIter(c.cmp, d.merge, iiter, snapshots, &c.rangeDelFrag, - allowZeroSeqNum, c.elideTombstone, c.elideRangeTombstone) + c.allowedZeroSeqNum, c.elideTombstone, c.elideRangeTombstone) var ( filenames []string @@ -1587,7 +1698,7 @@ func (d *DB) runCompaction( if meta.Largest.Trailer >= c.largest.Trailer { break } - if allowZeroSeqNum && meta.Largest.SeqNum() == 0 { + if c.allowedZeroSeqNum && meta.Largest.SeqNum() == 0 { break } fallthrough diff --git a/github.com/cockroachdb/pebble/compaction_picker.go b/github.com/cockroachdb/pebble/compaction_picker.go index ebfb0a5482..c65bea728e 100644 --- a/github.com/cockroachdb/pebble/compaction_picker.go +++ b/github.com/cockroachdb/pebble/compaction_picker.go @@ -9,6 +9,7 @@ import ( "fmt" "math" "sort" + "unsafe" "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/humanize" @@ -30,8 +31,8 @@ type compactionPicker interface { getBaseLevel() int getEstimatedMaxWAmp() float64 estimatedCompactionDebt(l0ExtraSize uint64) uint64 - pickAuto(env compactionEnv) (c *compaction) - pickManual(env compactionEnv, manual *manualCompaction) (c *compaction, retryLater bool) + pickAuto(env compactionEnv) (pc *pickedCompaction) + pickManual(env compactionEnv, manual *manualCompaction) (c *pickedCompaction, retryLater bool) forceBaseLevel1() } @@ -43,7 +44,7 @@ type compactionInfo struct { outputLevel int } -type sortCompactionLevelsDecreasingScore []pickedCompactionInfo +type sortCompactionLevelsDecreasingScore []candidateLevelInfo func (s sortCompactionLevelsDecreasingScore) Len() int { return len(s) @@ -58,6 +59,296 @@ func (s sortCompactionLevelsDecreasingScore) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +// pickedCompaction contains information about a compaction that has already +// been chosen, and is being constructed. Compaction construction info lives in +// this struct, and is copied over into the compaction struct when that's +// created. +type pickedCompaction struct { + cmp Compare + + // score of the chosen compaction. Taken from candidateLevelInfo. + score float64 + + // startLevel is the level that is being compacted. Inputs from startLevel + // and outputLevel will be merged to produce a set of outputLevel files. + startLevel *compactionLevel + // outputLevel is the level that files are being produced in. outputLevel is + // equal to startLevel+1 except when startLevel is 0 in which case it is + // equal to compactionPicker.baseLevel(). + outputLevel *compactionLevel + + inputs []compactionLevel + + // L0-specific compaction info. Set to a non-nil value for all compactions + // where startLevel == 0 that were generated by L0Sublevels. + lcf *manifest.L0CompactionFiles + + // maxOutputFileSize is the maximum size of an individual table created + // during compaction. + maxOutputFileSize uint64 + // maxOverlapBytes is the maximum number of bytes of overlap allowed for a + // single output table with the tables in the grandparent level. + maxOverlapBytes uint64 + // maxExpandedBytes is the maximum size of an expanded compaction. If growing + // a compaction results in a larger size, the original compaction is used + // instead. + maxExpandedBytes uint64 + + // The boundaries of the input data. + smallest InternalKey + largest InternalKey + + version *version +} + +func newPickedCompaction( + opts *Options, cur *version, startLevel, baseLevel int, +) *pickedCompaction { + if startLevel > 0 && startLevel < baseLevel { + panic(fmt.Sprintf("invalid compaction: start level %d should not be empty (base level %d)", + startLevel, baseLevel)) + } + + outputLevel := startLevel + 1 + if startLevel == 0 { + outputLevel = baseLevel + } + if outputLevel >= numLevels-1 { + outputLevel = numLevels - 1 + } + // Output level is in the range [baseLevel,numLevels]. For the purpose of + // determining the target output file size, overlap bytes, and expanded + // bytes, we want to adjust the range to [1,numLevels]. + adjustedOutputLevel := 1 + outputLevel - baseLevel + + pc := &pickedCompaction{ + cmp: opts.Comparer.Compare, + version: cur, + inputs: []compactionLevel{{level: startLevel}, {level: outputLevel}}, + maxOutputFileSize: uint64(opts.Level(adjustedOutputLevel).TargetFileSize), + maxOverlapBytes: maxGrandparentOverlapBytes(opts, adjustedOutputLevel), + maxExpandedBytes: expandedCompactionByteSizeLimit(opts, adjustedOutputLevel), + } + pc.startLevel = &pc.inputs[0] + pc.outputLevel = &pc.inputs[1] + return pc +} + +func newPickedCompactionFromL0(lcf *manifest.L0CompactionFiles, opts *Options, vers *version, baseLevel int, isBase bool) *pickedCompaction { + pc := newPickedCompaction(opts, vers, 0, baseLevel) + pc.lcf = lcf + if !isBase { + pc.outputLevel.level = 0 + } + + // Manually build the compaction as opposed to calling + // pickAutoHelper. This is because L0Sublevels has already added + // any overlapping L0 SSTables that need to be added, and + // because compactions built by L0SSTables do not necessarily + // pick contiguous sequences of files in pc.version.Levels[0]. + pc.startLevel.files = make([]*manifest.FileMetadata, 0, len(lcf.Files)) + for j := range lcf.FilesIncluded { + if lcf.FilesIncluded[j] { + pc.startLevel.files = append(pc.startLevel.files, vers.Levels[0][j]) + } + } + return pc +} + +func (pc *pickedCompaction) setupInputs() { + // Expand the initial inputs to a clean cut. + pc.startLevel.files = pc.expandInputs(pc.startLevel.level, pc.startLevel.files) + pc.smallest, pc.largest = manifest.KeyRange(pc.cmp, pc.startLevel.files) + + // Determine the sstables in the output level which overlap with the input + // sstables, and then expand those tables to a clean cut. No need to do + // this for intra-L0 compactions; outputLevel.files is left empty for those. + if pc.startLevel.level != pc.outputLevel.level { + pc.outputLevel.files = pc.version.Overlaps(pc.outputLevel.level, pc.cmp, pc.smallest.UserKey, pc.largest.UserKey).Collect() + pc.outputLevel.files = pc.expandInputs(pc.outputLevel.level, pc.outputLevel.files) + pc.smallest, pc.largest = manifest.KeyRange(pc.cmp, pc.startLevel.files, pc.outputLevel.files) + } + + // Grow the sstables in pc.startLevel.level as long as it doesn't affect the number + // of sstables included from pc.outputLevel.level. + if pc.lcf != nil && pc.startLevel.level == 0 && pc.outputLevel.level != 0 { + // Call the L0-specific compaction extension method. Similar logic as + // pc.grow. Additional L0 files are optionally added to the compaction at + // this step. Note that the bounds passed in are not the bounds of the + // compaction, but rather the smallest and largest internal keys that + // the compaction cannot include from L0 without pulling in more Lbase + // files. Consider this example: + // + // L0: c-d e+f g-h + // Lbase: a-b e+f i-j + // a b c d e f g h i j + // + // The e-f files have already been chosen in the compaction. As pulling + // in more LBase files is undesirable, the logic below will pass in + // smallest = b and largest = i to ExtendL0ForBaseCompactionTo, which + // will expand the compaction to include c-d and g-h from L0. The + // bounds passed in are exclusive; the compaction cannot be expanded + // to include files that "touch" it. + smallestBaseKey := base.InvalidInternalKey + largestBaseKey := base.InvalidInternalKey + baseFiles := pc.version.Levels[pc.outputLevel.level] + if len(baseFiles) != 0 { + // smallestBaseFileIdx is the index of the smallest + // (by key ordering) base file already in the compaction. + smallestBaseFileIdx := sort.Search(len(baseFiles), func(i int) bool { + return base.InternalCompare(pc.cmp, baseFiles[i].Largest, pc.smallest) >= 0 + }) + if smallestBaseFileIdx > 0 { + smallestBaseKey = baseFiles[smallestBaseFileIdx-1].Largest + } + // largestBaseFileIdx is the index that's one higher than the + // largest base file included in the compaction. + largestBaseFileIdx := sort.Search(len(baseFiles), func(i int) bool { + return base.InternalCompare(pc.cmp, baseFiles[i].Smallest, pc.largest) > 0 + }) + if largestBaseFileIdx < len(baseFiles) { + largestBaseKey = baseFiles[largestBaseFileIdx].Smallest + } + } + oldLcf := *pc.lcf + if pc.version.L0Sublevels.ExtendL0ForBaseCompactionTo(smallestBaseKey, largestBaseKey, pc.lcf) { + newStartLevelFiles := make([]*fileMetadata, 0, len(pc.startLevel.files)) + for j := range pc.lcf.FilesIncluded { + if pc.lcf.FilesIncluded[j] { + newStartLevelFiles = append(newStartLevelFiles, pc.version.Levels[0][j]) + } + } + if totalSize(newStartLevelFiles)+totalSize(pc.outputLevel.files) < pc.maxExpandedBytes { + pc.startLevel.files = newStartLevelFiles + pc.smallest, pc.largest = manifest.KeyRange(pc.cmp, pc.startLevel.files, pc.outputLevel.files) + } else { + *pc.lcf = oldLcf + } + } + } else if pc.grow(pc.smallest, pc.largest) { + pc.smallest, pc.largest = manifest.KeyRange(pc.cmp, pc.startLevel.files, pc.outputLevel.files) + } +} + +// grow grows the number of inputs at c.level without changing the number of +// c.level+1 files in the compaction, and returns whether the inputs grew. sm +// and la are the smallest and largest InternalKeys in all of the inputs. +func (pc *pickedCompaction) grow(sm, la InternalKey) bool { + if len(pc.outputLevel.files) == 0 { + return false + } + grow0 := pc.version.Overlaps(pc.startLevel.level, pc.cmp, sm.UserKey, la.UserKey).Collect() + grow0 = pc.expandInputs(pc.startLevel.level, grow0) + if len(grow0) <= len(pc.startLevel.files) { + return false + } + if totalSize(grow0)+totalSize(pc.outputLevel.files) >= pc.maxExpandedBytes { + return false + } + sm1, la1 := manifest.KeyRange(pc.cmp, grow0) + grow1 := pc.version.Overlaps(pc.outputLevel.level, pc.cmp, sm1.UserKey, la1.UserKey).Collect() + grow1 = pc.expandInputs(pc.outputLevel.level, grow1) + if len(grow1) != len(pc.outputLevel.files) { + return false + } + pc.startLevel.files = grow0 + pc.outputLevel.files = grow1 + return true +} + +// expandInputs expands the files in inputs in order to maintain the invariant +// that the versions of keys at level+1 are older than the versions of keys at +// level. This is achieved by adding tables to the right of the current input +// tables such that the rightmost table has a "clean cut". A clean cut is +// either a change in user keys, or when the largest key in the left sstable is +// a range tombstone sentinel key (InternalKeyRangeDeleteSentinel). +// +// In addition to maintaining the seqnum invariant, expandInputs is used to +// provide clean boundaries for range tombstone truncation during +// compaction. In order to achieve these clean boundaries, expandInputs needs +// to find a "clean cut" on the left edge of the compaction as well. This is +// necessary in order for "atomic compaction units" to always be compacted as a +// unit. Failure to do this leads to a subtle bug with truncation of range +// tombstones to atomic compaction unit boundaries. Consider the scenario: +// +// L3: +// 12:[a#2,15-b#1,1] +// 13:[b#0,15-d#72057594037927935,15] +// +// These sstables contain a range tombstone [a-d)#2 which spans the two +// sstables. The two sstables need to always be kept together. Compacting +// sstable 13 independently of sstable 12 would result in: +// +// L3: +// 12:[a#2,15-b#1,1] +// L4: +// 14:[b#0,15-d#72057594037927935,15] +// +// This state is still ok, but when sstable 12 is next compacted, its range +// tombstones will be truncated at "b" (the largest key in its atomic +// compaction unit). In the scenario here, that could result in b#1 becoming +// visible when it should be deleted. +func (pc *pickedCompaction) expandInputs(level int, inputs []*fileMetadata) []*fileMetadata { + if level == 0 { + // We already called version.Overlaps for L0 and that call guarantees that + // we get a "clean cut". + return inputs + } + if len(inputs) == 0 { + // Nothing to expand. + return inputs + } + files := pc.version.Levels[level] + // Pointer arithmetic to figure out the index if inputs[0] with + // files[0]. This requires that the inputs slice is a sub-slice of + // files. This is true for non-L0 files returned from version.overlaps. + if uintptr(unsafe.Pointer(&inputs[0])) < uintptr(unsafe.Pointer(&files[0])) { + panic("pebble: invalid input slice") + } + start := int((uintptr(unsafe.Pointer(&inputs[0])) - + uintptr(unsafe.Pointer(&files[0]))) / unsafe.Sizeof(inputs[0])) + if start >= len(files) { + panic("pebble: invalid input slice") + } + end := start + len(inputs) + + for ; start > 0; start-- { + cur := files[start] + prev := files[start-1] + if pc.cmp(prev.Largest.UserKey, cur.Smallest.UserKey) < 0 { + break + } + if prev.Largest.Trailer == InternalKeyRangeDeleteSentinel { + // The range deletion sentinel key is set for the largest key in a + // table when a range deletion tombstone straddles a table. It + // isn't necessary to include the prev table in the atomic + // compaction unit as prev.largest.UserKey does not actually exist + // in the prev table. + break + } + // prev.Largest.UserKey == cur.Smallest.UserKey, so we need to include prev + // in the compaction. + } + + for ; end < len(files); end++ { + cur := files[end-1] + next := files[end] + if pc.cmp(cur.Largest.UserKey, next.Smallest.UserKey) < 0 { + break + } + if cur.Largest.Trailer == InternalKeyRangeDeleteSentinel { + // The range deletion sentinel key is set for the largest key in a table + // when a range deletion tombstone straddles a table. It isn't necessary + // to include the next table in the compaction as cur.largest.UserKey + // does not actually exist in the table. + break + } + // cur.Largest.UserKey == next.Smallest.UserKey, so we need to include next + // in the compaction. + } + return files[start:end] +} + func newCompactionPicker( v *version, opts *Options, inProgressCompactions []compactionInfo, ) compactionPicker { @@ -69,9 +360,9 @@ func newCompactionPicker( return p } -// Information about a candidate compaction that has been identified by the -// compaction picker. -type pickedCompactionInfo struct { +// Information about a candidate compaction level that has been identified by +// the compaction picker. +type candidateLevelInfo struct { // The score of the level to be compacted. score float64 origScore float64 @@ -79,7 +370,7 @@ type pickedCompactionInfo struct { // The level to compact to. outputLevel int // The file in level that will be compacted. Additional files may be picked by the - // compaction. + // compaction, and a pickedCompaction created for the compaction. file int } @@ -213,7 +504,7 @@ func (p *compactionPickerByScore) initLevelMaxBytes(inProgressCompactions []comp } } for _, c := range inProgressCompactions { - if c.outputLevel == 0 { + if c.outputLevel == 0 || c.outputLevel == -1 { continue } if c.inputs[0].level == 0 && (firstNonEmptyLevel == -1 || c.outputLevel < firstNonEmptyLevel) { @@ -297,7 +588,9 @@ func calculateSizeAdjust(inProgressCompactions []compactionInfo) [numLevels]int6 if input.level != c.outputLevel { sizeAdjust[input.level] -= compensated - sizeAdjust[c.outputLevel] += real + if c.outputLevel != -1 { + sizeAdjust[c.outputLevel] += real + } } } } @@ -306,8 +599,8 @@ func calculateSizeAdjust(inProgressCompactions []compactionInfo) [numLevels]int6 func (p *compactionPickerByScore) calculateScores( inProgressCompactions []compactionInfo, -) [numLevels]pickedCompactionInfo { - var scores [numLevels]pickedCompactionInfo +) [numLevels]candidateLevelInfo { + var scores [numLevels]candidateLevelInfo for i := range scores { scores[i].level = i scores[i].outputLevel = i + 1 @@ -363,15 +656,16 @@ func (p *compactionPickerByScore) calculateScores( func (p *compactionPickerByScore) calculateL0Score( inProgressCompactions []compactionInfo, -) pickedCompactionInfo { - var info pickedCompactionInfo +) candidateLevelInfo { + var info candidateLevelInfo info.outputLevel = p.baseLevel if p.opts.Experimental.L0SublevelCompactions { // If L0Sublevels are present, we use the sublevel count as opposed to // the L0 file count to score this level. The base vs intra-L0 // compaction determination happens in pickAuto, not here. - info.score = float64(2 * p.vers.L0Sublevels.MaxDepthAfterOngoingCompactions()) + info.score = float64(2*p.vers.L0Sublevels.MaxDepthAfterOngoingCompactions()) / + float64(p.opts.L0CompactionThreshold) return info } @@ -523,7 +817,7 @@ func (p *compactionPickerByScore) pickFile(level, outputLevel int) int { // // If a score-based compaction cannot be found, pickAuto falls back to looking // for a forced compaction (identified by FileMetadata.MarkedForCompaction). -func (p *compactionPickerByScore) pickAuto(env compactionEnv) (c *compaction) { +func (p *compactionPickerByScore) pickAuto(env compactionEnv) (pc *pickedCompaction) { // Compaction concurrency is controlled by L0 read-amp. We allow one // additional compaction per L0CompactionConcurrency sublevels. Compaction // concurrency is tied to L0 sublevels as that signal is independent of the @@ -544,14 +838,14 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (c *compaction) { // TODO(peter): Either remove, or change this into an event sent to the // EventListener. - logCompaction := func(c *compaction) { + logCompaction := func(pc *pickedCompaction) { var buf bytes.Buffer for i := 0; i < numLevels; i++ { if i != 0 && i < p.baseLevel { continue } - var info *pickedCompactionInfo + var info *candidateLevelInfo for j := range scores { if scores[j].level == i { info = &scores[j] @@ -560,7 +854,7 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (c *compaction) { } marker := " " - if c.startLevel.level == info.level { + if pc.startLevel.level == info.level { marker = "*" } fmt.Fprintf(&buf, " %sL%d: %5.1f %5.1f %8s %8s", @@ -589,7 +883,7 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (c *compaction) { fmt.Fprintf(&buf, "\n") } p.opts.Logger.Infof("pickAuto: L%d->L%d\n%s", - c.startLevel.level, c.outputLevel.level, buf.String()) + pc.startLevel.level, pc.outputLevel.level, buf.String()) } // Check for a score-based compaction. "scores" has been sorted in order of @@ -605,16 +899,16 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (c *compaction) { } if info.level == 0 && p.opts.Experimental.L0SublevelCompactions { - c = pickL0(env, p.opts, p.vers, p.baseLevel) + pc = pickL0(env, p.opts, p.vers, p.baseLevel) // Fail-safe to protect against compacting the same sstable // concurrently. - if c != nil && !inputAlreadyCompacting(c) { - c.score = info.score + if pc != nil && !inputAlreadyCompacting(pc) { + pc.score = info.score // TODO(peter): remove if false { - logCompaction(c) + logCompaction(pc) } - return c + return pc } continue } @@ -624,15 +918,15 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (c *compaction) { continue } - c := pickAutoHelper(env, p.opts, p.vers, *info, p.baseLevel) + pc := pickAutoHelper(env, p.opts, p.vers, *info, p.baseLevel) // Fail-safe to protect against compacting the same sstable concurrently. - if c != nil && !inputAlreadyCompacting(c) { - c.score = info.score + if pc != nil && !inputAlreadyCompacting(pc) { + pc.score = info.score // TODO(peter): remove if false { - logCompaction(c) + logCompaction(pc) } - return c + return pc } } @@ -654,11 +948,11 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (c *compaction) { } info := &scores[i] info.file = file - c := pickAutoHelper(env, p.opts, p.vers, *info, p.baseLevel) + pc := pickAutoHelper(env, p.opts, p.vers, *info, p.baseLevel) // Fail-safe to protect against compacting the same sstable concurrently. - if c != nil && !inputAlreadyCompacting(c) { - c.score = info.score - return c + if pc != nil && !inputAlreadyCompacting(pc) { + pc.score = info.score + return pc } break } @@ -673,105 +967,88 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (c *compaction) { } func pickAutoHelper( - env compactionEnv, opts *Options, vers *version, cInfo pickedCompactionInfo, baseLevel int, -) (c *compaction) { + env compactionEnv, opts *Options, vers *version, cInfo candidateLevelInfo, baseLevel int, +) (pc *pickedCompaction) { if cInfo.outputLevel == 0 { return pickIntraL0(env, opts, vers) } - c = newCompaction(opts, vers, cInfo.level, baseLevel, env.bytesCompacted) - if c.outputLevel.level != cInfo.outputLevel { + pc = newPickedCompaction(opts, vers, cInfo.level, baseLevel) + if pc.outputLevel.level != cInfo.outputLevel { panic("pebble: compaction picked unexpected output level") } - c.startLevel.files = vers.Levels[c.startLevel.level][cInfo.file : cInfo.file+1] + pc.startLevel.files = vers.Levels[pc.startLevel.level][cInfo.file : cInfo.file+1] // Files in level 0 may overlap each other, so pick up all overlapping ones. - if c.startLevel.level == 0 { + if pc.startLevel.level == 0 { cmp := opts.Comparer.Compare - smallest, largest := manifest.KeyRange(cmp, c.startLevel.files, nil) - c.startLevel.files = vers.Overlaps(0, cmp, smallest.UserKey, largest.UserKey) - if len(c.startLevel.files) == 0 { + smallest, largest := manifest.KeyRange(cmp, pc.startLevel.files) + pc.startLevel.files = vers.Overlaps(0, cmp, smallest.UserKey, largest.UserKey).Collect() + if len(pc.startLevel.files) == 0 { panic("pebble: empty compaction") } } - c.setupInputs() - return c + pc.setupInputs() + return pc } // Helper method to pick compactions originating from L0. Uses information about // sublevels to generate a compaction. -func pickL0(env compactionEnv, opts *Options, vers *version, baseLevel int) (c *compaction) { +func pickL0(env compactionEnv, opts *Options, vers *version, baseLevel int) (pc *pickedCompaction) { // It is important to pass information about Lbase files to L0Sublevels // so it can pick a compaction that does not conflict with an Lbase => Lbase+1 // compaction. Without this, we observed reduced concurrency of L0=>Lbase // compactions, and increasing read amplification in L0. - lcf, err := vers.L0Sublevels.PickBaseCompaction( - opts.L0CompactionThreshold, vers.Levels[baseLevel]) + // + // TODO(bilal) Remove the minCompactionDepth parameter once fixing it at 1 + // has been shown to not cause a performance regression. + lcf, err := vers.L0Sublevels.PickBaseCompaction(1, vers.Levels[baseLevel]) if err != nil { opts.Logger.Infof("error when picking base compaction: %s", err) return } if lcf != nil { - // Manually build the compaction as opposed to calling - // pickAutoHelper. This is because L0Sublevels has already added - // any overlapping L0 SSTables that need to be added, and - // because compactions built by L0SSTables do not necessarily - // pick contiguous sequences of files in p.vers.Levels[0]. - c = newCompaction(opts, vers, 0, baseLevel, env.bytesCompacted) - c.lcf = lcf - if c.outputLevel.level != baseLevel { - opts.Logger.Fatalf("compaction picked unexpected output level: %d != %d", c.outputLevel.level, baseLevel) - } - c.startLevel.files = make([]*manifest.FileMetadata, 0, len(lcf.Files)) - for j := range lcf.FilesIncluded { - if lcf.FilesIncluded[j] { - c.startLevel.files = append(c.startLevel.files, vers.Levels[0][j]) - } - } - c.setupInputs() - if len(c.startLevel.files) == 0 { + pc = newPickedCompactionFromL0(lcf, opts, vers, baseLevel, true) + pc.setupInputs() + if len(pc.startLevel.files) == 0 { opts.Logger.Fatalf("empty compaction chosen") } - return c + return pc } // Couldn't choose a base compaction. Try choosing an intra-L0 - // compaction. + // compaction. Note that we pass in L0CompactionThreshold here as opposed to + // 1, since choosing a single sublevel intra-L0 compaction is + // counterproductive. lcf, err = vers.L0Sublevels.PickIntraL0Compaction(env.earliestUnflushedSeqNum, opts.L0CompactionThreshold) if err != nil { opts.Logger.Infof("error when picking intra-L0 compaction: %s", err) return } if lcf != nil { - c = newCompaction(opts, vers, 0, 0, env.bytesCompacted) - c.lcf = lcf - c.startLevel.files = make([]*manifest.FileMetadata, 0, len(lcf.Files)) - for j := range lcf.FilesIncluded { - if lcf.FilesIncluded[j] { - c.startLevel.files = append(c.startLevel.files, vers.Levels[0][j]) - } - } - if len(c.startLevel.files) == 0 { + pc = newPickedCompactionFromL0(lcf, opts, vers, 0, false) + pc.setupInputs() + if len(pc.startLevel.files) == 0 { opts.Logger.Fatalf("empty compaction chosen") - } else if len(c.startLevel.files) == 1 { + } + if len(pc.startLevel.files) == 1 { // A single-file intra-L0 compaction is unproductive. return nil } - c.smallest, c.largest = manifest.KeyRange(c.cmp, c.startLevel.files, nil) - c.setupInuseKeyRanges() + pc.smallest, pc.largest = manifest.KeyRange(pc.cmp, pc.startLevel.files) // Output only a single sstable for intra-L0 compactions. // Now that we have the ability to split flushes, we could conceivably // split the output of intra-L0 compactions too. This may be unnecessary // complexity -- the inputs to intra-L0 should be narrow in the key space // (unlike flushes), so writing a single sstable should be ok. - c.maxOutputFileSize = math.MaxUint64 - c.maxOverlapBytes = math.MaxUint64 - c.maxExpandedBytes = math.MaxUint64 + pc.maxOutputFileSize = math.MaxUint64 + pc.maxOverlapBytes = math.MaxUint64 + pc.maxExpandedBytes = math.MaxUint64 } - return c + return pc } -func pickIntraL0(env compactionEnv, opts *Options, vers *version) (c *compaction) { +func pickIntraL0(env compactionEnv, opts *Options, vers *version) (pc *pickedCompaction) { l0Files := vers.Levels[0] end := len(l0Files) for ; end >= 1; end-- { @@ -848,23 +1125,22 @@ func pickIntraL0(env compactionEnv, opts *Options, vers *version) (c *compaction return nil } - c = newCompaction(opts, vers, 0, 0, env.bytesCompacted) - c.startLevel.files = l0Files[begin:end] - c.smallest, c.largest = manifest.KeyRange(c.cmp, c.startLevel.files, nil) - c.setupInuseKeyRanges() + pc = newPickedCompaction(opts, vers, 0, 0) + pc.startLevel.files = l0Files[begin:end] + pc.smallest, pc.largest = manifest.KeyRange(pc.cmp, pc.startLevel.files) // Output only a single sstable for intra-L0 compactions. There is no current // benefit to outputting multiple tables, because other parts of the code // (i.e. iterators and comapction) expect L0 sstables to overlap and will // thus read all of the L0 sstables anyways, even if they are partitioned. - c.maxOutputFileSize = math.MaxUint64 - c.maxOverlapBytes = math.MaxUint64 - c.maxExpandedBytes = math.MaxUint64 - return c + pc.maxOutputFileSize = math.MaxUint64 + pc.maxOverlapBytes = math.MaxUint64 + pc.maxExpandedBytes = math.MaxUint64 + return pc } func (p *compactionPickerByScore) pickManual( env compactionEnv, manual *manualCompaction, -) (c *compaction, retryLater bool) { +) (pc *pickedCompaction, retryLater bool) { if p == nil { return nil, false } @@ -886,41 +1162,41 @@ func (p *compactionPickerByScore) pickManual( if conflictsWithInProgress(manual.level, outputLevel, env.inProgressCompactions) { return nil, true } - c = pickManualHelper(env, p.opts, manual, p.vers, p.baseLevel) - if c == nil { + pc = pickManualHelper(p.opts, manual, p.vers, p.baseLevel) + if pc == nil { return nil, false } - if c.outputLevel.level != outputLevel { + if pc.outputLevel.level != outputLevel { panic("pebble: compaction picked unexpected output level") } // Fail-safe to protect against compacting the same sstable concurrently. - if inputAlreadyCompacting(c) { + if inputAlreadyCompacting(pc) { return nil, true } - return c, false + return pc, false } func pickManualHelper( - env compactionEnv, opts *Options, manual *manualCompaction, vers *version, baseLevel int, -) (c *compaction) { - c = newCompaction(opts, vers, manual.level, baseLevel, env.bytesCompacted) - manual.outputLevel = c.outputLevel.level + opts *Options, manual *manualCompaction, vers *version, baseLevel int, +) (pc *pickedCompaction) { + pc = newPickedCompaction(opts, vers, manual.level, baseLevel) + manual.outputLevel = pc.outputLevel.level cmp := opts.Comparer.Compare - c.startLevel.files = vers.Overlaps(manual.level, cmp, manual.start.UserKey, manual.end.UserKey) - if len(c.startLevel.files) == 0 { + pc.startLevel.files = vers.Overlaps(manual.level, cmp, manual.start.UserKey, manual.end.UserKey).Collect() + if len(pc.startLevel.files) == 0 { // Nothing to do return nil } - c.setupInputs() - return c + pc.setupInputs() + return pc } func (p *compactionPickerByScore) forceBaseLevel1() { p.baseLevel = 1 } -func inputAlreadyCompacting(c *compaction) bool { - for _, cl := range c.inputs { +func inputAlreadyCompacting(pc *pickedCompaction) bool { + for _, cl := range pc.inputs { for _, f := range cl.files { if f.Compacting { return true diff --git a/github.com/cockroachdb/pebble/db.go b/github.com/cockroachdb/pebble/db.go index 0675af6bcc..19ae2f2bc6 100644 --- a/github.com/cockroachdb/pebble/db.go +++ b/github.com/cockroachdb/pebble/db.go @@ -292,6 +292,9 @@ type DB struct { flushing bool // The number of ongoing compactions. compactingCount int + // The list of deletion hints, suggesting ranges for delete-only + // compactions. + deletionHints []deleteCompactionHint // The list of manual compactions. The next manual compaction to perform // is at the start of the list. New entries are added to the end. manual []*manualCompaction @@ -929,7 +932,7 @@ func (d *DB) Compact( maxLevelWithFiles := 1 cur := d.mu.versions.currentVersion() for level := 0; level < numLevels; level++ { - if len(cur.Overlaps(level, d.cmp, start, end)) > 0 { + if !cur.Overlaps(level, d.cmp, start, end).Empty() { maxLevelWithFiles = level + 1 } } @@ -1136,19 +1139,15 @@ func (d *DB) EstimateDiskUsage(start, end []byte) (uint64, error) { var totalSize uint64 for level, files := range readState.current.Levels { + iter := manifest.SliceLevelIterator(files) if level > 0 { // We can only use `Overlaps` to restrict `files` at L1+ since at L0 it // expands the range iteratively until it has found a set of files that // do not overlap any other L0 files outside that set. - files = readState.current.Overlaps(level, d.opts.Comparer.Compare, start, end) + iter = readState.current.Overlaps(level, d.opts.Comparer.Compare, start, end) } - for fileIdx, file := range files { - if level > 0 && fileIdx > 0 && fileIdx < len(files)-1 { - // The files to the left and the right at least partially overlap - // with `file`, which means `file` is fully contained within the - // range specified by `[start, end]`. - totalSize += file.Size - } else if d.opts.Comparer.Compare(start, file.Smallest.UserKey) <= 0 && + for file := iter.First(); file != nil; file = iter.Next() { + if d.opts.Comparer.Compare(start, file.Smallest.UserKey) <= 0 && d.opts.Comparer.Compare(file.Largest.UserKey, end) <= 0 { // The range fully contains the file, so skip looking it up in // table cache/looking at its indexes, and add the full file size. @@ -1457,10 +1456,14 @@ func (d *DB) getEarliestUnflushedSeqNumLocked() uint64 { func (d *DB) getInProgressCompactionInfoLocked(finishing *compaction) (rv []compactionInfo) { for c := range d.mu.compact.inProgress { if len(c.flushing) == 0 && (finishing == nil || c != finishing) { - rv = append(rv, compactionInfo{ + info := compactionInfo{ inputs: c.inputs, - outputLevel: c.outputLevel.level, - }) + outputLevel: -1, + } + if c.outputLevel != nil { + info.outputLevel = c.outputLevel.level + } + rv = append(rv, info) } } return diff --git a/github.com/cockroachdb/pebble/event.go b/github.com/cockroachdb/pebble/event.go index 2032a018f3..46081de43c 100644 --- a/github.com/cockroachdb/pebble/event.go +++ b/github.com/cockroachdb/pebble/event.go @@ -5,14 +5,12 @@ package pebble import ( - "bytes" - "fmt" - "io" "strings" "time" "github.com/cockroachdb/pebble/internal/humanize" "github.com/cockroachdb/pebble/internal/manifest" + "github.com/cockroachdb/redact" ) // TableInfo exports the manifest.TableInfo type. @@ -44,10 +42,18 @@ type LevelInfo struct { } func (i LevelInfo) String() string { - return fmt.Sprintf("L%d [%s] (%s)", - i.Level, - formatFileNums(i.Tables), - humanize.Uint64(tablesTotalSize(i.Tables))) + return string(redact.Sprintfn(i.safeFormat)) +} + +func (i LevelInfo) safeFormat(w redact.SafePrinter) { + w.Printf("L%d [%s] (%s)", redact.Safe(i.Level), redact.Safe(formatFileNums(i.Tables)), + redact.Safe(humanize.Uint64(tablesTotalSize(i.Tables)))) +} + +func redactSprint(s redact.SafeFormatter) string { + return string(redact.Sprintfn(func(w redact.SafePrinter) { + s.SafeFormat(w, 's') + })) } // CompactionInfo contains the info for a compaction event. @@ -67,29 +73,40 @@ type CompactionInfo struct { } func (i CompactionInfo) String() string { + return redactSprint(i) +} + +// SafeFormat implements redact.SafeFormatter. +func (i CompactionInfo) SafeFormat(w redact.SafePrinter, _ rune) { if i.Err != nil { - return fmt.Sprintf("[JOB %d] compaction to L%d error: %s", - i.JobID, i.Output.Level, i.Err) + w.Printf("[JOB %d] compaction to L%d error: %s", + redact.Safe(i.JobID), redact.Safe(i.Output.Level), i.Err) + return + } + + if !i.Done { + w.Printf("[JOB %d] compacting ", redact.Safe(i.JobID)) + i.safeFormatInputs(w) + return } - var inputLevelsBuf bytes.Buffer + outputSize := tablesTotalSize(i.Output.Tables) + w.Printf("[JOB %d] compacted ", redact.Safe(i.JobID)) + i.safeFormatInputs(w) + w.Printf(" -> L%d [%s] (%s), in %.1fs, output rate %s/s", + redact.Safe(i.Output.Level), + redact.Safe(formatFileNums(i.Output.Tables)), + redact.Safe(humanize.Uint64(outputSize)), + redact.Safe(i.Duration.Seconds()), + redact.Safe(humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds())))) +} + +func (i CompactionInfo) safeFormatInputs(w redact.SafePrinter) { for j, levelInfo := range i.Input { if j > 0 { - io.WriteString(&inputLevelsBuf, " + ") + w.Printf(" + ") } - io.WriteString(&inputLevelsBuf, levelInfo.String()) - } - if !i.Done { - return fmt.Sprintf("[JOB %d] compacting %s", i.JobID, inputLevelsBuf.String()) + levelInfo.safeFormat(w) } - outputSize := tablesTotalSize(i.Output.Tables) - return fmt.Sprintf("[JOB %d] compacted %s -> L%d [%s] (%s), in %.1fs, output rate %s/s", - i.JobID, - inputLevelsBuf.String(), - i.Output.Level, - formatFileNums(i.Output.Tables), - humanize.Uint64(outputSize), - i.Duration.Seconds(), - humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))) } // FlushInfo contains the info for a flush event. @@ -109,25 +126,34 @@ type FlushInfo struct { } func (i FlushInfo) String() string { + return redactSprint(i) +} + +// SafeFormat implements redact.SafeFormatter. +func (i FlushInfo) SafeFormat(w redact.SafePrinter, _ rune) { if i.Err != nil { - return fmt.Sprintf("[JOB %d] flush error: %s", i.JobID, i.Err) + w.Printf("[JOB %d] flush error: %s", redact.Safe(i.JobID), i.Err) + return } - plural := "s" + plural := redact.SafeString("s") if i.Input == 1 { plural = "" } if !i.Done { - return fmt.Sprintf("[JOB %d] flushing %d memtable%s to L0", i.JobID, i.Input, plural) + w.Printf("[JOB %d] flushing %d memtable", redact.Safe(i.JobID), redact.Safe(i.Input)) + w.SafeString(plural) + w.Printf(" to L0") + return } outputSize := tablesTotalSize(i.Output) - return fmt.Sprintf("[JOB %d] flushed %d memtable%s to L0 [%s] (%s), in %.1fs, output rate %s/s", - i.JobID, i.Input, plural, - formatFileNums(i.Output), - humanize.Uint64(outputSize), - i.Duration.Seconds(), - humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))) + w.Printf("[JOB %d] flushed %d memtable%s to L0 [%s] (%s), in %.1fs, output rate %s/s", + redact.Safe(i.JobID), redact.Safe(i.Input), plural, + redact.Safe(formatFileNums(i.Output)), + redact.Safe(humanize.Uint64(outputSize)), + redact.Safe(i.Duration.Seconds()), + redact.Safe(humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds())))) } // ManifestCreateInfo contains info about a manifest creation event. @@ -141,10 +167,16 @@ type ManifestCreateInfo struct { } func (i ManifestCreateInfo) String() string { + return redactSprint(i) +} + +// SafeFormat implements redact.SafeFormatter. +func (i ManifestCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) { if i.Err != nil { - return fmt.Sprintf("[JOB %d] MANIFEST create error: %s", i.JobID, i.Err) + w.Printf("[JOB %d] MANIFEST create error: %s", redact.Safe(i.JobID), i.Err) + return } - return fmt.Sprintf("[JOB %d] MANIFEST created %s", i.JobID, i.FileNum) + w.Printf("[JOB %d] MANIFEST created %s", redact.Safe(i.JobID), redact.Safe(i.FileNum)) } // ManifestDeleteInfo contains the info for a Manifest deletion event. @@ -157,10 +189,16 @@ type ManifestDeleteInfo struct { } func (i ManifestDeleteInfo) String() string { + return redactSprint(i) +} + +// SafeFormat implements redact.SafeFormatter. +func (i ManifestDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) { if i.Err != nil { - return fmt.Sprintf("[JOB %d] MANIFEST delete error: %s", i.JobID, i.Err) + w.Printf("[JOB %d] MANIFEST delete error: %s", redact.Safe(i.JobID), i.Err) + return } - return fmt.Sprintf("[JOB %d] MANIFEST deleted %s", i.JobID, i.FileNum) + w.Printf("[JOB %d] MANIFEST deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum)) } // TableCreateInfo contains the info for a table creation event. @@ -174,7 +212,13 @@ type TableCreateInfo struct { } func (i TableCreateInfo) String() string { - return fmt.Sprintf("[JOB %d] %s: sstable created %s", i.JobID, i.Reason, i.FileNum) + return redactSprint(i) +} + +// SafeFormat implements redact.SafeFormatter. +func (i TableCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) { + w.Printf("[JOB %d] %s: sstable created %s", + redact.Safe(i.JobID), redact.Safe(i.Reason), redact.Safe(i.FileNum)) } // TableDeleteInfo contains the info for a table deletion event. @@ -186,11 +230,17 @@ type TableDeleteInfo struct { } func (i TableDeleteInfo) String() string { + return redactSprint(i) +} + +// SafeFormat implements redact.SafeFormatter. +func (i TableDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) { if i.Err != nil { - return fmt.Sprintf("[JOB %d] sstable delete error %s: %s", - i.JobID, i.FileNum, i.Err) + w.Printf("[JOB %d] sstable delete error %s: %s", + redact.Safe(i.JobID), redact.Safe(i.FileNum), i.Err) + return } - return fmt.Sprintf("[JOB %d] sstable deleted %s", i.JobID, i.FileNum) + w.Printf("[JOB %d] sstable deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum)) } // TableIngestInfo contains the info for a table ingestion event. @@ -208,20 +258,25 @@ type TableIngestInfo struct { } func (i TableIngestInfo) String() string { + return redactSprint(i) +} + +// SafeFormat implements redact.SafeFormatter. +func (i TableIngestInfo) SafeFormat(w redact.SafePrinter, _ rune) { if i.Err != nil { - return fmt.Sprintf("[JOB %d] ingest error: %s", i.JobID, i.Err) + w.Printf("[JOB %d] ingest error: %s", redact.Safe(i.JobID), i.Err) + return } - var buf bytes.Buffer - fmt.Fprintf(&buf, "[JOB %d] ingested", i.JobID) + w.Printf("[JOB %d] ingested", redact.Safe(i.JobID)) for j := range i.Tables { t := &i.Tables[j] if j > 0 { - fmt.Fprintf(&buf, ",") + w.Printf(",") } - fmt.Fprintf(&buf, " L%d:%s (%s)", t.Level, t.FileNum, humanize.Uint64(t.Size)) + w.Printf(" L%d:%s (%s)", redact.Safe(t.Level), redact.Safe(t.FileNum), + redact.Safe(humanize.Uint64(t.Size))) } - return buf.String() } // TableStatsInfo contains the info for a table stats loaded event. @@ -232,7 +287,12 @@ type TableStatsInfo struct { } func (i TableStatsInfo) String() string { - return fmt.Sprintf("[JOB %d] all initial table stats loaded", i.JobID) + return redactSprint(i) +} + +// SafeFormat implements redact.SafeFormatter. +func (i TableStatsInfo) SafeFormat(w redact.SafePrinter, _ rune) { + w.Printf("[JOB %d] all initial table stats loaded", redact.Safe(i.JobID)) } // WALCreateInfo contains info about a WAL creation event. @@ -249,16 +309,23 @@ type WALCreateInfo struct { } func (i WALCreateInfo) String() string { + return redactSprint(i) +} + +// SafeFormat implements redact.SafeFormatter. +func (i WALCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) { if i.Err != nil { - return fmt.Sprintf("[JOB %d] WAL create error: %s", i.JobID, i.Err) + w.Printf("[JOB %d] WAL create error: %s", redact.Safe(i.JobID), i.Err) + return } if i.RecycledFileNum == 0 { - return fmt.Sprintf("[JOB %d] WAL created %s", i.JobID, i.FileNum) + w.Printf("[JOB %d] WAL created %s", redact.Safe(i.JobID), redact.Safe(i.FileNum)) + return } - return fmt.Sprintf("[JOB %d] WAL created %s (recycled %s)", - i.JobID, i.FileNum, i.RecycledFileNum) + w.Printf("[JOB %d] WAL created %s (recycled %s)", + redact.Safe(i.JobID), redact.Safe(i.FileNum), redact.Safe(i.RecycledFileNum)) } // WALDeleteInfo contains the info for a WAL deletion event. @@ -271,10 +338,16 @@ type WALDeleteInfo struct { } func (i WALDeleteInfo) String() string { + return redactSprint(i) +} + +// SafeFormat implements redact.SafeFormatter. +func (i WALDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) { if i.Err != nil { - return fmt.Sprintf("[JOB %d] WAL delete error: %s", i.JobID, i.Err) + w.Printf("[JOB %d] WAL delete error: %s", redact.Safe(i.JobID), i.Err) + return } - return fmt.Sprintf("[JOB %d] WAL deleted %s", i.JobID, i.FileNum) + w.Printf("[JOB %d] WAL deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum)) } // WriteStallBeginInfo contains the info for a write stall begin event. @@ -283,7 +356,12 @@ type WriteStallBeginInfo struct { } func (i WriteStallBeginInfo) String() string { - return fmt.Sprintf("write stall beginning: %s", i.Reason) + return redactSprint(i) +} + +// SafeFormat implements redact.SafeFormatter. +func (i WriteStallBeginInfo) SafeFormat(w redact.SafePrinter, _ rune) { + w.Printf("write stall beginning: %s", redact.Safe(i.Reason)) } // EventListener contains a set of functions that will be invoked when various diff --git a/github.com/cockroachdb/pebble/go.mod b/github.com/cockroachdb/pebble/go.mod index baa617e878..e0ed903726 100644 --- a/github.com/cockroachdb/pebble/go.mod +++ b/github.com/cockroachdb/pebble/go.mod @@ -4,6 +4,7 @@ require ( github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 // indirect github.com/cockroachdb/errors v1.2.4 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect + github.com/cockroachdb/redact v0.0.0-20200622112456-cd282804bbd3 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd github.com/davecgh/go-spew v1.1.1 // indirect github.com/getsentry/raven-go v0.2.0 // indirect diff --git a/github.com/cockroachdb/pebble/go.sum b/github.com/cockroachdb/pebble/go.sum index d6d6767709..4f9c45333b 100644 --- a/github.com/cockroachdb/pebble/go.sum +++ b/github.com/cockroachdb/pebble/go.sum @@ -6,6 +6,8 @@ github.com/cockroachdb/errors v1.2.4 h1:Lap807SXTH5tri2TivECb/4abUkMZC9zRoLarvcK github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/cockroachdb/redact v0.0.0-20200622112456-cd282804bbd3 h1:2+dpIJzYMSbLi0587YXpi8tOJT52qCOI/1I0UNThc/I= +github.com/cockroachdb/redact v0.0.0-20200622112456-cd282804bbd3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/github.com/cockroachdb/pebble/ingest.go b/github.com/cockroachdb/pebble/ingest.go index 654e554189..b58b8f7ab8 100644 --- a/github.com/cockroachdb/pebble/ingest.go +++ b/github.com/cockroachdb/pebble/ingest.go @@ -118,7 +118,7 @@ func ingestLoad1( } } - iter, err := r.NewRangeDelIter() + iter, err := r.NewRawRangeDelIter() if err != nil { return nil, err } @@ -432,7 +432,7 @@ func ingestTargetLevel( } // Check boundary overlap. - if len(v.Overlaps(level, cmp, meta.Smallest.UserKey, meta.Largest.UserKey)) != 0 { + if !v.Overlaps(level, cmp, meta.Smallest.UserKey, meta.Largest.UserKey).Empty() { continue } @@ -445,7 +445,7 @@ func ingestTargetLevel( // negative (else we'd have returned earlier). overlaps := false for c := range compactions { - if level != c.outputLevel.level { + if c.outputLevel == nil || level != c.outputLevel.level { continue } if cmp(meta.Smallest.UserKey, c.largest.UserKey) <= 0 && diff --git a/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go b/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go index 0c260c6c3c..e448150695 100644 --- a/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go +++ b/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go @@ -15,12 +15,6 @@ import ( "github.com/cockroachdb/pebble/internal/base" ) -// TODO(bilal): -// - Integrate compaction picking logic with the rest of pebble. -// - Refactor away slicing and indexing for simplicity and stronger correctness -// guarantees, especially in extendCandidateToRectangle and -// Pick{Base,IntraL0}Compactions. - // Intervals are of the form [start, end) with no gap between intervals. Each // file overlaps perfectly with a sequence of intervals. This perfect overlap // occurs because the union of file boundary keys is used to pick intervals. @@ -32,6 +26,10 @@ import ( // - intervalKey{k, false} < intervalKey{k, true} // - k1 < k2 -> intervalKey{k1, _} < intervalKey{k2, _}. // +// Note that the file's largest key is exclusive if the internal key +// has a trailer matching the rangedel sentinel key. In this case, we set +// isLargest to false for end interval computation. +// // For example, consider three files with bounds [a,e], [b,g], and [e,j]. The // interval keys produced would be intervalKey{a, false}, intervalKey{b, false}, // intervalKey{e, false}, intervalKey{e, true}, intervalKey{g, true} and @@ -237,10 +235,13 @@ func NewL0Sublevels( s := &L0Sublevels{cmp: cmp, formatKey: formatKey} s.filesByAge = files keys := make([]intervalKey, 0, 2*len(files)) - for i := range s.filesByAge { - files[i].l0Index = i - keys = append(keys, intervalKey{key: files[i].Smallest.UserKey}) - keys = append(keys, intervalKey{key: files[i].Largest.UserKey, isLargest: true}) + for i, f := range s.filesByAge { + f.l0Index = i + keys = append(keys, intervalKey{key: f.Smallest.UserKey}) + keys = append(keys, intervalKey{ + key: f.Largest.UserKey, + isLargest: f.Largest.Trailer != base.InternalKeyRangeDeleteSentinel, + }) } keys = sortAndDedup(keys, cmp) // All interval indices reference s.orderedIntervals. @@ -265,7 +266,7 @@ func NewL0Sublevels( } f.maxIntervalIndex = sort.Search(len(keys), func(index int) bool { return intervalKeyCompare( - cmp, intervalKey{key: f.Largest.UserKey, isLargest: true}, keys[index]) <= 0 + cmp, intervalKey{key: f.Largest.UserKey, isLargest: f.Largest.Trailer != base.InternalKeyRangeDeleteSentinel}, keys[index]) <= 0 }) if f.maxIntervalIndex == len(keys) { return nil, errors.Errorf("expected sstable bound to be in interval keys: %s", f.Largest.UserKey) @@ -310,6 +311,9 @@ func NewL0Sublevels( } } var cumulativeBytes uint64 + // Multiply flushSplitMaxBytes by the number of sublevels. This prevents + // excessive flush splitting when the number of sublevels increases. + flushSplitMaxBytes *= int64(len(s.Levels)) for i := 0; i < len(s.orderedIntervals); i++ { interval := &s.orderedIntervals[i] if flushSplitMaxBytes > 0 && cumulativeBytes > uint64(flushSplitMaxBytes) && @@ -663,7 +667,7 @@ type L0CompactionFiles struct { filesAdded []*FileMetadata } -// Adds the specified file to the LCF. +// addFile adds the specified file to the LCF. func (l *L0CompactionFiles) addFile(f *FileMetadata) { if l.FilesIncluded[f.l0Index] { return @@ -1234,41 +1238,49 @@ func (s *L0Sublevels) intraL0CompactionUsingSeed( } // ExtendL0ForBaseCompactionTo extends the specified base compaction candidate -// L0CompactionFiles to cover all L0 files in the specified key interval, -// by calling extendCandidateToRectangle. +// L0CompactionFiles to optionally cover more files in L0 without "touching" +// any of the passed-in keys (i.e. the smallest/largest bounds are exclusive), +// as including any user keys for those internal keys +// could require choosing more files in LBase which is undesirable. Unbounded +// start/end keys are indicated by passing in the InvalidInternalKey. func (s *L0Sublevels) ExtendL0ForBaseCompactionTo( smallest, largest InternalKey, candidate *L0CompactionFiles, ) bool { - firstIntervalIndex := sort.Search(len(s.orderedIntervals), func(i int) bool { - // Need to start at >= smallest since if we widen too much we may miss - // an Lbase file that overlaps with an L0 file that will get picked in - // this widening, which would be bad. This interval will not start with - // an immediate successor key. - return s.cmp(smallest.UserKey, s.orderedIntervals[i].startKey.key) <= 0 - }) - // First interval that starts at or beyond the largest. This interval will not - // start with an immediate successor key. - var lastIntervalIndex int - if largest.Trailer == base.InternalKeyRangeDeleteSentinel { - // largest.UserKey is excluded. Do not expand L0 to include it. + firstIntervalIndex := 0 + lastIntervalIndex := len(s.orderedIntervals) - 1 + if smallest.Kind() != base.InternalKeyKindInvalid { + if smallest.Trailer == base.InternalKeyRangeDeleteSentinel { + // Starting at smallest.UserKey == interval.startKey is okay. + firstIntervalIndex = sort.Search(len(s.orderedIntervals), func(i int) bool { + return s.cmp(smallest.UserKey, s.orderedIntervals[i].startKey.key) <= 0 + }) + } else { + firstIntervalIndex = sort.Search(len(s.orderedIntervals), func(i int) bool { + // Need to start at >= smallest since if we widen too much we may miss + // an Lbase file that overlaps with an L0 file that will get picked in + // this widening, which would be bad. This interval will not start with + // an immediate successor key. + return s.cmp(smallest.UserKey, s.orderedIntervals[i].startKey.key) < 0 + }) + } + } + if largest.Kind() != base.InternalKeyKindInvalid { + // First interval that starts at or beyond the largest. This interval will not + // start with an immediate successor key. lastIntervalIndex = sort.Search(len(s.orderedIntervals), func(i int) bool { return s.cmp(largest.UserKey, s.orderedIntervals[i].startKey.key) <= 0 }) - } else { - lastIntervalIndex = sort.Search(len(s.orderedIntervals), func(i int) bool { - return s.cmp(largest.UserKey, s.orderedIntervals[i].startKey.key) < 0 - }) - } - // Right now, lastIntervalIndex has a startKey that extends beyond largest. - // The previous interval, by definition, has an end key higher than largest. - // Iterate back twice to get the last interval that's completely within - // [smallest, largest]. Except in the case where we went past the end of the - // list; in that case, the last interval to include is the very last - // interval in the list. - if lastIntervalIndex < len(s.orderedIntervals) { + // Right now, lastIntervalIndex has a startKey that extends beyond largest. + // The previous interval, by definition, has an end key higher than largest. + // Iterate back twice to get the last interval that's completely within + // (smallest, largest). Except in the case where we went past the end of the + // list; in that case, the last interval to include is the very last + // interval in the list. + if lastIntervalIndex < len(s.orderedIntervals) { + lastIntervalIndex-- + } lastIntervalIndex-- } - lastIntervalIndex-- if lastIntervalIndex < firstIntervalIndex { return false } @@ -1298,10 +1310,10 @@ func (s *L0Sublevels) ExtendL0ForBaseCompactionTo( // it's in the bounds, then add b-d, then a--d, and so on, to produce this: // // _____________ -// L0.3 |a--d g-j| _________ -// L0.2 | f--j| | r-t | -// L0.1 | b-d e---j| | | -// L0.0 |a--d f--j| l--o |p-----x| +// L0.3 |a--d g-j| +// L0.2 | f--j| r-t +// L0.1 | b-d e---j| +// L0.0 |a--d f--j| l--o p-----x // // Lbase a-------i m---------w // diff --git a/github.com/cockroachdb/pebble/internal/manifest/level_iterator.go b/github.com/cockroachdb/pebble/internal/manifest/level_iterator.go new file mode 100644 index 0000000000..2de2c526cc --- /dev/null +++ b/github.com/cockroachdb/pebble/internal/manifest/level_iterator.go @@ -0,0 +1,77 @@ +// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package manifest + +import "sort" + +// SliceLevelIterator constructs a LevelIterator over the provided slice. This +// function is expected to be a temporary adapter between interfaces. +// TODO(jackson): Revisit once the conversion of Version.Files to a btree is +// complete. +func SliceLevelIterator(files []*FileMetadata) LevelIterator { + return LevelIterator{files: files, cur: 0} +} + +// LevelIterator iterates over a set of files' metadata. +type LevelIterator struct { + files []*FileMetadata + cur int +} + +// Collect returns a slice of all the files in the iterator, irrespective of +// the current iterator position. The returned slice is owned by the iterator. +// This method is intended to be a temporary adatpter between interfaces, and +// callers should prefer using the iterator directory when possible. +// +// TODO(jackson): Revisit once the conversion of Version.Files to a btree is +// complete. +func (i LevelIterator) Collect() []*FileMetadata { + return i.files +} + +// SizeSum sums the size of all files in the iterator, irrespective of the +// current iterator position. +func (i LevelIterator) SizeSum() uint64 { + var sum uint64 + for _, f := range i.files { + sum += f.Size + } + return sum +} + +// Empty indicates whether there are remaining files in the iterator. +func (i LevelIterator) Empty() bool { + return i.cur >= len(i.files) +} + +// First seeks to the first file in the iterator and returns it. +func (i *LevelIterator) First() *FileMetadata { + i.cur = 0 + if i.cur >= len(i.files) { + return nil + } + return i.files[i.cur] +} + +// Next advances the iterator to the next file and returns it. +func (i *LevelIterator) Next() *FileMetadata { + i.cur++ + if i.cur >= len(i.files) { + return nil + } + return i.files[i.cur] +} + +// SeekGE seeks to the first file in the iterator's file set with a largest +// user key less than or equal to the provided user key. +func (i *LevelIterator) SeekGE(cmp Compare, userKey []byte) *FileMetadata { + i.cur = sort.Search(len(i.files), func(j int) bool { + return cmp(userKey, i.files[j].Largest.UserKey) <= 0 + }) + if i.cur >= len(i.files) { + return nil + } + return i.files[i.cur] +} diff --git a/github.com/cockroachdb/pebble/internal/manifest/version.go b/github.com/cockroachdb/pebble/internal/manifest/version.go index c67ecc3c52..079e2c4392 100644 --- a/github.com/cockroachdb/pebble/internal/manifest/version.go +++ b/github.com/cockroachdb/pebble/internal/manifest/version.go @@ -131,10 +131,10 @@ func (m *FileMetadata) lessSmallestKey(b *FileMetadata, cmp Compare) bool { // KeyRange returns the minimum smallest and maximum largest internalKey for // all the fileMetadata in f0 and f1. -func KeyRange(ucmp Compare, f0, f1 []*FileMetadata) (smallest, largest InternalKey) { +func KeyRange(ucmp Compare, fileSlices ...[]*FileMetadata) (smallest, largest InternalKey) { first := true - for _, f := range [2][]*FileMetadata{f0, f1} { - for _, meta := range f { + for _, files := range fileSlices { + for _, meta := range files { if first { first = false smallest, largest = meta.Smallest, meta.Largest @@ -379,12 +379,11 @@ func (v *Version) InitL0Sublevels( // searches among the files. If level is zero, Contains scans the entire // level. func (v *Version) Contains(level int, cmp Compare, m *FileMetadata) bool { - files := v.Levels[level] + iter := LevelIterator{files: v.Levels[level]} if level > 0 { - files = v.Overlaps(level, cmp, m.Smallest.UserKey, m.Largest.UserKey) + iter = v.Overlaps(level, cmp, m.Smallest.UserKey, m.Largest.UserKey) } - - for _, f := range files { + for f := iter.First(); f != nil; f = iter.Next() { if f == m { return true } @@ -400,11 +399,12 @@ func (v *Version) Contains(level int, cmp Compare, m *FileMetadata) bool { // and the computation is repeated until [start, end] stabilizes. // The returned files are a subsequence of the input files, i.e., the ordering // is not changed. -func (v *Version) Overlaps(level int, cmp Compare, start, end []byte) (ret []*FileMetadata) { +func (v *Version) Overlaps(level int, cmp Compare, start, end []byte) LevelIterator { if level == 0 { // Indices that have been selected as overlapping. selectedIndices := make([]bool, len(v.Levels[level])) numSelected := 0 + var iter LevelIterator for { restart := false for i, selected := range selectedIndices { @@ -442,25 +442,25 @@ func (v *Version) Overlaps(level int, cmp Compare, start, end []byte) (ret []*Fi } if !restart { - ret = make([]*FileMetadata, 0, numSelected) + iter.files = make([]*FileMetadata, 0, numSelected) for i, selected := range selectedIndices { if selected { - ret = append(ret, v.Levels[level][i]) + iter.files = append(iter.files, v.Levels[level][i]) } } break } // Continue looping to retry the files that were not selected. } - return + return iter } - files := v.Levels[level] - lower, upper := overlaps(files, cmp, start, end) - if lower >= upper { - return nil + var iter LevelIterator + lower, upper := overlaps(v.Levels[level], cmp, start, end) + if lower < upper { + iter.files = v.Levels[level][lower:upper] } - return files[lower:upper] + return iter } // CheckOrdering checks that the files are consistent with respect to diff --git a/github.com/cockroachdb/pebble/options.go b/github.com/cockroachdb/pebble/options.go index 57da2feb96..eca814dab2 100644 --- a/github.com/cockroachdb/pebble/options.go +++ b/github.com/cockroachdb/pebble/options.go @@ -289,16 +289,16 @@ type Options struct { // out of the experimental group, or made the non-adjustable default. These // options may change at any time, so do not rely on them. Experimental struct { - // FlushSplitBytes denotes the target number of bytes in each - // flush split interval (i.e. range between two flush split keys) in - // L0 sstables. When set to zero, only a single sstable is generated + // FlushSplitBytes denotes the target number of bytes per sublevel in + // each flush split interval (i.e. range between two flush split keys) + // in L0 sstables. When set to zero, only a single sstable is generated // by each flush. When set to a non-zero value, flushes are split at // points to meet L0's TargetFileSize, any grandparent-related overlap - // options, and at boundary keys of L0 flush split intervals (each of - // which are targeted to contain around FlushSplitBytes bytes across - // all L0 sstables). Splitting sstables during flush allows increased - // compaction flexibility and concurrency when those tables are - // compacted to lower levels. + // options, and at boundary keys of L0 flush split intervals (which are + // targeted to contain around FlushSplitBytes bytes in each sublevel + // between pairs of boundary keys). Splitting sstables during flush + // allows increased compaction flexibility and concurrency when those + // tables are compacted to lower levels. // // TODO(bilal): Experiment with this option to pick a good value. FlushSplitBytes int64 diff --git a/github.com/cockroachdb/pebble/sstable/reader.go b/github.com/cockroachdb/pebble/sstable/reader.go index 49529feb15..089d27f01e 100644 --- a/github.com/cockroachdb/pebble/sstable/reader.go +++ b/github.com/cockroachdb/pebble/sstable/reader.go @@ -1315,10 +1315,10 @@ func (r *Reader) NewCompactionIter(bytesIterated *uint64) (Iterator, error) { }, nil } -// NewRangeDelIter returns an internal iterator for the contents of the -// range-del block for the table. Returns nil if the table does not contain any -// range deletions. -func (r *Reader) NewRangeDelIter() (base.InternalIterator, error) { +// NewRawRangeDelIter returns an internal iterator for the contents of the +// range-del block for the table. Returns nil if the table does not contain +// any range deletions. +func (r *Reader) NewRawRangeDelIter() (base.InternalIterator, error) { if r.rangeDelBH.Length == 0 { return nil, nil } diff --git a/github.com/cockroachdb/pebble/table_cache.go b/github.com/cockroachdb/pebble/table_cache.go index 49c4003e6a..b47706241a 100644 --- a/github.com/cockroachdb/pebble/table_cache.go +++ b/github.com/cockroachdb/pebble/table_cache.go @@ -206,7 +206,7 @@ func (c *tableCacheShard) newIters( // NB: range-del iterator does not maintain a reference to the table, nor // does it need to read from it after creation. - rangeDelIter, err := v.reader.NewRangeDelIter() + rangeDelIter, err := v.reader.NewRawRangeDelIter() if err != nil { _ = iter.Close() return nil, nil, err diff --git a/github.com/cockroachdb/pebble/table_stats.go b/github.com/cockroachdb/pebble/table_stats.go index 78458448e2..9d85f5bb69 100644 --- a/github.com/cockroachdb/pebble/table_stats.go +++ b/github.com/cockroachdb/pebble/table_stats.go @@ -5,10 +5,12 @@ package pebble import ( + "math" "sync/atomic" "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/manifest" + "github.com/cockroachdb/pebble/internal/rangedel" "github.com/cockroachdb/pebble/sstable" ) @@ -97,12 +99,13 @@ func (d *DB) collectTableStats() { // Grab a read state to scan for tables. rs := d.loadReadState() var collected []collectedStats + var hints []deleteCompactionHint if len(pending) > 0 { - collected = d.loadNewFileStats(rs, pending) + collected, hints = d.loadNewFileStats(rs, pending) } else { var moreRemain bool var buf [maxTableStatsPerScan]collectedStats - collected, moreRemain = d.scanReadStateTableStats(rs, buf[:0]) + collected, hints, moreRemain = d.scanReadStateTableStats(rs, buf[:0]) loadedInitial = !moreRemain } rs.unref() @@ -125,6 +128,27 @@ func (d *DB) collectTableStats() { } d.mu.tableStats.cond.Broadcast() d.maybeCollectTableStats() + if len(hints) > 0 { + // Verify that all of the hint tombstones' files still exist in the + // current version. Otherwise, the tombstone itself may have been + // compacted into L6 and more recent keys may have had their sequence + // numbers zeroed. + // + // Note that it's possible that the tombstone file is being compacted + // presently. In that case, the file will be present in v. When the + // compaction finishes compacting the tombstone file, it will detect + // and clear the hint. + // + // See DB.maybeUpdateDeleteCompactionHints. + v := d.mu.versions.currentVersion() + keepHints := hints[:0] + for _, h := range hints { + if v.Contains(h.tombstoneLevel, d.cmp, h.tombstoneFile) { + keepHints = append(keepHints, h) + } + } + d.mu.compact.deletionHints = append(d.mu.compact.deletionHints, keepHints...) + } if maybeCompact { d.maybeScheduleCompaction() } @@ -135,7 +159,8 @@ type collectedStats struct { manifest.TableStats } -func (d *DB) loadNewFileStats(rs *readState, pending []manifest.NewFileEntry) []collectedStats { +func (d *DB) loadNewFileStats(rs *readState, pending []manifest.NewFileEntry) ([]collectedStats, []deleteCompactionHint) { + var hints []deleteCompactionHint collected := make([]collectedStats, 0, len(pending)) for _, nf := range pending { // A file's stats might have been populated by an earlier call to @@ -155,7 +180,7 @@ func (d *DB) loadNewFileStats(rs *readState, pending []manifest.NewFileEntry) [] continue } - stats, err := d.loadTableStats(rs.current, nf.Level, nf.Meta) + stats, newHints, err := d.loadTableStats(rs.current, nf.Level, nf.Meta) if err != nil { d.opts.EventListener.BackgroundError(err) continue @@ -167,17 +192,17 @@ func (d *DB) loadNewFileStats(rs *readState, pending []manifest.NewFileEntry) [] fileMetadata: nf.Meta, TableStats: stats, }) + hints = append(hints, newHints...) } - return collected + return collected, hints } // scanReadStateTableStats is run by an active stat collection job when there // are no pending new files, but there might be files that existed at Open for // which we haven't loaded table stats. -func (d *DB) scanReadStateTableStats( - rs *readState, fill []collectedStats, -) ([]collectedStats, bool) { +func (d *DB) scanReadStateTableStats(rs *readState, fill []collectedStats) ([]collectedStats, []deleteCompactionHint, bool) { moreRemain := false + var hints []deleteCompactionHint for l, ff := range rs.current.Levels { for _, f := range ff { // NB: We're not holding d.mu which protects f.Stats, but only the @@ -196,10 +221,10 @@ func (d *DB) scanReadStateTableStats( // work to do. if len(fill) == cap(fill) { moreRemain = true - return fill, moreRemain + return fill, hints, moreRemain } - stats, err := d.loadTableStats(rs.current, l, f) + stats, newHints, err := d.loadTableStats(rs.current, l, f) if err != nil { // Set `moreRemain` so we'll try again. moreRemain = true @@ -210,15 +235,17 @@ func (d *DB) scanReadStateTableStats( fileMetadata: f, TableStats: stats, }) + hints = append(hints, newHints...) } } - return fill, moreRemain + return fill, hints, moreRemain } func (d *DB) loadTableStats( v *version, level int, meta *fileMetadata, -) (manifest.TableStats, error) { +) (manifest.TableStats, []deleteCompactionHint, error) { var totalRangeDeletionEstimate uint64 + var compactionHints []deleteCompactionHint err := d.tableCache.withReader(meta, func(r *sstable.Reader) (err error) { if r.Properties.NumRangeDeletions == 0 { return nil @@ -229,33 +256,56 @@ func (d *DB) loadTableStats( // Also, merging abutting tombstones reduces the number of calls to // estimateSizeBeneath which is costly, and improves the accuracy of // our overall estimate. - rangeDelIter, err := r.NewRangeDelIter() + rangeDelIter, err := r.NewRawRangeDelIter() if err != nil { return err } defer rangeDelIter.Close() - err = foreachDefragmentedTombstone(rangeDelIter, d.cmp, func(startUserKey, endUserKey []byte) error { - estimate, err := d.estimateSizeBeneath(v, level, meta, startUserKey, endUserKey) - if err != nil { - return err - } - totalRangeDeletionEstimate += estimate - return nil - }) + // Truncate tombstones to the containing file's bounds if necessary. + // See docs/range_deletions.md for why this is necessary. + rangeDelIter = rangedel.Truncate(d.cmp, rangeDelIter, meta.Smallest.UserKey, meta.Largest.UserKey) + err = foreachDefragmentedTombstone(rangeDelIter, d.cmp, + func(startUserKey, endUserKey []byte, smallestSeqNum, largestSeqNum uint64) error { + estimate, hintSeqNum, err := d.estimateSizeBeneath(v, level, meta, startUserKey, endUserKey) + if err != nil { + return err + } + totalRangeDeletionEstimate += estimate + + // If any files were completely contained with the range, + // hintSeqNum is the smallest sequence number contained in any + // such file. + if hintSeqNum == math.MaxUint64 { + return nil + } + hint := deleteCompactionHint{ + start: make([]byte, len(startUserKey)), + end: make([]byte, len(endUserKey)), + tombstoneFile: meta, + tombstoneLevel: level, + tombstoneLargestSeqNum: largestSeqNum, + tombstoneSmallestSeqNum: smallestSeqNum, + fileSmallestSeqNum: hintSeqNum, + } + copy(hint.start, startUserKey) + copy(hint.end, endUserKey) + compactionHints = append(compactionHints, hint) + return nil + }) return err }) var stats manifest.TableStats if err != nil { - return stats, err + return stats, nil, err } stats.Valid = true stats.RangeDeletionsBytesEstimate = totalRangeDeletionEstimate - return stats, nil + return stats, compactionHints, nil } func (d *DB) estimateSizeBeneath( v *version, level int, meta *fileMetadata, start, end []byte, -) (uint64, error) { +) (estimate uint64, hintSeqNum uint64, err error) { // Find all files in lower levels that overlap with the deleted range. // // An overlapping file might be completely contained by the range @@ -264,21 +314,18 @@ func (d *DB) estimateSizeBeneath( // // Otherwise, estimating the range for the file requires // additional I/O to read the file's index blocks. - var estimate uint64 + hintSeqNum = math.MaxUint64 for l := level + 1; l < numLevels; l++ { - overlaps := v.Overlaps(l, d.cmp, start, end) - - for i, file := range overlaps { - if i > 0 && i < len(overlaps)-1 { - // The files to the left and the right at least - // partially overlap with the range tombstone, which - // means `file` is fully contained within the range. - estimate += file.Size - } else if d.cmp(start, file.Smallest.UserKey) <= 0 && + iter := v.Overlaps(l, d.cmp, start, end) + for file := iter.First(); file != nil; file = iter.Next() { + if d.cmp(start, file.Smallest.UserKey) <= 0 && d.cmp(file.Largest.UserKey, end) <= 0 { // The range fully contains the file, so skip looking it up in // table cache/looking at its indexes and add the full file size. estimate += file.Size + if hintSeqNum > file.SmallestSeqNum { + hintSeqNum = file.SmallestSeqNum + } } else if d.cmp(file.Smallest.UserKey, end) <= 0 && d.cmp(start, file.Largest.UserKey) <= 0 { var size uint64 err := d.tableCache.withReader(file, func(r *sstable.Reader) (err error) { @@ -286,27 +333,35 @@ func (d *DB) estimateSizeBeneath( return err }) if err != nil { - return 0, err + return 0, hintSeqNum, err } estimate += size } } } - return estimate, nil + return estimate, hintSeqNum, nil } func foreachDefragmentedTombstone( - rangeDelIter base.InternalIterator, cmp base.Compare, fn func([]byte, []byte) error, + rangeDelIter base.InternalIterator, cmp base.Compare, fn func([]byte, []byte, uint64, uint64) error, ) error { var startUserKey, endUserKey []byte + var smallestSeqNum, largestSeqNum uint64 var initialized bool for start, end := rangeDelIter.First(); start != nil; start, end = rangeDelIter.Next() { + // Range tombstones are fragmented such that any two tombstones // that share the same start key also share the same end key. // Multiple tombstones may exist at different sequence numbers. // If this tombstone starts or ends at the same point, it's a fragment // of the previous one. if cmp(startUserKey, start.UserKey) == 0 || cmp(endUserKey, end) == 0 { + if smallestSeqNum > start.SeqNum() { + smallestSeqNum = start.SeqNum() + } + if largestSeqNum < start.SeqNum() { + largestSeqNum = start.SeqNum() + } continue } @@ -314,6 +369,12 @@ func foreachDefragmentedTombstone( // tombstone ended, merge it and continue. if cmp(endUserKey, start.UserKey) == 0 { endUserKey = append(endUserKey[:0], end...) + if smallestSeqNum > start.SeqNum() { + smallestSeqNum = start.SeqNum() + } + if largestSeqNum < start.SeqNum() { + largestSeqNum = start.SeqNum() + } continue } @@ -322,18 +383,20 @@ func foreachDefragmentedTombstone( if !initialized { startUserKey = append(startUserKey[:0], start.UserKey...) endUserKey = append(endUserKey[:0], end...) + smallestSeqNum, largestSeqNum = start.SeqNum(), start.SeqNum() initialized = true continue } - if err := fn(startUserKey, endUserKey); err != nil { + if err := fn(startUserKey, endUserKey, smallestSeqNum, largestSeqNum); err != nil { return err } startUserKey = append(startUserKey[:0], start.UserKey...) endUserKey = append(endUserKey[:0], end...) + smallestSeqNum, largestSeqNum = start.SeqNum(), start.SeqNum() } if initialized { - if err := fn(startUserKey, endUserKey); err != nil { + if err := fn(startUserKey, endUserKey, smallestSeqNum, largestSeqNum); err != nil { return err } } diff --git a/github.com/cockroachdb/pebble/tool/find.go b/github.com/cockroachdb/pebble/tool/find.go index d0c43b79d9..76a1ea1528 100644 --- a/github.com/cockroachdb/pebble/tool/find.go +++ b/github.com/cockroachdb/pebble/tool/find.go @@ -423,7 +423,7 @@ func (f *findT) searchTables(searchKey []byte, refs []findRef) []findRef { // bit more work here to put them in a form that can be iterated in // parallel with the point records. rangeDelIter, err := func() (base.InternalIterator, error) { - iter, err := r.NewRangeDelIter() + iter, err := r.NewRawRangeDelIter() if err != nil { return nil, err } diff --git a/github.com/cockroachdb/pebble/tool/lsm.go b/github.com/cockroachdb/pebble/tool/lsm.go index 7f58d58ef8..10195747b4 100644 --- a/github.com/cockroachdb/pebble/tool/lsm.go +++ b/github.com/cockroachdb/pebble/tool/lsm.go @@ -36,7 +36,7 @@ type lsmVersionEdit struct { Added map[int][]base.FileNum `json:",omitempty"` // Map from level to files deleted from the level. Deleted map[int][]base.FileNum `json:",omitempty"` - // L0 sublevel structure for all files at this point. + // L0 sublevels for any files with changed sublevels so far. Sublevels map[base.FileNum]int `json:",omitempty"` } @@ -247,6 +247,12 @@ func (l *lsmT) buildEdits(edits []*manifest.VersionEdit) { edit.Sublevels = make(map[base.FileNum]int) for sublevel, files := range sublevels.Levels { for _, f := range files { + if len(l.state.Edits) > 0 { + lastEdit := l.state.Edits[len(l.state.Edits) - 1] + if sublevel2, ok := lastEdit.Sublevels[f.FileNum]; ok && sublevel == sublevel2 { + continue + } + } edit.Sublevels[f.FileNum] = sublevel } } diff --git a/github.com/cockroachdb/pebble/tool/lsm_data.go b/github.com/cockroachdb/pebble/tool/lsm_data.go index 5b7753f4e8..91e0c2a458 100644 --- a/github.com/cockroachdb/pebble/tool/lsm_data.go +++ b/github.com/cockroachdb/pebble/tool/lsm_data.go @@ -310,7 +310,10 @@ let version = { this.sublevels.push([]); } for (let file of this.levels[0]) { - let sublevel = data.Edits[this.index].Sublevels[file]; + let sublevel = null; + for (let i = index; i >= 0 && (sublevel === null || sublevel === undefined); i--) { + sublevel = data.Edits[i].Sublevels[file]; + } this.sublevels[sublevel].push(file); } diff --git a/github.com/cockroachdb/pebble/tool/sstable.go b/github.com/cockroachdb/pebble/tool/sstable.go index 7b178e5ebd..096a4f65e9 100644 --- a/github.com/cockroachdb/pebble/tool/sstable.go +++ b/github.com/cockroachdb/pebble/tool/sstable.go @@ -389,7 +389,7 @@ func (s *sstableT) runScan(cmd *cobra.Command, args []string) { // bit more work here to put them in a form that can be iterated in // parallel with the point records. rangeDelIter, err := func() (base.InternalIterator, error) { - iter, err := r.NewRangeDelIter() + iter, err := r.NewRawRangeDelIter() if err != nil { return nil, err } diff --git a/modules.txt b/modules.txt index cd46bb411f..d1bd277bd8 100644 --- a/modules.txt +++ b/modules.txt @@ -222,7 +222,7 @@ github.com/cockroachdb/gostdlib/x/tools/internal/span # github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f ## explicit github.com/cockroachdb/logtags -# github.com/cockroachdb/pebble v0.0.0-20200701140535-d845c8c2d870 +# github.com/cockroachdb/pebble v0.0.0-20200713154309-36c1b9c7a833 ## explicit github.com/cockroachdb/pebble github.com/cockroachdb/pebble/bloom From a591d5f1cf40aa524a0feaa9f92e91ed1c28ee21 Mon Sep 17 00:00:00 2001 From: Raphael 'kena' Poss Date: Tue, 14 Jul 2020 11:06:31 +0200 Subject: [PATCH 12/49] Bump datadriven --- github.com/cockroachdb/datadriven/datadriven.go | 4 ++-- modules.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/github.com/cockroachdb/datadriven/datadriven.go b/github.com/cockroachdb/datadriven/datadriven.go index b2e3fe81f3..d45cb3f614 100644 --- a/github.com/cockroachdb/datadriven/datadriven.go +++ b/github.com/cockroachdb/datadriven/datadriven.go @@ -46,7 +46,7 @@ var ( // Verbose returns true iff -datadriven-quiet was not passed. func Verbose() bool { - return !*quietLog + return testing.Verbose() && !*quietLog } // In CockroachDB we want to quiesce all the logs across all packages. @@ -366,7 +366,7 @@ func runDirective(t *testing.T, r *testDataReader, f func(*testing.T, *TestData) t.Logf("Failed to produce diff %v", err) } t.Fatalf("\n%s: %s\nexpected:\n%s\nfound:\n%s", d.Pos, d.Input, d.Expected, actual) - } else if !*quietLog { + } else if Verbose() { input := d.Input if input == "" { input = "" diff --git a/modules.txt b/modules.txt index d1bd277bd8..e829bddea5 100644 --- a/modules.txt +++ b/modules.txt @@ -177,7 +177,7 @@ github.com/cockroachdb/cockroach-go/crdb github.com/cockroachdb/crlfmt github.com/cockroachdb/crlfmt/internal/parser github.com/cockroachdb/crlfmt/internal/render -# github.com/cockroachdb/datadriven v0.0.0-20200708140406-a8a63ef9e03e +# github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5 ## explicit github.com/cockroachdb/datadriven # github.com/cockroachdb/errors v1.5.0 From 67d021529e202212b59d36177268bf131a79dbab Mon Sep 17 00:00:00 2001 From: Bilal Akhtar Date: Fri, 17 Jul 2020 09:53:04 -0400 Subject: [PATCH 13/49] Bump pebble to 073e15ee2ccf --- github.com/cockroachdb/pebble/checkpoint.go | 6 +- github.com/cockroachdb/pebble/compaction.go | 36 ++- .../cockroachdb/pebble/compaction_picker.go | 266 +++++++++--------- github.com/cockroachdb/pebble/db.go | 14 +- github.com/cockroachdb/pebble/event.go | 45 ++- github.com/cockroachdb/pebble/get_iter.go | 2 +- .../internal/manifest/level_iterator.go | 77 ----- .../internal/manifest/level_metadata.go | 203 +++++++++++++ .../pebble/internal/manifest/version.go | 141 +++++----- .../pebble/internal/manifest/version_edit.go | 2 +- .../cockroachdb/pebble/level_checker.go | 12 +- .../cockroachdb/pebble/sstable/reader.go | 117 +++++++- github.com/cockroachdb/pebble/table_cache.go | 7 +- github.com/cockroachdb/pebble/table_stats.go | 7 +- github.com/cockroachdb/pebble/tool/db.go | 3 +- .../cockroachdb/pebble/tool/manifest.go | 6 +- .../cockroachdb/pebble/vfs/fadvise_generic.go | 4 + .../cockroachdb/pebble/vfs/fadvise_linux.go | 5 + .../cockroachdb/pebble/vfs/prefetch_linux.go | 4 +- github.com/cockroachdb/pebble/vfs/vfs.go | 26 +- modules.txt | 2 +- 21 files changed, 626 insertions(+), 359 deletions(-) delete mode 100644 github.com/cockroachdb/pebble/internal/manifest/level_iterator.go create mode 100644 github.com/cockroachdb/pebble/internal/manifest/level_metadata.go diff --git a/github.com/cockroachdb/pebble/checkpoint.go b/github.com/cockroachdb/pebble/checkpoint.go index 605de15c86..b276194f4e 100644 --- a/github.com/cockroachdb/pebble/checkpoint.go +++ b/github.com/cockroachdb/pebble/checkpoint.go @@ -119,9 +119,9 @@ func (d *DB) Checkpoint(destDir string) (err error) { // Link or copy the sstables. for l := range current.Levels { - level := current.Levels[l] - for i := range level { - srcPath := base.MakeFilename(fs, d.dirname, fileTypeTable, level[i].FileNum) + iter := current.Levels[l].Iter() + for f := iter.First(); f != nil; f = iter.Next() { + srcPath := base.MakeFilename(fs, d.dirname, fileTypeTable, f.FileNum) destPath := fs.PathJoin(destDir, fs.PathBase(srcPath)) if err := vfs.LinkOrCopy(fs, srcPath, destPath); err != nil { return err diff --git a/github.com/cockroachdb/pebble/compaction.go b/github.com/cockroachdb/pebble/compaction.go index 3068b781bc..5ddbf5e18b 100644 --- a/github.com/cockroachdb/pebble/compaction.go +++ b/github.com/cockroachdb/pebble/compaction.go @@ -147,7 +147,7 @@ type compaction struct { // grandparents are the tables in level+2 that overlap with the files being // compacted. Used to determine output table boundaries. Do not assume that the actual files // in the grandparent when this compaction finishes will be the same. - grandparents manifest.LevelIterator + grandparents manifest.LevelSlice // Boundaries at which flushes to L0 should be split. Determined by // L0Sublevels. If nil, flushes aren't split. @@ -210,19 +210,26 @@ func newCompaction( largest: pc.largest, logger: opts.Logger, version: pc.version, - inputs: pc.inputs, maxOutputFileSize: pc.maxOutputFileSize, maxOverlapBytes: pc.maxOverlapBytes, maxExpandedBytes: pc.maxOverlapBytes, atomicBytesIterated: bytesCompacted, } + for _, in := range pc.inputs { + c.inputs = append(c.inputs, compactionLevel{ + level: in.level, + files: in.files.Collect(), + }) + } + c.startLevel = &c.inputs[0] c.outputLevel = &c.inputs[1] // Compute the set of outputLevel+1 files that overlap this compaction (these // are the grandparent sstables). if c.outputLevel.level+1 < numLevels { - c.grandparents = c.version.Overlaps(c.outputLevel.level+1, c.cmp, c.smallest.UserKey, c.largest.UserKey) + c.grandparents = c.version.Overlaps(c.outputLevel.level+1, c.cmp, + c.smallest.UserKey, c.largest.UserKey) } c.setupInuseKeyRanges() @@ -250,11 +257,11 @@ func newDeleteOnlyCompaction( } // Set c.smallest, c.largest. - levelFiles := make([][]*fileMetadata, 0, len(inputs)) + files := make([]manifest.LevelIterator, 0, len(inputs)) for _, in := range inputs { - levelFiles = append(levelFiles, in.files) + files = append(files, manifest.NewLevelSlice(in.files).Iter()) } - c.smallest, c.largest = manifest.KeyRange(opts.Comparer.Compare, levelFiles...) + c.smallest, c.largest = manifest.KeyRange(opts.Comparer.Compare, files...) return c } @@ -331,7 +338,8 @@ func newFlush( c.maxOutputFileSize = uint64(opts.Level(0).TargetFileSize) c.maxOverlapBytes = maxGrandparentOverlapBytes(opts, 0) c.maxExpandedBytes = expandedCompactionByteSizeLimit(opts, 0) - c.grandparents = c.version.Overlaps(baseLevel, c.cmp, c.smallest.UserKey, c.largest.UserKey) + c.grandparents = c.version.Overlaps(baseLevel, c.cmp, + c.smallest.UserKey, c.largest.UserKey) } c.setupInuseKeyRanges() @@ -350,7 +358,7 @@ func (c *compaction) setupInuseKeyRanges() { // levels. var input []userKeyRange for ; level < numLevels; level++ { - iter := c.version.Overlaps(level, c.cmp, c.smallest.UserKey, c.largest.UserKey) + iter := c.version.Overlaps(level, c.cmp, c.smallest.UserKey, c.largest.UserKey).Iter() for m := iter.First(); m != nil; m = iter.Next() { input = append(input, userKeyRange{m.Smallest.UserKey, m.Largest.UserKey}) } @@ -400,7 +408,7 @@ func (c *compaction) setupInuseKeyRanges() { // user-key. Perhaps this isn't a problem if the compaction picking heuristics // always pick the right (older) sibling for compaction first. func (c *compaction) findGrandparentLimit(start []byte) []byte { - iter := c.grandparents + iter := c.grandparents.Iter() var overlappedBytes uint64 for f := iter.SeekGE(c.cmp, start); f != nil; f = iter.Next() { overlappedBytes += f.Size @@ -584,11 +592,15 @@ func (c *compaction) newInputIter(newIters tableNewIters) (_ internalIterator, r // Check that the LSM ordering invariants are ok in order to prevent // generating corrupted sstables due to a violation of those invariants. if c.startLevel.level >= 0 { - if err := manifest.CheckOrdering(c.cmp, c.formatKey, manifest.Level(c.startLevel.level), c.startLevel.files); err != nil { + err := manifest.CheckOrdering(c.cmp, c.formatKey, manifest.Level(c.startLevel.level), + manifest.NewLevelSlice(c.startLevel.files).Iter()) + if err != nil { c.logger.Fatalf("%s", err) } } - if err := manifest.CheckOrdering(c.cmp, c.formatKey, manifest.Level(c.outputLevel.level), c.outputLevel.files); err != nil { + err := manifest.CheckOrdering(c.cmp, c.formatKey, manifest.Level(c.outputLevel.level), + manifest.NewLevelSlice(c.outputLevel.files).Iter()) + if err != nil { c.logger.Fatalf("%s", err) } @@ -1291,7 +1303,7 @@ func checkDeleteCompactionHints( // The hint h will be resolved and dropped, regardless of whether // there are any tables that can be deleted. for l := h.tombstoneLevel + 1; l < numLevels; l++ { - overlaps := v.Overlaps(l, cmp, h.start, h.end) + overlaps := v.Overlaps(l, cmp, h.start, h.end).Iter() for m := overlaps.First(); m != nil; m = overlaps.Next() { if m.Compacting || !h.canDelete(cmp, m, snapshots) || files[m] { continue diff --git a/github.com/cockroachdb/pebble/compaction_picker.go b/github.com/cockroachdb/pebble/compaction_picker.go index c65bea728e..873ba34979 100644 --- a/github.com/cockroachdb/pebble/compaction_picker.go +++ b/github.com/cockroachdb/pebble/compaction_picker.go @@ -9,7 +9,6 @@ import ( "fmt" "math" "sort" - "unsafe" "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/humanize" @@ -40,7 +39,7 @@ type compactionPicker interface { // Information about in-progress compactions provided to the compaction picker. These are used to // constrain the new compactions that will be picked. type compactionInfo struct { - inputs []compactionLevel + inputs []compactionInput outputLevel int } @@ -59,6 +58,13 @@ func (s sortCompactionLevelsDecreasingScore) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +// TODO(jackson): Consolidate this type with the compactionLevel type again +// once compactions are refactored to use LevelIterators too. +type compactionInput struct { + level int + files manifest.LevelSlice +} + // pickedCompaction contains information about a compaction that has already // been chosen, and is being constructed. Compaction construction info lives in // this struct, and is copied over into the compaction struct when that's @@ -71,13 +77,13 @@ type pickedCompaction struct { // startLevel is the level that is being compacted. Inputs from startLevel // and outputLevel will be merged to produce a set of outputLevel files. - startLevel *compactionLevel + startLevel *compactionInput // outputLevel is the level that files are being produced in. outputLevel is // equal to startLevel+1 except when startLevel is 0 in which case it is // equal to compactionPicker.baseLevel(). - outputLevel *compactionLevel + outputLevel *compactionInput - inputs []compactionLevel + inputs []compactionInput // L0-specific compaction info. Set to a non-nil value for all compactions // where startLevel == 0 that were generated by L0Sublevels. @@ -124,7 +130,7 @@ func newPickedCompaction( pc := &pickedCompaction{ cmp: opts.Comparer.Compare, version: cur, - inputs: []compactionLevel{{level: startLevel}, {level: outputLevel}}, + inputs: []compactionInput{{level: startLevel}, {level: outputLevel}}, maxOutputFileSize: uint64(opts.Level(adjustedOutputLevel).TargetFileSize), maxOverlapBytes: maxGrandparentOverlapBytes(opts, adjustedOutputLevel), maxExpandedBytes: expandedCompactionByteSizeLimit(opts, adjustedOutputLevel), @@ -146,27 +152,29 @@ func newPickedCompactionFromL0(lcf *manifest.L0CompactionFiles, opts *Options, v // any overlapping L0 SSTables that need to be added, and // because compactions built by L0SSTables do not necessarily // pick contiguous sequences of files in pc.version.Levels[0]. - pc.startLevel.files = make([]*manifest.FileMetadata, 0, len(lcf.Files)) + files := make([]*manifest.FileMetadata, 0, len(lcf.Files)) for j := range lcf.FilesIncluded { if lcf.FilesIncluded[j] { - pc.startLevel.files = append(pc.startLevel.files, vers.Levels[0][j]) + files = append(files, vers.Levels[0][j]) } } + pc.startLevel.files = manifest.NewLevelSlice(files) return pc } func (pc *pickedCompaction) setupInputs() { // Expand the initial inputs to a clean cut. - pc.startLevel.files = pc.expandInputs(pc.startLevel.level, pc.startLevel.files) - pc.smallest, pc.largest = manifest.KeyRange(pc.cmp, pc.startLevel.files) + pc.startLevel.files = expandInputs(pc.cmp, pc.startLevel.files) + pc.smallest, pc.largest = manifest.KeyRange(pc.cmp, pc.startLevel.files.Iter()) // Determine the sstables in the output level which overlap with the input // sstables, and then expand those tables to a clean cut. No need to do // this for intra-L0 compactions; outputLevel.files is left empty for those. if pc.startLevel.level != pc.outputLevel.level { - pc.outputLevel.files = pc.version.Overlaps(pc.outputLevel.level, pc.cmp, pc.smallest.UserKey, pc.largest.UserKey).Collect() - pc.outputLevel.files = pc.expandInputs(pc.outputLevel.level, pc.outputLevel.files) - pc.smallest, pc.largest = manifest.KeyRange(pc.cmp, pc.startLevel.files, pc.outputLevel.files) + pc.outputLevel.files = pc.version.Overlaps(pc.outputLevel.level, pc.cmp, pc.smallest.UserKey, pc.largest.UserKey) + pc.outputLevel.files = expandInputs(pc.cmp, pc.outputLevel.files) + pc.smallest, pc.largest = manifest.KeyRange(pc.cmp, + pc.startLevel.files.Iter(), pc.outputLevel.files.Iter()) } // Grow the sstables in pc.startLevel.level as long as it doesn't affect the number @@ -212,21 +220,23 @@ func (pc *pickedCompaction) setupInputs() { } oldLcf := *pc.lcf if pc.version.L0Sublevels.ExtendL0ForBaseCompactionTo(smallestBaseKey, largestBaseKey, pc.lcf) { - newStartLevelFiles := make([]*fileMetadata, 0, len(pc.startLevel.files)) + var newStartLevelFiles []*fileMetadata for j := range pc.lcf.FilesIncluded { if pc.lcf.FilesIncluded[j] { newStartLevelFiles = append(newStartLevelFiles, pc.version.Levels[0][j]) } } - if totalSize(newStartLevelFiles)+totalSize(pc.outputLevel.files) < pc.maxExpandedBytes { - pc.startLevel.files = newStartLevelFiles - pc.smallest, pc.largest = manifest.KeyRange(pc.cmp, pc.startLevel.files, pc.outputLevel.files) + if totalSize(newStartLevelFiles)+pc.outputLevel.files.SizeSum() < pc.maxExpandedBytes { + pc.startLevel.files = manifest.NewLevelSlice(newStartLevelFiles) + pc.smallest, pc.largest = manifest.KeyRange(pc.cmp, + pc.startLevel.files.Iter(), pc.outputLevel.files.Iter()) } else { *pc.lcf = oldLcf } } } else if pc.grow(pc.smallest, pc.largest) { - pc.smallest, pc.largest = manifest.KeyRange(pc.cmp, pc.startLevel.files, pc.outputLevel.files) + pc.smallest, pc.largest = manifest.KeyRange(pc.cmp, + pc.startLevel.files.Iter(), pc.outputLevel.files.Iter()) } } @@ -234,21 +244,21 @@ func (pc *pickedCompaction) setupInputs() { // c.level+1 files in the compaction, and returns whether the inputs grew. sm // and la are the smallest and largest InternalKeys in all of the inputs. func (pc *pickedCompaction) grow(sm, la InternalKey) bool { - if len(pc.outputLevel.files) == 0 { + if pc.outputLevel.files.Empty() { return false } - grow0 := pc.version.Overlaps(pc.startLevel.level, pc.cmp, sm.UserKey, la.UserKey).Collect() - grow0 = pc.expandInputs(pc.startLevel.level, grow0) - if len(grow0) <= len(pc.startLevel.files) { + grow0 := pc.version.Overlaps(pc.startLevel.level, pc.cmp, sm.UserKey, la.UserKey) + grow0 = expandInputs(pc.cmp, grow0) + if grow0.Len() <= pc.startLevel.files.Len() { return false } - if totalSize(grow0)+totalSize(pc.outputLevel.files) >= pc.maxExpandedBytes { + if grow0.SizeSum()+pc.outputLevel.files.SizeSum() >= pc.maxExpandedBytes { return false } - sm1, la1 := manifest.KeyRange(pc.cmp, grow0) - grow1 := pc.version.Overlaps(pc.outputLevel.level, pc.cmp, sm1.UserKey, la1.UserKey).Collect() - grow1 = pc.expandInputs(pc.outputLevel.level, grow1) - if len(grow1) != len(pc.outputLevel.files) { + sm1, la1 := manifest.KeyRange(pc.cmp, grow0.Iter()) + grow1 := pc.version.Overlaps(pc.outputLevel.level, pc.cmp, sm1.UserKey, la1.UserKey) + grow1 = expandInputs(pc.cmp, grow1) + if grow1.Len() != pc.outputLevel.files.Len() { return false } pc.startLevel.files = grow0 @@ -288,65 +298,52 @@ func (pc *pickedCompaction) grow(sm, la InternalKey) bool { // tombstones will be truncated at "b" (the largest key in its atomic // compaction unit). In the scenario here, that could result in b#1 becoming // visible when it should be deleted. -func (pc *pickedCompaction) expandInputs(level int, inputs []*fileMetadata) []*fileMetadata { - if level == 0 { - // We already called version.Overlaps for L0 and that call guarantees that - // we get a "clean cut". - return inputs - } - if len(inputs) == 0 { +func expandInputs(cmp Compare, inputs manifest.LevelSlice) manifest.LevelSlice { + // NB: Inputs for L0 can't be expanded and *version.Overlaps guarantees + // that we get a 'clean cut.' For L0, Overlaps will return a slice without + // access to the rest of the L0 files, so it's OK to try to reslice. + if inputs.Empty() { // Nothing to expand. return inputs } - files := pc.version.Levels[level] - // Pointer arithmetic to figure out the index if inputs[0] with - // files[0]. This requires that the inputs slice is a sub-slice of - // files. This is true for non-L0 files returned from version.overlaps. - if uintptr(unsafe.Pointer(&inputs[0])) < uintptr(unsafe.Pointer(&files[0])) { - panic("pebble: invalid input slice") - } - start := int((uintptr(unsafe.Pointer(&inputs[0])) - - uintptr(unsafe.Pointer(&files[0]))) / unsafe.Sizeof(inputs[0])) - if start >= len(files) { - panic("pebble: invalid input slice") - } - end := start + len(inputs) - for ; start > 0; start-- { - cur := files[start] - prev := files[start-1] - if pc.cmp(prev.Largest.UserKey, cur.Smallest.UserKey) < 0 { - break - } - if prev.Largest.Trailer == InternalKeyRangeDeleteSentinel { - // The range deletion sentinel key is set for the largest key in a - // table when a range deletion tombstone straddles a table. It - // isn't necessary to include the prev table in the atomic - // compaction unit as prev.largest.UserKey does not actually exist - // in the prev table. - break + return inputs.Reslice(func(start, end *manifest.LevelIterator) { + iter := start.Clone() + iter.Prev() + for cur, prev := start.Current(), iter.Current(); prev != nil; cur, prev = start.Prev(), iter.Prev() { + if cmp(prev.Largest.UserKey, cur.Smallest.UserKey) < 0 { + break + } + if prev.Largest.Trailer == InternalKeyRangeDeleteSentinel { + // The range deletion sentinel key is set for the largest key in a + // table when a range deletion tombstone straddles a table. It + // isn't necessary to include the prev table in the atomic + // compaction unit as prev.largest.UserKey does not actually exist + // in the prev table. + break + } + // prev.Largest.UserKey == cur.Smallest.UserKey, so we need to + // include prev in the compaction. } - // prev.Largest.UserKey == cur.Smallest.UserKey, so we need to include prev - // in the compaction. - } - for ; end < len(files); end++ { - cur := files[end-1] - next := files[end] - if pc.cmp(cur.Largest.UserKey, next.Smallest.UserKey) < 0 { - break - } - if cur.Largest.Trailer == InternalKeyRangeDeleteSentinel { - // The range deletion sentinel key is set for the largest key in a table - // when a range deletion tombstone straddles a table. It isn't necessary - // to include the next table in the compaction as cur.largest.UserKey - // does not actually exist in the table. - break + iter = end.Clone() + iter.Next() + for cur, next := end.Current(), iter.Current(); next != nil; cur, next = end.Next(), iter.Next() { + if cmp(cur.Largest.UserKey, next.Smallest.UserKey) < 0 { + break + } + if cur.Largest.Trailer == InternalKeyRangeDeleteSentinel { + // The range deletion sentinel key is set for the largest key + // in a table when a range deletion tombstone straddles a + // table. It isn't necessary to include the next table in the + // compaction as PeekPrev().Largest.UserKey does not actually + // exist in the table. + break + } + // cur.Largest.UserKey == next.Smallest.UserKey, so we need to + // include next in the compaction. } - // cur.Largest.UserKey == next.Smallest.UserKey, so we need to include next - // in the compaction. - } - return files[start:end] + }) } func newCompactionPicker( @@ -369,9 +366,10 @@ type candidateLevelInfo struct { level int // The level to compact to. outputLevel int - // The file in level that will be compacted. Additional files may be picked by the - // compaction, and a pickedCompaction created for the compaction. - file int + // The file in level that will be compacted. Additional files may be + // picked by the compaction, and a pickedCompaction created for the + // compaction. + file manifest.LevelSlice } // compensatedSize returns f's file size, inflated according to compaction @@ -384,9 +382,9 @@ func compensatedSize(f *fileMetadata) uint64 { return sz } -func totalCompensatedSize(files []*fileMetadata) uint64 { +func totalCompensatedSize(iter manifest.LevelIterator) uint64 { var sz uint64 - for _, f := range files { + for f := iter.First(); f != nil; f = iter.Next() { sz += compensatedSize(f) } return sz @@ -583,8 +581,8 @@ func calculateSizeAdjust(inProgressCompactions []compactionInfo) [numLevels]int6 c := &inProgressCompactions[i] for _, input := range c.inputs { - real := int64(totalSize(input.files)) - compensated := int64(totalCompensatedSize(input.files)) + real := int64(input.files.SizeSum()) + compensated := int64(totalCompensatedSize(input.files.Iter())) if input.level != c.outputLevel { sizeAdjust[input.level] -= compensated @@ -609,7 +607,7 @@ func (p *compactionPickerByScore) calculateScores( sizeAdjust := calculateSizeAdjust(inProgressCompactions) for level := 1; level < numLevels; level++ { - levelSize := int64(totalCompensatedSize(p.vers.Levels[level])) + sizeAdjust[level] + levelSize := int64(totalCompensatedSize(p.vers.Levels[level].Iter())) + sizeAdjust[level] scores[level].score = float64(levelSize) / float64(p.levelMaxBytes[level]) scores[level].origScore = scores[level].score } @@ -741,7 +739,7 @@ func (p *compactionPickerByScore) calculateL0Score( return info } -func (p *compactionPickerByScore) pickFile(level, outputLevel int) int { +func (p *compactionPickerByScore) pickFile(level, outputLevel int) manifest.LevelSlice { // Select the file within the level to compact. We want to minimize write // amplification, but also ensure that deletes are propagated to the // bottom level in a timely fashion so as to reclaim disk space. A table's @@ -765,31 +763,34 @@ func (p *compactionPickerByScore) pickFile(level, outputLevel int) int { // an in-progress compaction. cmp := p.opts.Comparer.Compare - outputLevelFiles := p.vers.Levels[outputLevel] + startIter := p.vers.Levels[level].Iter() + outputIter := p.vers.Levels[outputLevel].Iter() - file := -1 + var file manifest.LevelSlice smallestRatio := uint64(math.MaxUint64) - for i, f := range p.vers.Levels[level] { + outputFile := outputIter.First() + + for f := startIter.First(); f != nil; f = startIter.Next() { var overlappingBytes uint64 // Trim any output-level files smaller than f. - for len(outputLevelFiles) > 0 && base.InternalCompare(cmp, outputLevelFiles[0].Largest, f.Smallest) < 0 { - outputLevelFiles = outputLevelFiles[1:] + for outputFile != nil && base.InternalCompare(cmp, outputFile.Largest, f.Smallest) < 0 { + outputFile = outputIter.Next() } compacting := f.Compacting - for len(outputLevelFiles) > 0 && base.InternalCompare(cmp, outputLevelFiles[0].Smallest, f.Largest) < 0 { - overlappingBytes += outputLevelFiles[0].Size - compacting = compacting || outputLevelFiles[0].Compacting + for outputFile != nil && base.InternalCompare(cmp, outputFile.Smallest, f.Largest) < 0 { + overlappingBytes += outputFile.Size + compacting = compacting || outputFile.Compacting // If the file in the next level extends beyond f's largest key, - // break out and don't trim outputLevelFiles because f's - // successor might also overlap. - if base.InternalCompare(cmp, outputLevelFiles[0].Largest, f.Largest) > 0 { + // break out and don't advance outputIter because f's successor + // might also overlap. + if base.InternalCompare(cmp, outputFile.Largest, f.Largest) > 0 { break } - outputLevelFiles = outputLevelFiles[1:] + outputFile = outputIter.Next() } // If the input level file or one of the overlapping files is @@ -802,7 +803,7 @@ func (p *compactionPickerByScore) pickFile(level, outputLevel int) int { scaledRatio := overlappingBytes * 1024 / compensatedSize(f) if scaledRatio < smallestRatio && !f.Compacting { smallestRatio = scaledRatio - file = i + file = startIter.Take() } } return file @@ -859,7 +860,7 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (pc *pickedCompact } fmt.Fprintf(&buf, " %sL%d: %5.1f %5.1f %8s %8s", marker, info.level, info.score, info.origScore, - humanize.Int64(int64(totalCompensatedSize(p.vers.Levels[info.level]))), + humanize.Int64(int64(totalCompensatedSize(p.vers.Levels[info.level].Iter()))), humanize.Int64(p.levelMaxBytes[info.level]), ) @@ -914,7 +915,7 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (pc *pickedCompact } info.file = p.pickFile(info.level, info.outputLevel) - if info.file == -1 { + if info.file.Empty() { continue } @@ -938,25 +939,19 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (pc *pickedCompact // extremely wasteful in the common case. Could we maintain a // MarkedForCompaction map from fileNum to level? for level := 0; level < numLevels-1; level++ { - for file, f := range p.vers.Levels[level] { + iter := p.vers.Levels[level].Iter() + for f := iter.First(); f != nil; f = iter.Next() { if !f.MarkedForCompaction { continue } - for i := range scores { - if scores[i].level != level { - continue - } - info := &scores[i] - info.file = file - pc := pickAutoHelper(env, p.opts, p.vers, *info, p.baseLevel) - // Fail-safe to protect against compacting the same sstable concurrently. - if pc != nil && !inputAlreadyCompacting(pc) { - pc.score = info.score - return pc - } - break + info := &scores[level] + info.file = iter.Take() + pc := pickAutoHelper(env, p.opts, p.vers, *info, p.baseLevel) + // Fail-safe to protect against compacting the same sstable concurrently. + if pc != nil && !inputAlreadyCompacting(pc) { + pc.score = info.score + return pc } - break } } @@ -977,13 +972,13 @@ func pickAutoHelper( if pc.outputLevel.level != cInfo.outputLevel { panic("pebble: compaction picked unexpected output level") } - pc.startLevel.files = vers.Levels[pc.startLevel.level][cInfo.file : cInfo.file+1] + pc.startLevel.files = cInfo.file // Files in level 0 may overlap each other, so pick up all overlapping ones. if pc.startLevel.level == 0 { cmp := opts.Comparer.Compare - smallest, largest := manifest.KeyRange(cmp, pc.startLevel.files) - pc.startLevel.files = vers.Overlaps(0, cmp, smallest.UserKey, largest.UserKey).Collect() - if len(pc.startLevel.files) == 0 { + smallest, largest := manifest.KeyRange(cmp, pc.startLevel.files.Iter()) + pc.startLevel.files = vers.Overlaps(0, cmp, smallest.UserKey, largest.UserKey) + if pc.startLevel.files.Empty() { panic("pebble: empty compaction") } } @@ -1010,7 +1005,7 @@ func pickL0(env compactionEnv, opts *Options, vers *version, baseLevel int) (pc if lcf != nil { pc = newPickedCompactionFromL0(lcf, opts, vers, baseLevel, true) pc.setupInputs() - if len(pc.startLevel.files) == 0 { + if pc.startLevel.files.Empty() { opts.Logger.Fatalf("empty compaction chosen") } return pc @@ -1028,14 +1023,18 @@ func pickL0(env compactionEnv, opts *Options, vers *version, baseLevel int) (pc if lcf != nil { pc = newPickedCompactionFromL0(lcf, opts, vers, 0, false) pc.setupInputs() - if len(pc.startLevel.files) == 0 { + if pc.startLevel.files.Empty() { opts.Logger.Fatalf("empty compaction chosen") } - if len(pc.startLevel.files) == 1 { - // A single-file intra-L0 compaction is unproductive. - return nil + { + iter := pc.startLevel.files.Iter() + if iter.First() == nil || iter.Next() == nil { + // A single-file intra-L0 compaction is unproductive. + return nil + } } - pc.smallest, pc.largest = manifest.KeyRange(pc.cmp, pc.startLevel.files) + + pc.smallest, pc.largest = manifest.KeyRange(pc.cmp, pc.startLevel.files.Iter()) // Output only a single sstable for intra-L0 compactions. // Now that we have the ability to split flushes, we could conceivably // split the output of intra-L0 compactions too. This may be unnecessary @@ -1126,8 +1125,8 @@ func pickIntraL0(env compactionEnv, opts *Options, vers *version) (pc *pickedCom } pc = newPickedCompaction(opts, vers, 0, 0) - pc.startLevel.files = l0Files[begin:end] - pc.smallest, pc.largest = manifest.KeyRange(pc.cmp, pc.startLevel.files) + pc.startLevel.files = manifest.NewLevelSlice(l0Files[begin:end]) + pc.smallest, pc.largest = manifest.KeyRange(pc.cmp, pc.startLevel.files.Iter()) // Output only a single sstable for intra-L0 compactions. There is no current // benefit to outputting multiple tables, because other parts of the code // (i.e. iterators and comapction) expect L0 sstables to overlap and will @@ -1182,8 +1181,8 @@ func pickManualHelper( pc = newPickedCompaction(opts, vers, manual.level, baseLevel) manual.outputLevel = pc.outputLevel.level cmp := opts.Comparer.Compare - pc.startLevel.files = vers.Overlaps(manual.level, cmp, manual.start.UserKey, manual.end.UserKey).Collect() - if len(pc.startLevel.files) == 0 { + pc.startLevel.files = vers.Overlaps(manual.level, cmp, manual.start.UserKey, manual.end.UserKey) + if pc.startLevel.files.Empty() { // Nothing to do return nil } @@ -1197,7 +1196,8 @@ func (p *compactionPickerByScore) forceBaseLevel1() { func inputAlreadyCompacting(pc *pickedCompaction) bool { for _, cl := range pc.inputs { - for _, f := range cl.files { + iter := cl.files.Iter() + for f := iter.First(); f != nil; f = iter.Next() { if f.Compacting { return true } diff --git a/github.com/cockroachdb/pebble/db.go b/github.com/cockroachdb/pebble/db.go index 19ae2f2bc6..b9b8ff91b7 100644 --- a/github.com/cockroachdb/pebble/db.go +++ b/github.com/cockroachdb/pebble/db.go @@ -734,7 +734,7 @@ func (d *DB) newIterInternal( } } for level := 1; level < len(current.Levels); level++ { - if len(current.Levels[level]) == 0 { + if current.Levels[level].Iter().Empty() { continue } mlevels = append(mlevels, mergingIterLevel{}) @@ -1139,12 +1139,12 @@ func (d *DB) EstimateDiskUsage(start, end []byte) (uint64, error) { var totalSize uint64 for level, files := range readState.current.Levels { - iter := manifest.SliceLevelIterator(files) + iter := files.Iter() if level > 0 { // We can only use `Overlaps` to restrict `files` at L1+ since at L0 it // expands the range iteratively until it has found a set of files that // do not overlap any other L0 files outside that set. - iter = readState.current.Overlaps(level, d.opts.Comparer.Compare, start, end) + iter = readState.current.Overlaps(level, d.opts.Comparer.Compare, start, end).Iter() } for file := iter.First(); file != nil; file = iter.Next() { if d.opts.Comparer.Compare(start, file.Smallest.UserKey) <= 0 && @@ -1457,9 +1457,15 @@ func (d *DB) getInProgressCompactionInfoLocked(finishing *compaction) (rv []comp for c := range d.mu.compact.inProgress { if len(c.flushing) == 0 && (finishing == nil || c != finishing) { info := compactionInfo{ - inputs: c.inputs, + inputs: make([]compactionInput, 0, len(c.inputs)), outputLevel: -1, } + for _, in := range c.inputs { + info.inputs = append(info.inputs, compactionInput{ + level: in.level, + files: manifest.NewLevelSlice(in.files), + }) + } if c.outputLevel != nil { info.outputLevel = c.outputLevel.level } diff --git a/github.com/cockroachdb/pebble/event.go b/github.com/cockroachdb/pebble/event.go index 46081de43c..3ed371dcbc 100644 --- a/github.com/cockroachdb/pebble/event.go +++ b/github.com/cockroachdb/pebble/event.go @@ -42,20 +42,15 @@ type LevelInfo struct { } func (i LevelInfo) String() string { - return string(redact.Sprintfn(i.safeFormat)) + return redact.StringWithoutMarkers(i) } -func (i LevelInfo) safeFormat(w redact.SafePrinter) { +// SafeFormat implements redact.SafeFormatter. +func (i LevelInfo) SafeFormat(w redact.SafePrinter, _ rune) { w.Printf("L%d [%s] (%s)", redact.Safe(i.Level), redact.Safe(formatFileNums(i.Tables)), redact.Safe(humanize.Uint64(tablesTotalSize(i.Tables)))) } -func redactSprint(s redact.SafeFormatter) string { - return string(redact.Sprintfn(func(w redact.SafePrinter) { - s.SafeFormat(w, 's') - })) -} - // CompactionInfo contains the info for a compaction event. type CompactionInfo struct { // JobID is the ID of the compaction job. @@ -73,7 +68,7 @@ type CompactionInfo struct { } func (i CompactionInfo) String() string { - return redactSprint(i) + return redact.StringWithoutMarkers(i) } // SafeFormat implements redact.SafeFormatter. @@ -86,12 +81,12 @@ func (i CompactionInfo) SafeFormat(w redact.SafePrinter, _ rune) { if !i.Done { w.Printf("[JOB %d] compacting ", redact.Safe(i.JobID)) - i.safeFormatInputs(w) + w.Print(levelInfos(i.Input)) return } outputSize := tablesTotalSize(i.Output.Tables) w.Printf("[JOB %d] compacted ", redact.Safe(i.JobID)) - i.safeFormatInputs(w) + w.Print(levelInfos(i.Input)) w.Printf(" -> L%d [%s] (%s), in %.1fs, output rate %s/s", redact.Safe(i.Output.Level), redact.Safe(formatFileNums(i.Output.Tables)), @@ -100,12 +95,14 @@ func (i CompactionInfo) SafeFormat(w redact.SafePrinter, _ rune) { redact.Safe(humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds())))) } -func (i CompactionInfo) safeFormatInputs(w redact.SafePrinter) { - for j, levelInfo := range i.Input { +type levelInfos []LevelInfo + +func (i levelInfos) SafeFormat(w redact.SafePrinter, _ rune) { + for j, levelInfo := range i { if j > 0 { w.Printf(" + ") } - levelInfo.safeFormat(w) + w.Print(levelInfo) } } @@ -126,7 +123,7 @@ type FlushInfo struct { } func (i FlushInfo) String() string { - return redactSprint(i) + return redact.StringWithoutMarkers(i) } // SafeFormat implements redact.SafeFormatter. @@ -167,7 +164,7 @@ type ManifestCreateInfo struct { } func (i ManifestCreateInfo) String() string { - return redactSprint(i) + return redact.StringWithoutMarkers(i) } // SafeFormat implements redact.SafeFormatter. @@ -189,7 +186,7 @@ type ManifestDeleteInfo struct { } func (i ManifestDeleteInfo) String() string { - return redactSprint(i) + return redact.StringWithoutMarkers(i) } // SafeFormat implements redact.SafeFormatter. @@ -212,7 +209,7 @@ type TableCreateInfo struct { } func (i TableCreateInfo) String() string { - return redactSprint(i) + return redact.StringWithoutMarkers(i) } // SafeFormat implements redact.SafeFormatter. @@ -230,7 +227,7 @@ type TableDeleteInfo struct { } func (i TableDeleteInfo) String() string { - return redactSprint(i) + return redact.StringWithoutMarkers(i) } // SafeFormat implements redact.SafeFormatter. @@ -258,7 +255,7 @@ type TableIngestInfo struct { } func (i TableIngestInfo) String() string { - return redactSprint(i) + return redact.StringWithoutMarkers(i) } // SafeFormat implements redact.SafeFormatter. @@ -287,7 +284,7 @@ type TableStatsInfo struct { } func (i TableStatsInfo) String() string { - return redactSprint(i) + return redact.StringWithoutMarkers(i) } // SafeFormat implements redact.SafeFormatter. @@ -309,7 +306,7 @@ type WALCreateInfo struct { } func (i WALCreateInfo) String() string { - return redactSprint(i) + return redact.StringWithoutMarkers(i) } // SafeFormat implements redact.SafeFormatter. @@ -338,7 +335,7 @@ type WALDeleteInfo struct { } func (i WALDeleteInfo) String() string { - return redactSprint(i) + return redact.StringWithoutMarkers(i) } // SafeFormat implements redact.SafeFormatter. @@ -356,7 +353,7 @@ type WriteStallBeginInfo struct { } func (i WriteStallBeginInfo) String() string { - return redactSprint(i) + return redact.StringWithoutMarkers(i) } // SafeFormat implements redact.SafeFormatter. diff --git a/github.com/cockroachdb/pebble/get_iter.go b/github.com/cockroachdb/pebble/get_iter.go index 9d7f0d7164..37dbac5e87 100644 --- a/github.com/cockroachdb/pebble/get_iter.go +++ b/github.com/cockroachdb/pebble/get_iter.go @@ -155,7 +155,7 @@ func (g *getIter) Next() (*InternalKey, []byte) { if g.level >= numLevels { return nil, nil } - if len(g.version.Levels[g.level]) == 0 { + if g.version.Levels[g.level].Iter().Empty() { g.level++ continue } diff --git a/github.com/cockroachdb/pebble/internal/manifest/level_iterator.go b/github.com/cockroachdb/pebble/internal/manifest/level_iterator.go deleted file mode 100644 index 2de2c526cc..0000000000 --- a/github.com/cockroachdb/pebble/internal/manifest/level_iterator.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use -// of this source code is governed by a BSD-style license that can be found in -// the LICENSE file. - -package manifest - -import "sort" - -// SliceLevelIterator constructs a LevelIterator over the provided slice. This -// function is expected to be a temporary adapter between interfaces. -// TODO(jackson): Revisit once the conversion of Version.Files to a btree is -// complete. -func SliceLevelIterator(files []*FileMetadata) LevelIterator { - return LevelIterator{files: files, cur: 0} -} - -// LevelIterator iterates over a set of files' metadata. -type LevelIterator struct { - files []*FileMetadata - cur int -} - -// Collect returns a slice of all the files in the iterator, irrespective of -// the current iterator position. The returned slice is owned by the iterator. -// This method is intended to be a temporary adatpter between interfaces, and -// callers should prefer using the iterator directory when possible. -// -// TODO(jackson): Revisit once the conversion of Version.Files to a btree is -// complete. -func (i LevelIterator) Collect() []*FileMetadata { - return i.files -} - -// SizeSum sums the size of all files in the iterator, irrespective of the -// current iterator position. -func (i LevelIterator) SizeSum() uint64 { - var sum uint64 - for _, f := range i.files { - sum += f.Size - } - return sum -} - -// Empty indicates whether there are remaining files in the iterator. -func (i LevelIterator) Empty() bool { - return i.cur >= len(i.files) -} - -// First seeks to the first file in the iterator and returns it. -func (i *LevelIterator) First() *FileMetadata { - i.cur = 0 - if i.cur >= len(i.files) { - return nil - } - return i.files[i.cur] -} - -// Next advances the iterator to the next file and returns it. -func (i *LevelIterator) Next() *FileMetadata { - i.cur++ - if i.cur >= len(i.files) { - return nil - } - return i.files[i.cur] -} - -// SeekGE seeks to the first file in the iterator's file set with a largest -// user key less than or equal to the provided user key. -func (i *LevelIterator) SeekGE(cmp Compare, userKey []byte) *FileMetadata { - i.cur = sort.Search(len(i.files), func(j int) bool { - return cmp(userKey, i.files[j].Largest.UserKey) <= 0 - }) - if i.cur >= len(i.files) { - return nil - } - return i.files[i.cur] -} diff --git a/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go b/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go new file mode 100644 index 0000000000..079c7083ce --- /dev/null +++ b/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go @@ -0,0 +1,203 @@ +// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package manifest + +import "sort" + +// LevelMetadata contains metadata for all of the files within +// a level of the LSM. +// TODO(jackson): Convert to an opaque struct. +type LevelMetadata []*FileMetadata + +// Iter constructs a LevelIterator over the entire level. +func (lm LevelMetadata) Iter() LevelIterator { + return LevelIterator{files: lm, end: len(lm)} +} + +// Slice constructs a slice containing the entire level. +func (lm LevelMetadata) Slice() LevelSlice { + return LevelSlice{files: lm, end: len(lm)} +} + +// NewLevelSlice constructs a LevelSlice over the provided files. This +// function is expected to be a temporary adapter between interfaces. +// TODO(jackson): Revisit once the conversion of Version.Files to a btree is +// complete. +func NewLevelSlice(files []*FileMetadata) LevelSlice { + return LevelSlice{files: files, start: 0, end: len(files)} +} + +// LevelSlice contains a slice of the files within a level of the LSM. +type LevelSlice struct { + files []*FileMetadata + start int + end int +} + +// Collect returns a Go slice of all files in the slice. The returned slice is +// owned by the LevelSlice. This method is intended to be a temporary adatpter +// between interfaces, and callers should prefer using the iterator directory +// when possible. +// +// TODO(jackson): Revisit once the conversion of Version.Files to a btree is +// complete. +func (ls LevelSlice) Collect() []*FileMetadata { + return ls.files[ls.start:ls.end] +} + +// Each invokes fn for each element in the slice. +func (ls LevelSlice) Each(fn func(*FileMetadata)) { + iter := ls.Iter() + for f := iter.First(); f != nil; f = iter.Next() { + fn(f) + } +} + +// Empty indicates whether the slice contains any files. +func (ls LevelSlice) Empty() bool { + return ls.start >= ls.end +} + +// Iter constructs a LevelIterator that iterates over the slice. +func (ls LevelSlice) Iter() LevelIterator { + return LevelIterator{ + files: ls.files, + start: ls.start, + end: ls.end, + } +} + +// Len returns the number of files in the slice. +func (ls LevelSlice) Len() int { + return ls.end - ls.start +} + +// SizeSum sums the size of all files in the slice. +func (ls LevelSlice) SizeSum() uint64 { + var sum uint64 + for _, f := range ls.files[ls.start:ls.end] { + sum += f.Size + } + return sum +} + +// Reslice constructs a new slice backed by the same underlying level, with +// new start and end positions. Reslice invokes the provided function, passing +// two LevelIterators: one positioned to i's inclusive start and one +// positioned to i's inclusive end. The resliceFunc may move either iterator +// forward or backwards, including beyond the callee's original bounds to +// capture additional files from the underlying level. Reslice constructs and +// returns a new LevelSlice with the final bounds of the iterators after +// calling resliceFunc. +func (ls LevelSlice) Reslice(resliceFunc func(start, end *LevelIterator)) LevelSlice { + start := LevelIterator{ + files: ls.files, + cur: ls.start, + start: 0, + end: len(ls.files), + } + end := LevelIterator{ + files: ls.files, + cur: ls.end - 1, + start: 0, + end: len(ls.files), + } + resliceFunc(&start, &end) + return LevelSlice{ + files: ls.files, + start: start.cur, + end: end.cur + 1, + } +} + +// LevelIterator iterates over a set of files' metadata. Its zero value is an +// empty iterator. +type LevelIterator struct { + files []*FileMetadata + cur int + start int + end int +} + +// Clone copies the iterator, returning an independent iterator at the same +// position. +func (i *LevelIterator) Clone() LevelIterator { + return *i +} + +// Empty indicates whether there are remaining files in the iterator. +func (i LevelIterator) Empty() bool { + return i.cur >= i.end +} + +// Current returns the item at the current iterator position. +func (i LevelIterator) Current() *FileMetadata { + if i.cur < i.start || i.cur >= i.end { + return nil + } + return i.files[i.cur] +} + +// First seeks to the first file in the iterator and returns it. +func (i *LevelIterator) First() *FileMetadata { + i.cur = i.start + if i.cur < i.start || i.cur >= i.end { + return nil + } + return i.files[i.cur] +} + +// Last seeks to the last file in the iterator and returns it. +func (i *LevelIterator) Last() *FileMetadata { + i.cur = i.end - 1 + if i.cur < i.start || i.cur >= i.end { + return nil + } + return i.files[i.cur] +} + +// Next advances the iterator to the next file and returns it. +func (i *LevelIterator) Next() *FileMetadata { + i.cur++ + if i.cur < i.start || i.cur >= i.end { + return nil + } + return i.files[i.cur] +} + +// Prev moves the iterator the previous file and returns it. +func (i *LevelIterator) Prev() *FileMetadata { + i.cur-- + if i.cur < i.start || i.cur >= i.end { + return nil + } + return i.files[i.cur] +} + +// SeekGE seeks to the first file in the iterator's file set with a largest +// user key less than or equal to the provided user key. The iterator must +// have been constructed from L1+, because it requires the underlying files to +// be sorted by user keys and non-overlapping. +func (i *LevelIterator) SeekGE(cmp Compare, userKey []byte) *FileMetadata { + files := i.files[i.start:i.end] + i.cur = i.start + sort.Search(len(files), func(j int) bool { + return cmp(userKey, files[j].Largest.UserKey) <= 0 + }) + if i.cur >= i.end { + return nil + } + return i.files[i.cur] +} + +// Take constructs a new LevelSlice containing only the file at the iterator's +// current position. If the iterator is not currently positioned over a file, +// the returned iterator is empty. +func (i LevelIterator) Take() LevelSlice { + return LevelSlice{ + files: i.files, + start: i.cur, + end: i.cur + 1, + } +} diff --git a/github.com/cockroachdb/pebble/internal/manifest/version.go b/github.com/cockroachdb/pebble/internal/manifest/version.go index 079e2c4392..39c6f3bcef 100644 --- a/github.com/cockroachdb/pebble/internal/manifest/version.go +++ b/github.com/cockroachdb/pebble/internal/manifest/version.go @@ -130,20 +130,20 @@ func (m *FileMetadata) lessSmallestKey(b *FileMetadata, cmp Compare) bool { } // KeyRange returns the minimum smallest and maximum largest internalKey for -// all the fileMetadata in f0 and f1. -func KeyRange(ucmp Compare, fileSlices ...[]*FileMetadata) (smallest, largest InternalKey) { +// all the FileMetadata in iters. +func KeyRange(ucmp Compare, iters ...LevelIterator) (smallest, largest InternalKey) { first := true - for _, files := range fileSlices { - for _, meta := range files { + for _, iter := range iters { + for meta := iter.First(); meta != nil; meta = iter.Next() { if first { first = false smallest, largest = meta.Smallest, meta.Largest continue } - if base.InternalCompare(ucmp, meta.Smallest, smallest) < 0 { + if base.InternalCompare(ucmp, smallest, meta.Smallest) >= 0 { smallest = meta.Smallest } - if base.InternalCompare(ucmp, meta.Largest, largest) > 0 { + if base.InternalCompare(ucmp, largest, meta.Largest) <= 0 { largest = meta.Largest } } @@ -235,7 +235,7 @@ type Version struct { // in Files[0] are in L0Sublevels.Levels. L0Sublevels *L0Sublevels - Levels [NumLevels][]*FileMetadata + Levels [NumLevels]LevelMetadata // The callback to invoke when the last reference to a version is // removed. Will be called with list.mu held. @@ -255,25 +255,22 @@ func (v *Version) String() string { // Pretty returns a string representation of the version. func (v *Version) Pretty(format base.FormatKey) string { var buf bytes.Buffer - for level := 0; level < NumLevels; level++ { - if len(v.Levels[level]) == 0 { - continue - } - - if level == 0 { - for sublevel := len(v.L0Sublevels.Levels) - 1; sublevel >= 0; sublevel-- { - fmt.Fprintf(&buf, "0.%d:\n", sublevel) - for _, f := range v.L0Sublevels.Levels[sublevel] { - fmt.Fprintf(&buf, " %06d:[%s-%s]\n", f.FileNum, - format(f.Smallest.UserKey), format(f.Largest.UserKey)) - } + if v.L0Sublevels != nil { + for sublevel := len(v.L0Sublevels.Levels) - 1; sublevel >= 0; sublevel-- { + fmt.Fprintf(&buf, "0.%d:\n", sublevel) + for _, f := range v.L0Sublevels.Levels[sublevel] { + fmt.Fprintf(&buf, " %06d:[%s-%s]\n", f.FileNum, + format(f.Smallest.UserKey), format(f.Largest.UserKey)) } + } + } + for level := 1; level < NumLevels; level++ { + iter := v.Levels[level].Iter() + if iter.Empty() { continue } - fmt.Fprintf(&buf, "%d:\n", level) - for j := range v.Levels[level] { - f := v.Levels[level][j] + for f := iter.First(); f != nil; f = iter.Next() { fmt.Fprintf(&buf, " %s:[%s-%s]\n", f.FileNum, format(f.Smallest.UserKey), format(f.Largest.UserKey)) } @@ -285,25 +282,23 @@ func (v *Version) Pretty(format base.FormatKey) string { // sequence number and kind information for the sstable boundaries. func (v *Version) DebugString(format base.FormatKey) string { var buf bytes.Buffer - for level := 0; level < NumLevels; level++ { - if len(v.Levels[level]) == 0 { - continue - } - if level == 0 { - for sublevel := len(v.L0Sublevels.Levels) - 1; sublevel >= 0; sublevel-- { - fmt.Fprintf(&buf, "0.%d:\n", sublevel) - for _, f := range v.L0Sublevels.Levels[sublevel] { - fmt.Fprintf(&buf, " %06d:[%s-%s]\n", f.FileNum, - f.Smallest.Pretty(format), f.Largest.Pretty(format)) - } + if v.L0Sublevels != nil { + for sublevel := len(v.L0Sublevels.Levels) - 1; sublevel >= 0; sublevel-- { + fmt.Fprintf(&buf, "0.%d:\n", sublevel) + for _, f := range v.L0Sublevels.Levels[sublevel] { + fmt.Fprintf(&buf, " %06d:[%s-%s]\n", f.FileNum, + f.Smallest.Pretty(format), f.Largest.Pretty(format)) } + } + } + for level := 1; level < NumLevels; level++ { + iter := v.Levels[level].Iter() + if iter.Empty() { continue } - fmt.Fprintf(&buf, "%d:\n", level) - for j := range v.Levels[level] { - f := v.Levels[level][j] + for f := iter.First(); f != nil; f = iter.Next() { fmt.Fprintf(&buf, " %s:[%s-%s]\n", f.FileNum, f.Smallest.Pretty(format), f.Largest.Pretty(format)) } @@ -348,10 +343,12 @@ func (v *Version) UnrefLocked() { } func (v *Version) unrefFiles() []base.FileNum { + // TODO(jackson): Move responsibility of ref-ing of individual files into + // the LevelMetadata type. var obsolete []base.FileNum for _, files := range v.Levels { - for i := range files { - f := files[i] + iter := files.Iter() + for f := iter.First(); f != nil; f = iter.Next() { if atomic.AddInt32(&f.refs, -1) == 0 { obsolete = append(obsolete, f.FileNum) } @@ -379,9 +376,9 @@ func (v *Version) InitL0Sublevels( // searches among the files. If level is zero, Contains scans the entire // level. func (v *Version) Contains(level int, cmp Compare, m *FileMetadata) bool { - iter := LevelIterator{files: v.Levels[level]} + iter := v.Levels[level].Slice().Iter() if level > 0 { - iter = v.Overlaps(level, cmp, m.Smallest.UserKey, m.Largest.UserKey) + iter = v.Overlaps(level, cmp, m.Smallest.UserKey, m.Largest.UserKey).Iter() } for f := iter.First(); f != nil; f = iter.Next() { if f == m { @@ -399,12 +396,12 @@ func (v *Version) Contains(level int, cmp Compare, m *FileMetadata) bool { // and the computation is repeated until [start, end] stabilizes. // The returned files are a subsequence of the input files, i.e., the ordering // is not changed. -func (v *Version) Overlaps(level int, cmp Compare, start, end []byte) LevelIterator { +func (v *Version) Overlaps(level int, cmp Compare, start, end []byte) LevelSlice { if level == 0 { // Indices that have been selected as overlapping. selectedIndices := make([]bool, len(v.Levels[level])) numSelected := 0 - var iter LevelIterator + var slice LevelSlice for { restart := false for i, selected := range selectedIndices { @@ -442,25 +439,28 @@ func (v *Version) Overlaps(level int, cmp Compare, start, end []byte) LevelItera } if !restart { - iter.files = make([]*FileMetadata, 0, numSelected) + slice.files = make([]*FileMetadata, 0, numSelected) for i, selected := range selectedIndices { if selected { - iter.files = append(iter.files, v.Levels[level][i]) + slice.files = append(slice.files, v.Levels[level][i]) } } + slice.end = len(slice.files) break } // Continue looping to retry the files that were not selected. } - return iter + return slice } - var iter LevelIterator + var slice LevelSlice lower, upper := overlaps(v.Levels[level], cmp, start, end) if lower < upper { - iter.files = v.Levels[level][lower:upper] + slice.files = v.Levels[level] + slice.start = lower + slice.end = upper } - return iter + return slice } // CheckOrdering checks that the files are consistent with respect to @@ -468,13 +468,14 @@ func (v *Version) Overlaps(level int, cmp Compare, start, end []byte) LevelItera // overlapping internal key ranges (for level non-0 files). func (v *Version) CheckOrdering(cmp Compare, format base.FormatKey) error { for sublevel := len(v.L0Sublevels.Levels) - 1; sublevel >= 0; sublevel-- { - if err := CheckOrdering(cmp, format, L0Sublevel(sublevel), v.L0Sublevels.Levels[sublevel]); err != nil { + iter := NewLevelSlice(v.L0Sublevels.Levels[sublevel]).Iter() + if err := CheckOrdering(cmp, format, L0Sublevel(sublevel), iter); err != nil { return errors.Errorf("%s\n%s", err, v.DebugString(format)) } } - for level, files := range v.Levels { - if err := CheckOrdering(cmp, format, Level(level), files); err != nil { + for level, lm := range v.Levels { + if err := CheckOrdering(cmp, format, Level(level), lm.Iter()); err != nil { return errors.Errorf("%s\n%s", err, v.DebugString(format)) } } @@ -488,7 +489,8 @@ func (v *Version) CheckConsistency(dirname string, fs vfs.FS) error { var args []interface{} for level, files := range v.Levels { - for _, f := range files { + iter := files.Iter() + for f := iter.First(); f != nil; f = iter.Next() { path := base.MakeFilename(fs, dirname, base.FileTypeTable, f.FileNum) info, err := fs.Stat(path) if err != nil { @@ -573,7 +575,7 @@ func (l *VersionList) Remove(v *Version) { // CheckOrdering checks that the files are consistent with respect to // seqnums (for level 0 files -- see detailed comment below) and increasing and non- // overlapping internal key ranges (for non-level 0 files). -func CheckOrdering(cmp Compare, format base.FormatKey, level Level, files []*FileMetadata) error { +func CheckOrdering(cmp Compare, format base.FormatKey, level Level, files LevelIterator) error { // The invariants to check for L0 sublevels are the same as the ones to // check for all other levels. However, if L0 is not organized into // sublevels, or if all L0 files are being passed in, we do the legacy L0 @@ -623,31 +625,30 @@ func CheckOrdering(cmp Compare, format base.FormatKey, level Level, files []*Fil // with future versions of pebble, this method relaxes most L0 invariant // checks. - for i := range files { - f := files[i] - if i > 0 { - // Validate that the sorting is sane. - prev := files[i-1] - if prev.LargestSeqNum == 0 && f.LargestSeqNum == prev.LargestSeqNum { - // Multiple files satisfying case 2 mentioned above. - } else if !prev.lessSeqNum(f) { - return errors.Errorf("L0 files %s and %s are not properly ordered: <#%d-#%d> vs <#%d-#%d>", - errors.Safe(prev.FileNum), errors.Safe(f.FileNum), - errors.Safe(prev.SmallestSeqNum), errors.Safe(prev.LargestSeqNum), - errors.Safe(f.SmallestSeqNum), errors.Safe(f.LargestSeqNum)) - } + var prev *FileMetadata + for f := files.First(); f != nil; f, prev = files.Next(), f { + if prev == nil { + continue + } + // Validate that the sorting is sane. + if prev.LargestSeqNum == 0 && f.LargestSeqNum == prev.LargestSeqNum { + // Multiple files satisfying case 2 mentioned above. + } else if !prev.lessSeqNum(f) { + return errors.Errorf("L0 files %s and %s are not properly ordered: <#%d-#%d> vs <#%d-#%d>", + errors.Safe(prev.FileNum), errors.Safe(f.FileNum), + errors.Safe(prev.SmallestSeqNum), errors.Safe(prev.LargestSeqNum), + errors.Safe(f.SmallestSeqNum), errors.Safe(f.LargestSeqNum)) } } } else { - for i := range files { - f := files[i] + var prev *FileMetadata + for f := files.First(); f != nil; f, prev = files.Next(), f { if base.InternalCompare(cmp, f.Smallest, f.Largest) > 0 { return errors.Errorf("%s file %s has inconsistent bounds: %s vs %s", errors.Safe(level), errors.Safe(f.FileNum), f.Smallest.Pretty(format), f.Largest.Pretty(format)) } - if i > 0 { - prev := files[i-1] + if prev != nil { if !prev.lessSmallestKey(f, cmp) { return errors.Errorf("%s files %s and %s are not properly ordered: [%s-%s] vs [%s-%s]", errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum), diff --git a/github.com/cockroachdb/pebble/internal/manifest/version_edit.go b/github.com/cockroachdb/pebble/internal/manifest/version_edit.go index 37b31f7cb7..9bb01c7cfa 100644 --- a/github.com/cockroachdb/pebble/internal/manifest/version_edit.go +++ b/github.com/cockroachdb/pebble/internal/manifest/version_edit.go @@ -560,7 +560,7 @@ func (b *BulkVersionEdit) Apply( if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil { return nil, nil, errors.Wrap(err, "pebble: internal error") } - if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level]); err != nil { + if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter()); err != nil { return nil, nil, errors.Wrap(err, "pebble: internal error") } continue diff --git a/github.com/cockroachdb/pebble/level_checker.go b/github.com/cockroachdb/pebble/level_checker.go index 7a3b097ff2..29062c4524 100644 --- a/github.com/cockroachdb/pebble/level_checker.go +++ b/github.com/cockroachdb/pebble/level_checker.go @@ -368,10 +368,10 @@ func checkRangeTombstones(c *checkConfig) error { } current := c.readState.current - addTombstonesFromLevel := func(files *[]*fileMetadata, lsmLevel int) error { - for j := 0; j < len(*files); j++ { - lower, upper := getAtomicUnitBounds(c.cmp, *files, j) - f := (*files)[j] + addTombstonesFromLevel := func(files []*fileMetadata, lsmLevel int) error { + for j := 0; j < len(files); j++ { + lower, upper := getAtomicUnitBounds(c.cmp, files, j) + f := files[j] iterToClose, iter, err := c.newIters(f, nil, nil) if err != nil { return err @@ -405,7 +405,7 @@ func checkRangeTombstones(c *checkConfig) error { if len(current.L0Sublevels.Levels[i]) == 0 { continue } - if err := addTombstonesFromLevel(¤t.L0Sublevels.Levels[i], 0); err != nil { + if err := addTombstonesFromLevel(current.L0Sublevels.Levels[i], 0); err != nil { return err } level++ @@ -414,7 +414,7 @@ func checkRangeTombstones(c *checkConfig) error { if len(current.Levels[i]) == 0 { continue } - files := ¤t.Levels[i] + files := current.Levels[i] if err := addTombstonesFromLevel(files, i); err != nil { return err } diff --git a/github.com/cockroachdb/pebble/sstable/reader.go b/github.com/cockroachdb/pebble/sstable/reader.go index 089d27f01e..3f8c0e66f4 100644 --- a/github.com/cockroachdb/pebble/sstable/reader.go +++ b/github.com/cockroachdb/pebble/sstable/reader.go @@ -36,7 +36,7 @@ const ( // TODO(bilal): Have the initial size value be a factor of the block size, // as opposed to a hardcoded value. initialReadaheadSize = 64 << 10 /* 64KB */ - maxReadaheadSize = 512 << 10 /* 512KB */ + maxReadaheadSize = 256 << 10 /* 256KB */ ) // decodeBlockHandle returns the block handle encoded at the start of src, as @@ -498,6 +498,10 @@ func (i *singleLevelIterator) Close() error { } err = firstError(err, i.data.Close()) err = firstError(err, i.index.Close()) + if i.dataRS.sequentialFile != nil { + err = firstError(err, i.dataRS.sequentialFile.Close()) + i.dataRS.sequentialFile = nil + } err = firstError(err, i.err) *i = i.resetForReuse() singleLevelIterPool.Put(i) @@ -851,6 +855,10 @@ func (i *twoLevelIterator) Close() error { err = firstError(err, i.data.Close()) err = firstError(err, i.index.Close()) err = firstError(err, i.topLevelIndex.Close()) + if i.dataRS.sequentialFile != nil { + err = firstError(err, i.dataRS.sequentialFile.Close()) + i.dataRS.sequentialFile = nil + } err = firstError(err, i.err) *i = twoLevelIterator{ singleLevelIterator: i.singleLevelIterator.resetForReuse(), @@ -952,23 +960,66 @@ type readaheadState struct { // operation. Reads after this limit can benefit from a new call to // Prefetch. limit int64 + // sequentialFile holds a file descriptor to the same underlying File, + // except with fadvise(FADV_SEQUENTIAL) called on it to take advantage of + // OS-level readahead. Initialized when the iterator has been consistently + // reading blocks in a sequential access pattern. Once this is non-nil, + // the other variables in readaheadState don't matter much as we defer + // to OS-level readahead. + sequentialFile vfs.File +} + +func (rs *readaheadState) recordCacheHit(offset, blockLength int64) { + currentReadEnd := offset + blockLength + if rs.sequentialFile != nil { + // Using OS-level readahead instead, so do nothing. + return + } + if rs.numReads >= minFileReadsForReadahead { + if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize { + // This is a read that would have resulted in a readahead, had it + // not been a cache hit. + rs.limit = currentReadEnd + return + } + if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+maxReadaheadSize { + // We read too far away from rs.limit to benefit from readahead in + // any scenario. Reset all variables. + rs.numReads = 1 + rs.limit = currentReadEnd + rs.size = initialReadaheadSize + rs.prevSize = 0 + return + } + // Reads in the range [rs.limit - rs.prevSize, rs.limit] end up + // here. This is a read that is potentially benefitting from a past + // readahead. + return + } + if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize { + // Blocks are being read sequentially and would benefit from readahead + // down the line. + rs.numReads++ + return + } + // We read too far ahead of the last read, or before it. This indicates + // a random read, where readahead is not desirable. Reset all variables. + rs.numReads = 1 + rs.limit = currentReadEnd + rs.size = initialReadaheadSize + rs.prevSize = 0 } // maybeReadahead updates state and determines whether to issue a readahead / // prefetch call for a block read at offset for blockLength bytes. // Returns a size value (greater than 0) that should be prefetched if readahead // would be beneficial. -// -// TODO(bilal): This method is only called when there's a cache miss. Reads -// from the cache are completely invisible to the logic here and the state -// variables in readaheadState. This is a big reason why the -// readahead window in this method needs to be optimistic (that reading ahead -// will generally help) by always extending the window to -// rs.limit + maxReadaheadSize, to adjust for blocks that appear to be skipped -// but were actually read from the cache. Update this method or add another -// method to properly account for cache hits. func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 { currentReadEnd := offset + blockLength + if rs.sequentialFile != nil { + // Using OS-level readahead instead, so do nothing. + return 0 + } if rs.numReads >= minFileReadsForReadahead { // The minimum threshold of sequential reads to justify reading ahead // has been reached. @@ -1153,6 +1204,20 @@ func (c *cacheOpts) writerApply(w *Writer) { } } +// FileReopenOpt is specified if this reader is allowed to reopen additional +// file descriptors for this file. Used to take advantage of OS-level readahead. +type FileReopenOpt struct{ + FS vfs.FS + Filename string +} + +func (f FileReopenOpt) readerApply(r *Reader) { + if r.fs == nil { + r.fs = f.FS + r.filename = f.Filename + } +} + // rawTombstonesOpt is a Reader open option for specifying that range // tombstones returned by Reader.NewRangeDelIter() should not be // fragmented. Used by debug tools to get a raw view of the tombstones @@ -1175,6 +1240,8 @@ func init() { // Reader is a table reader. type Reader struct { file vfs.File + fs vfs.FS + filename string cacheID uint64 fileNum base.FileNum rawTombstones bool @@ -1350,18 +1417,42 @@ func (r *Reader) readBlock( bh BlockHandle, transform blockTransform, raState *readaheadState, ) (cache.Handle, error) { if h := r.opts.Cache.Get(r.cacheID, r.fileNum, bh.Offset); h.Get() != nil { + if raState != nil { + raState.recordCacheHit(int64(bh.Offset), int64(bh.Length+blockTrailerLen)) + } return h, nil } + file := r.file if raState != nil { - if readaheadSize := raState.maybeReadahead(int64(bh.Offset), int64(bh.Length+blockTrailerLen)); readaheadSize > 0 { - _ = vfs.Prefetch(r.file, bh.Offset, uint64(readaheadSize)) + if raState.sequentialFile != nil { + file = raState.sequentialFile + } else if readaheadSize := raState.maybeReadahead(int64(bh.Offset), int64(bh.Length+blockTrailerLen)); readaheadSize > 0 { + if readaheadSize >= maxReadaheadSize { + // We've reached the maximum readahead size. Beyond this + // point, rely on OS-level readahead. Note that we can only + // reopen a new file handle with this optimization if + // r.fs != nil. This reader must have been created with the + // FileReopenOpt for this field to be set. + if r.fs != nil { + f, err := r.fs.Open(r.filename, vfs.SequentialReadsOption) + if err == nil { + // Use this new file handle for all sequential reads by + // this iterator going forward. + raState.sequentialFile = f + file = f + } + } + } + if raState.sequentialFile != nil { + _ = vfs.Prefetch(r.file, bh.Offset, uint64(readaheadSize)) + } } } v := r.opts.Cache.Alloc(int(bh.Length + blockTrailerLen)) b := v.Buf() - if _, err := r.file.ReadAt(b, int64(bh.Offset)); err != nil { + if _, err := file.ReadAt(b, int64(bh.Offset)); err != nil { r.opts.Cache.Free(v) return cache.Handle{}, err } diff --git a/github.com/cockroachdb/pebble/table_cache.go b/github.com/cockroachdb/pebble/table_cache.go index b47706241a..901518d921 100644 --- a/github.com/cockroachdb/pebble/table_cache.go +++ b/github.com/cockroachdb/pebble/table_cache.go @@ -556,11 +556,12 @@ type tableCacheValue struct { func (v *tableCacheValue) load(meta *fileMetadata, c *tableCacheShard) { // Try opening the fileTypeTable first. var f vfs.File - f, v.err = c.fs.Open(base.MakeFilename(c.fs, c.dirname, fileTypeTable, meta.FileNum), - vfs.RandomReadsOption) + filename := base.MakeFilename(c.fs, c.dirname, fileTypeTable, meta.FileNum) + f, v.err = c.fs.Open(filename, vfs.RandomReadsOption) if v.err == nil { cacheOpts := private.SSTableCacheOpts(c.cacheID, meta.FileNum).(sstable.ReaderOption) - v.reader, v.err = sstable.NewReader(f, c.opts, cacheOpts, c.filterMetrics) + reopenOpt := sstable.FileReopenOpt{FS: c.fs, Filename: filename} + v.reader, v.err = sstable.NewReader(f, c.opts, cacheOpts, c.filterMetrics, reopenOpt) } if v.err == nil { if meta.SmallestSeqNum == meta.LargestSeqNum { diff --git a/github.com/cockroachdb/pebble/table_stats.go b/github.com/cockroachdb/pebble/table_stats.go index 9d85f5bb69..c9156ef944 100644 --- a/github.com/cockroachdb/pebble/table_stats.go +++ b/github.com/cockroachdb/pebble/table_stats.go @@ -203,8 +203,9 @@ func (d *DB) loadNewFileStats(rs *readState, pending []manifest.NewFileEntry) ([ func (d *DB) scanReadStateTableStats(rs *readState, fill []collectedStats) ([]collectedStats, []deleteCompactionHint, bool) { moreRemain := false var hints []deleteCompactionHint - for l, ff := range rs.current.Levels { - for _, f := range ff { + for l, levelMetadata := range rs.current.Levels { + iter := levelMetadata.Iter() + for f := iter.First(); f != nil; f = iter.Next() { // NB: We're not holding d.mu which protects f.Stats, but only the // active stats collection job updates f.Stats for active files, // and we ensure only one goroutine runs it at a time through @@ -316,7 +317,7 @@ func (d *DB) estimateSizeBeneath( // additional I/O to read the file's index blocks. hintSeqNum = math.MaxUint64 for l := level + 1; l < numLevels; l++ { - iter := v.Overlaps(l, d.cmp, start, end) + iter := v.Overlaps(l, d.cmp, start, end).Iter() for file := iter.First(); file != nil; file = iter.Next() { if d.cmp(start, file.Smallest.UserKey) <= 0 && d.cmp(file.Largest.UserKey, end) <= 0 { diff --git a/github.com/cockroachdb/pebble/tool/db.go b/github.com/cockroachdb/pebble/tool/db.go index 40830fec55..10eb8b1484 100644 --- a/github.com/cockroachdb/pebble/tool/db.go +++ b/github.com/cockroachdb/pebble/tool/db.go @@ -389,8 +389,9 @@ func (d *dbT) runProperties(cmd *cobra.Command, args []string) { var total props var all []props for _, l := range v.Levels { + iter := l.Iter() var level props - for _, t := range l { + for t := iter.First(); t != nil; t = iter.Next() { err := d.addProps(dirname, t, &level) if err != nil { return err diff --git a/github.com/cockroachdb/pebble/tool/manifest.go b/github.com/cockroachdb/pebble/tool/manifest.go index 571e7335c3..7b3db6f5d9 100644 --- a/github.com/cockroachdb/pebble/tool/manifest.go +++ b/github.com/cockroachdb/pebble/tool/manifest.go @@ -79,7 +79,8 @@ Check the contents of the MANIFEST files. func (m *manifestT) printLevels(v *manifest.Version) { for level := range v.Levels { - if level == 0 && v.L0Sublevels != nil && len(v.Levels[level]) > 0 { + iter := v.Levels[level].Iter() + if level == 0 && v.L0Sublevels != nil && !iter.Empty() { for sublevel := len(v.L0Sublevels.Levels) - 1; sublevel >= 0; sublevel-- { fmt.Fprintf(stdout, "--- L0.%d ---\n", sublevel) for _, f := range v.L0Sublevels.Levels[sublevel] { @@ -92,8 +93,7 @@ func (m *manifestT) printLevels(v *manifest.Version) { continue } fmt.Fprintf(stdout, "--- L%d ---\n", level) - for j := range v.Levels[level] { - f := v.Levels[level][j] + for f := iter.First(); f != nil; f = iter.Next() { fmt.Fprintf(stdout, " %s:%d", f.FileNum, f.Size) formatSeqNumRange(stdout, f.SmallestSeqNum, f.LargestSeqNum) formatKeyRange(stdout, m.fmtKey, &f.Smallest, &f.Largest) diff --git a/github.com/cockroachdb/pebble/vfs/fadvise_generic.go b/github.com/cockroachdb/pebble/vfs/fadvise_generic.go index b7aa995700..22b3de8eff 100644 --- a/github.com/cockroachdb/pebble/vfs/fadvise_generic.go +++ b/github.com/cockroachdb/pebble/vfs/fadvise_generic.go @@ -9,3 +9,7 @@ package vfs func fadviseRandom(f uintptr) error { return nil } + +func fadviseSequential(f uintptr) error { + return nil +} diff --git a/github.com/cockroachdb/pebble/vfs/fadvise_linux.go b/github.com/cockroachdb/pebble/vfs/fadvise_linux.go index 05d611d255..50c6547572 100644 --- a/github.com/cockroachdb/pebble/vfs/fadvise_linux.go +++ b/github.com/cockroachdb/pebble/vfs/fadvise_linux.go @@ -12,3 +12,8 @@ import "golang.org/x/sys/unix" func fadviseRandom(f uintptr) error { return unix.Fadvise(int(f), 0, 0, unix.FADV_RANDOM) } + +// Calls Fadvise with FADV_SEQUENTIAL to enable readahead on a file descriptor. +func fadviseSequential(f uintptr) error { + return unix.Fadvise(int(f), 0, 0, unix.FADV_SEQUENTIAL) +} diff --git a/github.com/cockroachdb/pebble/vfs/prefetch_linux.go b/github.com/cockroachdb/pebble/vfs/prefetch_linux.go index 6dcf828e7a..8bd9477e31 100644 --- a/github.com/cockroachdb/pebble/vfs/prefetch_linux.go +++ b/github.com/cockroachdb/pebble/vfs/prefetch_linux.go @@ -6,7 +6,9 @@ package vfs -import "syscall" +import ( + "syscall" +) // Prefetch signals the OS (on supported platforms) to fetch the next size // bytes in file after offset into cache. Any subsequent reads in that range diff --git a/github.com/cockroachdb/pebble/vfs/vfs.go b/github.com/cockroachdb/pebble/vfs/vfs.go index 1f0a416c0d..17de88c953 100644 --- a/github.com/cockroachdb/pebble/vfs/vfs.go +++ b/github.com/cockroachdb/pebble/vfs/vfs.go @@ -191,13 +191,33 @@ type randomReadsOption struct{} // RandomReadsOption is an OpenOption that optimizes opened file handle for // random reads, by calling fadvise() with POSIX_FADV_RANDOM on Linux systems -// to disable readahead. Only works when specified to defaultFS. +// to disable readahead. var RandomReadsOption OpenOption = &randomReadsOption{} // Apply implements the OpenOption interface. func (randomReadsOption) Apply(f File) { - if osFile, ok := f.(*os.File); ok { - _ = fadviseRandom(osFile.Fd()) + type fd interface { + Fd() uintptr + } + if fdFile, ok := f.(fd); ok { + _ = fadviseRandom(fdFile.Fd()) + } +} + +type sequentialReadsOption struct{} + +// SequentialReadsOption is an OpenOption that optimizes opened file handle for +// sequential reads, by calling fadvise() with POSIX_FADV_SEQUENTIAL on Linux +// systems to enable readahead. +var SequentialReadsOption OpenOption = &sequentialReadsOption{} + +// Apply implements the OpenOption interface. +func (sequentialReadsOption) Apply(f File) { + type fd interface { + Fd() uintptr + } + if fdFile, ok := f.(fd); ok { + _ = fadviseSequential(fdFile.Fd()) } } diff --git a/modules.txt b/modules.txt index e829bddea5..cb0a6626e7 100644 --- a/modules.txt +++ b/modules.txt @@ -222,7 +222,7 @@ github.com/cockroachdb/gostdlib/x/tools/internal/span # github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f ## explicit github.com/cockroachdb/logtags -# github.com/cockroachdb/pebble v0.0.0-20200713154309-36c1b9c7a833 +# github.com/cockroachdb/pebble v0.0.0-20200716211707-073e15ee2ccf ## explicit github.com/cockroachdb/pebble github.com/cockroachdb/pebble/bloom From cbc16420a69950da6486949f6d6d4aaad9dfd1c2 Mon Sep 17 00:00:00 2001 From: Raphael 'kena' Poss Date: Mon, 20 Jul 2020 09:24:32 +0200 Subject: [PATCH 14/49] bump gopsutil --- github.com/shirou/gopsutil/cpu/cpu.go | 22 +- github.com/shirou/gopsutil/cpu/cpu_darwin.go | 116 +- .../shirou/gopsutil/cpu/cpu_dragonfly.go | 161 + .../gopsutil/cpu/cpu_dragonfly_amd64.go | 9 + .../shirou/gopsutil/cpu/cpu_fallback.go | 7 +- github.com/shirou/gopsutil/cpu/cpu_freebsd.go | 7 +- .../shirou/gopsutil/cpu/cpu_freebsd_arm.go | 9 + .../shirou/gopsutil/cpu/cpu_freebsd_arm64.go | 9 + github.com/shirou/gopsutil/cpu/cpu_linux.go | 96 +- github.com/shirou/gopsutil/cpu/cpu_openbsd.go | 112 +- github.com/shirou/gopsutil/cpu/cpu_solaris.go | 100 +- github.com/shirou/gopsutil/cpu/cpu_windows.go | 156 +- .../shirou/gopsutil/disk/disk_darwin.go | 72 +- github.com/shirou/gopsutil/disk/disk_darwin.h | 164 - .../shirou/gopsutil/disk/disk_darwin_386.go | 59 - .../shirou/gopsutil/disk/disk_darwin_amd64.go | 58 - .../shirou/gopsutil/disk/disk_darwin_arm64.go | 58 - .../shirou/gopsutil/disk/disk_darwin_cgo.go | 78 +- .../shirou/gopsutil/disk/disk_freebsd.go | 76 +- .../shirou/gopsutil/disk/disk_freebsd_386.go | 50 - .../gopsutil/disk/disk_freebsd_amd64.go | 50 - .../shirou/gopsutil/disk/disk_freebsd_arm.go | 62 + .../gopsutil/disk/disk_freebsd_arm64.go | 65 + github.com/shirou/gopsutil/disk/disk_linux.go | 97 +- .../shirou/gopsutil/disk/disk_openbsd.go | 49 +- .../shirou/gopsutil/disk/disk_openbsd_386.go | 35 + .../gopsutil/disk/disk_openbsd_amd64.go | 57 +- github.com/shirou/gopsutil/disk/disk_unix.go | 2 +- .../shirou/gopsutil/disk/disk_windows.go | 95 +- .../shirou/gopsutil/disk/iostat_darwin.c | 131 + .../shirou/gopsutil/disk/iostat_darwin.h | 33 + .../gopsutil/host/freebsd_headers/utxdb.h | 63 + github.com/shirou/gopsutil/host/host.go | 1 + .../shirou/gopsutil/host/host_darwin.go | 73 +- .../shirou/gopsutil/host/host_darwin_cgo.go | 51 + .../shirou/gopsutil/host/host_darwin_nocgo.go | 18 + .../shirou/gopsutil/host/host_freebsd.go | 10 +- .../shirou/gopsutil/host/host_freebsd_386.go | 18 +- .../gopsutil/host/host_freebsd_amd64.go | 19 +- .../shirou/gopsutil/host/host_freebsd_arm.go | 19 +- .../gopsutil/host/host_freebsd_arm64.go | 39 + github.com/shirou/gopsutil/host/host_linux.go | 268 +- .../gopsutil/host/host_linux_riscv64.go | 47 + .../shirou/gopsutil/host/host_openbsd.go | 40 +- github.com/shirou/gopsutil/host/host_posix.go | 15 + .../shirou/gopsutil/host/host_solaris.go | 15 +- .../shirou/gopsutil/host/host_windows.go | 62 +- github.com/shirou/gopsutil/host/smc_darwin.c | 170 ++ github.com/shirou/gopsutil/host/smc_darwin.h | 32 + github.com/shirou/gopsutil/host/types.go | 25 + .../shirou/gopsutil/internal/common/common.go | 60 +- .../gopsutil/internal/common/common_darwin.go | 2 +- .../internal/common/common_freebsd.go | 18 +- .../gopsutil/internal/common/common_linux.go | 228 +- .../internal/common/common_openbsd.go | 2 +- .../gopsutil/internal/common/common_unix.go | 2 +- .../internal/common/common_windows.go | 51 +- github.com/shirou/gopsutil/load/load.go | 1 + .../shirou/gopsutil/load/load_darwin.go | 33 +- github.com/shirou/gopsutil/load/load_linux.go | 52 +- github.com/shirou/gopsutil/mem/mem.go | 4 + github.com/shirou/gopsutil/mem/mem_darwin.go | 47 +- github.com/shirou/gopsutil/mem/mem_freebsd.go | 79 +- github.com/shirou/gopsutil/mem/mem_linux.go | 130 +- github.com/shirou/gopsutil/mem/mem_solaris.go | 6 +- github.com/shirou/gopsutil/mem/mem_windows.go | 2 +- github.com/shirou/gopsutil/net/net.go | 186 +- github.com/shirou/gopsutil/net/net_aix.go | 425 +++ github.com/shirou/gopsutil/net/net_darwin.go | 15 +- .../shirou/gopsutil/net/net_fallback.go | 45 +- github.com/shirou/gopsutil/net/net_freebsd.go | 12 +- github.com/shirou/gopsutil/net/net_linux.go | 147 +- github.com/shirou/gopsutil/net/net_openbsd.go | 10 +- github.com/shirou/gopsutil/net/net_unix.go | 128 + github.com/shirou/gopsutil/net/net_windows.go | 171 +- github.com/shirou/gopsutil/process/process.go | 86 +- .../shirou/gopsutil/process/process_darwin.go | 125 +- .../gopsutil/process/process_darwin_cgo.go | 30 + .../gopsutil/process/process_darwin_nocgo.go | 34 + .../gopsutil/process/process_fallback.go | 47 +- .../gopsutil/process/process_freebsd.go | 60 +- .../gopsutil/process/process_freebsd_arm64.go | 201 ++ .../shirou/gopsutil/process/process_linux.go | 243 +- .../gopsutil/process/process_openbsd.go | 62 +- .../shirou/gopsutil/process/process_posix.go | 45 + .../gopsutil/process/process_windows.go | 578 ++-- .../gopsutil/process/process_windows_386.go | 86 + .../gopsutil/process/process_windows_amd64.go | 60 + github.com/shirou/w32/AUTHORS | 16 - github.com/shirou/w32/LICENSE | 23 - github.com/shirou/w32/README.md | 33 - github.com/shirou/w32/advapi32.go | 301 -- github.com/shirou/w32/comctl32.go | 111 - github.com/shirou/w32/comdlg32.go | 40 - github.com/shirou/w32/constants.go | 2661 ----------------- github.com/shirou/w32/dwmapi.go | 256 -- github.com/shirou/w32/gdi32.go | 511 ---- github.com/shirou/w32/gdiplus.go | 177 -- github.com/shirou/w32/idispatch.go | 45 - github.com/shirou/w32/istream.go | 33 - github.com/shirou/w32/iunknown.go | 29 - github.com/shirou/w32/kernel32.go | 316 -- github.com/shirou/w32/ole32.go | 65 - github.com/shirou/w32/oleaut32.go | 50 - github.com/shirou/w32/opengl32.go | 74 - github.com/shirou/w32/psapi.go | 27 - github.com/shirou/w32/shell32.go | 155 - github.com/shirou/w32/typedef.go | 901 ------ github.com/shirou/w32/user32.go | 950 ------ github.com/shirou/w32/utils.go | 203 -- github.com/shirou/w32/vars.go | 13 - modules.txt | 5 +- 112 files changed, 4654 insertions(+), 8934 deletions(-) create mode 100644 github.com/shirou/gopsutil/cpu/cpu_dragonfly.go create mode 100644 github.com/shirou/gopsutil/cpu/cpu_dragonfly_amd64.go create mode 100644 github.com/shirou/gopsutil/cpu/cpu_freebsd_arm.go create mode 100644 github.com/shirou/gopsutil/cpu/cpu_freebsd_arm64.go delete mode 100644 github.com/shirou/gopsutil/disk/disk_darwin.h delete mode 100644 github.com/shirou/gopsutil/disk/disk_darwin_386.go delete mode 100644 github.com/shirou/gopsutil/disk/disk_darwin_amd64.go delete mode 100644 github.com/shirou/gopsutil/disk/disk_darwin_arm64.go create mode 100644 github.com/shirou/gopsutil/disk/disk_freebsd_arm.go create mode 100644 github.com/shirou/gopsutil/disk/disk_freebsd_arm64.go create mode 100644 github.com/shirou/gopsutil/disk/disk_openbsd_386.go create mode 100644 github.com/shirou/gopsutil/disk/iostat_darwin.c create mode 100644 github.com/shirou/gopsutil/disk/iostat_darwin.h create mode 100644 github.com/shirou/gopsutil/host/freebsd_headers/utxdb.h create mode 100644 github.com/shirou/gopsutil/host/host_darwin_cgo.go create mode 100644 github.com/shirou/gopsutil/host/host_darwin_nocgo.go create mode 100644 github.com/shirou/gopsutil/host/host_freebsd_arm64.go create mode 100644 github.com/shirou/gopsutil/host/host_linux_riscv64.go create mode 100644 github.com/shirou/gopsutil/host/host_posix.go create mode 100644 github.com/shirou/gopsutil/host/smc_darwin.c create mode 100644 github.com/shirou/gopsutil/host/smc_darwin.h create mode 100644 github.com/shirou/gopsutil/host/types.go create mode 100644 github.com/shirou/gopsutil/net/net_aix.go create mode 100644 github.com/shirou/gopsutil/process/process_darwin_cgo.go create mode 100644 github.com/shirou/gopsutil/process/process_darwin_nocgo.go create mode 100644 github.com/shirou/gopsutil/process/process_freebsd_arm64.go delete mode 100644 github.com/shirou/w32/AUTHORS delete mode 100644 github.com/shirou/w32/LICENSE delete mode 100644 github.com/shirou/w32/README.md delete mode 100644 github.com/shirou/w32/advapi32.go delete mode 100644 github.com/shirou/w32/comctl32.go delete mode 100644 github.com/shirou/w32/comdlg32.go delete mode 100644 github.com/shirou/w32/constants.go delete mode 100644 github.com/shirou/w32/dwmapi.go delete mode 100644 github.com/shirou/w32/gdi32.go delete mode 100644 github.com/shirou/w32/gdiplus.go delete mode 100644 github.com/shirou/w32/idispatch.go delete mode 100644 github.com/shirou/w32/istream.go delete mode 100644 github.com/shirou/w32/iunknown.go delete mode 100644 github.com/shirou/w32/kernel32.go delete mode 100644 github.com/shirou/w32/ole32.go delete mode 100644 github.com/shirou/w32/oleaut32.go delete mode 100644 github.com/shirou/w32/opengl32.go delete mode 100644 github.com/shirou/w32/psapi.go delete mode 100644 github.com/shirou/w32/shell32.go delete mode 100644 github.com/shirou/w32/typedef.go delete mode 100644 github.com/shirou/w32/user32.go delete mode 100644 github.com/shirou/w32/utils.go delete mode 100644 github.com/shirou/w32/vars.go diff --git a/github.com/shirou/gopsutil/cpu/cpu.go b/github.com/shirou/gopsutil/cpu/cpu.go index ceaf77feeb..d3ea1f245c 100644 --- a/github.com/shirou/gopsutil/cpu/cpu.go +++ b/github.com/shirou/gopsutil/cpu/cpu.go @@ -4,7 +4,7 @@ import ( "context" "encoding/json" "fmt" - "runtime" + "math" "strconv" "strings" "sync" @@ -14,8 +14,7 @@ import ( ) // TimesStat contains the amounts of time the CPU has spent performing different -// kinds of work. Time units are in USER_HZ or Jiffies (typically hundredths of -// a second). It is based on linux /proc/stat file. +// kinds of work. Time units are in seconds. It is based on linux /proc/stat file. type TimesStat struct { CPU string `json:"cpu"` User float64 `json:"user"` @@ -28,7 +27,6 @@ type TimesStat struct { Steal float64 `json:"steal"` Guest float64 `json:"guest"` GuestNice float64 `json:"guestNice"` - Stolen float64 `json:"stolen"` } type InfoStat struct { @@ -63,14 +61,11 @@ func init() { lastCPUPercent.Unlock() } +// Counts returns the number of physical or logical cores in the system func Counts(logical bool) (int, error) { return CountsWithContext(context.Background(), logical) } -func CountsWithContext(ctx context.Context, logical bool) (int, error) { - return runtime.NumCPU(), nil -} - func (c TimesStat) String() string { v := []string{ `"cpu":"` + c.CPU + `"`, @@ -84,7 +79,6 @@ func (c TimesStat) String() string { `"steal":` + strconv.FormatFloat(c.Steal, 'f', 1, 64), `"guest":` + strconv.FormatFloat(c.Guest, 'f', 1, 64), `"guestNice":` + strconv.FormatFloat(c.GuestNice, 'f', 1, 64), - `"stolen":` + strconv.FormatFloat(c.Stolen, 'f', 1, 64), } return `{` + strings.Join(v, ",") + `}` @@ -92,8 +86,8 @@ func (c TimesStat) String() string { // Total returns the total number of seconds in a CPUTimesStat func (c TimesStat) Total() float64 { - total := c.User + c.System + c.Nice + c.Iowait + c.Irq + c.Softirq + c.Steal + - c.Guest + c.GuestNice + c.Idle + c.Stolen + total := c.User + c.System + c.Nice + c.Iowait + c.Irq + c.Softirq + + c.Steal + c.Idle return total } @@ -104,7 +98,7 @@ func (c InfoStat) String() string { func getAllBusy(t TimesStat) (float64, float64) { busy := t.User + t.System + t.Nice + t.Iowait + t.Irq + - t.Softirq + t.Steal + t.Guest + t.GuestNice + t.Stolen + t.Softirq + t.Steal return busy + t.Idle, busy } @@ -116,9 +110,9 @@ func calculateBusy(t1, t2 TimesStat) float64 { return 0 } if t2All <= t1All { - return 1 + return 100 } - return (t2Busy - t1Busy) / (t2All - t1All) * 100 + return math.Min(100, math.Max(0, (t2Busy-t1Busy)/(t2All-t1All)*100)) } func calculateAllBusy(t1, t2 []TimesStat) ([]float64, error) { diff --git a/github.com/shirou/gopsutil/cpu/cpu_darwin.go b/github.com/shirou/gopsutil/cpu/cpu_darwin.go index 74d2737317..3d3455ee68 100644 --- a/github.com/shirou/gopsutil/cpu/cpu_darwin.go +++ b/github.com/shirou/gopsutil/cpu/cpu_darwin.go @@ -7,6 +7,8 @@ import ( "os/exec" "strconv" "strings" + + "golang.org/x/sys/unix" ) // sys/resource.h @@ -22,6 +24,21 @@ const ( // default value. from time.h var ClocksPerSec = float64(128) +func init() { + getconf, err := exec.LookPath("getconf") + if err != nil { + return + } + out, err := invoke.Command(getconf, "CLK_TCK") + // ignore errors + if err == nil { + i, err := strconv.ParseFloat(strings.TrimSpace(string(out)), 64) + if err == nil { + ClocksPerSec = float64(i) + } + } +} + func Times(percpu bool) ([]TimesStat, error) { return TimesWithContext(context.Background(), percpu) } @@ -41,75 +58,62 @@ func Info() ([]InfoStat, error) { func InfoWithContext(ctx context.Context) ([]InfoStat, error) { var ret []InfoStat - sysctl, err := exec.LookPath("/usr/sbin/sysctl") - if err != nil { - return ret, err - } - out, err := invoke.CommandWithContext(ctx, sysctl, "machdep.cpu") - if err != nil { - return ret, err - } c := InfoStat{} - for _, line := range strings.Split(string(out), "\n") { - values := strings.Fields(line) - if len(values) < 1 { - continue + c.ModelName, _ = unix.Sysctl("machdep.cpu.brand_string") + family, _ := unix.SysctlUint32("machdep.cpu.family") + c.Family = strconv.FormatUint(uint64(family), 10) + model, _ := unix.SysctlUint32("machdep.cpu.model") + c.Model = strconv.FormatUint(uint64(model), 10) + stepping, _ := unix.SysctlUint32("machdep.cpu.stepping") + c.Stepping = int32(stepping) + features, err := unix.Sysctl("machdep.cpu.features") + if err == nil { + for _, v := range strings.Fields(features) { + c.Flags = append(c.Flags, strings.ToLower(v)) } - - t, err := strconv.ParseInt(values[1], 10, 64) - // err is not checked here because some value is string. - if strings.HasPrefix(line, "machdep.cpu.brand_string") { - c.ModelName = strings.Join(values[1:], " ") - } else if strings.HasPrefix(line, "machdep.cpu.family") { - c.Family = values[1] - } else if strings.HasPrefix(line, "machdep.cpu.model") { - c.Model = values[1] - } else if strings.HasPrefix(line, "machdep.cpu.stepping") { - if err != nil { - return ret, err - } - c.Stepping = int32(t) - } else if strings.HasPrefix(line, "machdep.cpu.features") { - for _, v := range values[1:] { - c.Flags = append(c.Flags, strings.ToLower(v)) - } - } else if strings.HasPrefix(line, "machdep.cpu.leaf7_features") { - for _, v := range values[1:] { - c.Flags = append(c.Flags, strings.ToLower(v)) - } - } else if strings.HasPrefix(line, "machdep.cpu.extfeatures") { - for _, v := range values[1:] { - c.Flags = append(c.Flags, strings.ToLower(v)) - } - } else if strings.HasPrefix(line, "machdep.cpu.core_count") { - if err != nil { - return ret, err - } - c.Cores = int32(t) - } else if strings.HasPrefix(line, "machdep.cpu.cache.size") { - if err != nil { - return ret, err - } - c.CacheSize = int32(t) - } else if strings.HasPrefix(line, "machdep.cpu.vendor") { - c.VendorID = values[1] + } + leaf7Features, err := unix.Sysctl("machdep.cpu.leaf7_features") + if err == nil { + for _, v := range strings.Fields(leaf7Features) { + c.Flags = append(c.Flags, strings.ToLower(v)) } } + extfeatures, err := unix.Sysctl("machdep.cpu.extfeatures") + if err == nil { + for _, v := range strings.Fields(extfeatures) { + c.Flags = append(c.Flags, strings.ToLower(v)) + } + } + cores, _ := unix.SysctlUint32("machdep.cpu.core_count") + c.Cores = int32(cores) + cacheSize, _ := unix.SysctlUint32("machdep.cpu.cache.size") + c.CacheSize = int32(cacheSize) + c.VendorID, _ = unix.Sysctl("machdep.cpu.vendor") // Use the rated frequency of the CPU. This is a static value and does not // account for low power or Turbo Boost modes. - out, err = invoke.CommandWithContext(ctx, sysctl, "hw.cpufrequency") + cpuFrequency, err := unix.SysctlUint64("hw.cpufrequency") if err != nil { return ret, err } + c.Mhz = float64(cpuFrequency) / 1000000.0 + + return append(ret, c), nil +} + +func CountsWithContext(ctx context.Context, logical bool) (int, error) { + var cpuArgument string + if logical { + cpuArgument = "hw.logicalcpu" + } else { + cpuArgument = "hw.physicalcpu" + } - values := strings.Fields(string(out)) - hz, err := strconv.ParseFloat(values[1], 64) + count, err := unix.SysctlUint32(cpuArgument) if err != nil { - return ret, err + return 0, err } - c.Mhz = hz / 1000000.0 - return append(ret, c), nil + return int(count), nil } diff --git a/github.com/shirou/gopsutil/cpu/cpu_dragonfly.go b/github.com/shirou/gopsutil/cpu/cpu_dragonfly.go new file mode 100644 index 0000000000..eed5beab78 --- /dev/null +++ b/github.com/shirou/gopsutil/cpu/cpu_dragonfly.go @@ -0,0 +1,161 @@ +package cpu + +import ( + "context" + "fmt" + "os/exec" + "reflect" + "regexp" + "runtime" + "strconv" + "strings" + "unsafe" + + "github.com/shirou/gopsutil/internal/common" + "golang.org/x/sys/unix" +) + +var ClocksPerSec = float64(128) +var cpuMatch = regexp.MustCompile(`^CPU:`) +var originMatch = regexp.MustCompile(`Origin\s*=\s*"(.+)"\s+Id\s*=\s*(.+)\s+Stepping\s*=\s*(.+)`) +var featuresMatch = regexp.MustCompile(`Features=.+<(.+)>`) +var featuresMatch2 = regexp.MustCompile(`Features2=[a-f\dx]+<(.+)>`) +var cpuEnd = regexp.MustCompile(`^Trying to mount root`) +var cpuTimesSize int +var emptyTimes cpuTimes + +func init() { + getconf, err := exec.LookPath("getconf") + if err != nil { + return + } + out, err := invoke.Command(getconf, "CLK_TCK") + // ignore errors + if err == nil { + i, err := strconv.ParseFloat(strings.TrimSpace(string(out)), 64) + if err == nil { + ClocksPerSec = float64(i) + } + } +} + +func timeStat(name string, t *cpuTimes) *TimesStat { + return &TimesStat{ + User: float64(t.User) / ClocksPerSec, + Nice: float64(t.Nice) / ClocksPerSec, + System: float64(t.Sys) / ClocksPerSec, + Idle: float64(t.Idle) / ClocksPerSec, + Irq: float64(t.Intr) / ClocksPerSec, + CPU: name, + } +} + +func Times(percpu bool) ([]TimesStat, error) { + return TimesWithContext(context.Background(), percpu) +} + +func TimesWithContext(ctx context.Context, percpu bool) ([]TimesStat, error) { + if percpu { + buf, err := unix.SysctlRaw("kern.cp_times") + if err != nil { + return nil, err + } + + // We can't do this in init due to the conflict with cpu.init() + if cpuTimesSize == 0 { + cpuTimesSize = int(reflect.TypeOf(cpuTimes{}).Size()) + } + + ncpus := len(buf) / cpuTimesSize + ret := make([]TimesStat, 0, ncpus) + for i := 0; i < ncpus; i++ { + times := (*cpuTimes)(unsafe.Pointer(&buf[i*cpuTimesSize])) + if *times == emptyTimes { + // CPU not present + continue + } + ret = append(ret, *timeStat(fmt.Sprintf("cpu%d", len(ret)), times)) + } + return ret, nil + } + + buf, err := unix.SysctlRaw("kern.cp_time") + if err != nil { + return nil, err + } + + times := (*cpuTimes)(unsafe.Pointer(&buf[0])) + return []TimesStat{*timeStat("cpu-total", times)}, nil +} + +// Returns only one InfoStat on DragonflyBSD. The information regarding core +// count, however is accurate and it is assumed that all InfoStat attributes +// are the same across CPUs. +func Info() ([]InfoStat, error) { + return InfoWithContext(context.Background()) +} + +func InfoWithContext(ctx context.Context) ([]InfoStat, error) { + const dmesgBoot = "/var/run/dmesg.boot" + + c, err := parseDmesgBoot(dmesgBoot) + if err != nil { + return nil, err + } + + var u32 uint32 + if u32, err = unix.SysctlUint32("hw.clockrate"); err != nil { + return nil, err + } + c.Mhz = float64(u32) + + var num int + var buf string + if buf, err = unix.Sysctl("hw.cpu_topology.tree"); err != nil { + return nil, err + } + num = strings.Count(buf, "CHIP") + c.Cores = int32(strings.Count(string(buf), "CORE") / num) + + if c.ModelName, err = unix.Sysctl("hw.model"); err != nil { + return nil, err + } + + ret := make([]InfoStat, num) + for i := 0; i < num; i++ { + ret[i] = c + } + + return ret, nil +} + +func parseDmesgBoot(fileName string) (InfoStat, error) { + c := InfoStat{} + lines, _ := common.ReadLines(fileName) + for _, line := range lines { + if matches := cpuEnd.FindStringSubmatch(line); matches != nil { + break + } else if matches := originMatch.FindStringSubmatch(line); matches != nil { + c.VendorID = matches[1] + t, err := strconv.ParseInt(matches[2], 10, 32) + if err != nil { + return c, fmt.Errorf("unable to parse DragonflyBSD CPU stepping information from %q: %v", line, err) + } + c.Stepping = int32(t) + } else if matches := featuresMatch.FindStringSubmatch(line); matches != nil { + for _, v := range strings.Split(matches[1], ",") { + c.Flags = append(c.Flags, strings.ToLower(v)) + } + } else if matches := featuresMatch2.FindStringSubmatch(line); matches != nil { + for _, v := range strings.Split(matches[1], ",") { + c.Flags = append(c.Flags, strings.ToLower(v)) + } + } + } + + return c, nil +} + +func CountsWithContext(ctx context.Context, logical bool) (int, error) { + return runtime.NumCPU(), nil +} diff --git a/github.com/shirou/gopsutil/cpu/cpu_dragonfly_amd64.go b/github.com/shirou/gopsutil/cpu/cpu_dragonfly_amd64.go new file mode 100644 index 0000000000..57e14528db --- /dev/null +++ b/github.com/shirou/gopsutil/cpu/cpu_dragonfly_amd64.go @@ -0,0 +1,9 @@ +package cpu + +type cpuTimes struct { + User uint64 + Nice uint64 + Sys uint64 + Intr uint64 + Idle uint64 +} diff --git a/github.com/shirou/gopsutil/cpu/cpu_fallback.go b/github.com/shirou/gopsutil/cpu/cpu_fallback.go index e9e7ada2ce..5551c49d1d 100644 --- a/github.com/shirou/gopsutil/cpu/cpu_fallback.go +++ b/github.com/shirou/gopsutil/cpu/cpu_fallback.go @@ -1,9 +1,10 @@ -// +build !darwin,!linux,!freebsd,!openbsd,!solaris,!windows +// +build !darwin,!linux,!freebsd,!openbsd,!solaris,!windows,!dragonfly package cpu import ( "context" + "runtime" "github.com/shirou/gopsutil/internal/common" ) @@ -23,3 +24,7 @@ func Info() ([]InfoStat, error) { func InfoWithContext(ctx context.Context) ([]InfoStat, error) { return []InfoStat{}, common.ErrNotImplementedError } + +func CountsWithContext(ctx context.Context, logical bool) (int, error) { + return runtime.NumCPU(), nil +} diff --git a/github.com/shirou/gopsutil/cpu/cpu_freebsd.go b/github.com/shirou/gopsutil/cpu/cpu_freebsd.go index b6c7186c7b..57beffae11 100644 --- a/github.com/shirou/gopsutil/cpu/cpu_freebsd.go +++ b/github.com/shirou/gopsutil/cpu/cpu_freebsd.go @@ -6,6 +6,7 @@ import ( "os/exec" "reflect" "regexp" + "runtime" "strconv" "strings" "unsafe" @@ -25,7 +26,7 @@ var cpuTimesSize int var emptyTimes cpuTimes func init() { - getconf, err := exec.LookPath("/usr/bin/getconf") + getconf, err := exec.LookPath("getconf") if err != nil { return } @@ -166,3 +167,7 @@ func parseDmesgBoot(fileName string) (InfoStat, int, error) { return c, cpuNum, nil } + +func CountsWithContext(ctx context.Context, logical bool) (int, error) { + return runtime.NumCPU(), nil +} diff --git a/github.com/shirou/gopsutil/cpu/cpu_freebsd_arm.go b/github.com/shirou/gopsutil/cpu/cpu_freebsd_arm.go new file mode 100644 index 0000000000..8b7f4c321e --- /dev/null +++ b/github.com/shirou/gopsutil/cpu/cpu_freebsd_arm.go @@ -0,0 +1,9 @@ +package cpu + +type cpuTimes struct { + User uint32 + Nice uint32 + Sys uint32 + Intr uint32 + Idle uint32 +} diff --git a/github.com/shirou/gopsutil/cpu/cpu_freebsd_arm64.go b/github.com/shirou/gopsutil/cpu/cpu_freebsd_arm64.go new file mode 100644 index 0000000000..57e14528db --- /dev/null +++ b/github.com/shirou/gopsutil/cpu/cpu_freebsd_arm64.go @@ -0,0 +1,9 @@ +package cpu + +type cpuTimes struct { + User uint64 + Nice uint64 + Sys uint64 + Intr uint64 + Idle uint64 +} diff --git a/github.com/shirou/gopsutil/cpu/cpu_linux.go b/github.com/shirou/gopsutil/cpu/cpu_linux.go index 2fffe85b99..735bd29ed1 100644 --- a/github.com/shirou/gopsutil/cpu/cpu_linux.go +++ b/github.com/shirou/gopsutil/cpu/cpu_linux.go @@ -13,10 +13,10 @@ import ( "github.com/shirou/gopsutil/internal/common" ) -var CPUTick = float64(100) +var ClocksPerSec = float64(100) func init() { - getconf, err := exec.LookPath("/usr/bin/getconf") + getconf, err := exec.LookPath("getconf") if err != nil { return } @@ -25,7 +25,7 @@ func init() { if err == nil { i, err := strconv.ParseFloat(strings.TrimSpace(string(out)), 64) if err == nil { - CPUTick = i + ClocksPerSec = i } } } @@ -87,7 +87,7 @@ func finishCPUInfo(c *InfoStat) error { lines, err = common.ReadLines(sysCPUPath(c.CPU, "cpufreq/cpuinfo_max_freq")) // if we encounter errors below such as there are no cpuinfo_max_freq file, // we just ignore. so let Mhz is 0. - if err != nil { + if err != nil || len(lines) == 0 { return nil } value, err = strconv.ParseFloat(lines[0], 64) @@ -250,35 +250,103 @@ func parseStatLine(line string) (*TimesStat, error) { ct := &TimesStat{ CPU: cpu, - User: user / CPUTick, - Nice: nice / CPUTick, - System: system / CPUTick, - Idle: idle / CPUTick, - Iowait: iowait / CPUTick, - Irq: irq / CPUTick, - Softirq: softirq / CPUTick, + User: user / ClocksPerSec, + Nice: nice / ClocksPerSec, + System: system / ClocksPerSec, + Idle: idle / ClocksPerSec, + Iowait: iowait / ClocksPerSec, + Irq: irq / ClocksPerSec, + Softirq: softirq / ClocksPerSec, } if len(fields) > 8 { // Linux >= 2.6.11 steal, err := strconv.ParseFloat(fields[8], 64) if err != nil { return nil, err } - ct.Steal = steal / CPUTick + ct.Steal = steal / ClocksPerSec } if len(fields) > 9 { // Linux >= 2.6.24 guest, err := strconv.ParseFloat(fields[9], 64) if err != nil { return nil, err } - ct.Guest = guest / CPUTick + ct.Guest = guest / ClocksPerSec } if len(fields) > 10 { // Linux >= 3.2.0 guestNice, err := strconv.ParseFloat(fields[10], 64) if err != nil { return nil, err } - ct.GuestNice = guestNice / CPUTick + ct.GuestNice = guestNice / ClocksPerSec } return ct, nil } + +func CountsWithContext(ctx context.Context, logical bool) (int, error) { + if logical { + ret := 0 + // https://github.com/giampaolo/psutil/blob/d01a9eaa35a8aadf6c519839e987a49d8be2d891/psutil/_pslinux.py#L599 + procCpuinfo := common.HostProc("cpuinfo") + lines, err := common.ReadLines(procCpuinfo) + if err == nil { + for _, line := range lines { + line = strings.ToLower(line) + if strings.HasPrefix(line, "processor") { + ret++ + } + } + } + if ret == 0 { + procStat := common.HostProc("stat") + lines, err = common.ReadLines(procStat) + if err != nil { + return 0, err + } + for _, line := range lines { + if len(line) >= 4 && strings.HasPrefix(line, "cpu") && '0' <= line[3] && line[3] <= '9' { // `^cpu\d` regexp matching + ret++ + } + } + } + return ret, nil + } + // physical cores https://github.com/giampaolo/psutil/blob/d01a9eaa35a8aadf6c519839e987a49d8be2d891/psutil/_pslinux.py#L628 + filename := common.HostProc("cpuinfo") + lines, err := common.ReadLines(filename) + if err != nil { + return 0, err + } + mapping := make(map[int]int) + currentInfo := make(map[string]int) + for _, line := range lines { + line = strings.ToLower(strings.TrimSpace(line)) + if line == "" { + // new section + id, okID := currentInfo["physical id"] + cores, okCores := currentInfo["cpu cores"] + if okID && okCores { + mapping[id] = cores + } + currentInfo = make(map[string]int) + continue + } + fields := strings.Split(line, ":") + if len(fields) < 2 { + continue + } + fields[0] = strings.TrimSpace(fields[0]) + if fields[0] == "physical id" || fields[0] == "cpu cores" { + val, err := strconv.Atoi(strings.TrimSpace(fields[1])) + if err != nil { + continue + } + currentInfo[fields[0]] = val + } + } + ret := 0 + for _, v := range mapping { + ret += v + } + return ret, nil +} diff --git a/github.com/shirou/gopsutil/cpu/cpu_openbsd.go b/github.com/shirou/gopsutil/cpu/cpu_openbsd.go index 82b920f668..92a8bd75c9 100644 --- a/github.com/shirou/gopsutil/cpu/cpu_openbsd.go +++ b/github.com/shirou/gopsutil/cpu/cpu_openbsd.go @@ -8,15 +8,17 @@ import ( "encoding/binary" "fmt" "os/exec" + "runtime" "strconv" "strings" + "syscall" "github.com/shirou/gopsutil/internal/common" "golang.org/x/sys/unix" ) // sys/sched.h -const ( +var ( CPUser = 0 CPNice = 1 CPSys = 2 @@ -28,6 +30,9 @@ const ( // sys/sysctl.h const ( CTLKern = 1 // "high kernel": proc, limits + CTLHw = 6 // CTL_HW + SMT = 24 // HW_SMT + NCpuOnline = 25 // HW_NCPUONLINE KernCptime = 40 // KERN_CPTIME KernCptime2 = 71 // KERN_CPTIME2 ) @@ -35,18 +40,52 @@ const ( var ClocksPerSec = float64(128) func init() { - getconf, err := exec.LookPath("/usr/bin/getconf") - if err != nil { - return - } - out, err := invoke.Command(getconf, "CLK_TCK") - // ignore errors - if err == nil { - i, err := strconv.ParseFloat(strings.TrimSpace(string(out)), 64) + func() { + getconf, err := exec.LookPath("getconf") + if err != nil { + return + } + out, err := invoke.Command(getconf, "CLK_TCK") + // ignore errors if err == nil { - ClocksPerSec = float64(i) + i, err := strconv.ParseFloat(strings.TrimSpace(string(out)), 64) + if err == nil { + ClocksPerSec = float64(i) + } + } + }() + func() { + v, err := unix.Sysctl("kern.osrelease") // can't reuse host.PlatformInformation because of circular import + if err != nil { + return + } + v = strings.ToLower(v) + version, err := strconv.ParseFloat(v, 64) + if err != nil { + return + } + if version >= 6.4 { + CPIntr = 4 + CPIdle = 5 + CPUStates = 6 } + }() +} + +func smt() (bool, error) { + mib := []int32{CTLHw, SMT} + buf, _, err := common.CallSyscall(mib) + if err != nil { + return false, err + } + + var ret bool + br := bytes.NewReader(buf) + if err := binary.Read(br, binary.LittleEndian, &ret); err != nil { + return false, err } + + return ret, nil } func Times(percpu bool) ([]TimesStat, error) { @@ -63,13 +102,27 @@ func TimesWithContext(ctx context.Context, percpu bool) ([]TimesStat, error) { ncpu = 1 } + smt, err := smt() + if err == syscall.EOPNOTSUPP { + // if hw.smt is not applicable for this platform (e.g. i386), + // pretend it's enabled + smt = true + } else if err != nil { + return nil, err + } + for i := 0; i < ncpu; i++ { - var cpuTimes [CPUStates]int64 + j := i + if !smt { + j *= 2 + } + + var cpuTimes = make([]int32, CPUStates) var mib []int32 if percpu { - mib = []int32{CTLKern, KernCptime} + mib = []int32{CTLKern, KernCptime2, int32(j)} } else { - mib = []int32{CTLKern, KernCptime2, int32(i)} + mib = []int32{CTLKern, KernCptime} } buf, _, err := common.CallSyscall(mib) if err != nil { @@ -88,10 +141,10 @@ func TimesWithContext(ctx context.Context, percpu bool) ([]TimesStat, error) { Idle: float64(cpuTimes[CPIdle]) / ClocksPerSec, Irq: float64(cpuTimes[CPIntr]) / ClocksPerSec, } - if !percpu { - c.CPU = "cpu-total" + if percpu { + c.CPU = fmt.Sprintf("cpu%d", j) } else { - c.CPU = fmt.Sprintf("cpu%d", i) + c.CPU = "cpu-total" } ret = append(ret, c) } @@ -106,14 +159,37 @@ func Info() ([]InfoStat, error) { func InfoWithContext(ctx context.Context) ([]InfoStat, error) { var ret []InfoStat + var err error c := InfoStat{} - v, err := unix.Sysctl("hw.model") + var u32 uint32 + if u32, err = unix.SysctlUint32("hw.cpuspeed"); err != nil { + return nil, err + } + c.Mhz = float64(u32) + + mib := []int32{CTLHw, NCpuOnline} + buf, _, err := common.CallSyscall(mib) if err != nil { return nil, err } - c.ModelName = v + + var ncpu int32 + br := bytes.NewReader(buf) + err = binary.Read(br, binary.LittleEndian, &ncpu) + if err != nil { + return nil, err + } + c.Cores = ncpu + + if c.ModelName, err = unix.Sysctl("hw.model"); err != nil { + return nil, err + } return append(ret, c), nil } + +func CountsWithContext(ctx context.Context, logical bool) (int, error) { + return runtime.NumCPU(), nil +} diff --git a/github.com/shirou/gopsutil/cpu/cpu_solaris.go b/github.com/shirou/gopsutil/cpu/cpu_solaris.go index 117fd909d4..3de0984240 100644 --- a/github.com/shirou/gopsutil/cpu/cpu_solaris.go +++ b/github.com/shirou/gopsutil/cpu/cpu_solaris.go @@ -6,17 +6,16 @@ import ( "fmt" "os/exec" "regexp" + "runtime" "sort" "strconv" "strings" - - "github.com/shirou/gopsutil/internal/common" ) var ClocksPerSec = float64(128) func init() { - getconf, err := exec.LookPath("/usr/bin/getconf") + getconf, err := exec.LookPath("getconf") if err != nil { return } @@ -30,12 +29,97 @@ func init() { } } +//sum all values in a float64 map with float64 keys +func msum(x map[float64]float64) float64 { + total := 0.0 + for _, y := range x { + total += y + } + return total +} + func Times(percpu bool) ([]TimesStat, error) { return TimesWithContext(context.Background(), percpu) } func TimesWithContext(ctx context.Context, percpu bool) ([]TimesStat, error) { - return []TimesStat{}, common.ErrNotImplementedError + kstatSys, err := exec.LookPath("kstat") + if err != nil { + return nil, fmt.Errorf("cannot find kstat: %s", err) + } + cpu := make(map[float64]float64) + idle := make(map[float64]float64) + user := make(map[float64]float64) + kern := make(map[float64]float64) + iowt := make(map[float64]float64) + //swap := make(map[float64]float64) + kstatSysOut, err := invoke.CommandWithContext(ctx, kstatSys, "-p", "cpu_stat:*:*:/^idle$|^user$|^kernel$|^iowait$|^swap$/") + if err != nil { + return nil, fmt.Errorf("cannot execute kstat: %s", err) + } + re := regexp.MustCompile(`[:\s]+`) + for _, line := range strings.Split(string(kstatSysOut), "\n") { + fields := re.Split(line, -1) + if fields[0] != "cpu_stat" { + continue + } + cpuNumber, err := strconv.ParseFloat(fields[1], 64) + if err != nil { + return nil, fmt.Errorf("cannot parse cpu number: %s", err) + } + cpu[cpuNumber] = cpuNumber + switch fields[3] { + case "idle": + idle[cpuNumber], err = strconv.ParseFloat(fields[4], 64) + if err != nil { + return nil, fmt.Errorf("cannot parse idle: %s", err) + } + case "user": + user[cpuNumber], err = strconv.ParseFloat(fields[4], 64) + if err != nil { + return nil, fmt.Errorf("cannot parse user: %s", err) + } + case "kernel": + kern[cpuNumber], err = strconv.ParseFloat(fields[4], 64) + if err != nil { + return nil, fmt.Errorf("cannot parse kernel: %s", err) + } + case "iowait": + iowt[cpuNumber], err = strconv.ParseFloat(fields[4], 64) + if err != nil { + return nil, fmt.Errorf("cannot parse iowait: %s", err) + } + //not sure how this translates, don't report, add to kernel, something else? + /*case "swap": + swap[cpuNumber], err = strconv.ParseFloat(fields[4], 64) + if err != nil { + return nil, fmt.Errorf("cannot parse swap: %s", err) + } */ + } + } + ret := make([]TimesStat, 0, len(cpu)) + if percpu { + for _, c := range cpu { + ct := &TimesStat{ + CPU: fmt.Sprintf("cpu%d", int(cpu[c])), + Idle: idle[c] / ClocksPerSec, + User: user[c] / ClocksPerSec, + System: kern[c] / ClocksPerSec, + Iowait: iowt[c] / ClocksPerSec, + } + ret = append(ret, *ct) + } + } else { + ct := &TimesStat{ + CPU: "cpu-total", + Idle: msum(idle) / ClocksPerSec, + User: msum(user) / ClocksPerSec, + System: msum(kern) / ClocksPerSec, + Iowait: msum(iowt) / ClocksPerSec, + } + ret = append(ret, *ct) + } + return ret, nil } func Info() ([]InfoStat, error) { @@ -43,7 +127,7 @@ func Info() ([]InfoStat, error) { } func InfoWithContext(ctx context.Context) ([]InfoStat, error) { - psrInfo, err := exec.LookPath("/usr/sbin/psrinfo") + psrInfo, err := exec.LookPath("psrinfo") if err != nil { return nil, fmt.Errorf("cannot find psrinfo: %s", err) } @@ -52,7 +136,7 @@ func InfoWithContext(ctx context.Context) ([]InfoStat, error) { return nil, fmt.Errorf("cannot execute psrinfo: %s", err) } - isaInfo, err := exec.LookPath("/usr/bin/isainfo") + isaInfo, err := exec.LookPath("isainfo") if err != nil { return nil, fmt.Errorf("cannot find isainfo: %s", err) } @@ -196,3 +280,7 @@ func parseProcessorInfo(cmdOutput string) ([]InfoStat, error) { } return result, nil } + +func CountsWithContext(ctx context.Context, logical bool) (int, error) { + return runtime.NumCPU(), nil +} diff --git a/github.com/shirou/gopsutil/cpu/cpu_windows.go b/github.com/shirou/gopsutil/cpu/cpu_windows.go index 3959212b18..ad1750b5c1 100644 --- a/github.com/shirou/gopsutil/cpu/cpu_windows.go +++ b/github.com/shirou/gopsutil/cpu/cpu_windows.go @@ -12,29 +12,35 @@ import ( "golang.org/x/sys/windows" ) +var ( + procGetActiveProcessorCount = common.Modkernel32.NewProc("GetActiveProcessorCount") + procGetNativeSystemInfo = common.Modkernel32.NewProc("GetNativeSystemInfo") +) + type Win32_Processor struct { LoadPercentage *uint16 Family uint16 Manufacturer string Name string NumberOfLogicalProcessors uint32 + NumberOfCores uint32 ProcessorID *string Stepping *string MaxClockSpeed uint32 } -type win32_PerfRawData_Counters_ProcessorInformation struct { - Name string - PercentDPCTime uint64 - PercentIdleTime uint64 - PercentUserTime uint64 - PercentProcessorTime uint64 - PercentInterruptTime uint64 - PercentPriorityTime uint64 - PercentPrivilegedTime uint64 - InterruptsPerSec uint32 - ProcessorFrequency uint32 - DPCRate uint32 +// SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION +// defined in windows api doc with the following +// https://docs.microsoft.com/en-us/windows/desktop/api/winternl/nf-winternl-ntquerysysteminformation#system_processor_performance_information +// additional fields documented here +// https://www.geoffchappell.com/studies/windows/km/ntoskrnl/api/ex/sysinfo/processor_performance.htm +type win32_SystemProcessorPerformanceInformation struct { + IdleTime int64 // idle time in 100ns (this is not a filetime). + KernelTime int64 // kernel time in 100ns. kernel time includes idle time. (this is not a filetime). + UserTime int64 // usertime in 100ns (this is not a filetime). + DpcTime int64 // dpc time in 100ns (this is not a filetime). + InterruptTime int64 // interrupt time in 100ns + InterruptCount uint32 } // Win32_PerfFormattedData_PerfOS_System struct to have count of processes and processor queue length @@ -44,7 +50,14 @@ type Win32_PerfFormattedData_PerfOS_System struct { } const ( - win32_TicksPerSecond = 10000000.0 + ClocksPerSec = 10000000.0 + + // systemProcessorPerformanceInformationClass information class to query with NTQuerySystemInformation + // https://processhacker.sourceforge.io/doc/ntexapi_8h.html#ad5d815b48e8f4da1ef2eb7a2f18a54e0 + win32_SystemProcessorPerformanceInformationClass = 8 + + // size of systemProcessorPerformanceInfoSize in memory + win32_SystemProcessorPerformanceInfoSize = uint32(unsafe.Sizeof(win32_SystemProcessorPerformanceInformation{})) ) // Times returns times stat per cpu and combined for all CPUs @@ -54,7 +67,7 @@ func Times(percpu bool) ([]TimesStat, error) { func TimesWithContext(ctx context.Context, percpu bool) ([]TimesStat, error) { if percpu { - return perCPUTimesWithContext(ctx) + return perCPUTimes() } var ret []TimesStat @@ -120,20 +133,6 @@ func InfoWithContext(ctx context.Context) ([]InfoStat, error) { return ret, nil } -// PerfInfo returns the performance counter's instance value for ProcessorInformation. -// Name property is the key by which overall, per cpu and per core metric is known. -func perfInfoWithContext(ctx context.Context) ([]win32_PerfRawData_Counters_ProcessorInformation, error) { - var ret []win32_PerfRawData_Counters_ProcessorInformation - - q := wmi.CreateQuery(&ret, "WHERE NOT Name LIKE '%_Total'") - err := common.WMIQueryWithContext(ctx, q, &ret) - if err != nil { - return []win32_PerfRawData_Counters_ProcessorInformation{}, err - } - - return ret, err -} - // ProcInfo returns processes count and processor queue length in the system. // There is a single queue for processor even on multiprocessors systems. func ProcInfo() ([]Win32_PerfFormattedData_PerfOS_System, error) { @@ -151,21 +150,106 @@ func ProcInfoWithContext(ctx context.Context) ([]Win32_PerfFormattedData_PerfOS_ } // perCPUTimes returns times stat per cpu, per core and overall for all CPUs -func perCPUTimesWithContext(ctx context.Context) ([]TimesStat, error) { +func perCPUTimes() ([]TimesStat, error) { var ret []TimesStat - stats, err := perfInfoWithContext(ctx) + stats, err := perfInfo() if err != nil { return nil, err } - for _, v := range stats { + for core, v := range stats { c := TimesStat{ - CPU: v.Name, - User: float64(v.PercentUserTime) / win32_TicksPerSecond, - System: float64(v.PercentPrivilegedTime) / win32_TicksPerSecond, - Idle: float64(v.PercentIdleTime) / win32_TicksPerSecond, - Irq: float64(v.PercentInterruptTime) / win32_TicksPerSecond, + CPU: fmt.Sprintf("cpu%d", core), + User: float64(v.UserTime) / ClocksPerSec, + System: float64(v.KernelTime-v.IdleTime) / ClocksPerSec, + Idle: float64(v.IdleTime) / ClocksPerSec, + Irq: float64(v.InterruptTime) / ClocksPerSec, } ret = append(ret, c) } return ret, nil } + +// makes call to Windows API function to retrieve performance information for each core +func perfInfo() ([]win32_SystemProcessorPerformanceInformation, error) { + // Make maxResults large for safety. + // We can't invoke the api call with a results array that's too small. + // If we have more than 2056 cores on a single host, then it's probably the future. + maxBuffer := 2056 + // buffer for results from the windows proc + resultBuffer := make([]win32_SystemProcessorPerformanceInformation, maxBuffer) + // size of the buffer in memory + bufferSize := uintptr(win32_SystemProcessorPerformanceInfoSize) * uintptr(maxBuffer) + // size of the returned response + var retSize uint32 + + // Invoke windows api proc. + // The returned err from the windows dll proc will always be non-nil even when successful. + // See https://godoc.org/golang.org/x/sys/windows#LazyProc.Call for more information + retCode, _, err := common.ProcNtQuerySystemInformation.Call( + win32_SystemProcessorPerformanceInformationClass, // System Information Class -> SystemProcessorPerformanceInformation + uintptr(unsafe.Pointer(&resultBuffer[0])), // pointer to first element in result buffer + bufferSize, // size of the buffer in memory + uintptr(unsafe.Pointer(&retSize)), // pointer to the size of the returned results the windows proc will set this + ) + + // check return code for errors + if retCode != 0 { + return nil, fmt.Errorf("call to NtQuerySystemInformation returned %d. err: %s", retCode, err.Error()) + } + + // calculate the number of returned elements based on the returned size + numReturnedElements := retSize / win32_SystemProcessorPerformanceInfoSize + + // trim results to the number of returned elements + resultBuffer = resultBuffer[:numReturnedElements] + + return resultBuffer, nil +} + +// SystemInfo is an equivalent representation of SYSTEM_INFO in the Windows API. +// https://msdn.microsoft.com/en-us/library/ms724958%28VS.85%29.aspx?f=255&MSPPError=-2147217396 +// https://github.com/elastic/go-windows/blob/bb1581babc04d5cb29a2bfa7a9ac6781c730c8dd/kernel32.go#L43 +type systemInfo struct { + wProcessorArchitecture uint16 + wReserved uint16 + dwPageSize uint32 + lpMinimumApplicationAddress uintptr + lpMaximumApplicationAddress uintptr + dwActiveProcessorMask uintptr + dwNumberOfProcessors uint32 + dwProcessorType uint32 + dwAllocationGranularity uint32 + wProcessorLevel uint16 + wProcessorRevision uint16 +} + +func CountsWithContext(ctx context.Context, logical bool) (int, error) { + if logical { + // https://github.com/giampaolo/psutil/blob/d01a9eaa35a8aadf6c519839e987a49d8be2d891/psutil/_psutil_windows.c#L97 + err := procGetActiveProcessorCount.Find() + if err == nil { // Win7+ + ret, _, _ := procGetActiveProcessorCount.Call(uintptr(0xffff)) // ALL_PROCESSOR_GROUPS is 0xffff according to Rust's winapi lib https://docs.rs/winapi/*/x86_64-pc-windows-msvc/src/winapi/shared/ntdef.rs.html#120 + if ret != 0 { + return int(ret), nil + } + } + var systemInfo systemInfo + _, _, err = procGetNativeSystemInfo.Call(uintptr(unsafe.Pointer(&systemInfo))) + if systemInfo.dwNumberOfProcessors == 0 { + return 0, err + } + return int(systemInfo.dwNumberOfProcessors), nil + } + // physical cores https://github.com/giampaolo/psutil/blob/d01a9eaa35a8aadf6c519839e987a49d8be2d891/psutil/_psutil_windows.c#L499 + // for the time being, try with unreliable and slow WMI call… + var dst []Win32_Processor + q := wmi.CreateQuery(&dst, "") + if err := common.WMIQueryWithContext(ctx, q, &dst); err != nil { + return 0, err + } + var count uint32 + for _, d := range dst { + count += d.NumberOfCores + } + return int(count), nil +} diff --git a/github.com/shirou/gopsutil/disk/disk_darwin.go b/github.com/shirou/gopsutil/disk/disk_darwin.go index 2b1d000db3..64c41d70ae 100644 --- a/github.com/shirou/gopsutil/disk/disk_darwin.go +++ b/github.com/shirou/gopsutil/disk/disk_darwin.go @@ -5,7 +5,6 @@ package disk import ( "context" "path" - "unsafe" "github.com/shirou/gopsutil/internal/common" "golang.org/x/sys/unix" @@ -18,63 +17,51 @@ func Partitions(all bool) ([]PartitionStat, error) { func PartitionsWithContext(ctx context.Context, all bool) ([]PartitionStat, error) { var ret []PartitionStat - count, err := Getfsstat(nil, MntWait) + count, err := unix.Getfsstat(nil, unix.MNT_WAIT) if err != nil { return ret, err } - fs := make([]Statfs, count) - if _, err = Getfsstat(fs, MntWait); err != nil { + fs := make([]unix.Statfs_t, count) + if _, err = unix.Getfsstat(fs, unix.MNT_WAIT); err != nil { return ret, err } for _, stat := range fs { opts := "rw" - if stat.Flags&MntReadOnly != 0 { + if stat.Flags&unix.MNT_RDONLY != 0 { opts = "ro" } - if stat.Flags&MntSynchronous != 0 { + if stat.Flags&unix.MNT_SYNCHRONOUS != 0 { opts += ",sync" } - if stat.Flags&MntNoExec != 0 { + if stat.Flags&unix.MNT_NOEXEC != 0 { opts += ",noexec" } - if stat.Flags&MntNoSuid != 0 { + if stat.Flags&unix.MNT_NOSUID != 0 { opts += ",nosuid" } - if stat.Flags&MntUnion != 0 { + if stat.Flags&unix.MNT_UNION != 0 { opts += ",union" } - if stat.Flags&MntAsync != 0 { + if stat.Flags&unix.MNT_ASYNC != 0 { opts += ",async" } - if stat.Flags&MntSuidDir != 0 { - opts += ",suiddir" + if stat.Flags&unix.MNT_DONTBROWSE != 0 { + opts += ",nobrowse" } - if stat.Flags&MntSoftDep != 0 { - opts += ",softdep" + if stat.Flags&unix.MNT_AUTOMOUNTED != 0 { + opts += ",automounted" } - if stat.Flags&MntNoSymFollow != 0 { - opts += ",nosymfollow" + if stat.Flags&unix.MNT_JOURNALED != 0 { + opts += ",journaled" } - if stat.Flags&MntGEOMJournal != 0 { - opts += ",gjounalc" - } - if stat.Flags&MntMultilabel != 0 { + if stat.Flags&unix.MNT_MULTILABEL != 0 { opts += ",multilabel" } - if stat.Flags&MntACLs != 0 { - opts += ",acls" - } - if stat.Flags&MntNoATime != 0 { - opts += ",noattime" - } - if stat.Flags&MntClusterRead != 0 { - opts += ",nocluster" + if stat.Flags&unix.MNT_NOATIME != 0 { + opts += ",noatime" } - if stat.Flags&MntClusterWrite != 0 { - opts += ",noclusterw" - } - if stat.Flags&MntNFS4ACLs != 0 { - opts += ",nfs4acls" + if stat.Flags&unix.MNT_NODEV != 0 { + opts += ",nodev" } d := PartitionStat{ Device: common.IntToString(stat.Mntfromname[:]), @@ -94,25 +81,6 @@ func PartitionsWithContext(ctx context.Context, all bool) ([]PartitionStat, erro return ret, nil } -func Getfsstat(buf []Statfs, flags int) (n int, err error) { - return GetfsstatWithContext(context.Background(), buf, flags) -} - -func GetfsstatWithContext(ctx context.Context, buf []Statfs, flags int) (n int, err error) { - var _p0 unsafe.Pointer - var bufsize uintptr - if len(buf) > 0 { - _p0 = unsafe.Pointer(&buf[0]) - bufsize = unsafe.Sizeof(Statfs{}) * uintptr(len(buf)) - } - r0, _, e1 := unix.Syscall(SYS_GETFSSTAT64, uintptr(_p0), bufsize, uintptr(flags)) - n = int(r0) - if e1 != 0 { - err = e1 - } - return -} - func getFsType(stat unix.Statfs_t) string { return common.IntToString(stat.Fstypename[:]) } diff --git a/github.com/shirou/gopsutil/disk/disk_darwin.h b/github.com/shirou/gopsutil/disk/disk_darwin.h deleted file mode 100644 index d0fe514e35..0000000000 --- a/github.com/shirou/gopsutil/disk/disk_darwin.h +++ /dev/null @@ -1,164 +0,0 @@ -#include -#include -#include -#include -#include - -// The iterator of all things disk. Allocated by StartIOCounterFetch, released -// by EndIOCounterFetch. -static io_iterator_t diskIter; - -// Begins fetching IO counters. -// -// Returns 1 if the fetch started successfully, false otherwise. -// -// If the fetch was started successfully, you must call EndIOCounterFetch once -// done to release resources. -int StartIOCounterFetch() -{ - if (IOServiceGetMatchingServices(kIOMasterPortDefault, - IOServiceMatching(kIOMediaClass), - &diskIter) != kIOReturnSuccess) { - return 0; - } - - return 1; -} - -// Releases resources from fetching IO counters. -void EndIOCounterFetch() -{ - IOObjectRelease(diskIter); -} - -// The current disk entry of interest. Allocated by FetchNextDisk(), released by -// ReadDiskInfo(). -static io_registry_entry_t diskEntry; - -// The parent of diskEntry. Same lifetimes. -static io_registry_entry_t parentEntry; - -// Fetches the next disk. Note that a disk entry is allocated, and will be held -// until it is processed and freed by ReadDiskInfo. -int FetchNextDisk() -{ - while ((diskEntry = IOIteratorNext(diskIter)) != 0) { - // We are iterating IOMedia. We need to get the parent too (IOBSD). - if (IORegistryEntryGetParentEntry(diskEntry, kIOServicePlane, &parentEntry) != kIOReturnSuccess) { - // something is wrong... - IOObjectRelease(diskEntry); - continue; - } - - if (!IOObjectConformsTo(parentEntry, "IOBlockStorageDriver")) { - // no use to us, try the next disk - IOObjectRelease(diskEntry); - IOObjectRelease(parentEntry); - continue; - } - - // Got a disk OK. - return 1; - } - - // No more disks. - return 0; -} - -// Reads the current disk (from iteration) info into DiskInfo struct. -// Once done, all resources from the current iteration of reading are freed, -// ready for FetchNextDisk() to be called again. -int ReadDiskInfo(DiskInfo *info) -{ - // Parent props. Allocated by us. - CFDictionaryRef parentProps = NULL; - - // Disk props. Allocated by us. - CFDictionaryRef diskProps = NULL; - - // Disk stats, fetched by us, but not allocated by us. - CFDictionaryRef stats = NULL; - - if (IORegistryEntryCreateCFProperties(diskEntry, (CFMutableDictionaryRef *)&parentProps, - kCFAllocatorDefault, kNilOptions) != kIOReturnSuccess) - { - // can't get parent props, give up - CFRelease(parentProps); - IOObjectRelease(diskEntry); - IOObjectRelease(parentEntry); - return -1; - } - - if (IORegistryEntryCreateCFProperties(parentEntry, (CFMutableDictionaryRef *)&diskProps, - kCFAllocatorDefault, kNilOptions) != kIOReturnSuccess) - { - // can't get disk props, give up - CFRelease(parentProps); - CFRelease(diskProps); - IOObjectRelease(diskEntry); - IOObjectRelease(parentEntry); - return -1; - } - - // Start fetching - CFStringRef cfDiskName = (CFStringRef)CFDictionaryGetValue(parentProps, CFSTR(kIOBSDNameKey)); - CFStringGetCString(cfDiskName, info->DiskName, MAX_DISK_NAME, CFStringGetSystemEncoding()); - stats = (CFDictionaryRef)CFDictionaryGetValue( diskProps, CFSTR(kIOBlockStorageDriverStatisticsKey)); - - if (stats == NULL) { - // stat fetch failed... - CFRelease(parentProps); - CFRelease(diskProps); - IOObjectRelease(parentEntry); - IOObjectRelease(diskEntry); - return -1; - } - - CFNumberRef cfnum; - - if ((cfnum = (CFNumberRef)CFDictionaryGetValue(stats, CFSTR(kIOBlockStorageDriverStatisticsReadsKey)))) { - CFNumberGetValue(cfnum, kCFNumberSInt64Type, &info->Reads); - } else { - info->Reads = 0; - } - - if ((cfnum = (CFNumberRef)CFDictionaryGetValue(stats, CFSTR(kIOBlockStorageDriverStatisticsWritesKey)))) { - CFNumberGetValue(cfnum, kCFNumberSInt64Type, &info->Writes); - } else { - info->Writes = 0; - } - - if ((cfnum = (CFNumberRef)CFDictionaryGetValue(stats, CFSTR(kIOBlockStorageDriverStatisticsBytesReadKey)))) { - CFNumberGetValue(cfnum, kCFNumberSInt64Type, &info->ReadBytes); - } else { - info->ReadBytes = 0; - } - - if ((cfnum = (CFNumberRef)CFDictionaryGetValue(stats, CFSTR(kIOBlockStorageDriverStatisticsBytesWrittenKey)))) { - CFNumberGetValue(cfnum, kCFNumberSInt64Type, &info->WriteBytes); - } else { - info->WriteBytes = 0; - } - - if ((cfnum = (CFNumberRef)CFDictionaryGetValue(stats, CFSTR(kIOBlockStorageDriverStatisticsTotalReadTimeKey)))) { - CFNumberGetValue(cfnum, kCFNumberSInt64Type, &info->ReadTime); - } else { - info->ReadTime = 0; - } - if ((cfnum = (CFNumberRef)CFDictionaryGetValue(stats, CFSTR(kIOBlockStorageDriverStatisticsTotalWriteTimeKey)))) { - CFNumberGetValue(cfnum, kCFNumberSInt64Type, &info->WriteTime); - } else { - info->WriteTime = 0; - } - - // note: read/write time are in ns, but we want ms. - info->ReadTime = info->ReadTime / 1000 / 1000; - info->WriteTime = info->WriteTime / 1000 / 1000; - - CFRelease(parentProps); - CFRelease(diskProps); - IOObjectRelease(parentEntry); - IOObjectRelease(diskEntry); - return 0; -} - diff --git a/github.com/shirou/gopsutil/disk/disk_darwin_386.go b/github.com/shirou/gopsutil/disk/disk_darwin_386.go deleted file mode 100644 index bd83a4a712..0000000000 --- a/github.com/shirou/gopsutil/disk/disk_darwin_386.go +++ /dev/null @@ -1,59 +0,0 @@ -// +build darwin -// +build 386 - -package disk - -const ( - MntWait = 1 - MfsNameLen = 15 /* length of fs type name, not inc. nul */ - MNameLen = 90 /* length of buffer for returned name */ - - MFSTYPENAMELEN = 16 /* length of fs type name including null */ - MAXPATHLEN = 1024 - MNAMELEN = MAXPATHLEN - - SYS_GETFSSTAT64 = 347 -) - -type Fsid struct{ val [2]int32 } /* file system id type */ -type uid_t int32 - -// sys/mount.h -const ( - MntReadOnly = 0x00000001 /* read only filesystem */ - MntSynchronous = 0x00000002 /* filesystem written synchronously */ - MntNoExec = 0x00000004 /* can't exec from filesystem */ - MntNoSuid = 0x00000008 /* don't honor setuid bits on fs */ - MntUnion = 0x00000020 /* union with underlying filesystem */ - MntAsync = 0x00000040 /* filesystem written asynchronously */ - MntSuidDir = 0x00100000 /* special handling of SUID on dirs */ - MntSoftDep = 0x00200000 /* soft updates being done */ - MntNoSymFollow = 0x00400000 /* do not follow symlinks */ - MntGEOMJournal = 0x02000000 /* GEOM journal support enabled */ - MntMultilabel = 0x04000000 /* MAC support for individual objects */ - MntACLs = 0x08000000 /* ACL support enabled */ - MntNoATime = 0x10000000 /* disable update of file access time */ - MntClusterRead = 0x40000000 /* disable cluster read */ - MntClusterWrite = 0x80000000 /* disable cluster write */ - MntNFS4ACLs = 0x00000010 -) - -// https://github.com/golang/go/blob/master/src/syscall/ztypes_darwin_386.go#L82 -type Statfs struct { - Bsize uint32 - Iosize int32 - Blocks uint64 - Bfree uint64 - Bavail uint64 - Files uint64 - Ffree uint64 - Fsid Fsid - Owner uint32 - Type uint32 - Flags uint32 - Fssubtype uint32 - Fstypename [16]int8 - Mntonname [1024]int8 - Mntfromname [1024]int8 - Reserved [8]uint32 -} diff --git a/github.com/shirou/gopsutil/disk/disk_darwin_amd64.go b/github.com/shirou/gopsutil/disk/disk_darwin_amd64.go deleted file mode 100644 index ec40a758a7..0000000000 --- a/github.com/shirou/gopsutil/disk/disk_darwin_amd64.go +++ /dev/null @@ -1,58 +0,0 @@ -// +build darwin -// +build amd64 - -package disk - -const ( - MntWait = 1 - MfsNameLen = 15 /* length of fs type name, not inc. nul */ - MNameLen = 90 /* length of buffer for returned name */ - - MFSTYPENAMELEN = 16 /* length of fs type name including null */ - MAXPATHLEN = 1024 - MNAMELEN = MAXPATHLEN - - SYS_GETFSSTAT64 = 347 -) - -type Fsid struct{ val [2]int32 } /* file system id type */ -type uid_t int32 - -// sys/mount.h -const ( - MntReadOnly = 0x00000001 /* read only filesystem */ - MntSynchronous = 0x00000002 /* filesystem written synchronously */ - MntNoExec = 0x00000004 /* can't exec from filesystem */ - MntNoSuid = 0x00000008 /* don't honor setuid bits on fs */ - MntUnion = 0x00000020 /* union with underlying filesystem */ - MntAsync = 0x00000040 /* filesystem written asynchronously */ - MntSuidDir = 0x00100000 /* special handling of SUID on dirs */ - MntSoftDep = 0x00200000 /* soft updates being done */ - MntNoSymFollow = 0x00400000 /* do not follow symlinks */ - MntGEOMJournal = 0x02000000 /* GEOM journal support enabled */ - MntMultilabel = 0x04000000 /* MAC support for individual objects */ - MntACLs = 0x08000000 /* ACL support enabled */ - MntNoATime = 0x10000000 /* disable update of file access time */ - MntClusterRead = 0x40000000 /* disable cluster read */ - MntClusterWrite = 0x80000000 /* disable cluster write */ - MntNFS4ACLs = 0x00000010 -) - -type Statfs struct { - Bsize uint32 - Iosize int32 - Blocks uint64 - Bfree uint64 - Bavail uint64 - Files uint64 - Ffree uint64 - Fsid Fsid - Owner uint32 - Type uint32 - Flags uint32 - Fssubtype uint32 - Fstypename [16]int8 - Mntonname [1024]int8 - Mntfromname [1024]int8 - Reserved [8]uint32 -} diff --git a/github.com/shirou/gopsutil/disk/disk_darwin_arm64.go b/github.com/shirou/gopsutil/disk/disk_darwin_arm64.go deleted file mode 100644 index 0e3f6700a6..0000000000 --- a/github.com/shirou/gopsutil/disk/disk_darwin_arm64.go +++ /dev/null @@ -1,58 +0,0 @@ -// +build darwin -// +build arm64 - -package disk - -const ( - MntWait = 1 - MfsNameLen = 15 /* length of fs type name, not inc. nul */ - MNameLen = 90 /* length of buffer for returned name */ - - MFSTYPENAMELEN = 16 /* length of fs type name including null */ - MAXPATHLEN = 1024 - MNAMELEN = MAXPATHLEN - - SYS_GETFSSTAT64 = 347 -) - -type Fsid struct{ val [2]int32 } /* file system id type */ -type uid_t int32 - -// sys/mount.h -const ( - MntReadOnly = 0x00000001 /* read only filesystem */ - MntSynchronous = 0x00000002 /* filesystem written synchronously */ - MntNoExec = 0x00000004 /* can't exec from filesystem */ - MntNoSuid = 0x00000008 /* don't honor setuid bits on fs */ - MntUnion = 0x00000020 /* union with underlying filesystem */ - MntAsync = 0x00000040 /* filesystem written asynchronously */ - MntSuidDir = 0x00100000 /* special handling of SUID on dirs */ - MntSoftDep = 0x00200000 /* soft updates being done */ - MntNoSymFollow = 0x00400000 /* do not follow symlinks */ - MntGEOMJournal = 0x02000000 /* GEOM journal support enabled */ - MntMultilabel = 0x04000000 /* MAC support for individual objects */ - MntACLs = 0x08000000 /* ACL support enabled */ - MntNoATime = 0x10000000 /* disable update of file access time */ - MntClusterRead = 0x40000000 /* disable cluster read */ - MntClusterWrite = 0x80000000 /* disable cluster write */ - MntNFS4ACLs = 0x00000010 -) - -type Statfs struct { - Bsize uint32 - Iosize int32 - Blocks uint64 - Bfree uint64 - Bavail uint64 - Files uint64 - Ffree uint64 - Fsid Fsid - Owner uint32 - Type uint32 - Flags uint32 - Fssubtype uint32 - Fstypename [16]int8 - Mntonname [1024]int8 - Mntfromname [1024]int8 - Reserved [8]uint32 -} diff --git a/github.com/shirou/gopsutil/disk/disk_darwin_cgo.go b/github.com/shirou/gopsutil/disk/disk_darwin_cgo.go index 480e237707..8a1848ed7d 100644 --- a/github.com/shirou/gopsutil/disk/disk_darwin_cgo.go +++ b/github.com/shirou/gopsutil/disk/disk_darwin_cgo.go @@ -4,32 +4,15 @@ package disk /* -#cgo LDFLAGS: -lobjc -framework Foundation -framework IOKit +#cgo LDFLAGS: -framework CoreFoundation -framework IOKit #include - -// ### enough? -const int MAX_DISK_NAME = 100; - -typedef struct -{ - char DiskName[MAX_DISK_NAME]; - int64_t Reads; - int64_t Writes; - int64_t ReadBytes; - int64_t WriteBytes; - int64_t ReadTime; - int64_t WriteTime; -} DiskInfo; - -#include "disk_darwin.h" +#include +#include "iostat_darwin.h" */ import "C" import ( "context" - "errors" - "strings" - "unsafe" "github.com/shirou/gopsutil/internal/common" ) @@ -39,57 +22,28 @@ func IOCounters(names ...string) (map[string]IOCountersStat, error) { } func IOCountersWithContext(ctx context.Context, names ...string) (map[string]IOCountersStat, error) { - if C.StartIOCounterFetch() == 0 { - return nil, errors.New("Unable to fetch disk list") + var buf [C.NDRIVE]C.DriveStats + n, err := C.readdrivestat(&buf[0], C.int(len(buf))) + if err != nil { + return nil, err } - - // Clean up when we are done. - defer C.EndIOCounterFetch() ret := make(map[string]IOCountersStat, 0) - - for { - res := C.FetchNextDisk() - if res == -1 { - return nil, errors.New("Unable to fetch disk information") - } else if res == 0 { - break // done - } - - di := C.DiskInfo{} - if C.ReadDiskInfo((*C.DiskInfo)(unsafe.Pointer(&di))) == -1 { - return nil, errors.New("Unable to fetch disk properties") - } - - // Used to only get the necessary part of the C string. - isRuneNull := func(r rune) bool { - return r == '\u0000' - } - - // Map from the darwin-specific C struct to the Go type - // - // ### missing: IopsInProgress, WeightedIO, MergedReadCount, - // MergedWriteCount, SerialNumber - // IOKit can give us at least the serial number I think... + for i := 0; i < int(n); i++ { d := IOCountersStat{ - // Note: The Go type wants unsigned values, but CFNumberGetValue - // doesn't appear to be able to give us unsigned values. So, we - // cast, and hope for the best. - ReadBytes: uint64(di.ReadBytes), - WriteBytes: uint64(di.WriteBytes), - ReadCount: uint64(di.Reads), - WriteCount: uint64(di.Writes), - ReadTime: uint64(di.ReadTime), - WriteTime: uint64(di.WriteTime), - IoTime: uint64(di.ReadTime + di.WriteTime), - Name: strings.TrimFunc(C.GoStringN(&di.DiskName[0], C.MAX_DISK_NAME), isRuneNull), + ReadBytes: uint64(buf[i].read), + WriteBytes: uint64(buf[i].written), + ReadCount: uint64(buf[i].nread), + WriteCount: uint64(buf[i].nwrite), + ReadTime: uint64(buf[i].readtime / 1000 / 1000), // note: read/write time are in ns, but we want ms. + WriteTime: uint64(buf[i].writetime / 1000 / 1000), + IoTime: uint64((buf[i].readtime + buf[i].writetime) / 1000 / 1000), + Name: C.GoString(&buf[i].name[0]), } - if len(names) > 0 && !common.StringsHas(names, d.Name) { continue } ret[d.Name] = d } - return ret, nil } diff --git a/github.com/shirou/gopsutil/disk/disk_freebsd.go b/github.com/shirou/gopsutil/disk/disk_freebsd.go index 2e0966a511..f172828486 100644 --- a/github.com/shirou/gopsutil/disk/disk_freebsd.go +++ b/github.com/shirou/gopsutil/disk/disk_freebsd.go @@ -8,7 +8,6 @@ import ( "encoding/binary" "path" "strconv" - "unsafe" "golang.org/x/sys/unix" @@ -23,71 +22,71 @@ func PartitionsWithContext(ctx context.Context, all bool) ([]PartitionStat, erro var ret []PartitionStat // get length - count, err := unix.Getfsstat(nil, MNT_WAIT) + count, err := unix.Getfsstat(nil, unix.MNT_WAIT) if err != nil { return ret, err } - fs := make([]Statfs, count) - if _, err = Getfsstat(fs, MNT_WAIT); err != nil { + fs := make([]unix.Statfs_t, count) + if _, err = unix.Getfsstat(fs, unix.MNT_WAIT); err != nil { return ret, err } for _, stat := range fs { opts := "rw" - if stat.Flags&MNT_RDONLY != 0 { + if stat.Flags&unix.MNT_RDONLY != 0 { opts = "ro" } - if stat.Flags&MNT_SYNCHRONOUS != 0 { + if stat.Flags&unix.MNT_SYNCHRONOUS != 0 { opts += ",sync" } - if stat.Flags&MNT_NOEXEC != 0 { + if stat.Flags&unix.MNT_NOEXEC != 0 { opts += ",noexec" } - if stat.Flags&MNT_NOSUID != 0 { + if stat.Flags&unix.MNT_NOSUID != 0 { opts += ",nosuid" } - if stat.Flags&MNT_UNION != 0 { + if stat.Flags&unix.MNT_UNION != 0 { opts += ",union" } - if stat.Flags&MNT_ASYNC != 0 { + if stat.Flags&unix.MNT_ASYNC != 0 { opts += ",async" } - if stat.Flags&MNT_SUIDDIR != 0 { + if stat.Flags&unix.MNT_SUIDDIR != 0 { opts += ",suiddir" } - if stat.Flags&MNT_SOFTDEP != 0 { + if stat.Flags&unix.MNT_SOFTDEP != 0 { opts += ",softdep" } - if stat.Flags&MNT_NOSYMFOLLOW != 0 { + if stat.Flags&unix.MNT_NOSYMFOLLOW != 0 { opts += ",nosymfollow" } - if stat.Flags&MNT_GJOURNAL != 0 { - opts += ",gjounalc" + if stat.Flags&unix.MNT_GJOURNAL != 0 { + opts += ",gjournal" } - if stat.Flags&MNT_MULTILABEL != 0 { + if stat.Flags&unix.MNT_MULTILABEL != 0 { opts += ",multilabel" } - if stat.Flags&MNT_ACLS != 0 { + if stat.Flags&unix.MNT_ACLS != 0 { opts += ",acls" } - if stat.Flags&MNT_NOATIME != 0 { - opts += ",noattime" + if stat.Flags&unix.MNT_NOATIME != 0 { + opts += ",noatime" } - if stat.Flags&MNT_NOCLUSTERR != 0 { - opts += ",nocluster" + if stat.Flags&unix.MNT_NOCLUSTERR != 0 { + opts += ",noclusterr" } - if stat.Flags&MNT_NOCLUSTERW != 0 { + if stat.Flags&unix.MNT_NOCLUSTERW != 0 { opts += ",noclusterw" } - if stat.Flags&MNT_NFS4ACLS != 0 { - opts += ",nfs4acls" + if stat.Flags&unix.MNT_NFS4ACLS != 0 { + opts += ",nfsv4acls" } d := PartitionStat{ - Device: common.IntToString(stat.Mntfromname[:]), - Mountpoint: common.IntToString(stat.Mntonname[:]), - Fstype: common.IntToString(stat.Fstypename[:]), + Device: common.ByteToString(stat.Mntfromname[:]), + Mountpoint: common.ByteToString(stat.Mntonname[:]), + Fstype: common.ByteToString(stat.Fstypename[:]), Opts: opts, } if all == false { @@ -158,27 +157,6 @@ func (b Bintime) Compute() float64 { // BT2LD(time) ((long double)(time).sec + (time).frac * BINTIME_SCALE) -// Getfsstat is borrowed from pkg/syscall/syscall_freebsd.go -// change Statfs_t to Statfs in order to get more information -func Getfsstat(buf []Statfs, flags int) (n int, err error) { - return GetfsstatWithContext(context.Background(), buf, flags) -} - -func GetfsstatWithContext(ctx context.Context, buf []Statfs, flags int) (n int, err error) { - var _p0 unsafe.Pointer - var bufsize uintptr - if len(buf) > 0 { - _p0 = unsafe.Pointer(&buf[0]) - bufsize = unsafe.Sizeof(Statfs{}) * uintptr(len(buf)) - } - r0, _, e1 := unix.Syscall(unix.SYS_GETFSSTAT, uintptr(_p0), bufsize, uintptr(flags)) - n = int(r0) - if e1 != 0 { - err = e1 - } - return -} - func parseDevstat(buf []byte) (Devstat, error) { var ds Devstat br := bytes.NewReader(buf) @@ -192,5 +170,5 @@ func parseDevstat(buf []byte) (Devstat, error) { } func getFsType(stat unix.Statfs_t) string { - return common.IntToString(stat.Fstypename[:]) + return common.ByteToString(stat.Fstypename[:]) } diff --git a/github.com/shirou/gopsutil/disk/disk_freebsd_386.go b/github.com/shirou/gopsutil/disk/disk_freebsd_386.go index 0b3f536c84..e2793a4fe6 100644 --- a/github.com/shirou/gopsutil/disk/disk_freebsd_386.go +++ b/github.com/shirou/gopsutil/disk/disk_freebsd_386.go @@ -15,28 +15,6 @@ const ( DEVSTAT_READ = 0x01 DEVSTAT_WRITE = 0x02 DEVSTAT_FREE = 0x03 - - MNT_RDONLY = 0x00000001 - MNT_SYNCHRONOUS = 0x00000002 - MNT_NOEXEC = 0x00000004 - MNT_NOSUID = 0x00000008 - MNT_UNION = 0x00000020 - MNT_ASYNC = 0x00000040 - MNT_SUIDDIR = 0x00100000 - MNT_SOFTDEP = 0x00200000 - MNT_NOSYMFOLLOW = 0x00400000 - MNT_GJOURNAL = 0x02000000 - MNT_MULTILABEL = 0x04000000 - MNT_ACLS = 0x08000000 - MNT_NOATIME = 0x10000000 - MNT_NOCLUSTERR = 0x40000000 - MNT_NOCLUSTERW = 0x80000000 - MNT_NFS4ACLS = 0x00000010 - - MNT_WAIT = 1 - MNT_NOWAIT = 2 - MNT_LAZY = 3 - MNT_SUSPEND = 4 ) const ( @@ -51,34 +29,6 @@ type ( _C_long_double int64 ) -type Statfs struct { - Version uint32 - Type uint32 - Flags uint64 - Bsize uint64 - Iosize uint64 - Blocks uint64 - Bfree uint64 - Bavail int64 - Files uint64 - Ffree int64 - Syncwrites uint64 - Asyncwrites uint64 - Syncreads uint64 - Asyncreads uint64 - Spare [10]uint64 - Namemax uint32 - Owner uint32 - Fsid Fsid - Charspare [80]int8 - Fstypename [16]int8 - Mntfromname [88]int8 - Mntonname [88]int8 -} -type Fsid struct { - Val [2]int32 -} - type Devstat struct { Sequence0 uint32 Allocated int32 diff --git a/github.com/shirou/gopsutil/disk/disk_freebsd_amd64.go b/github.com/shirou/gopsutil/disk/disk_freebsd_amd64.go index 89b617c9ca..e9613dc5c5 100644 --- a/github.com/shirou/gopsutil/disk/disk_freebsd_amd64.go +++ b/github.com/shirou/gopsutil/disk/disk_freebsd_amd64.go @@ -15,28 +15,6 @@ const ( DEVSTAT_READ = 0x01 DEVSTAT_WRITE = 0x02 DEVSTAT_FREE = 0x03 - - MNT_RDONLY = 0x00000001 - MNT_SYNCHRONOUS = 0x00000002 - MNT_NOEXEC = 0x00000004 - MNT_NOSUID = 0x00000008 - MNT_UNION = 0x00000020 - MNT_ASYNC = 0x00000040 - MNT_SUIDDIR = 0x00100000 - MNT_SOFTDEP = 0x00200000 - MNT_NOSYMFOLLOW = 0x00400000 - MNT_GJOURNAL = 0x02000000 - MNT_MULTILABEL = 0x04000000 - MNT_ACLS = 0x08000000 - MNT_NOATIME = 0x10000000 - MNT_NOCLUSTERR = 0x40000000 - MNT_NOCLUSTERW = 0x80000000 - MNT_NFS4ACLS = 0x00000010 - - MNT_WAIT = 1 - MNT_NOWAIT = 2 - MNT_LAZY = 3 - MNT_SUSPEND = 4 ) const ( @@ -51,34 +29,6 @@ type ( _C_long_double int64 ) -type Statfs struct { - Version uint32 - Type uint32 - Flags uint64 - Bsize uint64 - Iosize uint64 - Blocks uint64 - Bfree uint64 - Bavail int64 - Files uint64 - Ffree int64 - Syncwrites uint64 - Asyncwrites uint64 - Syncreads uint64 - Asyncreads uint64 - Spare [10]uint64 - Namemax uint32 - Owner uint32 - Fsid Fsid - Charspare [80]int8 - Fstypename [16]int8 - Mntfromname [88]int8 - Mntonname [88]int8 -} -type Fsid struct { - Val [2]int32 -} - type Devstat struct { Sequence0 uint32 Allocated int32 diff --git a/github.com/shirou/gopsutil/disk/disk_freebsd_arm.go b/github.com/shirou/gopsutil/disk/disk_freebsd_arm.go new file mode 100644 index 0000000000..e2793a4fe6 --- /dev/null +++ b/github.com/shirou/gopsutil/disk/disk_freebsd_arm.go @@ -0,0 +1,62 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs types_freebsd.go + +package disk + +const ( + sizeofPtr = 0x4 + sizeofShort = 0x2 + sizeofInt = 0x4 + sizeofLong = 0x4 + sizeofLongLong = 0x8 + sizeofLongDouble = 0x8 + + DEVSTAT_NO_DATA = 0x00 + DEVSTAT_READ = 0x01 + DEVSTAT_WRITE = 0x02 + DEVSTAT_FREE = 0x03 +) + +const ( + sizeOfDevstat = 0xf0 +) + +type ( + _C_short int16 + _C_int int32 + _C_long int32 + _C_long_long int64 + _C_long_double int64 +) + +type Devstat struct { + Sequence0 uint32 + Allocated int32 + Start_count uint32 + End_count uint32 + Busy_from Bintime + Dev_links _Ctype_struct___0 + Device_number uint32 + Device_name [16]int8 + Unit_number int32 + Bytes [4]uint64 + Operations [4]uint64 + Duration [4]Bintime + Busy_time Bintime + Creation_time Bintime + Block_size uint32 + Tag_types [3]uint64 + Flags uint32 + Device_type uint32 + Priority uint32 + Id *byte + Sequence1 uint32 +} +type Bintime struct { + Sec int32 + Frac uint64 +} + +type _Ctype_struct___0 struct { + Empty uint32 +} diff --git a/github.com/shirou/gopsutil/disk/disk_freebsd_arm64.go b/github.com/shirou/gopsutil/disk/disk_freebsd_arm64.go new file mode 100644 index 0000000000..1384131a8f --- /dev/null +++ b/github.com/shirou/gopsutil/disk/disk_freebsd_arm64.go @@ -0,0 +1,65 @@ +// +build freebsd +// +build arm64 +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs disk/types_freebsd.go + +package disk + +const ( + sizeofPtr = 0x8 + sizeofShort = 0x2 + sizeofInt = 0x4 + sizeofLong = 0x8 + sizeofLongLong = 0x8 + sizeofLongDouble = 0x8 + + DEVSTAT_NO_DATA = 0x00 + DEVSTAT_READ = 0x01 + DEVSTAT_WRITE = 0x02 + DEVSTAT_FREE = 0x03 +) + +const ( + sizeOfDevstat = 0x120 +) + +type ( + _C_short int16 + _C_int int32 + _C_long int64 + _C_long_long int64 + _C_long_double int64 +) + +type Devstat struct { + Sequence0 uint32 + Allocated int32 + Start_count uint32 + End_count uint32 + Busy_from Bintime + Dev_links _Ctype_struct___0 + Device_number uint32 + Device_name [16]int8 + Unit_number int32 + Bytes [4]uint64 + Operations [4]uint64 + Duration [4]Bintime + Busy_time Bintime + Creation_time Bintime + Block_size uint32 + Tag_types [3]uint64 + Flags uint32 + Device_type uint32 + Priority uint32 + Id *byte + Sequence1 uint32 + Pad_cgo_0 [4]byte +} +type Bintime struct { + Sec int64 + Frac uint64 +} + +type _Ctype_struct___0 struct { + Empty uint64 +} diff --git a/github.com/shirou/gopsutil/disk/disk_linux.go b/github.com/shirou/gopsutil/disk/disk_linux.go index 65392503ac..d23a4c4326 100644 --- a/github.com/shirou/gopsutil/disk/disk_linux.go +++ b/github.com/shirou/gopsutil/disk/disk_linux.go @@ -8,13 +8,13 @@ import ( "context" "fmt" "io/ioutil" + "os" "path/filepath" "strconv" "strings" - "golang.org/x/sys/unix" - "github.com/shirou/gopsutil/internal/common" + "golang.org/x/sys/unix" ) const ( @@ -47,6 +47,7 @@ const ( FUSE_SUPER_MAGIC = 0x65735546 FUTEXFS_SUPER_MAGIC = 0xBAD1DEA HFS_SUPER_MAGIC = 0x4244 + HFSPLUS_SUPER_MAGIC = 0x482b HOSTFS_SUPER_MAGIC = 0x00c0ffee HPFS_SUPER_MAGIC = 0xF995E849 HUGETLBFS_MAGIC = 0x958458f6 @@ -156,6 +157,7 @@ var fsTypeMap = map[int64]string{ GFS_SUPER_MAGIC: "gfs/gfs2", /* 0x1161970 remote */ GPFS_SUPER_MAGIC: "gpfs", /* 0x47504653 remote */ HFS_SUPER_MAGIC: "hfs", /* 0x4244 local */ + HFSPLUS_SUPER_MAGIC: "hfsplus", /* 0x482b local */ HPFS_SUPER_MAGIC: "hpfs", /* 0xF995E849 local */ HUGETLBFS_MAGIC: "hugetlbfs", /* 0x958458F6 local */ MTD_INODE_FS_SUPER_MAGIC: "inodefs", /* 0x11307854 local */ @@ -224,30 +226,95 @@ func Partitions(all bool) ([]PartitionStat, error) { } func PartitionsWithContext(ctx context.Context, all bool) ([]PartitionStat, error) { - filename := common.HostProc("self/mounts") + useMounts := false + + filename := common.HostProc("self/mountinfo") lines, err := common.ReadLines(filename) if err != nil { - return nil, err + if err != err.(*os.PathError) { + return nil, err + } + // if kernel does not support self/mountinfo, fallback to self/mounts (<2.6.26) + useMounts = true + filename = common.HostProc("self/mounts") + lines, err = common.ReadLines(filename) + if err != nil { + return nil, err + } } fs, err := getFileSystems() - if err != nil { + if err != nil && !all { return nil, err } ret := make([]PartitionStat, 0, len(lines)) for _, line := range lines { - fields := strings.Fields(line) - d := PartitionStat{ - Device: fields[0], - Mountpoint: unescapeFstab(fields[1]), - Fstype: fields[2], - Opts: fields[3], - } - if all == false { - if d.Device == "none" || !common.StringsHas(fs, d.Fstype) { - continue + var d PartitionStat + if useMounts { + fields := strings.Fields(line) + + d = PartitionStat{ + Device: fields[0], + Mountpoint: unescapeFstab(fields[1]), + Fstype: fields[2], + Opts: fields[3], + } + + if !all { + if d.Device == "none" || !common.StringsHas(fs, d.Fstype) { + continue + } + } + } else { + // a line of self/mountinfo has the following structure: + // 36 35 98:0 /mnt1 /mnt2 rw,noatime master:1 - ext3 /dev/root rw,errors=continue + // (1) (2) (3) (4) (5) (6) (7) (8) (9) (10) (11) + + // split the mountinfo line by the separator hyphen + parts := strings.Split(line, " - ") + if len(parts) != 2 { + return nil, fmt.Errorf("found invalid mountinfo line in file %s: %s ", filename, line) + } + + fields := strings.Fields(parts[0]) + blockDeviceID := fields[2] + mountPoint := fields[4] + mountOpts := fields[5] + + fields = strings.Fields(parts[1]) + fstype := fields[0] + device := fields[1] + + d = PartitionStat{ + Device: device, + Mountpoint: unescapeFstab(mountPoint), + Fstype: fstype, + Opts: mountOpts, + } + + if !all { + if d.Device == "none" || !common.StringsHas(fs, d.Fstype) { + continue + } + } + + if strings.HasPrefix(d.Device, "/dev/mapper/") { + devpath, err := filepath.EvalSymlinks(common.HostDev(strings.Replace(d.Device, "/dev", "", -1))) + if err == nil { + d.Device = devpath + } + } + + // /dev/root is not the real device name + // so we get the real device name from its major/minor number + if d.Device == "/dev/root" { + devpath, err := os.Readlink(common.HostSys("/dev/block/" + blockDeviceID)) + if err != nil { + return nil, err + } + d.Device = strings.Replace(d.Device, "root", filepath.Base(devpath), 1) } } ret = append(ret, d) diff --git a/github.com/shirou/gopsutil/disk/disk_openbsd.go b/github.com/shirou/gopsutil/disk/disk_openbsd.go index 6fdf38632e..cd9e294478 100644 --- a/github.com/shirou/gopsutil/disk/disk_openbsd.go +++ b/github.com/shirou/gopsutil/disk/disk_openbsd.go @@ -7,7 +7,6 @@ import ( "context" "encoding/binary" "path" - "unsafe" "github.com/shirou/gopsutil/internal/common" "golang.org/x/sys/unix" @@ -21,36 +20,45 @@ func PartitionsWithContext(ctx context.Context, all bool) ([]PartitionStat, erro var ret []PartitionStat // get length - count, err := unix.Getfsstat(nil, MNT_WAIT) + count, err := unix.Getfsstat(nil, unix.MNT_WAIT) if err != nil { return ret, err } - fs := make([]Statfs, count) - if _, err = Getfsstat(fs, MNT_WAIT); err != nil { + fs := make([]unix.Statfs_t, count) + if _, err = unix.Getfsstat(fs, unix.MNT_WAIT); err != nil { return ret, err } for _, stat := range fs { opts := "rw" - if stat.F_flags&MNT_RDONLY != 0 { + if stat.F_flags&unix.MNT_RDONLY != 0 { opts = "ro" } - if stat.F_flags&MNT_SYNCHRONOUS != 0 { + if stat.F_flags&unix.MNT_SYNCHRONOUS != 0 { opts += ",sync" } - if stat.F_flags&MNT_NOEXEC != 0 { + if stat.F_flags&unix.MNT_NOEXEC != 0 { opts += ",noexec" } - if stat.F_flags&MNT_NOSUID != 0 { + if stat.F_flags&unix.MNT_NOSUID != 0 { opts += ",nosuid" } - if stat.F_flags&MNT_NODEV != 0 { + if stat.F_flags&unix.MNT_NODEV != 0 { opts += ",nodev" } - if stat.F_flags&MNT_ASYNC != 0 { + if stat.F_flags&unix.MNT_ASYNC != 0 { opts += ",async" } + if stat.F_flags&unix.MNT_SOFTDEP != 0 { + opts += ",softdep" + } + if stat.F_flags&unix.MNT_NOATIME != 0 { + opts += ",noatime" + } + if stat.F_flags&unix.MNT_WXALLOWED != 0 { + opts += ",wxallowed" + } d := PartitionStat{ Device: common.IntToString(stat.F_mntfromname[:]), @@ -114,27 +122,6 @@ func IOCountersWithContext(ctx context.Context, names ...string) (map[string]IOC // BT2LD(time) ((long double)(time).sec + (time).frac * BINTIME_SCALE) -// Getfsstat is borrowed from pkg/syscall/syscall_freebsd.go -// change Statfs_t to Statfs in order to get more information -func Getfsstat(buf []Statfs, flags int) (n int, err error) { - return GetfsstatWithContext(context.Background(), buf, flags) -} - -func GetfsstatWithContext(ctx context.Context, buf []Statfs, flags int) (n int, err error) { - var _p0 unsafe.Pointer - var bufsize uintptr - if len(buf) > 0 { - _p0 = unsafe.Pointer(&buf[0]) - bufsize = unsafe.Sizeof(Statfs{}) * uintptr(len(buf)) - } - r0, _, e1 := unix.Syscall(unix.SYS_GETFSSTAT, uintptr(_p0), bufsize, uintptr(flags)) - n = int(r0) - if e1 != 0 { - err = e1 - } - return -} - func parseDiskstats(buf []byte) (Diskstats, error) { var ds Diskstats br := bytes.NewReader(buf) diff --git a/github.com/shirou/gopsutil/disk/disk_openbsd_386.go b/github.com/shirou/gopsutil/disk/disk_openbsd_386.go new file mode 100644 index 0000000000..2b1c7209d8 --- /dev/null +++ b/github.com/shirou/gopsutil/disk/disk_openbsd_386.go @@ -0,0 +1,35 @@ +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs types_openbsd.go + +package disk + +const ( + DEVSTAT_NO_DATA = 0x00 + DEVSTAT_READ = 0x01 + DEVSTAT_WRITE = 0x02 + DEVSTAT_FREE = 0x03 +) + +const ( + sizeOfDiskstats = 0x60 +) + +type Diskstats struct { + Name [16]int8 + Busy int32 + Rxfer uint64 + Wxfer uint64 + Seek uint64 + Rbytes uint64 + Wbytes uint64 + Attachtime Timeval + Timestamp Timeval + Time Timeval +} +type Timeval struct { + Sec int64 + Usec int32 +} + +type Diskstat struct{} +type Bintime struct{} diff --git a/github.com/shirou/gopsutil/disk/disk_openbsd_amd64.go b/github.com/shirou/gopsutil/disk/disk_openbsd_amd64.go index 07a845fbcb..7c9ceaa8db 100644 --- a/github.com/shirou/gopsutil/disk/disk_openbsd_amd64.go +++ b/github.com/shirou/gopsutil/disk/disk_openbsd_amd64.go @@ -1,71 +1,19 @@ -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs types_openbsd.go package disk const ( - sizeofPtr = 0x8 - sizeofShort = 0x2 - sizeofInt = 0x4 - sizeofLong = 0x8 - sizeofLongLong = 0x8 - sizeofLongDouble = 0x8 - DEVSTAT_NO_DATA = 0x00 DEVSTAT_READ = 0x01 DEVSTAT_WRITE = 0x02 DEVSTAT_FREE = 0x03 - - MNT_RDONLY = 0x00000001 - MNT_SYNCHRONOUS = 0x00000002 - MNT_NOEXEC = 0x00000004 - MNT_NOSUID = 0x00000008 - MNT_NODEV = 0x00000010 - MNT_ASYNC = 0x00000040 - - MNT_WAIT = 1 - MNT_NOWAIT = 2 - MNT_LAZY = 3 ) const ( sizeOfDiskstats = 0x70 ) -type ( - _C_short int16 - _C_int int32 - _C_long int64 - _C_long_long int64 - _C_long_double int64 -) - -type Statfs struct { - F_flags uint32 - F_bsize uint32 - F_iosize uint32 - Pad_cgo_0 [4]byte - F_blocks uint64 - F_bfree uint64 - F_bavail int64 - F_files uint64 - F_ffree uint64 - F_favail int64 - F_syncwrites uint64 - F_syncreads uint64 - F_asyncwrites uint64 - F_asyncreads uint64 - F_fsid Fsid - F_namemax uint32 - F_owner uint32 - F_ctime uint64 - F_fstypename [16]int8 - F_mntonname [90]int8 - F_mntfromname [90]int8 - F_mntfromspec [90]int8 - Pad_cgo_1 [2]byte - Mount_info [160]byte -} type Diskstats struct { Name [16]int8 Busy int32 @@ -79,9 +27,6 @@ type Diskstats struct { Timestamp Timeval Time Timeval } -type Fsid struct { - Val [2]int32 -} type Timeval struct { Sec int64 Usec int64 diff --git a/github.com/shirou/gopsutil/disk/disk_unix.go b/github.com/shirou/gopsutil/disk/disk_unix.go index 9b499b5287..86ab99cb65 100644 --- a/github.com/shirou/gopsutil/disk/disk_unix.go +++ b/github.com/shirou/gopsutil/disk/disk_unix.go @@ -9,7 +9,7 @@ import ( "golang.org/x/sys/unix" ) -// Usage returns a file system usage. path is a filessytem path such +// Usage returns a file system usage. path is a filesystem path such // as "/", not device file path like "/dev/vda1". If you want to use // a return value of disk.Partitions, use "Mountpoint" not "Device". func Usage(path string) (*UsageStat, error) { diff --git a/github.com/shirou/gopsutil/disk/disk_windows.go b/github.com/shirou/gopsutil/disk/disk_windows.go index 081443b7de..f871e9d8c3 100644 --- a/github.com/shirou/gopsutil/disk/disk_windows.go +++ b/github.com/shirou/gopsutil/disk/disk_windows.go @@ -5,6 +5,8 @@ package disk import ( "bytes" "context" + "fmt" + "syscall" "unsafe" "github.com/shirou/gopsutil/internal/common" @@ -23,25 +25,29 @@ var ( FileReadOnlyVolume = int64(524288) // 0x00080000 ) -type Win32_PerfFormattedData struct { - Name string - AvgDiskBytesPerRead uint64 - AvgDiskBytesPerWrite uint64 - AvgDiskReadQueueLength uint64 - AvgDiskWriteQueueLength uint64 - AvgDisksecPerRead uint64 - AvgDisksecPerWrite uint64 +// diskPerformance is an equivalent representation of DISK_PERFORMANCE in the Windows API. +// https://docs.microsoft.com/fr-fr/windows/win32/api/winioctl/ns-winioctl-disk_performance +type diskPerformance struct { + BytesRead int64 + BytesWritten int64 + ReadTime int64 + WriteTime int64 + IdleTime int64 + ReadCount uint32 + WriteCount uint32 + QueueDepth uint32 + SplitCount uint32 + QueryTime int64 + StorageDeviceNumber uint32 + StorageManagerName [8]uint16 + alignmentPadding uint32 // necessary for 32bit support, see https://github.com/elastic/beats/pull/16553 } -const WaitMSec = 500 - func Usage(path string) (*UsageStat, error) { return UsageWithContext(context.Background(), path) } func UsageWithContext(ctx context.Context, path string) (*UsageStat, error) { - ret := &UsageStat{} - lpFreeBytesAvailable := int64(0) lpTotalNumberOfBytes := int64(0) lpTotalNumberOfFreeBytes := int64(0) @@ -53,7 +59,7 @@ func UsageWithContext(ctx context.Context, path string) (*UsageStat, error) { if diskret == 0 { return nil, err } - ret = &UsageStat{ + ret := &UsageStat{ Path: path, Total: uint64(lpTotalNumberOfBytes), Free: uint64(lpTotalNumberOfFreeBytes), @@ -138,31 +144,52 @@ func IOCounters(names ...string) (map[string]IOCountersStat, error) { } func IOCountersWithContext(ctx context.Context, names ...string) (map[string]IOCountersStat, error) { - ret := make(map[string]IOCountersStat, 0) - var dst []Win32_PerfFormattedData + // https://github.com/giampaolo/psutil/blob/544e9daa4f66a9f80d7bf6c7886d693ee42f0a13/psutil/arch/windows/disk.c#L83 + drivemap := make(map[string]IOCountersStat, 0) + var diskPerformance diskPerformance - err := common.WMIQueryWithContext(ctx, "SELECT * FROM Win32_PerfFormattedData_PerfDisk_LogicalDisk", &dst) + lpBuffer := make([]uint16, 254) + lpBufferLen, err := windows.GetLogicalDriveStrings(uint32(len(lpBuffer)), &lpBuffer[0]) if err != nil { - return ret, err + return drivemap, err } - for _, d := range dst { - if len(d.Name) > 3 { // not get _Total or Harddrive - continue - } - - if len(names) > 0 && !common.StringsHas(names, d.Name) { - continue - } + for _, v := range lpBuffer[:lpBufferLen] { + if 'A' <= v && v <= 'Z' { + path := string(v) + ":" + typepath, _ := windows.UTF16PtrFromString(path) + typeret := windows.GetDriveType(typepath) + if typeret == 0 { + return drivemap, windows.GetLastError() + } + if typeret != windows.DRIVE_FIXED { + continue + } + szDevice := fmt.Sprintf(`\\.\%s`, path) + const IOCTL_DISK_PERFORMANCE = 0x70020 + h, err := windows.CreateFile(syscall.StringToUTF16Ptr(szDevice), 0, windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE, nil, windows.OPEN_EXISTING, 0, 0) + if err != nil { + if err == windows.ERROR_FILE_NOT_FOUND { + continue + } + return drivemap, err + } + defer windows.CloseHandle(h) - ret[d.Name] = IOCountersStat{ - Name: d.Name, - ReadCount: uint64(d.AvgDiskReadQueueLength), - WriteCount: d.AvgDiskWriteQueueLength, - ReadBytes: uint64(d.AvgDiskBytesPerRead), - WriteBytes: uint64(d.AvgDiskBytesPerWrite), - ReadTime: d.AvgDisksecPerRead, - WriteTime: d.AvgDisksecPerWrite, + var diskPerformanceSize uint32 + err = windows.DeviceIoControl(h, IOCTL_DISK_PERFORMANCE, nil, 0, (*byte)(unsafe.Pointer(&diskPerformance)), uint32(unsafe.Sizeof(diskPerformance)), &diskPerformanceSize, nil) + if err != nil { + return drivemap, err + } + drivemap[path] = IOCountersStat{ + ReadBytes: uint64(diskPerformance.BytesRead), + WriteBytes: uint64(diskPerformance.BytesWritten), + ReadCount: uint64(diskPerformance.ReadCount), + WriteCount: uint64(diskPerformance.WriteCount), + ReadTime: uint64(diskPerformance.ReadTime / 10000 / 1000), // convert to ms: https://github.com/giampaolo/psutil/issues/1012 + WriteTime: uint64(diskPerformance.WriteTime / 10000 / 1000), + Name: path, + } } } - return ret, nil + return drivemap, nil } diff --git a/github.com/shirou/gopsutil/disk/iostat_darwin.c b/github.com/shirou/gopsutil/disk/iostat_darwin.c new file mode 100644 index 0000000000..9619c6f47c --- /dev/null +++ b/github.com/shirou/gopsutil/disk/iostat_darwin.c @@ -0,0 +1,131 @@ +// https://github.com/lufia/iostat/blob/9f7362b77ad333b26c01c99de52a11bdb650ded2/iostat_darwin.c +#include +#include +#include "iostat_darwin.h" + +#define IOKIT 1 /* to get io_name_t in device_types.h */ + +#include +#include +#include +#include + +#include + +static int getdrivestat(io_registry_entry_t d, DriveStats *stat); +static int fillstat(io_registry_entry_t d, DriveStats *stat); + +int +readdrivestat(DriveStats a[], int n) +{ + mach_port_t port; + CFMutableDictionaryRef match; + io_iterator_t drives; + io_registry_entry_t d; + kern_return_t status; + int na, rv; + + IOMasterPort(bootstrap_port, &port); + match = IOServiceMatching("IOMedia"); + CFDictionaryAddValue(match, CFSTR(kIOMediaWholeKey), kCFBooleanTrue); + status = IOServiceGetMatchingServices(port, match, &drives); + if(status != KERN_SUCCESS) + return -1; + + na = 0; + while(na < n && (d=IOIteratorNext(drives)) > 0){ + rv = getdrivestat(d, &a[na]); + if(rv < 0) + return -1; + if(rv > 0) + na++; + IOObjectRelease(d); + } + IOObjectRelease(drives); + return na; +} + +static int +getdrivestat(io_registry_entry_t d, DriveStats *stat) +{ + io_registry_entry_t parent; + kern_return_t status; + CFDictionaryRef props; + CFStringRef name; + CFNumberRef num; + int rv; + + memset(stat, 0, sizeof *stat); + status = IORegistryEntryGetParentEntry(d, kIOServicePlane, &parent); + if(status != KERN_SUCCESS) + return -1; + if(!IOObjectConformsTo(parent, "IOBlockStorageDriver")){ + IOObjectRelease(parent); + return 0; + } + + status = IORegistryEntryCreateCFProperties(d, (CFMutableDictionaryRef *)&props, kCFAllocatorDefault, kNilOptions); + if(status != KERN_SUCCESS){ + IOObjectRelease(parent); + return -1; + } + name = (CFStringRef)CFDictionaryGetValue(props, CFSTR(kIOBSDNameKey)); + CFStringGetCString(name, stat->name, NAMELEN, CFStringGetSystemEncoding()); + num = (CFNumberRef)CFDictionaryGetValue(props, CFSTR(kIOMediaSizeKey)); + CFNumberGetValue(num, kCFNumberSInt64Type, &stat->size); + num = (CFNumberRef)CFDictionaryGetValue(props, CFSTR(kIOMediaPreferredBlockSizeKey)); + CFNumberGetValue(num, kCFNumberSInt64Type, &stat->blocksize); + CFRelease(props); + + rv = fillstat(parent, stat); + IOObjectRelease(parent); + if(rv < 0) + return -1; + return 1; +} + +static struct { + char *key; + size_t off; +} statstab[] = { + {kIOBlockStorageDriverStatisticsBytesReadKey, offsetof(DriveStats, read)}, + {kIOBlockStorageDriverStatisticsBytesWrittenKey, offsetof(DriveStats, written)}, + {kIOBlockStorageDriverStatisticsReadsKey, offsetof(DriveStats, nread)}, + {kIOBlockStorageDriverStatisticsWritesKey, offsetof(DriveStats, nwrite)}, + {kIOBlockStorageDriverStatisticsTotalReadTimeKey, offsetof(DriveStats, readtime)}, + {kIOBlockStorageDriverStatisticsTotalWriteTimeKey, offsetof(DriveStats, writetime)}, + {kIOBlockStorageDriverStatisticsLatentReadTimeKey, offsetof(DriveStats, readlat)}, + {kIOBlockStorageDriverStatisticsLatentWriteTimeKey, offsetof(DriveStats, writelat)}, +}; + +static int +fillstat(io_registry_entry_t d, DriveStats *stat) +{ + CFDictionaryRef props, v; + CFNumberRef num; + kern_return_t status; + typeof(statstab[0]) *bp, *ep; + + status = IORegistryEntryCreateCFProperties(d, (CFMutableDictionaryRef *)&props, kCFAllocatorDefault, kNilOptions); + if(status != KERN_SUCCESS) + return -1; + v = (CFDictionaryRef)CFDictionaryGetValue(props, CFSTR(kIOBlockStorageDriverStatisticsKey)); + if(v == NULL){ + CFRelease(props); + return -1; + } + + ep = &statstab[sizeof(statstab)/sizeof(statstab[0])]; + for(bp = &statstab[0]; bp < ep; bp++){ + CFStringRef s; + + s = CFStringCreateWithCString(kCFAllocatorDefault, bp->key, CFStringGetSystemEncoding()); + num = (CFNumberRef)CFDictionaryGetValue(v, s); + if(num) + CFNumberGetValue(num, kCFNumberSInt64Type, ((char*)stat)+bp->off); + CFRelease(s); + } + + CFRelease(props); + return 0; +} diff --git a/github.com/shirou/gopsutil/disk/iostat_darwin.h b/github.com/shirou/gopsutil/disk/iostat_darwin.h new file mode 100644 index 0000000000..c7208499d2 --- /dev/null +++ b/github.com/shirou/gopsutil/disk/iostat_darwin.h @@ -0,0 +1,33 @@ +// https://github.com/lufia/iostat/blob/9f7362b77ad333b26c01c99de52a11bdb650ded2/iostat_darwin.h +typedef struct DriveStats DriveStats; +typedef struct CPUStats CPUStats; + +enum { + NDRIVE = 16, + NAMELEN = 31 +}; + +struct DriveStats { + char name[NAMELEN+1]; + int64_t size; + int64_t blocksize; + + int64_t read; + int64_t written; + int64_t nread; + int64_t nwrite; + int64_t readtime; + int64_t writetime; + int64_t readlat; + int64_t writelat; +}; + +struct CPUStats { + natural_t user; + natural_t nice; + natural_t sys; + natural_t idle; +}; + +extern int readdrivestat(DriveStats a[], int n); +extern int readcpustat(CPUStats *cpu); diff --git a/github.com/shirou/gopsutil/host/freebsd_headers/utxdb.h b/github.com/shirou/gopsutil/host/freebsd_headers/utxdb.h new file mode 100644 index 0000000000..912dd0fa11 --- /dev/null +++ b/github.com/shirou/gopsutil/host/freebsd_headers/utxdb.h @@ -0,0 +1,63 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2010 Ed Schouten + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _UTXDB_H_ +#define _UTXDB_H_ + +#include + +#define _PATH_UTX_ACTIVE "/var/run/utx.active" +#define _PATH_UTX_LASTLOGIN "/var/log/utx.lastlogin" +#define _PATH_UTX_LOG "/var/log/utx.log" + +/* + * Entries in struct futx are ordered by how often they are used. In + * utx.log only entries will be written until the last non-zero byte, + * which means we want to put the hostname at the end. Most primitive + * records only store a ut_type and ut_tv, which means we want to store + * those at the front. + */ + +struct utmpx; + +struct futx { + uint8_t fu_type; + uint64_t fu_tv; + char fu_id[8]; + uint32_t fu_pid; + char fu_user[32]; + char fu_line[16]; + char fu_host[128]; +} __packed; + +void utx_to_futx(const struct utmpx *, struct futx *); +struct utmpx *futx_to_utx(const struct futx *); + +#endif /* !_UTXDB_H_ */ diff --git a/github.com/shirou/gopsutil/host/host.go b/github.com/shirou/gopsutil/host/host.go index 1e9e9bb686..e100bc5f71 100644 --- a/github.com/shirou/gopsutil/host/host.go +++ b/github.com/shirou/gopsutil/host/host.go @@ -20,6 +20,7 @@ type InfoStat struct { PlatformFamily string `json:"platformFamily"` // ex: debian, rhel PlatformVersion string `json:"platformVersion"` // version of the complete OS KernelVersion string `json:"kernelVersion"` // version of the OS kernel (if available) + KernelArch string `json:"kernelArch"` // native cpu architecture queried at runtime, as returned by `uname -m` or empty string in case of error VirtualizationSystem string `json:"virtualizationSystem"` VirtualizationRole string `json:"virtualizationRole"` // guest or host HostID string `json:"hostid"` // ex: uuid diff --git a/github.com/shirou/gopsutil/host/host_darwin.go b/github.com/shirou/gopsutil/host/host_darwin.go index 8241fc08bb..f019e3e791 100644 --- a/github.com/shirou/gopsutil/host/host_darwin.go +++ b/github.com/shirou/gopsutil/host/host_darwin.go @@ -10,7 +10,6 @@ import ( "os" "os/exec" "runtime" - "strconv" "strings" "sync/atomic" "time" @@ -18,6 +17,7 @@ import ( "github.com/shirou/gopsutil/internal/common" "github.com/shirou/gopsutil/process" + "golang.org/x/sys/unix" ) // from utmpx.h @@ -43,6 +43,11 @@ func InfoWithContext(ctx context.Context) (*InfoStat, error) { ret.KernelVersion = kernelVersion } + kernelArch, err := kernelArch() + if err == nil { + ret.KernelArch = kernelArch + } + platform, family, pver, err := PlatformInformation() if err == nil { ret.Platform = platform @@ -67,9 +72,9 @@ func InfoWithContext(ctx context.Context) (*InfoStat, error) { ret.Procs = uint64(len(procs)) } - values, err := common.DoSysctrlWithContext(ctx, "kern.uuid") - if err == nil && len(values) == 1 && values[0] != "" { - ret.HostID = strings.ToLower(values[0]) + uuid, err := unix.Sysctl("kern.uuid") + if err == nil && uuid != "" { + ret.HostID = strings.ToLower(uuid) } return ret, nil @@ -83,24 +88,22 @@ func BootTime() (uint64, error) { } func BootTimeWithContext(ctx context.Context) (uint64, error) { + // https://github.com/AaronO/dashd/blob/222e32ef9f7a1f9bea4a8da2c3627c4cb992f860/probe/probe_darwin.go t := atomic.LoadUint64(&cachedBootTime) if t != 0 { return t, nil } - values, err := common.DoSysctrlWithContext(ctx, "kern.boottime") - if err != nil { - return 0, err - } - // ex: { sec = 1392261637, usec = 627534 } Thu Feb 13 12:20:37 2014 - v := strings.Replace(values[2], ",", "", 1) - boottime, err := strconv.ParseInt(v, 10, 64) + value, err := unix.Sysctl("kern.boottime") if err != nil { return 0, err } - t = uint64(boottime) - atomic.StoreUint64(&cachedBootTime, t) + bytes := []byte(value[:]) + var boottime uint64 + boottime = uint64(bytes[0]) + uint64(bytes[1])*256 + uint64(bytes[2])*256*256 + uint64(bytes[3])*256*256*256 - return t, nil + atomic.StoreUint64(&cachedBootTime, boottime) + + return boottime, nil } func uptime(boot uint64) uint64 { @@ -112,7 +115,7 @@ func Uptime() (uint64, error) { } func UptimeWithContext(ctx context.Context) (uint64, error) { - boot, err := BootTime() + boot, err := BootTimeWithContext(ctx) if err != nil { return 0, err } @@ -180,21 +183,27 @@ func PlatformInformationWithContext(ctx context.Context) (string, string, string if err != nil { return "", "", "", err } - uname, err := exec.LookPath("uname") - if err != nil { - return "", "", "", err - } - out, err := invoke.CommandWithContext(ctx, uname, "-s") + p, err := unix.Sysctl("kern.ostype") if err == nil { - platform = strings.ToLower(strings.TrimSpace(string(out))) + platform = strings.ToLower(p) } - out, err = invoke.CommandWithContext(ctx, sw_vers, "-productVersion") + out, err := invoke.CommandWithContext(ctx, sw_vers, "-productVersion") if err == nil { pver = strings.ToLower(strings.TrimSpace(string(out))) } + // check if the macos server version file exists + _, err = os.Stat("/System/Library/CoreServices/ServerVersion.plist") + + // server file doesn't exist + if os.IsNotExist(err) { + family = "Standalone Workstation" + } else { + family = "Server" + } + return platform, family, pver, nil } @@ -211,22 +220,6 @@ func KernelVersion() (string, error) { } func KernelVersionWithContext(ctx context.Context) (string, error) { - uname, err := exec.LookPath("uname") - if err != nil { - return "", err - } - out, err := invoke.CommandWithContext(ctx, uname, "-r") - if err != nil { - return "", err - } - version := strings.ToLower(strings.TrimSpace(string(out))) - return version, err -} - -func SensorsTemperatures() ([]TemperatureStat, error) { - return SensorsTemperaturesWithContext(context.Background()) -} - -func SensorsTemperaturesWithContext(ctx context.Context) ([]TemperatureStat, error) { - return []TemperatureStat{}, common.ErrNotImplementedError + version, err := unix.Sysctl("kern.osrelease") + return strings.ToLower(version), err } diff --git a/github.com/shirou/gopsutil/host/host_darwin_cgo.go b/github.com/shirou/gopsutil/host/host_darwin_cgo.go new file mode 100644 index 0000000000..9b3b1c4369 --- /dev/null +++ b/github.com/shirou/gopsutil/host/host_darwin_cgo.go @@ -0,0 +1,51 @@ +// +build darwin +// +build cgo + +package host + +// #cgo LDFLAGS: -framework IOKit +// #include "smc_darwin.h" +import "C" +import "context" + +func SensorsTemperatures() ([]TemperatureStat, error) { + return SensorsTemperaturesWithContext(context.Background()) +} + +func SensorsTemperaturesWithContext(ctx context.Context) ([]TemperatureStat, error) { + temperatureKeys := []string{ + C.AMBIENT_AIR_0, + C.AMBIENT_AIR_1, + C.CPU_0_DIODE, + C.CPU_0_HEATSINK, + C.CPU_0_PROXIMITY, + C.ENCLOSURE_BASE_0, + C.ENCLOSURE_BASE_1, + C.ENCLOSURE_BASE_2, + C.ENCLOSURE_BASE_3, + C.GPU_0_DIODE, + C.GPU_0_HEATSINK, + C.GPU_0_PROXIMITY, + C.HARD_DRIVE_BAY, + C.MEMORY_SLOT_0, + C.MEMORY_SLOTS_PROXIMITY, + C.NORTHBRIDGE, + C.NORTHBRIDGE_DIODE, + C.NORTHBRIDGE_PROXIMITY, + C.THUNDERBOLT_0, + C.THUNDERBOLT_1, + C.WIRELESS_MODULE, + } + var temperatures []TemperatureStat + + C.open_smc() + defer C.close_smc() + + for _, key := range temperatureKeys { + temperatures = append(temperatures, TemperatureStat{ + SensorKey: key, + Temperature: float64(C.get_temperature(C.CString(key))), + }) + } + return temperatures, nil +} diff --git a/github.com/shirou/gopsutil/host/host_darwin_nocgo.go b/github.com/shirou/gopsutil/host/host_darwin_nocgo.go new file mode 100644 index 0000000000..7869f8c598 --- /dev/null +++ b/github.com/shirou/gopsutil/host/host_darwin_nocgo.go @@ -0,0 +1,18 @@ +// +build darwin +// +build !cgo + +package host + +import ( + "context" + + "github.com/shirou/gopsutil/internal/common" +) + +func SensorsTemperatures() ([]TemperatureStat, error) { + return SensorsTemperaturesWithContext(context.Background()) +} + +func SensorsTemperaturesWithContext(ctx context.Context) ([]TemperatureStat, error) { + return []TemperatureStat{}, common.ErrNotImplementedError +} diff --git a/github.com/shirou/gopsutil/host/host_freebsd.go b/github.com/shirou/gopsutil/host/host_freebsd.go index 00a8519068..6dc4bc182a 100644 --- a/github.com/shirou/gopsutil/host/host_freebsd.go +++ b/github.com/shirou/gopsutil/host/host_freebsd.go @@ -7,6 +7,7 @@ import ( "context" "encoding/binary" "io/ioutil" + "math" "os" "runtime" "strings" @@ -49,6 +50,11 @@ func InfoWithContext(ctx context.Context) (*InfoStat, error) { ret.KernelVersion = version } + kernelArch, err := kernelArch() + if err == nil { + ret.KernelArch = kernelArch + } + system, role, err := Virtualization() if err == nil { ret.VirtualizationSystem = system @@ -143,11 +149,11 @@ func UsersWithContext(ctx context.Context) ([]UserStat, error) { b := buf[i*sizeOfUtmpx : (i+1)*sizeOfUtmpx] var u Utmpx br := bytes.NewReader(b) - err := binary.Read(br, binary.LittleEndian, &u) + err := binary.Read(br, binary.BigEndian, &u) if err != nil || u.Type != 4 { continue } - sec := (binary.LittleEndian.Uint32(u.Tv.Sec[:])) / 2 // TODO: + sec := math.Floor(float64(u.Tv) / 1000000) user := UserStat{ User: common.IntToString(u.User[:]), Terminal: common.IntToString(u.Line[:]), diff --git a/github.com/shirou/gopsutil/host/host_freebsd_386.go b/github.com/shirou/gopsutil/host/host_freebsd_386.go index 7f06d8f8e2..88453d2a27 100644 --- a/github.com/shirou/gopsutil/host/host_freebsd_386.go +++ b/github.com/shirou/gopsutil/host/host_freebsd_386.go @@ -1,4 +1,4 @@ -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs types_freebsd.go package host @@ -9,7 +9,7 @@ const ( sizeofInt = 0x4 sizeofLong = 0x4 sizeofLongLong = 0x8 - sizeOfUtmpx = 197 // TODO why should 197 + sizeOfUtmpx = 0xc5 ) type ( @@ -27,17 +27,11 @@ type Utmp struct { } type Utmpx struct { - Type int16 - Tv Timeval + Type uint8 + Tv uint64 Id [8]int8 - Pid int32 + Pid uint32 User [32]int8 Line [16]int8 - Host [125]int8 - // X__ut_spare [64]int8 -} - -type Timeval struct { - Sec [4]byte - Usec [3]byte + Host [128]int8 } diff --git a/github.com/shirou/gopsutil/host/host_freebsd_amd64.go b/github.com/shirou/gopsutil/host/host_freebsd_amd64.go index 3f015f0fb5..8af74b0fe0 100644 --- a/github.com/shirou/gopsutil/host/host_freebsd_amd64.go +++ b/github.com/shirou/gopsutil/host/host_freebsd_amd64.go @@ -1,4 +1,4 @@ -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs types_freebsd.go package host @@ -9,7 +9,7 @@ const ( sizeofInt = 0x4 sizeofLong = 0x8 sizeofLongLong = 0x8 - sizeOfUtmpx = 197 // TODO: why should 197, not 0x118 + sizeOfUtmpx = 0xc5 ) type ( @@ -27,18 +27,11 @@ type Utmp struct { } type Utmpx struct { - Type int16 - Tv Timeval + Type uint8 + Tv uint64 Id [8]int8 - Pid int32 + Pid uint32 User [32]int8 Line [16]int8 - Host [125]int8 - // Host [128]int8 - // X__ut_spare [64]int8 -} - -type Timeval struct { - Sec [4]byte - Usec [3]byte + Host [128]int8 } diff --git a/github.com/shirou/gopsutil/host/host_freebsd_arm.go b/github.com/shirou/gopsutil/host/host_freebsd_arm.go index ac74980ae4..f7d6ede554 100644 --- a/github.com/shirou/gopsutil/host/host_freebsd_arm.go +++ b/github.com/shirou/gopsutil/host/host_freebsd_arm.go @@ -1,4 +1,4 @@ -// Created by cgo -godefs - DO NOT EDIT +// Code generated by cmd/cgo -godefs; DO NOT EDIT. // cgo -godefs types_freebsd.go package host @@ -9,7 +9,7 @@ const ( sizeofInt = 0x4 sizeofLong = 0x8 sizeofLongLong = 0x8 - sizeOfUtmpx = 197 // TODO: why should 197, not 0x118 + sizeOfUtmpx = 0xc5 ) type ( @@ -27,18 +27,11 @@ type Utmp struct { } type Utmpx struct { - Type int16 - Tv Timeval + Type uint8 + Tv uint64 Id [8]int8 - Pid int32 + Pid uint32 User [32]int8 Line [16]int8 - Host [125]int8 - // Host [128]int8 - // X__ut_spare [64]int8 -} - -type Timeval struct { - Sec [4]byte - Usec [3]byte + Host [128]int8 } diff --git a/github.com/shirou/gopsutil/host/host_freebsd_arm64.go b/github.com/shirou/gopsutil/host/host_freebsd_arm64.go new file mode 100644 index 0000000000..88dc11fca1 --- /dev/null +++ b/github.com/shirou/gopsutil/host/host_freebsd_arm64.go @@ -0,0 +1,39 @@ +// +build freebsd +// +build arm64 +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs host/types_freebsd.go + +package host + +const ( + sizeofPtr = 0x8 + sizeofShort = 0x2 + sizeofInt = 0x4 + sizeofLong = 0x8 + sizeofLongLong = 0x8 + sizeOfUtmpx = 0xc5 +) + +type ( + _C_short int16 + _C_int int32 + _C_long int64 + _C_long_long int64 +) + +type Utmp struct { + Line [8]int8 + Name [16]int8 + Host [16]int8 + Time int32 +} + +type Utmpx struct { + Type uint8 + Tv uint64 + Id [8]int8 + Pid uint32 + User [32]int8 + Line [16]int8 + Host [128]int8 +} diff --git a/github.com/shirou/gopsutil/host/host_linux.go b/github.com/shirou/gopsutil/host/host_linux.go index 2c1ac6ba72..6c76f905bf 100644 --- a/github.com/shirou/gopsutil/host/host_linux.go +++ b/github.com/shirou/gopsutil/host/host_linux.go @@ -15,10 +15,10 @@ import ( "runtime" "strconv" "strings" - "sync/atomic" "time" "github.com/shirou/gopsutil/internal/common" + "golang.org/x/sys/unix" ) type LSB struct { @@ -56,6 +56,11 @@ func InfoWithContext(ctx context.Context) (*InfoStat, error) { ret.KernelVersion = kernelVersion } + kernelArch, err := kernelArch() + if err == nil { + ret.KernelArch = kernelArch + } + system, role, err := Virtualization() if err == nil { ret.VirtualizationSystem = system @@ -74,6 +79,7 @@ func InfoWithContext(ctx context.Context) (*InfoStat, error) { sysProductUUID := common.HostSys("class/dmi/id/product_uuid") machineID := common.HostEtc("machine-id") + procSysKernelRandomBootID := common.HostProc("sys/kernel/random/boot_id") switch { // In order to read this file, needs to be supported by kernel/arch and run as root // so having fallback is important @@ -95,80 +101,22 @@ func InfoWithContext(ctx context.Context) (*InfoStat, error) { fallthrough // Not stable between reboot, but better than nothing default: - values, err := common.DoSysctrl("kernel.random.boot_id") - if err == nil && len(values) == 1 && values[0] != "" { - ret.HostID = strings.ToLower(values[0]) + lines, err := common.ReadLines(procSysKernelRandomBootID) + if err == nil && len(lines) > 0 && lines[0] != "" { + ret.HostID = strings.ToLower(lines[0]) } } return ret, nil } -// cachedBootTime must be accessed via atomic.Load/StoreUint64 -var cachedBootTime uint64 - // BootTime returns the system boot time expressed in seconds since the epoch. func BootTime() (uint64, error) { return BootTimeWithContext(context.Background()) } func BootTimeWithContext(ctx context.Context) (uint64, error) { - t := atomic.LoadUint64(&cachedBootTime) - if t != 0 { - return t, nil - } - - system, role, err := Virtualization() - if err != nil { - return 0, err - } - - statFile := "stat" - if system == "lxc" && role == "guest" { - // if lxc, /proc/uptime is used. - statFile = "uptime" - } else if system == "docker" && role == "guest" { - // also docker, guest - statFile = "uptime" - } - - filename := common.HostProc(statFile) - lines, err := common.ReadLines(filename) - if err != nil { - return 0, err - } - - if statFile == "stat" { - for _, line := range lines { - if strings.HasPrefix(line, "btime") { - f := strings.Fields(line) - if len(f) != 2 { - return 0, fmt.Errorf("wrong btime format") - } - b, err := strconv.ParseInt(f[1], 10, 64) - if err != nil { - return 0, err - } - t = uint64(b) - atomic.StoreUint64(&cachedBootTime, t) - return t, nil - } - } - } else if statFile == "uptime" { - if len(lines) != 1 { - return 0, fmt.Errorf("wrong uptime format") - } - f := strings.Fields(lines[0]) - b, err := strconv.ParseFloat(f[0], 64) - if err != nil { - return 0, err - } - t = uint64(time.Now().Unix()) - uint64(b) - atomic.StoreUint64(&cachedBootTime, t) - return t, nil - } - - return 0, fmt.Errorf("could not find btime") + return common.BootTimeWithContext(ctx) } func uptime(boot uint64) uint64 { @@ -234,26 +182,6 @@ func UsersWithContext(ctx context.Context) ([]UserStat, error) { } -func getOSRelease() (platform string, version string, err error) { - contents, err := common.ReadLines(common.HostEtc("os-release")) - if err != nil { - return "", "", nil // return empty - } - for _, line := range contents { - field := strings.Split(line, "=") - if len(field) < 2 { - continue - } - switch field[0] { - case "ID": // use ID for lowercase - platform = field[1] - case "VERSION": - version = field[1] - } - } - return platform, version, nil -} - func getLSB() (*LSB, error) { ret := &LSB{} if common.PathExists(common.HostEtc("lsb-release")) { @@ -278,7 +206,7 @@ func getLSB() (*LSB, error) { } } } else if common.PathExists("/usr/bin/lsb_release") { - lsb_release, err := exec.LookPath("/usr/bin/lsb_release") + lsb_release, err := exec.LookPath("lsb_release") if err != nil { return ret, err } @@ -352,7 +280,7 @@ func PlatformInformationWithContext(ctx context.Context) (platform string, famil platform = "debian" } contents, err := common.ReadLines(common.HostEtc("debian_version")) - if err == nil { + if err == nil && len(contents) > 0 && contents[0] != "" { version = contents[0] } } @@ -387,11 +315,11 @@ func PlatformInformationWithContext(ctx context.Context) (platform string, famil } else if common.PathExists(common.HostEtc("alpine-release")) { platform = "alpine" contents, err := common.ReadLines(common.HostEtc("alpine-release")) - if err == nil && len(contents) > 0 { + if err == nil && len(contents) > 0 && contents[0] != "" { version = contents[0] } } else if common.PathExists(common.HostEtc("os-release")) { - p, v, err := getOSRelease() + p, v, err := common.GetOSRelease() if err == nil { platform = p version = v @@ -420,7 +348,7 @@ func PlatformInformationWithContext(ctx context.Context) (platform string, famil family = "fedora" case "oracle", "centos", "redhat", "scientific", "enterpriseenterprise", "amazon", "xenserver", "cloudlinux", "ibm_powerkvm": family = "rhel" - case "suse", "opensuse": + case "suse", "opensuse", "sles": family = "suse" case "gentoo": family = "gentoo" @@ -434,6 +362,8 @@ func PlatformInformationWithContext(ctx context.Context) (platform string, famil family = "alpine" case "coreos": family = "coreos" + case "solus": + family = "solus" } return platform, family, version, nil @@ -445,19 +375,12 @@ func KernelVersion() (version string, err error) { } func KernelVersionWithContext(ctx context.Context) (version string, err error) { - filename := common.HostProc("sys/kernel/osrelease") - if common.PathExists(filename) { - contents, err := common.ReadLines(filename) - if err != nil { - return "", err - } - - if len(contents) > 0 { - version = contents[0] - } + var utsname unix.Utsname + err = unix.Uname(&utsname) + if err != nil { + return "", err } - - return version, nil + return string(utsname.Release[:bytes.IndexByte(utsname.Release[:], 0)]), nil } func getSlackwareVersion(contents []string) string { @@ -514,106 +437,7 @@ func Virtualization() (string, string, error) { } func VirtualizationWithContext(ctx context.Context) (string, string, error) { - var system string - var role string - - filename := common.HostProc("xen") - if common.PathExists(filename) { - system = "xen" - role = "guest" // assume guest - - if common.PathExists(filepath.Join(filename, "capabilities")) { - contents, err := common.ReadLines(filepath.Join(filename, "capabilities")) - if err == nil { - if common.StringsContains(contents, "control_d") { - role = "host" - } - } - } - } - - filename = common.HostProc("modules") - if common.PathExists(filename) { - contents, err := common.ReadLines(filename) - if err == nil { - if common.StringsContains(contents, "kvm") { - system = "kvm" - role = "host" - } else if common.StringsContains(contents, "vboxdrv") { - system = "vbox" - role = "host" - } else if common.StringsContains(contents, "vboxguest") { - system = "vbox" - role = "guest" - } else if common.StringsContains(contents, "vmware") { - system = "vmware" - role = "guest" - } - } - } - - filename = common.HostProc("cpuinfo") - if common.PathExists(filename) { - contents, err := common.ReadLines(filename) - if err == nil { - if common.StringsContains(contents, "QEMU Virtual CPU") || - common.StringsContains(contents, "Common KVM processor") || - common.StringsContains(contents, "Common 32-bit KVM processor") { - system = "kvm" - role = "guest" - } - } - } - - filename = common.HostProc() - if common.PathExists(filepath.Join(filename, "bc", "0")) { - system = "openvz" - role = "host" - } else if common.PathExists(filepath.Join(filename, "vz")) { - system = "openvz" - role = "guest" - } - - // not use dmidecode because it requires root - if common.PathExists(filepath.Join(filename, "self", "status")) { - contents, err := common.ReadLines(filepath.Join(filename, "self", "status")) - if err == nil { - - if common.StringsContains(contents, "s_context:") || - common.StringsContains(contents, "VxID:") { - system = "linux-vserver" - } - // TODO: guest or host - } - } - - if common.PathExists(filepath.Join(filename, "self", "cgroup")) { - contents, err := common.ReadLines(filepath.Join(filename, "self", "cgroup")) - if err == nil { - if common.StringsContains(contents, "lxc") { - system = "lxc" - role = "guest" - } else if common.StringsContains(contents, "docker") { - system = "docker" - role = "guest" - } else if common.StringsContains(contents, "machine-rkt") { - system = "rkt" - role = "guest" - } else if common.PathExists("/usr/bin/lxc-version") { - system = "lxc" - role = "host" - } - } - } - - if common.PathExists(common.HostEtc("os-release")) { - p, _, err := getOSRelease() - if err == nil && p == "coreos" { - system = "rkt" // Is it true? - role = "host" - } - } - return system, role, nil + return common.VirtualizationWithContext(ctx) } func SensorsTemperatures() ([]TemperatureStat, error) { @@ -634,6 +458,39 @@ func SensorsTemperaturesWithContext(ctx context.Context) ([]TemperatureStat, err return temperatures, err } } + var warns Warnings + + if len(files) == 0 { // handle distributions without hwmon, like raspbian #391, parse legacy thermal_zone files + files, err = filepath.Glob(common.HostSys("/class/thermal/thermal_zone*/")) + if err != nil { + return temperatures, err + } + for _, file := range files { + // Get the name of the temperature you are reading + name, err := ioutil.ReadFile(filepath.Join(file, "type")) + if err != nil { + warns.Add(err) + continue + } + // Get the temperature reading + current, err := ioutil.ReadFile(filepath.Join(file, "temp")) + if err != nil { + warns.Add(err) + continue + } + temperature, err := strconv.ParseInt(strings.TrimSpace(string(current)), 10, 64) + if err != nil { + warns.Add(err) + continue + } + + temperatures = append(temperatures, TemperatureStat{ + SensorKey: strings.TrimSpace(string(name)), + Temperature: float64(temperature) / 1000.0, + }) + } + return temperatures, warns.Reference() + } // example directory // device/ temp1_crit_alarm temp2_crit_alarm temp3_crit_alarm temp4_crit_alarm temp5_crit_alarm temp6_crit_alarm temp7_crit_alarm @@ -656,19 +513,22 @@ func SensorsTemperaturesWithContext(ctx context.Context) ([]TemperatureStat, err label = fmt.Sprintf("%s_", strings.Join(strings.Split(strings.TrimSpace(strings.ToLower(string(c))), " "), "")) } - // Get the name of the tempearture you are reading + // Get the name of the temperature you are reading name, err := ioutil.ReadFile(filepath.Join(filepath.Dir(file), "name")) if err != nil { - return temperatures, err + warns.Add(err) + continue } // Get the temperature reading current, err := ioutil.ReadFile(file) if err != nil { - return temperatures, err + warns.Add(err) + continue } temperature, err := strconv.ParseFloat(strings.TrimSpace(string(current)), 64) if err != nil { + warns.Add(err) continue } @@ -678,5 +538,5 @@ func SensorsTemperaturesWithContext(ctx context.Context) ([]TemperatureStat, err Temperature: temperature / 1000.0, }) } - return temperatures, nil + return temperatures, warns.Reference() } diff --git a/github.com/shirou/gopsutil/host/host_linux_riscv64.go b/github.com/shirou/gopsutil/host/host_linux_riscv64.go new file mode 100644 index 0000000000..79a077d66f --- /dev/null +++ b/github.com/shirou/gopsutil/host/host_linux_riscv64.go @@ -0,0 +1,47 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs types_linux.go + +package host + +const ( + sizeofPtr = 0x8 + sizeofShort = 0x2 + sizeofInt = 0x4 + sizeofLong = 0x8 + sizeofLongLong = 0x8 + sizeOfUtmp = 0x180 +) + +type ( + _C_short int16 + _C_int int32 + _C_long int64 + _C_long_long int64 +) + +type utmp struct { + Type int16 + Pid int32 + Line [32]int8 + Id [4]int8 + User [32]int8 + Host [256]int8 + Exit exit_status + Session int32 + Tv _Ctype_struct___0 + Addr_v6 [4]int32 + X__glibc_reserved [20]uint8 +} +type exit_status struct { + Termination int16 + Exit int16 +} +type timeval struct { + Sec int64 + Usec int64 +} + +type _Ctype_struct___0 struct { + Sec int32 + Usec int32 +} diff --git a/github.com/shirou/gopsutil/host/host_openbsd.go b/github.com/shirou/gopsutil/host/host_openbsd.go index 2ad64d77f6..d1501e9cab 100644 --- a/github.com/shirou/gopsutil/host/host_openbsd.go +++ b/github.com/shirou/gopsutil/host/host_openbsd.go @@ -8,15 +8,15 @@ import ( "encoding/binary" "io/ioutil" "os" - "os/exec" "runtime" - "strconv" "strings" + "sync/atomic" "time" "unsafe" "github.com/shirou/gopsutil/internal/common" "github.com/shirou/gopsutil/process" + "golang.org/x/sys/unix" ) const ( @@ -40,6 +40,11 @@ func InfoWithContext(ctx context.Context) (*InfoStat, error) { ret.Hostname = hostname } + kernelArch, err := kernelArch() + if err == nil { + ret.KernelArch = kernelArch + } + platform, family, version, err := PlatformInformation() if err == nil { ret.Platform = platform @@ -66,20 +71,28 @@ func InfoWithContext(ctx context.Context) (*InfoStat, error) { return ret, nil } +// cachedBootTime must be accessed via atomic.Load/StoreUint64 +var cachedBootTime uint64 + func BootTime() (uint64, error) { return BootTimeWithContext(context.Background()) } func BootTimeWithContext(ctx context.Context) (uint64, error) { - val, err := common.DoSysctrl("kern.boottime") - if err != nil { - return 0, err + // https://github.com/AaronO/dashd/blob/222e32ef9f7a1f9bea4a8da2c3627c4cb992f860/probe/probe_darwin.go + t := atomic.LoadUint64(&cachedBootTime) + if t != 0 { + return t, nil } - - boottime, err := strconv.ParseUint(val[0], 10, 64) + value, err := unix.Sysctl("kern.boottime") if err != nil { return 0, err } + bytes := []byte(value[:]) + var boottime uint64 + boottime = uint64(bytes[0]) + uint64(bytes[1])*256 + uint64(bytes[2])*256*256 + uint64(bytes[3])*256*256*256 + + atomic.StoreUint64(&cachedBootTime, boottime) return boottime, nil } @@ -108,19 +121,14 @@ func PlatformInformationWithContext(ctx context.Context) (string, string, string platform := "" family := "" version := "" - uname, err := exec.LookPath("uname") - if err != nil { - return "", "", "", err - } - out, err := invoke.CommandWithContext(ctx, uname, "-s") + p, err := unix.Sysctl("kern.ostype") if err == nil { - platform = strings.ToLower(strings.TrimSpace(string(out))) + platform = strings.ToLower(p) } - - out, err = invoke.CommandWithContext(ctx, uname, "-r") + v, err := unix.Sysctl("kern.osrelease") if err == nil { - version = strings.ToLower(strings.TrimSpace(string(out))) + version = strings.ToLower(v) } return platform, family, version, nil diff --git a/github.com/shirou/gopsutil/host/host_posix.go b/github.com/shirou/gopsutil/host/host_posix.go new file mode 100644 index 0000000000..a1b2479e1c --- /dev/null +++ b/github.com/shirou/gopsutil/host/host_posix.go @@ -0,0 +1,15 @@ +// +build linux freebsd openbsd darwin solaris + +package host + +import ( + "bytes" + + "golang.org/x/sys/unix" +) + +func kernelArch() (string, error) { + var utsname unix.Utsname + err := unix.Uname(&utsname) + return string(utsname.Machine[:bytes.IndexByte(utsname.Machine[:], 0)]), err +} diff --git a/github.com/shirou/gopsutil/host/host_solaris.go b/github.com/shirou/gopsutil/host/host_solaris.go index bb83bfc9c1..c6061b8e41 100644 --- a/github.com/shirou/gopsutil/host/host_solaris.go +++ b/github.com/shirou/gopsutil/host/host_solaris.go @@ -33,7 +33,7 @@ func InfoWithContext(ctx context.Context) (*InfoStat, error) { result.Hostname = hostname // Parse versions from output of `uname(1)` - uname, err := exec.LookPath("/usr/bin/uname") + uname, err := exec.LookPath("uname") if err != nil { return nil, err } @@ -54,6 +54,11 @@ func InfoWithContext(ctx context.Context) (*InfoStat, error) { result.PlatformVersion = fields[2] } + kernelArch, err := kernelArch() + if err == nil { + result.KernelArch = kernelArch + } + // Find distribution name from /etc/release fh, err := os.Open("/etc/release") if err != nil { @@ -85,7 +90,7 @@ func InfoWithContext(ctx context.Context) (*InfoStat, error) { switch result.Platform { case "SmartOS": // If everything works, use the current zone ID as the HostID if present. - zonename, err := exec.LookPath("/usr/bin/zonename") + zonename, err := exec.LookPath("zonename") if err == nil { out, err := invoke.CommandWithContext(ctx, zonename) if err == nil { @@ -112,7 +117,7 @@ func InfoWithContext(ctx context.Context) (*InfoStat, error) { // this point there are no hardware facilities available. This behavior // matches that of other supported OSes. if result.HostID == "" { - hostID, err := exec.LookPath("/usr/bin/hostid") + hostID, err := exec.LookPath("hostid") if err == nil { out, err := invoke.CommandWithContext(ctx, hostID) if err == nil { @@ -151,7 +156,7 @@ func BootTime() (uint64, error) { } func BootTimeWithContext(ctx context.Context) (uint64, error) { - kstat, err := exec.LookPath("/usr/bin/kstat") + kstat, err := exec.LookPath("kstat") if err != nil { return 0, err } @@ -215,7 +220,7 @@ func KernelVersion() (string, error) { func KernelVersionWithContext(ctx context.Context) (string, error) { // Parse versions from output of `uname(1)` - uname, err := exec.LookPath("/usr/bin/uname") + uname, err := exec.LookPath("uname") if err != nil { return "", err } diff --git a/github.com/shirou/gopsutil/host/host_windows.go b/github.com/shirou/gopsutil/host/host_windows.go index d89e191b7a..f9e1a16a2d 100644 --- a/github.com/shirou/gopsutil/host/host_windows.go +++ b/github.com/shirou/gopsutil/host/host_windows.go @@ -24,6 +24,7 @@ var ( procGetSystemTimeAsFileTime = common.Modkernel32.NewProc("GetSystemTimeAsFileTime") procGetTickCount32 = common.Modkernel32.NewProc("GetTickCount") procGetTickCount64 = common.Modkernel32.NewProc("GetTickCount64") + procGetNativeSystemInfo = common.Modkernel32.NewProc("GetNativeSystemInfo") procRtlGetVersion = common.ModNt.NewProc("RtlGetVersion") ) @@ -42,6 +43,20 @@ type osVersionInfoExW struct { wReserved uint8 } +type systemInfo struct { + wProcessorArchitecture uint16 + wReserved uint16 + dwPageSize uint32 + lpMinimumApplicationAddress uintptr + lpMaximumApplicationAddress uintptr + dwActiveProcessorMask uintptr + dwNumberOfProcessors uint32 + dwProcessorType uint32 + dwAllocationGranularity uint32 + wProcessorLevel uint16 + wProcessorRevision uint16 +} + type msAcpi_ThermalZoneTemperature struct { Active bool CriticalTripPoint uint32 @@ -77,7 +92,14 @@ func InfoWithContext(ctx context.Context) (*InfoStat, error) { } { - boot, err := BootTime() + kernelArch, err := kernelArch() + if err == nil { + ret.KernelArch = kernelArch + } + } + + { + boot, err := BootTimeWithContext(ctx) if err == nil { ret.BootTime = boot ret.Uptime, _ = Uptime() @@ -87,12 +109,12 @@ func InfoWithContext(ctx context.Context) (*InfoStat, error) { { hostID, err := getMachineGuid() if err == nil { - ret.HostID = strings.ToLower(hostID) + ret.HostID = hostID } } { - procs, err := process.Pids() + procs, err := process.PidsWithContext(ctx) if err == nil { ret.Procs = uint64(len(procs)) } @@ -128,7 +150,7 @@ func getMachineGuid() (string, error) { return "", fmt.Errorf("HostID incorrect: %q\n", hostID) } - return hostID, nil + return strings.ToLower(hostID), nil } func Uptime() (uint64, error) { @@ -293,3 +315,35 @@ func KernelVersionWithContext(ctx context.Context) (string, error) { _, _, version, err := PlatformInformation() return version, err } + +func kernelArch() (string, error) { + var systemInfo systemInfo + procGetNativeSystemInfo.Call(uintptr(unsafe.Pointer(&systemInfo))) + + const ( + PROCESSOR_ARCHITECTURE_INTEL = 0 + PROCESSOR_ARCHITECTURE_ARM = 5 + PROCESSOR_ARCHITECTURE_ARM64 = 12 + PROCESSOR_ARCHITECTURE_IA64 = 6 + PROCESSOR_ARCHITECTURE_AMD64 = 9 + ) + switch systemInfo.wProcessorArchitecture { + case PROCESSOR_ARCHITECTURE_INTEL: + if systemInfo.wProcessorLevel < 3 { + return "i386", nil + } + if systemInfo.wProcessorLevel > 6 { + return "i686", nil + } + return fmt.Sprintf("i%d86", systemInfo.wProcessorLevel), nil + case PROCESSOR_ARCHITECTURE_ARM: + return "arm", nil + case PROCESSOR_ARCHITECTURE_ARM64: + return "aarch64", nil + case PROCESSOR_ARCHITECTURE_IA64: + return "ia64", nil + case PROCESSOR_ARCHITECTURE_AMD64: + return "x86_64", nil + } + return "", nil +} diff --git a/github.com/shirou/gopsutil/host/smc_darwin.c b/github.com/shirou/gopsutil/host/smc_darwin.c new file mode 100644 index 0000000000..aedea8be95 --- /dev/null +++ b/github.com/shirou/gopsutil/host/smc_darwin.c @@ -0,0 +1,170 @@ +#include +#include +#include "smc_darwin.h" + +#define IOSERVICE_SMC "AppleSMC" +#define IOSERVICE_MODEL "IOPlatformExpertDevice" + +#define DATA_TYPE_SP78 "sp78" + +typedef enum { + kSMCUserClientOpen = 0, + kSMCUserClientClose = 1, + kSMCHandleYPCEvent = 2, + kSMCReadKey = 5, + kSMCWriteKey = 6, + kSMCGetKeyCount = 7, + kSMCGetKeyFromIndex = 8, + kSMCGetKeyInfo = 9, +} selector_t; + +typedef struct { + unsigned char major; + unsigned char minor; + unsigned char build; + unsigned char reserved; + unsigned short release; +} SMCVersion; + +typedef struct { + uint16_t version; + uint16_t length; + uint32_t cpuPLimit; + uint32_t gpuPLimit; + uint32_t memPLimit; +} SMCPLimitData; + +typedef struct { + IOByteCount data_size; + uint32_t data_type; + uint8_t data_attributes; +} SMCKeyInfoData; + +typedef struct { + uint32_t key; + SMCVersion vers; + SMCPLimitData p_limit_data; + SMCKeyInfoData key_info; + uint8_t result; + uint8_t status; + uint8_t data8; + uint32_t data32; + uint8_t bytes[32]; +} SMCParamStruct; + +typedef enum { + kSMCSuccess = 0, + kSMCError = 1, + kSMCKeyNotFound = 0x84, +} kSMC_t; + +typedef struct { + uint8_t data[32]; + uint32_t data_type; + uint32_t data_size; + kSMC_t kSMC; +} smc_return_t; + +static const int SMC_KEY_SIZE = 4; // number of characters in an SMC key. +static io_connect_t conn; // our connection to the SMC. + +kern_return_t open_smc(void) { + kern_return_t result; + io_service_t service; + + service = IOServiceGetMatchingService(kIOMasterPortDefault, + IOServiceMatching(IOSERVICE_SMC)); + if (service == 0) { + // Note: IOServiceMatching documents 0 on failure + printf("ERROR: %s NOT FOUND\n", IOSERVICE_SMC); + return kIOReturnError; + } + + result = IOServiceOpen(service, mach_task_self(), 0, &conn); + IOObjectRelease(service); + + return result; +} + +kern_return_t close_smc(void) { return IOServiceClose(conn); } + +static uint32_t to_uint32(char *key) { + uint32_t ans = 0; + uint32_t shift = 24; + + if (strlen(key) != SMC_KEY_SIZE) { + return 0; + } + + for (int i = 0; i < SMC_KEY_SIZE; i++) { + ans += key[i] << shift; + shift -= 8; + } + + return ans; +} + +static kern_return_t call_smc(SMCParamStruct *input, SMCParamStruct *output) { + kern_return_t result; + size_t input_cnt = sizeof(SMCParamStruct); + size_t output_cnt = sizeof(SMCParamStruct); + + result = IOConnectCallStructMethod(conn, kSMCHandleYPCEvent, input, input_cnt, + output, &output_cnt); + + if (result != kIOReturnSuccess) { + result = err_get_code(result); + } + return result; +} + +static kern_return_t read_smc(char *key, smc_return_t *result_smc) { + kern_return_t result; + SMCParamStruct input; + SMCParamStruct output; + + memset(&input, 0, sizeof(SMCParamStruct)); + memset(&output, 0, sizeof(SMCParamStruct)); + memset(result_smc, 0, sizeof(smc_return_t)); + + input.key = to_uint32(key); + input.data8 = kSMCGetKeyInfo; + + result = call_smc(&input, &output); + result_smc->kSMC = output.result; + + if (result != kIOReturnSuccess || output.result != kSMCSuccess) { + return result; + } + + result_smc->data_size = output.key_info.data_size; + result_smc->data_type = output.key_info.data_type; + + input.key_info.data_size = output.key_info.data_size; + input.data8 = kSMCReadKey; + + result = call_smc(&input, &output); + result_smc->kSMC = output.result; + + if (result != kIOReturnSuccess || output.result != kSMCSuccess) { + return result; + } + + memcpy(result_smc->data, output.bytes, sizeof(output.bytes)); + + return result; +} + +double get_temperature(char *key) { + kern_return_t result; + smc_return_t result_smc; + + result = read_smc(key, &result_smc); + + if (!(result == kIOReturnSuccess) && result_smc.data_size == 2 && + result_smc.data_type == to_uint32(DATA_TYPE_SP78)) { + return 0.0; + } + + return (double)result_smc.data[0]; +} diff --git a/github.com/shirou/gopsutil/host/smc_darwin.h b/github.com/shirou/gopsutil/host/smc_darwin.h new file mode 100644 index 0000000000..ab51ed9f7b --- /dev/null +++ b/github.com/shirou/gopsutil/host/smc_darwin.h @@ -0,0 +1,32 @@ +#ifndef __SMC_H__ +#define __SMC_H__ 1 + +#include + +#define AMBIENT_AIR_0 "TA0P" +#define AMBIENT_AIR_1 "TA1P" +#define CPU_0_DIODE "TC0D" +#define CPU_0_HEATSINK "TC0H" +#define CPU_0_PROXIMITY "TC0P" +#define ENCLOSURE_BASE_0 "TB0T" +#define ENCLOSURE_BASE_1 "TB1T" +#define ENCLOSURE_BASE_2 "TB2T" +#define ENCLOSURE_BASE_3 "TB3T" +#define GPU_0_DIODE "TG0D" +#define GPU_0_HEATSINK "TG0H" +#define GPU_0_PROXIMITY "TG0P" +#define HARD_DRIVE_BAY "TH0P" +#define MEMORY_SLOT_0 "TM0S" +#define MEMORY_SLOTS_PROXIMITY "TM0P" +#define NORTHBRIDGE "TN0H" +#define NORTHBRIDGE_DIODE "TN0D" +#define NORTHBRIDGE_PROXIMITY "TN0P" +#define THUNDERBOLT_0 "TI0P" +#define THUNDERBOLT_1 "TI1P" +#define WIRELESS_MODULE "TW0P" + +kern_return_t open_smc(void); +kern_return_t close_smc(void); +double get_temperature(char *); + +#endif // __SMC_H__ diff --git a/github.com/shirou/gopsutil/host/types.go b/github.com/shirou/gopsutil/host/types.go new file mode 100644 index 0000000000..1eff4755e2 --- /dev/null +++ b/github.com/shirou/gopsutil/host/types.go @@ -0,0 +1,25 @@ +package host + +import ( + "fmt" +) + +type Warnings struct { + List []error +} + +func (w *Warnings) Add(err error) { + w.List = append(w.List, err) +} + +func (w *Warnings) Reference() error { + if len(w.List) > 0 { + return w + } else { + return nil + } +} + +func (w *Warnings) Error() string { + return fmt.Sprintf("Number of warnings: %v", len(w.List)) +} diff --git a/github.com/shirou/gopsutil/internal/common/common.go b/github.com/shirou/gopsutil/internal/common/common.go index 71c2257f07..57e291b10b 100644 --- a/github.com/shirou/gopsutil/internal/common/common.go +++ b/github.com/shirou/gopsutil/internal/common/common.go @@ -94,6 +94,17 @@ func (i FakeInvoke) CommandWithContext(ctx context.Context, name string, arg ... var ErrNotImplementedError = errors.New("not implemented yet") +// ReadFile reads contents from a file +func ReadFile(filename string) (string, error) { + content, err := ioutil.ReadFile(filename) + + if err != nil { + return "", err + } + + return string(content), nil +} + // ReadLines reads contents from a file and splits them by new lines. // A convenience wrapper to ReadLinesOffsetN(filename, 0, -1). func ReadLines(filename string) ([]string, error) { @@ -213,6 +224,12 @@ func ReadInts(filename string) ([]int64, error) { return ret, nil } +// Parse Hex to uint32 without error +func HexToUint32(hex string) uint32 { + vv, _ := strconv.ParseUint(hex, 16, 32) + return uint32(vv) +} + // Parse to int32 without error func mustParseInt32(val string) int32 { vv, _ := strconv.ParseInt(val, 10, 32) @@ -332,47 +349,8 @@ func HostRun(combineWith ...string) string { return GetEnv("HOST_RUN", "/run", combineWith...) } -// https://gist.github.com/kylelemons/1525278 -func Pipeline(cmds ...*exec.Cmd) ([]byte, []byte, error) { - // Require at least one command - if len(cmds) < 1 { - return nil, nil, nil - } - - // Collect the output from the command(s) - var output bytes.Buffer - var stderr bytes.Buffer - - last := len(cmds) - 1 - for i, cmd := range cmds[:last] { - var err error - // Connect each command's stdin to the previous command's stdout - if cmds[i+1].Stdin, err = cmd.StdoutPipe(); err != nil { - return nil, nil, err - } - // Connect each command's stderr to a buffer - cmd.Stderr = &stderr - } - - // Connect the output and error for the last command - cmds[last].Stdout, cmds[last].Stderr = &output, &stderr - - // Start each command - for _, cmd := range cmds { - if err := cmd.Start(); err != nil { - return output.Bytes(), stderr.Bytes(), err - } - } - - // Wait for each command to complete - for _, cmd := range cmds { - if err := cmd.Wait(); err != nil { - return output.Bytes(), stderr.Bytes(), err - } - } - - // Return the pipeline output and the collected standard error - return output.Bytes(), stderr.Bytes(), nil +func HostDev(combineWith ...string) string { + return GetEnv("HOST_DEV", "/dev", combineWith...) } // getSysctrlEnv sets LC_ALL=C in a list of env vars for use when running diff --git a/github.com/shirou/gopsutil/internal/common/common_darwin.go b/github.com/shirou/gopsutil/internal/common/common_darwin.go index 3e85cc06b8..dde5c39037 100644 --- a/github.com/shirou/gopsutil/internal/common/common_darwin.go +++ b/github.com/shirou/gopsutil/internal/common/common_darwin.go @@ -13,7 +13,7 @@ import ( ) func DoSysctrlWithContext(ctx context.Context, mib string) ([]string, error) { - sysctl, err := exec.LookPath("/usr/sbin/sysctl") + sysctl, err := exec.LookPath("sysctl") if err != nil { return []string{}, err } diff --git a/github.com/shirou/gopsutil/internal/common/common_freebsd.go b/github.com/shirou/gopsutil/internal/common/common_freebsd.go index 107e2c9cf9..85bda0e22c 100644 --- a/github.com/shirou/gopsutil/internal/common/common_freebsd.go +++ b/github.com/shirou/gopsutil/internal/common/common_freebsd.go @@ -3,6 +3,7 @@ package common import ( + "fmt" "os" "os/exec" "strings" @@ -11,8 +12,23 @@ import ( "golang.org/x/sys/unix" ) +func SysctlUint(mib string) (uint64, error) { + buf, err := unix.SysctlRaw(mib) + if err != nil { + return 0, err + } + if len(buf) == 8 { // 64 bit + return *(*uint64)(unsafe.Pointer(&buf[0])), nil + } + if len(buf) == 4 { // 32bit + t := *(*uint32)(unsafe.Pointer(&buf[0])) + return uint64(t), nil + } + return 0, fmt.Errorf("unexpected size: %s, %d", mib, len(buf)) +} + func DoSysctrl(mib string) ([]string, error) { - sysctl, err := exec.LookPath("/sbin/sysctl") + sysctl, err := exec.LookPath("sysctl") if err != nil { return []string{}, err } diff --git a/github.com/shirou/gopsutil/internal/common/common_linux.go b/github.com/shirou/gopsutil/internal/common/common_linux.go index 4e829e057f..0f66f1c389 100644 --- a/github.com/shirou/gopsutil/internal/common/common_linux.go +++ b/github.com/shirou/gopsutil/internal/common/common_linux.go @@ -3,13 +3,18 @@ package common import ( + "context" + "fmt" "os" "os/exec" + "path/filepath" + "strconv" "strings" + "time" ) func DoSysctrl(mib string) ([]string, error) { - sysctl, err := exec.LookPath("/sbin/sysctl") + sysctl, err := exec.LookPath("sysctl") if err != nil { return []string{}, err } @@ -37,5 +42,224 @@ func NumProcs() (uint64, error) { if err != nil { return 0, err } - return uint64(len(list)), err + var cnt uint64 + + for _, v := range list { + if _, err = strconv.ParseUint(v, 10, 64); err == nil { + cnt++ + } + } + + return cnt, nil +} + +func BootTimeWithContext(ctx context.Context) (uint64, error) { + + system, role, err := Virtualization() + if err != nil { + return 0, err + } + + statFile := "stat" + if system == "lxc" && role == "guest" { + // if lxc, /proc/uptime is used. + statFile = "uptime" + } else if system == "docker" && role == "guest" { + // also docker, guest + statFile = "uptime" + } + + filename := HostProc(statFile) + lines, err := ReadLines(filename) + if err != nil { + return 0, err + } + + if statFile == "stat" { + for _, line := range lines { + if strings.HasPrefix(line, "btime") { + f := strings.Fields(line) + if len(f) != 2 { + return 0, fmt.Errorf("wrong btime format") + } + b, err := strconv.ParseInt(f[1], 10, 64) + if err != nil { + return 0, err + } + t := uint64(b) + return t, nil + } + } + } else if statFile == "uptime" { + if len(lines) != 1 { + return 0, fmt.Errorf("wrong uptime format") + } + f := strings.Fields(lines[0]) + b, err := strconv.ParseFloat(f[0], 64) + if err != nil { + return 0, err + } + t := uint64(time.Now().Unix()) - uint64(b) + return t, nil + } + + return 0, fmt.Errorf("could not find btime") +} + +func Virtualization() (string, string, error) { + return VirtualizationWithContext(context.Background()) +} + +func VirtualizationWithContext(ctx context.Context) (string, string, error) { + var system string + var role string + + filename := HostProc("xen") + if PathExists(filename) { + system = "xen" + role = "guest" // assume guest + + if PathExists(filepath.Join(filename, "capabilities")) { + contents, err := ReadLines(filepath.Join(filename, "capabilities")) + if err == nil { + if StringsContains(contents, "control_d") { + role = "host" + } + } + } + } + + filename = HostProc("modules") + if PathExists(filename) { + contents, err := ReadLines(filename) + if err == nil { + if StringsContains(contents, "kvm") { + system = "kvm" + role = "host" + } else if StringsContains(contents, "vboxdrv") { + system = "vbox" + role = "host" + } else if StringsContains(contents, "vboxguest") { + system = "vbox" + role = "guest" + } else if StringsContains(contents, "vmware") { + system = "vmware" + role = "guest" + } + } + } + + filename = HostProc("cpuinfo") + if PathExists(filename) { + contents, err := ReadLines(filename) + if err == nil { + if StringsContains(contents, "QEMU Virtual CPU") || + StringsContains(contents, "Common KVM processor") || + StringsContains(contents, "Common 32-bit KVM processor") { + system = "kvm" + role = "guest" + } + } + } + + filename = HostProc("bus/pci/devices") + if PathExists(filename) { + contents, err := ReadLines(filename) + if err == nil { + if StringsContains(contents, "virtio-pci") { + role = "guest" + } + } + } + + filename = HostProc() + if PathExists(filepath.Join(filename, "bc", "0")) { + system = "openvz" + role = "host" + } else if PathExists(filepath.Join(filename, "vz")) { + system = "openvz" + role = "guest" + } + + // not use dmidecode because it requires root + if PathExists(filepath.Join(filename, "self", "status")) { + contents, err := ReadLines(filepath.Join(filename, "self", "status")) + if err == nil { + + if StringsContains(contents, "s_context:") || + StringsContains(contents, "VxID:") { + system = "linux-vserver" + } + // TODO: guest or host + } + } + + if PathExists(filepath.Join(filename, "1", "environ")) { + contents, err := ReadFile(filepath.Join(filename, "1", "environ")) + + if err == nil { + if strings.Contains(contents, "container=lxc") { + system = "lxc" + role = "guest" + } + } + } + + if PathExists(filepath.Join(filename, "self", "cgroup")) { + contents, err := ReadLines(filepath.Join(filename, "self", "cgroup")) + if err == nil { + if StringsContains(contents, "lxc") { + system = "lxc" + role = "guest" + } else if StringsContains(contents, "docker") { + system = "docker" + role = "guest" + } else if StringsContains(contents, "machine-rkt") { + system = "rkt" + role = "guest" + } else if PathExists("/usr/bin/lxc-version") { + system = "lxc" + role = "host" + } + } + } + + if PathExists(HostEtc("os-release")) { + p, _, err := GetOSRelease() + if err == nil && p == "coreos" { + system = "rkt" // Is it true? + role = "host" + } + } + return system, role, nil +} + +func GetOSRelease() (platform string, version string, err error) { + contents, err := ReadLines(HostEtc("os-release")) + if err != nil { + return "", "", nil // return empty + } + for _, line := range contents { + field := strings.Split(line, "=") + if len(field) < 2 { + continue + } + switch field[0] { + case "ID": // use ID for lowercase + platform = trimQuotes(field[1]) + case "VERSION": + version = trimQuotes(field[1]) + } + } + return platform, version, nil +} + +// Remove quotes of the source string +func trimQuotes(s string) string { + if len(s) >= 2 { + if s[0] == '"' && s[len(s)-1] == '"' { + return s[1 : len(s)-1] + } + } + return s } diff --git a/github.com/shirou/gopsutil/internal/common/common_openbsd.go b/github.com/shirou/gopsutil/internal/common/common_openbsd.go index 398f78542e..ba73a7eb50 100644 --- a/github.com/shirou/gopsutil/internal/common/common_openbsd.go +++ b/github.com/shirou/gopsutil/internal/common/common_openbsd.go @@ -12,7 +12,7 @@ import ( ) func DoSysctrl(mib string) ([]string, error) { - sysctl, err := exec.LookPath("/sbin/sysctl") + sysctl, err := exec.LookPath("sysctl") if err != nil { return []string{}, err } diff --git a/github.com/shirou/gopsutil/internal/common/common_unix.go b/github.com/shirou/gopsutil/internal/common/common_unix.go index 750a5926d0..9e393bcfa8 100644 --- a/github.com/shirou/gopsutil/internal/common/common_unix.go +++ b/github.com/shirou/gopsutil/internal/common/common_unix.go @@ -23,7 +23,7 @@ func CallLsofWithContext(ctx context.Context, invoke Invoker, pid int32, args .. } out, err := invoke.CommandWithContext(ctx, lsof, cmd...) if err != nil { - // if no pid found, lsof returnes code 1. + // if no pid found, lsof returns code 1. if err.Error() == "exit status 1" && len(out) == 0 { return []string{}, nil } diff --git a/github.com/shirou/gopsutil/internal/common/common_windows.go b/github.com/shirou/gopsutil/internal/common/common_windows.go index 0997c9b8ab..9bc05ded88 100644 --- a/github.com/shirou/gopsutil/internal/common/common_windows.go +++ b/github.com/shirou/gopsutil/internal/common/common_windows.go @@ -4,6 +4,9 @@ package common import ( "context" + "path/filepath" + "strings" + "syscall" "unsafe" "github.com/StackExchange/wmi" @@ -46,19 +49,33 @@ const ( PDH_NO_DATA = 0x800007d5 ) +const ( + ProcessBasicInformation = 0 + ProcessWow64Information = 26 +) + var ( Modkernel32 = windows.NewLazySystemDLL("kernel32.dll") ModNt = windows.NewLazySystemDLL("ntdll.dll") ModPdh = windows.NewLazySystemDLL("pdh.dll") ModPsapi = windows.NewLazySystemDLL("psapi.dll") - ProcGetSystemTimes = Modkernel32.NewProc("GetSystemTimes") - ProcNtQuerySystemInformation = ModNt.NewProc("NtQuerySystemInformation") - PdhOpenQuery = ModPdh.NewProc("PdhOpenQuery") - PdhAddCounter = ModPdh.NewProc("PdhAddCounterW") - PdhCollectQueryData = ModPdh.NewProc("PdhCollectQueryData") - PdhGetFormattedCounterValue = ModPdh.NewProc("PdhGetFormattedCounterValue") - PdhCloseQuery = ModPdh.NewProc("PdhCloseQuery") + ProcGetSystemTimes = Modkernel32.NewProc("GetSystemTimes") + ProcNtQuerySystemInformation = ModNt.NewProc("NtQuerySystemInformation") + ProcRtlGetNativeSystemInformation = ModNt.NewProc("RtlGetNativeSystemInformation") + ProcRtlNtStatusToDosError = ModNt.NewProc("RtlNtStatusToDosError") + ProcNtQueryInformationProcess = ModNt.NewProc("NtQueryInformationProcess") + ProcNtReadVirtualMemory = ModNt.NewProc("NtReadVirtualMemory") + ProcNtWow64QueryInformationProcess64 = ModNt.NewProc("NtWow64QueryInformationProcess64") + ProcNtWow64ReadVirtualMemory64 = ModNt.NewProc("NtWow64ReadVirtualMemory64") + + PdhOpenQuery = ModPdh.NewProc("PdhOpenQuery") + PdhAddCounter = ModPdh.NewProc("PdhAddCounterW") + PdhCollectQueryData = ModPdh.NewProc("PdhCollectQueryData") + PdhGetFormattedCounterValue = ModPdh.NewProc("PdhGetFormattedCounterValue") + PdhCloseQuery = ModPdh.NewProc("PdhCloseQuery") + + procQueryDosDeviceW = Modkernel32.NewProc("QueryDosDeviceW") ) type FILETIME struct { @@ -133,3 +150,23 @@ func WMIQueryWithContext(ctx context.Context, query string, dst interface{}, con return err } } + +// Convert paths using native DOS format like: +// "\Device\HarddiskVolume1\Windows\systemew\file.txt" +// into: +// "C:\Windows\systemew\file.txt" +func ConvertDOSPath(p string) string { + rawDrive := strings.Join(strings.Split(p, `\`)[:3], `\`) + + for d := 'A'; d <= 'Z'; d++ { + szDeviceName := string(d) + ":" + szTarget := make([]uint16, 512) + ret, _, _ := procQueryDosDeviceW.Call(uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(szDeviceName))), + uintptr(unsafe.Pointer(&szTarget[0])), + uintptr(len(szTarget))) + if ret != 0 && windows.UTF16ToString(szTarget[:]) == rawDrive { + return filepath.Join(szDeviceName, p[len(rawDrive):]) + } + } + return p +} diff --git a/github.com/shirou/gopsutil/load/load.go b/github.com/shirou/gopsutil/load/load.go index 908588955f..86d9114208 100644 --- a/github.com/shirou/gopsutil/load/load.go +++ b/github.com/shirou/gopsutil/load/load.go @@ -20,6 +20,7 @@ func (l AvgStat) String() string { } type MiscStat struct { + ProcsTotal int `json:"procsTotal"` ProcsRunning int `json:"procsRunning"` ProcsBlocked int `json:"procsBlocked"` Ctxt int `json:"ctxt"` diff --git a/github.com/shirou/gopsutil/load/load_darwin.go b/github.com/shirou/gopsutil/load/load_darwin.go index cd7b74dfef..a205f2f8ae 100644 --- a/github.com/shirou/gopsutil/load/load_darwin.go +++ b/github.com/shirou/gopsutil/load/load_darwin.go @@ -5,10 +5,10 @@ package load import ( "context" "os/exec" - "strconv" "strings" + "unsafe" - "github.com/shirou/gopsutil/internal/common" + "golang.org/x/sys/unix" ) func Avg() (*AvgStat, error) { @@ -16,28 +16,23 @@ func Avg() (*AvgStat, error) { } func AvgWithContext(ctx context.Context) (*AvgStat, error) { - values, err := common.DoSysctrlWithContext(ctx, "vm.loadavg") - if err != nil { - return nil, err - } - - load1, err := strconv.ParseFloat(values[0], 64) - if err != nil { - return nil, err + // This SysctlRaw method borrowed from + // https://github.com/prometheus/node_exporter/blob/master/collector/loadavg_freebsd.go + // this implementation is common with BSDs + type loadavg struct { + load [3]uint32 + scale int } - load5, err := strconv.ParseFloat(values[1], 64) + b, err := unix.SysctlRaw("vm.loadavg") if err != nil { return nil, err } - load15, err := strconv.ParseFloat(values[2], 64) - if err != nil { - return nil, err - } - + load := *(*loadavg)(unsafe.Pointer((&b[0]))) + scale := float64(load.scale) ret := &AvgStat{ - Load1: float64(load1), - Load5: float64(load5), - Load15: float64(load15), + Load1: float64(load.load[0]) / scale, + Load5: float64(load.load[1]) / scale, + Load15: float64(load.load[2]) / scale, } return ret, nil diff --git a/github.com/shirou/gopsutil/load/load_linux.go b/github.com/shirou/gopsutil/load/load_linux.go index 63c26a2e21..081e2cc6d2 100644 --- a/github.com/shirou/gopsutil/load/load_linux.go +++ b/github.com/shirou/gopsutil/load/load_linux.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "strconv" "strings" + "syscall" "github.com/shirou/gopsutil/internal/common" ) @@ -16,13 +17,33 @@ func Avg() (*AvgStat, error) { } func AvgWithContext(ctx context.Context) (*AvgStat, error) { - filename := common.HostProc("loadavg") - line, err := ioutil.ReadFile(filename) + stat, err := fileAvgWithContext(ctx) + if err != nil { + stat, err = sysinfoAvgWithContext(ctx) + } + return stat, err +} + +func sysinfoAvgWithContext(ctx context.Context) (*AvgStat, error) { + var info syscall.Sysinfo_t + err := syscall.Sysinfo(&info) if err != nil { return nil, err } - values := strings.Fields(string(line)) + const si_load_shift = 16 + return &AvgStat{ + Load1: float64(info.Loads[0]) / float64(1<") - if len(addrs) == 0 { - return laddr, raddr, fmt.Errorf("wrong netaddr, %s", line) - } - laddr, err = parse(addrs[0]) - if len(addrs) == 2 { // remote addr exists - raddr, err = parse(addrs[1]) - if err != nil { - return laddr, raddr, err - } - } - - return laddr, raddr, err -} diff --git a/github.com/shirou/gopsutil/net/net_aix.go b/github.com/shirou/gopsutil/net/net_aix.go new file mode 100644 index 0000000000..4ac8497015 --- /dev/null +++ b/github.com/shirou/gopsutil/net/net_aix.go @@ -0,0 +1,425 @@ +// +build aix + +package net + +import ( + "context" + "fmt" + "os/exec" + "regexp" + "strconv" + "strings" + "syscall" + + "github.com/shirou/gopsutil/internal/common" +) + +func parseNetstatI(output string) ([]IOCountersStat, error) { + lines := strings.Split(string(output), "\n") + ret := make([]IOCountersStat, 0, len(lines)-1) + exists := make([]string, 0, len(ret)) + + // Check first line is header + if len(lines) > 0 && strings.Fields(lines[0])[0] != "Name" { + return nil, fmt.Errorf("not a 'netstat -i' output") + } + + for _, line := range lines[1:] { + values := strings.Fields(line) + if len(values) < 1 || values[0] == "Name" { + continue + } + if common.StringsHas(exists, values[0]) { + // skip if already get + continue + } + exists = append(exists, values[0]) + + if len(values) < 9 { + continue + } + + base := 1 + // sometimes Address is omitted + if len(values) < 10 { + base = 0 + } + + parsed := make([]uint64, 0, 5) + vv := []string{ + values[base+3], // Ipkts == PacketsRecv + values[base+4], // Ierrs == Errin + values[base+5], // Opkts == PacketsSent + values[base+6], // Oerrs == Errout + values[base+8], // Drops == Dropout + } + + for _, target := range vv { + if target == "-" { + parsed = append(parsed, 0) + continue + } + + t, err := strconv.ParseUint(target, 10, 64) + if err != nil { + return nil, err + } + parsed = append(parsed, t) + } + + n := IOCountersStat{ + Name: values[0], + PacketsRecv: parsed[0], + Errin: parsed[1], + PacketsSent: parsed[2], + Errout: parsed[3], + Dropout: parsed[4], + } + ret = append(ret, n) + } + return ret, nil +} + +func IOCounters(pernic bool) ([]IOCountersStat, error) { + return IOCountersWithContext(context.Background(), pernic) +} + +func IOCountersWithContext(ctx context.Context, pernic bool) ([]IOCountersStat, error) { + netstat, err := exec.LookPath("netstat") + if err != nil { + return nil, err + } + out, err := invoke.CommandWithContext(ctx, netstat, "-idn") + if err != nil { + return nil, err + } + + iocounters, err := parseNetstatI(string(out)) + if err != nil { + return nil, err + } + if pernic == false { + return getIOCountersAll(iocounters) + } + return iocounters, nil + +} + +// NetIOCountersByFile is an method which is added just a compatibility for linux. +func IOCountersByFile(pernic bool, filename string) ([]IOCountersStat, error) { + return IOCountersByFileWithContext(context.Background(), pernic, filename) +} + +func IOCountersByFileWithContext(ctx context.Context, pernic bool, filename string) ([]IOCountersStat, error) { + return IOCounters(pernic) +} + +func FilterCounters() ([]FilterStat, error) { + return FilterCountersWithContext(context.Background()) +} + +func FilterCountersWithContext(ctx context.Context) ([]FilterStat, error) { + return nil, common.ErrNotImplementedError +} + +func ConntrackStats(percpu bool) ([]ConntrackStat, error) { + return ConntrackStatsWithContext(context.Background(), percpu) +} + +func ConntrackStatsWithContext(ctx context.Context, percpu bool) ([]ConntrackStat, error) { + return nil, common.ErrNotImplementedError +} + +func ProtoCounters(protocols []string) ([]ProtoCountersStat, error) { + return ProtoCountersWithContext(context.Background(), protocols) +} + +func ProtoCountersWithContext(ctx context.Context, protocols []string) ([]ProtoCountersStat, error) { + return nil, common.ErrNotImplementedError +} + +func parseNetstatNetLine(line string) (ConnectionStat, error) { + f := strings.Fields(line) + if len(f) < 5 { + return ConnectionStat{}, fmt.Errorf("wrong line,%s", line) + } + + var netType, netFamily uint32 + switch f[0] { + case "tcp", "tcp4": + netType = syscall.SOCK_STREAM + netFamily = syscall.AF_INET + case "udp", "udp4": + netType = syscall.SOCK_DGRAM + netFamily = syscall.AF_INET + case "tcp6": + netType = syscall.SOCK_STREAM + netFamily = syscall.AF_INET6 + case "udp6": + netType = syscall.SOCK_DGRAM + netFamily = syscall.AF_INET6 + default: + return ConnectionStat{}, fmt.Errorf("unknown type, %s", f[0]) + } + + laddr, raddr, err := parseNetstatAddr(f[3], f[4], netFamily) + if err != nil { + return ConnectionStat{}, fmt.Errorf("failed to parse netaddr, %s %s", f[3], f[4]) + } + + n := ConnectionStat{ + Fd: uint32(0), // not supported + Family: uint32(netFamily), + Type: uint32(netType), + Laddr: laddr, + Raddr: raddr, + Pid: int32(0), // not supported + } + if len(f) == 6 { + n.Status = f[5] + } + + return n, nil +} + +var portMatch = regexp.MustCompile(`(.*)\.(\d+)$`) + +// This function only works for netstat returning addresses with a "." +// before the port (0.0.0.0.22 instead of 0.0.0.0:22). +func parseNetstatAddr(local string, remote string, family uint32) (laddr Addr, raddr Addr, err error) { + parse := func(l string) (Addr, error) { + matches := portMatch.FindStringSubmatch(l) + if matches == nil { + return Addr{}, fmt.Errorf("wrong addr, %s", l) + } + host := matches[1] + port := matches[2] + if host == "*" { + switch family { + case syscall.AF_INET: + host = "0.0.0.0" + case syscall.AF_INET6: + host = "::" + default: + return Addr{}, fmt.Errorf("unknown family, %d", family) + } + } + lport, err := strconv.Atoi(port) + if err != nil { + return Addr{}, err + } + return Addr{IP: host, Port: uint32(lport)}, nil + } + + laddr, err = parse(local) + if remote != "*.*" { // remote addr exists + raddr, err = parse(remote) + if err != nil { + return laddr, raddr, err + } + } + + return laddr, raddr, err +} + +func parseNetstatUnixLine(f []string) (ConnectionStat, error) { + if len(f) < 8 { + return ConnectionStat{}, fmt.Errorf("wrong number of fields: expected >=8 got %d", len(f)) + } + + var netType uint32 + + switch f[1] { + case "dgram": + netType = syscall.SOCK_DGRAM + case "stream": + netType = syscall.SOCK_STREAM + default: + return ConnectionStat{}, fmt.Errorf("unknown type: %s", f[1]) + } + + // Some Unix Socket don't have any address associated + addr := "" + if len(f) == 9 { + addr = f[8] + } + + c := ConnectionStat{ + Fd: uint32(0), // not supported + Family: uint32(syscall.AF_UNIX), + Type: uint32(netType), + Laddr: Addr{ + IP: addr, + }, + Status: "NONE", + Pid: int32(0), // not supported + } + + return c, nil +} + +// Return true if proto is the corresponding to the kind parameter +// Only for Inet lines +func hasCorrectInetProto(kind, proto string) bool { + switch kind { + case "all", "inet": + return true + case "unix": + return false + case "inet4": + return !strings.HasSuffix(proto, "6") + case "inet6": + return strings.HasSuffix(proto, "6") + case "tcp": + return proto == "tcp" || proto == "tcp4" || proto == "tcp6" + case "tcp4": + return proto == "tcp" || proto == "tcp4" + case "tcp6": + return proto == "tcp6" + case "udp": + return proto == "udp" || proto == "udp4" || proto == "udp6" + case "udp4": + return proto == "udp" || proto == "udp4" + case "udp6": + return proto == "udp6" + } + return false +} + +func parseNetstatA(output string, kind string) ([]ConnectionStat, error) { + var ret []ConnectionStat + lines := strings.Split(string(output), "\n") + + for _, line := range lines { + fields := strings.Fields(line) + if len(fields) < 1 { + continue + } + + if strings.HasPrefix(fields[0], "f1") { + // Unix lines + if len(fields) < 2 { + // every unix connections have two lines + continue + } + + c, err := parseNetstatUnixLine(fields) + if err != nil { + return nil, fmt.Errorf("failed to parse Unix Address (%s): %s", line, err) + } + + ret = append(ret, c) + + } else if strings.HasPrefix(fields[0], "tcp") || strings.HasPrefix(fields[0], "udp") { + // Inet lines + if !hasCorrectInetProto(kind, fields[0]) { + continue + } + + // On AIX, netstat display some connections with "*.*" as local addresses + // Skip them as they aren't real connections. + if fields[3] == "*.*" { + continue + } + + c, err := parseNetstatNetLine(line) + if err != nil { + return nil, fmt.Errorf("failed to parse Inet Address (%s): %s", line, err) + } + + ret = append(ret, c) + } else { + // Header lines + continue + } + } + + return ret, nil + +} + +func Connections(kind string) ([]ConnectionStat, error) { + return ConnectionsWithContext(context.Background(), kind) +} + +func ConnectionsWithContext(ctx context.Context, kind string) ([]ConnectionStat, error) { + + args := []string{"-na"} + switch strings.ToLower(kind) { + default: + fallthrough + case "": + kind = "all" + case "all": + // nothing to add + case "inet", "inet4", "inet6": + args = append(args, "-finet") + case "tcp", "tcp4", "tcp6": + args = append(args, "-finet") + case "udp", "udp4", "udp6": + args = append(args, "-finet") + case "unix": + args = append(args, "-funix") + } + + netstat, err := exec.LookPath("netstat") + if err != nil { + return nil, err + } + out, err := invoke.CommandWithContext(ctx, netstat, args...) + + if err != nil { + return nil, err + } + + ret, err := parseNetstatA(string(out), kind) + if err != nil { + return nil, err + } + + return ret, nil + +} + +func ConnectionsMax(kind string, max int) ([]ConnectionStat, error) { + return ConnectionsMaxWithContext(context.Background(), kind, max) +} + +func ConnectionsMaxWithContext(ctx context.Context, kind string, max int) ([]ConnectionStat, error) { + return []ConnectionStat{}, common.ErrNotImplementedError +} + +// Return a list of network connections opened, omitting `Uids`. +// WithoutUids functions are reliant on implementation details. They may be altered to be an alias for Connections or be +// removed from the API in the future. +func ConnectionsWithoutUids(kind string) ([]ConnectionStat, error) { + return ConnectionsWithoutUidsWithContext(context.Background(), kind) +} + +func ConnectionsWithoutUidsWithContext(ctx context.Context, kind string) ([]ConnectionStat, error) { + return ConnectionsMaxWithoutUidsWithContext(ctx, kind, 0) +} + +func ConnectionsMaxWithoutUidsWithContext(ctx context.Context, kind string, max int) ([]ConnectionStat, error) { + return ConnectionsPidMaxWithoutUidsWithContext(ctx, kind, 0, max) +} + +func ConnectionsPidWithoutUids(kind string, pid int32) ([]ConnectionStat, error) { + return ConnectionsPidWithoutUidsWithContext(context.Background(), kind, pid) +} + +func ConnectionsPidWithoutUidsWithContext(ctx context.Context, kind string, pid int32) ([]ConnectionStat, error) { + return ConnectionsPidMaxWithoutUidsWithContext(ctx, kind, pid, 0) +} + +func ConnectionsPidMaxWithoutUids(kind string, pid int32, max int) ([]ConnectionStat, error) { + return ConnectionsPidMaxWithoutUidsWithContext(context.Background(), kind, pid, max) +} + +func ConnectionsPidMaxWithoutUidsWithContext(ctx context.Context, kind string, pid int32, max int) ([]ConnectionStat, error) { + return connectionsPidMaxWithoutUidsWithContext(ctx, kind, pid, max) +} + +func connectionsPidMaxWithoutUidsWithContext(ctx context.Context, kind string, pid int32, max int) ([]ConnectionStat, error) { + return []ConnectionStat{}, common.ErrNotImplementedError +} diff --git a/github.com/shirou/gopsutil/net/net_darwin.go b/github.com/shirou/gopsutil/net/net_darwin.go index 0d89280f98..1daed86982 100644 --- a/github.com/shirou/gopsutil/net/net_darwin.go +++ b/github.com/shirou/gopsutil/net/net_darwin.go @@ -6,6 +6,7 @@ import ( "context" "errors" "fmt" + "github.com/shirou/gopsutil/internal/common" "os/exec" "regexp" "strconv" @@ -42,7 +43,7 @@ func parseNetstatLine(line string) (stat *IOCountersStat, linkID *uint, err erro base := 1 numberColumns := len(columns) - // sometimes Address is ommitted + // sometimes Address is omitted if numberColumns < 12 { base = 0 } @@ -174,7 +175,7 @@ func IOCountersWithContext(ctx context.Context, pernic bool) ([]IOCountersStat, retIndex int ) - netstat, err := exec.LookPath("/usr/sbin/netstat") + netstat, err := exec.LookPath("netstat") if err != nil { return nil, err } @@ -204,7 +205,7 @@ func IOCountersWithContext(ctx context.Context, pernic bool) ([]IOCountersStat, } } else { // duplicated interface, list all interfaces - ifconfig, err := exec.LookPath("/sbin/ifconfig") + ifconfig, err := exec.LookPath("ifconfig") if err != nil { return nil, err } @@ -271,6 +272,14 @@ func FilterCountersWithContext(ctx context.Context) ([]FilterStat, error) { return nil, errors.New("NetFilterCounters not implemented for darwin") } +func ConntrackStats(percpu bool) ([]ConntrackStat, error) { + return ConntrackStatsWithContext(context.Background(), percpu) +} + +func ConntrackStatsWithContext(ctx context.Context, percpu bool) ([]ConntrackStat, error) { + return nil, common.ErrNotImplementedError +} + // NetProtoCounters returns network statistics for the entire system // If protocols is empty then all protocols are returned, otherwise // just the protocols in the list are returned. diff --git a/github.com/shirou/gopsutil/net/net_fallback.go b/github.com/shirou/gopsutil/net/net_fallback.go index 7c5e632f87..7d9a265918 100644 --- a/github.com/shirou/gopsutil/net/net_fallback.go +++ b/github.com/shirou/gopsutil/net/net_fallback.go @@ -1,4 +1,4 @@ -// +build !darwin,!linux,!freebsd,!openbsd,!windows +// +build !aix,!darwin,!linux,!freebsd,!openbsd,!windows package net @@ -24,6 +24,14 @@ func FilterCountersWithContext(ctx context.Context) ([]FilterStat, error) { return []FilterStat{}, common.ErrNotImplementedError } +func ConntrackStats(percpu bool) ([]ConntrackStat, error) { + return ConntrackStatsWithContext(context.Background(), percpu) +} + +func ConntrackStatsWithContext(ctx context.Context, percpu bool) ([]ConntrackStat, error) { + return nil, common.ErrNotImplementedError +} + func ProtoCounters(protocols []string) ([]ProtoCountersStat, error) { return ProtoCountersWithContext(context.Background(), protocols) } @@ -47,3 +55,38 @@ func ConnectionsMax(kind string, max int) ([]ConnectionStat, error) { func ConnectionsMaxWithContext(ctx context.Context, kind string, max int) ([]ConnectionStat, error) { return []ConnectionStat{}, common.ErrNotImplementedError } + +// Return a list of network connections opened, omitting `Uids`. +// WithoutUids functions are reliant on implementation details. They may be altered to be an alias for Connections or be +// removed from the API in the future. +func ConnectionsWithoutUids(kind string) ([]ConnectionStat, error) { + return ConnectionsWithoutUidsWithContext(context.Background(), kind) +} + +func ConnectionsWithoutUidsWithContext(ctx context.Context, kind string) ([]ConnectionStat, error) { + return ConnectionsMaxWithoutUidsWithContext(ctx, kind, 0) +} + +func ConnectionsMaxWithoutUidsWithContext(ctx context.Context, kind string, max int) ([]ConnectionStat, error) { + return ConnectionsPidMaxWithoutUidsWithContext(ctx, kind, 0, max) +} + +func ConnectionsPidWithoutUids(kind string, pid int32) ([]ConnectionStat, error) { + return ConnectionsPidWithoutUidsWithContext(context.Background(), kind, pid) +} + +func ConnectionsPidWithoutUidsWithContext(ctx context.Context, kind string, pid int32) ([]ConnectionStat, error) { + return ConnectionsPidMaxWithoutUidsWithContext(ctx, kind, pid, 0) +} + +func ConnectionsPidMaxWithoutUids(kind string, pid int32, max int) ([]ConnectionStat, error) { + return ConnectionsPidMaxWithoutUidsWithContext(context.Background(), kind, pid, max) +} + +func ConnectionsPidMaxWithoutUidsWithContext(ctx context.Context, kind string, pid int32, max int) ([]ConnectionStat, error) { + return connectionsPidMaxWithoutUidsWithContext(ctx, kind, pid, max) +} + +func connectionsPidMaxWithoutUidsWithContext(ctx context.Context, kind string, pid int32, max int) ([]ConnectionStat, error) { + return []ConnectionStat{}, common.ErrNotImplementedError +} diff --git a/github.com/shirou/gopsutil/net/net_freebsd.go b/github.com/shirou/gopsutil/net/net_freebsd.go index ce0241576f..2284d982c8 100644 --- a/github.com/shirou/gopsutil/net/net_freebsd.go +++ b/github.com/shirou/gopsutil/net/net_freebsd.go @@ -17,7 +17,7 @@ func IOCounters(pernic bool) ([]IOCountersStat, error) { } func IOCountersWithContext(ctx context.Context, pernic bool) ([]IOCountersStat, error) { - netstat, err := exec.LookPath("/usr/bin/netstat") + netstat, err := exec.LookPath("netstat") if err != nil { return nil, err } @@ -45,7 +45,7 @@ func IOCountersWithContext(ctx context.Context, pernic bool) ([]IOCountersStat, continue } base := 1 - // sometimes Address is ommitted + // sometimes Address is omitted if len(values) < 13 { base = 0 } @@ -112,6 +112,14 @@ func FilterCountersWithContext(ctx context.Context) ([]FilterStat, error) { return nil, errors.New("NetFilterCounters not implemented for freebsd") } +func ConntrackStats(percpu bool) ([]ConntrackStat, error) { + return ConntrackStatsWithContext(context.Background(), percpu) +} + +func ConntrackStatsWithContext(ctx context.Context, percpu bool) ([]ConntrackStat, error) { + return nil, errors.New("ConntrackStats not implemented for freebsd") +} + // NetProtoCounters returns network statistics for the entire system // If protocols is empty then all protocols are returned, otherwise // just the protocols in the list are returned. diff --git a/github.com/shirou/gopsutil/net/net_linux.go b/github.com/shirou/gopsutil/net/net_linux.go index 71842fb4a2..f289a5dcc1 100644 --- a/github.com/shirou/gopsutil/net/net_linux.go +++ b/github.com/shirou/gopsutil/net/net_linux.go @@ -8,6 +8,7 @@ import ( "encoding/hex" "errors" "fmt" + "io" "io/ioutil" "net" "os" @@ -18,6 +19,26 @@ import ( "github.com/shirou/gopsutil/internal/common" ) +const ( // Conntrack Column numbers + CT_ENTRIES = iota + CT_SEARCHED + CT_FOUND + CT_NEW + CT_INVALID + CT_IGNORE + CT_DELETE + CT_DELETE_LIST + CT_INSERT + CT_INSERT_FAILED + CT_DROP + CT_EARLY_DROP + CT_ICMP_ERROR + CT_EXPECT_NEW + CT_EXPECT_CREATE + CT_EXPECT_DELETE + CT_SEARCH_RESTART +) + // NetIOCounters returnes network I/O statistics for every network // interface installed on the system. If pernic argument is false, // return only sum of all information (which name is 'all'). If true, @@ -29,7 +50,7 @@ func IOCounters(pernic bool) ([]IOCountersStat, error) { func IOCountersWithContext(ctx context.Context, pernic bool) ([]IOCountersStat, error) { filename := common.HostProc("net/dev") - return IOCountersByFile(pernic, filename) + return IOCountersByFileWithContext(ctx, pernic, filename) } func IOCountersByFile(pernic bool, filename string) ([]IOCountersStat, error) { @@ -232,6 +253,58 @@ func FilterCountersWithContext(ctx context.Context) ([]FilterStat, error) { return stats, nil } +// ConntrackStats returns more detailed info about the conntrack table +func ConntrackStats(percpu bool) ([]ConntrackStat, error) { + return ConntrackStatsWithContext(context.Background(), percpu) +} + +// ConntrackStatsWithContext returns more detailed info about the conntrack table +func ConntrackStatsWithContext(ctx context.Context, percpu bool) ([]ConntrackStat, error) { + return conntrackStatsFromFile(common.HostProc("net/stat/nf_conntrack"), percpu) +} + +// conntrackStatsFromFile returns more detailed info about the conntrack table +// from `filename` +// If 'percpu' is false, the result will contain exactly one item with totals/summary +func conntrackStatsFromFile(filename string, percpu bool) ([]ConntrackStat, error) { + lines, err := common.ReadLines(filename) + if err != nil { + return nil, err + } + + statlist := NewConntrackStatList() + + for _, line := range lines { + fields := strings.Fields(line) + if len(fields) == 17 && fields[0] != "entries" { + statlist.Append(NewConntrackStat( + common.HexToUint32(fields[CT_ENTRIES]), + common.HexToUint32(fields[CT_SEARCHED]), + common.HexToUint32(fields[CT_FOUND]), + common.HexToUint32(fields[CT_NEW]), + common.HexToUint32(fields[CT_INVALID]), + common.HexToUint32(fields[CT_IGNORE]), + common.HexToUint32(fields[CT_DELETE]), + common.HexToUint32(fields[CT_DELETE_LIST]), + common.HexToUint32(fields[CT_INSERT]), + common.HexToUint32(fields[CT_INSERT_FAILED]), + common.HexToUint32(fields[CT_DROP]), + common.HexToUint32(fields[CT_EARLY_DROP]), + common.HexToUint32(fields[CT_ICMP_ERROR]), + common.HexToUint32(fields[CT_EXPECT_NEW]), + common.HexToUint32(fields[CT_EXPECT_CREATE]), + common.HexToUint32(fields[CT_EXPECT_DELETE]), + common.HexToUint32(fields[CT_SEARCH_RESTART]), + )) + } + } + + if percpu { + return statlist.Items(), nil + } + return statlist.Summary(), nil +} + // http://students.mimuw.edu.pl/lxr/source/include/net/tcp_states.h var TCPStatuses = map[string]string{ "01": "ESTABLISHED", @@ -328,32 +401,36 @@ func ConnectionsMaxWithContext(ctx context.Context, kind string, max int) ([]Con return ConnectionsPidMax(kind, 0, max) } +// Return a list of network connections opened, omitting `Uids`. +// WithoutUids functions are reliant on implementation details. They may be altered to be an alias for Connections or be +// removed from the API in the future. +func ConnectionsWithoutUids(kind string) ([]ConnectionStat, error) { + return ConnectionsWithoutUidsWithContext(context.Background(), kind) +} + +func ConnectionsWithoutUidsWithContext(ctx context.Context, kind string) ([]ConnectionStat, error) { + return ConnectionsMaxWithoutUidsWithContext(ctx, kind, 0) +} + +func ConnectionsMaxWithoutUidsWithContext(ctx context.Context, kind string, max int) ([]ConnectionStat, error) { + return ConnectionsPidMaxWithoutUidsWithContext(ctx, kind, 0, max) +} + // Return a list of network connections opened by a process. func ConnectionsPid(kind string, pid int32) ([]ConnectionStat, error) { return ConnectionsPidWithContext(context.Background(), kind, pid) } +func ConnectionsPidWithoutUids(kind string, pid int32) ([]ConnectionStat, error) { + return ConnectionsPidWithoutUidsWithContext(context.Background(), kind, pid) +} + func ConnectionsPidWithContext(ctx context.Context, kind string, pid int32) ([]ConnectionStat, error) { - tmap, ok := netConnectionKindMap[kind] - if !ok { - return nil, fmt.Errorf("invalid kind, %s", kind) - } - root := common.HostProc() - var err error - var inodes map[string][]inodeMap - if pid == 0 { - inodes, err = getProcInodesAll(root, 0) - } else { - inodes, err = getProcInodes(root, pid, 0) - if len(inodes) == 0 { - // no connection for the pid - return []ConnectionStat{}, nil - } - } - if err != nil { - return nil, fmt.Errorf("cound not get pid(s), %d: %s", pid, err) - } - return statsFromInodes(root, pid, tmap, inodes) + return ConnectionsPidMaxWithContext(ctx, kind, pid, 0) +} + +func ConnectionsPidWithoutUidsWithContext(ctx context.Context, kind string, pid int32) ([]ConnectionStat, error) { + return ConnectionsPidMaxWithoutUidsWithContext(ctx, kind, pid, 0) } // Return up to `max` network connections opened by a process. @@ -361,7 +438,19 @@ func ConnectionsPidMax(kind string, pid int32, max int) ([]ConnectionStat, error return ConnectionsPidMaxWithContext(context.Background(), kind, pid, max) } +func ConnectionsPidMaxWithoutUids(kind string, pid int32, max int) ([]ConnectionStat, error) { + return ConnectionsPidMaxWithoutUidsWithContext(context.Background(), kind, pid, max) +} + func ConnectionsPidMaxWithContext(ctx context.Context, kind string, pid int32, max int) ([]ConnectionStat, error) { + return connectionsPidMaxWithoutUidsWithContext(ctx, kind, pid, max, false) +} + +func ConnectionsPidMaxWithoutUidsWithContext(ctx context.Context, kind string, pid int32, max int) ([]ConnectionStat, error) { + return connectionsPidMaxWithoutUidsWithContext(ctx, kind, pid, max, true) +} + +func connectionsPidMaxWithoutUidsWithContext(ctx context.Context, kind string, pid int32, max int, skipUids bool) ([]ConnectionStat, error) { tmap, ok := netConnectionKindMap[kind] if !ok { return nil, fmt.Errorf("invalid kind, %s", kind) @@ -379,12 +468,12 @@ func ConnectionsPidMaxWithContext(ctx context.Context, kind string, pid int32, m } } if err != nil { - return nil, fmt.Errorf("cound not get pid(s), %d", pid) + return nil, fmt.Errorf("cound not get pid(s), %d: %s", pid, err) } - return statsFromInodes(root, pid, tmap, inodes) + return statsFromInodes(root, pid, tmap, inodes, skipUids) } -func statsFromInodes(root string, pid int32, tmap []netConnectionKindType, inodes map[string][]inodeMap) ([]ConnectionStat, error) { +func statsFromInodes(root string, pid int32, tmap []netConnectionKindType, inodes map[string][]inodeMap, skipUids bool) ([]ConnectionStat, error) { dupCheckMap := make(map[string]struct{}) var ret []ConnectionStat @@ -431,9 +520,11 @@ func statsFromInodes(root string, pid int32, tmap []netConnectionKindType, inode conn.Pid = c.pid } - // fetch process owner Real, effective, saved set, and filesystem UIDs - proc := process{Pid: conn.Pid} - conn.Uids, _ = proc.getUids() + if !skipUids { + // fetch process owner Real, effective, saved set, and filesystem UIDs + proc := process{Pid: conn.Pid} + conn.Uids, _ = proc.getUids() + } ret = append(ret, conn) dupCheckMap[connKey] = struct{}{} @@ -581,7 +672,7 @@ func getProcInodesAll(root string, max int) (map[string][]inodeMap, error) { t, err := getProcInodes(root, pid, max) if err != nil { // skip if permission error or no longer exists - if os.IsPermission(err) || os.IsNotExist(err) { + if os.IsPermission(err) || os.IsNotExist(err) || err == io.EOF { continue } return ret, err diff --git a/github.com/shirou/gopsutil/net/net_openbsd.go b/github.com/shirou/gopsutil/net/net_openbsd.go index 3e74e8f3ad..3cf0a89d47 100644 --- a/github.com/shirou/gopsutil/net/net_openbsd.go +++ b/github.com/shirou/gopsutil/net/net_openbsd.go @@ -41,7 +41,7 @@ func ParseNetstat(output string, mode string, continue } base := 1 - // sometimes Address is ommitted + // sometimes Address is omitted if len(values) < columns { base = 0 } @@ -156,6 +156,14 @@ func FilterCountersWithContext(ctx context.Context) ([]FilterStat, error) { return nil, errors.New("NetFilterCounters not implemented for openbsd") } +func ConntrackStats(percpu bool) ([]ConntrackStat, error) { + return ConntrackStatsWithContext(context.Background(), percpu) +} + +func ConntrackStatsWithContext(ctx context.Context, percpu bool) ([]ConntrackStat, error) { + return nil, common.ErrNotImplementedError +} + // NetProtoCounters returns network statistics for the entire system // If protocols is empty then all protocols are returned, otherwise // just the protocols in the list are returned. diff --git a/github.com/shirou/gopsutil/net/net_unix.go b/github.com/shirou/gopsutil/net/net_unix.go index 4451b54574..d6e4303fdb 100644 --- a/github.com/shirou/gopsutil/net/net_unix.go +++ b/github.com/shirou/gopsutil/net/net_unix.go @@ -4,7 +4,11 @@ package net import ( "context" + "fmt" + "net" + "strconv" "strings" + "syscall" "github.com/shirou/gopsutil/internal/common" ) @@ -86,6 +90,95 @@ func ConnectionsPidWithContext(ctx context.Context, kind string, pid int32) ([]C return ret, nil } +var constMap = map[string]int{ + "unix": syscall.AF_UNIX, + "TCP": syscall.SOCK_STREAM, + "UDP": syscall.SOCK_DGRAM, + "IPv4": syscall.AF_INET, + "IPv6": syscall.AF_INET6, +} + +func parseNetLine(line string) (ConnectionStat, error) { + f := strings.Fields(line) + if len(f) < 8 { + return ConnectionStat{}, fmt.Errorf("wrong line,%s", line) + } + + if len(f) == 8 { + f = append(f, f[7]) + f[7] = "unix" + } + + pid, err := strconv.Atoi(f[1]) + if err != nil { + return ConnectionStat{}, err + } + fd, err := strconv.Atoi(strings.Trim(f[3], "u")) + if err != nil { + return ConnectionStat{}, fmt.Errorf("unknown fd, %s", f[3]) + } + netFamily, ok := constMap[f[4]] + if !ok { + return ConnectionStat{}, fmt.Errorf("unknown family, %s", f[4]) + } + netType, ok := constMap[f[7]] + if !ok { + return ConnectionStat{}, fmt.Errorf("unknown type, %s", f[7]) + } + + var laddr, raddr Addr + if f[7] == "unix" { + laddr.IP = f[8] + } else { + laddr, raddr, err = parseNetAddr(f[8]) + if err != nil { + return ConnectionStat{}, fmt.Errorf("failed to parse netaddr, %s", f[8]) + } + } + + n := ConnectionStat{ + Fd: uint32(fd), + Family: uint32(netFamily), + Type: uint32(netType), + Laddr: laddr, + Raddr: raddr, + Pid: int32(pid), + } + if len(f) == 10 { + n.Status = strings.Trim(f[9], "()") + } + + return n, nil +} + +func parseNetAddr(line string) (laddr Addr, raddr Addr, err error) { + parse := func(l string) (Addr, error) { + host, port, err := net.SplitHostPort(l) + if err != nil { + return Addr{}, fmt.Errorf("wrong addr, %s", l) + } + lport, err := strconv.Atoi(port) + if err != nil { + return Addr{}, err + } + return Addr{IP: host, Port: uint32(lport)}, nil + } + + addrs := strings.Split(line, "->") + if len(addrs) == 0 { + return laddr, raddr, fmt.Errorf("wrong netaddr, %s", line) + } + laddr, err = parse(addrs[0]) + if len(addrs) == 2 { // remote addr exists + raddr, err = parse(addrs[1]) + if err != nil { + return laddr, raddr, err + } + } + + return laddr, raddr, err +} + // Return up to `max` network connections opened by a process. func ConnectionsPidMax(kind string, pid int32, max int) ([]ConnectionStat, error) { return ConnectionsPidMaxWithContext(context.Background(), kind, pid, max) @@ -94,3 +187,38 @@ func ConnectionsPidMax(kind string, pid int32, max int) ([]ConnectionStat, error func ConnectionsPidMaxWithContext(ctx context.Context, kind string, pid int32, max int) ([]ConnectionStat, error) { return []ConnectionStat{}, common.ErrNotImplementedError } + +// Return a list of network connections opened, omitting `Uids`. +// WithoutUids functions are reliant on implementation details. They may be altered to be an alias for Connections or be +// removed from the API in the future. +func ConnectionsWithoutUids(kind string) ([]ConnectionStat, error) { + return ConnectionsWithoutUidsWithContext(context.Background(), kind) +} + +func ConnectionsWithoutUidsWithContext(ctx context.Context, kind string) ([]ConnectionStat, error) { + return ConnectionsMaxWithoutUidsWithContext(ctx, kind, 0) +} + +func ConnectionsMaxWithoutUidsWithContext(ctx context.Context, kind string, max int) ([]ConnectionStat, error) { + return ConnectionsPidMaxWithoutUidsWithContext(ctx, kind, 0, max) +} + +func ConnectionsPidWithoutUids(kind string, pid int32) ([]ConnectionStat, error) { + return ConnectionsPidWithoutUidsWithContext(context.Background(), kind, pid) +} + +func ConnectionsPidWithoutUidsWithContext(ctx context.Context, kind string, pid int32) ([]ConnectionStat, error) { + return ConnectionsPidMaxWithoutUidsWithContext(ctx, kind, pid, 0) +} + +func ConnectionsPidMaxWithoutUids(kind string, pid int32, max int) ([]ConnectionStat, error) { + return ConnectionsPidMaxWithoutUidsWithContext(context.Background(), kind, pid, max) +} + +func ConnectionsPidMaxWithoutUidsWithContext(ctx context.Context, kind string, pid int32, max int) ([]ConnectionStat, error) { + return connectionsPidMaxWithoutUidsWithContext(ctx, kind, pid, max) +} + +func connectionsPidMaxWithoutUidsWithContext(ctx context.Context, kind string, pid int32, max int) ([]ConnectionStat, error) { + return []ConnectionStat{}, common.ErrNotImplementedError +} diff --git a/github.com/shirou/gopsutil/net/net_windows.go b/github.com/shirou/gopsutil/net/net_windows.go index 61eb6ec20c..6ab45ab3a7 100644 --- a/github.com/shirou/gopsutil/net/net_windows.go +++ b/github.com/shirou/gopsutil/net/net_windows.go @@ -19,6 +19,7 @@ var ( modiphlpapi = windows.NewLazySystemDLL("iphlpapi.dll") procGetExtendedTCPTable = modiphlpapi.NewProc("GetExtendedTcpTable") procGetExtendedUDPTable = modiphlpapi.NewProc("GetExtendedUdpTable") + procGetIfEntry2 = modiphlpapi.NewProc("GetIfEntry2") ) const ( @@ -73,6 +74,65 @@ var netConnectionKindMap = map[string][]netConnectionKindType{ "inet6": {kindTCP6, kindUDP6}, } +// https://github.com/microsoft/ethr/blob/aecdaf923970e5a9b4c461b4e2e3963d781ad2cc/plt_windows.go#L114-L170 +type guid struct { + Data1 uint32 + Data2 uint16 + Data3 uint16 + Data4 [8]byte +} + +const ( + maxStringSize = 256 + maxPhysAddressLength = 32 + pad0for64_4for32 = 0 +) + +type mibIfRow2 struct { + InterfaceLuid uint64 + InterfaceIndex uint32 + InterfaceGuid guid + Alias [maxStringSize + 1]uint16 + Description [maxStringSize + 1]uint16 + PhysicalAddressLength uint32 + PhysicalAddress [maxPhysAddressLength]uint8 + PermanentPhysicalAddress [maxPhysAddressLength]uint8 + Mtu uint32 + Type uint32 + TunnelType uint32 + MediaType uint32 + PhysicalMediumType uint32 + AccessType uint32 + DirectionType uint32 + InterfaceAndOperStatusFlags uint32 + OperStatus uint32 + AdminStatus uint32 + MediaConnectState uint32 + NetworkGuid guid + ConnectionType uint32 + padding1 [pad0for64_4for32]byte + TransmitLinkSpeed uint64 + ReceiveLinkSpeed uint64 + InOctets uint64 + InUcastPkts uint64 + InNUcastPkts uint64 + InDiscards uint64 + InErrors uint64 + InUnknownProtos uint64 + InUcastOctets uint64 + InMulticastOctets uint64 + InBroadcastOctets uint64 + OutOctets uint64 + OutUcastPkts uint64 + OutNUcastPkts uint64 + OutDiscards uint64 + OutErrors uint64 + OutUcastOctets uint64 + OutMulticastOctets uint64 + OutBroadcastOctets uint64 + OutQLen uint64 +} + func IOCounters(pernic bool) ([]IOCountersStat, error) { return IOCountersWithContext(context.Background(), pernic) } @@ -82,34 +142,59 @@ func IOCountersWithContext(ctx context.Context, pernic bool) ([]IOCountersStat, if err != nil { return nil, err } - var ret []IOCountersStat + var counters []IOCountersStat + + err = procGetIfEntry2.Find() + if err == nil { // Vista+, uint64 values (issue#693) + for _, ifi := range ifs { + c := IOCountersStat{ + Name: ifi.Name, + } - for _, ifi := range ifs { - c := IOCountersStat{ - Name: ifi.Name, + row := mibIfRow2{InterfaceIndex: uint32(ifi.Index)} + ret, _, err := procGetIfEntry2.Call(uintptr(unsafe.Pointer(&row))) + if ret != 0 { + return nil, os.NewSyscallError("GetIfEntry2", err) + } + c.BytesSent = uint64(row.OutOctets) + c.BytesRecv = uint64(row.InOctets) + c.PacketsSent = uint64(row.OutUcastPkts) + c.PacketsRecv = uint64(row.InUcastPkts) + c.Errin = uint64(row.InErrors) + c.Errout = uint64(row.OutErrors) + c.Dropin = uint64(row.InDiscards) + c.Dropout = uint64(row.OutDiscards) + + counters = append(counters, c) } + } else { // WinXP fallback, uint32 values + for _, ifi := range ifs { + c := IOCountersStat{ + Name: ifi.Name, + } - row := windows.MibIfRow{Index: uint32(ifi.Index)} - e := windows.GetIfEntry(&row) - if e != nil { - return nil, os.NewSyscallError("GetIfEntry", e) + row := windows.MibIfRow{Index: uint32(ifi.Index)} + err = windows.GetIfEntry(&row) + if err != nil { + return nil, os.NewSyscallError("GetIfEntry", err) + } + c.BytesSent = uint64(row.OutOctets) + c.BytesRecv = uint64(row.InOctets) + c.PacketsSent = uint64(row.OutUcastPkts) + c.PacketsRecv = uint64(row.InUcastPkts) + c.Errin = uint64(row.InErrors) + c.Errout = uint64(row.OutErrors) + c.Dropin = uint64(row.InDiscards) + c.Dropout = uint64(row.OutDiscards) + + counters = append(counters, c) } - c.BytesSent = uint64(row.OutOctets) - c.BytesRecv = uint64(row.InOctets) - c.PacketsSent = uint64(row.OutUcastPkts) - c.PacketsRecv = uint64(row.InUcastPkts) - c.Errin = uint64(row.InErrors) - c.Errout = uint64(row.OutErrors) - c.Dropin = uint64(row.InDiscards) - c.Dropout = uint64(row.OutDiscards) - - ret = append(ret, c) } - if pernic == false { - return getIOCountersAll(ret) + if !pernic { + return getIOCountersAll(counters) } - return ret, nil + return counters, nil } // NetIOCountersByFile is an method which is added just a compatibility for linux. @@ -198,6 +283,41 @@ func ConnectionsMaxWithContext(ctx context.Context, kind string, max int) ([]Con return []ConnectionStat{}, common.ErrNotImplementedError } +// Return a list of network connections opened, omitting `Uids`. +// WithoutUids functions are reliant on implementation details. They may be altered to be an alias for Connections or be +// removed from the API in the future. +func ConnectionsWithoutUids(kind string) ([]ConnectionStat, error) { + return ConnectionsWithoutUidsWithContext(context.Background(), kind) +} + +func ConnectionsWithoutUidsWithContext(ctx context.Context, kind string) ([]ConnectionStat, error) { + return ConnectionsMaxWithoutUidsWithContext(ctx, kind, 0) +} + +func ConnectionsMaxWithoutUidsWithContext(ctx context.Context, kind string, max int) ([]ConnectionStat, error) { + return ConnectionsPidMaxWithoutUidsWithContext(ctx, kind, 0, max) +} + +func ConnectionsPidWithoutUids(kind string, pid int32) ([]ConnectionStat, error) { + return ConnectionsPidWithoutUidsWithContext(context.Background(), kind, pid) +} + +func ConnectionsPidWithoutUidsWithContext(ctx context.Context, kind string, pid int32) ([]ConnectionStat, error) { + return ConnectionsPidMaxWithoutUidsWithContext(ctx, kind, pid, 0) +} + +func ConnectionsPidMaxWithoutUids(kind string, pid int32, max int) ([]ConnectionStat, error) { + return ConnectionsPidMaxWithoutUidsWithContext(context.Background(), kind, pid, max) +} + +func ConnectionsPidMaxWithoutUidsWithContext(ctx context.Context, kind string, pid int32, max int) ([]ConnectionStat, error) { + return connectionsPidMaxWithoutUidsWithContext(ctx, kind, pid, max) +} + +func connectionsPidMaxWithoutUidsWithContext(ctx context.Context, kind string, pid int32, max int) ([]ConnectionStat, error) { + return []ConnectionStat{}, common.ErrNotImplementedError +} + func FilterCounters() ([]FilterStat, error) { return FilterCountersWithContext(context.Background()) } @@ -206,6 +326,15 @@ func FilterCountersWithContext(ctx context.Context) ([]FilterStat, error) { return nil, errors.New("NetFilterCounters not implemented for windows") } +func ConntrackStats(percpu bool) ([]ConntrackStat, error) { + return ConntrackStatsWithContext(context.Background(), percpu) +} + +func ConntrackStatsWithContext(ctx context.Context, percpu bool) ([]ConntrackStat, error) { + return nil, common.ErrNotImplementedError +} + + // NetProtoCounters returns network statistics for the entire system // If protocols is empty then all protocols are returned, otherwise // just the protocols in the list are returned. diff --git a/github.com/shirou/gopsutil/process/process.go b/github.com/shirou/gopsutil/process/process.go index 036ad917da..59441a0522 100644 --- a/github.com/shirou/gopsutil/process/process.go +++ b/github.com/shirou/gopsutil/process/process.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "runtime" + "sort" "time" "github.com/shirou/gopsutil/cpu" @@ -13,8 +14,9 @@ import ( ) var ( - invoke common.Invoker = common.Invoke{} - ErrorNoChildren = errors.New("process does not have children") + invoke common.Invoker = common.Invoke{} + ErrorNoChildren = errors.New("process does not have children") + ErrorProcessNotRunning = errors.New("process does not exist") ) type Process struct { @@ -28,6 +30,7 @@ type Process struct { numThreads int32 memInfo *MemoryInfoStat sigInfo *SignalInfoStat + createTime int64 lastCPUTimes *cpu.TimesStat lastCPUTime time.Time @@ -43,6 +46,7 @@ type OpenFilesStat struct { type MemoryInfoStat struct { RSS uint64 `json:"rss"` // bytes VMS uint64 `json:"vms"` // bytes + HWM uint64 `json:"hwm"` // bytes Data uint64 `json:"data"` // bytes Stack uint64 `json:"stack"` // bytes Locked uint64 `json:"locked"` // bytes @@ -76,6 +80,13 @@ type NumCtxSwitchesStat struct { Involuntary int64 `json:"involuntary"` } +type PageFaultsStat struct { + MinorFaults uint64 `json:"minorFaults"` + MajorFaults uint64 `json:"majorFaults"` + ChildMinorFaults uint64 `json:"childMinorFaults"` + ChildMajorFaults uint64 `json:"childMajorFaults"` +} + // Resource limit constants are from /usr/include/x86_64-linux-gnu/bits/resource.h // from libc6-dev package in Ubuntu 16.10 const ( @@ -127,23 +138,37 @@ func (p NumCtxSwitchesStat) String() string { return string(s) } -func PidExists(pid int32) (bool, error) { - return PidExistsWithContext(context.Background(), pid) +// Pids returns a slice of process ID list which are running now. +func Pids() ([]int32, error) { + return PidsWithContext(context.Background()) } -func PidExistsWithContext(ctx context.Context, pid int32) (bool, error) { - pids, err := Pids() +func PidsWithContext(ctx context.Context) ([]int32, error) { + pids, err := pidsWithContext(ctx) + sort.Slice(pids, func(i, j int) bool { return pids[i] < pids[j] }) + return pids, err +} + +// NewProcess creates a new Process instance, it only stores the pid and +// checks that the process exists. Other method on Process can be used +// to get more information about the process. An error will be returned +// if the process does not exist. +func NewProcess(pid int32) (*Process, error) { + p := &Process{Pid: pid} + + exists, err := PidExists(pid) if err != nil { - return false, err + return p, err } - - for _, i := range pids { - if i == pid { - return true, err - } + if !exists { + return p, ErrorProcessNotRunning } + p.CreateTime() + return p, nil +} - return false, err +func PidExists(pid int32) (bool, error) { + return PidExistsWithContext(context.Background(), pid) } // Background returns true if the process is in background, false otherwise. @@ -198,6 +223,41 @@ func (p *Process) PercentWithContext(ctx context.Context, interval time.Duration return ret, nil } +// IsRunning returns whether the process is still running or not. +func (p *Process) IsRunning() (bool, error) { + return p.IsRunningWithContext(context.Background()) +} + +func (p *Process) IsRunningWithContext(ctx context.Context) (bool, error) { + createTime, err := p.CreateTimeWithContext(ctx) + if err != nil { + return false, err + } + p2, err := NewProcess(p.Pid) + if err == ErrorProcessNotRunning { + return false, nil + } + createTime2, err := p2.CreateTimeWithContext(ctx) + if err != nil { + return false, err + } + return createTime == createTime2, nil +} + +// CreateTime returns created time of the process in milliseconds since the epoch, in UTC. +func (p *Process) CreateTime() (int64, error) { + return p.CreateTimeWithContext(context.Background()) +} + +func (p *Process) CreateTimeWithContext(ctx context.Context) (int64, error) { + if p.createTime != 0 { + return p.createTime, nil + } + createTime, err := p.createTimeWithContext(ctx) + p.createTime = createTime + return p.createTime, err +} + func calculatePercent(t1, t2 *cpu.TimesStat, delta float64, numcpu int) float64 { if delta == 0 { return 0 diff --git a/github.com/shirou/gopsutil/process/process_darwin.go b/github.com/shirou/gopsutil/process/process_darwin.go index ca0f18763e..3eb53e686b 100644 --- a/github.com/shirou/gopsutil/process/process_darwin.go +++ b/github.com/shirou/gopsutil/process/process_darwin.go @@ -8,6 +8,7 @@ import ( "encoding/binary" "fmt" "os/exec" + "path/filepath" "strconv" "strings" "time" @@ -44,11 +45,7 @@ type MemoryInfoExStat struct { type MemoryMapsStat struct { } -func Pids() ([]int32, error) { - return PidsWithContext(context.Background()) -} - -func PidsWithContext(ctx context.Context) ([]int32, error) { +func pidsWithContext(ctx context.Context) ([]int32, error) { var ret []int32 pids, err := callPsWithContext(ctx, "pid", 0, false) @@ -93,8 +90,24 @@ func (p *Process) NameWithContext(ctx context.Context) (string, error) { if err != nil { return "", err } + name := common.IntToString(k.Proc.P_comm[:]) + + if len(name) >= 15 { + cmdlineSlice, err := p.CmdlineSliceWithContext(ctx) + if err != nil { + return "", err + } + if len(cmdlineSlice) > 0 { + extendedName := filepath.Base(cmdlineSlice[0]) + if strings.HasPrefix(extendedName, p.name) { + name = extendedName + } else { + name = cmdlineSlice[0] + } + } + } - return common.IntToString(k.Proc.P_comm[:]), nil + return name, nil } func (p *Process) Tgid() (int32, error) { return 0, common.ErrNotImplementedError @@ -103,37 +116,6 @@ func (p *Process) Exe() (string, error) { return p.ExeWithContext(context.Background()) } -func (p *Process) ExeWithContext(ctx context.Context) (string, error) { - lsof_bin, err := exec.LookPath("lsof") - if err != nil { - return "", err - } - - awk_bin, err := exec.LookPath("awk") - if err != nil { - return "", err - } - - sed_bin, err := exec.LookPath("sed") - if err != nil { - return "", err - } - - lsof := exec.CommandContext(ctx, lsof_bin, "-p", strconv.Itoa(int(p.Pid)), "-Fpfn") - awk := exec.CommandContext(ctx, awk_bin, "NR==5{print}") - sed := exec.CommandContext(ctx, sed_bin, "s/n\\//\\//") - - output, _, err := common.Pipeline(lsof, awk, sed) - - if err != nil { - return "", err - } - - ret := strings.TrimSpace(string(output)) - - return ret, nil -} - // Cmdline returns the command line arguments of the process as a string with // each argument separated by 0x20 ascii character. func (p *Process) Cmdline() (string, error) { @@ -164,11 +146,8 @@ func (p *Process) CmdlineSliceWithContext(ctx context.Context) ([]string, error) } return r[0], err } -func (p *Process) CreateTime() (int64, error) { - return p.CreateTimeWithContext(context.Background()) -} -func (p *Process) CreateTimeWithContext(ctx context.Context) (int64, error) { +func (p *Process) createTimeWithContext(ctx context.Context) (int64, error) { r, err := callPsWithContext(ctx, "etime", p.Pid, false) if err != nil { return 0, err @@ -237,7 +216,7 @@ func (p *Process) StatusWithContext(ctx context.Context) (string, error) { return "", err } - return r[0][0], err + return r[0][0][0:1], err } func (p *Process) Foreground() (bool, error) { @@ -389,12 +368,32 @@ func convertCPUTimes(s string) (ret float64, err error) { var _tmp string if strings.Contains(s, ":") { _t := strings.Split(s, ":") - hour, err := strconv.Atoi(_t[0]) - if err != nil { - return ret, err + switch len(_t) { + case 3: + hour, err := strconv.Atoi(_t[0]) + if err != nil { + return ret, err + } + t += hour * 60 * 60 * ClockTicks + + mins, err := strconv.Atoi(_t[1]) + if err != nil { + return ret, err + } + t += mins * 60 * ClockTicks + _tmp = _t[2] + case 2: + mins, err := strconv.Atoi(_t[0]) + if err != nil { + return ret, err + } + t += mins * 60 * ClockTicks + _tmp = _t[1] + case 1, 0: + _tmp = s + default: + return ret, fmt.Errorf("wrong cpu time string") } - t += hour * 60 * 100 - _tmp = _t[1] } else { _tmp = s } @@ -404,7 +403,7 @@ func convertCPUTimes(s string) (ret float64, err error) { return ret, err } h, err := strconv.Atoi(_t[0]) - t += h * 100 + t += h * ClockTicks h, err = strconv.Atoi(_t[1]) t += h return float64(t) / ClockTicks, nil @@ -481,6 +480,14 @@ func (p *Process) MemoryInfoExWithContext(ctx context.Context) (*MemoryInfoExSta return nil, common.ErrNotImplementedError } +func (p *Process) PageFaults() (*PageFaultsStat, error) { + return p.PageFaultsWithContext(context.Background()) +} + +func (p *Process) PageFaultsWithContext(ctx context.Context) (*PageFaultsStat, error) { + return nil, common.ErrNotImplementedError +} + func (p *Process) Children() ([]*Process, error) { return p.ChildrenWithContext(context.Background()) } @@ -517,6 +524,15 @@ func (p *Process) ConnectionsWithContext(ctx context.Context) ([]net.ConnectionS return net.ConnectionsPid("all", p.Pid) } +// Connections returns a slice of net.ConnectionStat used by the process at most `max` +func (p *Process) ConnectionsMax(max int) ([]net.ConnectionStat, error) { + return p.ConnectionsMaxWithContext(context.Background(), max) +} + +func (p *Process) ConnectionsMaxWithContext(ctx context.Context, max int) ([]net.ConnectionStat, error) { + return net.ConnectionsPidMax("all", p.Pid, max) +} + func (p *Process) NetIOCounters(pernic bool) ([]net.IOCountersStat, error) { return p.NetIOCountersWithContext(context.Background(), pernic) } @@ -525,13 +541,6 @@ func (p *Process) NetIOCountersWithContext(ctx context.Context, pernic bool) ([] return nil, common.ErrNotImplementedError } -func (p *Process) IsRunning() (bool, error) { - return p.IsRunningWithContext(context.Background()) -} - -func (p *Process) IsRunningWithContext(ctx context.Context) (bool, error) { - return true, common.ErrNotImplementedError -} func (p *Process) MemoryMaps(grouped bool) (*[]MemoryMapsStat, error) { return p.MemoryMapsWithContext(context.Background(), grouped) } @@ -606,12 +615,6 @@ func (p *Process) getKProcWithContext(ctx context.Context) (*KinfoProc, error) { return &k, nil } -func NewProcess(pid int32) (*Process, error) { - p := &Process{Pid: pid} - - return p, nil -} - // call ps command. // Return value deletes Header line(you must not input wrong arg). // And splited by Space. Caller have responsibility to manage. diff --git a/github.com/shirou/gopsutil/process/process_darwin_cgo.go b/github.com/shirou/gopsutil/process/process_darwin_cgo.go new file mode 100644 index 0000000000..a80817755c --- /dev/null +++ b/github.com/shirou/gopsutil/process/process_darwin_cgo.go @@ -0,0 +1,30 @@ +// +build darwin +// +build cgo + +package process + +// #include +// #include +import "C" +import ( + "context" + "fmt" + "unsafe" +) + +func (p *Process) ExeWithContext(ctx context.Context) (string, error) { + var c C.char // need a var for unsafe.Sizeof need a var + const bufsize = C.PROC_PIDPATHINFO_MAXSIZE * unsafe.Sizeof(c) + buffer := (*C.char)(C.malloc(C.size_t(bufsize))) + defer C.free(unsafe.Pointer(buffer)) + + ret, err := C.proc_pidpath(C.int(p.Pid), unsafe.Pointer(buffer), C.uint32_t(bufsize)) + if err != nil { + return "", err + } + if ret <= 0 { + return "", fmt.Errorf("unknown error: proc_pidpath returned %d", ret) + } + + return C.GoString(buffer), nil +} diff --git a/github.com/shirou/gopsutil/process/process_darwin_nocgo.go b/github.com/shirou/gopsutil/process/process_darwin_nocgo.go new file mode 100644 index 0000000000..3583e19877 --- /dev/null +++ b/github.com/shirou/gopsutil/process/process_darwin_nocgo.go @@ -0,0 +1,34 @@ +// +build darwin +// +build !cgo + +package process + +import ( + "context" + "fmt" + "os/exec" + "strconv" + "strings" +) + +func (p *Process) ExeWithContext(ctx context.Context) (string, error) { + lsof_bin, err := exec.LookPath("lsof") + if err != nil { + return "", err + } + out, err := invoke.CommandWithContext(ctx, lsof_bin, "-p", strconv.Itoa(int(p.Pid)), "-Fpfn") + if err != nil { + return "", fmt.Errorf("bad call to lsof: %s", err) + } + txtFound := 0 + lines := strings.Split(string(out), "\n") + for i := 1; i < len(lines); i++ { + if lines[i] == "ftxt" { + txtFound++ + if txtFound == 2 { + return lines[i-1][1:], nil + } + } + } + return "", fmt.Errorf("missing txt data returned by lsof") +} diff --git a/github.com/shirou/gopsutil/process/process_fallback.go b/github.com/shirou/gopsutil/process/process_fallback.go index 8772440df4..1cb55c8b04 100644 --- a/github.com/shirou/gopsutil/process/process_fallback.go +++ b/github.com/shirou/gopsutil/process/process_fallback.go @@ -28,11 +28,7 @@ type MemoryMapsStat struct { type MemoryInfoExStat struct { } -func Pids() ([]int32, error) { - return PidsWithContext(context.Background()) -} - -func PidsWithContext(ctx context.Context) ([]int32, error) { +func pidsWithContext(ctx context.Context) ([]int32, error) { return []int32{}, common.ErrNotImplementedError } @@ -44,8 +40,19 @@ func ProcessesWithContext(ctx context.Context) ([]*Process, error) { return nil, common.ErrNotImplementedError } -func NewProcess(pid int32) (*Process, error) { - return nil, common.ErrNotImplementedError +func PidExistsWithContext(ctx context.Context, pid int32) (bool, error) { + pids, err := PidsWithContext(ctx) + if err != nil { + return false, err + } + + for _, i := range pids { + if i == pid { + return true, err + } + } + + return false, err } func (p *Process) Ppid() (int32, error) { @@ -86,11 +93,8 @@ func (p *Process) CmdlineSlice() ([]string, error) { func (p *Process) CmdlineSliceWithContext(ctx context.Context) ([]string, error) { return []string{}, common.ErrNotImplementedError } -func (p *Process) CreateTime() (int64, error) { - return p.CreateTimeWithContext(context.Background()) -} -func (p *Process) CreateTimeWithContext(ctx context.Context) (int64, error) { +func (p *Process) createTimeWithContext(ctx context.Context) (int64, error) { return 0, common.ErrNotImplementedError } func (p *Process) Cwd() (string, error) { @@ -233,6 +237,12 @@ func (p *Process) MemoryInfoEx() (*MemoryInfoExStat, error) { func (p *Process) MemoryInfoExWithContext(ctx context.Context) (*MemoryInfoExStat, error) { return nil, common.ErrNotImplementedError } +func (p *Process) PageFaults() (*PageFaultsStat, error) { + return p.PageFaultsWithContext(context.Background()) +} +func (p *Process) PageFaultsWithContext(ctx context.Context) (*PageFaultsStat, error) { + return nil, common.ErrNotImplementedError +} func (p *Process) Children() ([]*Process, error) { return p.ChildrenWithContext(context.Background()) } @@ -254,6 +264,15 @@ func (p *Process) Connections() ([]net.ConnectionStat, error) { func (p *Process) ConnectionsWithContext(ctx context.Context) ([]net.ConnectionStat, error) { return []net.ConnectionStat{}, common.ErrNotImplementedError } + +func (p *Process) ConnectionsMax(max int) ([]net.ConnectionStat, error) { + return p.ConnectionsMaxWithContext(context.Background(), max) +} + +func (p *Process) ConnectionsMaxWithContext(ctx context.Context, max int) ([]net.ConnectionStat, error) { + return []net.ConnectionStat{}, common.ErrNotImplementedError +} + func (p *Process) NetIOCounters(pernic bool) ([]net.IOCountersStat, error) { return p.NetIOCountersWithContext(context.Background(), pernic) } @@ -261,13 +280,7 @@ func (p *Process) NetIOCounters(pernic bool) ([]net.IOCountersStat, error) { func (p *Process) NetIOCountersWithContext(ctx context.Context, pernic bool) ([]net.IOCountersStat, error) { return []net.IOCountersStat{}, common.ErrNotImplementedError } -func (p *Process) IsRunning() (bool, error) { - return p.IsRunningWithContext(context.Background()) -} -func (p *Process) IsRunningWithContext(ctx context.Context) (bool, error) { - return true, common.ErrNotImplementedError -} func (p *Process) MemoryMaps(grouped bool) (*[]MemoryMapsStat, error) { return p.MemoryMapsWithContext(context.Background(), grouped) } diff --git a/github.com/shirou/gopsutil/process/process_freebsd.go b/github.com/shirou/gopsutil/process/process_freebsd.go index fd98a8ca6d..0cf1699dd5 100644 --- a/github.com/shirou/gopsutil/process/process_freebsd.go +++ b/github.com/shirou/gopsutil/process/process_freebsd.go @@ -7,6 +7,7 @@ import ( "context" "encoding/binary" "os/exec" + "path/filepath" "strconv" "strings" @@ -23,11 +24,7 @@ type MemoryInfoExStat struct { type MemoryMapsStat struct { } -func Pids() ([]int32, error) { - return PidsWithContext(context.Background()) -} - -func PidsWithContext(ctx context.Context) ([]int32, error) { +func pidsWithContext(ctx context.Context) ([]int32, error) { var ret []int32 procs, err := Processes() if err != nil { @@ -62,8 +59,24 @@ func (p *Process) NameWithContext(ctx context.Context) (string, error) { if err != nil { return "", err } + name := common.IntToString(k.Comm[:]) + + if len(name) >= 15 { + cmdlineSlice, err := p.CmdlineSliceWithContext(ctx) + if err != nil { + return "", err + } + if len(cmdlineSlice) > 0 { + extendedName := filepath.Base(cmdlineSlice[0]) + if strings.HasPrefix(extendedName, p.name) { + name = extendedName + } else { + name = cmdlineSlice[0] + } + } + } - return common.IntToString(k.Comm[:]), nil + return name, nil } func (p *Process) Tgid() (int32, error) { return 0, common.ErrNotImplementedError @@ -120,11 +133,8 @@ func (p *Process) CmdlineSliceWithContext(ctx context.Context) ([]string, error) return strParts, nil } -func (p *Process) CreateTime() (int64, error) { - return p.CreateTimeWithContext(context.Background()) -} -func (p *Process) CreateTimeWithContext(ctx context.Context) (int64, error) { +func (p *Process) createTimeWithContext(ctx context.Context) (int64, error) { return 0, common.ErrNotImplementedError } func (p *Process) Cwd() (string, error) { @@ -371,6 +381,14 @@ func (p *Process) MemoryInfoExWithContext(ctx context.Context) (*MemoryInfoExSta return nil, common.ErrNotImplementedError } +func (p *Process) PageFaults() (*PageFaultsStat, error) { + return p.PageFaultsWithContext(context.Background()) +} + +func (p *Process) PageFaultsWithContext(ctx context.Context) (*PageFaultsStat, error) { + return nil, common.ErrNotImplementedError +} + func (p *Process) Children() ([]*Process, error) { return p.ChildrenWithContext(context.Background()) } @@ -407,6 +425,15 @@ func (p *Process) ConnectionsWithContext(ctx context.Context) ([]net.ConnectionS return nil, common.ErrNotImplementedError } +// Connections returns a slice of net.ConnectionStat used by the process at most `max` +func (p *Process) ConnectionsMax(max int) ([]net.ConnectionStat, error) { + return p.ConnectionsMaxWithContext(context.Background(), max) +} + +func (p *Process) ConnectionsMaxWithContext(ctx context.Context, max int) ([]net.ConnectionStat, error) { + return []net.ConnectionStat{}, common.ErrNotImplementedError +} + func (p *Process) NetIOCounters(pernic bool) ([]net.IOCountersStat, error) { return p.NetIOCountersWithContext(context.Background(), pernic) } @@ -415,13 +442,6 @@ func (p *Process) NetIOCountersWithContext(ctx context.Context, pernic bool) ([] return nil, common.ErrNotImplementedError } -func (p *Process) IsRunning() (bool, error) { - return p.IsRunningWithContext(context.Background()) -} - -func (p *Process) IsRunningWithContext(ctx context.Context) (bool, error) { - return true, common.ErrNotImplementedError -} func (p *Process) MemoryMaps(grouped bool) (*[]MemoryMapsStat, error) { return p.MemoryMapsWithContext(context.Background(), grouped) } @@ -493,9 +513,3 @@ func (p *Process) getKProcWithContext(ctx context.Context) (*KinfoProc, error) { } return &k, nil } - -func NewProcess(pid int32) (*Process, error) { - p := &Process{Pid: pid} - - return p, nil -} diff --git a/github.com/shirou/gopsutil/process/process_freebsd_arm64.go b/github.com/shirou/gopsutil/process/process_freebsd_arm64.go new file mode 100644 index 0000000000..99781d1a2f --- /dev/null +++ b/github.com/shirou/gopsutil/process/process_freebsd_arm64.go @@ -0,0 +1,201 @@ +// +build freebsd +// +build arm64 +// Code generated by cmd/cgo -godefs; DO NOT EDIT. +// cgo -godefs process/types_freebsd.go + +package process + +const ( + CTLKern = 1 + KernProc = 14 + KernProcPID = 1 + KernProcProc = 8 + KernProcPathname = 12 + KernProcArgs = 7 +) + +const ( + sizeofPtr = 0x8 + sizeofShort = 0x2 + sizeofInt = 0x4 + sizeofLong = 0x8 + sizeofLongLong = 0x8 +) + +const ( + sizeOfKinfoVmentry = 0x488 + sizeOfKinfoProc = 0x440 +) + +const ( + SIDL = 1 + SRUN = 2 + SSLEEP = 3 + SSTOP = 4 + SZOMB = 5 + SWAIT = 6 + SLOCK = 7 +) + +type ( + _C_short int16 + _C_int int32 + _C_long int64 + _C_long_long int64 +) + +type Timespec struct { + Sec int64 + Nsec int64 +} + +type Timeval struct { + Sec int64 + Usec int64 +} + +type Rusage struct { + Utime Timeval + Stime Timeval + Maxrss int64 + Ixrss int64 + Idrss int64 + Isrss int64 + Minflt int64 + Majflt int64 + Nswap int64 + Inblock int64 + Oublock int64 + Msgsnd int64 + Msgrcv int64 + Nsignals int64 + Nvcsw int64 + Nivcsw int64 +} + +type Rlimit struct { + Cur int64 + Max int64 +} + +type KinfoProc struct { + Structsize int32 + Layout int32 + Args *int64 /* pargs */ + Paddr *int64 /* proc */ + Addr *int64 /* user */ + Tracep *int64 /* vnode */ + Textvp *int64 /* vnode */ + Fd *int64 /* filedesc */ + Vmspace *int64 /* vmspace */ + Wchan *byte + Pid int32 + Ppid int32 + Pgid int32 + Tpgid int32 + Sid int32 + Tsid int32 + Jobc int16 + Spare_short1 int16 + Tdev_freebsd11 uint32 + Siglist [16]byte /* sigset */ + Sigmask [16]byte /* sigset */ + Sigignore [16]byte /* sigset */ + Sigcatch [16]byte /* sigset */ + Uid uint32 + Ruid uint32 + Svuid uint32 + Rgid uint32 + Svgid uint32 + Ngroups int16 + Spare_short2 int16 + Groups [16]uint32 + Size uint64 + Rssize int64 + Swrss int64 + Tsize int64 + Dsize int64 + Ssize int64 + Xstat uint16 + Acflag uint16 + Pctcpu uint32 + Estcpu uint32 + Slptime uint32 + Swtime uint32 + Cow uint32 + Runtime uint64 + Start Timeval + Childtime Timeval + Flag int64 + Kiflag int64 + Traceflag int32 + Stat uint8 + Nice int8 + Lock uint8 + Rqindex uint8 + Oncpu_old uint8 + Lastcpu_old uint8 + Tdname [17]uint8 + Wmesg [9]uint8 + Login [18]uint8 + Lockname [9]uint8 + Comm [20]int8 + Emul [17]uint8 + Loginclass [18]uint8 + Moretdname [4]uint8 + Sparestrings [46]uint8 + Spareints [2]int32 + Tdev uint64 + Oncpu int32 + Lastcpu int32 + Tracer int32 + Flag2 int32 + Fibnum int32 + Cr_flags uint32 + Jid int32 + Numthreads int32 + Tid int32 + Pri Priority + Rusage Rusage + Rusage_ch Rusage + Pcb *int64 /* pcb */ + Kstack *byte + Udata *byte + Tdaddr *int64 /* thread */ + Spareptrs [6]*byte + Sparelongs [12]int64 + Sflag int64 + Tdflags int64 +} + +type Priority struct { + Class uint8 + Level uint8 + Native uint8 + User uint8 +} + +type KinfoVmentry struct { + Structsize int32 + Type int32 + Start uint64 + End uint64 + Offset uint64 + Vn_fileid uint64 + Vn_fsid_freebsd11 uint32 + Flags int32 + Resident int32 + Private_resident int32 + Protection int32 + Ref_count int32 + Shadow_count int32 + Vn_type int32 + Vn_size uint64 + Vn_rdev_freebsd11 uint32 + Vn_mode uint16 + Status uint16 + Vn_fsid uint64 + Vn_rdev uint64 + X_kve_ispare [8]int32 + Path [1024]uint8 +} diff --git a/github.com/shirou/gopsutil/process/process_linux.go b/github.com/shirou/gopsutil/process/process_linux.go index ecdd7b48bb..afd5e28e86 100644 --- a/github.com/shirou/gopsutil/process/process_linux.go +++ b/github.com/shirou/gopsutil/process/process_linux.go @@ -16,7 +16,6 @@ import ( "strings" "github.com/shirou/gopsutil/cpu" - "github.com/shirou/gopsutil/host" "github.com/shirou/gopsutil/internal/common" "github.com/shirou/gopsutil/net" "golang.org/x/sys/unix" @@ -65,26 +64,13 @@ func (m MemoryMapsStat) String() string { return string(s) } -// NewProcess creates a new Process instance, it only stores the pid and -// checks that the process exists. Other method on Process can be used -// to get more information about the process. An error will be returned -// if the process does not exist. -func NewProcess(pid int32) (*Process, error) { - p := &Process{ - Pid: int32(pid), - } - file, err := os.Open(common.HostProc(strconv.Itoa(int(p.Pid)))) - defer file.Close() - return p, err -} - // Ppid returns Parent Process ID of the process. func (p *Process) Ppid() (int32, error) { return p.PpidWithContext(context.Background()) } func (p *Process) PpidWithContext(ctx context.Context) (int32, error) { - _, ppid, _, _, _, _, err := p.fillFromStat() + _, ppid, _, _, _, _, _, err := p.fillFromStatWithContext(ctx) if err != nil { return -1, err } @@ -98,7 +84,7 @@ func (p *Process) Name() (string, error) { func (p *Process) NameWithContext(ctx context.Context) (string, error) { if p.name == "" { - if err := p.fillFromStatus(); err != nil { + if err := p.fillFromStatusWithContext(ctx); err != nil { return "", err } } @@ -108,7 +94,7 @@ func (p *Process) NameWithContext(ctx context.Context) (string, error) { // Tgid returns tgid, a Linux-synonym for user-space Pid func (p *Process) Tgid() (int32, error) { if p.tgid == 0 { - if err := p.fillFromStatus(); err != nil { + if err := p.fillFromStatusWithContext(context.Background()); err != nil { return 0, err } } @@ -121,7 +107,7 @@ func (p *Process) Exe() (string, error) { } func (p *Process) ExeWithContext(ctx context.Context) (string, error) { - return p.fillFromExe() + return p.fillFromExeWithContext(ctx) } // Cmdline returns the command line arguments of the process as a string with @@ -131,7 +117,7 @@ func (p *Process) Cmdline() (string, error) { } func (p *Process) CmdlineWithContext(ctx context.Context) (string, error) { - return p.fillFromCmdline() + return p.fillFromCmdlineWithContext(ctx) } // CmdlineSlice returns the command line arguments of the process as a slice with each @@ -141,16 +127,11 @@ func (p *Process) CmdlineSlice() ([]string, error) { } func (p *Process) CmdlineSliceWithContext(ctx context.Context) ([]string, error) { - return p.fillSliceFromCmdline() -} - -// CreateTime returns created time of the process in milliseconds since the epoch, in UTC. -func (p *Process) CreateTime() (int64, error) { - return p.CreateTimeWithContext(context.Background()) + return p.fillSliceFromCmdlineWithContext(ctx) } -func (p *Process) CreateTimeWithContext(ctx context.Context) (int64, error) { - _, _, _, createTime, _, _, err := p.fillFromStat() +func (p *Process) createTimeWithContext(ctx context.Context) (int64, error) { + _, _, _, createTime, _, _, _, err := p.fillFromStatWithContext(ctx) if err != nil { return 0, err } @@ -163,7 +144,7 @@ func (p *Process) Cwd() (string, error) { } func (p *Process) CwdWithContext(ctx context.Context) (string, error) { - return p.fillFromCwd() + return p.fillFromCwdWithContext(ctx) } // Parent returns parent Process of the process. @@ -172,7 +153,7 @@ func (p *Process) Parent() (*Process, error) { } func (p *Process) ParentWithContext(ctx context.Context) (*Process, error) { - err := p.fillFromStatus() + err := p.fillFromStatusWithContext(ctx) if err != nil { return nil, err } @@ -186,13 +167,13 @@ func (p *Process) ParentWithContext(ctx context.Context) (*Process, error) { // Return value could be one of these. // R: Running S: Sleep T: Stop I: Idle // Z: Zombie W: Wait L: Lock -// The charactor is same within all supported platforms. +// The character is same within all supported platforms. func (p *Process) Status() (string, error) { return p.StatusWithContext(context.Background()) } func (p *Process) StatusWithContext(ctx context.Context) (string, error) { - err := p.fillFromStatus() + err := p.fillFromStatusWithContext(ctx) if err != nil { return "", err } @@ -227,7 +208,7 @@ func (p *Process) Uids() ([]int32, error) { } func (p *Process) UidsWithContext(ctx context.Context) ([]int32, error) { - err := p.fillFromStatus() + err := p.fillFromStatusWithContext(ctx) if err != nil { return []int32{}, err } @@ -240,7 +221,7 @@ func (p *Process) Gids() ([]int32, error) { } func (p *Process) GidsWithContext(ctx context.Context) ([]int32, error) { - err := p.fillFromStatus() + err := p.fillFromStatusWithContext(ctx) if err != nil { return []int32{}, err } @@ -253,7 +234,7 @@ func (p *Process) Terminal() (string, error) { } func (p *Process) TerminalWithContext(ctx context.Context) (string, error) { - t, _, _, _, _, _, err := p.fillFromStat() + t, _, _, _, _, _, _, err := p.fillFromStatWithContext(ctx) if err != nil { return "", err } @@ -272,7 +253,7 @@ func (p *Process) Nice() (int32, error) { } func (p *Process) NiceWithContext(ctx context.Context) (int32, error) { - _, _, _, _, _, nice, err := p.fillFromStat() + _, _, _, _, _, nice, _, err := p.fillFromStatWithContext(ctx) if err != nil { return 0, err } @@ -305,16 +286,16 @@ func (p *Process) RlimitUsage(gatherUsed bool) ([]RlimitStat, error) { } func (p *Process) RlimitUsageWithContext(ctx context.Context, gatherUsed bool) ([]RlimitStat, error) { - rlimits, err := p.fillFromLimits() + rlimits, err := p.fillFromLimitsWithContext(ctx) if !gatherUsed || err != nil { return rlimits, err } - _, _, _, _, rtprio, nice, err := p.fillFromStat() + _, _, _, _, rtprio, nice, _, err := p.fillFromStatWithContext(ctx) if err != nil { return nil, err } - if err := p.fillFromStatus(); err != nil { + if err := p.fillFromStatusWithContext(ctx); err != nil { return nil, err } @@ -365,7 +346,7 @@ func (p *Process) IOCounters() (*IOCountersStat, error) { } func (p *Process) IOCountersWithContext(ctx context.Context) (*IOCountersStat, error) { - return p.fillFromIO() + return p.fillFromIOWithContext(ctx) } // NumCtxSwitches returns the number of the context switches of the process. @@ -374,7 +355,7 @@ func (p *Process) NumCtxSwitches() (*NumCtxSwitchesStat, error) { } func (p *Process) NumCtxSwitchesWithContext(ctx context.Context) (*NumCtxSwitchesStat, error) { - err := p.fillFromStatus() + err := p.fillFromStatusWithContext(ctx) if err != nil { return nil, err } @@ -387,7 +368,7 @@ func (p *Process) NumFDs() (int32, error) { } func (p *Process) NumFDsWithContext(ctx context.Context) (int32, error) { - _, fnames, err := p.fillFromfdList() + _, fnames, err := p.fillFromfdListWithContext(ctx) return int32(len(fnames)), err } @@ -397,7 +378,7 @@ func (p *Process) NumThreads() (int32, error) { } func (p *Process) NumThreadsWithContext(ctx context.Context) (int32, error) { - err := p.fillFromStatus() + err := p.fillFromStatusWithContext(ctx) if err != nil { return 0, err } @@ -418,7 +399,7 @@ func (p *Process) ThreadsWithContext(ctx context.Context) (map[int32]*cpu.TimesS } for _, tid := range tids { - _, _, cpuTimes, _, _, _, err := p.fillFromTIDStat(tid) + _, _, cpuTimes, _, _, _, _, err := p.fillFromTIDStatWithContext(ctx, tid) if err != nil { return nil, err } @@ -434,7 +415,7 @@ func (p *Process) Times() (*cpu.TimesStat, error) { } func (p *Process) TimesWithContext(ctx context.Context) (*cpu.TimesStat, error) { - _, _, cpuTimes, _, _, _, err := p.fillFromStat() + _, _, cpuTimes, _, _, _, _, err := p.fillFromStatWithContext(ctx) if err != nil { return nil, err } @@ -458,7 +439,7 @@ func (p *Process) MemoryInfo() (*MemoryInfoStat, error) { } func (p *Process) MemoryInfoWithContext(ctx context.Context) (*MemoryInfoStat, error) { - meminfo, _, err := p.fillFromStatm() + meminfo, _, err := p.fillFromStatmWithContext(ctx) if err != nil { return nil, err } @@ -471,13 +452,27 @@ func (p *Process) MemoryInfoEx() (*MemoryInfoExStat, error) { } func (p *Process) MemoryInfoExWithContext(ctx context.Context) (*MemoryInfoExStat, error) { - _, memInfoEx, err := p.fillFromStatm() + _, memInfoEx, err := p.fillFromStatmWithContext(ctx) if err != nil { return nil, err } return memInfoEx, nil } +// PageFaultsInfo returns the process's page fault counters +func (p *Process) PageFaults() (*PageFaultsStat, error) { + return p.PageFaultsWithContext(context.Background()) +} + +func (p *Process) PageFaultsWithContext(ctx context.Context) (*PageFaultsStat, error) { + _, _, _, _, _, _, pageFaults, err := p.fillFromStatWithContext(ctx) + if err != nil { + return nil, err + } + return pageFaults, nil + +} + // Children returns a slice of Process of the process. func (p *Process) Children() ([]*Process, error) { return p.ChildrenWithContext(context.Background()) @@ -509,7 +504,7 @@ func (p *Process) OpenFiles() ([]OpenFilesStat, error) { } func (p *Process) OpenFilesWithContext(ctx context.Context) ([]OpenFilesStat, error) { - _, ofs, err := p.fillFromfd() + _, ofs, err := p.fillFromfdWithContext(ctx) if err != nil { return nil, err } @@ -531,6 +526,15 @@ func (p *Process) ConnectionsWithContext(ctx context.Context) ([]net.ConnectionS return net.ConnectionsPid("all", p.Pid) } +// Connections returns a slice of net.ConnectionStat used by the process at most `max` +func (p *Process) ConnectionsMax(max int) ([]net.ConnectionStat, error) { + return p.ConnectionsMaxWithContext(context.Background(), max) +} + +func (p *Process) ConnectionsMaxWithContext(ctx context.Context, max int) ([]net.ConnectionStat, error) { + return net.ConnectionsPidMax("all", p.Pid, max) +} + // NetIOCounters returns NetIOCounters of the process. func (p *Process) NetIOCounters(pernic bool) ([]net.IOCountersStat, error) { return p.NetIOCountersWithContext(context.Background(), pernic) @@ -541,16 +545,6 @@ func (p *Process) NetIOCountersWithContext(ctx context.Context, pernic bool) ([] return net.IOCountersByFile(pernic, filename) } -// IsRunning returns whether the process is running or not. -// Not implemented yet. -func (p *Process) IsRunning() (bool, error) { - return p.IsRunningWithContext(context.Background()) -} - -func (p *Process) IsRunningWithContext(ctx context.Context) (bool, error) { - return true, common.ErrNotImplementedError -} - // MemoryMaps get memory maps from /proc/(pid)/smaps func (p *Process) MemoryMaps(grouped bool) (*[]MemoryMapsStat, error) { return p.MemoryMapsWithContext(context.Background(), grouped) @@ -559,6 +553,9 @@ func (p *Process) MemoryMaps(grouped bool) (*[]MemoryMapsStat, error) { func (p *Process) MemoryMapsWithContext(ctx context.Context, grouped bool) (*[]MemoryMapsStat, error) { pid := p.Pid var ret []MemoryMapsStat + if grouped { + ret = make([]MemoryMapsStat, 1) + } smapsPath := common.HostProc(strconv.Itoa(int(pid)), "smaps") contents, err := ioutil.ReadFile(smapsPath) if err != nil { @@ -621,7 +618,20 @@ func (p *Process) MemoryMapsWithContext(ctx context.Context, grouped bool) (*[]M if err != nil { return &ret, err } - ret = append(ret, g) + if grouped { + ret[0].Size += g.Size + ret[0].Rss += g.Rss + ret[0].Pss += g.Pss + ret[0].SharedClean += g.SharedClean + ret[0].SharedDirty += g.SharedDirty + ret[0].PrivateClean += g.PrivateClean + ret[0].PrivateDirty += g.PrivateDirty + ret[0].Referenced += g.Referenced + ret[0].Anonymous += g.Anonymous + ret[0].Swap += g.Swap + } else { + ret = append(ret, g) + } } // starts new block blocks = make([]string, 16) @@ -650,10 +660,6 @@ func limitToInt(val string) (int32, error) { } // Get num_fds from /proc/(pid)/limits -func (p *Process) fillFromLimits() ([]RlimitStat, error) { - return p.fillFromLimitsWithContext(context.Background()) -} - func (p *Process) fillFromLimitsWithContext(ctx context.Context) ([]RlimitStat, error) { pid := p.Pid limitsFile := common.HostProc(strconv.Itoa(int(pid)), "limits") @@ -747,10 +753,6 @@ func (p *Process) fillFromLimitsWithContext(ctx context.Context) ([]RlimitStat, } // Get list of /proc/(pid)/fd files -func (p *Process) fillFromfdList() (string, []string, error) { - return p.fillFromfdListWithContext(context.Background()) -} - func (p *Process) fillFromfdListWithContext(ctx context.Context) (string, []string, error) { pid := p.Pid statPath := common.HostProc(strconv.Itoa(int(pid)), "fd") @@ -764,12 +766,8 @@ func (p *Process) fillFromfdListWithContext(ctx context.Context) (string, []stri } // Get num_fds from /proc/(pid)/fd -func (p *Process) fillFromfd() (int32, []*OpenFilesStat, error) { - return p.fillFromfdWithContext(context.Background()) -} - func (p *Process) fillFromfdWithContext(ctx context.Context) (int32, []*OpenFilesStat, error) { - statPath, fnames, err := p.fillFromfdList() + statPath, fnames, err := p.fillFromfdListWithContext(ctx) if err != nil { return 0, nil, err } @@ -797,10 +795,6 @@ func (p *Process) fillFromfdWithContext(ctx context.Context) (int32, []*OpenFile } // Get cwd from /proc/(pid)/cwd -func (p *Process) fillFromCwd() (string, error) { - return p.fillFromCwdWithContext(context.Background()) -} - func (p *Process) fillFromCwdWithContext(ctx context.Context) (string, error) { pid := p.Pid cwdPath := common.HostProc(strconv.Itoa(int(pid)), "cwd") @@ -812,10 +806,6 @@ func (p *Process) fillFromCwdWithContext(ctx context.Context) (string, error) { } // Get exe from /proc/(pid)/exe -func (p *Process) fillFromExe() (string, error) { - return p.fillFromExeWithContext(context.Background()) -} - func (p *Process) fillFromExeWithContext(ctx context.Context) (string, error) { pid := p.Pid exePath := common.HostProc(strconv.Itoa(int(pid)), "exe") @@ -827,10 +817,6 @@ func (p *Process) fillFromExeWithContext(ctx context.Context) (string, error) { } // Get cmdline from /proc/(pid)/cmdline -func (p *Process) fillFromCmdline() (string, error) { - return p.fillFromCmdlineWithContext(context.Background()) -} - func (p *Process) fillFromCmdlineWithContext(ctx context.Context) (string, error) { pid := p.Pid cmdPath := common.HostProc(strconv.Itoa(int(pid)), "cmdline") @@ -848,10 +834,6 @@ func (p *Process) fillFromCmdlineWithContext(ctx context.Context) (string, error return strings.Join(ret, " "), nil } -func (p *Process) fillSliceFromCmdline() ([]string, error) { - return p.fillSliceFromCmdlineWithContext(context.Background()) -} - func (p *Process) fillSliceFromCmdlineWithContext(ctx context.Context) ([]string, error) { pid := p.Pid cmdPath := common.HostProc(strconv.Itoa(int(pid)), "cmdline") @@ -875,10 +857,6 @@ func (p *Process) fillSliceFromCmdlineWithContext(ctx context.Context) ([]string } // Get IO status from /proc/(pid)/io -func (p *Process) fillFromIO() (*IOCountersStat, error) { - return p.fillFromIOWithContext(context.Background()) -} - func (p *Process) fillFromIOWithContext(ctx context.Context) (*IOCountersStat, error) { pid := p.Pid ioPath := common.HostProc(strconv.Itoa(int(pid)), "io") @@ -918,10 +896,6 @@ func (p *Process) fillFromIOWithContext(ctx context.Context) (*IOCountersStat, e } // Get memory info from /proc/(pid)/statm -func (p *Process) fillFromStatm() (*MemoryInfoStat, *MemoryInfoExStat, error) { - return p.fillFromStatmWithContext(context.Background()) -} - func (p *Process) fillFromStatmWithContext(ctx context.Context) (*MemoryInfoStat, *MemoryInfoExStat, error) { pid := p.Pid memPath := common.HostProc(strconv.Itoa(int(pid)), "statm") @@ -974,10 +948,6 @@ func (p *Process) fillFromStatmWithContext(ctx context.Context) (*MemoryInfoStat } // Get various status from /proc/(pid)/status -func (p *Process) fillFromStatus() error { - return p.fillFromStatusWithContext(context.Background()) -} - func (p *Process) fillFromStatusWithContext(ctx context.Context) error { pid := p.Pid statPath := common.HostProc(strconv.Itoa(int(pid)), "status") @@ -1083,6 +1053,13 @@ func (p *Process) fillFromStatusWithContext(ctx context.Context) error { return err } p.memInfo.Swap = v * 1024 + case "VmHWM": + value := strings.Trim(value, " kB") // remove last "kB" + v, err := strconv.ParseUint(value, 10, 64) + if err != nil { + return err + } + p.memInfo.HWM = v * 1024 case "VmData": value := strings.Trim(value, " kB") // remove last "kB" v, err := strconv.ParseUint(value, 10, 64) @@ -1140,11 +1117,7 @@ func (p *Process) fillFromStatusWithContext(ctx context.Context) error { return nil } -func (p *Process) fillFromTIDStat(tid int32) (uint64, int32, *cpu.TimesStat, int64, uint32, int32, error) { - return p.fillFromTIDStatWithContext(context.Background(), tid) -} - -func (p *Process) fillFromTIDStatWithContext(ctx context.Context, tid int32) (uint64, int32, *cpu.TimesStat, int64, uint32, int32, error) { +func (p *Process) fillFromTIDStatWithContext(ctx context.Context, tid int32) (uint64, int32, *cpu.TimesStat, int64, uint32, int32, *PageFaultsStat, error) { pid := p.Pid var statPath string @@ -1156,7 +1129,7 @@ func (p *Process) fillFromTIDStatWithContext(ctx context.Context, tid int32) (ui contents, err := ioutil.ReadFile(statPath) if err != nil { - return 0, 0, nil, 0, 0, 0, err + return 0, 0, nil, 0, 0, 0, nil, err } fields := strings.Fields(string(contents)) @@ -1167,40 +1140,49 @@ func (p *Process) fillFromTIDStatWithContext(ctx context.Context, tid int32) (ui terminal, err := strconv.ParseUint(fields[i+5], 10, 64) if err != nil { - return 0, 0, nil, 0, 0, 0, err + return 0, 0, nil, 0, 0, 0, nil, err } ppid, err := strconv.ParseInt(fields[i+2], 10, 32) if err != nil { - return 0, 0, nil, 0, 0, 0, err + return 0, 0, nil, 0, 0, 0, nil, err } utime, err := strconv.ParseFloat(fields[i+12], 64) if err != nil { - return 0, 0, nil, 0, 0, 0, err + return 0, 0, nil, 0, 0, 0, nil, err } stime, err := strconv.ParseFloat(fields[i+13], 64) if err != nil { - return 0, 0, nil, 0, 0, 0, err + return 0, 0, nil, 0, 0, 0, nil, err + } + + // There is no such thing as iotime in stat file. As an approximation, we + // will use delayacct_blkio_ticks (aggregated block I/O delays, as per Linux + // docs). Note: I am assuming at least Linux 2.6.18 + iotime, err := strconv.ParseFloat(fields[i+40], 64) + if err != nil { + iotime = 0 // Ancient linux version, most likely } cpuTimes := &cpu.TimesStat{ CPU: "cpu", User: float64(utime / ClockTicks), System: float64(stime / ClockTicks), + Iowait: float64(iotime / ClockTicks), } - bootTime, _ := host.BootTime() + bootTime, _ := common.BootTimeWithContext(ctx) t, err := strconv.ParseUint(fields[i+20], 10, 64) if err != nil { - return 0, 0, nil, 0, 0, 0, err + return 0, 0, nil, 0, 0, 0, nil, err } ctime := (t / uint64(ClockTicks)) + uint64(bootTime) createTime := int64(ctime * 1000) rtpriority, err := strconv.ParseInt(fields[i+16], 10, 32) if err != nil { - return 0, 0, nil, 0, 0, 0, err + return 0, 0, nil, 0, 0, 0, nil, err } if rtpriority < 0 { rtpriority = rtpriority*-1 - 1 @@ -1213,23 +1195,38 @@ func (p *Process) fillFromTIDStatWithContext(ctx context.Context, tid int32) (ui snice, _ := unix.Getpriority(PrioProcess, int(pid)) nice := int32(snice) // FIXME: is this true? - return terminal, int32(ppid), cpuTimes, createTime, uint32(rtpriority), nice, nil -} + minFault, err := strconv.ParseUint(fields[i+8], 10, 64) + if err != nil { + return 0, 0, nil, 0, 0, 0, nil, err + } + cMinFault, err := strconv.ParseUint(fields[i+9], 10, 64) + if err != nil { + return 0, 0, nil, 0, 0, 0, nil, err + } + majFault, err := strconv.ParseUint(fields[i+10], 10, 64) + if err != nil { + return 0, 0, nil, 0, 0, 0, nil, err + } + cMajFault, err := strconv.ParseUint(fields[i+11], 10, 64) + if err != nil { + return 0, 0, nil, 0, 0, 0, nil, err + } -func (p *Process) fillFromStat() (uint64, int32, *cpu.TimesStat, int64, uint32, int32, error) { - return p.fillFromStatWithContext(context.Background()) -} + faults := &PageFaultsStat{ + MinorFaults: minFault, + MajorFaults: majFault, + ChildMinorFaults: cMinFault, + ChildMajorFaults: cMajFault, + } -func (p *Process) fillFromStatWithContext(ctx context.Context) (uint64, int32, *cpu.TimesStat, int64, uint32, int32, error) { - return p.fillFromTIDStat(-1) + return terminal, int32(ppid), cpuTimes, createTime, uint32(rtpriority), nice, faults, nil } -// Pids returns a slice of process ID list which are running now. -func Pids() ([]int32, error) { - return PidsWithContext(context.Background()) +func (p *Process) fillFromStatWithContext(ctx context.Context) (uint64, int32, *cpu.TimesStat, int64, uint32, int32, *PageFaultsStat, error) { + return p.fillFromTIDStatWithContext(ctx, -1) } -func PidsWithContext(ctx context.Context) ([]int32, error) { +func pidsWithContext(ctx context.Context) ([]int32, error) { return readPidsFromDir(common.HostProc()) } diff --git a/github.com/shirou/gopsutil/process/process_openbsd.go b/github.com/shirou/gopsutil/process/process_openbsd.go index f9e0a0867a..1f3c645be0 100644 --- a/github.com/shirou/gopsutil/process/process_openbsd.go +++ b/github.com/shirou/gopsutil/process/process_openbsd.go @@ -7,6 +7,9 @@ import ( "bytes" "context" "encoding/binary" + "os/exec" + "path/filepath" + "strconv" "strings" "unsafe" @@ -24,11 +27,7 @@ type MemoryInfoExStat struct { type MemoryMapsStat struct { } -func Pids() ([]int32, error) { - return PidsWithContext(context.Background()) -} - -func PidsWithContext(ctx context.Context) ([]int32, error) { +func pidsWithContext(ctx context.Context) ([]int32, error) { var ret []int32 procs, err := Processes() if err != nil { @@ -63,8 +62,24 @@ func (p *Process) NameWithContext(ctx context.Context) (string, error) { if err != nil { return "", err } + name := common.IntToString(k.Comm[:]) + + if len(name) >= 15 { + cmdlineSlice, err := p.CmdlineSliceWithContext(ctx) + if err != nil { + return "", err + } + if len(cmdlineSlice) > 0 { + extendedName := filepath.Base(cmdlineSlice[0]) + if strings.HasPrefix(extendedName, p.name) { + name = extendedName + } else { + name = cmdlineSlice[0] + } + } + } - return common.IntToString(k.Comm[:]), nil + return name, nil } func (p *Process) Tgid() (int32, error) { return 0, common.ErrNotImplementedError @@ -116,11 +131,7 @@ func (p *Process) CmdlineWithContext(ctx context.Context) (string, error) { return strings.Join(argv, " "), nil } -func (p *Process) CreateTime() (int64, error) { - return p.CreateTimeWithContext(context.Background()) -} - -func (p *Process) CreateTimeWithContext(ctx context.Context) (int64, error) { +func (p *Process) createTimeWithContext(ctx context.Context) (int64, error) { return 0, common.ErrNotImplementedError } func (p *Process) Cwd() (string, error) { @@ -357,6 +368,14 @@ func (p *Process) MemoryInfoExWithContext(ctx context.Context) (*MemoryInfoExSta return nil, common.ErrNotImplementedError } +func (p *Process) PageFaults() (*PageFaultsStat, error) { + return p.PageFaultsWithContext(context.Background()) +} + +func (p *Process) PageFaultsWithContext(ctx context.Context) (*PageFaultsStat, error) { + return nil, common.ErrNotImplementedError +} + func (p *Process) Children() ([]*Process, error) { return p.ChildrenWithContext(context.Background()) } @@ -393,6 +412,14 @@ func (p *Process) ConnectionsWithContext(ctx context.Context) ([]net.ConnectionS return nil, common.ErrNotImplementedError } +func (p *Process) ConnectionsMax(max int) ([]net.ConnectionStat, error) { + return p.ConnectionsMaxWithContext(context.Background(), max) +} + +func (p *Process) ConnectionsMaxWithContext(ctx context.Context, max int) ([]net.ConnectionStat, error) { + return []net.ConnectionStat{}, common.ErrNotImplementedError +} + func (p *Process) NetIOCounters(pernic bool) ([]net.IOCountersStat, error) { return p.NetIOCountersWithContext(context.Background(), pernic) } @@ -401,13 +428,6 @@ func (p *Process) NetIOCountersWithContext(ctx context.Context, pernic bool) ([] return nil, common.ErrNotImplementedError } -func (p *Process) IsRunning() (bool, error) { - return p.IsRunningWithContext(context.Background()) -} - -func (p *Process) IsRunningWithContext(ctx context.Context) (bool, error) { - return true, common.ErrNotImplementedError -} func (p *Process) MemoryMaps(grouped bool) (*[]MemoryMapsStat, error) { return p.MemoryMapsWithContext(context.Background(), grouped) } @@ -478,12 +498,6 @@ func (p *Process) getKProcWithContext(ctx context.Context) (*KinfoProc, error) { return &k, nil } -func NewProcess(pid int32) (*Process, error) { - p := &Process{Pid: pid} - - return p, nil -} - func CallKernProcSyscall(op int32, arg int32) ([]byte, uint64, error) { return CallKernProcSyscallWithContext(context.Background(), op, arg) } diff --git a/github.com/shirou/gopsutil/process/process_posix.go b/github.com/shirou/gopsutil/process/process_posix.go index 8ffb6b797d..109239a577 100644 --- a/github.com/shirou/gopsutil/process/process_posix.go +++ b/github.com/shirou/gopsutil/process/process_posix.go @@ -4,6 +4,7 @@ package process import ( "context" + "fmt" "os" "os/user" "path/filepath" @@ -11,6 +12,7 @@ import ( "strings" "syscall" + "github.com/shirou/gopsutil/internal/common" "golang.org/x/sys/unix" ) @@ -69,6 +71,49 @@ func getTerminalMap() (map[uint64]string, error) { return ret, nil } +func PidExistsWithContext(ctx context.Context, pid int32) (bool, error) { + if pid <= 0 { + return false, fmt.Errorf("invalid pid %v", pid) + } + proc, err := os.FindProcess(int(pid)) + if err != nil { + return false, err + } + + if _, err := os.Stat(common.HostProc()); err == nil { //Means that proc filesystem exist + // Checking PID existence based on existence of //proc/ folder + // This covers the case when running inside container with a different process namespace (by default) + + _, err := os.Stat(common.HostProc(strconv.Itoa(int(pid)))) + if os.IsNotExist(err) { + return false, nil + } + return err == nil, err + } + + //'/proc' filesystem is not exist, checking of PID existence is done via signalling the process + //Make sense only if we run in the same process namespace + err = proc.Signal(syscall.Signal(0)) + if err == nil { + return true, nil + } + if err.Error() == "os: process already finished" { + return false, nil + } + errno, ok := err.(syscall.Errno) + if !ok { + return false, err + } + switch errno { + case syscall.ESRCH: + return false, nil + case syscall.EPERM: + return true, nil + } + + return false, err +} + // SendSignal sends a unix.Signal to the process. // Currently, SIGSTOP, SIGCONT, SIGTERM and SIGKILL are supported. func (p *Process) SendSignal(sig syscall.Signal) error { diff --git a/github.com/shirou/gopsutil/process/process_windows.go b/github.com/shirou/gopsutil/process/process_windows.go index 48a12133cc..cdce609fba 100644 --- a/github.com/shirou/gopsutil/process/process_windows.go +++ b/github.com/shirou/gopsutil/process/process_windows.go @@ -4,33 +4,34 @@ package process import ( "context" + "errors" "fmt" "os" "strings" "syscall" - "time" "unsafe" - "github.com/StackExchange/wmi" cpu "github.com/shirou/gopsutil/cpu" "github.com/shirou/gopsutil/internal/common" net "github.com/shirou/gopsutil/net" - "github.com/shirou/w32" "golang.org/x/sys/windows" ) -const ( - NoMoreFiles = 0x12 - MaxPathLength = 260 -) - var ( - modpsapi = windows.NewLazySystemDLL("psapi.dll") - procGetProcessMemoryInfo = modpsapi.NewProc("GetProcessMemoryInfo") + modpsapi = windows.NewLazySystemDLL("psapi.dll") + procGetProcessMemoryInfo = modpsapi.NewProc("GetProcessMemoryInfo") + procGetProcessImageFileNameW = modpsapi.NewProc("GetProcessImageFileNameW") advapi32 = windows.NewLazySystemDLL("advapi32.dll") procLookupPrivilegeValue = advapi32.NewProc("LookupPrivilegeValueW") procAdjustTokenPrivileges = advapi32.NewProc("AdjustTokenPrivileges") + + procQueryFullProcessImageNameW = common.Modkernel32.NewProc("QueryFullProcessImageNameW") + procGetPriorityClass = common.Modkernel32.NewProc("GetPriorityClass") + procGetProcessIoCounters = common.Modkernel32.NewProc("GetProcessIoCounters") + procGetNativeSystemInfo = common.Modkernel32.NewProc("GetNativeSystemInfo") + + processorArchitecture uint ) type SystemProcessInformation struct { @@ -48,6 +49,28 @@ type SystemProcessInformation struct { Reserved6 [6]uint64 } +type systemProcessorInformation struct { + ProcessorArchitecture uint16 + ProcessorLevel uint16 + ProcessorRevision uint16 + Reserved uint16 + ProcessorFeatureBits uint16 +} + +type systemInfo struct { + wProcessorArchitecture uint16 + wReserved uint16 + dwPageSize uint32 + lpMinimumApplicationAddress uintptr + lpMaximumApplicationAddress uintptr + dwActiveProcessorMask uintptr + dwNumberOfProcessors uint32 + dwProcessorType uint32 + dwAllocationGranularity uint32 + wProcessorLevel uint16 + wProcessorRevision uint16 +} + // Memory_info_ex is different between OSes type MemoryInfoExStat struct { } @@ -55,43 +78,33 @@ type MemoryInfoExStat struct { type MemoryMapsStat struct { } -type Win32_Process struct { - Name string - ExecutablePath *string - CommandLine *string - Priority uint32 - CreationDate *time.Time - ProcessID uint32 - ThreadCount uint32 - Status *string - ReadOperationCount uint64 - ReadTransferCount uint64 - WriteOperationCount uint64 - WriteTransferCount uint64 - CSCreationClassName string - CSName string - Caption *string - CreationClassName string - Description *string - ExecutionState *uint16 - HandleCount uint32 - KernelModeTime uint64 - MaximumWorkingSetSize *uint32 - MinimumWorkingSetSize *uint32 - OSCreationClassName string - OSName string - OtherOperationCount uint64 - OtherTransferCount uint64 - PageFaults uint32 - PageFileUsage uint32 - ParentProcessID uint32 - PeakPageFileUsage uint32 - PeakVirtualSize uint64 - PeakWorkingSetSize uint32 - PrivatePageCount uint64 - TerminationDate *time.Time - UserModeTime uint64 - WorkingSetSize uint64 +// ioCounters is an equivalent representation of IO_COUNTERS in the Windows API. +// https://docs.microsoft.com/windows/win32/api/winnt/ns-winnt-io_counters +type ioCounters struct { + ReadOperationCount uint64 + WriteOperationCount uint64 + OtherOperationCount uint64 + ReadTransferCount uint64 + WriteTransferCount uint64 + OtherTransferCount uint64 +} + +type processBasicInformation32 struct { + Reserved1 uint32 + PebBaseAddress uint32 + Reserved2 uint32 + Reserved3 uint32 + UniqueProcessId uint32 + Reserved4 uint32 +} + +type processBasicInformation64 struct { + Reserved1 uint64 + PebBaseAddress uint64 + Reserved2 uint64 + Reserved3 uint64 + UniqueProcessId uint64 + Reserved4 uint64 } type winLUID struct { @@ -115,7 +128,10 @@ type winLong int32 type winDWord uint32 func init() { - wmi.DefaultClient.AllowMissingFields = true + var systemInfo systemInfo + + procGetNativeSystemInfo.Call(uintptr(unsafe.Pointer(&systemInfo))) + processorArchitecture = uint(systemInfo.wProcessorArchitecture) // enable SeDebugPrivilege https://github.com/midstar/proci/blob/6ec79f57b90ba3d9efa2a7b16ef9c9369d4be875/proci_windows.go#L80-L119 handle, err := syscall.GetCurrentProcess() @@ -151,11 +167,7 @@ func init() { 0) } -func Pids() ([]int32, error) { - return PidsWithContext(context.Background()) -} - -func PidsWithContext(ctx context.Context) ([]int32, error) { +func pidsWithContext(ctx context.Context) ([]int32, error) { // inspired by https://gist.github.com/henkman/3083408 // and https://github.com/giampaolo/psutil/blob/1c3a15f637521ba5c0031283da39c733fda53e4c/psutil/arch/windows/process_info.c#L315-L329 var ret []int32 @@ -165,8 +177,8 @@ func PidsWithContext(ctx context.Context) ([]int32, error) { for { ps := make([]uint32, psSize) - if !w32.EnumProcesses(ps, uint32(len(ps)), &read) { - return nil, fmt.Errorf("could not get w32.EnumProcesses") + if err := windows.EnumProcesses(ps, &read); err != nil { + return nil, err } if uint32(len(ps)) == read { // ps buffer was too small to host every results, retry with a bigger one psSize += 1024 @@ -181,6 +193,44 @@ func PidsWithContext(ctx context.Context) ([]int32, error) { } +func PidExistsWithContext(ctx context.Context, pid int32) (bool, error) { + if pid == 0 { // special case for pid 0 System Idle Process + return true, nil + } + if pid < 0 { + return false, fmt.Errorf("invalid pid %v", pid) + } + if pid%4 != 0 { + // OpenProcess will succeed even on non-existing pid here https://devblogs.microsoft.com/oldnewthing/20080606-00/?p=22043 + // so we list every pid just to be sure and be future-proof + pids, err := PidsWithContext(ctx) + if err != nil { + return false, err + } + for _, i := range pids { + if i == pid { + return true, err + } + } + return false, err + } + const STILL_ACTIVE = 259 // https://docs.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-getexitcodeprocess + h, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid)) + if err == windows.ERROR_ACCESS_DENIED { + return true, nil + } + if err == windows.ERROR_INVALID_PARAMETER { + return false, nil + } + if err != nil { + return false, err + } + defer syscall.CloseHandle(syscall.Handle(h)) + var exitCode uint32 + err = windows.GetExitCodeProcess(h, &exitCode) + return exitCode == STILL_ACTIVE, err +} + func (p *Process) Ppid() (int32, error) { return p.PpidWithContext(context.Background()) } @@ -193,26 +243,6 @@ func (p *Process) PpidWithContext(ctx context.Context) (int32, error) { return ppid, nil } -func GetWin32Proc(pid int32) ([]Win32_Process, error) { - return GetWin32ProcWithContext(context.Background(), pid) -} - -func GetWin32ProcWithContext(ctx context.Context, pid int32) ([]Win32_Process, error) { - var dst []Win32_Process - query := fmt.Sprintf("WHERE ProcessId = %d", pid) - q := wmi.CreateQuery(&dst, query) - err := common.WMIQueryWithContext(ctx, q, &dst) - if err != nil { - return []Win32_Process{}, fmt.Errorf("could not get win32Proc: %s", err) - } - - if len(dst) == 0 { - return []Win32_Process{}, fmt.Errorf("could not get win32Proc: empty") - } - - return dst, nil -} - func (p *Process) Name() (string, error) { return p.NameWithContext(context.Background()) } @@ -234,36 +264,42 @@ func (p *Process) Exe() (string, error) { } func (p *Process) ExeWithContext(ctx context.Context) (string, error) { - if p.Pid != 0 { // 0 or null is the current process for CreateToolhelp32Snapshot - snap := w32.CreateToolhelp32Snapshot(w32.TH32CS_SNAPMODULE|w32.TH32CS_SNAPMODULE32, uint32(p.Pid)) - if snap != 0 { // don't report errors here, fallback to WMI instead - defer w32.CloseHandle(snap) - var me32 w32.MODULEENTRY32 - me32.Size = uint32(unsafe.Sizeof(me32)) - - if w32.Module32First(snap, &me32) { - szexepath := windows.UTF16ToString(me32.SzExePath[:]) - return szexepath, nil - } + c, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(p.Pid)) + if err != nil { + return "", err + } + defer windows.CloseHandle(c) + buf := make([]uint16, syscall.MAX_LONG_PATH) + size := uint32(syscall.MAX_LONG_PATH) + if err := procQueryFullProcessImageNameW.Find(); err == nil { // Vista+ + ret, _, err := procQueryFullProcessImageNameW.Call( + uintptr(c), + uintptr(0), + uintptr(unsafe.Pointer(&buf[0])), + uintptr(unsafe.Pointer(&size))) + if ret == 0 { + return "", err } + return windows.UTF16ToString(buf[:]), nil } - dst, err := GetWin32ProcWithContext(ctx, p.Pid) - if err != nil { - return "", fmt.Errorf("could not get ExecutablePath: %s", err) + // XP fallback + ret, _, err := procGetProcessImageFileNameW.Call(uintptr(c), uintptr(unsafe.Pointer(&buf[0])), uintptr(size)) + if ret == 0 { + return "", err } - return *dst[0].ExecutablePath, nil + return common.ConvertDOSPath(windows.UTF16ToString(buf[:])), nil } func (p *Process) Cmdline() (string, error) { return p.CmdlineWithContext(context.Background()) } -func (p *Process) CmdlineWithContext(ctx context.Context) (string, error) { - dst, err := GetWin32ProcWithContext(ctx, p.Pid) +func (p *Process) CmdlineWithContext(_ context.Context) (string, error) { + cmdline, err := getProcessCommandLine(p.Pid) if err != nil { return "", fmt.Errorf("could not get CommandLine: %s", err) } - return *dst[0].CommandLine, nil + return cmdline, nil } // CmdlineSlice returns the command line arguments of the process as a slice with each @@ -281,11 +317,7 @@ func (p *Process) CmdlineSliceWithContext(ctx context.Context) ([]string, error) return strings.Split(cmdline, " "), nil } -func (p *Process) CreateTime() (int64, error) { - return p.CreateTimeWithContext(context.Background()) -} - -func (p *Process) CreateTimeWithContext(ctx context.Context) (int64, error) { +func (p *Process) createTimeWithContext(ctx context.Context) (int64, error) { ru, err := getRusage(p.Pid) if err != nil { return 0, fmt.Errorf("could not get CreationDate: %s", err) @@ -335,15 +367,14 @@ func (p *Process) Username() (string, error) { func (p *Process) UsernameWithContext(ctx context.Context) (string, error) { pid := p.Pid - // 0x1000 is PROCESS_QUERY_LIMITED_INFORMATION - c, err := syscall.OpenProcess(0x1000, false, uint32(pid)) + c, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid)) if err != nil { return "", err } - defer syscall.CloseHandle(c) + defer windows.CloseHandle(c) var token syscall.Token - err = syscall.OpenProcessToken(c, syscall.TOKEN_QUERY, &token) + err = syscall.OpenProcessToken(syscall.Handle(c), syscall.TOKEN_QUERY, &token) if err != nil { return "", err } @@ -382,17 +413,38 @@ func (p *Process) TerminalWithContext(ctx context.Context) (string, error) { return "", common.ErrNotImplementedError } +// priorityClasses maps a win32 priority class to its WMI equivalent Win32_Process.Priority +// https://docs.microsoft.com/en-us/windows/desktop/api/processthreadsapi/nf-processthreadsapi-getpriorityclass +// https://docs.microsoft.com/en-us/windows/desktop/cimwin32prov/win32-process +var priorityClasses = map[int]int32{ + 0x00008000: 10, // ABOVE_NORMAL_PRIORITY_CLASS + 0x00004000: 6, // BELOW_NORMAL_PRIORITY_CLASS + 0x00000080: 13, // HIGH_PRIORITY_CLASS + 0x00000040: 4, // IDLE_PRIORITY_CLASS + 0x00000020: 8, // NORMAL_PRIORITY_CLASS + 0x00000100: 24, // REALTIME_PRIORITY_CLASS +} + // Nice returns priority in Windows func (p *Process) Nice() (int32, error) { return p.NiceWithContext(context.Background()) } func (p *Process) NiceWithContext(ctx context.Context) (int32, error) { - dst, err := GetWin32ProcWithContext(ctx, p.Pid) + c, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(p.Pid)) if err != nil { - return 0, fmt.Errorf("could not get Priority: %s", err) + return 0, err + } + defer windows.CloseHandle(c) + ret, _, err := procGetPriorityClass.Call(uintptr(c)) + if ret == 0 { + return 0, err + } + priority, ok := priorityClasses[int(ret)] + if !ok { + return 0, fmt.Errorf("unknown priority class %v", ret) } - return int32(dst[0].Priority), nil + return priority, nil } func (p *Process) IOnice() (int32, error) { return p.IOniceWithContext(context.Background()) @@ -425,18 +477,24 @@ func (p *Process) IOCounters() (*IOCountersStat, error) { } func (p *Process) IOCountersWithContext(ctx context.Context) (*IOCountersStat, error) { - dst, err := GetWin32ProcWithContext(ctx, p.Pid) - if err != nil || len(dst) == 0 { - return nil, fmt.Errorf("could not get Win32Proc: %s", err) + c, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(p.Pid)) + if err != nil { + return nil, err } - ret := &IOCountersStat{ - ReadCount: uint64(dst[0].ReadOperationCount), - ReadBytes: uint64(dst[0].ReadTransferCount), - WriteCount: uint64(dst[0].WriteOperationCount), - WriteBytes: uint64(dst[0].WriteTransferCount), + defer windows.CloseHandle(c) + var ioCounters ioCounters + ret, _, err := procGetProcessIoCounters.Call(uintptr(c), uintptr(unsafe.Pointer(&ioCounters))) + if ret == 0 { + return nil, err + } + stats := &IOCountersStat{ + ReadCount: ioCounters.ReadOperationCount, + ReadBytes: ioCounters.ReadTransferCount, + WriteCount: ioCounters.WriteOperationCount, + WriteBytes: ioCounters.WriteTransferCount, } - return ret, nil + return stats, nil } func (p *Process) NumCtxSwitches() (*NumCtxSwitchesStat, error) { return p.NumCtxSwitchesWithContext(context.Background()) @@ -457,11 +515,11 @@ func (p *Process) NumThreads() (int32, error) { } func (p *Process) NumThreadsWithContext(ctx context.Context) (int32, error) { - dst, err := GetWin32ProcWithContext(ctx, p.Pid) + _, ret, _, err := getFromSnapProcess(p.Pid) if err != nil { - return 0, fmt.Errorf("could not get ThreadCount: %s", err) + return 0, err } - return int32(dst[0].ThreadCount), nil + return ret, nil } func (p *Process) Threads() (map[int32]*cpu.TimesStat, error) { return p.ThreadsWithContext(context.Background()) @@ -530,37 +588,40 @@ func (p *Process) MemoryInfoExWithContext(ctx context.Context) (*MemoryInfoExSta return nil, common.ErrNotImplementedError } +func (p *Process) PageFaults() (*PageFaultsStat, error) { + return p.PageFaultsWithContext(context.Background()) +} + +func (p *Process) PageFaultsWithContext(ctx context.Context) (*PageFaultsStat, error) { + return nil, common.ErrNotImplementedError +} + func (p *Process) Children() ([]*Process, error) { return p.ChildrenWithContext(context.Background()) } func (p *Process) ChildrenWithContext(ctx context.Context) ([]*Process, error) { out := []*Process{} - snap := w32.CreateToolhelp32Snapshot(w32.TH32CS_SNAPPROCESS, uint32(0)) - if snap == 0 { - return out, windows.GetLastError() - } - defer w32.CloseHandle(snap) - var pe32 w32.PROCESSENTRY32 - pe32.DwSize = uint32(unsafe.Sizeof(pe32)) - if w32.Process32First(snap, &pe32) == false { - return out, windows.GetLastError() + snap, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPPROCESS, uint32(0)) + if err != nil { + return out, err } - - if pe32.Th32ParentProcessID == uint32(p.Pid) { - p, err := NewProcess(int32(pe32.Th32ProcessID)) - if err == nil { - out = append(out, p) - } + defer windows.CloseHandle(snap) + var pe32 windows.ProcessEntry32 + pe32.Size = uint32(unsafe.Sizeof(pe32)) + if err := windows.Process32First(snap, &pe32); err != nil { + return out, err } - - for w32.Process32Next(snap, &pe32) { - if pe32.Th32ParentProcessID == uint32(p.Pid) { - p, err := NewProcess(int32(pe32.Th32ProcessID)) + for { + if pe32.ParentProcessID == uint32(p.Pid) { + p, err := NewProcess(int32(pe32.ProcessID)) if err == nil { out = append(out, p) } } + if err = windows.Process32Next(snap, &pe32); err != nil { + break + } } return out, nil } @@ -578,23 +639,23 @@ func (p *Process) Connections() ([]net.ConnectionStat, error) { } func (p *Process) ConnectionsWithContext(ctx context.Context) ([]net.ConnectionStat, error) { - return nil, common.ErrNotImplementedError + return net.ConnectionsPidWithContext(ctx, "all", p.Pid) } -func (p *Process) NetIOCounters(pernic bool) ([]net.IOCountersStat, error) { - return p.NetIOCountersWithContext(context.Background(), pernic) +func (p *Process) ConnectionsMax(max int) ([]net.ConnectionStat, error) { + return p.ConnectionsMaxWithContext(context.Background(), max) } -func (p *Process) NetIOCountersWithContext(ctx context.Context, pernic bool) ([]net.IOCountersStat, error) { - return nil, common.ErrNotImplementedError +func (p *Process) ConnectionsMaxWithContext(ctx context.Context, max int) ([]net.ConnectionStat, error) { + return []net.ConnectionStat{}, common.ErrNotImplementedError } -func (p *Process) IsRunning() (bool, error) { - return p.IsRunningWithContext(context.Background()) +func (p *Process) NetIOCounters(pernic bool) ([]net.IOCountersStat, error) { + return p.NetIOCountersWithContext(context.Background(), pernic) } -func (p *Process) IsRunningWithContext(ctx context.Context) (bool, error) { - return true, common.ErrNotImplementedError +func (p *Process) NetIOCountersWithContext(ctx context.Context, pernic bool) ([]net.IOCountersStat, error) { + return nil, common.ErrNotImplementedError } func (p *Process) MemoryMaps(grouped bool) (*[]MemoryMapsStat, error) { @@ -606,12 +667,6 @@ func (p *Process) MemoryMapsWithContext(ctx context.Context, grouped bool) (*[]M return &ret, common.ErrNotImplementedError } -func NewProcess(pid int32) (*Process, error) { - p := &Process{Pid: pid} - - return p, nil -} - func (p *Process) SendSignal(sig windows.Signal) error { return p.SendSignalWithContext(context.Background(), sig) } @@ -640,16 +695,13 @@ func (p *Process) Terminate() error { } func (p *Process) TerminateWithContext(ctx context.Context) error { - // PROCESS_TERMINATE = 0x0001 - proc := w32.OpenProcess(0x0001, false, uint32(p.Pid)) - ret := w32.TerminateProcess(proc, 0) - w32.CloseHandle(proc) - - if ret == false { - return windows.GetLastError() - } else { - return nil + proc, err := windows.OpenProcess(windows.PROCESS_TERMINATE, false, uint32(p.Pid)) + if err != nil { + return err } + err = windows.TerminateProcess(proc, 0) + windows.CloseHandle(proc) + return err } func (p *Process) Kill() error { @@ -662,29 +714,26 @@ func (p *Process) KillWithContext(ctx context.Context) error { } func getFromSnapProcess(pid int32) (int32, int32, string, error) { - snap := w32.CreateToolhelp32Snapshot(w32.TH32CS_SNAPPROCESS, uint32(pid)) - if snap == 0 { - return 0, 0, "", windows.GetLastError() - } - defer w32.CloseHandle(snap) - var pe32 w32.PROCESSENTRY32 - pe32.DwSize = uint32(unsafe.Sizeof(pe32)) - if w32.Process32First(snap, &pe32) == false { - return 0, 0, "", windows.GetLastError() + snap, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPPROCESS, uint32(pid)) + if err != nil { + return 0, 0, "", err } - - if pe32.Th32ProcessID == uint32(pid) { - szexe := windows.UTF16ToString(pe32.SzExeFile[:]) - return int32(pe32.Th32ParentProcessID), int32(pe32.CntThreads), szexe, nil + defer windows.CloseHandle(snap) + var pe32 windows.ProcessEntry32 + pe32.Size = uint32(unsafe.Sizeof(pe32)) + if err = windows.Process32First(snap, &pe32); err != nil { + return 0, 0, "", err } - - for w32.Process32Next(snap, &pe32) { - if pe32.Th32ProcessID == uint32(pid) { - szexe := windows.UTF16ToString(pe32.SzExeFile[:]) - return int32(pe32.Th32ParentProcessID), int32(pe32.CntThreads), szexe, nil + for { + if pe32.ProcessID == uint32(pid) { + szexe := windows.UTF16ToString(pe32.ExeFile[:]) + return int32(pe32.ParentProcessID), int32(pe32.Threads), szexe, nil + } + if err = windows.Process32Next(snap, &pe32); err != nil { + break } } - return 0, 0, "", fmt.Errorf("Couldn't find pid: %d", pid) + return 0, 0, "", fmt.Errorf("couldn't find pid: %d", pid) } // Get processes @@ -711,28 +760,10 @@ func ProcessesWithContext(ctx context.Context) ([]*Process, error) { return out, nil } -func getProcInfo(pid int32) (*SystemProcessInformation, error) { - initialBufferSize := uint64(0x4000) - bufferSize := initialBufferSize - buffer := make([]byte, bufferSize) - - var sysProcInfo SystemProcessInformation - ret, _, _ := common.ProcNtQuerySystemInformation.Call( - uintptr(unsafe.Pointer(&sysProcInfo)), - uintptr(unsafe.Pointer(&buffer[0])), - uintptr(unsafe.Pointer(&bufferSize)), - uintptr(unsafe.Pointer(&bufferSize))) - if ret != 0 { - return nil, windows.GetLastError() - } - - return &sysProcInfo, nil -} - func getRusage(pid int32) (*windows.Rusage, error) { var CPU windows.Rusage - c, err := windows.OpenProcess(windows.PROCESS_QUERY_INFORMATION, false, uint32(pid)) + c, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid)) if err != nil { return nil, err } @@ -747,8 +778,7 @@ func getRusage(pid int32) (*windows.Rusage, error) { func getMemoryInfo(pid int32) (PROCESS_MEMORY_COUNTERS, error) { var mem PROCESS_MEMORY_COUNTERS - // PROCESS_QUERY_LIMITED_INFORMATION is 0x1000 - c, err := windows.OpenProcess(0x1000, false, uint32(pid)) + c, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid)) if err != nil { return mem, err } @@ -782,8 +812,7 @@ type SYSTEM_TIMES struct { func getProcessCPUTimes(pid int32) (SYSTEM_TIMES, error) { var times SYSTEM_TIMES - // PROCESS_QUERY_LIMITED_INFORMATION is 0x1000 - h, err := windows.OpenProcess(0x1000, false, uint32(pid)) + h, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid)) if err != nil { return times, err } @@ -799,3 +828,148 @@ func getProcessCPUTimes(pid int32) (SYSTEM_TIMES, error) { return times, err } + +func is32BitProcess(procHandle syscall.Handle) bool { + var wow64 uint + + ret, _, _ := common.ProcNtQueryInformationProcess.Call( + uintptr(procHandle), + uintptr(common.ProcessWow64Information), + uintptr(unsafe.Pointer(&wow64)), + uintptr(unsafe.Sizeof(wow64)), + uintptr(0), + ) + if int(ret) >= 0 { + if wow64 != 0 { + return true + } + } else { + //if the OS does not support the call, we fallback into the bitness of the app + if unsafe.Sizeof(wow64) == 4 { + return true + } + } + return false +} + +func getProcessCommandLine(pid int32) (string, error) { + h, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION | windows.PROCESS_VM_READ, false, uint32(pid)) + if err == windows.ERROR_ACCESS_DENIED || err == windows.ERROR_INVALID_PARAMETER { + return "", nil + } + if err != nil { + return "", err + } + defer syscall.CloseHandle(syscall.Handle(h)) + + const ( + PROCESSOR_ARCHITECTURE_INTEL = 0 + PROCESSOR_ARCHITECTURE_ARM = 5 + PROCESSOR_ARCHITECTURE_ARM64 = 12 + PROCESSOR_ARCHITECTURE_IA64 = 6 + PROCESSOR_ARCHITECTURE_AMD64 = 9 + ) + + procIs32Bits := true + switch processorArchitecture { + case PROCESSOR_ARCHITECTURE_INTEL: + fallthrough + case PROCESSOR_ARCHITECTURE_ARM: + procIs32Bits = true + + case PROCESSOR_ARCHITECTURE_ARM64: + fallthrough + case PROCESSOR_ARCHITECTURE_IA64: + fallthrough + case PROCESSOR_ARCHITECTURE_AMD64: + procIs32Bits = is32BitProcess(syscall.Handle(h)) + + default: + //for other unknown platforms, we rely on process platform + if unsafe.Sizeof(processorArchitecture) == 8 { + procIs32Bits = false + } + } + + pebAddress := queryPebAddress(syscall.Handle(h), procIs32Bits) + if pebAddress == 0 { + return "", errors.New("cannot locate process PEB") + } + + if procIs32Bits { + buf := readProcessMemory(syscall.Handle(h), procIs32Bits, pebAddress + uint64(16), 4) + if len(buf) != 4 { + return "", errors.New("cannot locate process user parameters") + } + userProcParams := uint64(buf[0]) | (uint64(buf[1]) << 8) | (uint64(buf[2]) << 16) | (uint64(buf[3]) << 24) + + //read CommandLine field from PRTL_USER_PROCESS_PARAMETERS + remoteCmdLine := readProcessMemory(syscall.Handle(h), procIs32Bits, userProcParams + uint64(64), 8) + if len(remoteCmdLine) != 8 { + return "", errors.New("cannot read cmdline field") + } + + //remoteCmdLine is actually a UNICODE_STRING32 + //the first two bytes has the length + cmdLineLength := uint(remoteCmdLine[0]) | (uint(remoteCmdLine[1]) << 8) + if cmdLineLength > 0 { + //and, at offset 4, is the pointer to the buffer + bufferAddress := uint32(remoteCmdLine[4]) | (uint32(remoteCmdLine[5]) << 8) | + (uint32(remoteCmdLine[6]) << 16) | (uint32(remoteCmdLine[7]) << 24) + + cmdLine := readProcessMemory(syscall.Handle(h), procIs32Bits, uint64(bufferAddress), cmdLineLength) + if len(cmdLine) != int(cmdLineLength) { + return "", errors.New("cannot read cmdline") + } + + return convertUTF16ToString(cmdLine), nil + } + } else { + buf := readProcessMemory(syscall.Handle(h), procIs32Bits, pebAddress + uint64(32), 8) + if len(buf) != 8 { + return "", errors.New("cannot locate process user parameters") + } + userProcParams := uint64(buf[0]) | (uint64(buf[1]) << 8) | (uint64(buf[2]) << 16) | (uint64(buf[3]) << 24) | + (uint64(buf[4]) << 32) | (uint64(buf[5]) << 40) | (uint64(buf[6]) << 48) | (uint64(buf[7]) << 56) + + //read CommandLine field from PRTL_USER_PROCESS_PARAMETERS + remoteCmdLine := readProcessMemory(syscall.Handle(h), procIs32Bits, userProcParams + uint64(112), 16) + if len(remoteCmdLine) != 16 { + return "", errors.New("cannot read cmdline field") + } + + //remoteCmdLine is actually a UNICODE_STRING64 + //the first two bytes has the length + cmdLineLength := uint(remoteCmdLine[0]) | (uint(remoteCmdLine[1]) << 8) + if cmdLineLength > 0 { + //and, at offset 8, is the pointer to the buffer + bufferAddress := uint64(remoteCmdLine[8]) | (uint64(remoteCmdLine[9]) << 8) | + (uint64(remoteCmdLine[10]) << 16) | (uint64(remoteCmdLine[11]) << 24) | + (uint64(remoteCmdLine[12]) << 32) | (uint64(remoteCmdLine[13]) << 40) | + (uint64(remoteCmdLine[14]) << 48) | (uint64(remoteCmdLine[15]) << 56) + + cmdLine := readProcessMemory(syscall.Handle(h), procIs32Bits, bufferAddress, cmdLineLength) + if len(cmdLine) != int(cmdLineLength) { + return "", errors.New("cannot read cmdline") + } + + return convertUTF16ToString(cmdLine), nil + } + } + + //if we reach here, we have no command line + return "", nil +} + +func convertUTF16ToString(src []byte) string { + srcLen := len(src) / 2 + + codePoints := make([]uint16, srcLen) + + srcIdx := 0 + for i := 0; i < srcLen; i++ { + codePoints[i] = uint16(src[srcIdx]) | uint16(src[srcIdx + 1] << 8) + srcIdx += 2 + } + return syscall.UTF16ToString(codePoints) +} diff --git a/github.com/shirou/gopsutil/process/process_windows_386.go b/github.com/shirou/gopsutil/process/process_windows_386.go index 68f3153dc7..cd884968ef 100644 --- a/github.com/shirou/gopsutil/process/process_windows_386.go +++ b/github.com/shirou/gopsutil/process/process_windows_386.go @@ -2,6 +2,13 @@ package process +import ( + "syscall" + "unsafe" + + "github.com/shirou/gopsutil/internal/common" +) + type PROCESS_MEMORY_COUNTERS struct { CB uint32 PageFaultCount uint32 @@ -14,3 +21,82 @@ type PROCESS_MEMORY_COUNTERS struct { PagefileUsage uint32 PeakPagefileUsage uint32 } + +func queryPebAddress(procHandle syscall.Handle, is32BitProcess bool) uint64 { + if is32BitProcess { + //we are on a 32-bit process reading an external 32-bit process + var info processBasicInformation32 + + ret, _, _ := common.ProcNtQueryInformationProcess.Call( + uintptr(procHandle), + uintptr(common.ProcessBasicInformation), + uintptr(unsafe.Pointer(&info)), + uintptr(unsafe.Sizeof(info)), + uintptr(0), + ) + if int(ret) >= 0 { + return uint64(info.PebBaseAddress) + } + } else { + //we are on a 32-bit process reading an external 64-bit process + if common.ProcNtWow64QueryInformationProcess64.Find() == nil { //avoid panic + var info processBasicInformation64 + + ret, _, _ := common.ProcNtWow64QueryInformationProcess64.Call( + uintptr(procHandle), + uintptr(common.ProcessBasicInformation), + uintptr(unsafe.Pointer(&info)), + uintptr(unsafe.Sizeof(info)), + uintptr(0), + ) + if int(ret) >= 0 { + return info.PebBaseAddress + } + } + } + + //return 0 on error + return 0 +} + +func readProcessMemory(h syscall.Handle, is32BitProcess bool, address uint64, size uint) []byte { + if is32BitProcess { + var read uint + + buffer := make([]byte, size) + + ret, _, _ := common.ProcNtReadVirtualMemory.Call( + uintptr(h), + uintptr(address), + uintptr(unsafe.Pointer(&buffer[0])), + uintptr(size), + uintptr(unsafe.Pointer(&read)), + ) + if int(ret) >= 0 && read > 0 { + return buffer[:read] + } + } else { + //reading a 64-bit process from a 32-bit one + if common.ProcNtWow64ReadVirtualMemory64.Find() == nil { //avoid panic + var read uint64 + + buffer := make([]byte, size) + + ret, _, _ := common.ProcNtWow64ReadVirtualMemory64.Call( + uintptr(h), + uintptr(address & 0xFFFFFFFF), //the call expects a 64-bit value + uintptr(address >> 32), + uintptr(unsafe.Pointer(&buffer[0])), + uintptr(size), //the call expects a 64-bit value + uintptr(0), //but size is 32-bit so pass zero as the high dword + uintptr(unsafe.Pointer(&read)), + ) + if int(ret) >= 0 && read > 0 { + return buffer[:uint(read)] + } + } + } + + //if we reach here, an error happened + return nil +} diff --git a/github.com/shirou/gopsutil/process/process_windows_amd64.go b/github.com/shirou/gopsutil/process/process_windows_amd64.go index df286dfff2..3ee5be449c 100644 --- a/github.com/shirou/gopsutil/process/process_windows_amd64.go +++ b/github.com/shirou/gopsutil/process/process_windows_amd64.go @@ -2,6 +2,13 @@ package process +import ( + "syscall" + "unsafe" + + "github.com/shirou/gopsutil/internal/common" +) + type PROCESS_MEMORY_COUNTERS struct { CB uint32 PageFaultCount uint32 @@ -14,3 +21,56 @@ type PROCESS_MEMORY_COUNTERS struct { PagefileUsage uint64 PeakPagefileUsage uint64 } + +func queryPebAddress(procHandle syscall.Handle, is32BitProcess bool) uint64 { + if is32BitProcess { + //we are on a 64-bit process reading an external 32-bit process + var wow64 uint + + ret, _, _ := common.ProcNtQueryInformationProcess.Call( + uintptr(procHandle), + uintptr(common.ProcessWow64Information), + uintptr(unsafe.Pointer(&wow64)), + uintptr(unsafe.Sizeof(wow64)), + uintptr(0), + ) + if int(ret) >= 0 { + return uint64(wow64) + } + } else { + //we are on a 64-bit process reading an external 64-bit process + var info processBasicInformation64 + + ret, _, _ := common.ProcNtQueryInformationProcess.Call( + uintptr(procHandle), + uintptr(common.ProcessBasicInformation), + uintptr(unsafe.Pointer(&info)), + uintptr(unsafe.Sizeof(info)), + uintptr(0), + ) + if int(ret) >= 0 { + return info.PebBaseAddress + } + } + + //return 0 on error + return 0 +} + +func readProcessMemory(procHandle syscall.Handle, _ bool, address uint64, size uint) []byte { + var read uint + + buffer := make([]byte, size) + + ret, _, _ := common.ProcNtReadVirtualMemory.Call( + uintptr(procHandle), + uintptr(address), + uintptr(unsafe.Pointer(&buffer[0])), + uintptr(size), + uintptr(unsafe.Pointer(&read)), + ) + if int(ret) >= 0 && read > 0 { + return buffer[:read] + } + return nil +} diff --git a/github.com/shirou/w32/AUTHORS b/github.com/shirou/w32/AUTHORS deleted file mode 100644 index c0785e8203..0000000000 --- a/github.com/shirou/w32/AUTHORS +++ /dev/null @@ -1,16 +0,0 @@ -# This is the official list of 'w32' authors for copyright purposes. - -# Names should be added to this file as -# Name or Organization -# The email address is not required for organizations. - -# Please keep the list sorted. - -# Contributors -# ============ - -Allen Dang -Benny Siegert -Bruno Bigras -Gerald Rosenberg -Michael Henke \ No newline at end of file diff --git a/github.com/shirou/w32/LICENSE b/github.com/shirou/w32/LICENSE deleted file mode 100644 index 9f36608c87..0000000000 --- a/github.com/shirou/w32/LICENSE +++ /dev/null @@ -1,23 +0,0 @@ -Copyright (c) 2010-2012 The w32 Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. -3. The names of the authors may not be used to endorse or promote products - derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR -IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/github.com/shirou/w32/README.md b/github.com/shirou/w32/README.md deleted file mode 100644 index ed196e7661..0000000000 --- a/github.com/shirou/w32/README.md +++ /dev/null @@ -1,33 +0,0 @@ -About w32 -========== - -w32 is a wrapper of windows apis for the Go Programming Language. - -It wraps win32 apis to "Go style" to make them easier to use. - -Setup -===== - -1. Make sure you have a working Go installation and build environment, - see this go-nuts post for details: - http://groups.google.com/group/golang-nuts/msg/5c87630a84f4fd0c - - Updated versions of the Windows Go build are available here: - http://code.google.com/p/gomingw/downloads/list - -2. Create a "gopath" directory if you do not have one yet and set the - GOPATH variable accordingly. For example: - mkdir -p go-externals/src - export GOPATH=${PWD}/go-externals - -3. go get github.com/AllenDang/w32 - -4. go install github.com/AllenDang/w32... - -Contribute -========== - -Contributions in form of design, code, documentation, bug reporting or other -ways you see fit are very welcome. - -Thank You! diff --git a/github.com/shirou/w32/advapi32.go b/github.com/shirou/w32/advapi32.go deleted file mode 100644 index 35fd35a676..0000000000 --- a/github.com/shirou/w32/advapi32.go +++ /dev/null @@ -1,301 +0,0 @@ -// Copyright 2010-2012 The W32 Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build windows - -package w32 - -import ( - "errors" - "fmt" - "syscall" - "unsafe" -) - -var ( - modadvapi32 = syscall.NewLazyDLL("advapi32.dll") - - procRegCreateKeyEx = modadvapi32.NewProc("RegCreateKeyExW") - procRegOpenKeyEx = modadvapi32.NewProc("RegOpenKeyExW") - procRegCloseKey = modadvapi32.NewProc("RegCloseKey") - procRegGetValue = modadvapi32.NewProc("RegGetValueW") - procRegEnumKeyEx = modadvapi32.NewProc("RegEnumKeyExW") - // procRegSetKeyValue = modadvapi32.NewProc("RegSetKeyValueW") - procRegSetValueEx = modadvapi32.NewProc("RegSetValueExW") - procOpenEventLog = modadvapi32.NewProc("OpenEventLogW") - procReadEventLog = modadvapi32.NewProc("ReadEventLogW") - procCloseEventLog = modadvapi32.NewProc("CloseEventLog") - procOpenSCManager = modadvapi32.NewProc("OpenSCManagerW") - procCloseServiceHandle = modadvapi32.NewProc("CloseServiceHandle") - procOpenService = modadvapi32.NewProc("OpenServiceW") - procStartService = modadvapi32.NewProc("StartServiceW") - procControlService = modadvapi32.NewProc("ControlService") -) - -func RegCreateKey(hKey HKEY, subKey string) HKEY { - var result HKEY - ret, _, _ := procRegCreateKeyEx.Call( - uintptr(hKey), - uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(subKey))), - uintptr(0), - uintptr(0), - uintptr(0), - uintptr(KEY_ALL_ACCESS), - uintptr(0), - uintptr(unsafe.Pointer(&result)), - uintptr(0)) - _ = ret - return result -} - -func RegOpenKeyEx(hKey HKEY, subKey string, samDesired uint32) HKEY { - var result HKEY - ret, _, _ := procRegOpenKeyEx.Call( - uintptr(hKey), - uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(subKey))), - uintptr(0), - uintptr(samDesired), - uintptr(unsafe.Pointer(&result))) - - if ret != ERROR_SUCCESS { - panic(fmt.Sprintf("RegOpenKeyEx(%d, %s, %d) failed", hKey, subKey, samDesired)) - } - return result -} - -func RegCloseKey(hKey HKEY) error { - var err error - ret, _, _ := procRegCloseKey.Call( - uintptr(hKey)) - - if ret != ERROR_SUCCESS { - err = errors.New("RegCloseKey failed") - } - return err -} - -func RegGetRaw(hKey HKEY, subKey string, value string) []byte { - var bufLen uint32 - var valptr unsafe.Pointer - if len(value) > 0 { - valptr = unsafe.Pointer(syscall.StringToUTF16Ptr(value)) - } - procRegGetValue.Call( - uintptr(hKey), - uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(subKey))), - uintptr(valptr), - uintptr(RRF_RT_ANY), - 0, - 0, - uintptr(unsafe.Pointer(&bufLen))) - - if bufLen == 0 { - return nil - } - - buf := make([]byte, bufLen) - ret, _, _ := procRegGetValue.Call( - uintptr(hKey), - uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(subKey))), - uintptr(valptr), - uintptr(RRF_RT_ANY), - 0, - uintptr(unsafe.Pointer(&buf[0])), - uintptr(unsafe.Pointer(&bufLen))) - - if ret != ERROR_SUCCESS { - return nil - } - - return buf -} - -func RegSetBinary(hKey HKEY, subKey string, value []byte) (errno int) { - var lptr, vptr unsafe.Pointer - if len(subKey) > 0 { - lptr = unsafe.Pointer(syscall.StringToUTF16Ptr(subKey)) - } - if len(value) > 0 { - vptr = unsafe.Pointer(&value[0]) - } - ret, _, _ := procRegSetValueEx.Call( - uintptr(hKey), - uintptr(lptr), - uintptr(0), - uintptr(REG_BINARY), - uintptr(vptr), - uintptr(len(value))) - - return int(ret) -} - -func RegGetString(hKey HKEY, subKey string, value string) string { - var bufLen uint32 - procRegGetValue.Call( - uintptr(hKey), - uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(subKey))), - uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(value))), - uintptr(RRF_RT_REG_SZ), - 0, - 0, - uintptr(unsafe.Pointer(&bufLen))) - - if bufLen == 0 { - return "" - } - - buf := make([]uint16, bufLen) - ret, _, _ := procRegGetValue.Call( - uintptr(hKey), - uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(subKey))), - uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(value))), - uintptr(RRF_RT_REG_SZ), - 0, - uintptr(unsafe.Pointer(&buf[0])), - uintptr(unsafe.Pointer(&bufLen))) - - if ret != ERROR_SUCCESS { - return "" - } - - return syscall.UTF16ToString(buf) -} - -/* -func RegSetKeyValue(hKey HKEY, subKey string, valueName string, dwType uint32, data uintptr, cbData uint16) (errno int) { - ret, _, _ := procRegSetKeyValue.Call( - uintptr(hKey), - uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(subKey))), - uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(valueName))), - uintptr(dwType), - data, - uintptr(cbData)) - - return int(ret) -} -*/ - -func RegEnumKeyEx(hKey HKEY, index uint32) string { - var bufLen uint32 = 255 - buf := make([]uint16, bufLen) - procRegEnumKeyEx.Call( - uintptr(hKey), - uintptr(index), - uintptr(unsafe.Pointer(&buf[0])), - uintptr(unsafe.Pointer(&bufLen)), - 0, - 0, - 0, - 0) - return syscall.UTF16ToString(buf) -} - -func OpenEventLog(servername string, sourcename string) HANDLE { - ret, _, _ := procOpenEventLog.Call( - uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(servername))), - uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(sourcename)))) - - return HANDLE(ret) -} - -func ReadEventLog(eventlog HANDLE, readflags, recordoffset uint32, buffer []byte, numberofbytestoread uint32, bytesread, minnumberofbytesneeded *uint32) bool { - ret, _, _ := procReadEventLog.Call( - uintptr(eventlog), - uintptr(readflags), - uintptr(recordoffset), - uintptr(unsafe.Pointer(&buffer[0])), - uintptr(numberofbytestoread), - uintptr(unsafe.Pointer(bytesread)), - uintptr(unsafe.Pointer(minnumberofbytesneeded))) - - return ret != 0 -} - -func CloseEventLog(eventlog HANDLE) bool { - ret, _, _ := procCloseEventLog.Call( - uintptr(eventlog)) - - return ret != 0 -} - -func OpenSCManager(lpMachineName, lpDatabaseName string, dwDesiredAccess uint32) (HANDLE, error) { - var p1, p2 uintptr - if len(lpMachineName) > 0 { - p1 = uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(lpMachineName))) - } - if len(lpDatabaseName) > 0 { - p2 = uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(lpDatabaseName))) - } - ret, _, _ := procOpenSCManager.Call( - p1, - p2, - uintptr(dwDesiredAccess)) - - if ret == 0 { - return 0, syscall.GetLastError() - } - - return HANDLE(ret), nil -} - -func CloseServiceHandle(hSCObject HANDLE) error { - ret, _, _ := procCloseServiceHandle.Call(uintptr(hSCObject)) - if ret == 0 { - return syscall.GetLastError() - } - return nil -} - -func OpenService(hSCManager HANDLE, lpServiceName string, dwDesiredAccess uint32) (HANDLE, error) { - ret, _, _ := procOpenService.Call( - uintptr(hSCManager), - uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(lpServiceName))), - uintptr(dwDesiredAccess)) - - if ret == 0 { - return 0, syscall.GetLastError() - } - - return HANDLE(ret), nil -} - -func StartService(hService HANDLE, lpServiceArgVectors []string) error { - l := len(lpServiceArgVectors) - var ret uintptr - if l == 0 { - ret, _, _ = procStartService.Call( - uintptr(hService), - 0, - 0) - } else { - lpArgs := make([]uintptr, l) - for i := 0; i < l; i++ { - lpArgs[i] = uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(lpServiceArgVectors[i]))) - } - - ret, _, _ = procStartService.Call( - uintptr(hService), - uintptr(l), - uintptr(unsafe.Pointer(&lpArgs[0]))) - } - - if ret == 0 { - return syscall.GetLastError() - } - - return nil -} - -func ControlService(hService HANDLE, dwControl uint32, lpServiceStatus *SERVICE_STATUS) bool { - if lpServiceStatus == nil { - panic("ControlService:lpServiceStatus cannot be nil") - } - - ret, _, _ := procControlService.Call( - uintptr(hService), - uintptr(dwControl), - uintptr(unsafe.Pointer(lpServiceStatus))) - - return ret != 0 -} diff --git a/github.com/shirou/w32/comctl32.go b/github.com/shirou/w32/comctl32.go deleted file mode 100644 index 51395580e1..0000000000 --- a/github.com/shirou/w32/comctl32.go +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2010-2012 The W32 Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build windows - -package w32 - -import ( - "syscall" - "unsafe" -) - -var ( - modcomctl32 = syscall.NewLazyDLL("comctl32.dll") - - procInitCommonControlsEx = modcomctl32.NewProc("InitCommonControlsEx") - procImageList_Create = modcomctl32.NewProc("ImageList_Create") - procImageList_Destroy = modcomctl32.NewProc("ImageList_Destroy") - procImageList_GetImageCount = modcomctl32.NewProc("ImageList_GetImageCount") - procImageList_SetImageCount = modcomctl32.NewProc("ImageList_SetImageCount") - procImageList_Add = modcomctl32.NewProc("ImageList_Add") - procImageList_ReplaceIcon = modcomctl32.NewProc("ImageList_ReplaceIcon") - procImageList_Remove = modcomctl32.NewProc("ImageList_Remove") - procTrackMouseEvent = modcomctl32.NewProc("_TrackMouseEvent") -) - -func InitCommonControlsEx(lpInitCtrls *INITCOMMONCONTROLSEX) bool { - ret, _, _ := procInitCommonControlsEx.Call( - uintptr(unsafe.Pointer(lpInitCtrls))) - - return ret != 0 -} - -func ImageList_Create(cx, cy int, flags uint, cInitial, cGrow int) HIMAGELIST { - ret, _, _ := procImageList_Create.Call( - uintptr(cx), - uintptr(cy), - uintptr(flags), - uintptr(cInitial), - uintptr(cGrow)) - - if ret == 0 { - panic("Create image list failed") - } - - return HIMAGELIST(ret) -} - -func ImageList_Destroy(himl HIMAGELIST) bool { - ret, _, _ := procImageList_Destroy.Call( - uintptr(himl)) - - return ret != 0 -} - -func ImageList_GetImageCount(himl HIMAGELIST) int { - ret, _, _ := procImageList_GetImageCount.Call( - uintptr(himl)) - - return int(ret) -} - -func ImageList_SetImageCount(himl HIMAGELIST, uNewCount uint) bool { - ret, _, _ := procImageList_SetImageCount.Call( - uintptr(himl), - uintptr(uNewCount)) - - return ret != 0 -} - -func ImageList_Add(himl HIMAGELIST, hbmImage, hbmMask HBITMAP) int { - ret, _, _ := procImageList_Add.Call( - uintptr(himl), - uintptr(hbmImage), - uintptr(hbmMask)) - - return int(ret) -} - -func ImageList_ReplaceIcon(himl HIMAGELIST, i int, hicon HICON) int { - ret, _, _ := procImageList_ReplaceIcon.Call( - uintptr(himl), - uintptr(i), - uintptr(hicon)) - - return int(ret) -} - -func ImageList_AddIcon(himl HIMAGELIST, hicon HICON) int { - return ImageList_ReplaceIcon(himl, -1, hicon) -} - -func ImageList_Remove(himl HIMAGELIST, i int) bool { - ret, _, _ := procImageList_Remove.Call( - uintptr(himl), - uintptr(i)) - - return ret != 0 -} - -func ImageList_RemoveAll(himl HIMAGELIST) bool { - return ImageList_Remove(himl, -1) -} - -func TrackMouseEvent(tme *TRACKMOUSEEVENT) bool { - ret, _, _ := procTrackMouseEvent.Call( - uintptr(unsafe.Pointer(tme))) - - return ret != 0 -} diff --git a/github.com/shirou/w32/comdlg32.go b/github.com/shirou/w32/comdlg32.go deleted file mode 100644 index ad9f7762fd..0000000000 --- a/github.com/shirou/w32/comdlg32.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2010-2012 The W32 Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build windows - -package w32 - -import ( - "syscall" - "unsafe" -) - -var ( - modcomdlg32 = syscall.NewLazyDLL("comdlg32.dll") - - procGetSaveFileName = modcomdlg32.NewProc("GetSaveFileNameW") - procGetOpenFileName = modcomdlg32.NewProc("GetOpenFileNameW") - procCommDlgExtendedError = modcomdlg32.NewProc("CommDlgExtendedError") -) - -func GetOpenFileName(ofn *OPENFILENAME) bool { - ret, _, _ := procGetOpenFileName.Call( - uintptr(unsafe.Pointer(ofn))) - - return ret != 0 -} - -func GetSaveFileName(ofn *OPENFILENAME) bool { - ret, _, _ := procGetSaveFileName.Call( - uintptr(unsafe.Pointer(ofn))) - - return ret != 0 -} - -func CommDlgExtendedError() uint { - ret, _, _ := procCommDlgExtendedError.Call() - - return uint(ret) -} diff --git a/github.com/shirou/w32/constants.go b/github.com/shirou/w32/constants.go deleted file mode 100644 index 62d2d4b318..0000000000 --- a/github.com/shirou/w32/constants.go +++ /dev/null @@ -1,2661 +0,0 @@ -// Copyright 2010-2012 The W32 Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package w32 - -const ( - FALSE = 0 - TRUE = 1 -) - -const ( - NO_ERROR = 0 - ERROR_SUCCESS = 0 - ERROR_FILE_NOT_FOUND = 2 - ERROR_PATH_NOT_FOUND = 3 - ERROR_ACCESS_DENIED = 5 - ERROR_INVALID_HANDLE = 6 - ERROR_BAD_FORMAT = 11 - ERROR_INVALID_NAME = 123 - ERROR_MORE_DATA = 234 - ERROR_NO_MORE_ITEMS = 259 - ERROR_INVALID_SERVICE_CONTROL = 1052 - ERROR_SERVICE_REQUEST_TIMEOUT = 1053 - ERROR_SERVICE_NO_THREAD = 1054 - ERROR_SERVICE_DATABASE_LOCKED = 1055 - ERROR_SERVICE_ALREADY_RUNNING = 1056 - ERROR_SERVICE_DISABLED = 1058 - ERROR_SERVICE_DOES_NOT_EXIST = 1060 - ERROR_SERVICE_CANNOT_ACCEPT_CTRL = 1061 - ERROR_SERVICE_NOT_ACTIVE = 1062 - ERROR_DATABASE_DOES_NOT_EXIST = 1065 - ERROR_SERVICE_DEPENDENCY_FAIL = 1068 - ERROR_SERVICE_LOGON_FAILED = 1069 - ERROR_SERVICE_MARKED_FOR_DELETE = 1072 - ERROR_SERVICE_DEPENDENCY_DELETED = 1075 -) - -const ( - SE_ERR_FNF = 2 - SE_ERR_PNF = 3 - SE_ERR_ACCESSDENIED = 5 - SE_ERR_OOM = 8 - SE_ERR_DLLNOTFOUND = 32 - SE_ERR_SHARE = 26 - SE_ERR_ASSOCINCOMPLETE = 27 - SE_ERR_DDETIMEOUT = 28 - SE_ERR_DDEFAIL = 29 - SE_ERR_DDEBUSY = 30 - SE_ERR_NOASSOC = 31 -) - -const ( - CW_USEDEFAULT = ^0x7fffffff -) - -// ShowWindow constants -const ( - SW_HIDE = 0 - SW_NORMAL = 1 - SW_SHOWNORMAL = 1 - SW_SHOWMINIMIZED = 2 - SW_MAXIMIZE = 3 - SW_SHOWMAXIMIZED = 3 - SW_SHOWNOACTIVATE = 4 - SW_SHOW = 5 - SW_MINIMIZE = 6 - SW_SHOWMINNOACTIVE = 7 - SW_SHOWNA = 8 - SW_RESTORE = 9 - SW_SHOWDEFAULT = 10 - SW_FORCEMINIMIZE = 11 -) - -// Window class styles -const ( - CS_VREDRAW = 0x00000001 - CS_HREDRAW = 0x00000002 - CS_KEYCVTWINDOW = 0x00000004 - CS_DBLCLKS = 0x00000008 - CS_OWNDC = 0x00000020 - CS_CLASSDC = 0x00000040 - CS_PARENTDC = 0x00000080 - CS_NOKEYCVT = 0x00000100 - CS_NOCLOSE = 0x00000200 - CS_SAVEBITS = 0x00000800 - CS_BYTEALIGNCLIENT = 0x00001000 - CS_BYTEALIGNWINDOW = 0x00002000 - CS_GLOBALCLASS = 0x00004000 - CS_IME = 0x00010000 - CS_DROPSHADOW = 0x00020000 -) - -// Predefined cursor constants -const ( - IDC_ARROW = 32512 - IDC_IBEAM = 32513 - IDC_WAIT = 32514 - IDC_CROSS = 32515 - IDC_UPARROW = 32516 - IDC_SIZENWSE = 32642 - IDC_SIZENESW = 32643 - IDC_SIZEWE = 32644 - IDC_SIZENS = 32645 - IDC_SIZEALL = 32646 - IDC_NO = 32648 - IDC_HAND = 32649 - IDC_APPSTARTING = 32650 - IDC_HELP = 32651 - IDC_ICON = 32641 - IDC_SIZE = 32640 -) - -// Predefined icon constants -const ( - IDI_APPLICATION = 32512 - IDI_HAND = 32513 - IDI_QUESTION = 32514 - IDI_EXCLAMATION = 32515 - IDI_ASTERISK = 32516 - IDI_WINLOGO = 32517 - IDI_WARNING = IDI_EXCLAMATION - IDI_ERROR = IDI_HAND - IDI_INFORMATION = IDI_ASTERISK -) - -// Button style constants -const ( - BS_3STATE = 5 - BS_AUTO3STATE = 6 - BS_AUTOCHECKBOX = 3 - BS_AUTORADIOBUTTON = 9 - BS_BITMAP = 128 - BS_BOTTOM = 0X800 - BS_CENTER = 0X300 - BS_CHECKBOX = 2 - BS_DEFPUSHBUTTON = 1 - BS_GROUPBOX = 7 - BS_ICON = 64 - BS_LEFT = 256 - BS_LEFTTEXT = 32 - BS_MULTILINE = 0X2000 - BS_NOTIFY = 0X4000 - BS_OWNERDRAW = 0XB - BS_PUSHBUTTON = 0 - BS_PUSHLIKE = 4096 - BS_RADIOBUTTON = 4 - BS_RIGHT = 512 - BS_RIGHTBUTTON = 32 - BS_TEXT = 0 - BS_TOP = 0X400 - BS_USERBUTTON = 8 - BS_VCENTER = 0XC00 - BS_FLAT = 0X8000 -) - -// Button state constants -const ( - BST_CHECKED = 1 - BST_INDETERMINATE = 2 - BST_UNCHECKED = 0 - BST_FOCUS = 8 - BST_PUSHED = 4 -) - -// Predefined brushes constants -const ( - COLOR_3DDKSHADOW = 21 - COLOR_3DFACE = 15 - COLOR_3DHILIGHT = 20 - COLOR_3DHIGHLIGHT = 20 - COLOR_3DLIGHT = 22 - COLOR_BTNHILIGHT = 20 - COLOR_3DSHADOW = 16 - COLOR_ACTIVEBORDER = 10 - COLOR_ACTIVECAPTION = 2 - COLOR_APPWORKSPACE = 12 - COLOR_BACKGROUND = 1 - COLOR_DESKTOP = 1 - COLOR_BTNFACE = 15 - COLOR_BTNHIGHLIGHT = 20 - COLOR_BTNSHADOW = 16 - COLOR_BTNTEXT = 18 - COLOR_CAPTIONTEXT = 9 - COLOR_GRAYTEXT = 17 - COLOR_HIGHLIGHT = 13 - COLOR_HIGHLIGHTTEXT = 14 - COLOR_INACTIVEBORDER = 11 - COLOR_INACTIVECAPTION = 3 - COLOR_INACTIVECAPTIONTEXT = 19 - COLOR_INFOBK = 24 - COLOR_INFOTEXT = 23 - COLOR_MENU = 4 - COLOR_MENUTEXT = 7 - COLOR_SCROLLBAR = 0 - COLOR_WINDOW = 5 - COLOR_WINDOWFRAME = 6 - COLOR_WINDOWTEXT = 8 - COLOR_HOTLIGHT = 26 - COLOR_GRADIENTACTIVECAPTION = 27 - COLOR_GRADIENTINACTIVECAPTION = 28 -) - -// Button message constants -const ( - BM_CLICK = 245 - BM_GETCHECK = 240 - BM_GETIMAGE = 246 - BM_GETSTATE = 242 - BM_SETCHECK = 241 - BM_SETIMAGE = 247 - BM_SETSTATE = 243 - BM_SETSTYLE = 244 -) - -// Button notifications -const ( - BN_CLICKED = 0 - BN_PAINT = 1 - BN_HILITE = 2 - BN_PUSHED = BN_HILITE - BN_UNHILITE = 3 - BN_UNPUSHED = BN_UNHILITE - BN_DISABLE = 4 - BN_DOUBLECLICKED = 5 - BN_DBLCLK = BN_DOUBLECLICKED - BN_SETFOCUS = 6 - BN_KILLFOCUS = 7 -) - -// GetWindowLong and GetWindowLongPtr constants -const ( - GWL_EXSTYLE = -20 - GWL_STYLE = -16 - GWL_WNDPROC = -4 - GWLP_WNDPROC = -4 - GWL_HINSTANCE = -6 - GWLP_HINSTANCE = -6 - GWL_HWNDPARENT = -8 - GWLP_HWNDPARENT = -8 - GWL_ID = -12 - GWLP_ID = -12 - GWL_USERDATA = -21 - GWLP_USERDATA = -21 -) - -// Window style constants -const ( - WS_OVERLAPPED = 0X00000000 - WS_POPUP = 0X80000000 - WS_CHILD = 0X40000000 - WS_MINIMIZE = 0X20000000 - WS_VISIBLE = 0X10000000 - WS_DISABLED = 0X08000000 - WS_CLIPSIBLINGS = 0X04000000 - WS_CLIPCHILDREN = 0X02000000 - WS_MAXIMIZE = 0X01000000 - WS_CAPTION = 0X00C00000 - WS_BORDER = 0X00800000 - WS_DLGFRAME = 0X00400000 - WS_VSCROLL = 0X00200000 - WS_HSCROLL = 0X00100000 - WS_SYSMENU = 0X00080000 - WS_THICKFRAME = 0X00040000 - WS_GROUP = 0X00020000 - WS_TABSTOP = 0X00010000 - WS_MINIMIZEBOX = 0X00020000 - WS_MAXIMIZEBOX = 0X00010000 - WS_TILED = 0X00000000 - WS_ICONIC = 0X20000000 - WS_SIZEBOX = 0X00040000 - WS_OVERLAPPEDWINDOW = 0X00000000 | 0X00C00000 | 0X00080000 | 0X00040000 | 0X00020000 | 0X00010000 - WS_POPUPWINDOW = 0X80000000 | 0X00800000 | 0X00080000 - WS_CHILDWINDOW = 0X40000000 -) - -// Extended window style constants -const ( - WS_EX_DLGMODALFRAME = 0X00000001 - WS_EX_NOPARENTNOTIFY = 0X00000004 - WS_EX_TOPMOST = 0X00000008 - WS_EX_ACCEPTFILES = 0X00000010 - WS_EX_TRANSPARENT = 0X00000020 - WS_EX_MDICHILD = 0X00000040 - WS_EX_TOOLWINDOW = 0X00000080 - WS_EX_WINDOWEDGE = 0X00000100 - WS_EX_CLIENTEDGE = 0X00000200 - WS_EX_CONTEXTHELP = 0X00000400 - WS_EX_RIGHT = 0X00001000 - WS_EX_LEFT = 0X00000000 - WS_EX_RTLREADING = 0X00002000 - WS_EX_LTRREADING = 0X00000000 - WS_EX_LEFTSCROLLBAR = 0X00004000 - WS_EX_RIGHTSCROLLBAR = 0X00000000 - WS_EX_CONTROLPARENT = 0X00010000 - WS_EX_STATICEDGE = 0X00020000 - WS_EX_APPWINDOW = 0X00040000 - WS_EX_OVERLAPPEDWINDOW = 0X00000100 | 0X00000200 - WS_EX_PALETTEWINDOW = 0X00000100 | 0X00000080 | 0X00000008 - WS_EX_LAYERED = 0X00080000 - WS_EX_NOINHERITLAYOUT = 0X00100000 - WS_EX_LAYOUTRTL = 0X00400000 - WS_EX_NOACTIVATE = 0X08000000 -) - -// Window message constants -const ( - WM_APP = 32768 - WM_ACTIVATE = 6 - WM_ACTIVATEAPP = 28 - WM_AFXFIRST = 864 - WM_AFXLAST = 895 - WM_ASKCBFORMATNAME = 780 - WM_CANCELJOURNAL = 75 - WM_CANCELMODE = 31 - WM_CAPTURECHANGED = 533 - WM_CHANGECBCHAIN = 781 - WM_CHAR = 258 - WM_CHARTOITEM = 47 - WM_CHILDACTIVATE = 34 - WM_CLEAR = 771 - WM_CLOSE = 16 - WM_COMMAND = 273 - WM_COMMNOTIFY = 68 /* OBSOLETE */ - WM_COMPACTING = 65 - WM_COMPAREITEM = 57 - WM_CONTEXTMENU = 123 - WM_COPY = 769 - WM_COPYDATA = 74 - WM_CREATE = 1 - WM_CTLCOLORBTN = 309 - WM_CTLCOLORDLG = 310 - WM_CTLCOLOREDIT = 307 - WM_CTLCOLORLISTBOX = 308 - WM_CTLCOLORMSGBOX = 306 - WM_CTLCOLORSCROLLBAR = 311 - WM_CTLCOLORSTATIC = 312 - WM_CUT = 768 - WM_DEADCHAR = 259 - WM_DELETEITEM = 45 - WM_DESTROY = 2 - WM_DESTROYCLIPBOARD = 775 - WM_DEVICECHANGE = 537 - WM_DEVMODECHANGE = 27 - WM_DISPLAYCHANGE = 126 - WM_DRAWCLIPBOARD = 776 - WM_DRAWITEM = 43 - WM_DROPFILES = 563 - WM_ENABLE = 10 - WM_ENDSESSION = 22 - WM_ENTERIDLE = 289 - WM_ENTERMENULOOP = 529 - WM_ENTERSIZEMOVE = 561 - WM_ERASEBKGND = 20 - WM_EXITMENULOOP = 530 - WM_EXITSIZEMOVE = 562 - WM_FONTCHANGE = 29 - WM_GETDLGCODE = 135 - WM_GETFONT = 49 - WM_GETHOTKEY = 51 - WM_GETICON = 127 - WM_GETMINMAXINFO = 36 - WM_GETTEXT = 13 - WM_GETTEXTLENGTH = 14 - WM_HANDHELDFIRST = 856 - WM_HANDHELDLAST = 863 - WM_HELP = 83 - WM_HOTKEY = 786 - WM_HSCROLL = 276 - WM_HSCROLLCLIPBOARD = 782 - WM_ICONERASEBKGND = 39 - WM_INITDIALOG = 272 - WM_INITMENU = 278 - WM_INITMENUPOPUP = 279 - WM_INPUT = 0X00FF - WM_INPUTLANGCHANGE = 81 - WM_INPUTLANGCHANGEREQUEST = 80 - WM_KEYDOWN = 256 - WM_KEYUP = 257 - WM_KILLFOCUS = 8 - WM_MDIACTIVATE = 546 - WM_MDICASCADE = 551 - WM_MDICREATE = 544 - WM_MDIDESTROY = 545 - WM_MDIGETACTIVE = 553 - WM_MDIICONARRANGE = 552 - WM_MDIMAXIMIZE = 549 - WM_MDINEXT = 548 - WM_MDIREFRESHMENU = 564 - WM_MDIRESTORE = 547 - WM_MDISETMENU = 560 - WM_MDITILE = 550 - WM_MEASUREITEM = 44 - WM_GETOBJECT = 0X003D - WM_CHANGEUISTATE = 0X0127 - WM_UPDATEUISTATE = 0X0128 - WM_QUERYUISTATE = 0X0129 - WM_UNINITMENUPOPUP = 0X0125 - WM_MENURBUTTONUP = 290 - WM_MENUCOMMAND = 0X0126 - WM_MENUGETOBJECT = 0X0124 - WM_MENUDRAG = 0X0123 - WM_APPCOMMAND = 0X0319 - WM_MENUCHAR = 288 - WM_MENUSELECT = 287 - WM_MOVE = 3 - WM_MOVING = 534 - WM_NCACTIVATE = 134 - WM_NCCALCSIZE = 131 - WM_NCCREATE = 129 - WM_NCDESTROY = 130 - WM_NCHITTEST = 132 - WM_NCLBUTTONDBLCLK = 163 - WM_NCLBUTTONDOWN = 161 - WM_NCLBUTTONUP = 162 - WM_NCMBUTTONDBLCLK = 169 - WM_NCMBUTTONDOWN = 167 - WM_NCMBUTTONUP = 168 - WM_NCXBUTTONDOWN = 171 - WM_NCXBUTTONUP = 172 - WM_NCXBUTTONDBLCLK = 173 - WM_NCMOUSEHOVER = 0X02A0 - WM_NCMOUSELEAVE = 0X02A2 - WM_NCMOUSEMOVE = 160 - WM_NCPAINT = 133 - WM_NCRBUTTONDBLCLK = 166 - WM_NCRBUTTONDOWN = 164 - WM_NCRBUTTONUP = 165 - WM_NEXTDLGCTL = 40 - WM_NEXTMENU = 531 - WM_NOTIFY = 78 - WM_NOTIFYFORMAT = 85 - WM_NULL = 0 - WM_PAINT = 15 - WM_PAINTCLIPBOARD = 777 - WM_PAINTICON = 38 - WM_PALETTECHANGED = 785 - WM_PALETTEISCHANGING = 784 - WM_PARENTNOTIFY = 528 - WM_PASTE = 770 - WM_PENWINFIRST = 896 - WM_PENWINLAST = 911 - WM_POWER = 72 - WM_POWERBROADCAST = 536 - WM_PRINT = 791 - WM_PRINTCLIENT = 792 - WM_QUERYDRAGICON = 55 - WM_QUERYENDSESSION = 17 - WM_QUERYNEWPALETTE = 783 - WM_QUERYOPEN = 19 - WM_QUEUESYNC = 35 - WM_QUIT = 18 - WM_RENDERALLFORMATS = 774 - WM_RENDERFORMAT = 773 - WM_SETCURSOR = 32 - WM_SETFOCUS = 7 - WM_SETFONT = 48 - WM_SETHOTKEY = 50 - WM_SETICON = 128 - WM_SETREDRAW = 11 - WM_SETTEXT = 12 - WM_SETTINGCHANGE = 26 - WM_SHOWWINDOW = 24 - WM_SIZE = 5 - WM_SIZECLIPBOARD = 779 - WM_SIZING = 532 - WM_SPOOLERSTATUS = 42 - WM_STYLECHANGED = 125 - WM_STYLECHANGING = 124 - WM_SYSCHAR = 262 - WM_SYSCOLORCHANGE = 21 - WM_SYSCOMMAND = 274 - WM_SYSDEADCHAR = 263 - WM_SYSKEYDOWN = 260 - WM_SYSKEYUP = 261 - WM_TCARD = 82 - WM_THEMECHANGED = 794 - WM_TIMECHANGE = 30 - WM_TIMER = 275 - WM_UNDO = 772 - WM_USER = 1024 - WM_USERCHANGED = 84 - WM_VKEYTOITEM = 46 - WM_VSCROLL = 277 - WM_VSCROLLCLIPBOARD = 778 - WM_WINDOWPOSCHANGED = 71 - WM_WINDOWPOSCHANGING = 70 - WM_WININICHANGE = 26 - WM_KEYFIRST = 256 - WM_KEYLAST = 264 - WM_SYNCPAINT = 136 - WM_MOUSEACTIVATE = 33 - WM_MOUSEMOVE = 512 - WM_LBUTTONDOWN = 513 - WM_LBUTTONUP = 514 - WM_LBUTTONDBLCLK = 515 - WM_RBUTTONDOWN = 516 - WM_RBUTTONUP = 517 - WM_RBUTTONDBLCLK = 518 - WM_MBUTTONDOWN = 519 - WM_MBUTTONUP = 520 - WM_MBUTTONDBLCLK = 521 - WM_MOUSEWHEEL = 522 - WM_MOUSEFIRST = 512 - WM_XBUTTONDOWN = 523 - WM_XBUTTONUP = 524 - WM_XBUTTONDBLCLK = 525 - WM_MOUSELAST = 525 - WM_MOUSEHOVER = 0X2A1 - WM_MOUSELEAVE = 0X2A3 - WM_CLIPBOARDUPDATE = 0x031D -) - -// WM_ACTIVATE -const ( - WA_INACTIVE = 0 - WA_ACTIVE = 1 - WA_CLICKACTIVE = 2 -) - -const LF_FACESIZE = 32 - -// Font weight constants -const ( - FW_DONTCARE = 0 - FW_THIN = 100 - FW_EXTRALIGHT = 200 - FW_ULTRALIGHT = FW_EXTRALIGHT - FW_LIGHT = 300 - FW_NORMAL = 400 - FW_REGULAR = 400 - FW_MEDIUM = 500 - FW_SEMIBOLD = 600 - FW_DEMIBOLD = FW_SEMIBOLD - FW_BOLD = 700 - FW_EXTRABOLD = 800 - FW_ULTRABOLD = FW_EXTRABOLD - FW_HEAVY = 900 - FW_BLACK = FW_HEAVY -) - -// Charset constants -const ( - ANSI_CHARSET = 0 - DEFAULT_CHARSET = 1 - SYMBOL_CHARSET = 2 - SHIFTJIS_CHARSET = 128 - HANGEUL_CHARSET = 129 - HANGUL_CHARSET = 129 - GB2312_CHARSET = 134 - CHINESEBIG5_CHARSET = 136 - GREEK_CHARSET = 161 - TURKISH_CHARSET = 162 - HEBREW_CHARSET = 177 - ARABIC_CHARSET = 178 - BALTIC_CHARSET = 186 - RUSSIAN_CHARSET = 204 - THAI_CHARSET = 222 - EASTEUROPE_CHARSET = 238 - OEM_CHARSET = 255 - JOHAB_CHARSET = 130 - VIETNAMESE_CHARSET = 163 - MAC_CHARSET = 77 -) - -// Font output precision constants -const ( - OUT_DEFAULT_PRECIS = 0 - OUT_STRING_PRECIS = 1 - OUT_CHARACTER_PRECIS = 2 - OUT_STROKE_PRECIS = 3 - OUT_TT_PRECIS = 4 - OUT_DEVICE_PRECIS = 5 - OUT_RASTER_PRECIS = 6 - OUT_TT_ONLY_PRECIS = 7 - OUT_OUTLINE_PRECIS = 8 - OUT_PS_ONLY_PRECIS = 10 -) - -// Font clipping precision constants -const ( - CLIP_DEFAULT_PRECIS = 0 - CLIP_CHARACTER_PRECIS = 1 - CLIP_STROKE_PRECIS = 2 - CLIP_MASK = 15 - CLIP_LH_ANGLES = 16 - CLIP_TT_ALWAYS = 32 - CLIP_EMBEDDED = 128 -) - -// Font output quality constants -const ( - DEFAULT_QUALITY = 0 - DRAFT_QUALITY = 1 - PROOF_QUALITY = 2 - NONANTIALIASED_QUALITY = 3 - ANTIALIASED_QUALITY = 4 - CLEARTYPE_QUALITY = 5 -) - -// Font pitch constants -const ( - DEFAULT_PITCH = 0 - FIXED_PITCH = 1 - VARIABLE_PITCH = 2 -) - -// Font family constants -const ( - FF_DECORATIVE = 80 - FF_DONTCARE = 0 - FF_MODERN = 48 - FF_ROMAN = 16 - FF_SCRIPT = 64 - FF_SWISS = 32 -) - -// DeviceCapabilities capabilities -const ( - DC_FIELDS = 1 - DC_PAPERS = 2 - DC_PAPERSIZE = 3 - DC_MINEXTENT = 4 - DC_MAXEXTENT = 5 - DC_BINS = 6 - DC_DUPLEX = 7 - DC_SIZE = 8 - DC_EXTRA = 9 - DC_VERSION = 10 - DC_DRIVER = 11 - DC_BINNAMES = 12 - DC_ENUMRESOLUTIONS = 13 - DC_FILEDEPENDENCIES = 14 - DC_TRUETYPE = 15 - DC_PAPERNAMES = 16 - DC_ORIENTATION = 17 - DC_COPIES = 18 - DC_BINADJUST = 19 - DC_EMF_COMPLIANT = 20 - DC_DATATYPE_PRODUCED = 21 - DC_COLLATE = 22 - DC_MANUFACTURER = 23 - DC_MODEL = 24 - DC_PERSONALITY = 25 - DC_PRINTRATE = 26 - DC_PRINTRATEUNIT = 27 - DC_PRINTERMEM = 28 - DC_MEDIAREADY = 29 - DC_STAPLE = 30 - DC_PRINTRATEPPM = 31 - DC_COLORDEVICE = 32 - DC_NUP = 33 - DC_MEDIATYPENAMES = 34 - DC_MEDIATYPES = 35 -) - -// GetDeviceCaps index constants -const ( - DRIVERVERSION = 0 - TECHNOLOGY = 2 - HORZSIZE = 4 - VERTSIZE = 6 - HORZRES = 8 - VERTRES = 10 - LOGPIXELSX = 88 - LOGPIXELSY = 90 - BITSPIXEL = 12 - PLANES = 14 - NUMBRUSHES = 16 - NUMPENS = 18 - NUMFONTS = 22 - NUMCOLORS = 24 - NUMMARKERS = 20 - ASPECTX = 40 - ASPECTY = 42 - ASPECTXY = 44 - PDEVICESIZE = 26 - CLIPCAPS = 36 - SIZEPALETTE = 104 - NUMRESERVED = 106 - COLORRES = 108 - PHYSICALWIDTH = 110 - PHYSICALHEIGHT = 111 - PHYSICALOFFSETX = 112 - PHYSICALOFFSETY = 113 - SCALINGFACTORX = 114 - SCALINGFACTORY = 115 - VREFRESH = 116 - DESKTOPHORZRES = 118 - DESKTOPVERTRES = 117 - BLTALIGNMENT = 119 - SHADEBLENDCAPS = 120 - COLORMGMTCAPS = 121 - RASTERCAPS = 38 - CURVECAPS = 28 - LINECAPS = 30 - POLYGONALCAPS = 32 - TEXTCAPS = 34 -) - -// GetDeviceCaps TECHNOLOGY constants -const ( - DT_PLOTTER = 0 - DT_RASDISPLAY = 1 - DT_RASPRINTER = 2 - DT_RASCAMERA = 3 - DT_CHARSTREAM = 4 - DT_METAFILE = 5 - DT_DISPFILE = 6 -) - -// GetDeviceCaps SHADEBLENDCAPS constants -const ( - SB_NONE = 0x00 - SB_CONST_ALPHA = 0x01 - SB_PIXEL_ALPHA = 0x02 - SB_PREMULT_ALPHA = 0x04 - SB_GRAD_RECT = 0x10 - SB_GRAD_TRI = 0x20 -) - -// GetDeviceCaps COLORMGMTCAPS constants -const ( - CM_NONE = 0x00 - CM_DEVICE_ICM = 0x01 - CM_GAMMA_RAMP = 0x02 - CM_CMYK_COLOR = 0x04 -) - -// GetDeviceCaps RASTERCAPS constants -const ( - RC_BANDING = 2 - RC_BITBLT = 1 - RC_BITMAP64 = 8 - RC_DI_BITMAP = 128 - RC_DIBTODEV = 512 - RC_FLOODFILL = 4096 - RC_GDI20_OUTPUT = 16 - RC_PALETTE = 256 - RC_SCALING = 4 - RC_STRETCHBLT = 2048 - RC_STRETCHDIB = 8192 - RC_DEVBITS = 0x8000 - RC_OP_DX_OUTPUT = 0x4000 -) - -// GetDeviceCaps CURVECAPS constants -const ( - CC_NONE = 0 - CC_CIRCLES = 1 - CC_PIE = 2 - CC_CHORD = 4 - CC_ELLIPSES = 8 - CC_WIDE = 16 - CC_STYLED = 32 - CC_WIDESTYLED = 64 - CC_INTERIORS = 128 - CC_ROUNDRECT = 256 -) - -// GetDeviceCaps LINECAPS constants -const ( - LC_NONE = 0 - LC_POLYLINE = 2 - LC_MARKER = 4 - LC_POLYMARKER = 8 - LC_WIDE = 16 - LC_STYLED = 32 - LC_WIDESTYLED = 64 - LC_INTERIORS = 128 -) - -// GetDeviceCaps POLYGONALCAPS constants -const ( - PC_NONE = 0 - PC_POLYGON = 1 - PC_POLYPOLYGON = 256 - PC_PATHS = 512 - PC_RECTANGLE = 2 - PC_WINDPOLYGON = 4 - PC_SCANLINE = 8 - PC_TRAPEZOID = 4 - PC_WIDE = 16 - PC_STYLED = 32 - PC_WIDESTYLED = 64 - PC_INTERIORS = 128 -) - -// GetDeviceCaps TEXTCAPS constants -const ( - TC_OP_CHARACTER = 1 - TC_OP_STROKE = 2 - TC_CP_STROKE = 4 - TC_CR_90 = 8 - TC_CR_ANY = 16 - TC_SF_X_YINDEP = 32 - TC_SA_DOUBLE = 64 - TC_SA_INTEGER = 128 - TC_SA_CONTIN = 256 - TC_EA_DOUBLE = 512 - TC_IA_ABLE = 1024 - TC_UA_ABLE = 2048 - TC_SO_ABLE = 4096 - TC_RA_ABLE = 8192 - TC_VA_ABLE = 16384 - TC_RESERVED = 32768 - TC_SCROLLBLT = 65536 -) - -// Static control styles -const ( - SS_BITMAP = 14 - SS_BLACKFRAME = 7 - SS_BLACKRECT = 4 - SS_CENTER = 1 - SS_CENTERIMAGE = 512 - SS_EDITCONTROL = 0x2000 - SS_ENHMETAFILE = 15 - SS_ETCHEDFRAME = 18 - SS_ETCHEDHORZ = 16 - SS_ETCHEDVERT = 17 - SS_GRAYFRAME = 8 - SS_GRAYRECT = 5 - SS_ICON = 3 - SS_LEFT = 0 - SS_LEFTNOWORDWRAP = 0xc - SS_NOPREFIX = 128 - SS_NOTIFY = 256 - SS_OWNERDRAW = 0xd - SS_REALSIZECONTROL = 0x040 - SS_REALSIZEIMAGE = 0x800 - SS_RIGHT = 2 - SS_RIGHTJUST = 0x400 - SS_SIMPLE = 11 - SS_SUNKEN = 4096 - SS_WHITEFRAME = 9 - SS_WHITERECT = 6 - SS_USERITEM = 10 - SS_TYPEMASK = 0x0000001F - SS_ENDELLIPSIS = 0x00004000 - SS_PATHELLIPSIS = 0x00008000 - SS_WORDELLIPSIS = 0x0000C000 - SS_ELLIPSISMASK = 0x0000C000 -) - -// Edit styles -const ( - ES_LEFT = 0x0000 - ES_CENTER = 0x0001 - ES_RIGHT = 0x0002 - ES_MULTILINE = 0x0004 - ES_UPPERCASE = 0x0008 - ES_LOWERCASE = 0x0010 - ES_PASSWORD = 0x0020 - ES_AUTOVSCROLL = 0x0040 - ES_AUTOHSCROLL = 0x0080 - ES_NOHIDESEL = 0x0100 - ES_OEMCONVERT = 0x0400 - ES_READONLY = 0x0800 - ES_WANTRETURN = 0x1000 - ES_NUMBER = 0x2000 -) - -// Edit notifications -const ( - EN_SETFOCUS = 0x0100 - EN_KILLFOCUS = 0x0200 - EN_CHANGE = 0x0300 - EN_UPDATE = 0x0400 - EN_ERRSPACE = 0x0500 - EN_MAXTEXT = 0x0501 - EN_HSCROLL = 0x0601 - EN_VSCROLL = 0x0602 - EN_ALIGN_LTR_EC = 0x0700 - EN_ALIGN_RTL_EC = 0x0701 -) - -// Edit messages -const ( - EM_GETSEL = 0x00B0 - EM_SETSEL = 0x00B1 - EM_GETRECT = 0x00B2 - EM_SETRECT = 0x00B3 - EM_SETRECTNP = 0x00B4 - EM_SCROLL = 0x00B5 - EM_LINESCROLL = 0x00B6 - EM_SCROLLCARET = 0x00B7 - EM_GETMODIFY = 0x00B8 - EM_SETMODIFY = 0x00B9 - EM_GETLINECOUNT = 0x00BA - EM_LINEINDEX = 0x00BB - EM_SETHANDLE = 0x00BC - EM_GETHANDLE = 0x00BD - EM_GETTHUMB = 0x00BE - EM_LINELENGTH = 0x00C1 - EM_REPLACESEL = 0x00C2 - EM_GETLINE = 0x00C4 - EM_LIMITTEXT = 0x00C5 - EM_CANUNDO = 0x00C6 - EM_UNDO = 0x00C7 - EM_FMTLINES = 0x00C8 - EM_LINEFROMCHAR = 0x00C9 - EM_SETTABSTOPS = 0x00CB - EM_SETPASSWORDCHAR = 0x00CC - EM_EMPTYUNDOBUFFER = 0x00CD - EM_GETFIRSTVISIBLELINE = 0x00CE - EM_SETREADONLY = 0x00CF - EM_SETWORDBREAKPROC = 0x00D0 - EM_GETWORDBREAKPROC = 0x00D1 - EM_GETPASSWORDCHAR = 0x00D2 - EM_SETMARGINS = 0x00D3 - EM_GETMARGINS = 0x00D4 - EM_SETLIMITTEXT = EM_LIMITTEXT - EM_GETLIMITTEXT = 0x00D5 - EM_POSFROMCHAR = 0x00D6 - EM_CHARFROMPOS = 0x00D7 - EM_SETIMESTATUS = 0x00D8 - EM_GETIMESTATUS = 0x00D9 - EM_SETCUEBANNER = 0x1501 - EM_GETCUEBANNER = 0x1502 -) - -const ( - CCM_FIRST = 0x2000 - CCM_LAST = CCM_FIRST + 0x200 - CCM_SETBKCOLOR = 8193 - CCM_SETCOLORSCHEME = 8194 - CCM_GETCOLORSCHEME = 8195 - CCM_GETDROPTARGET = 8196 - CCM_SETUNICODEFORMAT = 8197 - CCM_GETUNICODEFORMAT = 8198 - CCM_SETVERSION = 0x2007 - CCM_GETVERSION = 0x2008 - CCM_SETNOTIFYWINDOW = 0x2009 - CCM_SETWINDOWTHEME = 0x200b - CCM_DPISCALE = 0x200c -) - -// Common controls styles -const ( - CCS_TOP = 1 - CCS_NOMOVEY = 2 - CCS_BOTTOM = 3 - CCS_NORESIZE = 4 - CCS_NOPARENTALIGN = 8 - CCS_ADJUSTABLE = 32 - CCS_NODIVIDER = 64 - CCS_VERT = 128 - CCS_LEFT = 129 - CCS_NOMOVEX = 130 - CCS_RIGHT = 131 -) - -// ProgressBar messages -const ( - PROGRESS_CLASS = "msctls_progress32" - PBM_SETPOS = WM_USER + 2 - PBM_DELTAPOS = WM_USER + 3 - PBM_SETSTEP = WM_USER + 4 - PBM_STEPIT = WM_USER + 5 - PBM_SETRANGE32 = 1030 - PBM_GETRANGE = 1031 - PBM_GETPOS = 1032 - PBM_SETBARCOLOR = 1033 - PBM_SETBKCOLOR = CCM_SETBKCOLOR - PBS_SMOOTH = 1 - PBS_VERTICAL = 4 -) - -// GetOpenFileName and GetSaveFileName extended flags -const ( - OFN_EX_NOPLACESBAR = 0x00000001 -) - -// GetOpenFileName and GetSaveFileName flags -const ( - OFN_ALLOWMULTISELECT = 0x00000200 - OFN_CREATEPROMPT = 0x00002000 - OFN_DONTADDTORECENT = 0x02000000 - OFN_ENABLEHOOK = 0x00000020 - OFN_ENABLEINCLUDENOTIFY = 0x00400000 - OFN_ENABLESIZING = 0x00800000 - OFN_ENABLETEMPLATE = 0x00000040 - OFN_ENABLETEMPLATEHANDLE = 0x00000080 - OFN_EXPLORER = 0x00080000 - OFN_EXTENSIONDIFFERENT = 0x00000400 - OFN_FILEMUSTEXIST = 0x00001000 - OFN_FORCESHOWHIDDEN = 0x10000000 - OFN_HIDEREADONLY = 0x00000004 - OFN_LONGNAMES = 0x00200000 - OFN_NOCHANGEDIR = 0x00000008 - OFN_NODEREFERENCELINKS = 0x00100000 - OFN_NOLONGNAMES = 0x00040000 - OFN_NONETWORKBUTTON = 0x00020000 - OFN_NOREADONLYRETURN = 0x00008000 - OFN_NOTESTFILECREATE = 0x00010000 - OFN_NOVALIDATE = 0x00000100 - OFN_OVERWRITEPROMPT = 0x00000002 - OFN_PATHMUSTEXIST = 0x00000800 - OFN_READONLY = 0x00000001 - OFN_SHAREAWARE = 0x00004000 - OFN_SHOWHELP = 0x00000010 -) - -//SHBrowseForFolder flags -const ( - BIF_RETURNONLYFSDIRS = 0x00000001 - BIF_DONTGOBELOWDOMAIN = 0x00000002 - BIF_STATUSTEXT = 0x00000004 - BIF_RETURNFSANCESTORS = 0x00000008 - BIF_EDITBOX = 0x00000010 - BIF_VALIDATE = 0x00000020 - BIF_NEWDIALOGSTYLE = 0x00000040 - BIF_BROWSEINCLUDEURLS = 0x00000080 - BIF_USENEWUI = BIF_EDITBOX | BIF_NEWDIALOGSTYLE - BIF_UAHINT = 0x00000100 - BIF_NONEWFOLDERBUTTON = 0x00000200 - BIF_NOTRANSLATETARGETS = 0x00000400 - BIF_BROWSEFORCOMPUTER = 0x00001000 - BIF_BROWSEFORPRINTER = 0x00002000 - BIF_BROWSEINCLUDEFILES = 0x00004000 - BIF_SHAREABLE = 0x00008000 - BIF_BROWSEFILEJUNCTIONS = 0x00010000 -) - -//MessageBox flags -const ( - MB_OK = 0x00000000 - MB_OKCANCEL = 0x00000001 - MB_ABORTRETRYIGNORE = 0x00000002 - MB_YESNOCANCEL = 0x00000003 - MB_YESNO = 0x00000004 - MB_RETRYCANCEL = 0x00000005 - MB_CANCELTRYCONTINUE = 0x00000006 - MB_ICONHAND = 0x00000010 - MB_ICONQUESTION = 0x00000020 - MB_ICONEXCLAMATION = 0x00000030 - MB_ICONASTERISK = 0x00000040 - MB_USERICON = 0x00000080 - MB_ICONWARNING = MB_ICONEXCLAMATION - MB_ICONERROR = MB_ICONHAND - MB_ICONINFORMATION = MB_ICONASTERISK - MB_ICONSTOP = MB_ICONHAND - MB_DEFBUTTON1 = 0x00000000 - MB_DEFBUTTON2 = 0x00000100 - MB_DEFBUTTON3 = 0x00000200 - MB_DEFBUTTON4 = 0x00000300 -) - -//COM -const ( - E_INVALIDARG = 0x80070057 - E_OUTOFMEMORY = 0x8007000E - E_UNEXPECTED = 0x8000FFFF -) - -const ( - S_OK = 0 - S_FALSE = 0x0001 - RPC_E_CHANGED_MODE = 0x80010106 -) - -// GetSystemMetrics constants -const ( - SM_CXSCREEN = 0 - SM_CYSCREEN = 1 - SM_CXVSCROLL = 2 - SM_CYHSCROLL = 3 - SM_CYCAPTION = 4 - SM_CXBORDER = 5 - SM_CYBORDER = 6 - SM_CXDLGFRAME = 7 - SM_CYDLGFRAME = 8 - SM_CYVTHUMB = 9 - SM_CXHTHUMB = 10 - SM_CXICON = 11 - SM_CYICON = 12 - SM_CXCURSOR = 13 - SM_CYCURSOR = 14 - SM_CYMENU = 15 - SM_CXFULLSCREEN = 16 - SM_CYFULLSCREEN = 17 - SM_CYKANJIWINDOW = 18 - SM_MOUSEPRESENT = 19 - SM_CYVSCROLL = 20 - SM_CXHSCROLL = 21 - SM_DEBUG = 22 - SM_SWAPBUTTON = 23 - SM_RESERVED1 = 24 - SM_RESERVED2 = 25 - SM_RESERVED3 = 26 - SM_RESERVED4 = 27 - SM_CXMIN = 28 - SM_CYMIN = 29 - SM_CXSIZE = 30 - SM_CYSIZE = 31 - SM_CXFRAME = 32 - SM_CYFRAME = 33 - SM_CXMINTRACK = 34 - SM_CYMINTRACK = 35 - SM_CXDOUBLECLK = 36 - SM_CYDOUBLECLK = 37 - SM_CXICONSPACING = 38 - SM_CYICONSPACING = 39 - SM_MENUDROPALIGNMENT = 40 - SM_PENWINDOWS = 41 - SM_DBCSENABLED = 42 - SM_CMOUSEBUTTONS = 43 - SM_CXFIXEDFRAME = SM_CXDLGFRAME - SM_CYFIXEDFRAME = SM_CYDLGFRAME - SM_CXSIZEFRAME = SM_CXFRAME - SM_CYSIZEFRAME = SM_CYFRAME - SM_SECURE = 44 - SM_CXEDGE = 45 - SM_CYEDGE = 46 - SM_CXMINSPACING = 47 - SM_CYMINSPACING = 48 - SM_CXSMICON = 49 - SM_CYSMICON = 50 - SM_CYSMCAPTION = 51 - SM_CXSMSIZE = 52 - SM_CYSMSIZE = 53 - SM_CXMENUSIZE = 54 - SM_CYMENUSIZE = 55 - SM_ARRANGE = 56 - SM_CXMINIMIZED = 57 - SM_CYMINIMIZED = 58 - SM_CXMAXTRACK = 59 - SM_CYMAXTRACK = 60 - SM_CXMAXIMIZED = 61 - SM_CYMAXIMIZED = 62 - SM_NETWORK = 63 - SM_CLEANBOOT = 67 - SM_CXDRAG = 68 - SM_CYDRAG = 69 - SM_SHOWSOUNDS = 70 - SM_CXMENUCHECK = 71 - SM_CYMENUCHECK = 72 - SM_SLOWMACHINE = 73 - SM_MIDEASTENABLED = 74 - SM_MOUSEWHEELPRESENT = 75 - SM_XVIRTUALSCREEN = 76 - SM_YVIRTUALSCREEN = 77 - SM_CXVIRTUALSCREEN = 78 - SM_CYVIRTUALSCREEN = 79 - SM_CMONITORS = 80 - SM_SAMEDISPLAYFORMAT = 81 - SM_IMMENABLED = 82 - SM_CXFOCUSBORDER = 83 - SM_CYFOCUSBORDER = 84 - SM_TABLETPC = 86 - SM_MEDIACENTER = 87 - SM_STARTER = 88 - SM_SERVERR2 = 89 - SM_CMETRICS = 91 - SM_REMOTESESSION = 0x1000 - SM_SHUTTINGDOWN = 0x2000 - SM_REMOTECONTROL = 0x2001 - SM_CARETBLINKINGENABLED = 0x2002 -) - -const ( - CLSCTX_INPROC_SERVER = 1 - CLSCTX_INPROC_HANDLER = 2 - CLSCTX_LOCAL_SERVER = 4 - CLSCTX_INPROC_SERVER16 = 8 - CLSCTX_REMOTE_SERVER = 16 - CLSCTX_ALL = CLSCTX_INPROC_SERVER | CLSCTX_INPROC_HANDLER | CLSCTX_LOCAL_SERVER - CLSCTX_INPROC = CLSCTX_INPROC_SERVER | CLSCTX_INPROC_HANDLER - CLSCTX_SERVER = CLSCTX_INPROC_SERVER | CLSCTX_LOCAL_SERVER | CLSCTX_REMOTE_SERVER -) - -const ( - COINIT_APARTMENTTHREADED = 0x2 - COINIT_MULTITHREADED = 0x0 - COINIT_DISABLE_OLE1DDE = 0x4 - COINIT_SPEED_OVER_MEMORY = 0x8 -) - -const ( - DISPATCH_METHOD = 1 - DISPATCH_PROPERTYGET = 2 - DISPATCH_PROPERTYPUT = 4 - DISPATCH_PROPERTYPUTREF = 8 -) - -const ( - CC_FASTCALL = iota - CC_CDECL - CC_MSCPASCAL - CC_PASCAL = CC_MSCPASCAL - CC_MACPASCAL - CC_STDCALL - CC_FPFASTCALL - CC_SYSCALL - CC_MPWCDECL - CC_MPWPASCAL - CC_MAX = CC_MPWPASCAL -) - -const ( - VT_EMPTY = 0x0 - VT_NULL = 0x1 - VT_I2 = 0x2 - VT_I4 = 0x3 - VT_R4 = 0x4 - VT_R8 = 0x5 - VT_CY = 0x6 - VT_DATE = 0x7 - VT_BSTR = 0x8 - VT_DISPATCH = 0x9 - VT_ERROR = 0xa - VT_BOOL = 0xb - VT_VARIANT = 0xc - VT_UNKNOWN = 0xd - VT_DECIMAL = 0xe - VT_I1 = 0x10 - VT_UI1 = 0x11 - VT_UI2 = 0x12 - VT_UI4 = 0x13 - VT_I8 = 0x14 - VT_UI8 = 0x15 - VT_INT = 0x16 - VT_UINT = 0x17 - VT_VOID = 0x18 - VT_HRESULT = 0x19 - VT_PTR = 0x1a - VT_SAFEARRAY = 0x1b - VT_CARRAY = 0x1c - VT_USERDEFINED = 0x1d - VT_LPSTR = 0x1e - VT_LPWSTR = 0x1f - VT_RECORD = 0x24 - VT_INT_PTR = 0x25 - VT_UINT_PTR = 0x26 - VT_FILETIME = 0x40 - VT_BLOB = 0x41 - VT_STREAM = 0x42 - VT_STORAGE = 0x43 - VT_STREAMED_OBJECT = 0x44 - VT_STORED_OBJECT = 0x45 - VT_BLOB_OBJECT = 0x46 - VT_CF = 0x47 - VT_CLSID = 0x48 - VT_BSTR_BLOB = 0xfff - VT_VECTOR = 0x1000 - VT_ARRAY = 0x2000 - VT_BYREF = 0x4000 - VT_RESERVED = 0x8000 - VT_ILLEGAL = 0xffff - VT_ILLEGALMASKED = 0xfff - VT_TYPEMASK = 0xfff -) - -const ( - DISPID_UNKNOWN = -1 - DISPID_VALUE = 0 - DISPID_PROPERTYPUT = -3 - DISPID_NEWENUM = -4 - DISPID_EVALUATE = -5 - DISPID_CONSTRUCTOR = -6 - DISPID_DESTRUCTOR = -7 - DISPID_COLLECT = -8 -) - -const ( - MONITOR_DEFAULTTONULL = 0x00000000 - MONITOR_DEFAULTTOPRIMARY = 0x00000001 - MONITOR_DEFAULTTONEAREST = 0x00000002 - - MONITORINFOF_PRIMARY = 0x00000001 -) - -const ( - CCHDEVICENAME = 32 - CCHFORMNAME = 32 -) - -const ( - IDOK = 1 - IDCANCEL = 2 - IDABORT = 3 - IDRETRY = 4 - IDIGNORE = 5 - IDYES = 6 - IDNO = 7 - IDCLOSE = 8 - IDHELP = 9 - IDTRYAGAIN = 10 - IDCONTINUE = 11 - IDTIMEOUT = 32000 -) - -// Generic WM_NOTIFY notification codes -const ( - NM_FIRST = 0 - NM_OUTOFMEMORY = NM_FIRST - 1 - NM_CLICK = NM_FIRST - 2 - NM_DBLCLK = NM_FIRST - 3 - NM_RETURN = NM_FIRST - 4 - NM_RCLICK = NM_FIRST - 5 - NM_RDBLCLK = NM_FIRST - 6 - NM_SETFOCUS = NM_FIRST - 7 - NM_KILLFOCUS = NM_FIRST - 8 - NM_CUSTOMDRAW = NM_FIRST - 12 - NM_HOVER = NM_FIRST - 13 - NM_NCHITTEST = NM_FIRST - 14 - NM_KEYDOWN = NM_FIRST - 15 - NM_RELEASEDCAPTURE = NM_FIRST - 16 - NM_SETCURSOR = NM_FIRST - 17 - NM_CHAR = NM_FIRST - 18 - NM_TOOLTIPSCREATED = NM_FIRST - 19 - NM_LAST = NM_FIRST - 99 -) - -// ListView messages -const ( - LVM_FIRST = 0x1000 - LVM_GETITEMCOUNT = LVM_FIRST + 4 - LVM_SETIMAGELIST = LVM_FIRST + 3 - LVM_GETIMAGELIST = LVM_FIRST + 2 - LVM_GETITEM = LVM_FIRST + 75 - LVM_SETITEM = LVM_FIRST + 76 - LVM_INSERTITEM = LVM_FIRST + 77 - LVM_DELETEITEM = LVM_FIRST + 8 - LVM_DELETEALLITEMS = LVM_FIRST + 9 - LVM_GETCALLBACKMASK = LVM_FIRST + 10 - LVM_SETCALLBACKMASK = LVM_FIRST + 11 - LVM_SETUNICODEFORMAT = CCM_SETUNICODEFORMAT - LVM_GETNEXTITEM = LVM_FIRST + 12 - LVM_FINDITEM = LVM_FIRST + 83 - LVM_GETITEMRECT = LVM_FIRST + 14 - LVM_GETSTRINGWIDTH = LVM_FIRST + 87 - LVM_HITTEST = LVM_FIRST + 18 - LVM_ENSUREVISIBLE = LVM_FIRST + 19 - LVM_SCROLL = LVM_FIRST + 20 - LVM_REDRAWITEMS = LVM_FIRST + 21 - LVM_ARRANGE = LVM_FIRST + 22 - LVM_EDITLABEL = LVM_FIRST + 118 - LVM_GETEDITCONTROL = LVM_FIRST + 24 - LVM_GETCOLUMN = LVM_FIRST + 95 - LVM_SETCOLUMN = LVM_FIRST + 96 - LVM_INSERTCOLUMN = LVM_FIRST + 97 - LVM_DELETECOLUMN = LVM_FIRST + 28 - LVM_GETCOLUMNWIDTH = LVM_FIRST + 29 - LVM_SETCOLUMNWIDTH = LVM_FIRST + 30 - LVM_GETHEADER = LVM_FIRST + 31 - LVM_CREATEDRAGIMAGE = LVM_FIRST + 33 - LVM_GETVIEWRECT = LVM_FIRST + 34 - LVM_GETTEXTCOLOR = LVM_FIRST + 35 - LVM_SETTEXTCOLOR = LVM_FIRST + 36 - LVM_GETTEXTBKCOLOR = LVM_FIRST + 37 - LVM_SETTEXTBKCOLOR = LVM_FIRST + 38 - LVM_GETTOPINDEX = LVM_FIRST + 39 - LVM_GETCOUNTPERPAGE = LVM_FIRST + 40 - LVM_GETORIGIN = LVM_FIRST + 41 - LVM_UPDATE = LVM_FIRST + 42 - LVM_SETITEMSTATE = LVM_FIRST + 43 - LVM_GETITEMSTATE = LVM_FIRST + 44 - LVM_GETITEMTEXT = LVM_FIRST + 115 - LVM_SETITEMTEXT = LVM_FIRST + 116 - LVM_SETITEMCOUNT = LVM_FIRST + 47 - LVM_SORTITEMS = LVM_FIRST + 48 - LVM_SETITEMPOSITION32 = LVM_FIRST + 49 - LVM_GETSELECTEDCOUNT = LVM_FIRST + 50 - LVM_GETITEMSPACING = LVM_FIRST + 51 - LVM_GETISEARCHSTRING = LVM_FIRST + 117 - LVM_SETICONSPACING = LVM_FIRST + 53 - LVM_SETEXTENDEDLISTVIEWSTYLE = LVM_FIRST + 54 - LVM_GETEXTENDEDLISTVIEWSTYLE = LVM_FIRST + 55 - LVM_GETSUBITEMRECT = LVM_FIRST + 56 - LVM_SUBITEMHITTEST = LVM_FIRST + 57 - LVM_SETCOLUMNORDERARRAY = LVM_FIRST + 58 - LVM_GETCOLUMNORDERARRAY = LVM_FIRST + 59 - LVM_SETHOTITEM = LVM_FIRST + 60 - LVM_GETHOTITEM = LVM_FIRST + 61 - LVM_SETHOTCURSOR = LVM_FIRST + 62 - LVM_GETHOTCURSOR = LVM_FIRST + 63 - LVM_APPROXIMATEVIEWRECT = LVM_FIRST + 64 - LVM_SETWORKAREAS = LVM_FIRST + 65 - LVM_GETWORKAREAS = LVM_FIRST + 70 - LVM_GETNUMBEROFWORKAREAS = LVM_FIRST + 73 - LVM_GETSELECTIONMARK = LVM_FIRST + 66 - LVM_SETSELECTIONMARK = LVM_FIRST + 67 - LVM_SETHOVERTIME = LVM_FIRST + 71 - LVM_GETHOVERTIME = LVM_FIRST + 72 - LVM_SETTOOLTIPS = LVM_FIRST + 74 - LVM_GETTOOLTIPS = LVM_FIRST + 78 - LVM_SORTITEMSEX = LVM_FIRST + 81 - LVM_SETBKIMAGE = LVM_FIRST + 138 - LVM_GETBKIMAGE = LVM_FIRST + 139 - LVM_SETSELECTEDCOLUMN = LVM_FIRST + 140 - LVM_SETVIEW = LVM_FIRST + 142 - LVM_GETVIEW = LVM_FIRST + 143 - LVM_INSERTGROUP = LVM_FIRST + 145 - LVM_SETGROUPINFO = LVM_FIRST + 147 - LVM_GETGROUPINFO = LVM_FIRST + 149 - LVM_REMOVEGROUP = LVM_FIRST + 150 - LVM_MOVEGROUP = LVM_FIRST + 151 - LVM_GETGROUPCOUNT = LVM_FIRST + 152 - LVM_GETGROUPINFOBYINDEX = LVM_FIRST + 153 - LVM_MOVEITEMTOGROUP = LVM_FIRST + 154 - LVM_GETGROUPRECT = LVM_FIRST + 98 - LVM_SETGROUPMETRICS = LVM_FIRST + 155 - LVM_GETGROUPMETRICS = LVM_FIRST + 156 - LVM_ENABLEGROUPVIEW = LVM_FIRST + 157 - LVM_SORTGROUPS = LVM_FIRST + 158 - LVM_INSERTGROUPSORTED = LVM_FIRST + 159 - LVM_REMOVEALLGROUPS = LVM_FIRST + 160 - LVM_HASGROUP = LVM_FIRST + 161 - LVM_GETGROUPSTATE = LVM_FIRST + 92 - LVM_GETFOCUSEDGROUP = LVM_FIRST + 93 - LVM_SETTILEVIEWINFO = LVM_FIRST + 162 - LVM_GETTILEVIEWINFO = LVM_FIRST + 163 - LVM_SETTILEINFO = LVM_FIRST + 164 - LVM_GETTILEINFO = LVM_FIRST + 165 - LVM_SETINSERTMARK = LVM_FIRST + 166 - LVM_GETINSERTMARK = LVM_FIRST + 167 - LVM_INSERTMARKHITTEST = LVM_FIRST + 168 - LVM_GETINSERTMARKRECT = LVM_FIRST + 169 - LVM_SETINSERTMARKCOLOR = LVM_FIRST + 170 - LVM_GETINSERTMARKCOLOR = LVM_FIRST + 171 - LVM_SETINFOTIP = LVM_FIRST + 173 - LVM_GETSELECTEDCOLUMN = LVM_FIRST + 174 - LVM_ISGROUPVIEWENABLED = LVM_FIRST + 175 - LVM_GETOUTLINECOLOR = LVM_FIRST + 176 - LVM_SETOUTLINECOLOR = LVM_FIRST + 177 - LVM_CANCELEDITLABEL = LVM_FIRST + 179 - LVM_MAPINDEXTOID = LVM_FIRST + 180 - LVM_MAPIDTOINDEX = LVM_FIRST + 181 - LVM_ISITEMVISIBLE = LVM_FIRST + 182 - LVM_GETNEXTITEMINDEX = LVM_FIRST + 211 -) - -// ListView notifications -const ( - LVN_FIRST = -100 - - LVN_ITEMCHANGING = LVN_FIRST - 0 - LVN_ITEMCHANGED = LVN_FIRST - 1 - LVN_INSERTITEM = LVN_FIRST - 2 - LVN_DELETEITEM = LVN_FIRST - 3 - LVN_DELETEALLITEMS = LVN_FIRST - 4 - LVN_BEGINLABELEDITA = LVN_FIRST - 5 - LVN_BEGINLABELEDITW = LVN_FIRST - 75 - LVN_ENDLABELEDITA = LVN_FIRST - 6 - LVN_ENDLABELEDITW = LVN_FIRST - 76 - LVN_COLUMNCLICK = LVN_FIRST - 8 - LVN_BEGINDRAG = LVN_FIRST - 9 - LVN_BEGINRDRAG = LVN_FIRST - 11 - LVN_ODCACHEHINT = LVN_FIRST - 13 - LVN_ODFINDITEMA = LVN_FIRST - 52 - LVN_ODFINDITEMW = LVN_FIRST - 79 - LVN_ITEMACTIVATE = LVN_FIRST - 14 - LVN_ODSTATECHANGED = LVN_FIRST - 15 - LVN_HOTTRACK = LVN_FIRST - 21 - LVN_GETDISPINFO = LVN_FIRST - 77 - LVN_SETDISPINFO = LVN_FIRST - 78 - LVN_KEYDOWN = LVN_FIRST - 55 - LVN_MARQUEEBEGIN = LVN_FIRST - 56 - LVN_GETINFOTIP = LVN_FIRST - 58 - LVN_INCREMENTALSEARCH = LVN_FIRST - 63 - LVN_BEGINSCROLL = LVN_FIRST - 80 - LVN_ENDSCROLL = LVN_FIRST - 81 -) - -// ListView LVNI constants -const ( - LVNI_ALL = 0 - LVNI_FOCUSED = 1 - LVNI_SELECTED = 2 - LVNI_CUT = 4 - LVNI_DROPHILITED = 8 - LVNI_ABOVE = 256 - LVNI_BELOW = 512 - LVNI_TOLEFT = 1024 - LVNI_TORIGHT = 2048 -) - -// ListView styles -const ( - LVS_ICON = 0x0000 - LVS_REPORT = 0x0001 - LVS_SMALLICON = 0x0002 - LVS_LIST = 0x0003 - LVS_TYPEMASK = 0x0003 - LVS_SINGLESEL = 0x0004 - LVS_SHOWSELALWAYS = 0x0008 - LVS_SORTASCENDING = 0x0010 - LVS_SORTDESCENDING = 0x0020 - LVS_SHAREIMAGELISTS = 0x0040 - LVS_NOLABELWRAP = 0x0080 - LVS_AUTOARRANGE = 0x0100 - LVS_EDITLABELS = 0x0200 - LVS_OWNERDATA = 0x1000 - LVS_NOSCROLL = 0x2000 - LVS_TYPESTYLEMASK = 0xfc00 - LVS_ALIGNTOP = 0x0000 - LVS_ALIGNLEFT = 0x0800 - LVS_ALIGNMASK = 0x0c00 - LVS_OWNERDRAWFIXED = 0x0400 - LVS_NOCOLUMNHEADER = 0x4000 - LVS_NOSORTHEADER = 0x8000 -) - -// ListView extended styles -const ( - LVS_EX_GRIDLINES = 0x00000001 - LVS_EX_SUBITEMIMAGES = 0x00000002 - LVS_EX_CHECKBOXES = 0x00000004 - LVS_EX_TRACKSELECT = 0x00000008 - LVS_EX_HEADERDRAGDROP = 0x00000010 - LVS_EX_FULLROWSELECT = 0x00000020 - LVS_EX_ONECLICKACTIVATE = 0x00000040 - LVS_EX_TWOCLICKACTIVATE = 0x00000080 - LVS_EX_FLATSB = 0x00000100 - LVS_EX_REGIONAL = 0x00000200 - LVS_EX_INFOTIP = 0x00000400 - LVS_EX_UNDERLINEHOT = 0x00000800 - LVS_EX_UNDERLINECOLD = 0x00001000 - LVS_EX_MULTIWORKAREAS = 0x00002000 - LVS_EX_LABELTIP = 0x00004000 - LVS_EX_BORDERSELECT = 0x00008000 - LVS_EX_DOUBLEBUFFER = 0x00010000 - LVS_EX_HIDELABELS = 0x00020000 - LVS_EX_SINGLEROW = 0x00040000 - LVS_EX_SNAPTOGRID = 0x00080000 - LVS_EX_SIMPLESELECT = 0x00100000 -) - -// ListView column flags -const ( - LVCF_FMT = 0x0001 - LVCF_WIDTH = 0x0002 - LVCF_TEXT = 0x0004 - LVCF_SUBITEM = 0x0008 - LVCF_IMAGE = 0x0010 - LVCF_ORDER = 0x0020 -) - -// ListView column format constants -const ( - LVCFMT_LEFT = 0x0000 - LVCFMT_RIGHT = 0x0001 - LVCFMT_CENTER = 0x0002 - LVCFMT_JUSTIFYMASK = 0x0003 - LVCFMT_IMAGE = 0x0800 - LVCFMT_BITMAP_ON_RIGHT = 0x1000 - LVCFMT_COL_HAS_IMAGES = 0x8000 -) - -// ListView item flags -const ( - LVIF_TEXT = 0x00000001 - LVIF_IMAGE = 0x00000002 - LVIF_PARAM = 0x00000004 - LVIF_STATE = 0x00000008 - LVIF_INDENT = 0x00000010 - LVIF_NORECOMPUTE = 0x00000800 - LVIF_GROUPID = 0x00000100 - LVIF_COLUMNS = 0x00000200 -) - -// ListView item states -const ( - LVIS_FOCUSED = 1 - LVIS_SELECTED = 2 - LVIS_CUT = 4 - LVIS_DROPHILITED = 8 - LVIS_OVERLAYMASK = 0xF00 - LVIS_STATEIMAGEMASK = 0xF000 -) - -// ListView hit test constants -const ( - LVHT_NOWHERE = 0x00000001 - LVHT_ONITEMICON = 0x00000002 - LVHT_ONITEMLABEL = 0x00000004 - LVHT_ONITEMSTATEICON = 0x00000008 - LVHT_ONITEM = LVHT_ONITEMICON | LVHT_ONITEMLABEL | LVHT_ONITEMSTATEICON - - LVHT_ABOVE = 0x00000008 - LVHT_BELOW = 0x00000010 - LVHT_TORIGHT = 0x00000020 - LVHT_TOLEFT = 0x00000040 -) - -// ListView image list types -const ( - LVSIL_NORMAL = 0 - LVSIL_SMALL = 1 - LVSIL_STATE = 2 - LVSIL_GROUPHEADER = 3 -) - -// InitCommonControlsEx flags -const ( - ICC_LISTVIEW_CLASSES = 1 - ICC_TREEVIEW_CLASSES = 2 - ICC_BAR_CLASSES = 4 - ICC_TAB_CLASSES = 8 - ICC_UPDOWN_CLASS = 16 - ICC_PROGRESS_CLASS = 32 - ICC_HOTKEY_CLASS = 64 - ICC_ANIMATE_CLASS = 128 - ICC_WIN95_CLASSES = 255 - ICC_DATE_CLASSES = 256 - ICC_USEREX_CLASSES = 512 - ICC_COOL_CLASSES = 1024 - ICC_INTERNET_CLASSES = 2048 - ICC_PAGESCROLLER_CLASS = 4096 - ICC_NATIVEFNTCTL_CLASS = 8192 - INFOTIPSIZE = 1024 - ICC_STANDARD_CLASSES = 0x00004000 - ICC_LINK_CLASS = 0x00008000 -) - -// Dialog Codes -const ( - DLGC_WANTARROWS = 0x0001 - DLGC_WANTTAB = 0x0002 - DLGC_WANTALLKEYS = 0x0004 - DLGC_WANTMESSAGE = 0x0004 - DLGC_HASSETSEL = 0x0008 - DLGC_DEFPUSHBUTTON = 0x0010 - DLGC_UNDEFPUSHBUTTON = 0x0020 - DLGC_RADIOBUTTON = 0x0040 - DLGC_WANTCHARS = 0x0080 - DLGC_STATIC = 0x0100 - DLGC_BUTTON = 0x2000 -) - -// Get/SetWindowWord/Long offsets for use with WC_DIALOG windows -const ( - DWL_MSGRESULT = 0 - DWL_DLGPROC = 4 - DWL_USER = 8 -) - -// Registry predefined keys -const ( - HKEY_CLASSES_ROOT HKEY = 0x80000000 - HKEY_CURRENT_USER HKEY = 0x80000001 - HKEY_LOCAL_MACHINE HKEY = 0x80000002 - HKEY_USERS HKEY = 0x80000003 - HKEY_PERFORMANCE_DATA HKEY = 0x80000004 - HKEY_CURRENT_CONFIG HKEY = 0x80000005 - HKEY_DYN_DATA HKEY = 0x80000006 -) - -// Registry Key Security and Access Rights -const ( - KEY_ALL_ACCESS = 0xF003F - KEY_CREATE_SUB_KEY = 0x0004 - KEY_ENUMERATE_SUB_KEYS = 0x0008 - KEY_NOTIFY = 0x0010 - KEY_QUERY_VALUE = 0x0001 - KEY_SET_VALUE = 0x0002 - KEY_READ = 0x20019 - KEY_WRITE = 0x20006 -) - -const ( - NFR_ANSI = 1 - NFR_UNICODE = 2 - NF_QUERY = 3 - NF_REQUERY = 4 -) - -// Registry value types -const ( - RRF_RT_REG_NONE = 0x00000001 - RRF_RT_REG_SZ = 0x00000002 - RRF_RT_REG_EXPAND_SZ = 0x00000004 - RRF_RT_REG_BINARY = 0x00000008 - RRF_RT_REG_DWORD = 0x00000010 - RRF_RT_REG_MULTI_SZ = 0x00000020 - RRF_RT_REG_QWORD = 0x00000040 - RRF_RT_DWORD = (RRF_RT_REG_BINARY | RRF_RT_REG_DWORD) - RRF_RT_QWORD = (RRF_RT_REG_BINARY | RRF_RT_REG_QWORD) - RRF_RT_ANY = 0x0000ffff - RRF_NOEXPAND = 0x10000000 - RRF_ZEROONFAILURE = 0x20000000 - REG_PROCESS_APPKEY = 0x00000001 - REG_MUI_STRING_TRUNCATE = 0x00000001 -) - -// PeekMessage wRemoveMsg value -const ( - PM_NOREMOVE = 0x000 - PM_REMOVE = 0x001 - PM_NOYIELD = 0x002 -) - -// ImageList flags -const ( - ILC_MASK = 0x00000001 - ILC_COLOR = 0x00000000 - ILC_COLORDDB = 0x000000FE - ILC_COLOR4 = 0x00000004 - ILC_COLOR8 = 0x00000008 - ILC_COLOR16 = 0x00000010 - ILC_COLOR24 = 0x00000018 - ILC_COLOR32 = 0x00000020 - ILC_PALETTE = 0x00000800 - ILC_MIRROR = 0x00002000 - ILC_PERITEMMIRROR = 0x00008000 - ILC_ORIGINALSIZE = 0x00010000 - ILC_HIGHQUALITYSCALE = 0x00020000 -) - -// Keystroke Message Flags -const ( - KF_EXTENDED = 0x0100 - KF_DLGMODE = 0x0800 - KF_MENUMODE = 0x1000 - KF_ALTDOWN = 0x2000 - KF_REPEAT = 0x4000 - KF_UP = 0x8000 -) - -// Virtual-Key Codes -const ( - VK_LBUTTON = 0x01 - VK_RBUTTON = 0x02 - VK_CANCEL = 0x03 - VK_MBUTTON = 0x04 - VK_XBUTTON1 = 0x05 - VK_XBUTTON2 = 0x06 - VK_BACK = 0x08 - VK_TAB = 0x09 - VK_CLEAR = 0x0C - VK_RETURN = 0x0D - VK_SHIFT = 0x10 - VK_CONTROL = 0x11 - VK_MENU = 0x12 - VK_PAUSE = 0x13 - VK_CAPITAL = 0x14 - VK_KANA = 0x15 - VK_HANGEUL = 0x15 - VK_HANGUL = 0x15 - VK_JUNJA = 0x17 - VK_FINAL = 0x18 - VK_HANJA = 0x19 - VK_KANJI = 0x19 - VK_ESCAPE = 0x1B - VK_CONVERT = 0x1C - VK_NONCONVERT = 0x1D - VK_ACCEPT = 0x1E - VK_MODECHANGE = 0x1F - VK_SPACE = 0x20 - VK_PRIOR = 0x21 - VK_NEXT = 0x22 - VK_END = 0x23 - VK_HOME = 0x24 - VK_LEFT = 0x25 - VK_UP = 0x26 - VK_RIGHT = 0x27 - VK_DOWN = 0x28 - VK_SELECT = 0x29 - VK_PRINT = 0x2A - VK_EXECUTE = 0x2B - VK_SNAPSHOT = 0x2C - VK_INSERT = 0x2D - VK_DELETE = 0x2E - VK_HELP = 0x2F - VK_LWIN = 0x5B - VK_RWIN = 0x5C - VK_APPS = 0x5D - VK_SLEEP = 0x5F - VK_NUMPAD0 = 0x60 - VK_NUMPAD1 = 0x61 - VK_NUMPAD2 = 0x62 - VK_NUMPAD3 = 0x63 - VK_NUMPAD4 = 0x64 - VK_NUMPAD5 = 0x65 - VK_NUMPAD6 = 0x66 - VK_NUMPAD7 = 0x67 - VK_NUMPAD8 = 0x68 - VK_NUMPAD9 = 0x69 - VK_MULTIPLY = 0x6A - VK_ADD = 0x6B - VK_SEPARATOR = 0x6C - VK_SUBTRACT = 0x6D - VK_DECIMAL = 0x6E - VK_DIVIDE = 0x6F - VK_F1 = 0x70 - VK_F2 = 0x71 - VK_F3 = 0x72 - VK_F4 = 0x73 - VK_F5 = 0x74 - VK_F6 = 0x75 - VK_F7 = 0x76 - VK_F8 = 0x77 - VK_F9 = 0x78 - VK_F10 = 0x79 - VK_F11 = 0x7A - VK_F12 = 0x7B - VK_F13 = 0x7C - VK_F14 = 0x7D - VK_F15 = 0x7E - VK_F16 = 0x7F - VK_F17 = 0x80 - VK_F18 = 0x81 - VK_F19 = 0x82 - VK_F20 = 0x83 - VK_F21 = 0x84 - VK_F22 = 0x85 - VK_F23 = 0x86 - VK_F24 = 0x87 - VK_NUMLOCK = 0x90 - VK_SCROLL = 0x91 - VK_OEM_NEC_EQUAL = 0x92 - VK_OEM_FJ_JISHO = 0x92 - VK_OEM_FJ_MASSHOU = 0x93 - VK_OEM_FJ_TOUROKU = 0x94 - VK_OEM_FJ_LOYA = 0x95 - VK_OEM_FJ_ROYA = 0x96 - VK_LSHIFT = 0xA0 - VK_RSHIFT = 0xA1 - VK_LCONTROL = 0xA2 - VK_RCONTROL = 0xA3 - VK_LMENU = 0xA4 - VK_RMENU = 0xA5 - VK_BROWSER_BACK = 0xA6 - VK_BROWSER_FORWARD = 0xA7 - VK_BROWSER_REFRESH = 0xA8 - VK_BROWSER_STOP = 0xA9 - VK_BROWSER_SEARCH = 0xAA - VK_BROWSER_FAVORITES = 0xAB - VK_BROWSER_HOME = 0xAC - VK_VOLUME_MUTE = 0xAD - VK_VOLUME_DOWN = 0xAE - VK_VOLUME_UP = 0xAF - VK_MEDIA_NEXT_TRACK = 0xB0 - VK_MEDIA_PREV_TRACK = 0xB1 - VK_MEDIA_STOP = 0xB2 - VK_MEDIA_PLAY_PAUSE = 0xB3 - VK_LAUNCH_MAIL = 0xB4 - VK_LAUNCH_MEDIA_SELECT = 0xB5 - VK_LAUNCH_APP1 = 0xB6 - VK_LAUNCH_APP2 = 0xB7 - VK_OEM_1 = 0xBA - VK_OEM_PLUS = 0xBB - VK_OEM_COMMA = 0xBC - VK_OEM_MINUS = 0xBD - VK_OEM_PERIOD = 0xBE - VK_OEM_2 = 0xBF - VK_OEM_3 = 0xC0 - VK_OEM_4 = 0xDB - VK_OEM_5 = 0xDC - VK_OEM_6 = 0xDD - VK_OEM_7 = 0xDE - VK_OEM_8 = 0xDF - VK_OEM_AX = 0xE1 - VK_OEM_102 = 0xE2 - VK_ICO_HELP = 0xE3 - VK_ICO_00 = 0xE4 - VK_PROCESSKEY = 0xE5 - VK_ICO_CLEAR = 0xE6 - VK_OEM_RESET = 0xE9 - VK_OEM_JUMP = 0xEA - VK_OEM_PA1 = 0xEB - VK_OEM_PA2 = 0xEC - VK_OEM_PA3 = 0xED - VK_OEM_WSCTRL = 0xEE - VK_OEM_CUSEL = 0xEF - VK_OEM_ATTN = 0xF0 - VK_OEM_FINISH = 0xF1 - VK_OEM_COPY = 0xF2 - VK_OEM_AUTO = 0xF3 - VK_OEM_ENLW = 0xF4 - VK_OEM_BACKTAB = 0xF5 - VK_ATTN = 0xF6 - VK_CRSEL = 0xF7 - VK_EXSEL = 0xF8 - VK_EREOF = 0xF9 - VK_PLAY = 0xFA - VK_ZOOM = 0xFB - VK_NONAME = 0xFC - VK_PA1 = 0xFD - VK_OEM_CLEAR = 0xFE -) - -// Registry Value Types -const ( - REG_NONE = 0 - REG_SZ = 1 - REG_EXPAND_SZ = 2 - REG_BINARY = 3 - REG_DWORD = 4 - REG_DWORD_LITTLE_ENDIAN = 4 - REG_DWORD_BIG_ENDIAN = 5 - REG_LINK = 6 - REG_MULTI_SZ = 7 - REG_RESOURCE_LIST = 8 - REG_FULL_RESOURCE_DESCRIPTOR = 9 - REG_RESOURCE_REQUIREMENTS_LIST = 10 - REG_QWORD = 11 - REG_QWORD_LITTLE_ENDIAN = 11 -) - -// Tooltip styles -const ( - TTS_ALWAYSTIP = 0x01 - TTS_NOPREFIX = 0x02 - TTS_NOANIMATE = 0x10 - TTS_NOFADE = 0x20 - TTS_BALLOON = 0x40 - TTS_CLOSE = 0x80 - TTS_USEVISUALSTYLE = 0x100 -) - -// Tooltip messages -const ( - TTM_ACTIVATE = (WM_USER + 1) - TTM_SETDELAYTIME = (WM_USER + 3) - TTM_ADDTOOL = (WM_USER + 50) - TTM_DELTOOL = (WM_USER + 51) - TTM_NEWTOOLRECT = (WM_USER + 52) - TTM_RELAYEVENT = (WM_USER + 7) - TTM_GETTOOLINFO = (WM_USER + 53) - TTM_SETTOOLINFO = (WM_USER + 54) - TTM_HITTEST = (WM_USER + 55) - TTM_GETTEXT = (WM_USER + 56) - TTM_UPDATETIPTEXT = (WM_USER + 57) - TTM_GETTOOLCOUNT = (WM_USER + 13) - TTM_ENUMTOOLS = (WM_USER + 58) - TTM_GETCURRENTTOOL = (WM_USER + 59) - TTM_WINDOWFROMPOINT = (WM_USER + 16) - TTM_TRACKACTIVATE = (WM_USER + 17) - TTM_TRACKPOSITION = (WM_USER + 18) - TTM_SETTIPBKCOLOR = (WM_USER + 19) - TTM_SETTIPTEXTCOLOR = (WM_USER + 20) - TTM_GETDELAYTIME = (WM_USER + 21) - TTM_GETTIPBKCOLOR = (WM_USER + 22) - TTM_GETTIPTEXTCOLOR = (WM_USER + 23) - TTM_SETMAXTIPWIDTH = (WM_USER + 24) - TTM_GETMAXTIPWIDTH = (WM_USER + 25) - TTM_SETMARGIN = (WM_USER + 26) - TTM_GETMARGIN = (WM_USER + 27) - TTM_POP = (WM_USER + 28) - TTM_UPDATE = (WM_USER + 29) - TTM_GETBUBBLESIZE = (WM_USER + 30) - TTM_ADJUSTRECT = (WM_USER + 31) - TTM_SETTITLE = (WM_USER + 33) - TTM_POPUP = (WM_USER + 34) - TTM_GETTITLE = (WM_USER + 35) -) - -// Tooltip icons -const ( - TTI_NONE = 0 - TTI_INFO = 1 - TTI_WARNING = 2 - TTI_ERROR = 3 - TTI_INFO_LARGE = 4 - TTI_WARNING_LARGE = 5 - TTI_ERROR_LARGE = 6 -) - -// Tooltip notifications -const ( - TTN_FIRST = -520 - TTN_LAST = -549 - TTN_GETDISPINFO = (TTN_FIRST - 10) - TTN_SHOW = (TTN_FIRST - 1) - TTN_POP = (TTN_FIRST - 2) - TTN_LINKCLICK = (TTN_FIRST - 3) - TTN_NEEDTEXT = TTN_GETDISPINFO -) - -const ( - TTF_IDISHWND = 0x0001 - TTF_CENTERTIP = 0x0002 - TTF_RTLREADING = 0x0004 - TTF_SUBCLASS = 0x0010 - TTF_TRACK = 0x0020 - TTF_ABSOLUTE = 0x0080 - TTF_TRANSPARENT = 0x0100 - TTF_PARSELINKS = 0x1000 - TTF_DI_SETITEM = 0x8000 -) - -const ( - SWP_NOSIZE = 0x0001 - SWP_NOMOVE = 0x0002 - SWP_NOZORDER = 0x0004 - SWP_NOREDRAW = 0x0008 - SWP_NOACTIVATE = 0x0010 - SWP_FRAMECHANGED = 0x0020 - SWP_SHOWWINDOW = 0x0040 - SWP_HIDEWINDOW = 0x0080 - SWP_NOCOPYBITS = 0x0100 - SWP_NOOWNERZORDER = 0x0200 - SWP_NOSENDCHANGING = 0x0400 - SWP_DRAWFRAME = SWP_FRAMECHANGED - SWP_NOREPOSITION = SWP_NOOWNERZORDER - SWP_DEFERERASE = 0x2000 - SWP_ASYNCWINDOWPOS = 0x4000 -) - -// Predefined window handles -const ( - HWND_BROADCAST = HWND(0xFFFF) - HWND_BOTTOM = HWND(1) - HWND_NOTOPMOST = ^HWND(1) // -2 - HWND_TOP = HWND(0) - HWND_TOPMOST = ^HWND(0) // -1 - HWND_DESKTOP = HWND(0) - HWND_MESSAGE = ^HWND(2) // -3 -) - -// Pen types -const ( - PS_COSMETIC = 0x00000000 - PS_GEOMETRIC = 0x00010000 - PS_TYPE_MASK = 0x000F0000 -) - -// Pen styles -const ( - PS_SOLID = 0 - PS_DASH = 1 - PS_DOT = 2 - PS_DASHDOT = 3 - PS_DASHDOTDOT = 4 - PS_NULL = 5 - PS_INSIDEFRAME = 6 - PS_USERSTYLE = 7 - PS_ALTERNATE = 8 - PS_STYLE_MASK = 0x0000000F -) - -// Pen cap types -const ( - PS_ENDCAP_ROUND = 0x00000000 - PS_ENDCAP_SQUARE = 0x00000100 - PS_ENDCAP_FLAT = 0x00000200 - PS_ENDCAP_MASK = 0x00000F00 -) - -// Pen join types -const ( - PS_JOIN_ROUND = 0x00000000 - PS_JOIN_BEVEL = 0x00001000 - PS_JOIN_MITER = 0x00002000 - PS_JOIN_MASK = 0x0000F000 -) - -// Hatch styles -const ( - HS_HORIZONTAL = 0 - HS_VERTICAL = 1 - HS_FDIAGONAL = 2 - HS_BDIAGONAL = 3 - HS_CROSS = 4 - HS_DIAGCROSS = 5 -) - -// Stock Logical Objects -const ( - WHITE_BRUSH = 0 - LTGRAY_BRUSH = 1 - GRAY_BRUSH = 2 - DKGRAY_BRUSH = 3 - BLACK_BRUSH = 4 - NULL_BRUSH = 5 - HOLLOW_BRUSH = NULL_BRUSH - WHITE_PEN = 6 - BLACK_PEN = 7 - NULL_PEN = 8 - OEM_FIXED_FONT = 10 - ANSI_FIXED_FONT = 11 - ANSI_VAR_FONT = 12 - SYSTEM_FONT = 13 - DEVICE_DEFAULT_FONT = 14 - DEFAULT_PALETTE = 15 - SYSTEM_FIXED_FONT = 16 - DEFAULT_GUI_FONT = 17 - DC_BRUSH = 18 - DC_PEN = 19 -) - -// Brush styles -const ( - BS_SOLID = 0 - BS_NULL = 1 - BS_HOLLOW = BS_NULL - BS_HATCHED = 2 - BS_PATTERN = 3 - BS_INDEXED = 4 - BS_DIBPATTERN = 5 - BS_DIBPATTERNPT = 6 - BS_PATTERN8X8 = 7 - BS_DIBPATTERN8X8 = 8 - BS_MONOPATTERN = 9 -) - -// TRACKMOUSEEVENT flags -const ( - TME_HOVER = 0x00000001 - TME_LEAVE = 0x00000002 - TME_NONCLIENT = 0x00000010 - TME_QUERY = 0x40000000 - TME_CANCEL = 0x80000000 - - HOVER_DEFAULT = 0xFFFFFFFF -) - -// WM_NCHITTEST and MOUSEHOOKSTRUCT Mouse Position Codes -const ( - HTERROR = (-2) - HTTRANSPARENT = (-1) - HTNOWHERE = 0 - HTCLIENT = 1 - HTCAPTION = 2 - HTSYSMENU = 3 - HTGROWBOX = 4 - HTSIZE = HTGROWBOX - HTMENU = 5 - HTHSCROLL = 6 - HTVSCROLL = 7 - HTMINBUTTON = 8 - HTMAXBUTTON = 9 - HTLEFT = 10 - HTRIGHT = 11 - HTTOP = 12 - HTTOPLEFT = 13 - HTTOPRIGHT = 14 - HTBOTTOM = 15 - HTBOTTOMLEFT = 16 - HTBOTTOMRIGHT = 17 - HTBORDER = 18 - HTREDUCE = HTMINBUTTON - HTZOOM = HTMAXBUTTON - HTSIZEFIRST = HTLEFT - HTSIZELAST = HTBOTTOMRIGHT - HTOBJECT = 19 - HTCLOSE = 20 - HTHELP = 21 -) - -// DrawText[Ex] format flags -const ( - DT_TOP = 0x00000000 - DT_LEFT = 0x00000000 - DT_CENTER = 0x00000001 - DT_RIGHT = 0x00000002 - DT_VCENTER = 0x00000004 - DT_BOTTOM = 0x00000008 - DT_WORDBREAK = 0x00000010 - DT_SINGLELINE = 0x00000020 - DT_EXPANDTABS = 0x00000040 - DT_TABSTOP = 0x00000080 - DT_NOCLIP = 0x00000100 - DT_EXTERNALLEADING = 0x00000200 - DT_CALCRECT = 0x00000400 - DT_NOPREFIX = 0x00000800 - DT_INTERNAL = 0x00001000 - DT_EDITCONTROL = 0x00002000 - DT_PATH_ELLIPSIS = 0x00004000 - DT_END_ELLIPSIS = 0x00008000 - DT_MODIFYSTRING = 0x00010000 - DT_RTLREADING = 0x00020000 - DT_WORD_ELLIPSIS = 0x00040000 - DT_NOFULLWIDTHCHARBREAK = 0x00080000 - DT_HIDEPREFIX = 0x00100000 - DT_PREFIXONLY = 0x00200000 -) - -const CLR_INVALID = 0xFFFFFFFF - -// Background Modes -const ( - TRANSPARENT = 1 - OPAQUE = 2 - BKMODE_LAST = 2 -) - -// Global Memory Flags -const ( - GMEM_FIXED = 0x0000 - GMEM_MOVEABLE = 0x0002 - GMEM_NOCOMPACT = 0x0010 - GMEM_NODISCARD = 0x0020 - GMEM_ZEROINIT = 0x0040 - GMEM_MODIFY = 0x0080 - GMEM_DISCARDABLE = 0x0100 - GMEM_NOT_BANKED = 0x1000 - GMEM_SHARE = 0x2000 - GMEM_DDESHARE = 0x2000 - GMEM_NOTIFY = 0x4000 - GMEM_LOWER = GMEM_NOT_BANKED - GMEM_VALID_FLAGS = 0x7F72 - GMEM_INVALID_HANDLE = 0x8000 - GHND = (GMEM_MOVEABLE | GMEM_ZEROINIT) - GPTR = (GMEM_FIXED | GMEM_ZEROINIT) -) - -// Ternary raster operations -const ( - SRCCOPY = 0x00CC0020 - SRCPAINT = 0x00EE0086 - SRCAND = 0x008800C6 - SRCINVERT = 0x00660046 - SRCERASE = 0x00440328 - NOTSRCCOPY = 0x00330008 - NOTSRCERASE = 0x001100A6 - MERGECOPY = 0x00C000CA - MERGEPAINT = 0x00BB0226 - PATCOPY = 0x00F00021 - PATPAINT = 0x00FB0A09 - PATINVERT = 0x005A0049 - DSTINVERT = 0x00550009 - BLACKNESS = 0x00000042 - WHITENESS = 0x00FF0062 - NOMIRRORBITMAP = 0x80000000 - CAPTUREBLT = 0x40000000 -) - -// Clipboard formats -const ( - CF_TEXT = 1 - CF_BITMAP = 2 - CF_METAFILEPICT = 3 - CF_SYLK = 4 - CF_DIF = 5 - CF_TIFF = 6 - CF_OEMTEXT = 7 - CF_DIB = 8 - CF_PALETTE = 9 - CF_PENDATA = 10 - CF_RIFF = 11 - CF_WAVE = 12 - CF_UNICODETEXT = 13 - CF_ENHMETAFILE = 14 - CF_HDROP = 15 - CF_LOCALE = 16 - CF_DIBV5 = 17 - CF_MAX = 18 - CF_OWNERDISPLAY = 0x0080 - CF_DSPTEXT = 0x0081 - CF_DSPBITMAP = 0x0082 - CF_DSPMETAFILEPICT = 0x0083 - CF_DSPENHMETAFILE = 0x008E - CF_PRIVATEFIRST = 0x0200 - CF_PRIVATELAST = 0x02FF - CF_GDIOBJFIRST = 0x0300 - CF_GDIOBJLAST = 0x03FF -) - -// Bitmap compression formats -const ( - BI_RGB = 0 - BI_RLE8 = 1 - BI_RLE4 = 2 - BI_BITFIELDS = 3 - BI_JPEG = 4 - BI_PNG = 5 -) - -// SetDIBitsToDevice fuColorUse -const ( - DIB_PAL_COLORS = 1 - DIB_RGB_COLORS = 0 -) - -const ( - STANDARD_RIGHTS_REQUIRED = 0x000F -) - -// Service Control Manager object specific access types -const ( - SC_MANAGER_CONNECT = 0x0001 - SC_MANAGER_CREATE_SERVICE = 0x0002 - SC_MANAGER_ENUMERATE_SERVICE = 0x0004 - SC_MANAGER_LOCK = 0x0008 - SC_MANAGER_QUERY_LOCK_STATUS = 0x0010 - SC_MANAGER_MODIFY_BOOT_CONFIG = 0x0020 - SC_MANAGER_ALL_ACCESS = STANDARD_RIGHTS_REQUIRED | SC_MANAGER_CONNECT | SC_MANAGER_CREATE_SERVICE | SC_MANAGER_ENUMERATE_SERVICE | SC_MANAGER_LOCK | SC_MANAGER_QUERY_LOCK_STATUS | SC_MANAGER_MODIFY_BOOT_CONFIG -) - -// Service Types (Bit Mask) -const ( - SERVICE_KERNEL_DRIVER = 0x00000001 - SERVICE_FILE_SYSTEM_DRIVER = 0x00000002 - SERVICE_ADAPTER = 0x00000004 - SERVICE_RECOGNIZER_DRIVER = 0x00000008 - SERVICE_DRIVER = SERVICE_KERNEL_DRIVER | SERVICE_FILE_SYSTEM_DRIVER | SERVICE_RECOGNIZER_DRIVER - SERVICE_WIN32_OWN_PROCESS = 0x00000010 - SERVICE_WIN32_SHARE_PROCESS = 0x00000020 - SERVICE_WIN32 = SERVICE_WIN32_OWN_PROCESS | SERVICE_WIN32_SHARE_PROCESS - SERVICE_INTERACTIVE_PROCESS = 0x00000100 - SERVICE_TYPE_ALL = SERVICE_WIN32 | SERVICE_ADAPTER | SERVICE_DRIVER | SERVICE_INTERACTIVE_PROCESS -) - -// Service State -- for CurrentState -const ( - SERVICE_STOPPED = 0x00000001 - SERVICE_START_PENDING = 0x00000002 - SERVICE_STOP_PENDING = 0x00000003 - SERVICE_RUNNING = 0x00000004 - SERVICE_CONTINUE_PENDING = 0x00000005 - SERVICE_PAUSE_PENDING = 0x00000006 - SERVICE_PAUSED = 0x00000007 -) - -// Controls Accepted (Bit Mask) -const ( - SERVICE_ACCEPT_STOP = 0x00000001 - SERVICE_ACCEPT_PAUSE_CONTINUE = 0x00000002 - SERVICE_ACCEPT_SHUTDOWN = 0x00000004 - SERVICE_ACCEPT_PARAMCHANGE = 0x00000008 - SERVICE_ACCEPT_NETBINDCHANGE = 0x00000010 - SERVICE_ACCEPT_HARDWAREPROFILECHANGE = 0x00000020 - SERVICE_ACCEPT_POWEREVENT = 0x00000040 - SERVICE_ACCEPT_SESSIONCHANGE = 0x00000080 - SERVICE_ACCEPT_PRESHUTDOWN = 0x00000100 - SERVICE_ACCEPT_TIMECHANGE = 0x00000200 - SERVICE_ACCEPT_TRIGGEREVENT = 0x00000400 -) - -// Service object specific access type -const ( - SERVICE_QUERY_CONFIG = 0x0001 - SERVICE_CHANGE_CONFIG = 0x0002 - SERVICE_QUERY_STATUS = 0x0004 - SERVICE_ENUMERATE_DEPENDENTS = 0x0008 - SERVICE_START = 0x0010 - SERVICE_STOP = 0x0020 - SERVICE_PAUSE_CONTINUE = 0x0040 - SERVICE_INTERROGATE = 0x0080 - SERVICE_USER_DEFINED_CONTROL = 0x0100 - - SERVICE_ALL_ACCESS = STANDARD_RIGHTS_REQUIRED | - SERVICE_QUERY_CONFIG | - SERVICE_CHANGE_CONFIG | - SERVICE_QUERY_STATUS | - SERVICE_ENUMERATE_DEPENDENTS | - SERVICE_START | - SERVICE_STOP | - SERVICE_PAUSE_CONTINUE | - SERVICE_INTERROGATE | - SERVICE_USER_DEFINED_CONTROL -) - -// MapVirtualKey maptypes -const ( - MAPVK_VK_TO_CHAR = 2 - MAPVK_VK_TO_VSC = 0 - MAPVK_VSC_TO_VK = 1 - MAPVK_VSC_TO_VK_EX = 3 -) - -// ReadEventLog Flags -const ( - EVENTLOG_SEEK_READ = 0x0002 - EVENTLOG_SEQUENTIAL_READ = 0x0001 - EVENTLOG_FORWARDS_READ = 0x0004 - EVENTLOG_BACKWARDS_READ = 0x0008 -) - -// CreateToolhelp32Snapshot flags -const ( - TH32CS_SNAPHEAPLIST = 0x00000001 - TH32CS_SNAPPROCESS = 0x00000002 - TH32CS_SNAPTHREAD = 0x00000004 - TH32CS_SNAPMODULE = 0x00000008 - TH32CS_SNAPMODULE32 = 0x00000010 - TH32CS_INHERIT = 0x80000000 - TH32CS_SNAPALL = TH32CS_SNAPHEAPLIST | TH32CS_SNAPMODULE | TH32CS_SNAPPROCESS | TH32CS_SNAPTHREAD -) - -const ( - MAX_MODULE_NAME32 = 255 - MAX_PATH = 260 -) - -const ( - FOREGROUND_BLUE = 0x0001 - FOREGROUND_GREEN = 0x0002 - FOREGROUND_RED = 0x0004 - FOREGROUND_INTENSITY = 0x0008 - BACKGROUND_BLUE = 0x0010 - BACKGROUND_GREEN = 0x0020 - BACKGROUND_RED = 0x0040 - BACKGROUND_INTENSITY = 0x0080 - COMMON_LVB_LEADING_BYTE = 0x0100 - COMMON_LVB_TRAILING_BYTE = 0x0200 - COMMON_LVB_GRID_HORIZONTAL = 0x0400 - COMMON_LVB_GRID_LVERTICAL = 0x0800 - COMMON_LVB_GRID_RVERTICAL = 0x1000 - COMMON_LVB_REVERSE_VIDEO = 0x4000 - COMMON_LVB_UNDERSCORE = 0x8000 -) - -// Flags used by the DWM_BLURBEHIND structure to indicate -// which of its members contain valid information. -const ( - DWM_BB_ENABLE = 0x00000001 // A value for the fEnable member has been specified. - DWM_BB_BLURREGION = 0x00000002 // A value for the hRgnBlur member has been specified. - DWM_BB_TRANSITIONONMAXIMIZED = 0x00000004 // A value for the fTransitionOnMaximized member has been specified. -) - -// Flags used by the DwmEnableComposition function -// to change the state of Desktop Window Manager (DWM) composition. -const ( - DWM_EC_DISABLECOMPOSITION = 0 // Disable composition - DWM_EC_ENABLECOMPOSITION = 1 // Enable composition -) - -// enum-lite implementation for the following constant structure -type DWM_SHOWCONTACT int32 - -const ( - DWMSC_DOWN = 0x00000001 - DWMSC_UP = 0x00000002 - DWMSC_DRAG = 0x00000004 - DWMSC_HOLD = 0x00000008 - DWMSC_PENBARREL = 0x00000010 - DWMSC_NONE = 0x00000000 - DWMSC_ALL = 0xFFFFFFFF -) - -// enum-lite implementation for the following constant structure -type DWM_SOURCE_FRAME_SAMPLING int32 - -// TODO: need to verify this construction -// Flags used by the DwmSetPresentParameters function -// to specify the frame sampling type -const ( - DWM_SOURCE_FRAME_SAMPLING_POINT = iota + 1 - DWM_SOURCE_FRAME_SAMPLING_COVERAGE - DWM_SOURCE_FRAME_SAMPLING_LAST -) - -// Flags used by the DWM_THUMBNAIL_PROPERTIES structure to -// indicate which of its members contain valid information. -const ( - DWM_TNP_RECTDESTINATION = 0x00000001 // A value for the rcDestination member has been specified - DWM_TNP_RECTSOURCE = 0x00000002 // A value for the rcSource member has been specified - DWM_TNP_OPACITY = 0x00000004 // A value for the opacity member has been specified - DWM_TNP_VISIBLE = 0x00000008 // A value for the fVisible member has been specified - DWM_TNP_SOURCECLIENTAREAONLY = 0x00000010 // A value for the fSourceClientAreaOnly member has been specified -) - -// enum-lite implementation for the following constant structure -type DWMFLIP3DWINDOWPOLICY int32 - -// TODO: need to verify this construction -// Flags used by the DwmSetWindowAttribute function -// to specify the Flip3D window policy -const ( - DWMFLIP3D_DEFAULT = iota + 1 - DWMFLIP3D_EXCLUDEBELOW - DWMFLIP3D_EXCLUDEABOVE - DWMFLIP3D_LAST -) - -// enum-lite implementation for the following constant structure -type DWMNCRENDERINGPOLICY int32 - -// TODO: need to verify this construction -// Flags used by the DwmSetWindowAttribute function -// to specify the non-client area rendering policy -const ( - DWMNCRP_USEWINDOWSTYLE = iota + 1 - DWMNCRP_DISABLED - DWMNCRP_ENABLED - DWMNCRP_LAST -) - -// enum-lite implementation for the following constant structure -type DWMTRANSITION_OWNEDWINDOW_TARGET int32 - -const ( - DWMTRANSITION_OWNEDWINDOW_NULL = -1 - DWMTRANSITION_OWNEDWINDOW_REPOSITION = 0 -) - -// enum-lite implementation for the following constant structure -type DWMWINDOWATTRIBUTE int32 - -// TODO: need to verify this construction -// Flags used by the DwmGetWindowAttribute and DwmSetWindowAttribute functions -// to specify window attributes for non-client rendering -const ( - DWMWA_NCRENDERING_ENABLED = iota + 1 - DWMWA_NCRENDERING_POLICY - DWMWA_TRANSITIONS_FORCEDISABLED - DWMWA_ALLOW_NCPAINT - DWMWA_CAPTION_BUTTON_BOUNDS - DWMWA_NONCLIENT_RTL_LAYOUT - DWMWA_FORCE_ICONIC_REPRESENTATION - DWMWA_FLIP3D_POLICY - DWMWA_EXTENDED_FRAME_BOUNDS - DWMWA_HAS_ICONIC_BITMAP - DWMWA_DISALLOW_PEEK - DWMWA_EXCLUDED_FROM_PEEK - DWMWA_CLOAK - DWMWA_CLOAKED - DWMWA_FREEZE_REPRESENTATION - DWMWA_LAST -) - -// enum-lite implementation for the following constant structure -type GESTURE_TYPE int32 - -// TODO: use iota? -// Identifies the gesture type -const ( - GT_PEN_TAP = 0 - GT_PEN_DOUBLETAP = 1 - GT_PEN_RIGHTTAP = 2 - GT_PEN_PRESSANDHOLD = 3 - GT_PEN_PRESSANDHOLDABORT = 4 - GT_TOUCH_TAP = 5 - GT_TOUCH_DOUBLETAP = 6 - GT_TOUCH_RIGHTTAP = 7 - GT_TOUCH_PRESSANDHOLD = 8 - GT_TOUCH_PRESSANDHOLDABORT = 9 - GT_TOUCH_PRESSANDTAP = 10 -) - -// Icons -const ( - ICON_SMALL = 0 - ICON_BIG = 1 - ICON_SMALL2 = 2 -) - -const ( - SIZE_RESTORED = 0 - SIZE_MINIMIZED = 1 - SIZE_MAXIMIZED = 2 - SIZE_MAXSHOW = 3 - SIZE_MAXHIDE = 4 -) - -// XButton values -const ( - XBUTTON1 = 1 - XBUTTON2 = 2 -) - -// Devmode -const ( - DM_SPECVERSION = 0x0401 - - DM_ORIENTATION = 0x00000001 - DM_PAPERSIZE = 0x00000002 - DM_PAPERLENGTH = 0x00000004 - DM_PAPERWIDTH = 0x00000008 - DM_SCALE = 0x00000010 - DM_POSITION = 0x00000020 - DM_NUP = 0x00000040 - DM_DISPLAYORIENTATION = 0x00000080 - DM_COPIES = 0x00000100 - DM_DEFAULTSOURCE = 0x00000200 - DM_PRINTQUALITY = 0x00000400 - DM_COLOR = 0x00000800 - DM_DUPLEX = 0x00001000 - DM_YRESOLUTION = 0x00002000 - DM_TTOPTION = 0x00004000 - DM_COLLATE = 0x00008000 - DM_FORMNAME = 0x00010000 - DM_LOGPIXELS = 0x00020000 - DM_BITSPERPEL = 0x00040000 - DM_PELSWIDTH = 0x00080000 - DM_PELSHEIGHT = 0x00100000 - DM_DISPLAYFLAGS = 0x00200000 - DM_DISPLAYFREQUENCY = 0x00400000 - DM_ICMMETHOD = 0x00800000 - DM_ICMINTENT = 0x01000000 - DM_MEDIATYPE = 0x02000000 - DM_DITHERTYPE = 0x04000000 - DM_PANNINGWIDTH = 0x08000000 - DM_PANNINGHEIGHT = 0x10000000 - DM_DISPLAYFIXEDOUTPUT = 0x20000000 -) - -// ChangeDisplaySettings -const ( - CDS_UPDATEREGISTRY = 0x00000001 - CDS_TEST = 0x00000002 - CDS_FULLSCREEN = 0x00000004 - CDS_GLOBAL = 0x00000008 - CDS_SET_PRIMARY = 0x00000010 - CDS_VIDEOPARAMETERS = 0x00000020 - CDS_RESET = 0x40000000 - CDS_NORESET = 0x10000000 - - DISP_CHANGE_SUCCESSFUL = 0 - DISP_CHANGE_RESTART = 1 - DISP_CHANGE_FAILED = -1 - DISP_CHANGE_BADMODE = -2 - DISP_CHANGE_NOTUPDATED = -3 - DISP_CHANGE_BADFLAGS = -4 - DISP_CHANGE_BADPARAM = -5 - DISP_CHANGE_BADDUALVIEW = -6 -) - -const ( - ENUM_CURRENT_SETTINGS = 0xFFFFFFFF - ENUM_REGISTRY_SETTINGS = 0xFFFFFFFE -) - -// PIXELFORMATDESCRIPTOR -const ( - PFD_TYPE_RGBA = 0 - PFD_TYPE_COLORINDEX = 1 - - PFD_MAIN_PLANE = 0 - PFD_OVERLAY_PLANE = 1 - PFD_UNDERLAY_PLANE = -1 - - PFD_DOUBLEBUFFER = 0x00000001 - PFD_STEREO = 0x00000002 - PFD_DRAW_TO_WINDOW = 0x00000004 - PFD_DRAW_TO_BITMAP = 0x00000008 - PFD_SUPPORT_GDI = 0x00000010 - PFD_SUPPORT_OPENGL = 0x00000020 - PFD_GENERIC_FORMAT = 0x00000040 - PFD_NEED_PALETTE = 0x00000080 - PFD_NEED_SYSTEM_PALETTE = 0x00000100 - PFD_SWAP_EXCHANGE = 0x00000200 - PFD_SWAP_COPY = 0x00000400 - PFD_SWAP_LAYER_BUFFERS = 0x00000800 - PFD_GENERIC_ACCELERATED = 0x00001000 - PFD_SUPPORT_DIRECTDRAW = 0x00002000 - PFD_DIRECT3D_ACCELERATED = 0x00004000 - PFD_SUPPORT_COMPOSITION = 0x00008000 - - PFD_DEPTH_DONTCARE = 0x20000000 - PFD_DOUBLEBUFFER_DONTCARE = 0x40000000 - PFD_STEREO_DONTCARE = 0x80000000 -) - -const ( - INPUT_MOUSE = 0 - INPUT_KEYBOARD = 1 - INPUT_HARDWARE = 2 -) - -const ( - MOUSEEVENTF_ABSOLUTE = 0x8000 - MOUSEEVENTF_HWHEEL = 0x01000 - MOUSEEVENTF_MOVE = 0x0001 - MOUSEEVENTF_MOVE_NOCOALESCE = 0x2000 - MOUSEEVENTF_LEFTDOWN = 0x0002 - MOUSEEVENTF_LEFTUP = 0x0004 - MOUSEEVENTF_RIGHTDOWN = 0x0008 - MOUSEEVENTF_RIGHTUP = 0x0010 - MOUSEEVENTF_MIDDLEDOWN = 0x0020 - MOUSEEVENTF_MIDDLEUP = 0x0040 - MOUSEEVENTF_VIRTUALDESK = 0x4000 - MOUSEEVENTF_WHEEL = 0x0800 - MOUSEEVENTF_XDOWN = 0x0080 - MOUSEEVENTF_XUP = 0x0100 -) diff --git a/github.com/shirou/w32/dwmapi.go b/github.com/shirou/w32/dwmapi.go deleted file mode 100644 index 139b93745a..0000000000 --- a/github.com/shirou/w32/dwmapi.go +++ /dev/null @@ -1,256 +0,0 @@ -// Copyright 2010-2012 The W32 Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build windows - -package w32 - -import ( - "fmt" - "syscall" - "unsafe" -) - -// DEFINED IN THE DWM API BUT NOT IMPLEMENTED BY MS: -// DwmAttachMilContent -// DwmDetachMilContent -// DwmEnableComposition -// DwmGetGraphicsStreamClient -// DwmGetGraphicsStreamTransformHint - -var ( - moddwmapi = syscall.NewLazyDLL("dwmapi.dll") - - procDwmDefWindowProc = moddwmapi.NewProc("DwmDefWindowProc") - procDwmEnableBlurBehindWindow = moddwmapi.NewProc("DwmEnableBlurBehindWindow") - procDwmEnableMMCSS = moddwmapi.NewProc("DwmEnableMMCSS") - procDwmExtendFrameIntoClientArea = moddwmapi.NewProc("DwmExtendFrameIntoClientArea") - procDwmFlush = moddwmapi.NewProc("DwmFlush") - procDwmGetColorizationColor = moddwmapi.NewProc("DwmGetColorizationColor") - procDwmGetCompositionTimingInfo = moddwmapi.NewProc("DwmGetCompositionTimingInfo") - procDwmGetTransportAttributes = moddwmapi.NewProc("DwmGetTransportAttributes") - procDwmGetWindowAttribute = moddwmapi.NewProc("DwmGetWindowAttribute") - procDwmInvalidateIconicBitmaps = moddwmapi.NewProc("DwmInvalidateIconicBitmaps") - procDwmIsCompositionEnabled = moddwmapi.NewProc("DwmIsCompositionEnabled") - procDwmModifyPreviousDxFrameDuration = moddwmapi.NewProc("DwmModifyPreviousDxFrameDuration") - procDwmQueryThumbnailSourceSize = moddwmapi.NewProc("DwmQueryThumbnailSourceSize") - procDwmRegisterThumbnail = moddwmapi.NewProc("DwmRegisterThumbnail") - procDwmRenderGesture = moddwmapi.NewProc("DwmRenderGesture") - procDwmSetDxFrameDuration = moddwmapi.NewProc("DwmSetDxFrameDuration") - procDwmSetIconicLivePreviewBitmap = moddwmapi.NewProc("DwmSetIconicLivePreviewBitmap") - procDwmSetIconicThumbnail = moddwmapi.NewProc("DwmSetIconicThumbnail") - procDwmSetPresentParameters = moddwmapi.NewProc("DwmSetPresentParameters") - procDwmSetWindowAttribute = moddwmapi.NewProc("DwmSetWindowAttribute") - procDwmShowContact = moddwmapi.NewProc("DwmShowContact") - procDwmTetherContact = moddwmapi.NewProc("DwmTetherContact") - procDwmTransitionOwnedWindow = moddwmapi.NewProc("DwmTransitionOwnedWindow") - procDwmUnregisterThumbnail = moddwmapi.NewProc("DwmUnregisterThumbnail") - procDwmUpdateThumbnailProperties = moddwmapi.NewProc("DwmUpdateThumbnailProperties") -) - -func DwmDefWindowProc(hWnd HWND, msg uint, wParam, lParam uintptr) (bool, uint) { - var result uint - ret, _, _ := procDwmDefWindowProc.Call( - uintptr(hWnd), - uintptr(msg), - wParam, - lParam, - uintptr(unsafe.Pointer(&result))) - return ret != 0, result -} - -func DwmEnableBlurBehindWindow(hWnd HWND, pBlurBehind *DWM_BLURBEHIND) HRESULT { - ret, _, _ := procDwmEnableBlurBehindWindow.Call( - uintptr(hWnd), - uintptr(unsafe.Pointer(pBlurBehind))) - return HRESULT(ret) -} - -func DwmEnableMMCSS(fEnableMMCSS bool) HRESULT { - ret, _, _ := procDwmEnableMMCSS.Call( - uintptr(BoolToBOOL(fEnableMMCSS))) - return HRESULT(ret) -} - -func DwmExtendFrameIntoClientArea(hWnd HWND, pMarInset *MARGINS) HRESULT { - ret, _, _ := procDwmExtendFrameIntoClientArea.Call( - uintptr(hWnd), - uintptr(unsafe.Pointer(pMarInset))) - return HRESULT(ret) -} - -func DwmFlush() HRESULT { - ret, _, _ := procDwmFlush.Call() - return HRESULT(ret) -} - -func DwmGetColorizationColor(pcrColorization *uint32, pfOpaqueBlend *BOOL) HRESULT { - ret, _, _ := procDwmGetColorizationColor.Call( - uintptr(unsafe.Pointer(pcrColorization)), - uintptr(unsafe.Pointer(pfOpaqueBlend))) - return HRESULT(ret) -} - -func DwmGetCompositionTimingInfo(hWnd HWND, pTimingInfo *DWM_TIMING_INFO) HRESULT { - ret, _, _ := procDwmGetCompositionTimingInfo.Call( - uintptr(hWnd), - uintptr(unsafe.Pointer(pTimingInfo))) - return HRESULT(ret) -} - -func DwmGetTransportAttributes(pfIsRemoting *BOOL, pfIsConnected *BOOL, pDwGeneration *uint32) HRESULT { - ret, _, _ := procDwmGetTransportAttributes.Call( - uintptr(unsafe.Pointer(pfIsRemoting)), - uintptr(unsafe.Pointer(pfIsConnected)), - uintptr(unsafe.Pointer(pDwGeneration))) - return HRESULT(ret) -} - -// TODO: verify handling of variable arguments -func DwmGetWindowAttribute(hWnd HWND, dwAttribute uint32) (pAttribute interface{}, result HRESULT) { - var pvAttribute, pvAttrSize uintptr - switch dwAttribute { - case DWMWA_NCRENDERING_ENABLED: - v := new(BOOL) - pAttribute = v - pvAttribute = uintptr(unsafe.Pointer(v)) - pvAttrSize = unsafe.Sizeof(*v) - case DWMWA_CAPTION_BUTTON_BOUNDS, DWMWA_EXTENDED_FRAME_BOUNDS: - v := new(RECT) - pAttribute = v - pvAttribute = uintptr(unsafe.Pointer(v)) - pvAttrSize = unsafe.Sizeof(*v) - case DWMWA_CLOAKED: - panic(fmt.Sprintf("DwmGetWindowAttribute(%d) is not currently supported.", dwAttribute)) - default: - panic(fmt.Sprintf("DwmGetWindowAttribute(%d) is not valid.", dwAttribute)) - } - - ret, _, _ := procDwmGetWindowAttribute.Call( - uintptr(hWnd), - uintptr(dwAttribute), - pvAttribute, - pvAttrSize) - result = HRESULT(ret) - return -} - -func DwmInvalidateIconicBitmaps(hWnd HWND) HRESULT { - ret, _, _ := procDwmInvalidateIconicBitmaps.Call( - uintptr(hWnd)) - return HRESULT(ret) -} - -func DwmIsCompositionEnabled(pfEnabled *BOOL) HRESULT { - ret, _, _ := procDwmIsCompositionEnabled.Call( - uintptr(unsafe.Pointer(pfEnabled))) - return HRESULT(ret) -} - -func DwmModifyPreviousDxFrameDuration(hWnd HWND, cRefreshes int, fRelative bool) HRESULT { - ret, _, _ := procDwmModifyPreviousDxFrameDuration.Call( - uintptr(hWnd), - uintptr(cRefreshes), - uintptr(BoolToBOOL(fRelative))) - return HRESULT(ret) -} - -func DwmQueryThumbnailSourceSize(hThumbnail HTHUMBNAIL, pSize *SIZE) HRESULT { - ret, _, _ := procDwmQueryThumbnailSourceSize.Call( - uintptr(hThumbnail), - uintptr(unsafe.Pointer(pSize))) - return HRESULT(ret) -} - -func DwmRegisterThumbnail(hWndDestination HWND, hWndSource HWND, phThumbnailId *HTHUMBNAIL) HRESULT { - ret, _, _ := procDwmRegisterThumbnail.Call( - uintptr(hWndDestination), - uintptr(hWndSource), - uintptr(unsafe.Pointer(phThumbnailId))) - return HRESULT(ret) -} - -func DwmRenderGesture(gt GESTURE_TYPE, cContacts uint, pdwPointerID *uint32, pPoints *POINT) { - procDwmRenderGesture.Call( - uintptr(gt), - uintptr(cContacts), - uintptr(unsafe.Pointer(pdwPointerID)), - uintptr(unsafe.Pointer(pPoints))) - return -} - -func DwmSetDxFrameDuration(hWnd HWND, cRefreshes int) HRESULT { - ret, _, _ := procDwmSetDxFrameDuration.Call( - uintptr(hWnd), - uintptr(cRefreshes)) - return HRESULT(ret) -} - -func DwmSetIconicLivePreviewBitmap(hWnd HWND, hbmp HBITMAP, pptClient *POINT, dwSITFlags uint32) HRESULT { - ret, _, _ := procDwmSetIconicLivePreviewBitmap.Call( - uintptr(hWnd), - uintptr(hbmp), - uintptr(unsafe.Pointer(pptClient)), - uintptr(dwSITFlags)) - return HRESULT(ret) -} - -func DwmSetIconicThumbnail(hWnd HWND, hbmp HBITMAP, dwSITFlags uint32) HRESULT { - ret, _, _ := procDwmSetIconicThumbnail.Call( - uintptr(hWnd), - uintptr(hbmp), - uintptr(dwSITFlags)) - return HRESULT(ret) -} - -func DwmSetPresentParameters(hWnd HWND, pPresentParams *DWM_PRESENT_PARAMETERS) HRESULT { - ret, _, _ := procDwmSetPresentParameters.Call( - uintptr(hWnd), - uintptr(unsafe.Pointer(pPresentParams))) - return HRESULT(ret) -} - -func DwmSetWindowAttribute(hWnd HWND, dwAttribute uint32, pvAttribute LPCVOID, cbAttribute uint32) HRESULT { - ret, _, _ := procDwmSetWindowAttribute.Call( - uintptr(hWnd), - uintptr(dwAttribute), - uintptr(pvAttribute), - uintptr(cbAttribute)) - return HRESULT(ret) -} - -func DwmShowContact(dwPointerID uint32, eShowContact DWM_SHOWCONTACT) { - procDwmShowContact.Call( - uintptr(dwPointerID), - uintptr(eShowContact)) - return -} - -func DwmTetherContact(dwPointerID uint32, fEnable bool, ptTether POINT) { - procDwmTetherContact.Call( - uintptr(dwPointerID), - uintptr(BoolToBOOL(fEnable)), - uintptr(unsafe.Pointer(&ptTether))) - return -} - -func DwmTransitionOwnedWindow(hWnd HWND, target DWMTRANSITION_OWNEDWINDOW_TARGET) { - procDwmTransitionOwnedWindow.Call( - uintptr(hWnd), - uintptr(target)) - return -} - -func DwmUnregisterThumbnail(hThumbnailId HTHUMBNAIL) HRESULT { - ret, _, _ := procDwmUnregisterThumbnail.Call( - uintptr(hThumbnailId)) - return HRESULT(ret) -} - -func DwmUpdateThumbnailProperties(hThumbnailId HTHUMBNAIL, ptnProperties *DWM_THUMBNAIL_PROPERTIES) HRESULT { - ret, _, _ := procDwmUpdateThumbnailProperties.Call( - uintptr(hThumbnailId), - uintptr(unsafe.Pointer(ptnProperties))) - return HRESULT(ret) -} diff --git a/github.com/shirou/w32/gdi32.go b/github.com/shirou/w32/gdi32.go deleted file mode 100644 index 34f032c7bc..0000000000 --- a/github.com/shirou/w32/gdi32.go +++ /dev/null @@ -1,511 +0,0 @@ -// Copyright 2010-2012 The W32 Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build windows - -package w32 - -import ( - "syscall" - "unsafe" -) - -var ( - modgdi32 = syscall.NewLazyDLL("gdi32.dll") - - procGetDeviceCaps = modgdi32.NewProc("GetDeviceCaps") - procDeleteObject = modgdi32.NewProc("DeleteObject") - procCreateFontIndirect = modgdi32.NewProc("CreateFontIndirectW") - procAbortDoc = modgdi32.NewProc("AbortDoc") - procBitBlt = modgdi32.NewProc("BitBlt") - procCloseEnhMetaFile = modgdi32.NewProc("CloseEnhMetaFile") - procCopyEnhMetaFile = modgdi32.NewProc("CopyEnhMetaFileW") - procCreateBrushIndirect = modgdi32.NewProc("CreateBrushIndirect") - procCreateCompatibleDC = modgdi32.NewProc("CreateCompatibleDC") - procCreateDC = modgdi32.NewProc("CreateDCW") - procCreateDIBSection = modgdi32.NewProc("CreateDIBSection") - procCreateEnhMetaFile = modgdi32.NewProc("CreateEnhMetaFileW") - procCreateIC = modgdi32.NewProc("CreateICW") - procDeleteDC = modgdi32.NewProc("DeleteDC") - procDeleteEnhMetaFile = modgdi32.NewProc("DeleteEnhMetaFile") - procEllipse = modgdi32.NewProc("Ellipse") - procEndDoc = modgdi32.NewProc("EndDoc") - procEndPage = modgdi32.NewProc("EndPage") - procExtCreatePen = modgdi32.NewProc("ExtCreatePen") - procGetEnhMetaFile = modgdi32.NewProc("GetEnhMetaFileW") - procGetEnhMetaFileHeader = modgdi32.NewProc("GetEnhMetaFileHeader") - procGetObject = modgdi32.NewProc("GetObjectW") - procGetStockObject = modgdi32.NewProc("GetStockObject") - procGetTextExtentExPoint = modgdi32.NewProc("GetTextExtentExPointW") - procGetTextExtentPoint32 = modgdi32.NewProc("GetTextExtentPoint32W") - procGetTextMetrics = modgdi32.NewProc("GetTextMetricsW") - procLineTo = modgdi32.NewProc("LineTo") - procMoveToEx = modgdi32.NewProc("MoveToEx") - procPlayEnhMetaFile = modgdi32.NewProc("PlayEnhMetaFile") - procRectangle = modgdi32.NewProc("Rectangle") - procResetDC = modgdi32.NewProc("ResetDCW") - procSelectObject = modgdi32.NewProc("SelectObject") - procSetBkMode = modgdi32.NewProc("SetBkMode") - procSetBrushOrgEx = modgdi32.NewProc("SetBrushOrgEx") - procSetStretchBltMode = modgdi32.NewProc("SetStretchBltMode") - procSetTextColor = modgdi32.NewProc("SetTextColor") - procSetBkColor = modgdi32.NewProc("SetBkColor") - procStartDoc = modgdi32.NewProc("StartDocW") - procStartPage = modgdi32.NewProc("StartPage") - procStretchBlt = modgdi32.NewProc("StretchBlt") - procSetDIBitsToDevice = modgdi32.NewProc("SetDIBitsToDevice") - procChoosePixelFormat = modgdi32.NewProc("ChoosePixelFormat") - procDescribePixelFormat = modgdi32.NewProc("DescribePixelFormat") - procGetEnhMetaFilePixelFormat = modgdi32.NewProc("GetEnhMetaFilePixelFormat") - procGetPixelFormat = modgdi32.NewProc("GetPixelFormat") - procSetPixelFormat = modgdi32.NewProc("SetPixelFormat") - procSwapBuffers = modgdi32.NewProc("SwapBuffers") -) - -func GetDeviceCaps(hdc HDC, index int) int { - ret, _, _ := procGetDeviceCaps.Call( - uintptr(hdc), - uintptr(index)) - - return int(ret) -} - -func DeleteObject(hObject HGDIOBJ) bool { - ret, _, _ := procDeleteObject.Call( - uintptr(hObject)) - - return ret != 0 -} - -func CreateFontIndirect(logFont *LOGFONT) HFONT { - ret, _, _ := procCreateFontIndirect.Call( - uintptr(unsafe.Pointer(logFont))) - - return HFONT(ret) -} - -func AbortDoc(hdc HDC) int { - ret, _, _ := procAbortDoc.Call( - uintptr(hdc)) - - return int(ret) -} - -func BitBlt(hdcDest HDC, nXDest, nYDest, nWidth, nHeight int, hdcSrc HDC, nXSrc, nYSrc int, dwRop uint) { - ret, _, _ := procBitBlt.Call( - uintptr(hdcDest), - uintptr(nXDest), - uintptr(nYDest), - uintptr(nWidth), - uintptr(nHeight), - uintptr(hdcSrc), - uintptr(nXSrc), - uintptr(nYSrc), - uintptr(dwRop)) - - if ret == 0 { - panic("BitBlt failed") - } -} - -func CloseEnhMetaFile(hdc HDC) HENHMETAFILE { - ret, _, _ := procCloseEnhMetaFile.Call( - uintptr(hdc)) - - return HENHMETAFILE(ret) -} - -func CopyEnhMetaFile(hemfSrc HENHMETAFILE, lpszFile *uint16) HENHMETAFILE { - ret, _, _ := procCopyEnhMetaFile.Call( - uintptr(hemfSrc), - uintptr(unsafe.Pointer(lpszFile))) - - return HENHMETAFILE(ret) -} - -func CreateBrushIndirect(lplb *LOGBRUSH) HBRUSH { - ret, _, _ := procCreateBrushIndirect.Call( - uintptr(unsafe.Pointer(lplb))) - - return HBRUSH(ret) -} - -func CreateCompatibleDC(hdc HDC) HDC { - ret, _, _ := procCreateCompatibleDC.Call( - uintptr(hdc)) - - if ret == 0 { - panic("Create compatible DC failed") - } - - return HDC(ret) -} - -func CreateDC(lpszDriver, lpszDevice, lpszOutput *uint16, lpInitData *DEVMODE) HDC { - ret, _, _ := procCreateDC.Call( - uintptr(unsafe.Pointer(lpszDriver)), - uintptr(unsafe.Pointer(lpszDevice)), - uintptr(unsafe.Pointer(lpszOutput)), - uintptr(unsafe.Pointer(lpInitData))) - - return HDC(ret) -} - -func CreateDIBSection(hdc HDC, pbmi *BITMAPINFO, iUsage uint, ppvBits *unsafe.Pointer, hSection HANDLE, dwOffset uint) HBITMAP { - ret, _, _ := procCreateDIBSection.Call( - uintptr(hdc), - uintptr(unsafe.Pointer(pbmi)), - uintptr(iUsage), - uintptr(unsafe.Pointer(ppvBits)), - uintptr(hSection), - uintptr(dwOffset)) - - return HBITMAP(ret) -} - -func CreateEnhMetaFile(hdcRef HDC, lpFilename *uint16, lpRect *RECT, lpDescription *uint16) HDC { - ret, _, _ := procCreateEnhMetaFile.Call( - uintptr(hdcRef), - uintptr(unsafe.Pointer(lpFilename)), - uintptr(unsafe.Pointer(lpRect)), - uintptr(unsafe.Pointer(lpDescription))) - - return HDC(ret) -} - -func CreateIC(lpszDriver, lpszDevice, lpszOutput *uint16, lpdvmInit *DEVMODE) HDC { - ret, _, _ := procCreateIC.Call( - uintptr(unsafe.Pointer(lpszDriver)), - uintptr(unsafe.Pointer(lpszDevice)), - uintptr(unsafe.Pointer(lpszOutput)), - uintptr(unsafe.Pointer(lpdvmInit))) - - return HDC(ret) -} - -func DeleteDC(hdc HDC) bool { - ret, _, _ := procDeleteDC.Call( - uintptr(hdc)) - - return ret != 0 -} - -func DeleteEnhMetaFile(hemf HENHMETAFILE) bool { - ret, _, _ := procDeleteEnhMetaFile.Call( - uintptr(hemf)) - - return ret != 0 -} - -func Ellipse(hdc HDC, nLeftRect, nTopRect, nRightRect, nBottomRect int) bool { - ret, _, _ := procEllipse.Call( - uintptr(hdc), - uintptr(nLeftRect), - uintptr(nTopRect), - uintptr(nRightRect), - uintptr(nBottomRect)) - - return ret != 0 -} - -func EndDoc(hdc HDC) int { - ret, _, _ := procEndDoc.Call( - uintptr(hdc)) - - return int(ret) -} - -func EndPage(hdc HDC) int { - ret, _, _ := procEndPage.Call( - uintptr(hdc)) - - return int(ret) -} - -func ExtCreatePen(dwPenStyle, dwWidth uint, lplb *LOGBRUSH, dwStyleCount uint, lpStyle *uint) HPEN { - ret, _, _ := procExtCreatePen.Call( - uintptr(dwPenStyle), - uintptr(dwWidth), - uintptr(unsafe.Pointer(lplb)), - uintptr(dwStyleCount), - uintptr(unsafe.Pointer(lpStyle))) - - return HPEN(ret) -} - -func GetEnhMetaFile(lpszMetaFile *uint16) HENHMETAFILE { - ret, _, _ := procGetEnhMetaFile.Call( - uintptr(unsafe.Pointer(lpszMetaFile))) - - return HENHMETAFILE(ret) -} - -func GetEnhMetaFileHeader(hemf HENHMETAFILE, cbBuffer uint, lpemh *ENHMETAHEADER) uint { - ret, _, _ := procGetEnhMetaFileHeader.Call( - uintptr(hemf), - uintptr(cbBuffer), - uintptr(unsafe.Pointer(lpemh))) - - return uint(ret) -} - -func GetObject(hgdiobj HGDIOBJ, cbBuffer uintptr, lpvObject unsafe.Pointer) int { - ret, _, _ := procGetObject.Call( - uintptr(hgdiobj), - uintptr(cbBuffer), - uintptr(lpvObject)) - - return int(ret) -} - -func GetStockObject(fnObject int) HGDIOBJ { - ret, _, _ := procGetDeviceCaps.Call( - uintptr(fnObject)) - - return HGDIOBJ(ret) -} - -func GetTextExtentExPoint(hdc HDC, lpszStr *uint16, cchString, nMaxExtent int, lpnFit, alpDx *int, lpSize *SIZE) bool { - ret, _, _ := procGetTextExtentExPoint.Call( - uintptr(hdc), - uintptr(unsafe.Pointer(lpszStr)), - uintptr(cchString), - uintptr(nMaxExtent), - uintptr(unsafe.Pointer(lpnFit)), - uintptr(unsafe.Pointer(alpDx)), - uintptr(unsafe.Pointer(lpSize))) - - return ret != 0 -} - -func GetTextExtentPoint32(hdc HDC, lpString *uint16, c int, lpSize *SIZE) bool { - ret, _, _ := procGetTextExtentPoint32.Call( - uintptr(hdc), - uintptr(unsafe.Pointer(lpString)), - uintptr(c), - uintptr(unsafe.Pointer(lpSize))) - - return ret != 0 -} - -func GetTextMetrics(hdc HDC, lptm *TEXTMETRIC) bool { - ret, _, _ := procGetTextMetrics.Call( - uintptr(hdc), - uintptr(unsafe.Pointer(lptm))) - - return ret != 0 -} - -func LineTo(hdc HDC, nXEnd, nYEnd int) bool { - ret, _, _ := procLineTo.Call( - uintptr(hdc), - uintptr(nXEnd), - uintptr(nYEnd)) - - return ret != 0 -} - -func MoveToEx(hdc HDC, x, y int, lpPoint *POINT) bool { - ret, _, _ := procMoveToEx.Call( - uintptr(hdc), - uintptr(x), - uintptr(y), - uintptr(unsafe.Pointer(lpPoint))) - - return ret != 0 -} - -func PlayEnhMetaFile(hdc HDC, hemf HENHMETAFILE, lpRect *RECT) bool { - ret, _, _ := procPlayEnhMetaFile.Call( - uintptr(hdc), - uintptr(hemf), - uintptr(unsafe.Pointer(lpRect))) - - return ret != 0 -} - -func Rectangle(hdc HDC, nLeftRect, nTopRect, nRightRect, nBottomRect int) bool { - ret, _, _ := procRectangle.Call( - uintptr(hdc), - uintptr(nLeftRect), - uintptr(nTopRect), - uintptr(nRightRect), - uintptr(nBottomRect)) - - return ret != 0 -} - -func ResetDC(hdc HDC, lpInitData *DEVMODE) HDC { - ret, _, _ := procResetDC.Call( - uintptr(hdc), - uintptr(unsafe.Pointer(lpInitData))) - - return HDC(ret) -} - -func SelectObject(hdc HDC, hgdiobj HGDIOBJ) HGDIOBJ { - ret, _, _ := procSelectObject.Call( - uintptr(hdc), - uintptr(hgdiobj)) - - if ret == 0 { - panic("SelectObject failed") - } - - return HGDIOBJ(ret) -} - -func SetBkMode(hdc HDC, iBkMode int) int { - ret, _, _ := procSetBkMode.Call( - uintptr(hdc), - uintptr(iBkMode)) - - if ret == 0 { - panic("SetBkMode failed") - } - - return int(ret) -} - -func SetBrushOrgEx(hdc HDC, nXOrg, nYOrg int, lppt *POINT) bool { - ret, _, _ := procSetBrushOrgEx.Call( - uintptr(hdc), - uintptr(nXOrg), - uintptr(nYOrg), - uintptr(unsafe.Pointer(lppt))) - - return ret != 0 -} - -func SetStretchBltMode(hdc HDC, iStretchMode int) int { - ret, _, _ := procSetStretchBltMode.Call( - uintptr(hdc), - uintptr(iStretchMode)) - - return int(ret) -} - -func SetTextColor(hdc HDC, crColor COLORREF) COLORREF { - ret, _, _ := procSetTextColor.Call( - uintptr(hdc), - uintptr(crColor)) - - if ret == CLR_INVALID { - panic("SetTextColor failed") - } - - return COLORREF(ret) -} - -func SetBkColor(hdc HDC, crColor COLORREF) COLORREF { - ret, _, _ := procSetBkColor.Call( - uintptr(hdc), - uintptr(crColor)) - - if ret == CLR_INVALID { - panic("SetBkColor failed") - } - - return COLORREF(ret) -} - -func StartDoc(hdc HDC, lpdi *DOCINFO) int { - ret, _, _ := procStartDoc.Call( - uintptr(hdc), - uintptr(unsafe.Pointer(lpdi))) - - return int(ret) -} - -func StartPage(hdc HDC) int { - ret, _, _ := procStartPage.Call( - uintptr(hdc)) - - return int(ret) -} - -func StretchBlt(hdcDest HDC, nXOriginDest, nYOriginDest, nWidthDest, nHeightDest int, hdcSrc HDC, nXOriginSrc, nYOriginSrc, nWidthSrc, nHeightSrc int, dwRop uint) { - ret, _, _ := procStretchBlt.Call( - uintptr(hdcDest), - uintptr(nXOriginDest), - uintptr(nYOriginDest), - uintptr(nWidthDest), - uintptr(nHeightDest), - uintptr(hdcSrc), - uintptr(nXOriginSrc), - uintptr(nYOriginSrc), - uintptr(nWidthSrc), - uintptr(nHeightSrc), - uintptr(dwRop)) - - if ret == 0 { - panic("StretchBlt failed") - } -} - -func SetDIBitsToDevice(hdc HDC, xDest, yDest, dwWidth, dwHeight, xSrc, ySrc int, uStartScan, cScanLines uint, lpvBits []byte, lpbmi *BITMAPINFO, fuColorUse uint) int { - ret, _, _ := procSetDIBitsToDevice.Call( - uintptr(hdc), - uintptr(xDest), - uintptr(yDest), - uintptr(dwWidth), - uintptr(dwHeight), - uintptr(xSrc), - uintptr(ySrc), - uintptr(uStartScan), - uintptr(cScanLines), - uintptr(unsafe.Pointer(&lpvBits[0])), - uintptr(unsafe.Pointer(lpbmi)), - uintptr(fuColorUse)) - - return int(ret) -} - -func ChoosePixelFormat(hdc HDC, pfd *PIXELFORMATDESCRIPTOR) int { - ret, _, _ := procChoosePixelFormat.Call( - uintptr(hdc), - uintptr(unsafe.Pointer(pfd)), - ) - return int(ret) -} - -func DescribePixelFormat(hdc HDC, iPixelFormat int, nBytes uint, pfd *PIXELFORMATDESCRIPTOR) int { - ret, _, _ := procDescribePixelFormat.Call( - uintptr(hdc), - uintptr(iPixelFormat), - uintptr(nBytes), - uintptr(unsafe.Pointer(pfd)), - ) - return int(ret) -} - -func GetEnhMetaFilePixelFormat(hemf HENHMETAFILE, cbBuffer uint32, pfd *PIXELFORMATDESCRIPTOR) uint { - ret, _, _ := procGetEnhMetaFilePixelFormat.Call( - uintptr(hemf), - uintptr(cbBuffer), - uintptr(unsafe.Pointer(pfd)), - ) - return uint(ret) -} - -func GetPixelFormat(hdc HDC) int { - ret, _, _ := procGetPixelFormat.Call( - uintptr(hdc), - ) - return int(ret) -} - -func SetPixelFormat(hdc HDC, iPixelFormat int, pfd *PIXELFORMATDESCRIPTOR) bool { - ret, _, _ := procSetPixelFormat.Call( - uintptr(hdc), - uintptr(iPixelFormat), - uintptr(unsafe.Pointer(pfd)), - ) - return ret == TRUE -} - -func SwapBuffers(hdc HDC) bool { - ret, _, _ := procSwapBuffers.Call(uintptr(hdc)) - return ret == TRUE -} diff --git a/github.com/shirou/w32/gdiplus.go b/github.com/shirou/w32/gdiplus.go deleted file mode 100644 index 443334b0b7..0000000000 --- a/github.com/shirou/w32/gdiplus.go +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright 2010-2012 The W32 Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build windows - -package w32 - -import ( - "errors" - "fmt" - "syscall" - "unsafe" -) - -const ( - Ok = 0 - GenericError = 1 - InvalidParameter = 2 - OutOfMemory = 3 - ObjectBusy = 4 - InsufficientBuffer = 5 - NotImplemented = 6 - Win32Error = 7 - WrongState = 8 - Aborted = 9 - FileNotFound = 10 - ValueOverflow = 11 - AccessDenied = 12 - UnknownImageFormat = 13 - FontFamilyNotFound = 14 - FontStyleNotFound = 15 - NotTrueTypeFont = 16 - UnsupportedGdiplusVersion = 17 - GdiplusNotInitialized = 18 - PropertyNotFound = 19 - PropertyNotSupported = 20 - ProfileNotFound = 21 -) - -func GetGpStatus(s int32) string { - switch s { - case Ok: - return "Ok" - case GenericError: - return "GenericError" - case InvalidParameter: - return "InvalidParameter" - case OutOfMemory: - return "OutOfMemory" - case ObjectBusy: - return "ObjectBusy" - case InsufficientBuffer: - return "InsufficientBuffer" - case NotImplemented: - return "NotImplemented" - case Win32Error: - return "Win32Error" - case WrongState: - return "WrongState" - case Aborted: - return "Aborted" - case FileNotFound: - return "FileNotFound" - case ValueOverflow: - return "ValueOverflow" - case AccessDenied: - return "AccessDenied" - case UnknownImageFormat: - return "UnknownImageFormat" - case FontFamilyNotFound: - return "FontFamilyNotFound" - case FontStyleNotFound: - return "FontStyleNotFound" - case NotTrueTypeFont: - return "NotTrueTypeFont" - case UnsupportedGdiplusVersion: - return "UnsupportedGdiplusVersion" - case GdiplusNotInitialized: - return "GdiplusNotInitialized" - case PropertyNotFound: - return "PropertyNotFound" - case PropertyNotSupported: - return "PropertyNotSupported" - case ProfileNotFound: - return "ProfileNotFound" - } - return "Unknown Status Value" -} - -var ( - token uintptr - - modgdiplus = syscall.NewLazyDLL("gdiplus.dll") - - procGdipCreateBitmapFromFile = modgdiplus.NewProc("GdipCreateBitmapFromFile") - procGdipCreateBitmapFromHBITMAP = modgdiplus.NewProc("GdipCreateBitmapFromHBITMAP") - procGdipCreateHBITMAPFromBitmap = modgdiplus.NewProc("GdipCreateHBITMAPFromBitmap") - procGdipCreateBitmapFromResource = modgdiplus.NewProc("GdipCreateBitmapFromResource") - procGdipCreateBitmapFromStream = modgdiplus.NewProc("GdipCreateBitmapFromStream") - procGdipDisposeImage = modgdiplus.NewProc("GdipDisposeImage") - procGdiplusShutdown = modgdiplus.NewProc("GdiplusShutdown") - procGdiplusStartup = modgdiplus.NewProc("GdiplusStartup") -) - -func GdipCreateBitmapFromFile(filename string) (*uintptr, error) { - var bitmap *uintptr - ret, _, _ := procGdipCreateBitmapFromFile.Call( - uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(filename))), - uintptr(unsafe.Pointer(&bitmap))) - - if ret != Ok { - return nil, errors.New(fmt.Sprintf("GdipCreateBitmapFromFile failed with status '%s' for file '%s'", GetGpStatus(int32(ret)), filename)) - } - - return bitmap, nil -} - -func GdipCreateBitmapFromResource(instance HINSTANCE, resId *uint16) (*uintptr, error) { - var bitmap *uintptr - ret, _, _ := procGdipCreateBitmapFromResource.Call( - uintptr(instance), - uintptr(unsafe.Pointer(resId)), - uintptr(unsafe.Pointer(&bitmap))) - - if ret != Ok { - return nil, errors.New(fmt.Sprintf("GdiCreateBitmapFromResource failed with status '%s'", GetGpStatus(int32(ret)))) - } - - return bitmap, nil -} - -func GdipCreateBitmapFromStream(stream *IStream) (*uintptr, error) { - var bitmap *uintptr - ret, _, _ := procGdipCreateBitmapFromStream.Call( - uintptr(unsafe.Pointer(stream)), - uintptr(unsafe.Pointer(&bitmap))) - - if ret != Ok { - return nil, errors.New(fmt.Sprintf("GdipCreateBitmapFromStream failed with status '%s'", GetGpStatus(int32(ret)))) - } - - return bitmap, nil -} - -func GdipCreateHBITMAPFromBitmap(bitmap *uintptr, background uint32) (HBITMAP, error) { - var hbitmap HBITMAP - ret, _, _ := procGdipCreateHBITMAPFromBitmap.Call( - uintptr(unsafe.Pointer(bitmap)), - uintptr(unsafe.Pointer(&hbitmap)), - uintptr(background)) - - if ret != Ok { - return 0, errors.New(fmt.Sprintf("GdipCreateHBITMAPFromBitmap failed with status '%s'", GetGpStatus(int32(ret)))) - } - - return hbitmap, nil -} - -func GdipDisposeImage(image *uintptr) { - procGdipDisposeImage.Call(uintptr(unsafe.Pointer(image))) -} - -func GdiplusShutdown() { - procGdiplusShutdown.Call(token) -} - -func GdiplusStartup(input *GdiplusStartupInput, output *GdiplusStartupOutput) { - ret, _, _ := procGdiplusStartup.Call( - uintptr(unsafe.Pointer(&token)), - uintptr(unsafe.Pointer(input)), - uintptr(unsafe.Pointer(output))) - - if ret != Ok { - panic("GdiplusStartup failed with status " + GetGpStatus(int32(ret))) - } -} diff --git a/github.com/shirou/w32/idispatch.go b/github.com/shirou/w32/idispatch.go deleted file mode 100644 index d6c2504d8b..0000000000 --- a/github.com/shirou/w32/idispatch.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2010-2012 The W32 Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build windows - -package w32 - -import ( - "unsafe" -) - -type pIDispatchVtbl struct { - pQueryInterface uintptr - pAddRef uintptr - pRelease uintptr - pGetTypeInfoCount uintptr - pGetTypeInfo uintptr - pGetIDsOfNames uintptr - pInvoke uintptr -} - -type IDispatch struct { - lpVtbl *pIDispatchVtbl -} - -func (this *IDispatch) QueryInterface(id *GUID) *IDispatch { - return ComQueryInterface((*IUnknown)(unsafe.Pointer(this)), id) -} - -func (this *IDispatch) AddRef() int32 { - return ComAddRef((*IUnknown)(unsafe.Pointer(this))) -} - -func (this *IDispatch) Release() int32 { - return ComRelease((*IUnknown)(unsafe.Pointer(this))) -} - -func (this *IDispatch) GetIDsOfName(names []string) []int32 { - return ComGetIDsOfName(this, names) -} - -func (this *IDispatch) Invoke(dispid int32, dispatch int16, params ...interface{}) *VARIANT { - return ComInvoke(this, dispid, dispatch, params...) -} diff --git a/github.com/shirou/w32/istream.go b/github.com/shirou/w32/istream.go deleted file mode 100644 index 0bb2822225..0000000000 --- a/github.com/shirou/w32/istream.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2010-2012 The W32 Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build windows - -package w32 - -import ( - "unsafe" -) - -type pIStreamVtbl struct { - pQueryInterface uintptr - pAddRef uintptr - pRelease uintptr -} - -type IStream struct { - lpVtbl *pIStreamVtbl -} - -func (this *IStream) QueryInterface(id *GUID) *IDispatch { - return ComQueryInterface((*IUnknown)(unsafe.Pointer(this)), id) -} - -func (this *IStream) AddRef() int32 { - return ComAddRef((*IUnknown)(unsafe.Pointer(this))) -} - -func (this *IStream) Release() int32 { - return ComRelease((*IUnknown)(unsafe.Pointer(this))) -} diff --git a/github.com/shirou/w32/iunknown.go b/github.com/shirou/w32/iunknown.go deleted file mode 100644 index 847fba7ecd..0000000000 --- a/github.com/shirou/w32/iunknown.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2010-2012 The W32 Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build windows - -package w32 - -type pIUnknownVtbl struct { - pQueryInterface uintptr - pAddRef uintptr - pRelease uintptr -} - -type IUnknown struct { - lpVtbl *pIUnknownVtbl -} - -func (this *IUnknown) QueryInterface(id *GUID) *IDispatch { - return ComQueryInterface(this, id) -} - -func (this *IUnknown) AddRef() int32 { - return ComAddRef(this) -} - -func (this *IUnknown) Release() int32 { - return ComRelease(this) -} diff --git a/github.com/shirou/w32/kernel32.go b/github.com/shirou/w32/kernel32.go deleted file mode 100644 index 5d5b4d8aae..0000000000 --- a/github.com/shirou/w32/kernel32.go +++ /dev/null @@ -1,316 +0,0 @@ -// Copyright 2010-2012 The W32 Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build windows - -package w32 - -import ( - "syscall" - "unsafe" -) - -var ( - modkernel32 = syscall.NewLazyDLL("kernel32.dll") - - procGetModuleHandle = modkernel32.NewProc("GetModuleHandleW") - procMulDiv = modkernel32.NewProc("MulDiv") - procGetConsoleWindow = modkernel32.NewProc("GetConsoleWindow") - procGetCurrentThread = modkernel32.NewProc("GetCurrentThread") - procGetLogicalDrives = modkernel32.NewProc("GetLogicalDrives") - procGetUserDefaultLCID = modkernel32.NewProc("GetUserDefaultLCID") - procLstrlen = modkernel32.NewProc("lstrlenW") - procLstrcpy = modkernel32.NewProc("lstrcpyW") - procGlobalAlloc = modkernel32.NewProc("GlobalAlloc") - procGlobalFree = modkernel32.NewProc("GlobalFree") - procGlobalLock = modkernel32.NewProc("GlobalLock") - procGlobalUnlock = modkernel32.NewProc("GlobalUnlock") - procMoveMemory = modkernel32.NewProc("RtlMoveMemory") - procFindResource = modkernel32.NewProc("FindResourceW") - procSizeofResource = modkernel32.NewProc("SizeofResource") - procLockResource = modkernel32.NewProc("LockResource") - procLoadResource = modkernel32.NewProc("LoadResource") - procGetLastError = modkernel32.NewProc("GetLastError") - procOpenProcess = modkernel32.NewProc("OpenProcess") - procTerminateProcess = modkernel32.NewProc("TerminateProcess") - procCloseHandle = modkernel32.NewProc("CloseHandle") - procCreateToolhelp32Snapshot = modkernel32.NewProc("CreateToolhelp32Snapshot") - procModule32First = modkernel32.NewProc("Module32FirstW") - procModule32Next = modkernel32.NewProc("Module32NextW") - procProcess32First = modkernel32.NewProc("Process32FirstW") - procProcess32Next = modkernel32.NewProc("Process32NextW") - procGetSystemTimes = modkernel32.NewProc("GetSystemTimes") - procGetConsoleScreenBufferInfo = modkernel32.NewProc("GetConsoleScreenBufferInfo") - procSetConsoleTextAttribute = modkernel32.NewProc("SetConsoleTextAttribute") - procGetDiskFreeSpaceEx = modkernel32.NewProc("GetDiskFreeSpaceExW") - procGetProcessTimes = modkernel32.NewProc("GetProcessTimes") -) - -func GetModuleHandle(modulename string) HINSTANCE { - var mn uintptr - if modulename == "" { - mn = 0 - } else { - mn = uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(modulename))) - } - ret, _, _ := procGetModuleHandle.Call(mn) - return HINSTANCE(ret) -} - -func MulDiv(number, numerator, denominator int) int { - ret, _, _ := procMulDiv.Call( - uintptr(number), - uintptr(numerator), - uintptr(denominator)) - - return int(ret) -} - -func GetConsoleWindow() HWND { - ret, _, _ := procGetConsoleWindow.Call() - - return HWND(ret) -} - -func GetCurrentThread() HANDLE { - ret, _, _ := procGetCurrentThread.Call() - - return HANDLE(ret) -} - -func GetLogicalDrives() uint32 { - ret, _, _ := procGetLogicalDrives.Call() - - return uint32(ret) -} - -func GetUserDefaultLCID() uint32 { - ret, _, _ := procGetUserDefaultLCID.Call() - - return uint32(ret) -} - -func Lstrlen(lpString *uint16) int { - ret, _, _ := procLstrlen.Call(uintptr(unsafe.Pointer(lpString))) - - return int(ret) -} - -func Lstrcpy(buf []uint16, lpString *uint16) { - procLstrcpy.Call( - uintptr(unsafe.Pointer(&buf[0])), - uintptr(unsafe.Pointer(lpString))) -} - -func GlobalAlloc(uFlags uint, dwBytes uint32) HGLOBAL { - ret, _, _ := procGlobalAlloc.Call( - uintptr(uFlags), - uintptr(dwBytes)) - - if ret == 0 { - panic("GlobalAlloc failed") - } - - return HGLOBAL(ret) -} - -func GlobalFree(hMem HGLOBAL) { - ret, _, _ := procGlobalFree.Call(uintptr(hMem)) - - if ret != 0 { - panic("GlobalFree failed") - } -} - -func GlobalLock(hMem HGLOBAL) unsafe.Pointer { - ret, _, _ := procGlobalLock.Call(uintptr(hMem)) - - if ret == 0 { - panic("GlobalLock failed") - } - - return unsafe.Pointer(ret) -} - -func GlobalUnlock(hMem HGLOBAL) bool { - ret, _, _ := procGlobalUnlock.Call(uintptr(hMem)) - - return ret != 0 -} - -func MoveMemory(destination, source unsafe.Pointer, length uint32) { - procMoveMemory.Call( - uintptr(unsafe.Pointer(destination)), - uintptr(source), - uintptr(length)) -} - -func FindResource(hModule HMODULE, lpName, lpType *uint16) (HRSRC, error) { - ret, _, _ := procFindResource.Call( - uintptr(hModule), - uintptr(unsafe.Pointer(lpName)), - uintptr(unsafe.Pointer(lpType))) - - if ret == 0 { - return 0, syscall.GetLastError() - } - - return HRSRC(ret), nil -} - -func SizeofResource(hModule HMODULE, hResInfo HRSRC) uint32 { - ret, _, _ := procSizeofResource.Call( - uintptr(hModule), - uintptr(hResInfo)) - - if ret == 0 { - panic("SizeofResource failed") - } - - return uint32(ret) -} - -func LockResource(hResData HGLOBAL) unsafe.Pointer { - ret, _, _ := procLockResource.Call(uintptr(hResData)) - - if ret == 0 { - panic("LockResource failed") - } - - return unsafe.Pointer(ret) -} - -func LoadResource(hModule HMODULE, hResInfo HRSRC) HGLOBAL { - ret, _, _ := procLoadResource.Call( - uintptr(hModule), - uintptr(hResInfo)) - - if ret == 0 { - panic("LoadResource failed") - } - - return HGLOBAL(ret) -} - -func GetLastError() uint32 { - ret, _, _ := procGetLastError.Call() - return uint32(ret) -} - -func OpenProcess(desiredAccess uint32, inheritHandle bool, processId uint32) HANDLE { - inherit := 0 - if inheritHandle { - inherit = 1 - } - - ret, _, _ := procOpenProcess.Call( - uintptr(desiredAccess), - uintptr(inherit), - uintptr(processId)) - return HANDLE(ret) -} - -func TerminateProcess(hProcess HANDLE, uExitCode uint) bool { - ret, _, _ := procTerminateProcess.Call( - uintptr(hProcess), - uintptr(uExitCode)) - return ret != 0 -} - -func CloseHandle(object HANDLE) bool { - ret, _, _ := procCloseHandle.Call( - uintptr(object)) - return ret != 0 -} - -func CreateToolhelp32Snapshot(flags, processId uint32) HANDLE { - ret, _, _ := procCreateToolhelp32Snapshot.Call( - uintptr(flags), - uintptr(processId)) - - if ret <= 0 { - return HANDLE(0) - } - - return HANDLE(ret) -} - -func Module32First(snapshot HANDLE, me *MODULEENTRY32) bool { - ret, _, _ := procModule32First.Call( - uintptr(snapshot), - uintptr(unsafe.Pointer(me))) - - return ret != 0 -} - -func Module32Next(snapshot HANDLE, me *MODULEENTRY32) bool { - ret, _, _ := procModule32Next.Call( - uintptr(snapshot), - uintptr(unsafe.Pointer(me))) - - return ret != 0 -} -func Process32First(snapshot HANDLE, pe *PROCESSENTRY32) bool { - ret, _, _ := procProcess32First.Call( - uintptr(snapshot), - uintptr(unsafe.Pointer(pe))) - - return ret != 0 -} - -func Process32Next(snapshot HANDLE, pe *PROCESSENTRY32) bool { - ret, _, _ := procProcess32Next.Call( - uintptr(snapshot), - uintptr(unsafe.Pointer(pe))) - - return ret != 0 -} -func GetSystemTimes(lpIdleTime, lpKernelTime, lpUserTime *FILETIME) bool { - ret, _, _ := procGetSystemTimes.Call( - uintptr(unsafe.Pointer(lpIdleTime)), - uintptr(unsafe.Pointer(lpKernelTime)), - uintptr(unsafe.Pointer(lpUserTime))) - - return ret != 0 -} - -func GetProcessTimes(hProcess HANDLE, lpCreationTime, lpExitTime, lpKernelTime, lpUserTime *FILETIME) bool { - ret, _, _ := procGetProcessTimes.Call( - uintptr(hProcess), - uintptr(unsafe.Pointer(lpCreationTime)), - uintptr(unsafe.Pointer(lpExitTime)), - uintptr(unsafe.Pointer(lpKernelTime)), - uintptr(unsafe.Pointer(lpUserTime))) - - return ret != 0 -} - -func GetConsoleScreenBufferInfo(hConsoleOutput HANDLE) *CONSOLE_SCREEN_BUFFER_INFO { - var csbi CONSOLE_SCREEN_BUFFER_INFO - ret, _, _ := procGetConsoleScreenBufferInfo.Call( - uintptr(hConsoleOutput), - uintptr(unsafe.Pointer(&csbi))) - if ret == 0 { - return nil - } - return &csbi -} - -func SetConsoleTextAttribute(hConsoleOutput HANDLE, wAttributes uint16) bool { - ret, _, _ := procSetConsoleTextAttribute.Call( - uintptr(hConsoleOutput), - uintptr(wAttributes)) - return ret != 0 -} - -func GetDiskFreeSpaceEx(dirName string) (r bool, - freeBytesAvailable, totalNumberOfBytes, totalNumberOfFreeBytes uint64) { - ret, _, _ := procGetDiskFreeSpaceEx.Call( - uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(dirName))), - uintptr(unsafe.Pointer(&freeBytesAvailable)), - uintptr(unsafe.Pointer(&totalNumberOfBytes)), - uintptr(unsafe.Pointer(&totalNumberOfFreeBytes))) - return ret != 0, - freeBytesAvailable, totalNumberOfBytes, totalNumberOfFreeBytes -} diff --git a/github.com/shirou/w32/ole32.go b/github.com/shirou/w32/ole32.go deleted file mode 100644 index 48589848c1..0000000000 --- a/github.com/shirou/w32/ole32.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2010-2012 The W32 Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build windows - -package w32 - -import ( - "syscall" - "unsafe" -) - -var ( - modole32 = syscall.NewLazyDLL("ole32.dll") - - procCoInitializeEx = modole32.NewProc("CoInitializeEx") - procCoInitialize = modole32.NewProc("CoInitialize") - procCoUninitialize = modole32.NewProc("CoUninitialize") - procCreateStreamOnHGlobal = modole32.NewProc("CreateStreamOnHGlobal") -) - -func CoInitializeEx(coInit uintptr) HRESULT { - ret, _, _ := procCoInitializeEx.Call( - 0, - coInit) - - switch uint32(ret) { - case E_INVALIDARG: - panic("CoInitializeEx failed with E_INVALIDARG") - case E_OUTOFMEMORY: - panic("CoInitializeEx failed with E_OUTOFMEMORY") - case E_UNEXPECTED: - panic("CoInitializeEx failed with E_UNEXPECTED") - } - - return HRESULT(ret) -} - -func CoInitialize() { - procCoInitialize.Call(0) -} - -func CoUninitialize() { - procCoUninitialize.Call() -} - -func CreateStreamOnHGlobal(hGlobal HGLOBAL, fDeleteOnRelease bool) *IStream { - stream := new(IStream) - ret, _, _ := procCreateStreamOnHGlobal.Call( - uintptr(hGlobal), - uintptr(BoolToBOOL(fDeleteOnRelease)), - uintptr(unsafe.Pointer(&stream))) - - switch uint32(ret) { - case E_INVALIDARG: - panic("CreateStreamOnHGlobal failed with E_INVALIDARG") - case E_OUTOFMEMORY: - panic("CreateStreamOnHGlobal failed with E_OUTOFMEMORY") - case E_UNEXPECTED: - panic("CreateStreamOnHGlobal failed with E_UNEXPECTED") - } - - return stream -} diff --git a/github.com/shirou/w32/oleaut32.go b/github.com/shirou/w32/oleaut32.go deleted file mode 100644 index cdfcb00381..0000000000 --- a/github.com/shirou/w32/oleaut32.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2010-2012 The W32 Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build windows - -package w32 - -import ( - "syscall" - "unsafe" -) - -var ( - modoleaut32 = syscall.NewLazyDLL("oleaut32") - - procVariantInit = modoleaut32.NewProc("VariantInit") - procSysAllocString = modoleaut32.NewProc("SysAllocString") - procSysFreeString = modoleaut32.NewProc("SysFreeString") - procSysStringLen = modoleaut32.NewProc("SysStringLen") - procCreateDispTypeInfo = modoleaut32.NewProc("CreateDispTypeInfo") - procCreateStdDispatch = modoleaut32.NewProc("CreateStdDispatch") -) - -func VariantInit(v *VARIANT) { - hr, _, _ := procVariantInit.Call(uintptr(unsafe.Pointer(v))) - if hr != 0 { - panic("Invoke VariantInit error.") - } - return -} - -func SysAllocString(v string) (ss *int16) { - pss, _, _ := procSysAllocString.Call(uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(v)))) - ss = (*int16)(unsafe.Pointer(pss)) - return -} - -func SysFreeString(v *int16) { - hr, _, _ := procSysFreeString.Call(uintptr(unsafe.Pointer(v))) - if hr != 0 { - panic("Invoke SysFreeString error.") - } - return -} - -func SysStringLen(v *int16) uint { - l, _, _ := procSysStringLen.Call(uintptr(unsafe.Pointer(v))) - return uint(l) -} diff --git a/github.com/shirou/w32/opengl32.go b/github.com/shirou/w32/opengl32.go deleted file mode 100644 index 4f35f19ef8..0000000000 --- a/github.com/shirou/w32/opengl32.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2010-2012 The W32 Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build windows - -package w32 - -import ( - "syscall" - "unsafe" -) - -var ( - modopengl32 = syscall.NewLazyDLL("opengl32.dll") - - procwglCreateContext = modopengl32.NewProc("wglCreateContext") - procwglCreateLayerContext = modopengl32.NewProc("wglCreateLayerContext") - procwglDeleteContext = modopengl32.NewProc("wglDeleteContext") - procwglGetProcAddress = modopengl32.NewProc("wglGetProcAddress") - procwglMakeCurrent = modopengl32.NewProc("wglMakeCurrent") - procwglShareLists = modopengl32.NewProc("wglShareLists") -) - -func WglCreateContext(hdc HDC) HGLRC { - ret, _, _ := procwglCreateContext.Call( - uintptr(hdc), - ) - - return HGLRC(ret) -} - -func WglCreateLayerContext(hdc HDC, iLayerPlane int) HGLRC { - ret, _, _ := procwglCreateLayerContext.Call( - uintptr(hdc), - uintptr(iLayerPlane), - ) - - return HGLRC(ret) -} - -func WglDeleteContext(hglrc HGLRC) bool { - ret, _, _ := procwglDeleteContext.Call( - uintptr(hglrc), - ) - - return ret == TRUE -} - -func WglGetProcAddress(szProc string) uintptr { - ret, _, _ := procwglGetProcAddress.Call( - uintptr(unsafe.Pointer(syscall.StringBytePtr(szProc))), - ) - - return ret -} - -func WglMakeCurrent(hdc HDC, hglrc HGLRC) bool { - ret, _, _ := procwglMakeCurrent.Call( - uintptr(hdc), - uintptr(hglrc), - ) - - return ret == TRUE -} - -func WglShareLists(hglrc1, hglrc2 HGLRC) bool { - ret, _, _ := procwglShareLists.Call( - uintptr(hglrc1), - uintptr(hglrc2), - ) - - return ret == TRUE -} diff --git a/github.com/shirou/w32/psapi.go b/github.com/shirou/w32/psapi.go deleted file mode 100644 index ab7858cb55..0000000000 --- a/github.com/shirou/w32/psapi.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2010-2012 The W32 Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build windows - -package w32 - -import ( - "syscall" - "unsafe" -) - -var ( - modpsapi = syscall.NewLazyDLL("psapi.dll") - - procEnumProcesses = modpsapi.NewProc("EnumProcesses") -) - -func EnumProcesses(processIds []uint32, cb uint32, bytesReturned *uint32) bool { - ret, _, _ := procEnumProcesses.Call( - uintptr(unsafe.Pointer(&processIds[0])), - uintptr(cb), - uintptr(unsafe.Pointer(bytesReturned))) - - return ret != 0 -} diff --git a/github.com/shirou/w32/shell32.go b/github.com/shirou/w32/shell32.go deleted file mode 100644 index 0f5ce8cbd0..0000000000 --- a/github.com/shirou/w32/shell32.go +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright 2010-2012 The W32 Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build windows - -package w32 - -import ( - "errors" - "fmt" - "syscall" - "unsafe" -) - -var ( - modshell32 = syscall.NewLazyDLL("shell32.dll") - - procSHBrowseForFolder = modshell32.NewProc("SHBrowseForFolderW") - procSHGetPathFromIDList = modshell32.NewProc("SHGetPathFromIDListW") - procDragAcceptFiles = modshell32.NewProc("DragAcceptFiles") - procDragQueryFile = modshell32.NewProc("DragQueryFileW") - procDragQueryPoint = modshell32.NewProc("DragQueryPoint") - procDragFinish = modshell32.NewProc("DragFinish") - procShellExecute = modshell32.NewProc("ShellExecuteW") - procExtractIcon = modshell32.NewProc("ExtractIconW") -) - -func SHBrowseForFolder(bi *BROWSEINFO) uintptr { - ret, _, _ := procSHBrowseForFolder.Call(uintptr(unsafe.Pointer(bi))) - - return ret -} - -func SHGetPathFromIDList(idl uintptr) string { - buf := make([]uint16, 1024) - procSHGetPathFromIDList.Call( - idl, - uintptr(unsafe.Pointer(&buf[0]))) - - return syscall.UTF16ToString(buf) -} - -func DragAcceptFiles(hwnd HWND, accept bool) { - procDragAcceptFiles.Call( - uintptr(hwnd), - uintptr(BoolToBOOL(accept))) -} - -func DragQueryFile(hDrop HDROP, iFile uint) (fileName string, fileCount uint) { - ret, _, _ := procDragQueryFile.Call( - uintptr(hDrop), - uintptr(iFile), - 0, - 0) - - fileCount = uint(ret) - - if iFile != 0xFFFFFFFF { - buf := make([]uint16, fileCount+1) - - ret, _, _ := procDragQueryFile.Call( - uintptr(hDrop), - uintptr(iFile), - uintptr(unsafe.Pointer(&buf[0])), - uintptr(fileCount+1)) - - if ret == 0 { - panic("Invoke DragQueryFile error.") - } - - fileName = syscall.UTF16ToString(buf) - } - - return -} - -func DragQueryPoint(hDrop HDROP) (x, y int, isClientArea bool) { - var pt POINT - ret, _, _ := procDragQueryPoint.Call( - uintptr(hDrop), - uintptr(unsafe.Pointer(&pt))) - - return int(pt.X), int(pt.Y), (ret == 1) -} - -func DragFinish(hDrop HDROP) { - procDragFinish.Call(uintptr(hDrop)) -} - -func ShellExecute(hwnd HWND, lpOperation, lpFile, lpParameters, lpDirectory string, nShowCmd int) error { - var op, param, directory uintptr - if len(lpOperation) != 0 { - op = uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(lpOperation))) - } - if len(lpParameters) != 0 { - param = uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(lpParameters))) - } - if len(lpDirectory) != 0 { - directory = uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(lpDirectory))) - } - - ret, _, _ := procShellExecute.Call( - uintptr(hwnd), - op, - uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(lpFile))), - param, - directory, - uintptr(nShowCmd)) - - errorMsg := "" - if ret != 0 && ret <= 32 { - switch int(ret) { - case ERROR_FILE_NOT_FOUND: - errorMsg = "The specified file was not found." - case ERROR_PATH_NOT_FOUND: - errorMsg = "The specified path was not found." - case ERROR_BAD_FORMAT: - errorMsg = "The .exe file is invalid (non-Win32 .exe or error in .exe image)." - case SE_ERR_ACCESSDENIED: - errorMsg = "The operating system denied access to the specified file." - case SE_ERR_ASSOCINCOMPLETE: - errorMsg = "The file name association is incomplete or invalid." - case SE_ERR_DDEBUSY: - errorMsg = "The DDE transaction could not be completed because other DDE transactions were being processed." - case SE_ERR_DDEFAIL: - errorMsg = "The DDE transaction failed." - case SE_ERR_DDETIMEOUT: - errorMsg = "The DDE transaction could not be completed because the request timed out." - case SE_ERR_DLLNOTFOUND: - errorMsg = "The specified DLL was not found." - case SE_ERR_NOASSOC: - errorMsg = "There is no application associated with the given file name extension. This error will also be returned if you attempt to print a file that is not printable." - case SE_ERR_OOM: - errorMsg = "There was not enough memory to complete the operation." - case SE_ERR_SHARE: - errorMsg = "A sharing violation occurred." - default: - errorMsg = fmt.Sprintf("Unknown error occurred with error code %v", ret) - } - } else { - return nil - } - - return errors.New(errorMsg) -} - -func ExtractIcon(lpszExeFileName string, nIconIndex int) HICON { - ret, _, _ := procExtractIcon.Call( - 0, - uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(lpszExeFileName))), - uintptr(nIconIndex)) - - return HICON(ret) -} diff --git a/github.com/shirou/w32/typedef.go b/github.com/shirou/w32/typedef.go deleted file mode 100644 index 65f5111292..0000000000 --- a/github.com/shirou/w32/typedef.go +++ /dev/null @@ -1,901 +0,0 @@ -// Copyright 2010-2012 The W32 Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package w32 - -import ( - "unsafe" -) - -// From MSDN: Windows Data Types -// http://msdn.microsoft.com/en-us/library/s3f49ktz.aspx -// http://msdn.microsoft.com/en-us/library/windows/desktop/aa383751.aspx -// ATOM WORD -// BOOL int32 -// BOOLEAN byte -// BYTE byte -// CCHAR int8 -// CHAR int8 -// COLORREF DWORD -// DWORD uint32 -// DWORDLONG ULONGLONG -// DWORD_PTR ULONG_PTR -// DWORD32 uint32 -// DWORD64 uint64 -// FLOAT float32 -// HACCEL HANDLE -// HALF_PTR struct{} // ??? -// HANDLE PVOID -// HBITMAP HANDLE -// HBRUSH HANDLE -// HCOLORSPACE HANDLE -// HCONV HANDLE -// HCONVLIST HANDLE -// HCURSOR HANDLE -// HDC HANDLE -// HDDEDATA HANDLE -// HDESK HANDLE -// HDROP HANDLE -// HDWP HANDLE -// HENHMETAFILE HANDLE -// HFILE HANDLE -// HFONT HANDLE -// HGDIOBJ HANDLE -// HGLOBAL HANDLE -// HHOOK HANDLE -// HICON HANDLE -// HINSTANCE HANDLE -// HKEY HANDLE -// HKL HANDLE -// HLOCAL HANDLE -// HMENU HANDLE -// HMETAFILE HANDLE -// HMODULE HANDLE -// HPALETTE HANDLE -// HPEN HANDLE -// HRESULT int32 -// HRGN HANDLE -// HSZ HANDLE -// HWINSTA HANDLE -// HWND HANDLE -// INT int32 -// INT_PTR uintptr -// INT8 int8 -// INT16 int16 -// INT32 int32 -// INT64 int64 -// LANGID WORD -// LCID DWORD -// LCTYPE DWORD -// LGRPID DWORD -// LONG int32 -// LONGLONG int64 -// LONG_PTR uintptr -// LONG32 int32 -// LONG64 int64 -// LPARAM LONG_PTR -// LPBOOL *BOOL -// LPBYTE *BYTE -// LPCOLORREF *COLORREF -// LPCSTR *int8 -// LPCTSTR LPCWSTR -// LPCVOID unsafe.Pointer -// LPCWSTR *WCHAR -// LPDWORD *DWORD -// LPHANDLE *HANDLE -// LPINT *INT -// LPLONG *LONG -// LPSTR *CHAR -// LPTSTR LPWSTR -// LPVOID unsafe.Pointer -// LPWORD *WORD -// LPWSTR *WCHAR -// LRESULT LONG_PTR -// PBOOL *BOOL -// PBOOLEAN *BOOLEAN -// PBYTE *BYTE -// PCHAR *CHAR -// PCSTR *CHAR -// PCTSTR PCWSTR -// PCWSTR *WCHAR -// PDWORD *DWORD -// PDWORDLONG *DWORDLONG -// PDWORD_PTR *DWORD_PTR -// PDWORD32 *DWORD32 -// PDWORD64 *DWORD64 -// PFLOAT *FLOAT -// PHALF_PTR *HALF_PTR -// PHANDLE *HANDLE -// PHKEY *HKEY -// PINT_PTR *INT_PTR -// PINT8 *INT8 -// PINT16 *INT16 -// PINT32 *INT32 -// PINT64 *INT64 -// PLCID *LCID -// PLONG *LONG -// PLONGLONG *LONGLONG -// PLONG_PTR *LONG_PTR -// PLONG32 *LONG32 -// PLONG64 *LONG64 -// POINTER_32 struct{} // ??? -// POINTER_64 struct{} // ??? -// POINTER_SIGNED uintptr -// POINTER_UNSIGNED uintptr -// PSHORT *SHORT -// PSIZE_T *SIZE_T -// PSSIZE_T *SSIZE_T -// PSTR *CHAR -// PTBYTE *TBYTE -// PTCHAR *TCHAR -// PTSTR PWSTR -// PUCHAR *UCHAR -// PUHALF_PTR *UHALF_PTR -// PUINT *UINT -// PUINT_PTR *UINT_PTR -// PUINT8 *UINT8 -// PUINT16 *UINT16 -// PUINT32 *UINT32 -// PUINT64 *UINT64 -// PULONG *ULONG -// PULONGLONG *ULONGLONG -// PULONG_PTR *ULONG_PTR -// PULONG32 *ULONG32 -// PULONG64 *ULONG64 -// PUSHORT *USHORT -// PVOID unsafe.Pointer -// PWCHAR *WCHAR -// PWORD *WORD -// PWSTR *WCHAR -// QWORD uint64 -// SC_HANDLE HANDLE -// SC_LOCK LPVOID -// SERVICE_STATUS_HANDLE HANDLE -// SHORT int16 -// SIZE_T ULONG_PTR -// SSIZE_T LONG_PTR -// TBYTE WCHAR -// TCHAR WCHAR -// UCHAR uint8 -// UHALF_PTR struct{} // ??? -// UINT uint32 -// UINT_PTR uintptr -// UINT8 uint8 -// UINT16 uint16 -// UINT32 uint32 -// UINT64 uint64 -// ULONG uint32 -// ULONGLONG uint64 -// ULONG_PTR uintptr -// ULONG32 uint32 -// ULONG64 uint64 -// USHORT uint16 -// USN LONGLONG -// WCHAR uint16 -// WORD uint16 -// WPARAM UINT_PTR -type ( - ATOM uint16 - BOOL int32 - COLORREF uint32 - DWM_FRAME_COUNT uint64 - HACCEL HANDLE - HANDLE uintptr - HBITMAP HANDLE - HBRUSH HANDLE - HCURSOR HANDLE - HDC HANDLE - HDROP HANDLE - HDWP HANDLE - HENHMETAFILE HANDLE - HFONT HANDLE - HGDIOBJ HANDLE - HGLOBAL HANDLE - HGLRC HANDLE - HICON HANDLE - HIMAGELIST HANDLE - HINSTANCE HANDLE - HKEY HANDLE - HKL HANDLE - HMENU HANDLE - HMODULE HANDLE - HMONITOR HANDLE - HPEN HANDLE - HRESULT int32 - HRGN HANDLE - HRSRC HANDLE - HTHUMBNAIL HANDLE - HWND HANDLE - LPCVOID unsafe.Pointer - PVOID unsafe.Pointer - QPC_TIME uint64 -) - -// http://msdn.microsoft.com/en-us/library/windows/desktop/dd162805.aspx -type POINT struct { - X, Y int32 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/dd162897.aspx -type RECT struct { - Left, Top, Right, Bottom int32 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/ms633577.aspx -type WNDCLASSEX struct { - Size uint32 - Style uint32 - WndProc uintptr - ClsExtra int32 - WndExtra int32 - Instance HINSTANCE - Icon HICON - Cursor HCURSOR - Background HBRUSH - MenuName *uint16 - ClassName *uint16 - IconSm HICON -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/ms644958.aspx -type MSG struct { - Hwnd HWND - Message uint32 - WParam uintptr - LParam uintptr - Time uint32 - Pt POINT -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/dd145037.aspx -type LOGFONT struct { - Height int32 - Width int32 - Escapement int32 - Orientation int32 - Weight int32 - Italic byte - Underline byte - StrikeOut byte - CharSet byte - OutPrecision byte - ClipPrecision byte - Quality byte - PitchAndFamily byte - FaceName [LF_FACESIZE]uint16 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/ms646839.aspx -type OPENFILENAME struct { - StructSize uint32 - Owner HWND - Instance HINSTANCE - Filter *uint16 - CustomFilter *uint16 - MaxCustomFilter uint32 - FilterIndex uint32 - File *uint16 - MaxFile uint32 - FileTitle *uint16 - MaxFileTitle uint32 - InitialDir *uint16 - Title *uint16 - Flags uint32 - FileOffset uint16 - FileExtension uint16 - DefExt *uint16 - CustData uintptr - FnHook uintptr - TemplateName *uint16 - PvReserved unsafe.Pointer - DwReserved uint32 - FlagsEx uint32 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/bb773205.aspx -type BROWSEINFO struct { - Owner HWND - Root *uint16 - DisplayName *uint16 - Title *uint16 - Flags uint32 - CallbackFunc uintptr - LParam uintptr - Image int32 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/aa373931.aspx -type GUID struct { - Data1 uint32 - Data2 uint16 - Data3 uint16 - Data4 [8]byte -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/ms221627.aspx -type VARIANT struct { - VT uint16 // 2 - WReserved1 uint16 // 4 - WReserved2 uint16 // 6 - WReserved3 uint16 // 8 - Val int64 // 16 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/ms221416.aspx -type DISPPARAMS struct { - Rgvarg uintptr - RgdispidNamedArgs uintptr - CArgs uint32 - CNamedArgs uint32 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/ms221133.aspx -type EXCEPINFO struct { - WCode uint16 - WReserved uint16 - BstrSource *uint16 - BstrDescription *uint16 - BstrHelpFile *uint16 - DwHelpContext uint32 - PvReserved uintptr - PfnDeferredFillIn uintptr - Scode int32 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/dd145035.aspx -type LOGBRUSH struct { - LbStyle uint32 - LbColor COLORREF - LbHatch uintptr -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/dd183565.aspx -type DEVMODE struct { - DmDeviceName [CCHDEVICENAME]uint16 - DmSpecVersion uint16 - DmDriverVersion uint16 - DmSize uint16 - DmDriverExtra uint16 - DmFields uint32 - DmOrientation int16 - DmPaperSize int16 - DmPaperLength int16 - DmPaperWidth int16 - DmScale int16 - DmCopies int16 - DmDefaultSource int16 - DmPrintQuality int16 - DmColor int16 - DmDuplex int16 - DmYResolution int16 - DmTTOption int16 - DmCollate int16 - DmFormName [CCHFORMNAME]uint16 - DmLogPixels uint16 - DmBitsPerPel uint32 - DmPelsWidth uint32 - DmPelsHeight uint32 - DmDisplayFlags uint32 - DmDisplayFrequency uint32 - DmICMMethod uint32 - DmICMIntent uint32 - DmMediaType uint32 - DmDitherType uint32 - DmReserved1 uint32 - DmReserved2 uint32 - DmPanningWidth uint32 - DmPanningHeight uint32 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/dd183376.aspx -type BITMAPINFOHEADER struct { - BiSize uint32 - BiWidth int32 - BiHeight int32 - BiPlanes uint16 - BiBitCount uint16 - BiCompression uint32 - BiSizeImage uint32 - BiXPelsPerMeter int32 - BiYPelsPerMeter int32 - BiClrUsed uint32 - BiClrImportant uint32 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/dd162938.aspx -type RGBQUAD struct { - RgbBlue byte - RgbGreen byte - RgbRed byte - RgbReserved byte -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/dd183375.aspx -type BITMAPINFO struct { - BmiHeader BITMAPINFOHEADER - BmiColors *RGBQUAD -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/dd183371.aspx -type BITMAP struct { - BmType int32 - BmWidth int32 - BmHeight int32 - BmWidthBytes int32 - BmPlanes uint16 - BmBitsPixel uint16 - BmBits unsafe.Pointer -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/dd183567.aspx -type DIBSECTION struct { - DsBm BITMAP - DsBmih BITMAPINFOHEADER - DsBitfields [3]uint32 - DshSection HANDLE - DsOffset uint32 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/dd162607.aspx -type ENHMETAHEADER struct { - IType uint32 - NSize uint32 - RclBounds RECT - RclFrame RECT - DSignature uint32 - NVersion uint32 - NBytes uint32 - NRecords uint32 - NHandles uint16 - SReserved uint16 - NDescription uint32 - OffDescription uint32 - NPalEntries uint32 - SzlDevice SIZE - SzlMillimeters SIZE - CbPixelFormat uint32 - OffPixelFormat uint32 - BOpenGL uint32 - SzlMicrometers SIZE -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/dd145106.aspx -type SIZE struct { - CX, CY int32 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/dd145132.aspx -type TEXTMETRIC struct { - TmHeight int32 - TmAscent int32 - TmDescent int32 - TmInternalLeading int32 - TmExternalLeading int32 - TmAveCharWidth int32 - TmMaxCharWidth int32 - TmWeight int32 - TmOverhang int32 - TmDigitizedAspectX int32 - TmDigitizedAspectY int32 - TmFirstChar uint16 - TmLastChar uint16 - TmDefaultChar uint16 - TmBreakChar uint16 - TmItalic byte - TmUnderlined byte - TmStruckOut byte - TmPitchAndFamily byte - TmCharSet byte -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/dd183574.aspx -type DOCINFO struct { - CbSize int32 - LpszDocName *uint16 - LpszOutput *uint16 - LpszDatatype *uint16 - FwType uint32 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/bb775514.aspx -type NMHDR struct { - HwndFrom HWND - IdFrom uintptr - Code uint32 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/bb774743.aspx -type LVCOLUMN struct { - Mask uint32 - Fmt int32 - Cx int32 - PszText *uint16 - CchTextMax int32 - ISubItem int32 - IImage int32 - IOrder int32 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/bb774760.aspx -type LVITEM struct { - Mask uint32 - IItem int32 - ISubItem int32 - State uint32 - StateMask uint32 - PszText *uint16 - CchTextMax int32 - IImage int32 - LParam uintptr - IIndent int32 - IGroupId int32 - CColumns uint32 - PuColumns uint32 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/bb774754.aspx -type LVHITTESTINFO struct { - Pt POINT - Flags uint32 - IItem int32 - ISubItem int32 - IGroup int32 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/bb774771.aspx -type NMITEMACTIVATE struct { - Hdr NMHDR - IItem int32 - ISubItem int32 - UNewState uint32 - UOldState uint32 - UChanged uint32 - PtAction POINT - LParam uintptr - UKeyFlags uint32 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/bb774773.aspx -type NMLISTVIEW struct { - Hdr NMHDR - IItem int32 - ISubItem int32 - UNewState uint32 - UOldState uint32 - UChanged uint32 - PtAction POINT - LParam uintptr -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/bb774780.aspx -type NMLVDISPINFO struct { - Hdr NMHDR - Item LVITEM -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/bb775507.aspx -type INITCOMMONCONTROLSEX struct { - DwSize uint32 - DwICC uint32 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/bb760256.aspx -type TOOLINFO struct { - CbSize uint32 - UFlags uint32 - Hwnd HWND - UId uintptr - Rect RECT - Hinst HINSTANCE - LpszText *uint16 - LParam uintptr - LpReserved unsafe.Pointer -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/ms645604.aspx -type TRACKMOUSEEVENT struct { - CbSize uint32 - DwFlags uint32 - HwndTrack HWND - DwHoverTime uint32 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/ms534067.aspx -type GdiplusStartupInput struct { - GdiplusVersion uint32 - DebugEventCallback uintptr - SuppressBackgroundThread BOOL - SuppressExternalCodecs BOOL -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/ms534068.aspx -type GdiplusStartupOutput struct { - NotificationHook uintptr - NotificationUnhook uintptr -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/dd162768.aspx -type PAINTSTRUCT struct { - Hdc HDC - FErase BOOL - RcPaint RECT - FRestore BOOL - FIncUpdate BOOL - RgbReserved [32]byte -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/aa363646.aspx -type EVENTLOGRECORD struct { - Length uint32 - Reserved uint32 - RecordNumber uint32 - TimeGenerated uint32 - TimeWritten uint32 - EventID uint32 - EventType uint16 - NumStrings uint16 - EventCategory uint16 - ReservedFlags uint16 - ClosingRecordNumber uint32 - StringOffset uint32 - UserSidLength uint32 - UserSidOffset uint32 - DataLength uint32 - DataOffset uint32 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/ms685996.aspx -type SERVICE_STATUS struct { - DwServiceType uint32 - DwCurrentState uint32 - DwControlsAccepted uint32 - DwWin32ExitCode uint32 - DwServiceSpecificExitCode uint32 - DwCheckPoint uint32 - DwWaitHint uint32 -} -type PROCESSENTRY32 struct { - DwSize uint32 - CntUsage uint32 - Th32ProcessID uint32 - Th32DefaultHeapID uintptr - Th32ModuleID uint32 - CntThreads uint32 - Th32ParentProcessID uint32 - PcPriClassBase int32 - DwFlags uint32 - SzExeFile [MAX_PATH]uint16 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/ms684225.aspx -type MODULEENTRY32 struct { - Size uint32 - ModuleID uint32 - ProcessID uint32 - GlblcntUsage uint32 - ProccntUsage uint32 - ModBaseAddr *uint8 - ModBaseSize uint32 - HModule HMODULE - SzModule [MAX_MODULE_NAME32 + 1]uint16 - SzExePath [MAX_PATH]uint16 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/ms724284.aspx -type FILETIME struct { - DwLowDateTime uint32 - DwHighDateTime uint32 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/ms682119.aspx -type COORD struct { - X, Y int16 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/ms686311.aspx -type SMALL_RECT struct { - Left, Top, Right, Bottom int16 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/ms682093.aspx -type CONSOLE_SCREEN_BUFFER_INFO struct { - DwSize COORD - DwCursorPosition COORD - WAttributes uint16 - SrWindow SMALL_RECT - DwMaximumWindowSize COORD -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/bb773244.aspx -type MARGINS struct { - CxLeftWidth, CxRightWidth, CyTopHeight, CyBottomHeight int32 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/aa969500.aspx -type DWM_BLURBEHIND struct { - DwFlags uint32 - fEnable BOOL - hRgnBlur HRGN - fTransitionOnMaximized BOOL -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/aa969501.aspx -type DWM_PRESENT_PARAMETERS struct { - cbSize uint32 - fQueue BOOL - cRefreshStart DWM_FRAME_COUNT - cBuffer uint32 - fUseSourceRate BOOL - rateSource UNSIGNED_RATIO - cRefreshesPerFrame uint32 - eSampling DWM_SOURCE_FRAME_SAMPLING -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/aa969502.aspx -type DWM_THUMBNAIL_PROPERTIES struct { - dwFlags uint32 - rcDestination RECT - rcSource RECT - opacity byte - fVisible BOOL - fSourceClientAreaOnly BOOL -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/aa969503.aspx -type DWM_TIMING_INFO struct { - cbSize uint32 - rateRefresh UNSIGNED_RATIO - qpcRefreshPeriod QPC_TIME - rateCompose UNSIGNED_RATIO - qpcVBlank QPC_TIME - cRefresh DWM_FRAME_COUNT - cDXRefresh uint32 - qpcCompose QPC_TIME - cFrame DWM_FRAME_COUNT - cDXPresent uint32 - cRefreshFrame DWM_FRAME_COUNT - cFrameSubmitted DWM_FRAME_COUNT - cDXPresentSubmitted uint32 - cFrameConfirmed DWM_FRAME_COUNT - cDXPresentConfirmed uint32 - cRefreshConfirmed DWM_FRAME_COUNT - cDXRefreshConfirmed uint32 - cFramesLate DWM_FRAME_COUNT - cFramesOutstanding uint32 - cFrameDisplayed DWM_FRAME_COUNT - qpcFrameDisplayed QPC_TIME - cRefreshFrameDisplayed DWM_FRAME_COUNT - cFrameComplete DWM_FRAME_COUNT - qpcFrameComplete QPC_TIME - cFramePending DWM_FRAME_COUNT - qpcFramePending QPC_TIME - cFramesDisplayed DWM_FRAME_COUNT - cFramesComplete DWM_FRAME_COUNT - cFramesPending DWM_FRAME_COUNT - cFramesAvailable DWM_FRAME_COUNT - cFramesDropped DWM_FRAME_COUNT - cFramesMissed DWM_FRAME_COUNT - cRefreshNextDisplayed DWM_FRAME_COUNT - cRefreshNextPresented DWM_FRAME_COUNT - cRefreshesDisplayed DWM_FRAME_COUNT - cRefreshesPresented DWM_FRAME_COUNT - cRefreshStarted DWM_FRAME_COUNT - cPixelsReceived uint64 - cPixelsDrawn uint64 - cBuffersEmpty DWM_FRAME_COUNT -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/dd389402.aspx -type MilMatrix3x2D struct { - S_11, S_12, S_21, S_22 float64 - DX, DY float64 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/aa969505.aspx -type UNSIGNED_RATIO struct { - uiNumerator uint32 - uiDenominator uint32 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/ms632603.aspx -type CREATESTRUCT struct { - CreateParams uintptr - Instance HINSTANCE - Menu HMENU - Parent HWND - Cy, Cx int32 - Y, X int32 - Style int32 - Name *uint16 - Class *uint16 - dwExStyle uint32 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/dd145065.aspx -type MONITORINFO struct { - CbSize uint32 - RcMonitor RECT - RcWork RECT - DwFlags uint32 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/dd145066.aspx -type MONITORINFOEX struct { - MONITORINFO - SzDevice [CCHDEVICENAME]uint16 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/dd368826.aspx -type PIXELFORMATDESCRIPTOR struct { - Size uint16 - Version uint16 - DwFlags uint32 - IPixelType byte - ColorBits byte - RedBits, RedShift byte - GreenBits, GreenShift byte - BlueBits, BlueShift byte - AlphaBits, AlphaShift byte - AccumBits byte - AccumRedBits byte - AccumGreenBits byte - AccumBlueBits byte - AccumAlphaBits byte - DepthBits, StencilBits byte - AuxBuffers byte - ILayerType byte - Reserved byte - DwLayerMask uint32 - DwVisibleMask uint32 - DwDamageMask uint32 -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/ms646270(v=vs.85).aspx -type INPUT struct { - Type uint32 - Mi MOUSEINPUT - Ki KEYBDINPUT - Hi HARDWAREINPUT -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/ms646273(v=vs.85).aspx -type MOUSEINPUT struct { - Dx int32 - Dy int32 - MouseData uint32 - DwFlags uint32 - Time uint32 - DwExtraInfo uintptr -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/ms646271(v=vs.85).aspx -type KEYBDINPUT struct { - WVk uint16 - WScan uint16 - DwFlags uint32 - Time uint32 - DwExtraInfo uintptr -} - -// http://msdn.microsoft.com/en-us/library/windows/desktop/ms646269(v=vs.85).aspx -type HARDWAREINPUT struct { - UMsg uint32 - WParamL uint16 - WParamH uint16 -} - -type KbdInput struct { - typ uint32 - ki KEYBDINPUT -} - -type MouseInput struct { - typ uint32 - mi MOUSEINPUT -} - -type HardwareInput struct { - typ uint32 - hi HARDWAREINPUT -} diff --git a/github.com/shirou/w32/user32.go b/github.com/shirou/w32/user32.go deleted file mode 100644 index 6aa7cd7059..0000000000 --- a/github.com/shirou/w32/user32.go +++ /dev/null @@ -1,950 +0,0 @@ -// Copyright 2010-2012 The W32 Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build windows - -package w32 - -import ( - "fmt" - "syscall" - "unsafe" -) - -var ( - moduser32 = syscall.NewLazyDLL("user32.dll") - - procRegisterClassEx = moduser32.NewProc("RegisterClassExW") - procLoadIcon = moduser32.NewProc("LoadIconW") - procLoadCursor = moduser32.NewProc("LoadCursorW") - procShowWindow = moduser32.NewProc("ShowWindow") - procUpdateWindow = moduser32.NewProc("UpdateWindow") - procCreateWindowEx = moduser32.NewProc("CreateWindowExW") - procAdjustWindowRect = moduser32.NewProc("AdjustWindowRect") - procAdjustWindowRectEx = moduser32.NewProc("AdjustWindowRectEx") - procDestroyWindow = moduser32.NewProc("DestroyWindow") - procDefWindowProc = moduser32.NewProc("DefWindowProcW") - procDefDlgProc = moduser32.NewProc("DefDlgProcW") - procPostQuitMessage = moduser32.NewProc("PostQuitMessage") - procGetMessage = moduser32.NewProc("GetMessageW") - procTranslateMessage = moduser32.NewProc("TranslateMessage") - procDispatchMessage = moduser32.NewProc("DispatchMessageW") - procSendMessage = moduser32.NewProc("SendMessageW") - procPostMessage = moduser32.NewProc("PostMessageW") - procWaitMessage = moduser32.NewProc("WaitMessage") - procSetWindowText = moduser32.NewProc("SetWindowTextW") - procGetWindowTextLength = moduser32.NewProc("GetWindowTextLengthW") - procGetWindowText = moduser32.NewProc("GetWindowTextW") - procGetWindowRect = moduser32.NewProc("GetWindowRect") - procMoveWindow = moduser32.NewProc("MoveWindow") - procScreenToClient = moduser32.NewProc("ScreenToClient") - procCallWindowProc = moduser32.NewProc("CallWindowProcW") - procSetWindowLong = moduser32.NewProc("SetWindowLongW") - procSetWindowLongPtr = moduser32.NewProc("SetWindowLongW") - procGetWindowLong = moduser32.NewProc("GetWindowLongW") - procGetWindowLongPtr = moduser32.NewProc("GetWindowLongW") - procEnableWindow = moduser32.NewProc("EnableWindow") - procIsWindowEnabled = moduser32.NewProc("IsWindowEnabled") - procIsWindowVisible = moduser32.NewProc("IsWindowVisible") - procSetFocus = moduser32.NewProc("SetFocus") - procInvalidateRect = moduser32.NewProc("InvalidateRect") - procGetClientRect = moduser32.NewProc("GetClientRect") - procGetDC = moduser32.NewProc("GetDC") - procReleaseDC = moduser32.NewProc("ReleaseDC") - procSetCapture = moduser32.NewProc("SetCapture") - procReleaseCapture = moduser32.NewProc("ReleaseCapture") - procGetWindowThreadProcessId = moduser32.NewProc("GetWindowThreadProcessId") - procMessageBox = moduser32.NewProc("MessageBoxW") - procGetSystemMetrics = moduser32.NewProc("GetSystemMetrics") - procCopyRect = moduser32.NewProc("CopyRect") - procEqualRect = moduser32.NewProc("EqualRect") - procInflateRect = moduser32.NewProc("InflateRect") - procIntersectRect = moduser32.NewProc("IntersectRect") - procIsRectEmpty = moduser32.NewProc("IsRectEmpty") - procOffsetRect = moduser32.NewProc("OffsetRect") - procPtInRect = moduser32.NewProc("PtInRect") - procSetRect = moduser32.NewProc("SetRect") - procSetRectEmpty = moduser32.NewProc("SetRectEmpty") - procSubtractRect = moduser32.NewProc("SubtractRect") - procUnionRect = moduser32.NewProc("UnionRect") - procCreateDialogParam = moduser32.NewProc("CreateDialogParamW") - procDialogBoxParam = moduser32.NewProc("DialogBoxParamW") - procGetDlgItem = moduser32.NewProc("GetDlgItem") - procDrawIcon = moduser32.NewProc("DrawIcon") - procClientToScreen = moduser32.NewProc("ClientToScreen") - procIsDialogMessage = moduser32.NewProc("IsDialogMessageW") - procIsWindow = moduser32.NewProc("IsWindow") - procEndDialog = moduser32.NewProc("EndDialog") - procPeekMessage = moduser32.NewProc("PeekMessageW") - procTranslateAccelerator = moduser32.NewProc("TranslateAcceleratorW") - procSetWindowPos = moduser32.NewProc("SetWindowPos") - procFillRect = moduser32.NewProc("FillRect") - procDrawText = moduser32.NewProc("DrawTextW") - procAddClipboardFormatListener = moduser32.NewProc("AddClipboardFormatListener") - procRemoveClipboardFormatListener = moduser32.NewProc("RemoveClipboardFormatListener") - procOpenClipboard = moduser32.NewProc("OpenClipboard") - procCloseClipboard = moduser32.NewProc("CloseClipboard") - procEnumClipboardFormats = moduser32.NewProc("EnumClipboardFormats") - procGetClipboardData = moduser32.NewProc("GetClipboardData") - procSetClipboardData = moduser32.NewProc("SetClipboardData") - procEmptyClipboard = moduser32.NewProc("EmptyClipboard") - procGetClipboardFormatName = moduser32.NewProc("GetClipboardFormatNameW") - procIsClipboardFormatAvailable = moduser32.NewProc("IsClipboardFormatAvailable") - procBeginPaint = moduser32.NewProc("BeginPaint") - procEndPaint = moduser32.NewProc("EndPaint") - procGetKeyboardState = moduser32.NewProc("GetKeyboardState") - procMapVirtualKey = moduser32.NewProc("MapVirtualKeyExW") - procGetAsyncKeyState = moduser32.NewProc("GetAsyncKeyState") - procToAscii = moduser32.NewProc("ToAscii") - procSwapMouseButton = moduser32.NewProc("SwapMouseButton") - procGetCursorPos = moduser32.NewProc("GetCursorPos") - procSetCursorPos = moduser32.NewProc("SetCursorPos") - procSetCursor = moduser32.NewProc("SetCursor") - procCreateIcon = moduser32.NewProc("CreateIcon") - procDestroyIcon = moduser32.NewProc("DestroyIcon") - procMonitorFromPoint = moduser32.NewProc("MonitorFromPoint") - procMonitorFromRect = moduser32.NewProc("MonitorFromRect") - procMonitorFromWindow = moduser32.NewProc("MonitorFromWindow") - procGetMonitorInfo = moduser32.NewProc("GetMonitorInfoW") - procEnumDisplayMonitors = moduser32.NewProc("EnumDisplayMonitors") - procEnumDisplaySettingsEx = moduser32.NewProc("EnumDisplaySettingsExW") - procChangeDisplaySettingsEx = moduser32.NewProc("ChangeDisplaySettingsExW") - procSendInput = moduser32.NewProc("SendInput") -) - -func RegisterClassEx(wndClassEx *WNDCLASSEX) ATOM { - ret, _, _ := procRegisterClassEx.Call(uintptr(unsafe.Pointer(wndClassEx))) - return ATOM(ret) -} - -func LoadIcon(instance HINSTANCE, iconName *uint16) HICON { - ret, _, _ := procLoadIcon.Call( - uintptr(instance), - uintptr(unsafe.Pointer(iconName))) - - return HICON(ret) - -} - -func LoadCursor(instance HINSTANCE, cursorName *uint16) HCURSOR { - ret, _, _ := procLoadCursor.Call( - uintptr(instance), - uintptr(unsafe.Pointer(cursorName))) - - return HCURSOR(ret) - -} - -func ShowWindow(hwnd HWND, cmdshow int) bool { - ret, _, _ := procShowWindow.Call( - uintptr(hwnd), - uintptr(cmdshow)) - - return ret != 0 - -} - -func UpdateWindow(hwnd HWND) bool { - ret, _, _ := procUpdateWindow.Call( - uintptr(hwnd)) - return ret != 0 -} - -func CreateWindowEx(exStyle uint, className, windowName *uint16, - style uint, x, y, width, height int, parent HWND, menu HMENU, - instance HINSTANCE, param unsafe.Pointer) HWND { - ret, _, _ := procCreateWindowEx.Call( - uintptr(exStyle), - uintptr(unsafe.Pointer(className)), - uintptr(unsafe.Pointer(windowName)), - uintptr(style), - uintptr(x), - uintptr(y), - uintptr(width), - uintptr(height), - uintptr(parent), - uintptr(menu), - uintptr(instance), - uintptr(param)) - - return HWND(ret) -} - -func AdjustWindowRectEx(rect *RECT, style uint, menu bool, exStyle uint) bool { - ret, _, _ := procAdjustWindowRectEx.Call( - uintptr(unsafe.Pointer(rect)), - uintptr(style), - uintptr(BoolToBOOL(menu)), - uintptr(exStyle)) - - return ret != 0 -} - -func AdjustWindowRect(rect *RECT, style uint, menu bool) bool { - ret, _, _ := procAdjustWindowRect.Call( - uintptr(unsafe.Pointer(rect)), - uintptr(style), - uintptr(BoolToBOOL(menu))) - - return ret != 0 -} - -func DestroyWindow(hwnd HWND) bool { - ret, _, _ := procDestroyWindow.Call( - uintptr(hwnd)) - - return ret != 0 -} - -func DefWindowProc(hwnd HWND, msg uint32, wParam, lParam uintptr) uintptr { - ret, _, _ := procDefWindowProc.Call( - uintptr(hwnd), - uintptr(msg), - wParam, - lParam) - - return ret -} - -func DefDlgProc(hwnd HWND, msg uint32, wParam, lParam uintptr) uintptr { - ret, _, _ := procDefDlgProc.Call( - uintptr(hwnd), - uintptr(msg), - wParam, - lParam) - - return ret -} - -func PostQuitMessage(exitCode int) { - procPostQuitMessage.Call( - uintptr(exitCode)) -} - -func GetMessage(msg *MSG, hwnd HWND, msgFilterMin, msgFilterMax uint32) int { - ret, _, _ := procGetMessage.Call( - uintptr(unsafe.Pointer(msg)), - uintptr(hwnd), - uintptr(msgFilterMin), - uintptr(msgFilterMax)) - - return int(ret) -} - -func TranslateMessage(msg *MSG) bool { - ret, _, _ := procTranslateMessage.Call( - uintptr(unsafe.Pointer(msg))) - - return ret != 0 - -} - -func DispatchMessage(msg *MSG) uintptr { - ret, _, _ := procDispatchMessage.Call( - uintptr(unsafe.Pointer(msg))) - - return ret - -} - -func SendMessage(hwnd HWND, msg uint32, wParam, lParam uintptr) uintptr { - ret, _, _ := procSendMessage.Call( - uintptr(hwnd), - uintptr(msg), - wParam, - lParam) - - return ret -} - -func PostMessage(hwnd HWND, msg uint32, wParam, lParam uintptr) bool { - ret, _, _ := procPostMessage.Call( - uintptr(hwnd), - uintptr(msg), - wParam, - lParam) - - return ret != 0 -} - -func WaitMessage() bool { - ret, _, _ := procWaitMessage.Call() - return ret != 0 -} - -func SetWindowText(hwnd HWND, text string) { - procSetWindowText.Call( - uintptr(hwnd), - uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(text)))) -} - -func GetWindowTextLength(hwnd HWND) int { - ret, _, _ := procGetWindowTextLength.Call( - uintptr(hwnd)) - - return int(ret) -} - -func GetWindowText(hwnd HWND) string { - textLen := GetWindowTextLength(hwnd) + 1 - - buf := make([]uint16, textLen) - procGetWindowText.Call( - uintptr(hwnd), - uintptr(unsafe.Pointer(&buf[0])), - uintptr(textLen)) - - return syscall.UTF16ToString(buf) -} - -func GetWindowRect(hwnd HWND) *RECT { - var rect RECT - procGetWindowRect.Call( - uintptr(hwnd), - uintptr(unsafe.Pointer(&rect))) - - return &rect -} - -func MoveWindow(hwnd HWND, x, y, width, height int, repaint bool) bool { - ret, _, _ := procMoveWindow.Call( - uintptr(hwnd), - uintptr(x), - uintptr(y), - uintptr(width), - uintptr(height), - uintptr(BoolToBOOL(repaint))) - - return ret != 0 - -} - -func ScreenToClient(hwnd HWND, x, y int) (X, Y int, ok bool) { - pt := POINT{X: int32(x), Y: int32(y)} - ret, _, _ := procScreenToClient.Call( - uintptr(hwnd), - uintptr(unsafe.Pointer(&pt))) - - return int(pt.X), int(pt.Y), ret != 0 -} - -func CallWindowProc(preWndProc uintptr, hwnd HWND, msg uint32, wParam, lParam uintptr) uintptr { - ret, _, _ := procCallWindowProc.Call( - preWndProc, - uintptr(hwnd), - uintptr(msg), - wParam, - lParam) - - return ret -} - -func SetWindowLong(hwnd HWND, index int, value uint32) uint32 { - ret, _, _ := procSetWindowLong.Call( - uintptr(hwnd), - uintptr(index), - uintptr(value)) - - return uint32(ret) -} - -func SetWindowLongPtr(hwnd HWND, index int, value uintptr) uintptr { - ret, _, _ := procSetWindowLongPtr.Call( - uintptr(hwnd), - uintptr(index), - value) - - return ret -} - -func GetWindowLong(hwnd HWND, index int) int32 { - ret, _, _ := procGetWindowLong.Call( - uintptr(hwnd), - uintptr(index)) - - return int32(ret) -} - -func GetWindowLongPtr(hwnd HWND, index int) uintptr { - ret, _, _ := procGetWindowLongPtr.Call( - uintptr(hwnd), - uintptr(index)) - - return ret -} - -func EnableWindow(hwnd HWND, b bool) bool { - ret, _, _ := procEnableWindow.Call( - uintptr(hwnd), - uintptr(BoolToBOOL(b))) - return ret != 0 -} - -func IsWindowEnabled(hwnd HWND) bool { - ret, _, _ := procIsWindowEnabled.Call( - uintptr(hwnd)) - - return ret != 0 -} - -func IsWindowVisible(hwnd HWND) bool { - ret, _, _ := procIsWindowVisible.Call( - uintptr(hwnd)) - - return ret != 0 -} - -func SetFocus(hwnd HWND) HWND { - ret, _, _ := procSetFocus.Call( - uintptr(hwnd)) - - return HWND(ret) -} - -func InvalidateRect(hwnd HWND, rect *RECT, erase bool) bool { - ret, _, _ := procInvalidateRect.Call( - uintptr(hwnd), - uintptr(unsafe.Pointer(rect)), - uintptr(BoolToBOOL(erase))) - - return ret != 0 -} - -func GetClientRect(hwnd HWND) *RECT { - var rect RECT - ret, _, _ := procGetClientRect.Call( - uintptr(hwnd), - uintptr(unsafe.Pointer(&rect))) - - if ret == 0 { - panic(fmt.Sprintf("GetClientRect(%d) failed", hwnd)) - } - - return &rect -} - -func GetDC(hwnd HWND) HDC { - ret, _, _ := procGetDC.Call( - uintptr(hwnd)) - - return HDC(ret) -} - -func ReleaseDC(hwnd HWND, hDC HDC) bool { - ret, _, _ := procReleaseDC.Call( - uintptr(hwnd), - uintptr(hDC)) - - return ret != 0 -} - -func SetCapture(hwnd HWND) HWND { - ret, _, _ := procSetCapture.Call( - uintptr(hwnd)) - - return HWND(ret) -} - -func ReleaseCapture() bool { - ret, _, _ := procReleaseCapture.Call() - - return ret != 0 -} - -func GetWindowThreadProcessId(hwnd HWND) (HANDLE, int) { - var processId int - ret, _, _ := procGetWindowThreadProcessId.Call( - uintptr(hwnd), - uintptr(unsafe.Pointer(&processId))) - - return HANDLE(ret), processId -} - -func MessageBox(hwnd HWND, title, caption string, flags uint) int { - ret, _, _ := procMessageBox.Call( - uintptr(hwnd), - uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(title))), - uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(caption))), - uintptr(flags)) - - return int(ret) -} - -func GetSystemMetrics(index int) int { - ret, _, _ := procGetSystemMetrics.Call( - uintptr(index)) - - return int(ret) -} - -func CopyRect(dst, src *RECT) bool { - ret, _, _ := procCopyRect.Call( - uintptr(unsafe.Pointer(dst)), - uintptr(unsafe.Pointer(src))) - - return ret != 0 -} - -func EqualRect(rect1, rect2 *RECT) bool { - ret, _, _ := procEqualRect.Call( - uintptr(unsafe.Pointer(rect1)), - uintptr(unsafe.Pointer(rect2))) - - return ret != 0 -} - -func InflateRect(rect *RECT, dx, dy int) bool { - ret, _, _ := procInflateRect.Call( - uintptr(unsafe.Pointer(rect)), - uintptr(dx), - uintptr(dy)) - - return ret != 0 -} - -func IntersectRect(dst, src1, src2 *RECT) bool { - ret, _, _ := procIntersectRect.Call( - uintptr(unsafe.Pointer(dst)), - uintptr(unsafe.Pointer(src1)), - uintptr(unsafe.Pointer(src2))) - - return ret != 0 -} - -func IsRectEmpty(rect *RECT) bool { - ret, _, _ := procIsRectEmpty.Call( - uintptr(unsafe.Pointer(rect))) - - return ret != 0 -} - -func OffsetRect(rect *RECT, dx, dy int) bool { - ret, _, _ := procOffsetRect.Call( - uintptr(unsafe.Pointer(rect)), - uintptr(dx), - uintptr(dy)) - - return ret != 0 -} - -func PtInRect(rect *RECT, x, y int) bool { - pt := POINT{X: int32(x), Y: int32(y)} - ret, _, _ := procPtInRect.Call( - uintptr(unsafe.Pointer(rect)), - uintptr(unsafe.Pointer(&pt))) - - return ret != 0 -} - -func SetRect(rect *RECT, left, top, right, bottom int) bool { - ret, _, _ := procSetRect.Call( - uintptr(unsafe.Pointer(rect)), - uintptr(left), - uintptr(top), - uintptr(right), - uintptr(bottom)) - - return ret != 0 -} - -func SetRectEmpty(rect *RECT) bool { - ret, _, _ := procSetRectEmpty.Call( - uintptr(unsafe.Pointer(rect))) - - return ret != 0 -} - -func SubtractRect(dst, src1, src2 *RECT) bool { - ret, _, _ := procSubtractRect.Call( - uintptr(unsafe.Pointer(dst)), - uintptr(unsafe.Pointer(src1)), - uintptr(unsafe.Pointer(src2))) - - return ret != 0 -} - -func UnionRect(dst, src1, src2 *RECT) bool { - ret, _, _ := procUnionRect.Call( - uintptr(unsafe.Pointer(dst)), - uintptr(unsafe.Pointer(src1)), - uintptr(unsafe.Pointer(src2))) - - return ret != 0 -} - -func CreateDialog(hInstance HINSTANCE, lpTemplate *uint16, hWndParent HWND, lpDialogProc uintptr) HWND { - ret, _, _ := procCreateDialogParam.Call( - uintptr(hInstance), - uintptr(unsafe.Pointer(lpTemplate)), - uintptr(hWndParent), - lpDialogProc, - 0) - - return HWND(ret) -} - -func DialogBox(hInstance HINSTANCE, lpTemplateName *uint16, hWndParent HWND, lpDialogProc uintptr) int { - ret, _, _ := procDialogBoxParam.Call( - uintptr(hInstance), - uintptr(unsafe.Pointer(lpTemplateName)), - uintptr(hWndParent), - lpDialogProc, - 0) - - return int(ret) -} - -func GetDlgItem(hDlg HWND, nIDDlgItem int) HWND { - ret, _, _ := procGetDlgItem.Call( - uintptr(unsafe.Pointer(hDlg)), - uintptr(nIDDlgItem)) - - return HWND(ret) -} - -func DrawIcon(hDC HDC, x, y int, hIcon HICON) bool { - ret, _, _ := procDrawIcon.Call( - uintptr(unsafe.Pointer(hDC)), - uintptr(x), - uintptr(y), - uintptr(unsafe.Pointer(hIcon))) - - return ret != 0 -} - -func ClientToScreen(hwnd HWND, x, y int) (int, int) { - pt := POINT{X: int32(x), Y: int32(y)} - - procClientToScreen.Call( - uintptr(hwnd), - uintptr(unsafe.Pointer(&pt))) - - return int(pt.X), int(pt.Y) -} - -func IsDialogMessage(hwnd HWND, msg *MSG) bool { - ret, _, _ := procIsDialogMessage.Call( - uintptr(hwnd), - uintptr(unsafe.Pointer(msg))) - - return ret != 0 -} - -func IsWindow(hwnd HWND) bool { - ret, _, _ := procIsWindow.Call( - uintptr(hwnd)) - - return ret != 0 -} - -func EndDialog(hwnd HWND, nResult uintptr) bool { - ret, _, _ := procEndDialog.Call( - uintptr(hwnd), - nResult) - - return ret != 0 -} - -func PeekMessage(lpMsg *MSG, hwnd HWND, wMsgFilterMin, wMsgFilterMax, wRemoveMsg uint32) bool { - ret, _, _ := procPeekMessage.Call( - uintptr(unsafe.Pointer(lpMsg)), - uintptr(hwnd), - uintptr(wMsgFilterMin), - uintptr(wMsgFilterMax), - uintptr(wRemoveMsg)) - - return ret != 0 -} - -func TranslateAccelerator(hwnd HWND, hAccTable HACCEL, lpMsg *MSG) bool { - ret, _, _ := procTranslateMessage.Call( - uintptr(hwnd), - uintptr(hAccTable), - uintptr(unsafe.Pointer(lpMsg))) - - return ret != 0 -} - -func SetWindowPos(hwnd, hWndInsertAfter HWND, x, y, cx, cy int, uFlags uint) bool { - ret, _, _ := procSetWindowPos.Call( - uintptr(hwnd), - uintptr(hWndInsertAfter), - uintptr(x), - uintptr(y), - uintptr(cx), - uintptr(cy), - uintptr(uFlags)) - - return ret != 0 -} - -func FillRect(hDC HDC, lprc *RECT, hbr HBRUSH) bool { - ret, _, _ := procFillRect.Call( - uintptr(hDC), - uintptr(unsafe.Pointer(lprc)), - uintptr(hbr)) - - return ret != 0 -} - -func DrawText(hDC HDC, text string, uCount int, lpRect *RECT, uFormat uint) int { - ret, _, _ := procDrawText.Call( - uintptr(hDC), - uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(text))), - uintptr(uCount), - uintptr(unsafe.Pointer(lpRect)), - uintptr(uFormat)) - - return int(ret) -} - -func AddClipboardFormatListener(hwnd HWND) bool { - ret, _, _ := procAddClipboardFormatListener.Call( - uintptr(hwnd)) - return ret != 0 -} - -func RemoveClipboardFormatListener(hwnd HWND) bool { - ret, _, _ := procRemoveClipboardFormatListener.Call( - uintptr(hwnd)) - return ret != 0 -} - -func OpenClipboard(hWndNewOwner HWND) bool { - ret, _, _ := procOpenClipboard.Call( - uintptr(hWndNewOwner)) - return ret != 0 -} - -func CloseClipboard() bool { - ret, _, _ := procCloseClipboard.Call() - return ret != 0 -} - -func EnumClipboardFormats(format uint) uint { - ret, _, _ := procEnumClipboardFormats.Call( - uintptr(format)) - return uint(ret) -} - -func GetClipboardData(uFormat uint) HANDLE { - ret, _, _ := procGetClipboardData.Call( - uintptr(uFormat)) - return HANDLE(ret) -} - -func SetClipboardData(uFormat uint, hMem HANDLE) HANDLE { - ret, _, _ := procSetClipboardData.Call( - uintptr(uFormat), - uintptr(hMem)) - return HANDLE(ret) -} - -func EmptyClipboard() bool { - ret, _, _ := procEmptyClipboard.Call() - return ret != 0 -} - -func GetClipboardFormatName(format uint) (string, bool) { - cchMaxCount := 255 - buf := make([]uint16, cchMaxCount) - ret, _, _ := procGetClipboardFormatName.Call( - uintptr(format), - uintptr(unsafe.Pointer(&buf[0])), - uintptr(cchMaxCount)) - - if ret > 0 { - return syscall.UTF16ToString(buf), true - } - - return "Requested format does not exist or is predefined", false -} - -func IsClipboardFormatAvailable(format uint) bool { - ret, _, _ := procIsClipboardFormatAvailable.Call(uintptr(format)) - return ret != 0 -} - -func BeginPaint(hwnd HWND, paint *PAINTSTRUCT) HDC { - ret, _, _ := procBeginPaint.Call( - uintptr(hwnd), - uintptr(unsafe.Pointer(paint))) - return HDC(ret) -} - -func EndPaint(hwnd HWND, paint *PAINTSTRUCT) { - procBeginPaint.Call( - uintptr(hwnd), - uintptr(unsafe.Pointer(paint))) -} - -func GetKeyboardState(lpKeyState *[]byte) bool { - ret, _, _ := procGetKeyboardState.Call( - uintptr(unsafe.Pointer(&(*lpKeyState)[0]))) - return ret != 0 -} - -func MapVirtualKeyEx(uCode, uMapType uint, dwhkl HKL) uint { - ret, _, _ := procMapVirtualKey.Call( - uintptr(uCode), - uintptr(uMapType), - uintptr(dwhkl)) - return uint(ret) -} - -func GetAsyncKeyState(vKey int) uint16 { - ret, _, _ := procGetAsyncKeyState.Call(uintptr(vKey)) - return uint16(ret) -} - -func ToAscii(uVirtKey, uScanCode uint, lpKeyState *byte, lpChar *uint16, uFlags uint) int { - ret, _, _ := procToAscii.Call( - uintptr(uVirtKey), - uintptr(uScanCode), - uintptr(unsafe.Pointer(lpKeyState)), - uintptr(unsafe.Pointer(lpChar)), - uintptr(uFlags)) - return int(ret) -} - -func SwapMouseButton(fSwap bool) bool { - ret, _, _ := procSwapMouseButton.Call( - uintptr(BoolToBOOL(fSwap))) - return ret != 0 -} - -func GetCursorPos() (x, y int, ok bool) { - pt := POINT{} - ret, _, _ := procGetCursorPos.Call(uintptr(unsafe.Pointer(&pt))) - return int(pt.X), int(pt.Y), ret != 0 -} - -func SetCursorPos(x, y int) bool { - ret, _, _ := procSetCursorPos.Call( - uintptr(x), - uintptr(y), - ) - return ret != 0 -} - -func SetCursor(cursor HCURSOR) HCURSOR { - ret, _, _ := procSetCursor.Call( - uintptr(cursor), - ) - return HCURSOR(ret) -} - -func CreateIcon(instance HINSTANCE, nWidth, nHeight int, cPlanes, cBitsPerPixel byte, ANDbits, XORbits *byte) HICON { - ret, _, _ := procCreateIcon.Call( - uintptr(instance), - uintptr(nWidth), - uintptr(nHeight), - uintptr(cPlanes), - uintptr(cBitsPerPixel), - uintptr(unsafe.Pointer(ANDbits)), - uintptr(unsafe.Pointer(XORbits)), - ) - return HICON(ret) -} - -func DestroyIcon(icon HICON) bool { - ret, _, _ := procDestroyIcon.Call( - uintptr(icon), - ) - return ret != 0 -} - -func MonitorFromPoint(x, y int, dwFlags uint32) HMONITOR { - ret, _, _ := procMonitorFromPoint.Call( - uintptr(x), - uintptr(y), - uintptr(dwFlags), - ) - return HMONITOR(ret) -} - -func MonitorFromRect(rc *RECT, dwFlags uint32) HMONITOR { - ret, _, _ := procMonitorFromRect.Call( - uintptr(unsafe.Pointer(rc)), - uintptr(dwFlags), - ) - return HMONITOR(ret) -} - -func MonitorFromWindow(hwnd HWND, dwFlags uint32) HMONITOR { - ret, _, _ := procMonitorFromWindow.Call( - uintptr(hwnd), - uintptr(dwFlags), - ) - return HMONITOR(ret) -} - -func GetMonitorInfo(hMonitor HMONITOR, lmpi *MONITORINFO) bool { - ret, _, _ := procGetMonitorInfo.Call( - uintptr(hMonitor), - uintptr(unsafe.Pointer(lmpi)), - ) - return ret != 0 -} - -func EnumDisplayMonitors(hdc HDC, clip *RECT, fnEnum, dwData uintptr) bool { - ret, _, _ := procEnumDisplayMonitors.Call( - uintptr(hdc), - uintptr(unsafe.Pointer(clip)), - fnEnum, - dwData, - ) - return ret != 0 -} - -func EnumDisplaySettingsEx(szDeviceName *uint16, iModeNum uint32, devMode *DEVMODE, dwFlags uint32) bool { - ret, _, _ := procEnumDisplaySettingsEx.Call( - uintptr(unsafe.Pointer(szDeviceName)), - uintptr(iModeNum), - uintptr(unsafe.Pointer(devMode)), - uintptr(dwFlags), - ) - return ret != 0 -} - -func ChangeDisplaySettingsEx(szDeviceName *uint16, devMode *DEVMODE, hwnd HWND, dwFlags uint32, lParam uintptr) int32 { - ret, _, _ := procChangeDisplaySettingsEx.Call( - uintptr(unsafe.Pointer(szDeviceName)), - uintptr(unsafe.Pointer(devMode)), - uintptr(hwnd), - uintptr(dwFlags), - lParam, - ) - return int32(ret) -} - -/* remove to build without cgo -func SendInput(inputs []INPUT) uint32 { - var validInputs []C.INPUT - - for _, oneInput := range inputs { - input := C.INPUT{_type: C.DWORD(oneInput.Type)} - - switch oneInput.Type { - case INPUT_MOUSE: - (*MouseInput)(unsafe.Pointer(&input)).mi = oneInput.Mi - case INPUT_KEYBOARD: - (*KbdInput)(unsafe.Pointer(&input)).ki = oneInput.Ki - case INPUT_HARDWARE: - (*HardwareInput)(unsafe.Pointer(&input)).hi = oneInput.Hi - default: - panic("unkown type") - } - - validInputs = append(validInputs, input) - } - - ret, _, _ := procSendInput.Call( - uintptr(len(validInputs)), - uintptr(unsafe.Pointer(&validInputs[0])), - uintptr(unsafe.Sizeof(C.INPUT{})), - ) - return uint32(ret) -} -*/ diff --git a/github.com/shirou/w32/utils.go b/github.com/shirou/w32/utils.go deleted file mode 100644 index 69aa31a46d..0000000000 --- a/github.com/shirou/w32/utils.go +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright 2010-2012 The W32 Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build windows - -package w32 - -import ( - "syscall" - "unicode/utf16" - "unsafe" -) - -func MakeIntResource(id uint16) *uint16 { - return (*uint16)(unsafe.Pointer(uintptr(id))) -} - -func LOWORD(dw uint32) uint16 { - return uint16(dw) -} - -func HIWORD(dw uint32) uint16 { - return uint16(dw >> 16 & 0xffff) -} - -func BoolToBOOL(value bool) BOOL { - if value { - return 1 - } - - return 0 -} - -func UTF16PtrToString(cstr *uint16) string { - if cstr != nil { - us := make([]uint16, 0, 256) - for p := uintptr(unsafe.Pointer(cstr)); ; p += 2 { - u := *(*uint16)(unsafe.Pointer(p)) - if u == 0 { - return string(utf16.Decode(us)) - } - us = append(us, u) - } - } - - return "" -} - -func ComAddRef(unknown *IUnknown) int32 { - ret, _, _ := syscall.Syscall(unknown.lpVtbl.pAddRef, 1, - uintptr(unsafe.Pointer(unknown)), - 0, - 0) - return int32(ret) -} - -func ComRelease(unknown *IUnknown) int32 { - ret, _, _ := syscall.Syscall(unknown.lpVtbl.pRelease, 1, - uintptr(unsafe.Pointer(unknown)), - 0, - 0) - return int32(ret) -} - -func ComQueryInterface(unknown *IUnknown, id *GUID) *IDispatch { - var disp *IDispatch - hr, _, _ := syscall.Syscall(unknown.lpVtbl.pQueryInterface, 3, - uintptr(unsafe.Pointer(unknown)), - uintptr(unsafe.Pointer(id)), - uintptr(unsafe.Pointer(&disp))) - if hr != 0 { - panic("Invoke QieryInterface error.") - } - return disp -} - -func ComGetIDsOfName(disp *IDispatch, names []string) []int32 { - wnames := make([]*uint16, len(names)) - dispid := make([]int32, len(names)) - for i := 0; i < len(names); i++ { - wnames[i] = syscall.StringToUTF16Ptr(names[i]) - } - hr, _, _ := syscall.Syscall6(disp.lpVtbl.pGetIDsOfNames, 6, - uintptr(unsafe.Pointer(disp)), - uintptr(unsafe.Pointer(IID_NULL)), - uintptr(unsafe.Pointer(&wnames[0])), - uintptr(len(names)), - uintptr(GetUserDefaultLCID()), - uintptr(unsafe.Pointer(&dispid[0]))) - if hr != 0 { - panic("Invoke GetIDsOfName error.") - } - return dispid -} - -func ComInvoke(disp *IDispatch, dispid int32, dispatch int16, params ...interface{}) (result *VARIANT) { - var dispparams DISPPARAMS - - if dispatch&DISPATCH_PROPERTYPUT != 0 { - dispnames := [1]int32{DISPID_PROPERTYPUT} - dispparams.RgdispidNamedArgs = uintptr(unsafe.Pointer(&dispnames[0])) - dispparams.CNamedArgs = 1 - } - var vargs []VARIANT - if len(params) > 0 { - vargs = make([]VARIANT, len(params)) - for i, v := range params { - //n := len(params)-i-1 - n := len(params) - i - 1 - VariantInit(&vargs[n]) - switch v.(type) { - case bool: - if v.(bool) { - vargs[n] = VARIANT{VT_BOOL, 0, 0, 0, 0xffff} - } else { - vargs[n] = VARIANT{VT_BOOL, 0, 0, 0, 0} - } - case *bool: - vargs[n] = VARIANT{VT_BOOL | VT_BYREF, 0, 0, 0, int64(uintptr(unsafe.Pointer(v.(*bool))))} - case byte: - vargs[n] = VARIANT{VT_I1, 0, 0, 0, int64(v.(byte))} - case *byte: - vargs[n] = VARIANT{VT_I1 | VT_BYREF, 0, 0, 0, int64(uintptr(unsafe.Pointer(v.(*byte))))} - case int16: - vargs[n] = VARIANT{VT_I2, 0, 0, 0, int64(v.(int16))} - case *int16: - vargs[n] = VARIANT{VT_I2 | VT_BYREF, 0, 0, 0, int64(uintptr(unsafe.Pointer(v.(*int16))))} - case uint16: - vargs[n] = VARIANT{VT_UI2, 0, 0, 0, int64(v.(int16))} - case *uint16: - vargs[n] = VARIANT{VT_UI2 | VT_BYREF, 0, 0, 0, int64(uintptr(unsafe.Pointer(v.(*uint16))))} - case int, int32: - vargs[n] = VARIANT{VT_UI4, 0, 0, 0, int64(v.(int))} - case *int, *int32: - vargs[n] = VARIANT{VT_I4 | VT_BYREF, 0, 0, 0, int64(uintptr(unsafe.Pointer(v.(*int))))} - case uint, uint32: - vargs[n] = VARIANT{VT_UI4, 0, 0, 0, int64(v.(uint))} - case *uint, *uint32: - vargs[n] = VARIANT{VT_UI4 | VT_BYREF, 0, 0, 0, int64(uintptr(unsafe.Pointer(v.(*uint))))} - case int64: - vargs[n] = VARIANT{VT_I8, 0, 0, 0, v.(int64)} - case *int64: - vargs[n] = VARIANT{VT_I8 | VT_BYREF, 0, 0, 0, int64(uintptr(unsafe.Pointer(v.(*int64))))} - case uint64: - vargs[n] = VARIANT{VT_UI8, 0, 0, 0, int64(v.(uint64))} - case *uint64: - vargs[n] = VARIANT{VT_UI8 | VT_BYREF, 0, 0, 0, int64(uintptr(unsafe.Pointer(v.(*uint64))))} - case float32: - vargs[n] = VARIANT{VT_R4, 0, 0, 0, int64(v.(float32))} - case *float32: - vargs[n] = VARIANT{VT_R4 | VT_BYREF, 0, 0, 0, int64(uintptr(unsafe.Pointer(v.(*float32))))} - case float64: - vargs[n] = VARIANT{VT_R8, 0, 0, 0, int64(v.(float64))} - case *float64: - vargs[n] = VARIANT{VT_R8 | VT_BYREF, 0, 0, 0, int64(uintptr(unsafe.Pointer(v.(*float64))))} - case string: - vargs[n] = VARIANT{VT_BSTR, 0, 0, 0, int64(uintptr(unsafe.Pointer(SysAllocString(v.(string)))))} - case *string: - vargs[n] = VARIANT{VT_BSTR | VT_BYREF, 0, 0, 0, int64(uintptr(unsafe.Pointer(v.(*string))))} - case *IDispatch: - vargs[n] = VARIANT{VT_DISPATCH, 0, 0, 0, int64(uintptr(unsafe.Pointer(v.(*IDispatch))))} - case **IDispatch: - vargs[n] = VARIANT{VT_DISPATCH | VT_BYREF, 0, 0, 0, int64(uintptr(unsafe.Pointer(v.(**IDispatch))))} - case nil: - vargs[n] = VARIANT{VT_NULL, 0, 0, 0, 0} - case *VARIANT: - vargs[n] = VARIANT{VT_VARIANT | VT_BYREF, 0, 0, 0, int64(uintptr(unsafe.Pointer(v.(*VARIANT))))} - default: - panic("unknown type") - } - } - dispparams.Rgvarg = uintptr(unsafe.Pointer(&vargs[0])) - dispparams.CArgs = uint32(len(params)) - } - - var ret VARIANT - var excepInfo EXCEPINFO - VariantInit(&ret) - hr, _, _ := syscall.Syscall9(disp.lpVtbl.pInvoke, 8, - uintptr(unsafe.Pointer(disp)), - uintptr(dispid), - uintptr(unsafe.Pointer(IID_NULL)), - uintptr(GetUserDefaultLCID()), - uintptr(dispatch), - uintptr(unsafe.Pointer(&dispparams)), - uintptr(unsafe.Pointer(&ret)), - uintptr(unsafe.Pointer(&excepInfo)), - 0) - if hr != 0 { - if excepInfo.BstrDescription != nil { - bs := UTF16PtrToString(excepInfo.BstrDescription) - panic(bs) - } - } - for _, varg := range vargs { - if varg.VT == VT_BSTR && varg.Val != 0 { - SysFreeString(((*int16)(unsafe.Pointer(uintptr(varg.Val))))) - } - } - result = &ret - return -} diff --git a/github.com/shirou/w32/vars.go b/github.com/shirou/w32/vars.go deleted file mode 100644 index 2dab2e3963..0000000000 --- a/github.com/shirou/w32/vars.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2010-2012 The W32 Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package w32 - -var ( - IID_NULL = &GUID{0x00000000, 0x0000, 0x0000, [8]byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}} - IID_IUnknown = &GUID{0x00000000, 0x0000, 0x0000, [8]byte{0xC0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46}} - IID_IDispatch = &GUID{0x00020400, 0x0000, 0x0000, [8]byte{0xC0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46}} - IID_IConnectionPointContainer = &GUID{0xB196B284, 0xBAB4, 0x101A, [8]byte{0xB6, 0x9C, 0x00, 0xAA, 0x00, 0x34, 0x1D, 0x07}} - IID_IConnectionPoint = &GUID{0xB196B286, 0xBAB4, 0x101A, [8]byte{0xB6, 0x9C, 0x00, 0xAA, 0x00, 0x34, 0x1D, 0x07}} -) diff --git a/modules.txt b/modules.txt index cb0a6626e7..93e8a2dea6 100644 --- a/modules.txt +++ b/modules.txt @@ -722,7 +722,7 @@ github.com/russross/blackfriday # github.com/sasha-s/go-deadlock v0.2.0 ## explicit github.com/sasha-s/go-deadlock -# github.com/shirou/gopsutil v2.18.12+incompatible +# github.com/shirou/gopsutil v2.20.6+incompatible ## explicit github.com/shirou/gopsutil/cpu github.com/shirou/gopsutil/disk @@ -732,9 +732,6 @@ github.com/shirou/gopsutil/load github.com/shirou/gopsutil/mem github.com/shirou/gopsutil/net github.com/shirou/gopsutil/process -# github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 -## explicit -github.com/shirou/w32 # github.com/shopspring/decimal v1.2.0 ## explicit # github.com/sirupsen/logrus v1.4.2 From 358bdd568c9944ec043f4f3492523a41fa2d070d Mon Sep 17 00:00:00 2001 From: Matt Jibson Date: Mon, 20 Jul 2020 06:36:16 -0600 Subject: [PATCH 15/49] lib/pq 1.7.1 --- github.com/alexbrainman/sspi/LICENSE | 27 + github.com/alexbrainman/sspi/README.md | 1 + github.com/alexbrainman/sspi/buffer.go | 57 ++ github.com/alexbrainman/sspi/mksyscall.go | 7 + .../alexbrainman/sspi/negotiate/negotiate.go | 462 +++++++++++ github.com/alexbrainman/sspi/sspi.go | 226 ++++++ github.com/alexbrainman/sspi/syscall.go | 174 +++++ .../alexbrainman/sspi/zsyscall_windows.go | 152 ++++ github.com/jcmturner/aescts/v2/LICENSE | 201 +++++ github.com/jcmturner/aescts/v2/aescts.go | 186 +++++ github.com/jcmturner/aescts/v2/go.mod | 5 + github.com/jcmturner/aescts/v2/go.sum | 10 + github.com/jcmturner/dnsutils/v2/LICENSE | 201 +++++ github.com/jcmturner/dnsutils/v2/go.mod | 5 + github.com/jcmturner/dnsutils/v2/go.sum | 10 + github.com/jcmturner/dnsutils/v2/srv.go | 95 +++ github.com/jcmturner/goidentity/v6/LICENSE | 201 +++++ github.com/jcmturner/goidentity/v6/README.md | 7 + .../jcmturner/goidentity/v6/authenticator.go | 6 + github.com/jcmturner/goidentity/v6/go.mod | 8 + github.com/jcmturner/goidentity/v6/go.sum | 12 + .../jcmturner/goidentity/v6/identity.go | 52 ++ github.com/jcmturner/goidentity/v6/user.go | 172 +++++ github.com/jcmturner/gokrb5/v8/LICENSE | 201 +++++ .../jcmturner/gokrb5/v8/asn1tools/tools.go | 86 +++ .../jcmturner/gokrb5/v8/client/ASExchange.go | 182 +++++ .../jcmturner/gokrb5/v8/client/TGSExchange.go | 103 +++ .../jcmturner/gokrb5/v8/client/cache.go | 134 ++++ .../jcmturner/gokrb5/v8/client/client.go | 329 ++++++++ .../jcmturner/gokrb5/v8/client/network.go | 213 +++++ .../jcmturner/gokrb5/v8/client/passwd.go | 95 +++ .../jcmturner/gokrb5/v8/client/session.go | 295 +++++++ .../jcmturner/gokrb5/v8/client/settings.go | 93 +++ .../jcmturner/gokrb5/v8/config/error.go | 30 + .../jcmturner/gokrb5/v8/config/hosts.go | 141 ++++ .../jcmturner/gokrb5/v8/config/krb5conf.go | 728 ++++++++++++++++++ .../jcmturner/gokrb5/v8/credentials/ccache.go | 333 ++++++++ .../gokrb5/v8/credentials/credentials.go | 405 ++++++++++ .../v8/crypto/aes128-cts-hmac-sha1-96.go | 129 ++++ .../v8/crypto/aes128-cts-hmac-sha256-128.go | 132 ++++ .../v8/crypto/aes256-cts-hmac-sha1-96.go | 129 ++++ .../v8/crypto/aes256-cts-hmac-sha384-192.go | 132 ++++ .../gokrb5/v8/crypto/common/common.go | 132 ++++ .../jcmturner/gokrb5/v8/crypto/crypto.go | 175 +++++ .../gokrb5/v8/crypto/des3-cbc-sha1-kd.go | 139 ++++ .../jcmturner/gokrb5/v8/crypto/etype/etype.go | 29 + .../jcmturner/gokrb5/v8/crypto/rc4-hmac.go | 133 ++++ .../gokrb5/v8/crypto/rfc3961/encryption.go | 119 +++ .../gokrb5/v8/crypto/rfc3961/keyDerivation.go | 169 ++++ .../gokrb5/v8/crypto/rfc3961/nfold.go | 107 +++ .../gokrb5/v8/crypto/rfc3962/encryption.go | 89 +++ .../gokrb5/v8/crypto/rfc3962/keyDerivation.go | 51 ++ .../gokrb5/v8/crypto/rfc4757/checksum.go | 40 + .../gokrb5/v8/crypto/rfc4757/encryption.go | 80 ++ .../gokrb5/v8/crypto/rfc4757/keyDerivation.go | 40 + .../gokrb5/v8/crypto/rfc4757/msgtype.go | 20 + .../gokrb5/v8/crypto/rfc8009/encryption.go | 125 +++ .../gokrb5/v8/crypto/rfc8009/keyDerivation.go | 135 ++++ .../jcmturner/gokrb5/v8/gssapi/MICToken.go | 174 +++++ .../jcmturner/gokrb5/v8/gssapi/README.md | 20 + .../gokrb5/v8/gssapi/contextFlags.go | 25 + .../jcmturner/gokrb5/v8/gssapi/gssapi.go | 202 +++++ .../jcmturner/gokrb5/v8/gssapi/wrapToken.go | 193 +++++ .../gokrb5/v8/iana/addrtype/constants.go | 15 + .../gokrb5/v8/iana/adtype/constants.go | 23 + .../gokrb5/v8/iana/asnAppTag/constants.go | 24 + .../gokrb5/v8/iana/chksumtype/constants.go | 32 + .../jcmturner/gokrb5/v8/iana/constants.go | 5 + .../gokrb5/v8/iana/errorcode/constants.go | 155 ++++ .../gokrb5/v8/iana/etypeID/constants.go | 101 +++ .../gokrb5/v8/iana/flags/constants.go | 36 + .../gokrb5/v8/iana/keyusage/constants.go | 42 + .../gokrb5/v8/iana/msgtype/constants.go | 18 + .../gokrb5/v8/iana/nametype/constants.go | 15 + .../gokrb5/v8/iana/patype/constants.go | 77 ++ .../gokrb5/v8/kadmin/changepasswddata.go | 23 + .../jcmturner/gokrb5/v8/kadmin/message.go | 114 +++ .../jcmturner/gokrb5/v8/kadmin/passwd.go | 68 ++ .../jcmturner/gokrb5/v8/keytab/keytab.go | 470 +++++++++++ .../jcmturner/gokrb5/v8/krberror/error.go | 67 ++ .../jcmturner/gokrb5/v8/messages/APRep.go | 49 ++ .../jcmturner/gokrb5/v8/messages/APReq.go | 199 +++++ .../jcmturner/gokrb5/v8/messages/KDCRep.go | 308 ++++++++ .../jcmturner/gokrb5/v8/messages/KDCReq.go | 432 +++++++++++ .../jcmturner/gokrb5/v8/messages/KRBCred.go | 102 +++ .../jcmturner/gokrb5/v8/messages/KRBError.go | 83 ++ .../jcmturner/gokrb5/v8/messages/KRBPriv.go | 108 +++ .../jcmturner/gokrb5/v8/messages/KRBSafe.go | 43 ++ .../jcmturner/gokrb5/v8/messages/Ticket.go | 265 +++++++ .../jcmturner/gokrb5/v8/pac/client_claims.go | 33 + .../jcmturner/gokrb5/v8/pac/client_info.go | 31 + .../gokrb5/v8/pac/credentials_info.go | 82 ++ .../jcmturner/gokrb5/v8/pac/device_claims.go | 33 + .../jcmturner/gokrb5/v8/pac/device_info.go | 32 + .../gokrb5/v8/pac/kerb_validation_info.go | 110 +++ .../jcmturner/gokrb5/v8/pac/pac_type.go | 251 ++++++ .../gokrb5/v8/pac/s4u_delegation_info.go | 26 + .../jcmturner/gokrb5/v8/pac/signature_data.go | 67 ++ .../gokrb5/v8/pac/supplemental_cred.go | 87 +++ .../jcmturner/gokrb5/v8/pac/upn_dns_info.go | 73 ++ .../jcmturner/gokrb5/v8/service/APExchange.go | 61 ++ .../gokrb5/v8/service/authenticator.go | 118 +++ .../jcmturner/gokrb5/v8/service/cache.go | 128 +++ .../jcmturner/gokrb5/v8/service/settings.go | 163 ++++ github.com/jcmturner/gokrb5/v8/spnego/http.go | 373 +++++++++ .../jcmturner/gokrb5/v8/spnego/krb5Token.go | 218 ++++++ .../gokrb5/v8/spnego/negotiationToken.go | 302 ++++++++ .../jcmturner/gokrb5/v8/spnego/spnego.go | 203 +++++ .../gokrb5/v8/types/Authenticator.go | 81 ++ .../gokrb5/v8/types/AuthorizationData.go | 55 ++ .../jcmturner/gokrb5/v8/types/Cryptosystem.go | 55 ++ .../jcmturner/gokrb5/v8/types/HostAddress.go | 180 +++++ .../gokrb5/v8/types/KerberosFlags.go | 68 ++ .../jcmturner/gokrb5/v8/types/PAData.go | 155 ++++ .../gokrb5/v8/types/PrincipalName.go | 64 ++ .../jcmturner/gokrb5/v8/types/TypedData.go | 18 + github.com/jcmturner/rpc/v2/LICENSE | 201 +++++ github.com/jcmturner/rpc/v2/mstypes/claims.go | 152 ++++ github.com/jcmturner/rpc/v2/mstypes/common.go | 12 + .../jcmturner/rpc/v2/mstypes/filetime.go | 52 ++ .../rpc/v2/mstypes/group_membership.go | 19 + .../rpc/v2/mstypes/kerb_sid_and_attributes.go | 23 + github.com/jcmturner/rpc/v2/mstypes/reader.go | 109 +++ .../rpc/v2/mstypes/rpc_unicode_string.go | 13 + github.com/jcmturner/rpc/v2/mstypes/sid.go | 32 + .../rpc/v2/mstypes/user_session_key.go | 11 + github.com/jcmturner/rpc/v2/ndr/arrays.go | 413 ++++++++++ github.com/jcmturner/rpc/v2/ndr/decoder.go | 393 ++++++++++ github.com/jcmturner/rpc/v2/ndr/error.go | 18 + github.com/jcmturner/rpc/v2/ndr/header.go | 116 +++ github.com/jcmturner/rpc/v2/ndr/pipe.go | 31 + github.com/jcmturner/rpc/v2/ndr/primitives.go | 211 +++++ github.com/jcmturner/rpc/v2/ndr/rawbytes.go | 61 ++ github.com/jcmturner/rpc/v2/ndr/strings.go | 70 ++ github.com/jcmturner/rpc/v2/ndr/tags.go | 69 ++ github.com/jcmturner/rpc/v2/ndr/union.go | 57 ++ github.com/lib/pq/README.md | 4 + github.com/lib/pq/auth/kerberos/go.mod | 8 + github.com/lib/pq/auth/kerberos/go.sum | 40 + github.com/lib/pq/auth/kerberos/krb.go | 29 + github.com/lib/pq/auth/kerberos/krb_unix.go | 127 +++ .../lib/pq/auth/kerberos/krb_windows.go | 66 ++ github.com/lib/pq/conn.go | 65 +- github.com/lib/pq/connector.go | 7 +- github.com/lib/pq/copy.go | 25 +- github.com/lib/pq/doc.go | 18 + github.com/lib/pq/go.mod | 2 + github.com/lib/pq/krb.go | 27 + modules.txt | 52 +- 149 files changed, 16828 insertions(+), 10 deletions(-) create mode 100644 github.com/alexbrainman/sspi/LICENSE create mode 100644 github.com/alexbrainman/sspi/README.md create mode 100644 github.com/alexbrainman/sspi/buffer.go create mode 100644 github.com/alexbrainman/sspi/mksyscall.go create mode 100644 github.com/alexbrainman/sspi/negotiate/negotiate.go create mode 100644 github.com/alexbrainman/sspi/sspi.go create mode 100644 github.com/alexbrainman/sspi/syscall.go create mode 100644 github.com/alexbrainman/sspi/zsyscall_windows.go create mode 100644 github.com/jcmturner/aescts/v2/LICENSE create mode 100644 github.com/jcmturner/aescts/v2/aescts.go create mode 100644 github.com/jcmturner/aescts/v2/go.mod create mode 100644 github.com/jcmturner/aescts/v2/go.sum create mode 100644 github.com/jcmturner/dnsutils/v2/LICENSE create mode 100644 github.com/jcmturner/dnsutils/v2/go.mod create mode 100644 github.com/jcmturner/dnsutils/v2/go.sum create mode 100644 github.com/jcmturner/dnsutils/v2/srv.go create mode 100644 github.com/jcmturner/goidentity/v6/LICENSE create mode 100644 github.com/jcmturner/goidentity/v6/README.md create mode 100644 github.com/jcmturner/goidentity/v6/authenticator.go create mode 100644 github.com/jcmturner/goidentity/v6/go.mod create mode 100644 github.com/jcmturner/goidentity/v6/go.sum create mode 100644 github.com/jcmturner/goidentity/v6/identity.go create mode 100644 github.com/jcmturner/goidentity/v6/user.go create mode 100644 github.com/jcmturner/gokrb5/v8/LICENSE create mode 100644 github.com/jcmturner/gokrb5/v8/asn1tools/tools.go create mode 100644 github.com/jcmturner/gokrb5/v8/client/ASExchange.go create mode 100644 github.com/jcmturner/gokrb5/v8/client/TGSExchange.go create mode 100644 github.com/jcmturner/gokrb5/v8/client/cache.go create mode 100644 github.com/jcmturner/gokrb5/v8/client/client.go create mode 100644 github.com/jcmturner/gokrb5/v8/client/network.go create mode 100644 github.com/jcmturner/gokrb5/v8/client/passwd.go create mode 100644 github.com/jcmturner/gokrb5/v8/client/session.go create mode 100644 github.com/jcmturner/gokrb5/v8/client/settings.go create mode 100644 github.com/jcmturner/gokrb5/v8/config/error.go create mode 100644 github.com/jcmturner/gokrb5/v8/config/hosts.go create mode 100644 github.com/jcmturner/gokrb5/v8/config/krb5conf.go create mode 100644 github.com/jcmturner/gokrb5/v8/credentials/ccache.go create mode 100644 github.com/jcmturner/gokrb5/v8/credentials/credentials.go create mode 100644 github.com/jcmturner/gokrb5/v8/crypto/aes128-cts-hmac-sha1-96.go create mode 100644 github.com/jcmturner/gokrb5/v8/crypto/aes128-cts-hmac-sha256-128.go create mode 100644 github.com/jcmturner/gokrb5/v8/crypto/aes256-cts-hmac-sha1-96.go create mode 100644 github.com/jcmturner/gokrb5/v8/crypto/aes256-cts-hmac-sha384-192.go create mode 100644 github.com/jcmturner/gokrb5/v8/crypto/common/common.go create mode 100644 github.com/jcmturner/gokrb5/v8/crypto/crypto.go create mode 100644 github.com/jcmturner/gokrb5/v8/crypto/des3-cbc-sha1-kd.go create mode 100644 github.com/jcmturner/gokrb5/v8/crypto/etype/etype.go create mode 100644 github.com/jcmturner/gokrb5/v8/crypto/rc4-hmac.go create mode 100644 github.com/jcmturner/gokrb5/v8/crypto/rfc3961/encryption.go create mode 100644 github.com/jcmturner/gokrb5/v8/crypto/rfc3961/keyDerivation.go create mode 100644 github.com/jcmturner/gokrb5/v8/crypto/rfc3961/nfold.go create mode 100644 github.com/jcmturner/gokrb5/v8/crypto/rfc3962/encryption.go create mode 100644 github.com/jcmturner/gokrb5/v8/crypto/rfc3962/keyDerivation.go create mode 100644 github.com/jcmturner/gokrb5/v8/crypto/rfc4757/checksum.go create mode 100644 github.com/jcmturner/gokrb5/v8/crypto/rfc4757/encryption.go create mode 100644 github.com/jcmturner/gokrb5/v8/crypto/rfc4757/keyDerivation.go create mode 100644 github.com/jcmturner/gokrb5/v8/crypto/rfc4757/msgtype.go create mode 100644 github.com/jcmturner/gokrb5/v8/crypto/rfc8009/encryption.go create mode 100644 github.com/jcmturner/gokrb5/v8/crypto/rfc8009/keyDerivation.go create mode 100644 github.com/jcmturner/gokrb5/v8/gssapi/MICToken.go create mode 100644 github.com/jcmturner/gokrb5/v8/gssapi/README.md create mode 100644 github.com/jcmturner/gokrb5/v8/gssapi/contextFlags.go create mode 100644 github.com/jcmturner/gokrb5/v8/gssapi/gssapi.go create mode 100644 github.com/jcmturner/gokrb5/v8/gssapi/wrapToken.go create mode 100644 github.com/jcmturner/gokrb5/v8/iana/addrtype/constants.go create mode 100644 github.com/jcmturner/gokrb5/v8/iana/adtype/constants.go create mode 100644 github.com/jcmturner/gokrb5/v8/iana/asnAppTag/constants.go create mode 100644 github.com/jcmturner/gokrb5/v8/iana/chksumtype/constants.go create mode 100644 github.com/jcmturner/gokrb5/v8/iana/constants.go create mode 100644 github.com/jcmturner/gokrb5/v8/iana/errorcode/constants.go create mode 100644 github.com/jcmturner/gokrb5/v8/iana/etypeID/constants.go create mode 100644 github.com/jcmturner/gokrb5/v8/iana/flags/constants.go create mode 100644 github.com/jcmturner/gokrb5/v8/iana/keyusage/constants.go create mode 100644 github.com/jcmturner/gokrb5/v8/iana/msgtype/constants.go create mode 100644 github.com/jcmturner/gokrb5/v8/iana/nametype/constants.go create mode 100644 github.com/jcmturner/gokrb5/v8/iana/patype/constants.go create mode 100644 github.com/jcmturner/gokrb5/v8/kadmin/changepasswddata.go create mode 100644 github.com/jcmturner/gokrb5/v8/kadmin/message.go create mode 100644 github.com/jcmturner/gokrb5/v8/kadmin/passwd.go create mode 100644 github.com/jcmturner/gokrb5/v8/keytab/keytab.go create mode 100644 github.com/jcmturner/gokrb5/v8/krberror/error.go create mode 100644 github.com/jcmturner/gokrb5/v8/messages/APRep.go create mode 100644 github.com/jcmturner/gokrb5/v8/messages/APReq.go create mode 100644 github.com/jcmturner/gokrb5/v8/messages/KDCRep.go create mode 100644 github.com/jcmturner/gokrb5/v8/messages/KDCReq.go create mode 100644 github.com/jcmturner/gokrb5/v8/messages/KRBCred.go create mode 100644 github.com/jcmturner/gokrb5/v8/messages/KRBError.go create mode 100644 github.com/jcmturner/gokrb5/v8/messages/KRBPriv.go create mode 100644 github.com/jcmturner/gokrb5/v8/messages/KRBSafe.go create mode 100644 github.com/jcmturner/gokrb5/v8/messages/Ticket.go create mode 100644 github.com/jcmturner/gokrb5/v8/pac/client_claims.go create mode 100644 github.com/jcmturner/gokrb5/v8/pac/client_info.go create mode 100644 github.com/jcmturner/gokrb5/v8/pac/credentials_info.go create mode 100644 github.com/jcmturner/gokrb5/v8/pac/device_claims.go create mode 100644 github.com/jcmturner/gokrb5/v8/pac/device_info.go create mode 100644 github.com/jcmturner/gokrb5/v8/pac/kerb_validation_info.go create mode 100644 github.com/jcmturner/gokrb5/v8/pac/pac_type.go create mode 100644 github.com/jcmturner/gokrb5/v8/pac/s4u_delegation_info.go create mode 100644 github.com/jcmturner/gokrb5/v8/pac/signature_data.go create mode 100644 github.com/jcmturner/gokrb5/v8/pac/supplemental_cred.go create mode 100644 github.com/jcmturner/gokrb5/v8/pac/upn_dns_info.go create mode 100644 github.com/jcmturner/gokrb5/v8/service/APExchange.go create mode 100644 github.com/jcmturner/gokrb5/v8/service/authenticator.go create mode 100644 github.com/jcmturner/gokrb5/v8/service/cache.go create mode 100644 github.com/jcmturner/gokrb5/v8/service/settings.go create mode 100644 github.com/jcmturner/gokrb5/v8/spnego/http.go create mode 100644 github.com/jcmturner/gokrb5/v8/spnego/krb5Token.go create mode 100644 github.com/jcmturner/gokrb5/v8/spnego/negotiationToken.go create mode 100644 github.com/jcmturner/gokrb5/v8/spnego/spnego.go create mode 100644 github.com/jcmturner/gokrb5/v8/types/Authenticator.go create mode 100644 github.com/jcmturner/gokrb5/v8/types/AuthorizationData.go create mode 100644 github.com/jcmturner/gokrb5/v8/types/Cryptosystem.go create mode 100644 github.com/jcmturner/gokrb5/v8/types/HostAddress.go create mode 100644 github.com/jcmturner/gokrb5/v8/types/KerberosFlags.go create mode 100644 github.com/jcmturner/gokrb5/v8/types/PAData.go create mode 100644 github.com/jcmturner/gokrb5/v8/types/PrincipalName.go create mode 100644 github.com/jcmturner/gokrb5/v8/types/TypedData.go create mode 100644 github.com/jcmturner/rpc/v2/LICENSE create mode 100644 github.com/jcmturner/rpc/v2/mstypes/claims.go create mode 100644 github.com/jcmturner/rpc/v2/mstypes/common.go create mode 100644 github.com/jcmturner/rpc/v2/mstypes/filetime.go create mode 100644 github.com/jcmturner/rpc/v2/mstypes/group_membership.go create mode 100644 github.com/jcmturner/rpc/v2/mstypes/kerb_sid_and_attributes.go create mode 100644 github.com/jcmturner/rpc/v2/mstypes/reader.go create mode 100644 github.com/jcmturner/rpc/v2/mstypes/rpc_unicode_string.go create mode 100644 github.com/jcmturner/rpc/v2/mstypes/sid.go create mode 100644 github.com/jcmturner/rpc/v2/mstypes/user_session_key.go create mode 100644 github.com/jcmturner/rpc/v2/ndr/arrays.go create mode 100644 github.com/jcmturner/rpc/v2/ndr/decoder.go create mode 100644 github.com/jcmturner/rpc/v2/ndr/error.go create mode 100644 github.com/jcmturner/rpc/v2/ndr/header.go create mode 100644 github.com/jcmturner/rpc/v2/ndr/pipe.go create mode 100644 github.com/jcmturner/rpc/v2/ndr/primitives.go create mode 100644 github.com/jcmturner/rpc/v2/ndr/rawbytes.go create mode 100644 github.com/jcmturner/rpc/v2/ndr/strings.go create mode 100644 github.com/jcmturner/rpc/v2/ndr/tags.go create mode 100644 github.com/jcmturner/rpc/v2/ndr/union.go create mode 100644 github.com/lib/pq/auth/kerberos/go.mod create mode 100644 github.com/lib/pq/auth/kerberos/go.sum create mode 100644 github.com/lib/pq/auth/kerberos/krb.go create mode 100644 github.com/lib/pq/auth/kerberos/krb_unix.go create mode 100644 github.com/lib/pq/auth/kerberos/krb_windows.go create mode 100644 github.com/lib/pq/krb.go diff --git a/github.com/alexbrainman/sspi/LICENSE b/github.com/alexbrainman/sspi/LICENSE new file mode 100644 index 0000000000..7448756763 --- /dev/null +++ b/github.com/alexbrainman/sspi/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/github.com/alexbrainman/sspi/README.md b/github.com/alexbrainman/sspi/README.md new file mode 100644 index 0000000000..848b6b586b --- /dev/null +++ b/github.com/alexbrainman/sspi/README.md @@ -0,0 +1 @@ +This repository holds Go packages for accessing Security Support Provider Interface on Windows. diff --git a/github.com/alexbrainman/sspi/buffer.go b/github.com/alexbrainman/sspi/buffer.go new file mode 100644 index 0000000000..f4b0ef326d --- /dev/null +++ b/github.com/alexbrainman/sspi/buffer.go @@ -0,0 +1,57 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build windows + +package sspi + +import ( + "io" + "unsafe" +) + +func (b *SecBuffer) Set(buftype uint32, data []byte) { + b.BufferType = buftype + if len(data) > 0 { + b.Buffer = &data[0] + b.BufferSize = uint32(len(data)) + } else { + b.Buffer = nil + b.BufferSize = 0 + } +} + +func (b *SecBuffer) Free() error { + if b.Buffer == nil { + return nil + } + return FreeContextBuffer((*byte)(unsafe.Pointer(b.Buffer))) +} + +func (b *SecBuffer) Bytes() []byte { + if b.Buffer == nil || b.BufferSize <= 0 { + return nil + } + return (*[2 << 20]byte)(unsafe.Pointer(b.Buffer))[:b.BufferSize] +} + +func (b *SecBuffer) WriteAll(w io.Writer) (int, error) { + if b.BufferSize == 0 || b.Buffer == nil { + return 0, nil + } + data := b.Bytes() + total := 0 + for { + n, err := w.Write(data) + total += n + if err != nil { + return total, err + } + if n >= len(data) { + break + } + data = data[n:] + } + return total, nil +} diff --git a/github.com/alexbrainman/sspi/mksyscall.go b/github.com/alexbrainman/sspi/mksyscall.go new file mode 100644 index 0000000000..19e1195984 --- /dev/null +++ b/github.com/alexbrainman/sspi/mksyscall.go @@ -0,0 +1,7 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sspi + +//go:generate go run $GOROOT/src/syscall/mksyscall_windows.go -systemdll=false -output=zsyscall_windows.go syscall.go diff --git a/github.com/alexbrainman/sspi/negotiate/negotiate.go b/github.com/alexbrainman/sspi/negotiate/negotiate.go new file mode 100644 index 0000000000..e04126087e --- /dev/null +++ b/github.com/alexbrainman/sspi/negotiate/negotiate.go @@ -0,0 +1,462 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build windows + +// Package negotiate provides access to the Microsoft Negotiate SSP Package. +// +package negotiate + +import ( + "errors" + "syscall" + "time" + "unsafe" + + "github.com/alexbrainman/sspi" +) + +// TODO: maybe (if possible) move all winapi related out of sspi and into sspi/internal/winapi + +// PackageInfo contains Negotiate SSP package description. +var PackageInfo *sspi.PackageInfo + +func init() { + var err error + PackageInfo, err = sspi.QueryPackageInfo(sspi.NEGOSSP_NAME) + if err != nil { + panic("failed to fetch Negotiate package info: " + err.Error()) + } +} + +func acquireCredentials(principalName string, creduse uint32, ai *sspi.SEC_WINNT_AUTH_IDENTITY) (*sspi.Credentials, error) { + c, err := sspi.AcquireCredentials(principalName, sspi.NEGOSSP_NAME, creduse, (*byte)(unsafe.Pointer(ai))) + if err != nil { + return nil, err + } + return c, nil +} + +// AcquireCurrentUserCredentials acquires credentials of currently +// logged on user. These will be used by the client to authenticate +// itself to the server. It will also be used by the server +// to impersonate the user. +func AcquireCurrentUserCredentials() (*sspi.Credentials, error) { + return acquireCredentials("", sspi.SECPKG_CRED_OUTBOUND, nil) +} + +// TODO: see if I can share this common ntlm and negotiate code + +// AcquireUserCredentials acquires credentials of user described by +// domain, username and password. These will be used by the client to +// authenticate itself to the server. It will also be used by the +// server to impersonate the user. +func AcquireUserCredentials(domain, username, password string) (*sspi.Credentials, error) { + if len(username) == 0 { + return nil, errors.New("username parameter cannot be empty") + } + d, err := syscall.UTF16FromString(domain) + if err != nil { + return nil, err + } + u, err := syscall.UTF16FromString(username) + if err != nil { + return nil, err + } + p, err := syscall.UTF16FromString(password) + if err != nil { + return nil, err + } + ai := sspi.SEC_WINNT_AUTH_IDENTITY{ + User: &u[0], + UserLength: uint32(len(u) - 1), // do not count terminating 0 + Domain: &d[0], + DomainLength: uint32(len(d) - 1), // do not count terminating 0 + Password: &p[0], + PasswordLength: uint32(len(p) - 1), // do not count terminating 0 + Flags: sspi.SEC_WINNT_AUTH_IDENTITY_UNICODE, + } + return acquireCredentials("", sspi.SECPKG_CRED_OUTBOUND, &ai) +} + +// AcquireServerCredentials acquires server credentials that will +// be used to authenticate clients. +// The principalName parameter is passed to the underlying call to +// the winapi AcquireCredentialsHandle function (and specifies the +// name of the principal whose credentials the underlying handle +// will reference). +// As a special case, using an empty string for the principal name +// will require the credential of the user under whose security context +// the current process is running. +func AcquireServerCredentials(principalName string) (*sspi.Credentials, error) { + return acquireCredentials(principalName, sspi.SECPKG_CRED_INBOUND, nil) +} + +func updateContext(c *sspi.Context, dst, src []byte, targetName *uint16) (authCompleted bool, n int, err error) { + var inBuf, outBuf [1]sspi.SecBuffer + inBuf[0].Set(sspi.SECBUFFER_TOKEN, src) + inBufs := &sspi.SecBufferDesc{ + Version: sspi.SECBUFFER_VERSION, + BuffersCount: 1, + Buffers: &inBuf[0], + } + outBuf[0].Set(sspi.SECBUFFER_TOKEN, dst) + outBufs := &sspi.SecBufferDesc{ + Version: sspi.SECBUFFER_VERSION, + BuffersCount: 1, + Buffers: &outBuf[0], + } + ret := c.Update(targetName, outBufs, inBufs) + switch ret { + case sspi.SEC_E_OK: + // session established -> return success + return true, int(outBuf[0].BufferSize), nil + case sspi.SEC_I_COMPLETE_NEEDED, sspi.SEC_I_COMPLETE_AND_CONTINUE: + ret = sspi.CompleteAuthToken(c.Handle, outBufs) + if ret != sspi.SEC_E_OK { + return false, 0, ret + } + case sspi.SEC_I_CONTINUE_NEEDED: + default: + return false, 0, ret + } + return false, int(outBuf[0].BufferSize), nil +} + +func makeSignature(c *sspi.Context, msg []byte, qop, seqno uint32) ([]byte, error) { + _, maxSignature, _, _, err := c.Sizes() + if err != nil { + return nil, err + } + + if maxSignature == 0 { + return nil, errors.New("integrity services are not requested or unavailable") + } + + var b [2]sspi.SecBuffer + b[0].Set(sspi.SECBUFFER_DATA, msg) + b[1].Set(sspi.SECBUFFER_TOKEN, make([]byte, maxSignature)) + + ret := sspi.MakeSignature(c.Handle, qop, sspi.NewSecBufferDesc(b[:]), seqno) + if ret != sspi.SEC_E_OK { + return nil, ret + } + + return b[1].Bytes(), nil +} + +func encryptMessage(c *sspi.Context, msg []byte, qop, seqno uint32) ([]byte, error) { + _ /*maxToken*/, maxSignature, cBlockSize, cSecurityTrailer, err := c.Sizes() + if err != nil { + return nil, err + } + + if maxSignature == 0 { + return nil, errors.New("integrity services are not requested or unavailable") + } + + var b [3]sspi.SecBuffer + b[0].Set(sspi.SECBUFFER_TOKEN, make([]byte, cSecurityTrailer)) + b[1].Set(sspi.SECBUFFER_DATA, msg) + b[2].Set(sspi.SECBUFFER_PADDING, make([]byte, cBlockSize)) + + ret := sspi.EncryptMessage(c.Handle, qop, sspi.NewSecBufferDesc(b[:]), seqno) + if ret != sspi.SEC_E_OK { + return nil, ret + } + + r0, r1, r2 := b[0].Bytes(), b[1].Bytes(), b[2].Bytes() + res := make([]byte, 0, len(r0)+len(r1)+len(r2)) + res = append(res, r0...) + res = append(res, r1...) + res = append(res, r2...) + + return res, nil +} + +func decryptMessage(c *sspi.Context, msg []byte, seqno uint32) (uint32, []byte, error) { + var b [2]sspi.SecBuffer + b[0].Set(sspi.SECBUFFER_STREAM, msg) + b[1].Set(sspi.SECBUFFER_DATA, []byte{}) + + var qop uint32 + ret := sspi.DecryptMessage(c.Handle, sspi.NewSecBufferDesc(b[:]), seqno, &qop) + if ret != sspi.SEC_E_OK { + return qop, nil, ret + } + + return qop, b[1].Bytes(), nil +} + +func verifySignature(c *sspi.Context, msg, token []byte, seqno uint32) (uint32, error) { + var b [2]sspi.SecBuffer + b[0].Set(sspi.SECBUFFER_DATA, msg) + b[1].Set(sspi.SECBUFFER_TOKEN, token) + + var qop uint32 + + ret := sspi.VerifySignature(c.Handle, sspi.NewSecBufferDesc(b[:]), seqno, &qop) + if ret != sspi.SEC_E_OK { + return 0, ret + } + + return qop, nil +} + +// ClientContext is used by the client to manage all steps of Negotiate negotiation. +type ClientContext struct { + sctxt *sspi.Context + targetName *uint16 +} + +// NewClientContext creates a new client context. It uses client +// credentials cred generated by AcquireCurrentUserCredentials or +// AcquireUserCredentials and SPN to start a client Negotiate +// negotiation sequence. targetName is the service principal name +// (SPN) or the security context of the destination server. +// NewClientContext returns a new token to be sent to the server. +func NewClientContext(cred *sspi.Credentials, targetName string) (cc *ClientContext, outputToken []byte, err error) { + return NewClientContextWithFlags(cred, targetName, sspi.ISC_REQ_CONNECTION) +} + +// NewClientContextWithFlags creates a new client context. It uses client +// credentials cred generated by AcquireCurrentUserCredentials or +// AcquireUserCredentials and SPN to start a client Negotiate +// negotiation sequence. targetName is the service principal name +// (SPN) or the security context of the destination server. +// The flags parameter is used to indicate requests for the context +// (for example sspi.ISC_REQ_CONFIDENTIALITY|sspi.ISC_REQ_REPLAY_DETECT) +// NewClientContextWithFlags returns a new token to be sent to the server. +func NewClientContextWithFlags(cred *sspi.Credentials, targetName string, flags uint32) (cc *ClientContext, outputToken []byte, err error) { + var tname *uint16 + if len(targetName) > 0 { + p, err2 := syscall.UTF16FromString(targetName) + if err2 != nil { + return nil, nil, err2 + } + if len(p) > 0 { + tname = &p[0] + } + } + otoken := make([]byte, PackageInfo.MaxToken) + c := sspi.NewClientContext(cred, flags) + + authCompleted, n, err2 := updateContext(c, otoken, nil, tname) + if err2 != nil { + return nil, nil, err2 + } + if authCompleted { + c.Release() + return nil, nil, errors.New("negotiate authentication should not be completed yet") + } + if n == 0 { + c.Release() + return nil, nil, errors.New("negotiate token should not be empty") + } + otoken = otoken[:n] + return &ClientContext{sctxt: c, targetName: tname}, otoken, nil +} + +// Release free up resources associated with client context c. +func (c *ClientContext) Release() error { + if c == nil { + return nil + } + return c.sctxt.Release() +} + +// Expiry returns c expiry time. +func (c *ClientContext) Expiry() time.Time { + return c.sctxt.Expiry() +} + +// Update advances client part of Negotiate negotiation c. It uses +// token received from the server and returns true if client part +// of authentication is complete. It also returns new token to be +// sent to the server. +func (c *ClientContext) Update(token []byte) (authCompleted bool, outputToken []byte, err error) { + otoken := make([]byte, PackageInfo.MaxToken) + authDone, n, err2 := updateContext(c.sctxt, otoken, token, c.targetName) + if err2 != nil { + return false, nil, err2 + } + if n == 0 && !authDone { + return false, nil, errors.New("negotiate token should not be empty") + } + otoken = otoken[:n] + return authDone, otoken, nil +} + +// Sizes queries the client context for the sizes used in per-message +// functions. It returns the maximum token size used in authentication +// exchanges, the maximum signature size, the preferred integral size of +// messages, the size of any security trailer, and any error. +func (c *ClientContext) Sizes() (uint32, uint32, uint32, uint32, error) { + return c.sctxt.Sizes() +} + +// MakeSignature uses the established client context to create a signature +// for the given message using the provided quality of protection flags and +// sequence number. It returns the signature token in addition to any error. +func (c *ClientContext) MakeSignature(msg []byte, qop, seqno uint32) ([]byte, error) { + return makeSignature(c.sctxt, msg, qop, seqno) +} + +// VerifySignature uses the established client context and signature token +// to check that the provided message hasn't been tampered or received out +// of sequence. It returns any quality of protection flags and any error +// that occurred. +func (c *ClientContext) VerifySignature(msg, token []byte, seqno uint32) (uint32, error) { + return verifySignature(c.sctxt, msg, token, seqno) +} + +// EncryptMessage uses the established client context to encrypt a message +// using the provided quality of protection flags and sequence number. +// It returns the signature token in addition to any error. +// IMPORTANT: the input msg parameter is updated in place by the low-level windows api +// so must be copied if the initial content should not be modified. +func (c *ClientContext) EncryptMessage(msg []byte, qop, seqno uint32) ([]byte, error) { + return encryptMessage(c.sctxt, msg, qop, seqno) +} + +// DecryptMessage uses the established client context to decrypt a message +// using the provided sequence number. +// It returns the quality of protection flag and the decrypted message in addition to any error. +func (c *ClientContext) DecryptMessage(msg []byte, seqno uint32) (uint32, []byte, error) { + return decryptMessage(c.sctxt, msg, seqno) +} + +// VerifyFlags determines if all flags used to construct the client context +// were honored (see NewClientContextWithFlags). It should be called after c.Update. +func (c *ClientContext) VerifyFlags() error { + return c.sctxt.VerifyFlags() +} + +// VerifySelectiveFlags determines if the given flags were honored (see NewClientContextWithFlags). +// It should be called after c.Update. +func (c *ClientContext) VerifySelectiveFlags(flags uint32) error { + return c.sctxt.VerifySelectiveFlags(flags) +} + +// ServerContext is used by the server to manage all steps of Negotiate +// negotiation. Once authentication is completed the context can be +// used to impersonate client. +type ServerContext struct { + sctxt *sspi.Context +} + +// NewServerContext creates new server context. It uses server +// credentials created by AcquireServerCredentials and token from +// the client to start server Negotiate negotiation sequence. +// It also returns new token to be sent to the client. +func NewServerContext(cred *sspi.Credentials, token []byte) (sc *ServerContext, authDone bool, outputToken []byte, err error) { + otoken := make([]byte, PackageInfo.MaxToken) + c := sspi.NewServerContext(cred, sspi.ASC_REQ_CONNECTION) + authDone, n, err2 := updateContext(c, otoken, token, nil) + if err2 != nil { + return nil, false, nil, err2 + } + otoken = otoken[:n] + return &ServerContext{sctxt: c}, authDone, otoken, nil +} + +// Release free up resources associated with server context c. +func (c *ServerContext) Release() error { + if c == nil { + return nil + } + return c.sctxt.Release() +} + +// Expiry returns c expiry time. +func (c *ServerContext) Expiry() time.Time { + return c.sctxt.Expiry() +} + +// Update advances server part of Negotiate negotiation c. It uses +// token received from the client and returns true if server part +// of authentication is complete. It also returns new token to be +// sent to the client. +func (c *ServerContext) Update(token []byte) (authCompleted bool, outputToken []byte, err error) { + otoken := make([]byte, PackageInfo.MaxToken) + authDone, n, err2 := updateContext(c.sctxt, otoken, token, nil) + if err2 != nil { + return false, nil, err2 + } + if n == 0 && !authDone { + return false, nil, errors.New("negotiate token should not be empty") + } + otoken = otoken[:n] + return authDone, otoken, nil +} + +const _SECPKG_ATTR_NATIVE_NAMES = 13 + +type _SecPkgContext_NativeNames struct { + ClientName *uint16 + ServerName *uint16 +} + +// GetUsername returns the username corresponding to the authenticated client +func (c *ServerContext) GetUsername() (string, error) { + var ns _SecPkgContext_NativeNames + ret := sspi.QueryContextAttributes(c.sctxt.Handle, _SECPKG_ATTR_NATIVE_NAMES, (*byte)(unsafe.Pointer(&ns))) + if ret != sspi.SEC_E_OK { + return "", ret + } + sspi.FreeContextBuffer((*byte)(unsafe.Pointer(ns.ServerName))) + defer sspi.FreeContextBuffer((*byte)(unsafe.Pointer(ns.ClientName))) + return syscall.UTF16ToString((*[2 << 20]uint16)(unsafe.Pointer(ns.ClientName))[:]), nil +} + +// ImpersonateUser changes current OS thread user. New user is +// the user as specified by client credentials. +func (c *ServerContext) ImpersonateUser() error { + return c.sctxt.ImpersonateUser() +} + +// RevertToSelf stops impersonation. It changes current OS thread +// user to what it was before ImpersonateUser was executed. +func (c *ServerContext) RevertToSelf() error { + return c.sctxt.RevertToSelf() +} + +// Sizes queries the server context for the sizes used in per-message +// functions. It returns the maximum token size used in authentication +// exchanges, the maximum signature size, the preferred integral size of +// messages, the size of any security trailer, and any error. +func (c *ServerContext) Sizes() (uint32, uint32, uint32, uint32, error) { + return c.sctxt.Sizes() +} + +// MakeSignature uses the established server context to create a signature +// for the given message using the provided quality of protection flags and +// sequence number. It returns the signature token in addition to any error. +func (c *ServerContext) MakeSignature(msg []byte, qop, seqno uint32) ([]byte, error) { + return makeSignature(c.sctxt, msg, qop, seqno) +} + +// VerifySignature uses the established server context and signature token +// to check that the provided message hasn't been tampered or received out +// of sequence. It returns any quality of protection flags and any error +// that occurred. +func (c *ServerContext) VerifySignature(msg, token []byte, seqno uint32) (uint32, error) { + return verifySignature(c.sctxt, msg, token, seqno) +} + +// EncryptMessage uses the established server context to encrypt a message +// using the provided quality of protection flags and sequence number. +// It returns the signature token in addition to any error. +// IMPORTANT: the input msg parameter is updated in place by the low-level windows api +// so must be copied if the initial content should not be modified. +func (c *ServerContext) EncryptMessage(msg []byte, qop, seqno uint32) ([]byte, error) { + return encryptMessage(c.sctxt, msg, qop, seqno) +} + +// DecryptMessage uses the established server context to decrypt a message +// using the provided sequence number. +// It returns the quality of protection flag and the decrypted message in addition to any error. +func (c *ServerContext) DecryptMessage(msg []byte, seqno uint32) (uint32, []byte, error) { + return decryptMessage(c.sctxt, msg, seqno) +} diff --git a/github.com/alexbrainman/sspi/sspi.go b/github.com/alexbrainman/sspi/sspi.go new file mode 100644 index 0000000000..04f20b75ae --- /dev/null +++ b/github.com/alexbrainman/sspi/sspi.go @@ -0,0 +1,226 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build windows + +package sspi + +import ( + "fmt" + "syscall" + "time" + "unsafe" +) + +// TODO: add documentation + +type PackageInfo struct { + Capabilities uint32 + Version uint16 + RPCID uint16 + MaxToken uint32 + Name string + Comment string +} + +func QueryPackageInfo(pkgname string) (*PackageInfo, error) { + name, err := syscall.UTF16PtrFromString(pkgname) + if err != nil { + return nil, err + } + var pi *SecPkgInfo + ret := QuerySecurityPackageInfo(name, &pi) + if ret != SEC_E_OK { + return nil, ret + } + defer FreeContextBuffer((*byte)(unsafe.Pointer(pi))) + + return &PackageInfo{ + Capabilities: pi.Capabilities, + Version: pi.Version, + RPCID: pi.RPCID, + MaxToken: pi.MaxToken, + Name: syscall.UTF16ToString((*[2 << 12]uint16)(unsafe.Pointer(pi.Name))[:]), + Comment: syscall.UTF16ToString((*[2 << 12]uint16)(unsafe.Pointer(pi.Comment))[:]), + }, nil +} + +type Credentials struct { + Handle CredHandle + expiry syscall.Filetime +} + +// AcquireCredentials calls the windows AcquireCredentialsHandle function and +// returns Credentials containing a security handle that can be used for +// InitializeSecurityContext or AcceptSecurityContext operations. +// As a special case, passing an empty string as the principal parameter will +// pass a null string to the underlying function. +func AcquireCredentials(principal string, pkgname string, creduse uint32, authdata *byte) (*Credentials, error) { + var principalName *uint16 + if principal != "" { + var err error + principalName, err = syscall.UTF16PtrFromString(principal) + if err != nil { + return nil, err + } + } + name, err := syscall.UTF16PtrFromString(pkgname) + if err != nil { + return nil, err + } + var c Credentials + ret := AcquireCredentialsHandle(principalName, name, creduse, nil, authdata, 0, 0, &c.Handle, &c.expiry) + if ret != SEC_E_OK { + return nil, ret + } + return &c, nil +} + +func (c *Credentials) Release() error { + if c == nil { + return nil + } + ret := FreeCredentialsHandle(&c.Handle) + if ret != SEC_E_OK { + return ret + } + return nil +} + +func (c *Credentials) Expiry() time.Time { + return time.Unix(0, c.expiry.Nanoseconds()) +} + +// TODO: add functions to display and manage RequestedFlags and EstablishedFlags fields. +// TODO: maybe get rid of RequestedFlags and EstablishedFlags fields, and replace them with input parameter for New...Context and return value of Update (instead of current bool parameter). + +type updateFunc func(c *Context, targname *uint16, h, newh *CtxtHandle, out, in *SecBufferDesc) syscall.Errno + +type Context struct { + Cred *Credentials + Handle *CtxtHandle + handle CtxtHandle + updFn updateFunc + expiry syscall.Filetime + RequestedFlags uint32 + EstablishedFlags uint32 +} + +func NewClientContext(cred *Credentials, flags uint32) *Context { + return &Context{ + Cred: cred, + updFn: initialize, + RequestedFlags: flags, + } +} + +func NewServerContext(cred *Credentials, flags uint32) *Context { + return &Context{ + Cred: cred, + updFn: accept, + RequestedFlags: flags, + } +} + +func initialize(c *Context, targname *uint16, h, newh *CtxtHandle, out, in *SecBufferDesc) syscall.Errno { + return InitializeSecurityContext(&c.Cred.Handle, h, targname, c.RequestedFlags, + 0, SECURITY_NATIVE_DREP, in, 0, newh, out, &c.EstablishedFlags, &c.expiry) +} + +func accept(c *Context, targname *uint16, h, newh *CtxtHandle, out, in *SecBufferDesc) syscall.Errno { + return AcceptSecurityContext(&c.Cred.Handle, h, in, c.RequestedFlags, + SECURITY_NATIVE_DREP, newh, out, &c.EstablishedFlags, &c.expiry) +} + +func (c *Context) Update(targname *uint16, out, in *SecBufferDesc) syscall.Errno { + h := c.Handle + if c.Handle == nil { + c.Handle = &c.handle + } + return c.updFn(c, targname, h, c.Handle, out, in) +} + +func (c *Context) Release() error { + if c == nil { + return nil + } + ret := DeleteSecurityContext(c.Handle) + if ret != SEC_E_OK { + return ret + } + return nil +} + +func (c *Context) Expiry() time.Time { + return time.Unix(0, c.expiry.Nanoseconds()) +} + +// TODO: add comment to function doco that this "impersonation" is applied to current OS thread. +func (c *Context) ImpersonateUser() error { + ret := ImpersonateSecurityContext(c.Handle) + if ret != SEC_E_OK { + return ret + } + return nil +} + +func (c *Context) RevertToSelf() error { + ret := RevertSecurityContext(c.Handle) + if ret != SEC_E_OK { + return ret + } + return nil +} + +// Sizes queries the context for the sizes used in per-message functions. +// It returns the maximum token size used in authentication exchanges, the +// maximum signature size, the preferred integral size of messages, the +// size of any security trailer, and any error. +func (c *Context) Sizes() (uint32, uint32, uint32, uint32, error) { + var s _SecPkgContext_Sizes + ret := QueryContextAttributes(c.Handle, _SECPKG_ATTR_SIZES, (*byte)(unsafe.Pointer(&s))) + if ret != SEC_E_OK { + return 0, 0, 0, 0, ret + } + return s.MaxToken, s.MaxSignature, s.BlockSize, s.SecurityTrailer, nil +} + +// VerifyFlags determines if all flags used to construct the context +// were honored (see NewClientContext). It should be called after c.Update. +func (c *Context) VerifyFlags() error { + return c.VerifySelectiveFlags(c.RequestedFlags) +} + +// VerifySelectiveFlags determines if the given flags were honored (see NewClientContext). +// It should be called after c.Update. +func (c *Context) VerifySelectiveFlags(flags uint32) error { + if valid, missing, extra := verifySelectiveFlags(flags, c.RequestedFlags); !valid { + return fmt.Errorf("sspi: invalid flags check: desired=%b requested=%b missing=%b extra=%b", flags, c.RequestedFlags, missing, extra) + } + if valid, missing, extra := verifySelectiveFlags(flags, c.EstablishedFlags); !valid { + return fmt.Errorf("sspi: invalid flags: desired=%b established=%b missing=%b extra=%b", flags, c.EstablishedFlags, missing, extra) + } + return nil +} + +// verifySelectiveFlags determines if all bits requested in flags are set in establishedFlags. +// missing represents the bits set in flags that are not set in establishedFlags. +// extra represents the bits set in establishedFlags that are not set in flags. +// valid is true and missing is zero when establishedFlags has all of the requested flags. +func verifySelectiveFlags(flags, establishedFlags uint32) (valid bool, missing, extra uint32) { + missing = flags&establishedFlags ^ flags + extra = flags | establishedFlags ^ flags + valid = missing == 0 + return valid, missing, extra +} + +// NewSecBufferDesc returns an initialized SecBufferDesc describing the +// provided SecBuffer. +func NewSecBufferDesc(b []SecBuffer) *SecBufferDesc { + return &SecBufferDesc{ + Version: SECBUFFER_VERSION, + BuffersCount: uint32(len(b)), + Buffers: &b[0], + } +} diff --git a/github.com/alexbrainman/sspi/syscall.go b/github.com/alexbrainman/sspi/syscall.go new file mode 100644 index 0000000000..04660df2ae --- /dev/null +++ b/github.com/alexbrainman/sspi/syscall.go @@ -0,0 +1,174 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build windows + +package sspi + +import ( + "syscall" +) + +const ( + SEC_E_OK = syscall.Errno(0) + + SEC_I_COMPLETE_AND_CONTINUE = syscall.Errno(590612) + SEC_I_COMPLETE_NEEDED = syscall.Errno(590611) + SEC_I_CONTINUE_NEEDED = syscall.Errno(590610) + + SEC_E_LOGON_DENIED = syscall.Errno(0x8009030c) + SEC_E_CONTEXT_EXPIRED = syscall.Errno(0x80090317) // not sure if the value is valid + SEC_E_INCOMPLETE_MESSAGE = syscall.Errno(0x80090318) + + NTLMSP_NAME = "NTLM" + MICROSOFT_KERBEROS_NAME = "Kerberos" + NEGOSSP_NAME = "Negotiate" + UNISP_NAME = "Microsoft Unified Security Protocol Provider" + + _SECPKG_ATTR_SIZES = 0 + _SECPKG_ATTR_NAMES = 1 + _SECPKG_ATTR_LIFESPAN = 2 + _SECPKG_ATTR_DCE_INFO = 3 + _SECPKG_ATTR_STREAM_SIZES = 4 + _SECPKG_ATTR_KEY_INFO = 5 + _SECPKG_ATTR_AUTHORITY = 6 + _SECPKG_ATTR_PROTO_INFO = 7 + _SECPKG_ATTR_PASSWORD_EXPIRY = 8 + _SECPKG_ATTR_SESSION_KEY = 9 + _SECPKG_ATTR_PACKAGE_INFO = 10 + _SECPKG_ATTR_USER_FLAGS = 11 + _SECPKG_ATTR_NEGOTIATION_INFO = 12 + _SECPKG_ATTR_NATIVE_NAMES = 13 + _SECPKG_ATTR_FLAGS = 14 +) + +type SecPkgInfo struct { + Capabilities uint32 + Version uint16 + RPCID uint16 + MaxToken uint32 + Name *uint16 + Comment *uint16 +} + +type _SecPkgContext_Sizes struct { + MaxToken uint32 + MaxSignature uint32 + BlockSize uint32 + SecurityTrailer uint32 +} + +//sys QuerySecurityPackageInfo(pkgname *uint16, pkginfo **SecPkgInfo) (ret syscall.Errno) = secur32.QuerySecurityPackageInfoW +//sys FreeContextBuffer(buf *byte) (ret syscall.Errno) = secur32.FreeContextBuffer + +const ( + SECPKG_CRED_INBOUND = 1 + SECPKG_CRED_OUTBOUND = 2 + SECPKG_CRED_BOTH = (SECPKG_CRED_OUTBOUND | SECPKG_CRED_INBOUND) + + SEC_WINNT_AUTH_IDENTITY_UNICODE = 0x2 +) + +type SEC_WINNT_AUTH_IDENTITY struct { + User *uint16 + UserLength uint32 + Domain *uint16 + DomainLength uint32 + Password *uint16 + PasswordLength uint32 + Flags uint32 +} + +type LUID struct { + LowPart uint32 + HighPart int32 +} + +type CredHandle struct { + Lower uintptr + Upper uintptr +} + +//sys AcquireCredentialsHandle(principal *uint16, pkgname *uint16, creduse uint32, logonid *LUID, authdata *byte, getkeyfn uintptr, getkeyarg uintptr, handle *CredHandle, expiry *syscall.Filetime) (ret syscall.Errno) = secur32.AcquireCredentialsHandleW +//sys FreeCredentialsHandle(handle *CredHandle) (ret syscall.Errno) = secur32.FreeCredentialsHandle + +const ( + SECURITY_NATIVE_DREP = 16 + + SECBUFFER_DATA = 1 + SECBUFFER_TOKEN = 2 + SECBUFFER_PKG_PARAMS = 3 + SECBUFFER_MISSING = 4 + SECBUFFER_EXTRA = 5 + SECBUFFER_STREAM_TRAILER = 6 + SECBUFFER_STREAM_HEADER = 7 + SECBUFFER_PADDING = 9 + SECBUFFER_STREAM = 10 + SECBUFFER_READONLY = 0x80000000 + SECBUFFER_ATTRMASK = 0xf0000000 + SECBUFFER_VERSION = 0 + SECBUFFER_EMPTY = 0 + + ISC_REQ_DELEGATE = 1 + ISC_REQ_MUTUAL_AUTH = 2 + ISC_REQ_REPLAY_DETECT = 4 + ISC_REQ_SEQUENCE_DETECT = 8 + ISC_REQ_CONFIDENTIALITY = 16 + ISC_REQ_USE_SESSION_KEY = 32 + ISC_REQ_PROMPT_FOR_CREDS = 64 + ISC_REQ_USE_SUPPLIED_CREDS = 128 + ISC_REQ_ALLOCATE_MEMORY = 256 + ISC_REQ_USE_DCE_STYLE = 512 + ISC_REQ_DATAGRAM = 1024 + ISC_REQ_CONNECTION = 2048 + ISC_REQ_EXTENDED_ERROR = 16384 + ISC_REQ_STREAM = 32768 + ISC_REQ_INTEGRITY = 65536 + ISC_REQ_MANUAL_CRED_VALIDATION = 524288 + ISC_REQ_HTTP = 268435456 + + ASC_REQ_DELEGATE = 1 + ASC_REQ_MUTUAL_AUTH = 2 + ASC_REQ_REPLAY_DETECT = 4 + ASC_REQ_SEQUENCE_DETECT = 8 + ASC_REQ_CONFIDENTIALITY = 16 + ASC_REQ_USE_SESSION_KEY = 32 + ASC_REQ_ALLOCATE_MEMORY = 256 + ASC_REQ_USE_DCE_STYLE = 512 + ASC_REQ_DATAGRAM = 1024 + ASC_REQ_CONNECTION = 2048 + ASC_REQ_EXTENDED_ERROR = 32768 + ASC_REQ_STREAM = 65536 + ASC_REQ_INTEGRITY = 131072 +) + +type CtxtHandle struct { + Lower uintptr + Upper uintptr +} + +type SecBuffer struct { + BufferSize uint32 + BufferType uint32 + Buffer *byte +} + +type SecBufferDesc struct { + Version uint32 + BuffersCount uint32 + Buffers *SecBuffer +} + +//sys InitializeSecurityContext(credential *CredHandle, context *CtxtHandle, targname *uint16, contextreq uint32, reserved1 uint32, targdatarep uint32, input *SecBufferDesc, reserved2 uint32, newcontext *CtxtHandle, output *SecBufferDesc, contextattr *uint32, expiry *syscall.Filetime) (ret syscall.Errno) = secur32.InitializeSecurityContextW +//sys AcceptSecurityContext(credential *CredHandle, context *CtxtHandle, input *SecBufferDesc, contextreq uint32, targdatarep uint32, newcontext *CtxtHandle, output *SecBufferDesc, contextattr *uint32, expiry *syscall.Filetime) (ret syscall.Errno) = secur32.AcceptSecurityContext +//sys CompleteAuthToken(context *CtxtHandle, token *SecBufferDesc) (ret syscall.Errno) = secur32.CompleteAuthToken +//sys DeleteSecurityContext(context *CtxtHandle) (ret syscall.Errno) = secur32.DeleteSecurityContext +//sys ImpersonateSecurityContext(context *CtxtHandle) (ret syscall.Errno) = secur32.ImpersonateSecurityContext +//sys RevertSecurityContext(context *CtxtHandle) (ret syscall.Errno) = secur32.RevertSecurityContext +//sys QueryContextAttributes(context *CtxtHandle, attribute uint32, buf *byte) (ret syscall.Errno) = secur32.QueryContextAttributesW +//sys EncryptMessage(context *CtxtHandle, qop uint32, message *SecBufferDesc, messageseqno uint32) (ret syscall.Errno) = secur32.EncryptMessage +//sys DecryptMessage(context *CtxtHandle, message *SecBufferDesc, messageseqno uint32, qop *uint32) (ret syscall.Errno) = secur32.DecryptMessage +//sys ApplyControlToken(context *CtxtHandle, input *SecBufferDesc) (ret syscall.Errno) = secur32.ApplyControlToken +//sys MakeSignature(context *CtxtHandle, qop uint32, message *SecBufferDesc, messageseqno uint32) (ret syscall.Errno) = secur32.MakeSignature +//sys VerifySignature(context *CtxtHandle, message *SecBufferDesc, messageseqno uint32, qop *uint32) (ret syscall.Errno) = secur32.VerifySignature diff --git a/github.com/alexbrainman/sspi/zsyscall_windows.go b/github.com/alexbrainman/sspi/zsyscall_windows.go new file mode 100644 index 0000000000..55e8209970 --- /dev/null +++ b/github.com/alexbrainman/sspi/zsyscall_windows.go @@ -0,0 +1,152 @@ +// MACHINE GENERATED BY 'go generate' COMMAND; DO NOT EDIT + +package sspi + +import ( + "syscall" + "unsafe" +) + +var _ unsafe.Pointer + +// Do the interface allocations only once for common +// Errno values. +const ( + errnoERROR_IO_PENDING = 997 +) + +var ( + errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING) +) + +// errnoErr returns common boxed Errno values, to prevent +// allocations at runtime. +func errnoErr(e syscall.Errno) error { + switch e { + case 0: + return nil + case errnoERROR_IO_PENDING: + return errERROR_IO_PENDING + } + // TODO: add more here, after collecting data on the common + // error values see on Windows. (perhaps when running + // all.bat?) + return e +} + +var ( + modsecur32 = syscall.NewLazyDLL("secur32.dll") + + procQuerySecurityPackageInfoW = modsecur32.NewProc("QuerySecurityPackageInfoW") + procFreeContextBuffer = modsecur32.NewProc("FreeContextBuffer") + procAcquireCredentialsHandleW = modsecur32.NewProc("AcquireCredentialsHandleW") + procFreeCredentialsHandle = modsecur32.NewProc("FreeCredentialsHandle") + procInitializeSecurityContextW = modsecur32.NewProc("InitializeSecurityContextW") + procAcceptSecurityContext = modsecur32.NewProc("AcceptSecurityContext") + procCompleteAuthToken = modsecur32.NewProc("CompleteAuthToken") + procDeleteSecurityContext = modsecur32.NewProc("DeleteSecurityContext") + procImpersonateSecurityContext = modsecur32.NewProc("ImpersonateSecurityContext") + procRevertSecurityContext = modsecur32.NewProc("RevertSecurityContext") + procQueryContextAttributesW = modsecur32.NewProc("QueryContextAttributesW") + procEncryptMessage = modsecur32.NewProc("EncryptMessage") + procDecryptMessage = modsecur32.NewProc("DecryptMessage") + procApplyControlToken = modsecur32.NewProc("ApplyControlToken") + procMakeSignature = modsecur32.NewProc("MakeSignature") + procVerifySignature = modsecur32.NewProc("VerifySignature") +) + +func QuerySecurityPackageInfo(pkgname *uint16, pkginfo **SecPkgInfo) (ret syscall.Errno) { + r0, _, _ := syscall.Syscall(procQuerySecurityPackageInfoW.Addr(), 2, uintptr(unsafe.Pointer(pkgname)), uintptr(unsafe.Pointer(pkginfo)), 0) + ret = syscall.Errno(r0) + return +} + +func FreeContextBuffer(buf *byte) (ret syscall.Errno) { + r0, _, _ := syscall.Syscall(procFreeContextBuffer.Addr(), 1, uintptr(unsafe.Pointer(buf)), 0, 0) + ret = syscall.Errno(r0) + return +} + +func AcquireCredentialsHandle(principal *uint16, pkgname *uint16, creduse uint32, logonid *LUID, authdata *byte, getkeyfn uintptr, getkeyarg uintptr, handle *CredHandle, expiry *syscall.Filetime) (ret syscall.Errno) { + r0, _, _ := syscall.Syscall9(procAcquireCredentialsHandleW.Addr(), 9, uintptr(unsafe.Pointer(principal)), uintptr(unsafe.Pointer(pkgname)), uintptr(creduse), uintptr(unsafe.Pointer(logonid)), uintptr(unsafe.Pointer(authdata)), uintptr(getkeyfn), uintptr(getkeyarg), uintptr(unsafe.Pointer(handle)), uintptr(unsafe.Pointer(expiry))) + ret = syscall.Errno(r0) + return +} + +func FreeCredentialsHandle(handle *CredHandle) (ret syscall.Errno) { + r0, _, _ := syscall.Syscall(procFreeCredentialsHandle.Addr(), 1, uintptr(unsafe.Pointer(handle)), 0, 0) + ret = syscall.Errno(r0) + return +} + +func InitializeSecurityContext(credential *CredHandle, context *CtxtHandle, targname *uint16, contextreq uint32, reserved1 uint32, targdatarep uint32, input *SecBufferDesc, reserved2 uint32, newcontext *CtxtHandle, output *SecBufferDesc, contextattr *uint32, expiry *syscall.Filetime) (ret syscall.Errno) { + r0, _, _ := syscall.Syscall12(procInitializeSecurityContextW.Addr(), 12, uintptr(unsafe.Pointer(credential)), uintptr(unsafe.Pointer(context)), uintptr(unsafe.Pointer(targname)), uintptr(contextreq), uintptr(reserved1), uintptr(targdatarep), uintptr(unsafe.Pointer(input)), uintptr(reserved2), uintptr(unsafe.Pointer(newcontext)), uintptr(unsafe.Pointer(output)), uintptr(unsafe.Pointer(contextattr)), uintptr(unsafe.Pointer(expiry))) + ret = syscall.Errno(r0) + return +} + +func AcceptSecurityContext(credential *CredHandle, context *CtxtHandle, input *SecBufferDesc, contextreq uint32, targdatarep uint32, newcontext *CtxtHandle, output *SecBufferDesc, contextattr *uint32, expiry *syscall.Filetime) (ret syscall.Errno) { + r0, _, _ := syscall.Syscall9(procAcceptSecurityContext.Addr(), 9, uintptr(unsafe.Pointer(credential)), uintptr(unsafe.Pointer(context)), uintptr(unsafe.Pointer(input)), uintptr(contextreq), uintptr(targdatarep), uintptr(unsafe.Pointer(newcontext)), uintptr(unsafe.Pointer(output)), uintptr(unsafe.Pointer(contextattr)), uintptr(unsafe.Pointer(expiry))) + ret = syscall.Errno(r0) + return +} + +func CompleteAuthToken(context *CtxtHandle, token *SecBufferDesc) (ret syscall.Errno) { + r0, _, _ := syscall.Syscall(procCompleteAuthToken.Addr(), 2, uintptr(unsafe.Pointer(context)), uintptr(unsafe.Pointer(token)), 0) + ret = syscall.Errno(r0) + return +} + +func DeleteSecurityContext(context *CtxtHandle) (ret syscall.Errno) { + r0, _, _ := syscall.Syscall(procDeleteSecurityContext.Addr(), 1, uintptr(unsafe.Pointer(context)), 0, 0) + ret = syscall.Errno(r0) + return +} + +func ImpersonateSecurityContext(context *CtxtHandle) (ret syscall.Errno) { + r0, _, _ := syscall.Syscall(procImpersonateSecurityContext.Addr(), 1, uintptr(unsafe.Pointer(context)), 0, 0) + ret = syscall.Errno(r0) + return +} + +func RevertSecurityContext(context *CtxtHandle) (ret syscall.Errno) { + r0, _, _ := syscall.Syscall(procRevertSecurityContext.Addr(), 1, uintptr(unsafe.Pointer(context)), 0, 0) + ret = syscall.Errno(r0) + return +} + +func QueryContextAttributes(context *CtxtHandle, attribute uint32, buf *byte) (ret syscall.Errno) { + r0, _, _ := syscall.Syscall(procQueryContextAttributesW.Addr(), 3, uintptr(unsafe.Pointer(context)), uintptr(attribute), uintptr(unsafe.Pointer(buf))) + ret = syscall.Errno(r0) + return +} + +func EncryptMessage(context *CtxtHandle, qop uint32, message *SecBufferDesc, messageseqno uint32) (ret syscall.Errno) { + r0, _, _ := syscall.Syscall6(procEncryptMessage.Addr(), 4, uintptr(unsafe.Pointer(context)), uintptr(qop), uintptr(unsafe.Pointer(message)), uintptr(messageseqno), 0, 0) + ret = syscall.Errno(r0) + return +} + +func DecryptMessage(context *CtxtHandle, message *SecBufferDesc, messageseqno uint32, qop *uint32) (ret syscall.Errno) { + r0, _, _ := syscall.Syscall6(procDecryptMessage.Addr(), 4, uintptr(unsafe.Pointer(context)), uintptr(unsafe.Pointer(message)), uintptr(messageseqno), uintptr(unsafe.Pointer(qop)), 0, 0) + ret = syscall.Errno(r0) + return +} + +func ApplyControlToken(context *CtxtHandle, input *SecBufferDesc) (ret syscall.Errno) { + r0, _, _ := syscall.Syscall(procApplyControlToken.Addr(), 2, uintptr(unsafe.Pointer(context)), uintptr(unsafe.Pointer(input)), 0) + ret = syscall.Errno(r0) + return +} + +func MakeSignature(context *CtxtHandle, qop uint32, message *SecBufferDesc, messageseqno uint32) (ret syscall.Errno) { + r0, _, _ := syscall.Syscall6(procMakeSignature.Addr(), 4, uintptr(unsafe.Pointer(context)), uintptr(qop), uintptr(unsafe.Pointer(message)), uintptr(messageseqno), 0, 0) + ret = syscall.Errno(r0) + return +} + +func VerifySignature(context *CtxtHandle, message *SecBufferDesc, messageseqno uint32, qop *uint32) (ret syscall.Errno) { + r0, _, _ := syscall.Syscall6(procVerifySignature.Addr(), 4, uintptr(unsafe.Pointer(context)), uintptr(unsafe.Pointer(message)), uintptr(messageseqno), uintptr(unsafe.Pointer(qop)), 0, 0) + ret = syscall.Errno(r0) + return +} diff --git a/github.com/jcmturner/aescts/v2/LICENSE b/github.com/jcmturner/aescts/v2/LICENSE new file mode 100644 index 0000000000..8dada3edaf --- /dev/null +++ b/github.com/jcmturner/aescts/v2/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/github.com/jcmturner/aescts/v2/aescts.go b/github.com/jcmturner/aescts/v2/aescts.go new file mode 100644 index 0000000000..fee3b43e91 --- /dev/null +++ b/github.com/jcmturner/aescts/v2/aescts.go @@ -0,0 +1,186 @@ +// Package aescts provides AES CBC CipherText Stealing encryption and decryption methods +package aescts + +import ( + "crypto/aes" + "crypto/cipher" + "errors" + "fmt" +) + +// Encrypt the message with the key and the initial vector. +// Returns: next iv, ciphertext bytes, error +func Encrypt(key, iv, plaintext []byte) ([]byte, []byte, error) { + l := len(plaintext) + + block, err := aes.NewCipher(key) + if err != nil { + return []byte{}, []byte{}, fmt.Errorf("error creating cipher: %v", err) + } + mode := cipher.NewCBCEncrypter(block, iv) + + m := make([]byte, len(plaintext)) + copy(m, plaintext) + + /*For consistency, ciphertext stealing is always used for the last two + blocks of the data to be encrypted, as in [RC5]. If the data length + is a multiple of the block size, this is equivalent to plain CBC mode + with the last two ciphertext blocks swapped.*/ + /*The initial vector carried out from one encryption for use in a + subsequent encryption is the next-to-last block of the encryption + output; this is the encrypted form of the last plaintext block.*/ + if l <= aes.BlockSize { + m, _ = zeroPad(m, aes.BlockSize) + mode.CryptBlocks(m, m) + return m, m, nil + } + if l%aes.BlockSize == 0 { + mode.CryptBlocks(m, m) + iv = m[len(m)-aes.BlockSize:] + rb, _ := swapLastTwoBlocks(m, aes.BlockSize) + return iv, rb, nil + } + m, _ = zeroPad(m, aes.BlockSize) + rb, pb, lb, err := tailBlocks(m, aes.BlockSize) + if err != nil { + return []byte{}, []byte{}, fmt.Errorf("error tailing blocks: %v", err) + } + var ct []byte + if rb != nil { + // Encrpt all but the lats 2 blocks and update the rolling iv + mode.CryptBlocks(rb, rb) + iv = rb[len(rb)-aes.BlockSize:] + mode = cipher.NewCBCEncrypter(block, iv) + ct = append(ct, rb...) + } + mode.CryptBlocks(pb, pb) + mode = cipher.NewCBCEncrypter(block, pb) + mode.CryptBlocks(lb, lb) + // Cipher Text Stealing (CTS) - Ref: https://en.wikipedia.org/wiki/Ciphertext_stealing#CBC_ciphertext_stealing + // Swap the last two cipher blocks + // Truncate the ciphertext to the length of the original plaintext + ct = append(ct, lb...) + ct = append(ct, pb...) + return lb, ct[:l], nil +} + +// Decrypt the ciphertext with the key and the initial vector. +func Decrypt(key, iv, ciphertext []byte) ([]byte, error) { + // Copy the cipher text as golang slices even when passed by value to this method can result in the backing arrays of the calling code value being updated. + ct := make([]byte, len(ciphertext)) + copy(ct, ciphertext) + if len(ct) < aes.BlockSize { + return []byte{}, fmt.Errorf("ciphertext is not large enough. It is less that one block size. Blocksize:%v; Ciphertext:%v", aes.BlockSize, len(ct)) + } + // Configure the CBC + block, err := aes.NewCipher(key) + if err != nil { + return nil, fmt.Errorf("error creating cipher: %v", err) + } + var mode cipher.BlockMode + + //If ciphertext is multiple of blocksize we just need to swap back the last two blocks and then do CBC + //If the ciphertext is just one block we can't swap so we just decrypt + if len(ct)%aes.BlockSize == 0 { + if len(ct) > aes.BlockSize { + ct, _ = swapLastTwoBlocks(ct, aes.BlockSize) + } + mode = cipher.NewCBCDecrypter(block, iv) + message := make([]byte, len(ct)) + mode.CryptBlocks(message, ct) + return message[:len(ct)], nil + } + + // Cipher Text Stealing (CTS) using CBC interface. Ref: https://en.wikipedia.org/wiki/Ciphertext_stealing#CBC_ciphertext_stealing + // Get ciphertext of the 2nd to last (penultimate) block (cpb), the last block (clb) and the rest (crb) + crb, cpb, clb, _ := tailBlocks(ct, aes.BlockSize) + v := make([]byte, len(iv), len(iv)) + copy(v, iv) + var message []byte + if crb != nil { + //If there is more than just the last and the penultimate block we decrypt it and the last bloc of this becomes the iv for later + rb := make([]byte, len(crb)) + mode = cipher.NewCBCDecrypter(block, v) + v = crb[len(crb)-aes.BlockSize:] + mode.CryptBlocks(rb, crb) + message = append(message, rb...) + } + + // We need to modify the cipher text + // Decryt the 2nd to last (penultimate) block with a the original iv + pb := make([]byte, aes.BlockSize) + mode = cipher.NewCBCDecrypter(block, iv) + mode.CryptBlocks(pb, cpb) + // number of byte needed to pad + npb := aes.BlockSize - len(ct)%aes.BlockSize + //pad last block using the number of bytes needed from the tail of the plaintext 2nd to last (penultimate) block + clb = append(clb, pb[len(pb)-npb:]...) + + // Now decrypt the last block in the penultimate position (iv will be from the crb, if the is no crb it's zeros) + // iv for the penultimate block decrypted in the last position becomes the modified last block + lb := make([]byte, aes.BlockSize) + mode = cipher.NewCBCDecrypter(block, v) + v = clb + mode.CryptBlocks(lb, clb) + message = append(message, lb...) + + // Now decrypt the penultimate block in the last position (iv will be from the modified last block) + mode = cipher.NewCBCDecrypter(block, v) + mode.CryptBlocks(cpb, cpb) + message = append(message, cpb...) + + // Truncate to the size of the original cipher text + return message[:len(ct)], nil +} + +func tailBlocks(b []byte, c int) ([]byte, []byte, []byte, error) { + if len(b) <= c { + return []byte{}, []byte{}, []byte{}, errors.New("bytes slice is not larger than one block so cannot tail") + } + // Get size of last block + var lbs int + if l := len(b) % aes.BlockSize; l == 0 { + lbs = aes.BlockSize + } else { + lbs = l + } + // Get last block + lb := b[len(b)-lbs:] + // Get 2nd to last (penultimate) block + pb := b[len(b)-lbs-c : len(b)-lbs] + if len(b) > 2*c { + rb := b[:len(b)-lbs-c] + return rb, pb, lb, nil + } + return nil, pb, lb, nil +} + +func swapLastTwoBlocks(b []byte, c int) ([]byte, error) { + rb, pb, lb, err := tailBlocks(b, c) + if err != nil { + return nil, err + } + var out []byte + if rb != nil { + out = append(out, rb...) + } + out = append(out, lb...) + out = append(out, pb...) + return out, nil +} + +// zeroPad pads bytes with zeros to nearest multiple of message size m. +func zeroPad(b []byte, m int) ([]byte, error) { + if m <= 0 { + return nil, errors.New("invalid message block size when padding") + } + if b == nil || len(b) == 0 { + return nil, errors.New("data not valid to pad: Zero size") + } + if l := len(b) % m; l != 0 { + n := m - l + z := make([]byte, n) + b = append(b, z...) + } + return b, nil +} diff --git a/github.com/jcmturner/aescts/v2/go.mod b/github.com/jcmturner/aescts/v2/go.mod new file mode 100644 index 0000000000..034c3cec60 --- /dev/null +++ b/github.com/jcmturner/aescts/v2/go.mod @@ -0,0 +1,5 @@ +module github.com/jcmturner/aescts/v2 + +go 1.13 + +require github.com/stretchr/testify v1.4.0 diff --git a/github.com/jcmturner/aescts/v2/go.sum b/github.com/jcmturner/aescts/v2/go.sum new file mode 100644 index 0000000000..e863f517e1 --- /dev/null +++ b/github.com/jcmturner/aescts/v2/go.sum @@ -0,0 +1,10 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/github.com/jcmturner/dnsutils/v2/LICENSE b/github.com/jcmturner/dnsutils/v2/LICENSE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/github.com/jcmturner/dnsutils/v2/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/github.com/jcmturner/dnsutils/v2/go.mod b/github.com/jcmturner/dnsutils/v2/go.mod new file mode 100644 index 0000000000..f75ac6d264 --- /dev/null +++ b/github.com/jcmturner/dnsutils/v2/go.mod @@ -0,0 +1,5 @@ +module github.com/jcmturner/dnsutils/v2 + +go 1.13 + +require github.com/stretchr/testify v1.4.0 diff --git a/github.com/jcmturner/dnsutils/v2/go.sum b/github.com/jcmturner/dnsutils/v2/go.sum new file mode 100644 index 0000000000..e863f517e1 --- /dev/null +++ b/github.com/jcmturner/dnsutils/v2/go.sum @@ -0,0 +1,10 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/github.com/jcmturner/dnsutils/v2/srv.go b/github.com/jcmturner/dnsutils/v2/srv.go new file mode 100644 index 0000000000..15ea912d10 --- /dev/null +++ b/github.com/jcmturner/dnsutils/v2/srv.go @@ -0,0 +1,95 @@ +package dnsutils + +import ( + "math/rand" + "net" + "sort" +) + +// OrderedSRV returns a count of the results and a map keyed on the order they should be used. +// This based on the records' priority and randomised selection based on their relative weighting. +// The function's inputs are the same as those for net.LookupSRV +// To use in the correct order: +// +// count, orderedSRV, err := OrderedSRV(service, proto, name) +// i := 1 +// for i <= count { +// srv := orderedSRV[i] +// // Do something such as dial this SRV. If fails move on the the next or break if it succeeds. +// i += 1 +// } +func OrderedSRV(service, proto, name string) (int, map[int]*net.SRV, error) { + _, addrs, err := net.LookupSRV(service, proto, name) + if err != nil { + return 0, make(map[int]*net.SRV), err + } + index, osrv := orderSRV(addrs) + return index, osrv, nil +} + +func orderSRV(addrs []*net.SRV) (int, map[int]*net.SRV) { + // Initialise the ordered map + var o int + osrv := make(map[int]*net.SRV) + + prioMap := make(map[int][]*net.SRV, 0) + for _, srv := range addrs { + prioMap[int(srv.Priority)] = append(prioMap[int(srv.Priority)], srv) + } + + priorities := make([]int, 0) + for p := range prioMap { + priorities = append(priorities, p) + } + + var count int + sort.Ints(priorities) + for _, p := range priorities { + tos := weightedOrder(prioMap[p]) + for i, s := range tos { + count += 1 + osrv[o+i] = s + } + o += len(tos) + } + return count, osrv +} + +func weightedOrder(srvs []*net.SRV) map[int]*net.SRV { + // Get the total weight + var tw int + for _, s := range srvs { + tw += int(s.Weight) + } + + // Initialise the ordered map + o := 1 + osrv := make(map[int]*net.SRV) + + // Whilst there are still entries to be ordered + l := len(srvs) + for l > 0 { + i := rand.Intn(l) + s := srvs[i] + var rw int + if tw > 0 { + // Greater the weight the more likely this will be zero or less + rw = rand.Intn(tw) - int(s.Weight) + } + if rw <= 0 { + // Put entry in position + osrv[o] = s + if len(srvs) > 1 { + // Remove the entry from the source slice by swapping with the last entry and truncating + srvs[len(srvs)-1], srvs[i] = srvs[i], srvs[len(srvs)-1] + srvs = srvs[:len(srvs)-1] + l = len(srvs) + } else { + l = 0 + } + o += 1 + tw = tw - int(s.Weight) + } + } + return osrv +} diff --git a/github.com/jcmturner/goidentity/v6/LICENSE b/github.com/jcmturner/goidentity/v6/LICENSE new file mode 100644 index 0000000000..8dada3edaf --- /dev/null +++ b/github.com/jcmturner/goidentity/v6/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/github.com/jcmturner/goidentity/v6/README.md b/github.com/jcmturner/goidentity/v6/README.md new file mode 100644 index 0000000000..3e15042eac --- /dev/null +++ b/github.com/jcmturner/goidentity/v6/README.md @@ -0,0 +1,7 @@ +# goidentity +[![GoDoc](https://godoc.org/github.com/jcmturner/goidentity/v6?status.svg)](https://godoc.org/github.com/jcmturner/goidentity/v6) [![Go Report Card](https://goreportcard.com/badge/github.com/jcmturner/goidentity/v6)](https://goreportcard.com/report/github.com/jcmturner/goidentity/v6) + +Please import as below +``` +import "github.com/jcmturner/goidentity/v6" +``` diff --git a/github.com/jcmturner/goidentity/v6/authenticator.go b/github.com/jcmturner/goidentity/v6/authenticator.go new file mode 100644 index 0000000000..42ec79b061 --- /dev/null +++ b/github.com/jcmturner/goidentity/v6/authenticator.go @@ -0,0 +1,6 @@ +package goidentity + +type Authenticator interface { + Authenticate() (Identity, bool, error) + Mechanism() string // gives the name of the type of authentication mechanism +} diff --git a/github.com/jcmturner/goidentity/v6/go.mod b/github.com/jcmturner/goidentity/v6/go.mod new file mode 100644 index 0000000000..73cb36b60c --- /dev/null +++ b/github.com/jcmturner/goidentity/v6/go.mod @@ -0,0 +1,8 @@ +module github.com/jcmturner/goidentity/v6 + +go 1.13 + +require ( + github.com/hashicorp/go-uuid v1.0.2 + github.com/stretchr/testify v1.4.0 +) diff --git a/github.com/jcmturner/goidentity/v6/go.sum b/github.com/jcmturner/goidentity/v6/go.sum new file mode 100644 index 0000000000..92979e4a96 --- /dev/null +++ b/github.com/jcmturner/goidentity/v6/go.sum @@ -0,0 +1,12 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/github.com/jcmturner/goidentity/v6/identity.go b/github.com/jcmturner/goidentity/v6/identity.go new file mode 100644 index 0000000000..f55d3b4e0a --- /dev/null +++ b/github.com/jcmturner/goidentity/v6/identity.go @@ -0,0 +1,52 @@ +package goidentity + +import ( + "context" + "net/http" + "time" +) + +const ( + CTXKey = "jcmturner/goidentity" +) + +type Identity interface { + UserName() string + SetUserName(s string) + Domain() string + SetDomain(s string) + DisplayName() string + SetDisplayName(s string) + Human() bool + SetHuman(b bool) + AuthTime() time.Time + SetAuthTime(t time.Time) + AuthzAttributes() []string + AddAuthzAttribute(a string) + RemoveAuthzAttribute(a string) + Authenticated() bool + SetAuthenticated(b bool) + Authorized(a string) bool + SessionID() string + Expired() bool + Attributes() map[string]interface{} + SetAttribute(k string, v interface{}) + SetAttributes(map[string]interface{}) + RemoveAttribute(k string) + Marshal() ([]byte, error) + Unmarshal([]byte) error +} + +func AddToHTTPRequestContext(id Identity, r *http.Request) *http.Request { + ctx := r.Context() + ctx = context.WithValue(ctx, CTXKey, id) + return r.WithContext(ctx) +} + +func FromHTTPRequestContext(r *http.Request) Identity { + ctx := r.Context() + if id, ok := ctx.Value(CTXKey).(Identity); ok { + return id + } + return nil +} diff --git a/github.com/jcmturner/goidentity/v6/user.go b/github.com/jcmturner/goidentity/v6/user.go new file mode 100644 index 0000000000..88a73889bf --- /dev/null +++ b/github.com/jcmturner/goidentity/v6/user.go @@ -0,0 +1,172 @@ +package goidentity + +import ( + "bytes" + "encoding/gob" + "github.com/hashicorp/go-uuid" + "time" +) + +type User struct { + authenticated bool + domain string + userName string + displayName string + email string + human bool + groupMembership map[string]bool + authTime time.Time + sessionID string + expiry time.Time + attributes map[string]interface{} +} + +func NewUser(username string) User { + uuid, err := uuid.GenerateUUID() + if err != nil { + uuid = "00unique-sess-ions-uuid-unavailable0" + } + return User{ + userName: username, + groupMembership: make(map[string]bool), + sessionID: uuid, + } +} + +func (u *User) UserName() string { + return u.userName +} + +func (u *User) SetUserName(s string) { + u.userName = s +} + +func (u *User) Domain() string { + return u.domain +} + +func (u *User) SetDomain(s string) { + u.domain = s +} + +func (u *User) DisplayName() string { + if u.displayName == "" { + return u.userName + } + return u.displayName +} + +func (u *User) SetDisplayName(s string) { + u.displayName = s +} + +func (u *User) Human() bool { + return u.human +} + +func (u *User) SetHuman(b bool) { + u.human = b +} + +func (u *User) AuthTime() time.Time { + return u.authTime +} + +func (u *User) SetAuthTime(t time.Time) { + u.authTime = t +} + +func (u *User) AuthzAttributes() []string { + s := make([]string, len(u.groupMembership)) + i := 0 + for a := range u.groupMembership { + s[i] = a + i++ + } + return s +} + +func (u *User) Authenticated() bool { + return u.authenticated +} + +func (u *User) SetAuthenticated(b bool) { + u.authenticated = b +} + +func (u *User) AddAuthzAttribute(a string) { + u.groupMembership[a] = true +} + +func (u *User) RemoveAuthzAttribute(a string) { + if _, ok := u.groupMembership[a]; !ok { + return + } + delete(u.groupMembership, a) +} + +func (u *User) EnableAuthzAttribute(a string) { + if enabled, ok := u.groupMembership[a]; ok && !enabled { + u.groupMembership[a] = true + } +} + +func (u *User) DisableAuthzAttribute(a string) { + if enabled, ok := u.groupMembership[a]; ok && enabled { + u.groupMembership[a] = false + } +} + +func (u *User) Authorized(a string) bool { + if enabled, ok := u.groupMembership[a]; ok && enabled { + return true + } + return false +} + +func (u *User) SessionID() string { + return u.sessionID +} + +func (u *User) SetExpiry(t time.Time) { + u.expiry = t +} + +func (u *User) Expired() bool { + if !u.expiry.IsZero() && time.Now().UTC().After(u.expiry) { + return true + } + return false +} + +func (u *User) Attributes() map[string]interface{} { + return u.attributes +} + +func (u *User) SetAttribute(k string, v interface{}) { + u.attributes[k] = v +} + +func (u *User) SetAttributes(a map[string]interface{}) { + u.attributes = a +} + +func (u *User) RemoveAttribute(k string) { + delete(u.attributes, k) +} + +func (u *User) Marshal() ([]byte, error) { + buf := new(bytes.Buffer) + enc := gob.NewEncoder(buf) + err := enc.Encode(u) + if err != nil { + return []byte{}, err + } + return buf.Bytes(), nil +} + +func (u *User) Unmarshal(b []byte) error { + buf := bytes.NewBuffer(b) + dec := gob.NewDecoder(buf) + return dec.Decode(u) +} diff --git a/github.com/jcmturner/gokrb5/v8/LICENSE b/github.com/jcmturner/gokrb5/v8/LICENSE new file mode 100644 index 0000000000..8dada3edaf --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/github.com/jcmturner/gokrb5/v8/asn1tools/tools.go b/github.com/jcmturner/gokrb5/v8/asn1tools/tools.go new file mode 100644 index 0000000000..f27740b9bd --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/asn1tools/tools.go @@ -0,0 +1,86 @@ +// Package asn1tools provides tools for managing ASN1 marshaled data. +package asn1tools + +import ( + "github.com/jcmturner/gofork/encoding/asn1" +) + +// MarshalLengthBytes returns the ASN1 encoded bytes for the length 'l' +// +// There are two forms: short (for lengths between 0 and 127), and long definite (for lengths between 0 and 2^1008 -1). +// +// Short form: One octet. Bit 8 has value "0" and bits 7-1 give the length. +// +// Long form: Two to 127 octets. Bit 8 of first octet has value "1" and bits 7-1 give the number of additional length octets. Second and following octets give the length, base 256, most significant digit first. +func MarshalLengthBytes(l int) []byte { + if l <= 127 { + return []byte{byte(l)} + } + var b []byte + p := 1 + for i := 1; i < 127; { + b = append([]byte{byte((l % (p * 256)) / p)}, b...) + p = p * 256 + l = l - l%p + if l <= 0 { + break + } + } + return append([]byte{byte(128 + len(b))}, b...) +} + +// GetLengthFromASN returns the length of a slice of ASN1 encoded bytes from the ASN1 length header it contains. +func GetLengthFromASN(b []byte) int { + if int(b[1]) <= 127 { + return int(b[1]) + } + // The bytes that indicate the length + lb := b[2 : 2+int(b[1])-128] + base := 1 + l := 0 + for i := len(lb) - 1; i >= 0; i-- { + l += int(lb[i]) * base + base = base * 256 + } + return l +} + +// GetNumberBytesInLengthHeader returns the number of bytes in the ASn1 header that indicate the length. +func GetNumberBytesInLengthHeader(b []byte) int { + if int(b[1]) <= 127 { + return 1 + } + // The bytes that indicate the length + return 1 + int(b[1]) - 128 +} + +// AddASNAppTag adds an ASN1 encoding application tag value to the raw bytes provided. +func AddASNAppTag(b []byte, tag int) []byte { + r := asn1.RawValue{ + Class: asn1.ClassApplication, + IsCompound: true, + Tag: tag, + Bytes: b, + } + ab, _ := asn1.Marshal(r) + return ab +} + +/* +// The Marshal method of golang's asn1 package does not enable you to define wrapping the output in an application tag. +// This method adds that wrapping tag. +func AddASNAppTag(b []byte, tag int) []byte { + // The ASN1 wrapping consists of 2 bytes: + // 1st byte -> Identifier Octet - Application Tag + // 2nd byte -> The length (this will be the size indicated in the input bytes + 2 for the additional bytes we add here. + // Application Tag: + //| Bit: | 8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | + //| Value: | 0 | 1 | 1 | From the RFC spec 4120 | + //| Explanation | Defined by the ASN1 encoding rules for an application tag | A value of 1 indicates a constructed type | The ASN Application tag value | + // Therefore the value of the byte is an integer = ( Application tag value + 96 ) + //b = append(MarshalLengthBytes(int(b[1])+2), b...) + b = append(MarshalLengthBytes(len(b)), b...) + b = append([]byte{byte(96 + tag)}, b...) + return b +} +*/ diff --git a/github.com/jcmturner/gokrb5/v8/client/ASExchange.go b/github.com/jcmturner/gokrb5/v8/client/ASExchange.go new file mode 100644 index 0000000000..5becccc4dd --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/client/ASExchange.go @@ -0,0 +1,182 @@ +package client + +import ( + "github.com/jcmturner/gokrb5/v8/crypto" + "github.com/jcmturner/gokrb5/v8/crypto/etype" + "github.com/jcmturner/gokrb5/v8/iana/errorcode" + "github.com/jcmturner/gokrb5/v8/iana/keyusage" + "github.com/jcmturner/gokrb5/v8/iana/patype" + "github.com/jcmturner/gokrb5/v8/krberror" + "github.com/jcmturner/gokrb5/v8/messages" + "github.com/jcmturner/gokrb5/v8/types" +) + +// ASExchange performs an AS exchange for the client to retrieve a TGT. +func (cl *Client) ASExchange(realm string, ASReq messages.ASReq, referral int) (messages.ASRep, error) { + if ok, err := cl.IsConfigured(); !ok { + return messages.ASRep{}, krberror.Errorf(err, krberror.ConfigError, "AS Exchange cannot be performed") + } + + // Set PAData if required + err := setPAData(cl, nil, &ASReq) + if err != nil { + return messages.ASRep{}, krberror.Errorf(err, krberror.KRBMsgError, "AS Exchange Error: issue with setting PAData on AS_REQ") + } + + b, err := ASReq.Marshal() + if err != nil { + return messages.ASRep{}, krberror.Errorf(err, krberror.EncodingError, "AS Exchange Error: failed marshaling AS_REQ") + } + var ASRep messages.ASRep + + rb, err := cl.sendToKDC(b, realm) + if err != nil { + if e, ok := err.(messages.KRBError); ok { + switch e.ErrorCode { + case errorcode.KDC_ERR_PREAUTH_REQUIRED, errorcode.KDC_ERR_PREAUTH_FAILED: + // From now on assume this client will need to do this pre-auth and set the PAData + cl.settings.assumePreAuthentication = true + err = setPAData(cl, &e, &ASReq) + if err != nil { + return messages.ASRep{}, krberror.Errorf(err, krberror.KRBMsgError, "AS Exchange Error: failed setting AS_REQ PAData for pre-authentication required") + } + b, err := ASReq.Marshal() + if err != nil { + return messages.ASRep{}, krberror.Errorf(err, krberror.EncodingError, "AS Exchange Error: failed marshaling AS_REQ with PAData") + } + rb, err = cl.sendToKDC(b, realm) + if err != nil { + if _, ok := err.(messages.KRBError); ok { + return messages.ASRep{}, krberror.Errorf(err, krberror.KDCError, "AS Exchange Error: kerberos error response from KDC") + } + return messages.ASRep{}, krberror.Errorf(err, krberror.NetworkingError, "AS Exchange Error: failed sending AS_REQ to KDC") + } + case errorcode.KDC_ERR_WRONG_REALM: + // Client referral https://tools.ietf.org/html/rfc6806.html#section-7 + if referral > 5 { + return messages.ASRep{}, krberror.Errorf(err, krberror.KRBMsgError, "maximum number of client referrals exceeded") + } + referral++ + return cl.ASExchange(e.CRealm, ASReq, referral) + default: + return messages.ASRep{}, krberror.Errorf(err, krberror.KDCError, "AS Exchange Error: kerberos error response from KDC") + } + } else { + return messages.ASRep{}, krberror.Errorf(err, krberror.NetworkingError, "AS Exchange Error: failed sending AS_REQ to KDC") + } + } + err = ASRep.Unmarshal(rb) + if err != nil { + return messages.ASRep{}, krberror.Errorf(err, krberror.EncodingError, "AS Exchange Error: failed to process the AS_REP") + } + if ok, err := ASRep.Verify(cl.Config, cl.Credentials, ASReq); !ok { + return messages.ASRep{}, krberror.Errorf(err, krberror.KRBMsgError, "AS Exchange Error: AS_REP is not valid or client password/keytab incorrect") + } + return ASRep, nil +} + +// setPAData adds pre-authentication data to the AS_REQ. +func setPAData(cl *Client, krberr *messages.KRBError, ASReq *messages.ASReq) error { + if !cl.settings.DisablePAFXFAST() { + pa := types.PAData{PADataType: patype.PA_REQ_ENC_PA_REP} + ASReq.PAData = append(ASReq.PAData, pa) + } + if cl.settings.AssumePreAuthentication() { + // Identify the etype to use to encrypt the PA Data + var et etype.EType + var err error + var key types.EncryptionKey + var kvno int + if krberr == nil { + // This is not in response to an error from the KDC. It is preemptive or renewal + // There is no KRB Error that tells us the etype to use + etn := cl.settings.preAuthEType // Use the etype that may have previously been negotiated + if etn == 0 { + etn = int32(cl.Config.LibDefaults.PreferredPreauthTypes[0]) // Resort to config + } + et, err = crypto.GetEtype(etn) + if err != nil { + return krberror.Errorf(err, krberror.EncryptingError, "error getting etype for pre-auth encryption") + } + key, kvno, err = cl.Key(et, 0, nil) + if err != nil { + return krberror.Errorf(err, krberror.EncryptingError, "error getting key from credentials") + } + } else { + // Get the etype to use from the PA data in the KRBError e-data + et, err = preAuthEType(krberr) + if err != nil { + return krberror.Errorf(err, krberror.EncryptingError, "error getting etype for pre-auth encryption") + } + cl.settings.preAuthEType = et.GetETypeID() // Set the etype that has been defined for potential future use + key, kvno, err = cl.Key(et, 0, krberr) + if err != nil { + return krberror.Errorf(err, krberror.EncryptingError, "error getting key from credentials") + } + } + // Generate the PA data + paTSb, err := types.GetPAEncTSEncAsnMarshalled() + if err != nil { + return krberror.Errorf(err, krberror.KRBMsgError, "error creating PAEncTSEnc for Pre-Authentication") + } + paEncTS, err := crypto.GetEncryptedData(paTSb, key, keyusage.AS_REQ_PA_ENC_TIMESTAMP, kvno) + if err != nil { + return krberror.Errorf(err, krberror.EncryptingError, "error encrypting pre-authentication timestamp") + } + pb, err := paEncTS.Marshal() + if err != nil { + return krberror.Errorf(err, krberror.EncodingError, "error marshaling the PAEncTSEnc encrypted data") + } + pa := types.PAData{ + PADataType: patype.PA_ENC_TIMESTAMP, + PADataValue: pb, + } + // Look for and delete any exiting patype.PA_ENC_TIMESTAMP + for i, pa := range ASReq.PAData { + if pa.PADataType == patype.PA_ENC_TIMESTAMP { + ASReq.PAData[i] = ASReq.PAData[len(ASReq.PAData)-1] + ASReq.PAData = ASReq.PAData[:len(ASReq.PAData)-1] + } + } + ASReq.PAData = append(ASReq.PAData, pa) + } + return nil +} + +// preAuthEType establishes what encryption type to use for pre-authentication from the KRBError returned from the KDC. +func preAuthEType(krberr *messages.KRBError) (etype etype.EType, err error) { + //RFC 4120 5.2.7.5 covers the preference order of ETYPE-INFO2 and ETYPE-INFO. + var etypeID int32 + var pas types.PADataSequence + e := pas.Unmarshal(krberr.EData) + if e != nil { + err = krberror.Errorf(e, krberror.EncodingError, "error unmashalling KRBError data") + return + } +Loop: + for _, pa := range pas { + switch pa.PADataType { + case patype.PA_ETYPE_INFO2: + info, e := pa.GetETypeInfo2() + if e != nil { + err = krberror.Errorf(e, krberror.EncodingError, "error unmashalling ETYPE-INFO2 data") + return + } + etypeID = info[0].EType + break Loop + case patype.PA_ETYPE_INFO: + info, e := pa.GetETypeInfo() + if e != nil { + err = krberror.Errorf(e, krberror.EncodingError, "error unmashalling ETYPE-INFO data") + return + } + etypeID = info[0].EType + } + } + etype, e = crypto.GetEtype(etypeID) + if e != nil { + err = krberror.Errorf(e, krberror.EncryptingError, "error creating etype") + return + } + return etype, nil +} diff --git a/github.com/jcmturner/gokrb5/v8/client/TGSExchange.go b/github.com/jcmturner/gokrb5/v8/client/TGSExchange.go new file mode 100644 index 0000000000..e4571ce866 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/client/TGSExchange.go @@ -0,0 +1,103 @@ +package client + +import ( + "github.com/jcmturner/gokrb5/v8/iana/flags" + "github.com/jcmturner/gokrb5/v8/iana/nametype" + "github.com/jcmturner/gokrb5/v8/krberror" + "github.com/jcmturner/gokrb5/v8/messages" + "github.com/jcmturner/gokrb5/v8/types" +) + +// TGSREQGenerateAndExchange generates the TGS_REQ and performs a TGS exchange to retrieve a ticket to the specified SPN. +func (cl *Client) TGSREQGenerateAndExchange(spn types.PrincipalName, kdcRealm string, tgt messages.Ticket, sessionKey types.EncryptionKey, renewal bool) (tgsReq messages.TGSReq, tgsRep messages.TGSRep, err error) { + tgsReq, err = messages.NewTGSReq(cl.Credentials.CName(), kdcRealm, cl.Config, tgt, sessionKey, spn, renewal) + if err != nil { + return tgsReq, tgsRep, krberror.Errorf(err, krberror.KRBMsgError, "TGS Exchange Error: failed to generate a new TGS_REQ") + } + return cl.TGSExchange(tgsReq, kdcRealm, tgsRep.Ticket, sessionKey, 0) +} + +// TGSExchange exchanges the provided TGS_REQ with the KDC to retrieve a TGS_REP. +// Referrals are automatically handled. +// The client's cache is updated with the ticket received. +func (cl *Client) TGSExchange(tgsReq messages.TGSReq, kdcRealm string, tgt messages.Ticket, sessionKey types.EncryptionKey, referral int) (messages.TGSReq, messages.TGSRep, error) { + var tgsRep messages.TGSRep + b, err := tgsReq.Marshal() + if err != nil { + return tgsReq, tgsRep, krberror.Errorf(err, krberror.EncodingError, "TGS Exchange Error: failed to marshal TGS_REQ") + } + r, err := cl.sendToKDC(b, kdcRealm) + if err != nil { + if _, ok := err.(messages.KRBError); ok { + return tgsReq, tgsRep, krberror.Errorf(err, krberror.KDCError, "TGS Exchange Error: kerberos error response from KDC when requesting for %s", tgsReq.ReqBody.SName.PrincipalNameString()) + } + return tgsReq, tgsRep, krberror.Errorf(err, krberror.NetworkingError, "TGS Exchange Error: issue sending TGS_REQ to KDC") + } + err = tgsRep.Unmarshal(r) + if err != nil { + return tgsReq, tgsRep, krberror.Errorf(err, krberror.EncodingError, "TGS Exchange Error: failed to process the TGS_REP") + } + err = tgsRep.DecryptEncPart(sessionKey) + if err != nil { + return tgsReq, tgsRep, krberror.Errorf(err, krberror.EncodingError, "TGS Exchange Error: failed to process the TGS_REP") + } + if ok, err := tgsRep.Verify(cl.Config, tgsReq); !ok { + return tgsReq, tgsRep, krberror.Errorf(err, krberror.EncodingError, "TGS Exchange Error: TGS_REP is not valid") + } + + if tgsRep.Ticket.SName.NameString[0] == "krbtgt" && !tgsRep.Ticket.SName.Equal(tgsReq.ReqBody.SName) { + if referral > 5 { + return tgsReq, tgsRep, krberror.Errorf(err, krberror.KRBMsgError, "TGS Exchange Error: maximum number of referrals exceeded") + } + // Server referral https://tools.ietf.org/html/rfc6806.html#section-8 + // The TGS Rep contains a TGT for another domain as the service resides in that domain. + cl.addSession(tgsRep.Ticket, tgsRep.DecryptedEncPart) + realm := tgsRep.Ticket.SName.NameString[len(tgsRep.Ticket.SName.NameString)-1] + referral++ + if types.IsFlagSet(&tgsReq.ReqBody.KDCOptions, flags.EncTktInSkey) && len(tgsReq.ReqBody.AdditionalTickets) > 0 { + tgsReq, err = messages.NewUser2UserTGSReq(cl.Credentials.CName(), kdcRealm, cl.Config, tgt, sessionKey, tgsReq.ReqBody.SName, tgsReq.Renewal, tgsReq.ReqBody.AdditionalTickets[0]) + if err != nil { + return tgsReq, tgsRep, err + } + } + tgsReq, err = messages.NewTGSReq(cl.Credentials.CName(), realm, cl.Config, tgsRep.Ticket, tgsRep.DecryptedEncPart.Key, tgsReq.ReqBody.SName, tgsReq.Renewal) + if err != nil { + return tgsReq, tgsRep, err + } + return cl.TGSExchange(tgsReq, realm, tgsRep.Ticket, tgsRep.DecryptedEncPart.Key, referral) + } + cl.cache.addEntry( + tgsRep.Ticket, + tgsRep.DecryptedEncPart.AuthTime, + tgsRep.DecryptedEncPart.StartTime, + tgsRep.DecryptedEncPart.EndTime, + tgsRep.DecryptedEncPart.RenewTill, + tgsRep.DecryptedEncPart.Key, + ) + cl.Log("ticket added to cache for %s (EndTime: %v)", tgsRep.Ticket.SName.PrincipalNameString(), tgsRep.DecryptedEncPart.EndTime) + return tgsReq, tgsRep, err +} + +// GetServiceTicket makes a request to get a service ticket for the SPN specified +// SPN format: / Eg. HTTP/www.example.com +// The ticket will be added to the client's ticket cache +func (cl *Client) GetServiceTicket(spn string) (messages.Ticket, types.EncryptionKey, error) { + var tkt messages.Ticket + var skey types.EncryptionKey + if tkt, skey, ok := cl.GetCachedTicket(spn); ok { + // Already a valid ticket in the cache + return tkt, skey, nil + } + princ := types.NewPrincipalName(nametype.KRB_NT_PRINCIPAL, spn) + realm := cl.Config.ResolveRealm(princ.NameString[len(princ.NameString)-1]) + + tgt, skey, err := cl.sessionTGT(realm) + if err != nil { + return tkt, skey, err + } + _, tgsRep, err := cl.TGSREQGenerateAndExchange(princ, realm, tgt, skey, false) + if err != nil { + return tkt, skey, err + } + return tgsRep.Ticket, tgsRep.DecryptedEncPart.Key, nil +} diff --git a/github.com/jcmturner/gokrb5/v8/client/cache.go b/github.com/jcmturner/gokrb5/v8/client/cache.go new file mode 100644 index 0000000000..552e73e41e --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/client/cache.go @@ -0,0 +1,134 @@ +package client + +import ( + "encoding/json" + "errors" + "sort" + "sync" + "time" + + "github.com/jcmturner/gokrb5/v8/messages" + "github.com/jcmturner/gokrb5/v8/types" +) + +// Cache for service tickets held by the client. +type Cache struct { + Entries map[string]CacheEntry + mux sync.RWMutex +} + +// CacheEntry holds details for a cache entry. +type CacheEntry struct { + SPN string + Ticket messages.Ticket `json:"-"` + AuthTime time.Time + StartTime time.Time + EndTime time.Time + RenewTill time.Time + SessionKey types.EncryptionKey `json:"-"` +} + +// NewCache creates a new client ticket cache instance. +func NewCache() *Cache { + return &Cache{ + Entries: map[string]CacheEntry{}, + } +} + +// getEntry returns a cache entry that matches the SPN. +func (c *Cache) getEntry(spn string) (CacheEntry, bool) { + c.mux.RLock() + defer c.mux.RUnlock() + e, ok := (*c).Entries[spn] + return e, ok +} + +// JSON returns information about the cached service tickets in a JSON format. +func (c *Cache) JSON() (string, error) { + c.mux.RLock() + defer c.mux.RUnlock() + var es []CacheEntry + keys := make([]string, 0, len(c.Entries)) + for k := range c.Entries { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + es = append(es, c.Entries[k]) + } + b, err := json.MarshalIndent(&es, "", " ") + if err != nil { + return "", err + } + return string(b), nil +} + +// addEntry adds a ticket to the cache. +func (c *Cache) addEntry(tkt messages.Ticket, authTime, startTime, endTime, renewTill time.Time, sessionKey types.EncryptionKey) CacheEntry { + spn := tkt.SName.PrincipalNameString() + c.mux.Lock() + defer c.mux.Unlock() + (*c).Entries[spn] = CacheEntry{ + SPN: spn, + Ticket: tkt, + AuthTime: authTime, + StartTime: startTime, + EndTime: endTime, + RenewTill: renewTill, + SessionKey: sessionKey, + } + return c.Entries[spn] +} + +// clear deletes all the cache entries +func (c *Cache) clear() { + c.mux.Lock() + defer c.mux.Unlock() + for k := range c.Entries { + delete(c.Entries, k) + } +} + +// RemoveEntry removes the cache entry for the defined SPN. +func (c *Cache) RemoveEntry(spn string) { + c.mux.Lock() + defer c.mux.Unlock() + delete(c.Entries, spn) +} + +// GetCachedTicket returns a ticket from the cache for the SPN. +// Only a ticket that is currently valid will be returned. +func (cl *Client) GetCachedTicket(spn string) (messages.Ticket, types.EncryptionKey, bool) { + if e, ok := cl.cache.getEntry(spn); ok { + //If within time window of ticket return it + if time.Now().UTC().After(e.StartTime) && time.Now().UTC().Before(e.EndTime) { + cl.Log("ticket received from cache for %s", spn) + return e.Ticket, e.SessionKey, true + } else if time.Now().UTC().Before(e.RenewTill) { + e, err := cl.renewTicket(e) + if err != nil { + return e.Ticket, e.SessionKey, false + } + return e.Ticket, e.SessionKey, true + } + } + var tkt messages.Ticket + var key types.EncryptionKey + return tkt, key, false +} + +// renewTicket renews a cache entry ticket. +// To renew from outside the client package use GetCachedTicket +func (cl *Client) renewTicket(e CacheEntry) (CacheEntry, error) { + spn := e.Ticket.SName + _, _, err := cl.TGSREQGenerateAndExchange(spn, e.Ticket.Realm, e.Ticket, e.SessionKey, true) + if err != nil { + return e, err + } + e, ok := cl.cache.getEntry(e.Ticket.SName.PrincipalNameString()) + if !ok { + return e, errors.New("ticket was not added to cache") + } + cl.Log("ticket renewed for %s (EndTime: %v)", spn.PrincipalNameString(), e.EndTime) + return e, nil +} diff --git a/github.com/jcmturner/gokrb5/v8/client/client.go b/github.com/jcmturner/gokrb5/v8/client/client.go new file mode 100644 index 0000000000..074e3f1245 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/client/client.go @@ -0,0 +1,329 @@ +// Package client provides a client library and methods for Kerberos 5 authentication. +package client + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "strings" + "time" + + "github.com/jcmturner/gokrb5/v8/config" + "github.com/jcmturner/gokrb5/v8/credentials" + "github.com/jcmturner/gokrb5/v8/crypto" + "github.com/jcmturner/gokrb5/v8/crypto/etype" + "github.com/jcmturner/gokrb5/v8/iana/errorcode" + "github.com/jcmturner/gokrb5/v8/iana/nametype" + "github.com/jcmturner/gokrb5/v8/keytab" + "github.com/jcmturner/gokrb5/v8/krberror" + "github.com/jcmturner/gokrb5/v8/messages" + "github.com/jcmturner/gokrb5/v8/types" +) + +// Client side configuration and state. +type Client struct { + Credentials *credentials.Credentials + Config *config.Config + settings *Settings + sessions *sessions + cache *Cache +} + +// NewWithPassword creates a new client from a password credential. +// Set the realm to empty string to use the default realm from config. +func NewWithPassword(username, realm, password string, krb5conf *config.Config, settings ...func(*Settings)) *Client { + creds := credentials.New(username, realm) + return &Client{ + Credentials: creds.WithPassword(password), + Config: krb5conf, + settings: NewSettings(settings...), + sessions: &sessions{ + Entries: make(map[string]*session), + }, + cache: NewCache(), + } +} + +// NewWithKeytab creates a new client from a keytab credential. +func NewWithKeytab(username, realm string, kt *keytab.Keytab, krb5conf *config.Config, settings ...func(*Settings)) *Client { + creds := credentials.New(username, realm) + return &Client{ + Credentials: creds.WithKeytab(kt), + Config: krb5conf, + settings: NewSettings(settings...), + sessions: &sessions{ + Entries: make(map[string]*session), + }, + cache: NewCache(), + } +} + +// NewFromCCache create a client from a populated client cache. +// +// WARNING: A client created from CCache does not automatically renew TGTs and a failure will occur after the TGT expires. +func NewFromCCache(c *credentials.CCache, krb5conf *config.Config, settings ...func(*Settings)) (*Client, error) { + cl := &Client{ + Credentials: c.GetClientCredentials(), + Config: krb5conf, + settings: NewSettings(settings...), + sessions: &sessions{ + Entries: make(map[string]*session), + }, + cache: NewCache(), + } + spn := types.PrincipalName{ + NameType: nametype.KRB_NT_SRV_INST, + NameString: []string{"krbtgt", c.DefaultPrincipal.Realm}, + } + cred, ok := c.GetEntry(spn) + if !ok { + return cl, errors.New("TGT not found in CCache") + } + var tgt messages.Ticket + err := tgt.Unmarshal(cred.Ticket) + if err != nil { + return cl, fmt.Errorf("TGT bytes in cache are not valid: %v", err) + } + cl.sessions.Entries[c.DefaultPrincipal.Realm] = &session{ + realm: c.DefaultPrincipal.Realm, + authTime: cred.AuthTime, + endTime: cred.EndTime, + renewTill: cred.RenewTill, + tgt: tgt, + sessionKey: cred.Key, + } + for _, cred := range c.GetEntries() { + var tkt messages.Ticket + err = tkt.Unmarshal(cred.Ticket) + if err != nil { + return cl, fmt.Errorf("cache entry ticket bytes are not valid: %v", err) + } + cl.cache.addEntry( + tkt, + cred.AuthTime, + cred.StartTime, + cred.EndTime, + cred.RenewTill, + cred.Key, + ) + } + return cl, nil +} + +// Key returns the client's encryption key for the specified encryption type and its kvno (kvno of zero will find latest). +// The key can be retrieved either from the keytab or generated from the client's password. +// If the client has both a keytab and a password defined the keytab is favoured as the source for the key +// A KRBError can be passed in the event the KDC returns one of type KDC_ERR_PREAUTH_REQUIRED and is required to derive +// the key for pre-authentication from the client's password. If a KRBError is not available, pass nil to this argument. +func (cl *Client) Key(etype etype.EType, kvno int, krberr *messages.KRBError) (types.EncryptionKey, int, error) { + if cl.Credentials.HasKeytab() && etype != nil { + return cl.Credentials.Keytab().GetEncryptionKey(cl.Credentials.CName(), cl.Credentials.Domain(), kvno, etype.GetETypeID()) + } else if cl.Credentials.HasPassword() { + if krberr != nil && krberr.ErrorCode == errorcode.KDC_ERR_PREAUTH_REQUIRED { + var pas types.PADataSequence + err := pas.Unmarshal(krberr.EData) + if err != nil { + return types.EncryptionKey{}, 0, fmt.Errorf("could not get PAData from KRBError to generate key from password: %v", err) + } + key, _, err := crypto.GetKeyFromPassword(cl.Credentials.Password(), krberr.CName, krberr.CRealm, etype.GetETypeID(), pas) + return key, 0, err + } + key, _, err := crypto.GetKeyFromPassword(cl.Credentials.Password(), cl.Credentials.CName(), cl.Credentials.Domain(), etype.GetETypeID(), types.PADataSequence{}) + return key, 0, err + } + return types.EncryptionKey{}, 0, errors.New("credential has neither keytab or password to generate key") +} + +// IsConfigured indicates if the client has the values required set. +func (cl *Client) IsConfigured() (bool, error) { + if cl.Credentials.UserName() == "" { + return false, errors.New("client does not have a username") + } + if cl.Credentials.Domain() == "" { + return false, errors.New("client does not have a define realm") + } + // Client needs to have either a password, keytab or a session already (later when loading from CCache) + if !cl.Credentials.HasPassword() && !cl.Credentials.HasKeytab() { + authTime, _, _, _, err := cl.sessionTimes(cl.Credentials.Domain()) + if err != nil || authTime.IsZero() { + return false, errors.New("client has neither a keytab nor a password set and no session") + } + } + if !cl.Config.LibDefaults.DNSLookupKDC { + for _, r := range cl.Config.Realms { + if r.Realm == cl.Credentials.Domain() { + if len(r.KDC) > 0 { + return true, nil + } + return false, errors.New("client krb5 config does not have any defined KDCs for the default realm") + } + } + } + return true, nil +} + +// Login the client with the KDC via an AS exchange. +func (cl *Client) Login() error { + if ok, err := cl.IsConfigured(); !ok { + return err + } + if !cl.Credentials.HasPassword() && !cl.Credentials.HasKeytab() { + _, endTime, _, _, err := cl.sessionTimes(cl.Credentials.Domain()) + if err != nil { + return krberror.Errorf(err, krberror.KRBMsgError, "no user credentials available and error getting any existing session") + } + if time.Now().UTC().After(endTime) { + return krberror.New(krberror.KRBMsgError, "cannot login, no user credentials available and no valid existing session") + } + // no credentials but there is a session with tgt already + return nil + } + ASReq, err := messages.NewASReqForTGT(cl.Credentials.Domain(), cl.Config, cl.Credentials.CName()) + if err != nil { + return krberror.Errorf(err, krberror.KRBMsgError, "error generating new AS_REQ") + } + ASRep, err := cl.ASExchange(cl.Credentials.Domain(), ASReq, 0) + if err != nil { + return err + } + cl.addSession(ASRep.Ticket, ASRep.DecryptedEncPart) + return nil +} + +// AffirmLogin will only perform an AS exchange with the KDC if the client does not already have a TGT. +func (cl *Client) AffirmLogin() error { + _, endTime, _, _, err := cl.sessionTimes(cl.Credentials.Domain()) + if err != nil || time.Now().UTC().After(endTime) { + err := cl.Login() + if err != nil { + return fmt.Errorf("could not get valid TGT for client's realm: %v", err) + } + } + return nil +} + +// realmLogin obtains or renews a TGT and establishes a session for the realm specified. +func (cl *Client) realmLogin(realm string) error { + if realm == cl.Credentials.Domain() { + return cl.Login() + } + _, endTime, _, _, err := cl.sessionTimes(cl.Credentials.Domain()) + if err != nil || time.Now().UTC().After(endTime) { + err := cl.Login() + if err != nil { + return fmt.Errorf("could not get valid TGT for client's realm: %v", err) + } + } + tgt, skey, err := cl.sessionTGT(cl.Credentials.Domain()) + if err != nil { + return err + } + + spn := types.PrincipalName{ + NameType: nametype.KRB_NT_SRV_INST, + NameString: []string{"krbtgt", realm}, + } + + _, tgsRep, err := cl.TGSREQGenerateAndExchange(spn, cl.Credentials.Domain(), tgt, skey, false) + if err != nil { + return err + } + cl.addSession(tgsRep.Ticket, tgsRep.DecryptedEncPart) + + return nil +} + +// Destroy stops the auto-renewal of all sessions and removes the sessions and cache entries from the client. +func (cl *Client) Destroy() { + creds := credentials.New("", "") + cl.sessions.destroy() + cl.cache.clear() + cl.Credentials = creds + cl.Log("client destroyed") +} + +// Diagnostics runs a set of checks that the client is properly configured and writes details to the io.Writer provided. +func (cl *Client) Diagnostics(w io.Writer) error { + cl.Print(w) + var errs []string + if cl.Credentials.HasKeytab() { + var loginRealmEncTypes []int32 + for _, e := range cl.Credentials.Keytab().Entries { + if e.Principal.Realm == cl.Credentials.Realm() { + loginRealmEncTypes = append(loginRealmEncTypes, e.Key.KeyType) + } + } + for _, et := range cl.Config.LibDefaults.DefaultTktEnctypeIDs { + var etInKt bool + for _, val := range loginRealmEncTypes { + if val == et { + etInKt = true + break + } + } + if !etInKt { + errs = append(errs, fmt.Sprintf("default_tkt_enctypes specifies %d but this enctype is not available in the client's keytab", et)) + } + } + for _, et := range cl.Config.LibDefaults.PreferredPreauthTypes { + var etInKt bool + for _, val := range loginRealmEncTypes { + if int(val) == et { + etInKt = true + break + } + } + if !etInKt { + errs = append(errs, fmt.Sprintf("preferred_preauth_types specifies %d but this enctype is not available in the client's keytab", et)) + } + } + } + udpCnt, udpKDC, err := cl.Config.GetKDCs(cl.Credentials.Realm(), false) + if err != nil { + errs = append(errs, fmt.Sprintf("error when resolving KDCs for UDP communication: %v", err)) + } + if udpCnt < 1 { + errs = append(errs, "no KDCs resolved for communication via UDP.") + } else { + b, _ := json.MarshalIndent(&udpKDC, "", " ") + fmt.Fprintf(w, "UDP KDCs: %s\n", string(b)) + } + tcpCnt, tcpKDC, err := cl.Config.GetKDCs(cl.Credentials.Realm(), false) + if err != nil { + errs = append(errs, fmt.Sprintf("error when resolving KDCs for TCP communication: %v", err)) + } + if tcpCnt < 1 { + errs = append(errs, "no KDCs resolved for communication via TCP.") + } else { + b, _ := json.MarshalIndent(&tcpKDC, "", " ") + fmt.Fprintf(w, "TCP KDCs: %s\n", string(b)) + } + + if errs == nil || len(errs) < 1 { + return nil + } + err = fmt.Errorf(strings.Join(errs, "\n")) + return err +} + +// Print writes the details of the client to the io.Writer provided. +func (cl *Client) Print(w io.Writer) { + c, _ := cl.Credentials.JSON() + fmt.Fprintf(w, "Credentials:\n%s\n", c) + + s, _ := cl.sessions.JSON() + fmt.Fprintf(w, "TGT Sessions:\n%s\n", s) + + c, _ = cl.cache.JSON() + fmt.Fprintf(w, "Service ticket cache:\n%s\n", c) + + s, _ = cl.settings.JSON() + fmt.Fprintf(w, "Settings:\n%s\n", s) + + j, _ := cl.Config.JSON() + fmt.Fprintf(w, "Krb5 config:\n%s\n", j) + + k, _ := cl.Credentials.Keytab().JSON() + fmt.Fprintf(w, "Keytab:\n%s\n", k) +} diff --git a/github.com/jcmturner/gokrb5/v8/client/network.go b/github.com/jcmturner/gokrb5/v8/client/network.go new file mode 100644 index 0000000000..9ca0e37ace --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/client/network.go @@ -0,0 +1,213 @@ +package client + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "net" + "time" + + "github.com/jcmturner/gokrb5/v8/iana/errorcode" + "github.com/jcmturner/gokrb5/v8/messages" +) + +// SendToKDC performs network actions to send data to the KDC. +func (cl *Client) sendToKDC(b []byte, realm string) ([]byte, error) { + var rb []byte + if cl.Config.LibDefaults.UDPPreferenceLimit == 1 { + //1 means we should always use TCP + rb, errtcp := cl.sendKDCTCP(realm, b) + if errtcp != nil { + if e, ok := errtcp.(messages.KRBError); ok { + return rb, e + } + return rb, fmt.Errorf("communication error with KDC via TCP: %v", errtcp) + } + return rb, nil + } + if len(b) <= cl.Config.LibDefaults.UDPPreferenceLimit { + //Try UDP first, TCP second + rb, errudp := cl.sendKDCUDP(realm, b) + if errudp != nil { + if e, ok := errudp.(messages.KRBError); ok && e.ErrorCode != errorcode.KRB_ERR_RESPONSE_TOO_BIG { + // Got a KRBError from KDC + // If this is not a KRB_ERR_RESPONSE_TOO_BIG we will return immediately otherwise will try TCP. + return rb, e + } + // Try TCP + r, errtcp := cl.sendKDCTCP(realm, b) + if errtcp != nil { + if e, ok := errtcp.(messages.KRBError); ok { + // Got a KRBError + return r, e + } + return r, fmt.Errorf("failed to communicate with KDC. Attempts made with UDP (%v) and then TCP (%v)", errudp, errtcp) + } + rb = r + } + return rb, nil + } + //Try TCP first, UDP second + rb, errtcp := cl.sendKDCTCP(realm, b) + if errtcp != nil { + if e, ok := errtcp.(messages.KRBError); ok { + // Got a KRBError from KDC so returning and not trying UDP. + return rb, e + } + rb, errudp := cl.sendKDCUDP(realm, b) + if errudp != nil { + if e, ok := errudp.(messages.KRBError); ok { + // Got a KRBError + return rb, e + } + return rb, fmt.Errorf("failed to communicate with KDC. Attempts made with TCP (%v) and then UDP (%v)", errtcp, errudp) + } + } + return rb, nil +} + +// dialKDCTCP establishes a UDP connection to a KDC. +func dialKDCUDP(count int, kdcs map[int]string) (*net.UDPConn, error) { + i := 1 + for i <= count { + udpAddr, err := net.ResolveUDPAddr("udp", kdcs[i]) + if err != nil { + return nil, fmt.Errorf("error resolving KDC address: %v", err) + } + + conn, err := net.DialTimeout("udp", udpAddr.String(), 5*time.Second) + if err == nil { + if err := conn.SetDeadline(time.Now().Add(5 * time.Second)); err != nil { + return nil, err + } + // conn is guaranteed to be a UDPConn + return conn.(*net.UDPConn), nil + } + i++ + } + return nil, errors.New("error in getting a UDP connection to any of the KDCs") +} + +// dialKDCTCP establishes a TCP connection to a KDC. +func dialKDCTCP(count int, kdcs map[int]string) (*net.TCPConn, error) { + i := 1 + for i <= count { + tcpAddr, err := net.ResolveTCPAddr("tcp", kdcs[i]) + if err != nil { + return nil, fmt.Errorf("error resolving KDC address: %v", err) + } + + conn, err := net.DialTimeout("tcp", tcpAddr.String(), 5*time.Second) + if err == nil { + if err := conn.SetDeadline(time.Now().Add(5 * time.Second)); err != nil { + return nil, err + } + // conn is guaranteed to be a TCPConn + return conn.(*net.TCPConn), nil + } + i++ + } + return nil, errors.New("error in getting a TCP connection to any of the KDCs") +} + +// sendKDCUDP sends bytes to the KDC via UDP. +func (cl *Client) sendKDCUDP(realm string, b []byte) ([]byte, error) { + var r []byte + count, kdcs, err := cl.Config.GetKDCs(realm, false) + if err != nil { + return r, err + } + conn, err := dialKDCUDP(count, kdcs) + if err != nil { + return r, err + } + r, err = cl.sendUDP(conn, b) + if err != nil { + return r, err + } + return checkForKRBError(r) +} + +// sendKDCTCP sends bytes to the KDC via TCP. +func (cl *Client) sendKDCTCP(realm string, b []byte) ([]byte, error) { + var r []byte + count, kdcs, err := cl.Config.GetKDCs(realm, true) + if err != nil { + return r, err + } + conn, err := dialKDCTCP(count, kdcs) + if err != nil { + return r, err + } + rb, err := cl.sendTCP(conn, b) + if err != nil { + return r, err + } + return checkForKRBError(rb) +} + +// sendUDP sends bytes to connection over UDP. +func (cl *Client) sendUDP(conn *net.UDPConn, b []byte) ([]byte, error) { + var r []byte + defer conn.Close() + _, err := conn.Write(b) + if err != nil { + return r, fmt.Errorf("error sending to (%s): %v", conn.RemoteAddr().String(), err) + } + udpbuf := make([]byte, 4096) + n, _, err := conn.ReadFrom(udpbuf) + r = udpbuf[:n] + if err != nil { + return r, fmt.Errorf("sending over UDP failed to %s: %v", conn.RemoteAddr().String(), err) + } + if len(r) < 1 { + return r, fmt.Errorf("no response data from %s", conn.RemoteAddr().String()) + } + return r, nil +} + +// sendTCP sends bytes to connection over TCP. +func (cl *Client) sendTCP(conn *net.TCPConn, b []byte) ([]byte, error) { + defer conn.Close() + var r []byte + // RFC 4120 7.2.2 specifies the first 4 bytes indicate the length of the message in big endian order. + var buf bytes.Buffer + err := binary.Write(&buf, binary.BigEndian, uint32(len(b))) + if err != nil { + return r, err + } + b = append(buf.Bytes(), b...) + + _, err = conn.Write(b) + if err != nil { + return r, fmt.Errorf("error sending to KDC (%s): %v", conn.RemoteAddr().String(), err) + } + + sh := make([]byte, 4, 4) + _, err = conn.Read(sh) + if err != nil { + return r, fmt.Errorf("error reading response size header: %v", err) + } + s := binary.BigEndian.Uint32(sh) + + rb := make([]byte, s, s) + _, err = io.ReadFull(conn, rb) + if err != nil { + return r, fmt.Errorf("error reading response: %v", err) + } + if len(rb) < 1 { + return r, fmt.Errorf("no response data from KDC %s", conn.RemoteAddr().String()) + } + return rb, nil +} + +// checkForKRBError checks if the response bytes from the KDC are a KRBError. +func checkForKRBError(b []byte) ([]byte, error) { + var KRBErr messages.KRBError + if err := KRBErr.Unmarshal(b); err == nil { + return b, KRBErr + } + return b, nil +} diff --git a/github.com/jcmturner/gokrb5/v8/client/passwd.go b/github.com/jcmturner/gokrb5/v8/client/passwd.go new file mode 100644 index 0000000000..35069c77ed --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/client/passwd.go @@ -0,0 +1,95 @@ +package client + +import ( + "fmt" + "net" + + "github.com/jcmturner/gokrb5/v8/kadmin" + "github.com/jcmturner/gokrb5/v8/messages" +) + +// Kpasswd server response codes. +const ( + KRB5_KPASSWD_SUCCESS = 0 + KRB5_KPASSWD_MALFORMED = 1 + KRB5_KPASSWD_HARDERROR = 2 + KRB5_KPASSWD_AUTHERROR = 3 + KRB5_KPASSWD_SOFTERROR = 4 + KRB5_KPASSWD_ACCESSDENIED = 5 + KRB5_KPASSWD_BAD_VERSION = 6 + KRB5_KPASSWD_INITIAL_FLAG_NEEDED = 7 +) + +// ChangePasswd changes the password of the client to the value provided. +func (cl *Client) ChangePasswd(newPasswd string) (bool, error) { + ASReq, err := messages.NewASReqForChgPasswd(cl.Credentials.Domain(), cl.Config, cl.Credentials.CName()) + if err != nil { + return false, err + } + ASRep, err := cl.ASExchange(cl.Credentials.Domain(), ASReq, 0) + if err != nil { + return false, err + } + + msg, key, err := kadmin.ChangePasswdMsg(cl.Credentials.CName(), cl.Credentials.Domain(), newPasswd, ASRep.Ticket, ASRep.DecryptedEncPart.Key) + if err != nil { + return false, err + } + r, err := cl.sendToKPasswd(msg) + if err != nil { + return false, err + } + err = r.Decrypt(key) + if err != nil { + return false, err + } + if r.ResultCode != KRB5_KPASSWD_SUCCESS { + return false, fmt.Errorf("error response from kadmin: code: %d; result: %s; krberror: %v", r.ResultCode, r.Result, r.KRBError) + } + cl.Credentials.WithPassword(newPasswd) + return true, nil +} + +func (cl *Client) sendToKPasswd(msg kadmin.Request) (r kadmin.Reply, err error) { + _, kps, err := cl.Config.GetKpasswdServers(cl.Credentials.Domain(), true) + if err != nil { + return + } + addr := kps[1] + b, err := msg.Marshal() + if err != nil { + return + } + if len(b) <= cl.Config.LibDefaults.UDPPreferenceLimit { + return cl.sendKPasswdUDP(b, addr) + } + return cl.sendKPasswdTCP(b, addr) +} + +func (cl *Client) sendKPasswdTCP(b []byte, kadmindAddr string) (r kadmin.Reply, err error) { + tcpAddr, err := net.ResolveTCPAddr("tcp", kadmindAddr) + if err != nil { + return + } + conn, err := net.DialTCP("tcp", nil, tcpAddr) + if err != nil { + return + } + rb, err := cl.sendTCP(conn, b) + err = r.Unmarshal(rb) + return +} + +func (cl *Client) sendKPasswdUDP(b []byte, kadmindAddr string) (r kadmin.Reply, err error) { + udpAddr, err := net.ResolveUDPAddr("udp", kadmindAddr) + if err != nil { + return + } + conn, err := net.DialUDP("udp", nil, udpAddr) + if err != nil { + return + } + rb, err := cl.sendUDP(conn, b) + err = r.Unmarshal(rb) + return +} diff --git a/github.com/jcmturner/gokrb5/v8/client/session.go b/github.com/jcmturner/gokrb5/v8/client/session.go new file mode 100644 index 0000000000..f7654d0d07 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/client/session.go @@ -0,0 +1,295 @@ +package client + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "sync" + "time" + + "github.com/jcmturner/gokrb5/v8/iana/nametype" + "github.com/jcmturner/gokrb5/v8/krberror" + "github.com/jcmturner/gokrb5/v8/messages" + "github.com/jcmturner/gokrb5/v8/types" +) + +// sessions hold TGTs and are keyed on the realm name +type sessions struct { + Entries map[string]*session + mux sync.RWMutex +} + +// destroy erases all sessions +func (s *sessions) destroy() { + s.mux.Lock() + defer s.mux.Unlock() + for k, e := range s.Entries { + e.destroy() + delete(s.Entries, k) + } +} + +// update replaces a session with the one provided or adds it as a new one +func (s *sessions) update(sess *session) { + s.mux.Lock() + defer s.mux.Unlock() + // if a session already exists for this, cancel its auto renew. + if i, ok := s.Entries[sess.realm]; ok { + if i != sess { + // Session in the sessions cache is not the same as one provided. + // Cancel the one in the cache and add this one. + i.mux.Lock() + defer i.mux.Unlock() + i.cancel <- true + s.Entries[sess.realm] = sess + return + } + } + // No session for this realm was found so just add it + s.Entries[sess.realm] = sess +} + +// get returns the session for the realm specified +func (s *sessions) get(realm string) (*session, bool) { + s.mux.RLock() + defer s.mux.RUnlock() + sess, ok := s.Entries[realm] + return sess, ok +} + +// session holds the TGT details for a realm +type session struct { + realm string + authTime time.Time + endTime time.Time + renewTill time.Time + tgt messages.Ticket + sessionKey types.EncryptionKey + sessionKeyExpiration time.Time + cancel chan bool + mux sync.RWMutex +} + +// jsonSession is used to enable marshaling some information of a session in a JSON format +type jsonSession struct { + Realm string + AuthTime time.Time + EndTime time.Time + RenewTill time.Time + SessionKeyExpiration time.Time +} + +// AddSession adds a session for a realm with a TGT to the client's session cache. +// A goroutine is started to automatically renew the TGT before expiry. +func (cl *Client) addSession(tgt messages.Ticket, dep messages.EncKDCRepPart) { + if strings.ToLower(tgt.SName.NameString[0]) != "krbtgt" { + // Not a TGT + return + } + realm := tgt.SName.NameString[len(tgt.SName.NameString)-1] + s := &session{ + realm: realm, + authTime: dep.AuthTime, + endTime: dep.EndTime, + renewTill: dep.RenewTill, + tgt: tgt, + sessionKey: dep.Key, + sessionKeyExpiration: dep.KeyExpiration, + } + cl.sessions.update(s) + cl.enableAutoSessionRenewal(s) + cl.Log("TGT session added for %s (EndTime: %v)", realm, dep.EndTime) +} + +// update overwrites the session details with those from the TGT and decrypted encPart +func (s *session) update(tgt messages.Ticket, dep messages.EncKDCRepPart) { + s.mux.Lock() + defer s.mux.Unlock() + s.authTime = dep.AuthTime + s.endTime = dep.EndTime + s.renewTill = dep.RenewTill + s.tgt = tgt + s.sessionKey = dep.Key + s.sessionKeyExpiration = dep.KeyExpiration +} + +// destroy will cancel any auto renewal of the session and set the expiration times to the current time +func (s *session) destroy() { + s.mux.Lock() + defer s.mux.Unlock() + if s.cancel != nil { + s.cancel <- true + } + s.endTime = time.Now().UTC() + s.renewTill = s.endTime + s.sessionKeyExpiration = s.endTime +} + +// valid informs if the TGT is still within the valid time window +func (s *session) valid() bool { + s.mux.RLock() + defer s.mux.RUnlock() + t := time.Now().UTC() + if t.Before(s.endTime) && s.authTime.Before(t) { + return true + } + return false +} + +// tgtDetails is a thread safe way to get the session's realm, TGT and session key values +func (s *session) tgtDetails() (string, messages.Ticket, types.EncryptionKey) { + s.mux.RLock() + defer s.mux.RUnlock() + return s.realm, s.tgt, s.sessionKey +} + +// timeDetails is a thread safe way to get the session's validity time values +func (s *session) timeDetails() (string, time.Time, time.Time, time.Time, time.Time) { + s.mux.RLock() + defer s.mux.RUnlock() + return s.realm, s.authTime, s.endTime, s.renewTill, s.sessionKeyExpiration +} + +// JSON return information about the held sessions in a JSON format. +func (s *sessions) JSON() (string, error) { + s.mux.RLock() + defer s.mux.RUnlock() + var js []jsonSession + keys := make([]string, 0, len(s.Entries)) + for k := range s.Entries { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + r, at, et, rt, kt := s.Entries[k].timeDetails() + j := jsonSession{ + Realm: r, + AuthTime: at, + EndTime: et, + RenewTill: rt, + SessionKeyExpiration: kt, + } + js = append(js, j) + } + b, err := json.MarshalIndent(js, "", " ") + if err != nil { + return "", err + } + return string(b), nil +} + +// enableAutoSessionRenewal turns on the automatic renewal for the client's TGT session. +func (cl *Client) enableAutoSessionRenewal(s *session) { + var timer *time.Timer + s.mux.Lock() + s.cancel = make(chan bool, 1) + s.mux.Unlock() + go func(s *session) { + for { + s.mux.RLock() + w := (s.endTime.Sub(time.Now().UTC()) * 5) / 6 + s.mux.RUnlock() + if w < 0 { + return + } + timer = time.NewTimer(w) + select { + case <-timer.C: + renewal, err := cl.refreshSession(s) + if err != nil { + cl.Log("error refreshing session: %v", err) + } + if !renewal && err == nil { + // end this goroutine as there will have been a new login and new auto renewal goroutine created. + return + } + case <-s.cancel: + // cancel has been called. Stop the timer and exit. + timer.Stop() + return + } + } + }(s) +} + +// renewTGT renews the client's TGT session. +func (cl *Client) renewTGT(s *session) error { + realm, tgt, skey := s.tgtDetails() + spn := types.PrincipalName{ + NameType: nametype.KRB_NT_SRV_INST, + NameString: []string{"krbtgt", realm}, + } + _, tgsRep, err := cl.TGSREQGenerateAndExchange(spn, cl.Credentials.Domain(), tgt, skey, true) + if err != nil { + return krberror.Errorf(err, krberror.KRBMsgError, "error renewing TGT for %s", realm) + } + s.update(tgsRep.Ticket, tgsRep.DecryptedEncPart) + cl.sessions.update(s) + cl.Log("TGT session renewed for %s (EndTime: %v)", realm, tgsRep.DecryptedEncPart.EndTime) + return nil +} + +// refreshSession updates either through renewal or creating a new login. +// The boolean indicates if the update was a renewal. +func (cl *Client) refreshSession(s *session) (bool, error) { + s.mux.RLock() + realm := s.realm + renewTill := s.renewTill + s.mux.RUnlock() + cl.Log("refreshing TGT session for %s", realm) + if time.Now().UTC().Before(renewTill) { + err := cl.renewTGT(s) + return true, err + } + err := cl.realmLogin(realm) + return false, err +} + +// ensureValidSession makes sure there is a valid session for the realm +func (cl *Client) ensureValidSession(realm string) error { + s, ok := cl.sessions.get(realm) + if ok { + s.mux.RLock() + d := s.endTime.Sub(s.authTime) / 6 + if s.endTime.Sub(time.Now().UTC()) > d { + s.mux.RUnlock() + return nil + } + s.mux.RUnlock() + _, err := cl.refreshSession(s) + return err + } + return cl.realmLogin(realm) +} + +// sessionTGTDetails is a thread safe way to get the TGT and session key values for a realm +func (cl *Client) sessionTGT(realm string) (tgt messages.Ticket, sessionKey types.EncryptionKey, err error) { + err = cl.ensureValidSession(realm) + if err != nil { + return + } + s, ok := cl.sessions.get(realm) + if !ok { + err = fmt.Errorf("could not find TGT session for %s", realm) + return + } + _, tgt, sessionKey = s.tgtDetails() + return +} + +// sessionTimes provides the timing information with regards to a session for the realm specified. +func (cl *Client) sessionTimes(realm string) (authTime, endTime, renewTime, sessionExp time.Time, err error) { + s, ok := cl.sessions.get(realm) + if !ok { + err = fmt.Errorf("could not find TGT session for %s", realm) + return + } + _, authTime, endTime, renewTime, sessionExp = s.timeDetails() + return +} + +// spnRealm resolves the realm name of a service principal name +func (cl *Client) spnRealm(spn types.PrincipalName) string { + return cl.Config.ResolveRealm(spn.NameString[len(spn.NameString)-1]) +} diff --git a/github.com/jcmturner/gokrb5/v8/client/settings.go b/github.com/jcmturner/gokrb5/v8/client/settings.go new file mode 100644 index 0000000000..bcd3945431 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/client/settings.go @@ -0,0 +1,93 @@ +package client + +import ( + "encoding/json" + "fmt" + "log" +) + +// Settings holds optional client settings. +type Settings struct { + disablePAFXFast bool + assumePreAuthentication bool + preAuthEType int32 + logger *log.Logger +} + +// jsonSettings is used when marshaling the Settings details to JSON format. +type jsonSettings struct { + DisablePAFXFast bool + AssumePreAuthentication bool +} + +// NewSettings creates a new client settings struct. +func NewSettings(settings ...func(*Settings)) *Settings { + s := new(Settings) + for _, set := range settings { + set(s) + } + return s +} + +// DisablePAFXFAST used to configure the client to not use PA_FX_FAST. +// +// s := NewSettings(DisablePAFXFAST(true)) +func DisablePAFXFAST(b bool) func(*Settings) { + return func(s *Settings) { + s.disablePAFXFast = b + } +} + +// DisablePAFXFAST indicates is the client should disable the use of PA_FX_FAST. +func (s *Settings) DisablePAFXFAST() bool { + return s.disablePAFXFast +} + +// AssumePreAuthentication used to configure the client to assume pre-authentication is required. +// +// s := NewSettings(AssumePreAuthentication(true)) +func AssumePreAuthentication(b bool) func(*Settings) { + return func(s *Settings) { + s.assumePreAuthentication = b + } +} + +// AssumePreAuthentication indicates if the client should proactively assume using pre-authentication. +func (s *Settings) AssumePreAuthentication() bool { + return s.assumePreAuthentication +} + +// Logger used to configure client with a logger. +// +// s := NewSettings(kt, Logger(l)) +func Logger(l *log.Logger) func(*Settings) { + return func(s *Settings) { + s.logger = l + } +} + +// Logger returns the client logger instance. +func (s *Settings) Logger() *log.Logger { + return s.logger +} + +// Log will write to the service's logger if it is configured. +func (cl *Client) Log(format string, v ...interface{}) { + if cl.settings.Logger() != nil { + cl.settings.Logger().Output(2, fmt.Sprintf(format, v...)) + } +} + +// JSON returns a JSON representation of the settings. +func (s *Settings) JSON() (string, error) { + js := jsonSettings{ + DisablePAFXFast: s.disablePAFXFast, + AssumePreAuthentication: s.assumePreAuthentication, + } + b, err := json.MarshalIndent(js, "", " ") + if err != nil { + return "", err + } + return string(b), nil + +} diff --git a/github.com/jcmturner/gokrb5/v8/config/error.go b/github.com/jcmturner/gokrb5/v8/config/error.go new file mode 100644 index 0000000000..1fbda51f38 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/config/error.go @@ -0,0 +1,30 @@ +package config + +import "fmt" + +// UnsupportedDirective error. +type UnsupportedDirective struct { + text string +} + +// Error implements the error interface for unsupported directives. +func (e UnsupportedDirective) Error() string { + return e.text +} + +// Invalid config error. +type Invalid struct { + text string +} + +// Error implements the error interface for invalid config error. +func (e Invalid) Error() string { + return e.text +} + +// InvalidErrorf creates a new Invalid error. +func InvalidErrorf(format string, a ...interface{}) Invalid { + return Invalid{ + text: fmt.Sprintf("invalid krb5 config "+format, a...), + } +} diff --git a/github.com/jcmturner/gokrb5/v8/config/hosts.go b/github.com/jcmturner/gokrb5/v8/config/hosts.go new file mode 100644 index 0000000000..3f22c70c41 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/config/hosts.go @@ -0,0 +1,141 @@ +package config + +import ( + "fmt" + "math/rand" + "net" + "strconv" + "strings" + + "github.com/jcmturner/dnsutils/v2" +) + +// GetKDCs returns the count of KDCs available and a map of KDC host names keyed on preference order. +func (c *Config) GetKDCs(realm string, tcp bool) (int, map[int]string, error) { + if realm == "" { + realm = c.LibDefaults.DefaultRealm + } + kdcs := make(map[int]string) + var count int + + // Get the KDCs from the krb5.conf. + var ks []string + for _, r := range c.Realms { + if r.Realm != realm { + continue + } + ks = r.KDC + } + count = len(ks) + + if count > 0 { + // Order the kdcs randomly for preference. + kdcs = randServOrder(ks) + return count, kdcs, nil + } + + if !c.LibDefaults.DNSLookupKDC { + return count, kdcs, fmt.Errorf("no KDCs defined in configuration for realm %s", realm) + } + + // Use DNS to resolve kerberos SRV records. + proto := "udp" + if tcp { + proto = "tcp" + } + index, addrs, err := dnsutils.OrderedSRV("kerberos", proto, realm) + if err != nil { + return count, kdcs, err + } + if len(addrs) < 1 { + return count, kdcs, fmt.Errorf("no KDC SRV records found for realm %s", realm) + } + count = index + for k, v := range addrs { + kdcs[k] = strings.TrimRight(v.Target, ".") + ":" + strconv.Itoa(int(v.Port)) + } + return count, kdcs, nil +} + +// GetKpasswdServers returns the count of kpasswd servers available and a map of kpasswd host names keyed on preference order. +// https://web.mit.edu/kerberos/krb5-latest/doc/admin/conf_files/krb5_conf.html#realms - see kpasswd_server section +func (c *Config) GetKpasswdServers(realm string, tcp bool) (int, map[int]string, error) { + kdcs := make(map[int]string) + var count int + + // Use DNS to resolve kerberos SRV records if configured to do so in krb5.conf. + if c.LibDefaults.DNSLookupKDC { + proto := "udp" + if tcp { + proto = "tcp" + } + c, addrs, err := dnsutils.OrderedSRV("kpasswd", proto, realm) + if err != nil { + return count, kdcs, err + } + if c < 1 { + c, addrs, err = dnsutils.OrderedSRV("kerberos-adm", proto, realm) + if err != nil { + return count, kdcs, err + } + } + if len(addrs) < 1 { + return count, kdcs, fmt.Errorf("no kpasswd or kadmin SRV records found for realm %s", realm) + } + count = c + for k, v := range addrs { + kdcs[k] = strings.TrimRight(v.Target, ".") + ":" + strconv.Itoa(int(v.Port)) + } + } else { + // Get the KDCs from the krb5.conf an order them randomly for preference. + var ks []string + var ka []string + for _, r := range c.Realms { + if r.Realm == realm { + ks = r.KPasswdServer + ka = r.AdminServer + break + } + } + if len(ks) < 1 { + for _, k := range ka { + h, _, err := net.SplitHostPort(k) + if err != nil { + continue + } + ks = append(ks, h+":464") + } + } + count = len(ks) + if count < 1 { + return count, kdcs, fmt.Errorf("no kpasswd or kadmin defined in configuration for realm %s", realm) + } + kdcs = randServOrder(ks) + } + return count, kdcs, nil +} + +func randServOrder(ks []string) map[int]string { + kdcs := make(map[int]string) + count := len(ks) + i := 1 + if count > 1 { + l := len(ks) + for l > 0 { + ri := rand.Intn(l) + kdcs[i] = ks[ri] + if l > 1 { + // Remove the entry from the source slice by swapping with the last entry and truncating + ks[len(ks)-1], ks[ri] = ks[ri], ks[len(ks)-1] + ks = ks[:len(ks)-1] + l = len(ks) + } else { + l = 0 + } + i++ + } + } else { + kdcs[i] = ks[0] + } + return kdcs +} diff --git a/github.com/jcmturner/gokrb5/v8/config/krb5conf.go b/github.com/jcmturner/gokrb5/v8/config/krb5conf.go new file mode 100644 index 0000000000..a7638433d5 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/config/krb5conf.go @@ -0,0 +1,728 @@ +// Package config implements KRB5 client and service configuration as described at https://web.mit.edu/kerberos/krb5-latest/doc/admin/conf_files/krb5_conf.html +package config + +import ( + "bufio" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "os" + "os/user" + "regexp" + "strconv" + "strings" + "time" + + "github.com/jcmturner/gofork/encoding/asn1" + "github.com/jcmturner/gokrb5/v8/iana/etypeID" +) + +// Config represents the KRB5 configuration. +type Config struct { + LibDefaults LibDefaults + Realms []Realm + DomainRealm DomainRealm + //CaPaths + //AppDefaults + //Plugins +} + +// WeakETypeList is a list of encryption types that have been deemed weak. +const WeakETypeList = "des-cbc-crc des-cbc-md4 des-cbc-md5 des-cbc-raw des3-cbc-raw des-hmac-sha1 arcfour-hmac-exp rc4-hmac-exp arcfour-hmac-md5-exp des" + +// New creates a new config struct instance. +func New() *Config { + d := make(DomainRealm) + return &Config{ + LibDefaults: newLibDefaults(), + DomainRealm: d, + } +} + +// LibDefaults represents the [libdefaults] section of the configuration. +type LibDefaults struct { + AllowWeakCrypto bool //default false + // ap_req_checksum_type int //unlikely to support this + Canonicalize bool //default false + CCacheType int //default is 4. unlikely to implement older + Clockskew time.Duration //max allowed skew in seconds, default 300 + //Default_ccache_name string // default /tmp/krb5cc_%{uid} //Not implementing as will hold in memory + DefaultClientKeytabName string //default /usr/local/var/krb5/user/%{euid}/client.keytab + DefaultKeytabName string //default /etc/krb5.keytab + DefaultRealm string + DefaultTGSEnctypes []string //default aes256-cts-hmac-sha1-96 aes128-cts-hmac-sha1-96 des3-cbc-sha1 arcfour-hmac-md5 camellia256-cts-cmac camellia128-cts-cmac des-cbc-crc des-cbc-md5 des-cbc-md4 + DefaultTktEnctypes []string //default aes256-cts-hmac-sha1-96 aes128-cts-hmac-sha1-96 des3-cbc-sha1 arcfour-hmac-md5 camellia256-cts-cmac camellia128-cts-cmac des-cbc-crc des-cbc-md5 des-cbc-md4 + DefaultTGSEnctypeIDs []int32 //default aes256-cts-hmac-sha1-96 aes128-cts-hmac-sha1-96 des3-cbc-sha1 arcfour-hmac-md5 camellia256-cts-cmac camellia128-cts-cmac des-cbc-crc des-cbc-md5 des-cbc-md4 + DefaultTktEnctypeIDs []int32 //default aes256-cts-hmac-sha1-96 aes128-cts-hmac-sha1-96 des3-cbc-sha1 arcfour-hmac-md5 camellia256-cts-cmac camellia128-cts-cmac des-cbc-crc des-cbc-md5 des-cbc-md4 + DNSCanonicalizeHostname bool //default true + DNSLookupKDC bool //default false + DNSLookupRealm bool + ExtraAddresses []net.IP //Not implementing yet + Forwardable bool //default false + IgnoreAcceptorHostname bool //default false + K5LoginAuthoritative bool //default false + K5LoginDirectory string //default user's home directory. Must be owned by the user or root + KDCDefaultOptions asn1.BitString //default 0x00000010 (KDC_OPT_RENEWABLE_OK) + KDCTimeSync int //default 1 + //kdc_req_checksum_type int //unlikely to implement as for very old KDCs + NoAddresses bool //default true + PermittedEnctypes []string //default aes256-cts-hmac-sha1-96 aes128-cts-hmac-sha1-96 des3-cbc-sha1 arcfour-hmac-md5 camellia256-cts-cmac camellia128-cts-cmac des-cbc-crc des-cbc-md5 des-cbc-md4 + PermittedEnctypeIDs []int32 + //plugin_base_dir string //not supporting plugins + PreferredPreauthTypes []int //default “17, 16, 15, 14â€, which forces libkrb5 to attempt to use PKINIT if it is supported + Proxiable bool //default false + RDNS bool //default true + RealmTryDomains int //default -1 + RenewLifetime time.Duration //default 0 + SafeChecksumType int //default 8 + TicketLifetime time.Duration //default 1 day + UDPPreferenceLimit int // 1 means to always use tcp. MIT krb5 has a default value of 1465, and it prevents user setting more than 32700. + VerifyAPReqNofail bool //default false +} + +// Create a new LibDefaults struct. +func newLibDefaults() LibDefaults { + uid := "0" + var hdir string + usr, _ := user.Current() + if usr != nil { + uid = usr.Uid + hdir = usr.HomeDir + } + opts := asn1.BitString{} + opts.Bytes, _ = hex.DecodeString("00000010") + opts.BitLength = len(opts.Bytes) * 8 + return LibDefaults{ + CCacheType: 4, + Clockskew: time.Duration(300) * time.Second, + DefaultClientKeytabName: fmt.Sprintf("/usr/local/var/krb5/user/%s/client.keytab", uid), + DefaultKeytabName: "/etc/krb5.keytab", + DefaultTGSEnctypes: []string{"aes256-cts-hmac-sha1-96", "aes128-cts-hmac-sha1-96", "des3-cbc-sha1", "arcfour-hmac-md5", "camellia256-cts-cmac", "camellia128-cts-cmac", "des-cbc-crc", "des-cbc-md5", "des-cbc-md4"}, + DefaultTktEnctypes: []string{"aes256-cts-hmac-sha1-96", "aes128-cts-hmac-sha1-96", "des3-cbc-sha1", "arcfour-hmac-md5", "camellia256-cts-cmac", "camellia128-cts-cmac", "des-cbc-crc", "des-cbc-md5", "des-cbc-md4"}, + DNSCanonicalizeHostname: true, + K5LoginDirectory: hdir, + KDCDefaultOptions: opts, + KDCTimeSync: 1, + NoAddresses: true, + PermittedEnctypes: []string{"aes256-cts-hmac-sha1-96", "aes128-cts-hmac-sha1-96", "des3-cbc-sha1", "arcfour-hmac-md5", "camellia256-cts-cmac", "camellia128-cts-cmac", "des-cbc-crc", "des-cbc-md5", "des-cbc-md4"}, + RDNS: true, + RealmTryDomains: -1, + SafeChecksumType: 8, + TicketLifetime: time.Duration(24) * time.Hour, + UDPPreferenceLimit: 1465, + PreferredPreauthTypes: []int{17, 16, 15, 14}, + } +} + +// Parse the lines of the [libdefaults] section of the configuration into the LibDefaults struct. +func (l *LibDefaults) parseLines(lines []string) error { + for _, line := range lines { + //Remove comments after the values + if idx := strings.IndexAny(line, "#;"); idx != -1 { + line = line[:idx] + } + line = strings.TrimSpace(line) + if line == "" { + continue + } + if !strings.Contains(line, "=") { + return InvalidErrorf("libdefaults section line (%s)", line) + } + + p := strings.Split(line, "=") + key := strings.TrimSpace(strings.ToLower(p[0])) + switch key { + case "allow_weak_crypto": + v, err := parseBoolean(p[1]) + if err != nil { + return InvalidErrorf("libdefaults section line (%s): %v", line, err) + } + l.AllowWeakCrypto = v + case "canonicalize": + v, err := parseBoolean(p[1]) + if err != nil { + return InvalidErrorf("libdefaults section line (%s): %v", line, err) + } + l.Canonicalize = v + case "ccache_type": + p[1] = strings.TrimSpace(p[1]) + v, err := strconv.ParseUint(p[1], 10, 32) + if err != nil || v < 0 || v > 4 { + return InvalidErrorf("libdefaults section line (%s)", line) + } + l.CCacheType = int(v) + case "clockskew": + d, err := parseDuration(p[1]) + if err != nil { + return InvalidErrorf("libdefaults section line (%s): %v", line, err) + } + l.Clockskew = d + case "default_client_keytab_name": + l.DefaultClientKeytabName = strings.TrimSpace(p[1]) + case "default_keytab_name": + l.DefaultKeytabName = strings.TrimSpace(p[1]) + case "default_realm": + l.DefaultRealm = strings.TrimSpace(p[1]) + case "default_tgs_enctypes": + l.DefaultTGSEnctypes = strings.Fields(p[1]) + case "default_tkt_enctypes": + l.DefaultTktEnctypes = strings.Fields(p[1]) + case "dns_canonicalize_hostname": + v, err := parseBoolean(p[1]) + if err != nil { + return InvalidErrorf("libdefaults section line (%s): %v", line, err) + } + l.DNSCanonicalizeHostname = v + case "dns_lookup_kdc": + v, err := parseBoolean(p[1]) + if err != nil { + return InvalidErrorf("libdefaults section line (%s): %v", line, err) + } + l.DNSLookupKDC = v + case "dns_lookup_realm": + v, err := parseBoolean(p[1]) + if err != nil { + return InvalidErrorf("libdefaults section line (%s): %v", line, err) + } + l.DNSLookupRealm = v + case "extra_addresses": + ipStr := strings.TrimSpace(p[1]) + for _, ip := range strings.Split(ipStr, ",") { + if eip := net.ParseIP(ip); eip != nil { + l.ExtraAddresses = append(l.ExtraAddresses, eip) + } + } + case "forwardable": + v, err := parseBoolean(p[1]) + if err != nil { + return InvalidErrorf("libdefaults section line (%s): %v", line, err) + } + l.Forwardable = v + case "ignore_acceptor_hostname": + v, err := parseBoolean(p[1]) + if err != nil { + return InvalidErrorf("libdefaults section line (%s): %v", line, err) + } + l.IgnoreAcceptorHostname = v + case "k5login_authoritative": + v, err := parseBoolean(p[1]) + if err != nil { + return InvalidErrorf("libdefaults section line (%s): %v", line, err) + } + l.K5LoginAuthoritative = v + case "k5login_directory": + l.K5LoginDirectory = strings.TrimSpace(p[1]) + case "kdc_default_options": + v := strings.TrimSpace(p[1]) + v = strings.Replace(v, "0x", "", -1) + b, err := hex.DecodeString(v) + if err != nil { + return InvalidErrorf("libdefaults section line (%s): %v", line, err) + } + l.KDCDefaultOptions.Bytes = b + l.KDCDefaultOptions.BitLength = len(b) * 8 + case "kdc_timesync": + p[1] = strings.TrimSpace(p[1]) + v, err := strconv.ParseInt(p[1], 10, 32) + if err != nil || v < 0 { + return InvalidErrorf("libdefaults section line (%s)", line) + } + l.KDCTimeSync = int(v) + case "noaddresses": + v, err := parseBoolean(p[1]) + if err != nil { + return InvalidErrorf("libdefaults section line (%s): %v", line, err) + } + l.NoAddresses = v + case "permitted_enctypes": + l.PermittedEnctypes = strings.Fields(p[1]) + case "preferred_preauth_types": + p[1] = strings.TrimSpace(p[1]) + t := strings.Split(p[1], ",") + var v []int + for _, s := range t { + i, err := strconv.ParseInt(s, 10, 32) + if err != nil { + return InvalidErrorf("libdefaults section line (%s): %v", line, err) + } + v = append(v, int(i)) + } + l.PreferredPreauthTypes = v + case "proxiable": + v, err := parseBoolean(p[1]) + if err != nil { + return InvalidErrorf("libdefaults section line (%s): %v", line, err) + } + l.Proxiable = v + case "rdns": + v, err := parseBoolean(p[1]) + if err != nil { + return InvalidErrorf("libdefaults section line (%s): %v", line, err) + } + l.RDNS = v + case "realm_try_domains": + p[1] = strings.TrimSpace(p[1]) + v, err := strconv.ParseInt(p[1], 10, 32) + if err != nil || v < -1 { + return InvalidErrorf("libdefaults section line (%s)", line) + } + l.RealmTryDomains = int(v) + case "renew_lifetime": + d, err := parseDuration(p[1]) + if err != nil { + return InvalidErrorf("libdefaults section line (%s): %v", line, err) + } + l.RenewLifetime = d + case "safe_checksum_type": + p[1] = strings.TrimSpace(p[1]) + v, err := strconv.ParseInt(p[1], 10, 32) + if err != nil || v < 0 { + return InvalidErrorf("libdefaults section line (%s)", line) + } + l.SafeChecksumType = int(v) + case "ticket_lifetime": + d, err := parseDuration(p[1]) + if err != nil { + return InvalidErrorf("libdefaults section line (%s): %v", line, err) + } + l.TicketLifetime = d + case "udp_preference_limit": + p[1] = strings.TrimSpace(p[1]) + v, err := strconv.ParseUint(p[1], 10, 32) + if err != nil || v > 32700 { + return InvalidErrorf("libdefaults section line (%s)", line) + } + l.UDPPreferenceLimit = int(v) + case "verify_ap_req_nofail": + v, err := parseBoolean(p[1]) + if err != nil { + return InvalidErrorf("libdefaults section line (%s): %v", line, err) + } + l.VerifyAPReqNofail = v + } + } + l.DefaultTGSEnctypeIDs = parseETypes(l.DefaultTGSEnctypes, l.AllowWeakCrypto) + l.DefaultTktEnctypeIDs = parseETypes(l.DefaultTktEnctypes, l.AllowWeakCrypto) + l.PermittedEnctypeIDs = parseETypes(l.PermittedEnctypes, l.AllowWeakCrypto) + return nil +} + +// Realm represents an entry in the [realms] section of the configuration. +type Realm struct { + Realm string + AdminServer []string + //auth_to_local //Not implementing for now + //auth_to_local_names //Not implementing for now + DefaultDomain string + KDC []string + KPasswdServer []string //default admin_server:464 + MasterKDC []string +} + +// Parse the lines of a [realms] entry into the Realm struct. +func (r *Realm) parseLines(name string, lines []string) (err error) { + r.Realm = name + var adminServerFinal bool + var KDCFinal bool + var kpasswdServerFinal bool + var masterKDCFinal bool + var ignore bool + var c int // counts the depth of blocks within brackets { } + for _, line := range lines { + if ignore && c > 0 && !strings.Contains(line, "{") && !strings.Contains(line, "}") { + continue + } + //Remove comments after the values + if idx := strings.IndexAny(line, "#;"); idx != -1 { + line = line[:idx] + } + line = strings.TrimSpace(line) + if line == "" { + continue + } + if !strings.Contains(line, "=") && !strings.Contains(line, "}") { + return InvalidErrorf("realms section line (%s)", line) + } + if strings.Contains(line, "v4_") { + ignore = true + err = UnsupportedDirective{"v4 configurations are not supported"} + } + if strings.Contains(line, "{") { + c++ + if ignore { + continue + } + } + if strings.Contains(line, "}") { + c-- + if c < 0 { + return InvalidErrorf("unpaired curly brackets") + } + if ignore { + if c < 1 { + c = 0 + ignore = false + } + continue + } + } + + p := strings.Split(line, "=") + key := strings.TrimSpace(strings.ToLower(p[0])) + v := strings.TrimSpace(p[1]) + switch key { + case "admin_server": + appendUntilFinal(&r.AdminServer, v, &adminServerFinal) + case "default_domain": + r.DefaultDomain = v + case "kdc": + if !strings.Contains(v, ":") { + // No port number specified default to 88 + if strings.HasSuffix(v, `*`) { + v = strings.TrimSpace(strings.TrimSuffix(v, `*`)) + ":88*" + } else { + v = strings.TrimSpace(v) + ":88" + } + } + appendUntilFinal(&r.KDC, v, &KDCFinal) + case "kpasswd_server": + appendUntilFinal(&r.KPasswdServer, v, &kpasswdServerFinal) + case "master_kdc": + appendUntilFinal(&r.MasterKDC, v, &masterKDCFinal) + } + } + //default for Kpasswd_server = admin_server:464 + if len(r.KPasswdServer) < 1 { + for _, a := range r.AdminServer { + s := strings.Split(a, ":") + r.KPasswdServer = append(r.KPasswdServer, s[0]+":464") + } + } + return +} + +// Parse the lines of the [realms] section of the configuration into an slice of Realm structs. +func parseRealms(lines []string) (realms []Realm, err error) { + var name string + var start int + var c int + for i, l := range lines { + //Remove comments after the values + if idx := strings.IndexAny(l, "#;"); idx != -1 { + l = l[:idx] + } + l = strings.TrimSpace(l) + if l == "" { + continue + } + //if strings.Contains(l, "v4_") { + // return nil, errors.New("v4 configurations are not supported in Realms section") + //} + if strings.Contains(l, "{") { + c++ + if !strings.Contains(l, "=") { + return nil, fmt.Errorf("realm configuration line invalid: %s", l) + } + if c == 1 { + start = i + p := strings.Split(l, "=") + name = strings.TrimSpace(p[0]) + } + } + if strings.Contains(l, "}") { + if c < 1 { + // but not started a block!!! + return nil, errors.New("invalid Realms section in configuration") + } + c-- + if c == 0 { + var r Realm + e := r.parseLines(name, lines[start+1:i]) + if e != nil { + if _, ok := e.(UnsupportedDirective); !ok { + err = e + return + } + err = e + } + realms = append(realms, r) + } + } + } + return +} + +// DomainRealm maps the domains to realms representing the [domain_realm] section of the configuration. +type DomainRealm map[string]string + +// Parse the lines of the [domain_realm] section of the configuration and add to the mapping. +func (d *DomainRealm) parseLines(lines []string) error { + for _, line := range lines { + //Remove comments after the values + if idx := strings.IndexAny(line, "#;"); idx != -1 { + line = line[:idx] + } + if strings.TrimSpace(line) == "" { + continue + } + if !strings.Contains(line, "=") { + return InvalidErrorf("realm line (%s)", line) + } + p := strings.Split(line, "=") + domain := strings.TrimSpace(strings.ToLower(p[0])) + realm := strings.TrimSpace(p[1]) + d.addMapping(domain, realm) + } + return nil +} + +// Add a domain to realm mapping. +func (d *DomainRealm) addMapping(domain, realm string) { + (*d)[domain] = realm +} + +// Delete a domain to realm mapping. +func (d *DomainRealm) deleteMapping(domain, realm string) { + delete(*d, domain) +} + +// ResolveRealm resolves the kerberos realm for the specified domain name from the domain to realm mapping. +// The most specific mapping is returned. +func (c *Config) ResolveRealm(domainName string) string { + domainName = strings.TrimSuffix(domainName, ".") + + // Try to match the entire hostname first + if r, ok := c.DomainRealm[domainName]; ok { + return r + } + + // Try to match all DNS domain parts + periods := strings.Count(domainName, ".") + 1 + for i := 2; i <= periods; i++ { + z := strings.SplitN(domainName, ".", i) + if r, ok := c.DomainRealm["."+z[len(z)-1]]; ok { + return r + } + } + return c.LibDefaults.DefaultRealm +} + +// Load the KRB5 configuration from the specified file path. +func Load(cfgPath string) (*Config, error) { + fh, err := os.Open(cfgPath) + if err != nil { + return nil, errors.New("configuration file could not be opened: " + cfgPath + " " + err.Error()) + } + defer fh.Close() + scanner := bufio.NewScanner(fh) + return NewFromScanner(scanner) +} + +// NewFromString creates a new Config struct from a string. +func NewFromString(s string) (*Config, error) { + reader := strings.NewReader(s) + return NewFromReader(reader) +} + +// NewFromReader creates a new Config struct from an io.Reader. +func NewFromReader(r io.Reader) (*Config, error) { + scanner := bufio.NewScanner(r) + return NewFromScanner(scanner) +} + +// NewFromScanner creates a new Config struct from a bufio.Scanner. +func NewFromScanner(scanner *bufio.Scanner) (*Config, error) { + c := New() + var e error + sections := make(map[int]string) + var sectionLineNum []int + var lines []string + for scanner.Scan() { + // Skip comments and blank lines + if matched, _ := regexp.MatchString(`^\s*(#|;|\n)`, scanner.Text()); matched { + continue + } + if matched, _ := regexp.MatchString(`^\s*\[libdefaults\]\s*`, scanner.Text()); matched { + sections[len(lines)] = "libdefaults" + sectionLineNum = append(sectionLineNum, len(lines)) + continue + } + if matched, _ := regexp.MatchString(`^\s*\[realms\]\s*`, scanner.Text()); matched { + sections[len(lines)] = "realms" + sectionLineNum = append(sectionLineNum, len(lines)) + continue + } + if matched, _ := regexp.MatchString(`^\s*\[domain_realm\]\s*`, scanner.Text()); matched { + sections[len(lines)] = "domain_realm" + sectionLineNum = append(sectionLineNum, len(lines)) + continue + } + if matched, _ := regexp.MatchString(`^\s*\[.*\]\s*`, scanner.Text()); matched { + sections[len(lines)] = "unknown_section" + sectionLineNum = append(sectionLineNum, len(lines)) + continue + } + lines = append(lines, scanner.Text()) + } + for i, start := range sectionLineNum { + var end int + if i+1 >= len(sectionLineNum) { + end = len(lines) + } else { + end = sectionLineNum[i+1] + } + switch section := sections[start]; section { + case "libdefaults": + err := c.LibDefaults.parseLines(lines[start:end]) + if err != nil { + if _, ok := err.(UnsupportedDirective); !ok { + return nil, fmt.Errorf("error processing libdefaults section: %v", err) + } + e = err + } + case "realms": + realms, err := parseRealms(lines[start:end]) + if err != nil { + if _, ok := err.(UnsupportedDirective); !ok { + return nil, fmt.Errorf("error processing realms section: %v", err) + } + e = err + } + c.Realms = realms + case "domain_realm": + err := c.DomainRealm.parseLines(lines[start:end]) + if err != nil { + if _, ok := err.(UnsupportedDirective); !ok { + return nil, fmt.Errorf("error processing domaain_realm section: %v", err) + } + e = err + } + } + } + return c, e +} + +// Parse a space delimited list of ETypes into a list of EType numbers optionally filtering out weak ETypes. +func parseETypes(s []string, w bool) []int32 { + var eti []int32 + for _, et := range s { + if !w { + var weak bool + for _, wet := range strings.Fields(WeakETypeList) { + if et == wet { + weak = true + break + } + } + if weak { + continue + } + } + i := etypeID.EtypeSupported(et) + if i != 0 { + eti = append(eti, i) + } + } + return eti +} + +// Parse a time duration string in the configuration to a golang time.Duration. +func parseDuration(s string) (time.Duration, error) { + s = strings.Replace(strings.TrimSpace(s), " ", "", -1) + + // handle Nd[NmNs] + if strings.Contains(s, "d") { + ds := strings.SplitN(s, "d", 2) + dn, err := strconv.ParseUint(ds[0], 10, 32) + if err != nil { + return time.Duration(0), errors.New("invalid time duration") + } + d := time.Duration(dn*24) * time.Hour + if ds[1] != "" { + dp, err := time.ParseDuration(ds[1]) + if err != nil { + return time.Duration(0), errors.New("invalid time duration") + } + d = d + dp + } + return d, nil + } + + // handle Nm[Ns] + d, err := time.ParseDuration(s) + if err == nil { + return d, nil + } + + // handle N + v, err := strconv.ParseUint(s, 10, 32) + if err == nil && v > 0 { + return time.Duration(v) * time.Second, nil + } + + // handle h:m[:s] + if strings.Contains(s, ":") { + t := strings.Split(s, ":") + if 2 > len(t) || len(t) > 3 { + return time.Duration(0), errors.New("invalid time duration value") + } + var i []int + for _, n := range t { + j, err := strconv.ParseInt(n, 10, 16) + if err != nil { + return time.Duration(0), errors.New("invalid time duration value") + } + i = append(i, int(j)) + } + d := time.Duration(i[0])*time.Hour + time.Duration(i[1])*time.Minute + if len(i) == 3 { + d = d + time.Duration(i[2])*time.Second + } + return d, nil + } + return time.Duration(0), errors.New("invalid time duration value") +} + +// Parse possible boolean values to golang bool. +func parseBoolean(s string) (bool, error) { + s = strings.TrimSpace(s) + v, err := strconv.ParseBool(s) + if err == nil { + return v, nil + } + switch strings.ToLower(s) { + case "yes": + return true, nil + case "y": + return true, nil + case "no": + return false, nil + case "n": + return false, nil + } + return false, errors.New("invalid boolean value") +} + +// Parse array of strings but stop if an asterisk is placed at the end of a line. +func appendUntilFinal(s *[]string, value string, final *bool) { + if *final { + return + } + if last := len(value) - 1; last >= 0 && value[last] == '*' { + *final = true + value = value[:len(value)-1] + } + *s = append(*s, value) +} + +// JSON return details of the config in a JSON format. +func (c *Config) JSON() (string, error) { + b, err := json.MarshalIndent(c, "", " ") + if err != nil { + return "", err + } + return string(b), nil +} diff --git a/github.com/jcmturner/gokrb5/v8/credentials/ccache.go b/github.com/jcmturner/gokrb5/v8/credentials/ccache.go new file mode 100644 index 0000000000..c3b35c77a1 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/credentials/ccache.go @@ -0,0 +1,333 @@ +package credentials + +import ( + "bytes" + "encoding/binary" + "errors" + "io/ioutil" + "strings" + "time" + "unsafe" + + "github.com/jcmturner/gofork/encoding/asn1" + "github.com/jcmturner/gokrb5/v8/types" +) + +const ( + headerFieldTagKDCOffset = 1 +) + +// CCache is the file credentials cache as define here: https://web.mit.edu/kerberos/krb5-latest/doc/formats/ccache_file_format.html +type CCache struct { + Version uint8 + Header header + DefaultPrincipal principal + Credentials []*Credential + Path string +} + +type header struct { + length uint16 + fields []headerField +} + +type headerField struct { + tag uint16 + length uint16 + value []byte +} + +// Credential cache entry principal struct. +type principal struct { + Realm string + PrincipalName types.PrincipalName +} + +// Credential holds a Kerberos client's ccache credential information. +type Credential struct { + Client principal + Server principal + Key types.EncryptionKey + AuthTime time.Time + StartTime time.Time + EndTime time.Time + RenewTill time.Time + IsSKey bool + TicketFlags asn1.BitString + Addresses []types.HostAddress + AuthData []types.AuthorizationDataEntry + Ticket []byte + SecondTicket []byte +} + +// LoadCCache loads a credential cache file into a CCache type. +func LoadCCache(cpath string) (*CCache, error) { + c := new(CCache) + b, err := ioutil.ReadFile(cpath) + if err != nil { + return c, err + } + err = c.Unmarshal(b) + return c, err +} + +// Unmarshal a byte slice of credential cache data into CCache type. +func (c *CCache) Unmarshal(b []byte) error { + p := 0 + //The first byte of the file always has the value 5 + if int8(b[p]) != 5 { + return errors.New("Invalid credential cache data. First byte does not equal 5") + } + p++ + //Get credential cache version + //The second byte contains the version number (1 to 4) + c.Version = b[p] + if c.Version < 1 || c.Version > 4 { + return errors.New("Invalid credential cache data. Keytab version is not within 1 to 4") + } + p++ + //Version 1 or 2 of the file format uses native byte order for integer representations. Versions 3 & 4 always uses big-endian byte order + var endian binary.ByteOrder + endian = binary.BigEndian + if (c.Version == 1 || c.Version == 2) && isNativeEndianLittle() { + endian = binary.LittleEndian + } + if c.Version == 4 { + err := parseHeader(b, &p, c, &endian) + if err != nil { + return err + } + } + c.DefaultPrincipal = parsePrincipal(b, &p, c, &endian) + for p < len(b) { + cred, err := parseCredential(b, &p, c, &endian) + if err != nil { + return err + } + c.Credentials = append(c.Credentials, cred) + } + return nil +} + +func parseHeader(b []byte, p *int, c *CCache, e *binary.ByteOrder) error { + if c.Version != 4 { + return errors.New("Credentials cache version is not 4 so there is no header to parse.") + } + h := header{} + h.length = uint16(readInt16(b, p, e)) + for *p <= int(h.length) { + f := headerField{} + f.tag = uint16(readInt16(b, p, e)) + f.length = uint16(readInt16(b, p, e)) + f.value = b[*p : *p+int(f.length)] + *p += int(f.length) + if !f.valid() { + return errors.New("Invalid credential cache header found") + } + h.fields = append(h.fields, f) + } + c.Header = h + return nil +} + +// Parse the Keytab bytes of a principal into a Keytab entry's principal. +func parsePrincipal(b []byte, p *int, c *CCache, e *binary.ByteOrder) (princ principal) { + if c.Version != 1 { + //Name Type is omitted in version 1 + princ.PrincipalName.NameType = readInt32(b, p, e) + } + nc := int(readInt32(b, p, e)) + if c.Version == 1 { + //In version 1 the number of components includes the realm. Minus 1 to make consistent with version 2 + nc-- + } + lenRealm := readInt32(b, p, e) + princ.Realm = string(readBytes(b, p, int(lenRealm), e)) + for i := 0; i < nc; i++ { + l := readInt32(b, p, e) + princ.PrincipalName.NameString = append(princ.PrincipalName.NameString, string(readBytes(b, p, int(l), e))) + } + return princ +} + +func parseCredential(b []byte, p *int, c *CCache, e *binary.ByteOrder) (cred *Credential, err error) { + cred = new(Credential) + cred.Client = parsePrincipal(b, p, c, e) + cred.Server = parsePrincipal(b, p, c, e) + key := types.EncryptionKey{} + key.KeyType = int32(readInt16(b, p, e)) + if c.Version == 3 { + //repeated twice in version 3 + key.KeyType = int32(readInt16(b, p, e)) + } + key.KeyValue = readData(b, p, e) + cred.Key = key + cred.AuthTime = readTimestamp(b, p, e) + cred.StartTime = readTimestamp(b, p, e) + cred.EndTime = readTimestamp(b, p, e) + cred.RenewTill = readTimestamp(b, p, e) + if ik := readInt8(b, p, e); ik == 0 { + cred.IsSKey = false + } else { + cred.IsSKey = true + } + cred.TicketFlags = types.NewKrbFlags() + cred.TicketFlags.Bytes = readBytes(b, p, 4, e) + l := int(readInt32(b, p, e)) + cred.Addresses = make([]types.HostAddress, l, l) + for i := range cred.Addresses { + cred.Addresses[i] = readAddress(b, p, e) + } + l = int(readInt32(b, p, e)) + cred.AuthData = make([]types.AuthorizationDataEntry, l, l) + for i := range cred.AuthData { + cred.AuthData[i] = readAuthDataEntry(b, p, e) + } + cred.Ticket = readData(b, p, e) + cred.SecondTicket = readData(b, p, e) + return +} + +// GetClientPrincipalName returns a PrincipalName type for the client the credentials cache is for. +func (c *CCache) GetClientPrincipalName() types.PrincipalName { + return c.DefaultPrincipal.PrincipalName +} + +// GetClientRealm returns the reals of the client the credentials cache is for. +func (c *CCache) GetClientRealm() string { + return c.DefaultPrincipal.Realm +} + +// GetClientCredentials returns a Credentials object representing the client of the credentials cache. +func (c *CCache) GetClientCredentials() *Credentials { + return &Credentials{ + username: c.DefaultPrincipal.PrincipalName.PrincipalNameString(), + realm: c.GetClientRealm(), + cname: c.DefaultPrincipal.PrincipalName, + } +} + +// Contains tests if the cache contains a credential for the provided server PrincipalName +func (c *CCache) Contains(p types.PrincipalName) bool { + for _, cred := range c.Credentials { + if cred.Server.PrincipalName.Equal(p) { + return true + } + } + return false +} + +// GetEntry returns a specific credential for the PrincipalName provided. +func (c *CCache) GetEntry(p types.PrincipalName) (*Credential, bool) { + cred := new(Credential) + var found bool + for i := range c.Credentials { + if c.Credentials[i].Server.PrincipalName.Equal(p) { + cred = c.Credentials[i] + found = true + break + } + } + if !found { + return cred, false + } + return cred, true +} + +// GetEntries filters out configuration entries an returns a slice of credentials. +func (c *CCache) GetEntries() []*Credential { + creds := make([]*Credential, 0) + for _, cred := range c.Credentials { + // Filter out configuration entries + if strings.HasPrefix(cred.Server.Realm, "X-CACHECONF") { + continue + } + creds = append(creds, cred) + } + return creds +} + +func (h *headerField) valid() bool { + // See https://web.mit.edu/kerberos/krb5-latest/doc/formats/ccache_file_format.html - Header format + switch h.tag { + case headerFieldTagKDCOffset: + if h.length != 8 || len(h.value) != 8 { + return false + } + return true + } + return false +} + +func readData(b []byte, p *int, e *binary.ByteOrder) []byte { + l := readInt32(b, p, e) + return readBytes(b, p, int(l), e) +} + +func readAddress(b []byte, p *int, e *binary.ByteOrder) types.HostAddress { + a := types.HostAddress{} + a.AddrType = int32(readInt16(b, p, e)) + a.Address = readData(b, p, e) + return a +} + +func readAuthDataEntry(b []byte, p *int, e *binary.ByteOrder) types.AuthorizationDataEntry { + a := types.AuthorizationDataEntry{} + a.ADType = int32(readInt16(b, p, e)) + a.ADData = readData(b, p, e) + return a +} + +// Read bytes representing a timestamp. +func readTimestamp(b []byte, p *int, e *binary.ByteOrder) time.Time { + return time.Unix(int64(readInt32(b, p, e)), 0) +} + +// Read bytes representing an eight bit integer. +func readInt8(b []byte, p *int, e *binary.ByteOrder) (i int8) { + buf := bytes.NewBuffer(b[*p : *p+1]) + binary.Read(buf, *e, &i) + *p++ + return +} + +// Read bytes representing a sixteen bit integer. +func readInt16(b []byte, p *int, e *binary.ByteOrder) (i int16) { + buf := bytes.NewBuffer(b[*p : *p+2]) + binary.Read(buf, *e, &i) + *p += 2 + return +} + +// Read bytes representing a thirty two bit integer. +func readInt32(b []byte, p *int, e *binary.ByteOrder) (i int32) { + buf := bytes.NewBuffer(b[*p : *p+4]) + binary.Read(buf, *e, &i) + *p += 4 + return +} + +func readBytes(b []byte, p *int, s int, e *binary.ByteOrder) []byte { + buf := bytes.NewBuffer(b[*p : *p+s]) + r := make([]byte, s) + binary.Read(buf, *e, &r) + *p += s + return r +} + +func isNativeEndianLittle() bool { + var x = 0x012345678 + var p = unsafe.Pointer(&x) + var bp = (*[4]byte)(p) + + var endian bool + if 0x01 == bp[0] { + endian = false + } else if (0x78 & 0xff) == (bp[0] & 0xff) { + endian = true + } else { + // Default to big endian + endian = false + } + return endian +} diff --git a/github.com/jcmturner/gokrb5/v8/credentials/credentials.go b/github.com/jcmturner/gokrb5/v8/credentials/credentials.go new file mode 100644 index 0000000000..bddbc7e3e3 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/credentials/credentials.go @@ -0,0 +1,405 @@ +// Package credentials provides credentials management for Kerberos 5 authentication. +package credentials + +import ( + "bytes" + "encoding/gob" + "encoding/json" + "time" + + "github.com/hashicorp/go-uuid" + "github.com/jcmturner/gokrb5/v8/iana/nametype" + "github.com/jcmturner/gokrb5/v8/keytab" + "github.com/jcmturner/gokrb5/v8/types" +) + +const ( + // AttributeKeyADCredentials assigned number for AD credentials. + AttributeKeyADCredentials = "gokrb5AttributeKeyADCredentials" +) + +// Credentials struct for a user. +// Contains either a keytab, password or both. +// Keytabs are used over passwords if both are defined. +type Credentials struct { + username string + displayName string + realm string + cname types.PrincipalName + keytab *keytab.Keytab + password string + attributes map[string]interface{} + validUntil time.Time + authenticated bool + human bool + authTime time.Time + groupMembership map[string]bool + sessionID string +} + +// marshalCredentials is used to enable marshaling and unmarshaling of credentials +// without having exported fields on the Credentials struct +type marshalCredentials struct { + Username string + DisplayName string + Realm string + CName types.PrincipalName `json:"-"` + Keytab bool + Password bool + Attributes map[string]interface{} `json:"-"` + ValidUntil time.Time + Authenticated bool + Human bool + AuthTime time.Time + GroupMembership map[string]bool `json:"-"` + SessionID string +} + +// ADCredentials contains information obtained from the PAC. +type ADCredentials struct { + EffectiveName string + FullName string + UserID int + PrimaryGroupID int + LogOnTime time.Time + LogOffTime time.Time + PasswordLastSet time.Time + GroupMembershipSIDs []string + LogonDomainName string + LogonDomainID string + LogonServer string +} + +// New creates a new Credentials instance. +func New(username string, realm string) *Credentials { + uid, err := uuid.GenerateUUID() + if err != nil { + uid = "00unique-sess-ions-uuid-unavailable0" + } + return &Credentials{ + username: username, + displayName: username, + realm: realm, + cname: types.NewPrincipalName(nametype.KRB_NT_PRINCIPAL, username), + keytab: keytab.New(), + attributes: make(map[string]interface{}), + groupMembership: make(map[string]bool), + sessionID: uid, + human: true, + } +} + +// NewFromPrincipalName creates a new Credentials instance with the user details provides as a PrincipalName type. +func NewFromPrincipalName(cname types.PrincipalName, realm string) *Credentials { + c := New(cname.PrincipalNameString(), realm) + c.cname = cname + return c +} + +// WithKeytab sets the Keytab in the Credentials struct. +func (c *Credentials) WithKeytab(kt *keytab.Keytab) *Credentials { + c.keytab = kt + c.password = "" + return c +} + +// Keytab returns the credential's Keytab. +func (c *Credentials) Keytab() *keytab.Keytab { + return c.keytab +} + +// HasKeytab queries if the Credentials has a keytab defined. +func (c *Credentials) HasKeytab() bool { + if c.keytab != nil && len(c.keytab.Entries) > 0 { + return true + } + return false +} + +// WithPassword sets the password in the Credentials struct. +func (c *Credentials) WithPassword(password string) *Credentials { + c.password = password + c.keytab = keytab.New() // clear any keytab + return c +} + +// Password returns the credential's password. +func (c *Credentials) Password() string { + return c.password +} + +// HasPassword queries if the Credentials has a password defined. +func (c *Credentials) HasPassword() bool { + if c.password != "" { + return true + } + return false +} + +// SetValidUntil sets the expiry time of the credentials +func (c *Credentials) SetValidUntil(t time.Time) { + c.validUntil = t +} + +// SetADCredentials adds ADCredentials attributes to the credentials +func (c *Credentials) SetADCredentials(a ADCredentials) { + c.SetAttribute(AttributeKeyADCredentials, a) + if a.FullName != "" { + c.SetDisplayName(a.FullName) + } + if a.EffectiveName != "" { + c.SetUserName(a.EffectiveName) + } + for i := range a.GroupMembershipSIDs { + c.AddAuthzAttribute(a.GroupMembershipSIDs[i]) + } +} + +// GetADCredentials returns ADCredentials attributes sorted in the credential +func (c *Credentials) GetADCredentials() ADCredentials { + if a, ok := c.attributes[AttributeKeyADCredentials].(ADCredentials); ok { + return a + } + return ADCredentials{} +} + +// Methods to implement goidentity.Identity interface + +// UserName returns the credential's username. +func (c *Credentials) UserName() string { + return c.username +} + +// SetUserName sets the username value on the credential. +func (c *Credentials) SetUserName(s string) { + c.username = s +} + +// CName returns the credential's client principal name. +func (c *Credentials) CName() types.PrincipalName { + return c.cname +} + +// SetCName sets the client principal name on the credential. +func (c *Credentials) SetCName(pn types.PrincipalName) { + c.cname = pn +} + +// Domain returns the credential's domain. +func (c *Credentials) Domain() string { + return c.realm +} + +// SetDomain sets the domain value on the credential. +func (c *Credentials) SetDomain(s string) { + c.realm = s +} + +// Realm returns the credential's realm. Same as the domain. +func (c *Credentials) Realm() string { + return c.Domain() +} + +// SetRealm sets the realm value on the credential. Same as the domain +func (c *Credentials) SetRealm(s string) { + c.SetDomain(s) +} + +// DisplayName returns the credential's display name. +func (c *Credentials) DisplayName() string { + return c.displayName +} + +// SetDisplayName sets the display name value on the credential. +func (c *Credentials) SetDisplayName(s string) { + c.displayName = s +} + +// Human returns if the credential represents a human or not. +func (c *Credentials) Human() bool { + return c.human +} + +// SetHuman sets the credential as human. +func (c *Credentials) SetHuman(b bool) { + c.human = b +} + +// AuthTime returns the time the credential was authenticated. +func (c *Credentials) AuthTime() time.Time { + return c.authTime +} + +// SetAuthTime sets the time the credential was authenticated. +func (c *Credentials) SetAuthTime(t time.Time) { + c.authTime = t +} + +// AuthzAttributes returns the credentials authorizing attributes. +func (c *Credentials) AuthzAttributes() []string { + s := make([]string, len(c.groupMembership)) + i := 0 + for a := range c.groupMembership { + s[i] = a + i++ + } + return s +} + +// Authenticated indicates if the credential has been successfully authenticated or not. +func (c *Credentials) Authenticated() bool { + return c.authenticated +} + +// SetAuthenticated sets the credential as having been successfully authenticated. +func (c *Credentials) SetAuthenticated(b bool) { + c.authenticated = b +} + +// AddAuthzAttribute adds an authorization attribute to the credential. +func (c *Credentials) AddAuthzAttribute(a string) { + c.groupMembership[a] = true +} + +// RemoveAuthzAttribute removes an authorization attribute from the credential. +func (c *Credentials) RemoveAuthzAttribute(a string) { + if _, ok := c.groupMembership[a]; !ok { + return + } + delete(c.groupMembership, a) +} + +// EnableAuthzAttribute toggles an authorization attribute to an enabled state on the credential. +func (c *Credentials) EnableAuthzAttribute(a string) { + if enabled, ok := c.groupMembership[a]; ok && !enabled { + c.groupMembership[a] = true + } +} + +// DisableAuthzAttribute toggles an authorization attribute to a disabled state on the credential. +func (c *Credentials) DisableAuthzAttribute(a string) { + if enabled, ok := c.groupMembership[a]; ok && enabled { + c.groupMembership[a] = false + } +} + +// Authorized indicates if the credential has the specified authorizing attribute. +func (c *Credentials) Authorized(a string) bool { + if enabled, ok := c.groupMembership[a]; ok && enabled { + return true + } + return false +} + +// SessionID returns the credential's session ID. +func (c *Credentials) SessionID() string { + return c.sessionID +} + +// Expired indicates if the credential has expired. +func (c *Credentials) Expired() bool { + if !c.validUntil.IsZero() && time.Now().UTC().After(c.validUntil) { + return true + } + return false +} + +// ValidUntil returns the credential's valid until date +func (c *Credentials) ValidUntil() time.Time { + return c.validUntil +} + +// Attributes returns the Credentials' attributes map. +func (c *Credentials) Attributes() map[string]interface{} { + return c.attributes +} + +// SetAttribute sets the value of an attribute. +func (c *Credentials) SetAttribute(k string, v interface{}) { + c.attributes[k] = v +} + +// SetAttributes replaces the attributes map with the one provided. +func (c *Credentials) SetAttributes(a map[string]interface{}) { + c.attributes = a +} + +// RemoveAttribute deletes an attribute from the attribute map that has the key provided. +func (c *Credentials) RemoveAttribute(k string) { + delete(c.attributes, k) +} + +// Marshal the Credentials into a byte slice +func (c *Credentials) Marshal() ([]byte, error) { + gob.Register(map[string]interface{}{}) + gob.Register(ADCredentials{}) + buf := new(bytes.Buffer) + enc := gob.NewEncoder(buf) + mc := marshalCredentials{ + Username: c.username, + DisplayName: c.displayName, + Realm: c.realm, + CName: c.cname, + Keytab: c.HasKeytab(), + Password: c.HasPassword(), + Attributes: c.attributes, + ValidUntil: c.validUntil, + Authenticated: c.authenticated, + Human: c.human, + AuthTime: c.authTime, + GroupMembership: c.groupMembership, + SessionID: c.sessionID, + } + err := enc.Encode(&mc) + if err != nil { + return []byte{}, err + } + return buf.Bytes(), nil +} + +// Unmarshal a byte slice into Credentials +func (c *Credentials) Unmarshal(b []byte) error { + gob.Register(map[string]interface{}{}) + gob.Register(ADCredentials{}) + mc := new(marshalCredentials) + buf := bytes.NewBuffer(b) + dec := gob.NewDecoder(buf) + err := dec.Decode(mc) + if err != nil { + return err + } + c.username = mc.Username + c.displayName = mc.DisplayName + c.realm = mc.Realm + c.cname = mc.CName + c.attributes = mc.Attributes + c.validUntil = mc.ValidUntil + c.authenticated = mc.Authenticated + c.human = mc.Human + c.authTime = mc.AuthTime + c.groupMembership = mc.GroupMembership + c.sessionID = mc.SessionID + return nil +} + +// JSON return details of the Credentials in a JSON format. +func (c *Credentials) JSON() (string, error) { + mc := marshalCredentials{ + Username: c.username, + DisplayName: c.displayName, + Realm: c.realm, + CName: c.cname, + Keytab: c.HasKeytab(), + Password: c.HasPassword(), + ValidUntil: c.validUntil, + Authenticated: c.authenticated, + Human: c.human, + AuthTime: c.authTime, + SessionID: c.sessionID, + } + b, err := json.MarshalIndent(mc, "", " ") + if err != nil { + return "", err + } + return string(b), nil +} diff --git a/github.com/jcmturner/gokrb5/v8/crypto/aes128-cts-hmac-sha1-96.go b/github.com/jcmturner/gokrb5/v8/crypto/aes128-cts-hmac-sha1-96.go new file mode 100644 index 0000000000..dd8babd5df --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/crypto/aes128-cts-hmac-sha1-96.go @@ -0,0 +1,129 @@ +package crypto + +import ( + "crypto/aes" + "crypto/hmac" + "crypto/sha1" + "hash" + + "github.com/jcmturner/gokrb5/v8/crypto/common" + "github.com/jcmturner/gokrb5/v8/crypto/rfc3961" + "github.com/jcmturner/gokrb5/v8/crypto/rfc3962" + "github.com/jcmturner/gokrb5/v8/iana/chksumtype" + "github.com/jcmturner/gokrb5/v8/iana/etypeID" +) + +// RFC 3962 + +// Aes128CtsHmacSha96 implements Kerberos encryption type aes128-cts-hmac-sha1-96 +type Aes128CtsHmacSha96 struct { +} + +// GetETypeID returns the EType ID number. +func (e Aes128CtsHmacSha96) GetETypeID() int32 { + return etypeID.AES128_CTS_HMAC_SHA1_96 +} + +// GetHashID returns the checksum type ID number. +func (e Aes128CtsHmacSha96) GetHashID() int32 { + return chksumtype.HMAC_SHA1_96_AES128 +} + +// GetKeyByteSize returns the number of bytes for key of this etype. +func (e Aes128CtsHmacSha96) GetKeyByteSize() int { + return 128 / 8 +} + +// GetKeySeedBitLength returns the number of bits for the seed for key generation. +func (e Aes128CtsHmacSha96) GetKeySeedBitLength() int { + return e.GetKeyByteSize() * 8 +} + +// GetHashFunc returns the hash function for this etype. +func (e Aes128CtsHmacSha96) GetHashFunc() func() hash.Hash { + return sha1.New +} + +// GetMessageBlockByteSize returns the block size for the etype's messages. +func (e Aes128CtsHmacSha96) GetMessageBlockByteSize() int { + return 1 +} + +// GetDefaultStringToKeyParams returns the default key derivation parameters in string form. +func (e Aes128CtsHmacSha96) GetDefaultStringToKeyParams() string { + return "00001000" +} + +// GetConfounderByteSize returns the byte count for confounder to be used during cryptographic operations. +func (e Aes128CtsHmacSha96) GetConfounderByteSize() int { + return aes.BlockSize +} + +// GetHMACBitLength returns the bit count size of the integrity hash. +func (e Aes128CtsHmacSha96) GetHMACBitLength() int { + return 96 +} + +// GetCypherBlockBitLength returns the bit count size of the cypher block. +func (e Aes128CtsHmacSha96) GetCypherBlockBitLength() int { + return aes.BlockSize * 8 +} + +// StringToKey returns a key derived from the string provided. +func (e Aes128CtsHmacSha96) StringToKey(secret string, salt string, s2kparams string) ([]byte, error) { + return rfc3962.StringToKey(secret, salt, s2kparams, e) +} + +// RandomToKey returns a key from the bytes provided. +func (e Aes128CtsHmacSha96) RandomToKey(b []byte) []byte { + return rfc3961.RandomToKey(b) +} + +// EncryptData encrypts the data provided. +func (e Aes128CtsHmacSha96) EncryptData(key, data []byte) ([]byte, []byte, error) { + return rfc3962.EncryptData(key, data, e) +} + +// EncryptMessage encrypts the message provided and concatenates it with the integrity hash to create an encrypted message. +func (e Aes128CtsHmacSha96) EncryptMessage(key, message []byte, usage uint32) ([]byte, []byte, error) { + return rfc3962.EncryptMessage(key, message, usage, e) +} + +// DecryptData decrypts the data provided. +func (e Aes128CtsHmacSha96) DecryptData(key, data []byte) ([]byte, error) { + return rfc3962.DecryptData(key, data, e) +} + +// DecryptMessage decrypts the message provided and verifies the integrity of the message. +func (e Aes128CtsHmacSha96) DecryptMessage(key, ciphertext []byte, usage uint32) ([]byte, error) { + return rfc3962.DecryptMessage(key, ciphertext, usage, e) +} + +// DeriveKey derives a key from the protocol key based on the usage value. +func (e Aes128CtsHmacSha96) DeriveKey(protocolKey, usage []byte) ([]byte, error) { + return rfc3961.DeriveKey(protocolKey, usage, e) +} + +// DeriveRandom generates data needed for key generation. +func (e Aes128CtsHmacSha96) DeriveRandom(protocolKey, usage []byte) ([]byte, error) { + return rfc3961.DeriveRandom(protocolKey, usage, e) +} + +// VerifyIntegrity checks the integrity of the plaintext message. +func (e Aes128CtsHmacSha96) VerifyIntegrity(protocolKey, ct, pt []byte, usage uint32) bool { + return rfc3961.VerifyIntegrity(protocolKey, ct, pt, usage, e) +} + +// GetChecksumHash returns a keyed checksum hash of the bytes provided. +func (e Aes128CtsHmacSha96) GetChecksumHash(protocolKey, data []byte, usage uint32) ([]byte, error) { + return common.GetHash(data, protocolKey, common.GetUsageKc(usage), e) +} + +// VerifyChecksum compares the checksum of the message bytes is the same as the checksum provided. +func (e Aes128CtsHmacSha96) VerifyChecksum(protocolKey, data, chksum []byte, usage uint32) bool { + c, err := e.GetChecksumHash(protocolKey, data, usage) + if err != nil { + return false + } + return hmac.Equal(chksum, c) +} diff --git a/github.com/jcmturner/gokrb5/v8/crypto/aes128-cts-hmac-sha256-128.go b/github.com/jcmturner/gokrb5/v8/crypto/aes128-cts-hmac-sha256-128.go new file mode 100644 index 0000000000..b05af7d3d4 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/crypto/aes128-cts-hmac-sha256-128.go @@ -0,0 +1,132 @@ +package crypto + +import ( + "crypto/aes" + "crypto/hmac" + "crypto/sha256" + "hash" + + "github.com/jcmturner/gokrb5/v8/crypto/common" + "github.com/jcmturner/gokrb5/v8/crypto/rfc8009" + "github.com/jcmturner/gokrb5/v8/iana/chksumtype" + "github.com/jcmturner/gokrb5/v8/iana/etypeID" +) + +// RFC https://tools.ietf.org/html/rfc8009 + +// Aes128CtsHmacSha256128 implements Kerberos encryption type aes128-cts-hmac-sha256-128 +type Aes128CtsHmacSha256128 struct { +} + +// GetETypeID returns the EType ID number. +func (e Aes128CtsHmacSha256128) GetETypeID() int32 { + return etypeID.AES128_CTS_HMAC_SHA256_128 +} + +// GetHashID returns the checksum type ID number. +func (e Aes128CtsHmacSha256128) GetHashID() int32 { + return chksumtype.HMAC_SHA256_128_AES128 +} + +// GetKeyByteSize returns the number of bytes for key of this etype. +func (e Aes128CtsHmacSha256128) GetKeyByteSize() int { + return 128 / 8 +} + +// GetKeySeedBitLength returns the number of bits for the seed for key generation. +func (e Aes128CtsHmacSha256128) GetKeySeedBitLength() int { + return e.GetKeyByteSize() * 8 +} + +// GetHashFunc returns the hash function for this etype. +func (e Aes128CtsHmacSha256128) GetHashFunc() func() hash.Hash { + return sha256.New +} + +// GetMessageBlockByteSize returns the block size for the etype's messages. +func (e Aes128CtsHmacSha256128) GetMessageBlockByteSize() int { + return 1 +} + +// GetDefaultStringToKeyParams returns the default key derivation parameters in string form. +func (e Aes128CtsHmacSha256128) GetDefaultStringToKeyParams() string { + return "00008000" +} + +// GetConfounderByteSize returns the byte count for confounder to be used during cryptographic operations. +func (e Aes128CtsHmacSha256128) GetConfounderByteSize() int { + return aes.BlockSize +} + +// GetHMACBitLength returns the bit count size of the integrity hash. +func (e Aes128CtsHmacSha256128) GetHMACBitLength() int { + return 128 +} + +// GetCypherBlockBitLength returns the bit count size of the cypher block. +func (e Aes128CtsHmacSha256128) GetCypherBlockBitLength() int { + return aes.BlockSize * 8 +} + +// StringToKey returns a key derived from the string provided. +func (e Aes128CtsHmacSha256128) StringToKey(secret string, salt string, s2kparams string) ([]byte, error) { + saltp := rfc8009.GetSaltP(salt, "aes128-cts-hmac-sha256-128") + return rfc8009.StringToKey(secret, saltp, s2kparams, e) +} + +// RandomToKey returns a key from the bytes provided. +func (e Aes128CtsHmacSha256128) RandomToKey(b []byte) []byte { + return rfc8009.RandomToKey(b) +} + +// EncryptData encrypts the data provided. +func (e Aes128CtsHmacSha256128) EncryptData(key, data []byte) ([]byte, []byte, error) { + return rfc8009.EncryptData(key, data, e) +} + +// EncryptMessage encrypts the message provided and concatenates it with the integrity hash to create an encrypted message. +func (e Aes128CtsHmacSha256128) EncryptMessage(key, message []byte, usage uint32) ([]byte, []byte, error) { + return rfc8009.EncryptMessage(key, message, usage, e) +} + +// DecryptData decrypts the data provided. +func (e Aes128CtsHmacSha256128) DecryptData(key, data []byte) ([]byte, error) { + return rfc8009.DecryptData(key, data, e) +} + +// DecryptMessage decrypts the message provided and verifies the integrity of the message. +func (e Aes128CtsHmacSha256128) DecryptMessage(key, ciphertext []byte, usage uint32) ([]byte, error) { + return rfc8009.DecryptMessage(key, ciphertext, usage, e) +} + +// DeriveKey derives a key from the protocol key based on the usage value. +func (e Aes128CtsHmacSha256128) DeriveKey(protocolKey, usage []byte) ([]byte, error) { + return rfc8009.DeriveKey(protocolKey, usage, e), nil +} + +// DeriveRandom generates data needed for key generation. +func (e Aes128CtsHmacSha256128) DeriveRandom(protocolKey, usage []byte) ([]byte, error) { + return rfc8009.DeriveRandom(protocolKey, usage, e) +} + +// VerifyIntegrity checks the integrity of the ciphertext message. +// As the hash is calculated over the iv concatenated with the AES cipher output not the plaintext the pt value to this +// interface method is not use. Pass any []byte. +func (e Aes128CtsHmacSha256128) VerifyIntegrity(protocolKey, ct, pt []byte, usage uint32) bool { + // We don't need ib just there for the interface + return rfc8009.VerifyIntegrity(protocolKey, ct, usage, e) +} + +// GetChecksumHash returns a keyed checksum hash of the bytes provided. +func (e Aes128CtsHmacSha256128) GetChecksumHash(protocolKey, data []byte, usage uint32) ([]byte, error) { + return common.GetHash(data, protocolKey, common.GetUsageKc(usage), e) +} + +// VerifyChecksum compares the checksum of the message bytes is the same as the checksum provided. +func (e Aes128CtsHmacSha256128) VerifyChecksum(protocolKey, data, chksum []byte, usage uint32) bool { + c, err := e.GetChecksumHash(protocolKey, data, usage) + if err != nil { + return false + } + return hmac.Equal(chksum, c) +} diff --git a/github.com/jcmturner/gokrb5/v8/crypto/aes256-cts-hmac-sha1-96.go b/github.com/jcmturner/gokrb5/v8/crypto/aes256-cts-hmac-sha1-96.go new file mode 100644 index 0000000000..45e439a434 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/crypto/aes256-cts-hmac-sha1-96.go @@ -0,0 +1,129 @@ +package crypto + +import ( + "crypto/aes" + "crypto/hmac" + "crypto/sha1" + "hash" + + "github.com/jcmturner/gokrb5/v8/crypto/common" + "github.com/jcmturner/gokrb5/v8/crypto/rfc3961" + "github.com/jcmturner/gokrb5/v8/crypto/rfc3962" + "github.com/jcmturner/gokrb5/v8/iana/chksumtype" + "github.com/jcmturner/gokrb5/v8/iana/etypeID" +) + +// RFC 3962 + +// Aes256CtsHmacSha96 implements Kerberos encryption type aes256-cts-hmac-sha1-96 +type Aes256CtsHmacSha96 struct { +} + +// GetETypeID returns the EType ID number. +func (e Aes256CtsHmacSha96) GetETypeID() int32 { + return etypeID.AES256_CTS_HMAC_SHA1_96 +} + +// GetHashID returns the checksum type ID number. +func (e Aes256CtsHmacSha96) GetHashID() int32 { + return chksumtype.HMAC_SHA1_96_AES256 +} + +// GetKeyByteSize returns the number of bytes for key of this etype. +func (e Aes256CtsHmacSha96) GetKeyByteSize() int { + return 256 / 8 +} + +// GetKeySeedBitLength returns the number of bits for the seed for key generation. +func (e Aes256CtsHmacSha96) GetKeySeedBitLength() int { + return e.GetKeyByteSize() * 8 +} + +// GetHashFunc returns the hash function for this etype. +func (e Aes256CtsHmacSha96) GetHashFunc() func() hash.Hash { + return sha1.New +} + +// GetMessageBlockByteSize returns the block size for the etype's messages. +func (e Aes256CtsHmacSha96) GetMessageBlockByteSize() int { + return 1 +} + +// GetDefaultStringToKeyParams returns the default key derivation parameters in string form. +func (e Aes256CtsHmacSha96) GetDefaultStringToKeyParams() string { + return "00001000" +} + +// GetConfounderByteSize returns the byte count for confounder to be used during cryptographic operations. +func (e Aes256CtsHmacSha96) GetConfounderByteSize() int { + return aes.BlockSize +} + +// GetHMACBitLength returns the bit count size of the integrity hash. +func (e Aes256CtsHmacSha96) GetHMACBitLength() int { + return 96 +} + +// GetCypherBlockBitLength returns the bit count size of the cypher block. +func (e Aes256CtsHmacSha96) GetCypherBlockBitLength() int { + return aes.BlockSize * 8 +} + +// StringToKey returns a key derived from the string provided. +func (e Aes256CtsHmacSha96) StringToKey(secret string, salt string, s2kparams string) ([]byte, error) { + return rfc3962.StringToKey(secret, salt, s2kparams, e) +} + +// RandomToKey returns a key from the bytes provided. +func (e Aes256CtsHmacSha96) RandomToKey(b []byte) []byte { + return rfc3961.RandomToKey(b) +} + +// EncryptData encrypts the data provided. +func (e Aes256CtsHmacSha96) EncryptData(key, data []byte) ([]byte, []byte, error) { + return rfc3962.EncryptData(key, data, e) +} + +// EncryptMessage encrypts the message provided and concatenates it with the integrity hash to create an encrypted message. +func (e Aes256CtsHmacSha96) EncryptMessage(key, message []byte, usage uint32) ([]byte, []byte, error) { + return rfc3962.EncryptMessage(key, message, usage, e) +} + +// DecryptData decrypts the data provided. +func (e Aes256CtsHmacSha96) DecryptData(key, data []byte) ([]byte, error) { + return rfc3962.DecryptData(key, data, e) +} + +// DecryptMessage decrypts the message provided and verifies the integrity of the message. +func (e Aes256CtsHmacSha96) DecryptMessage(key, ciphertext []byte, usage uint32) ([]byte, error) { + return rfc3962.DecryptMessage(key, ciphertext, usage, e) +} + +// DeriveKey derives a key from the protocol key based on the usage value. +func (e Aes256CtsHmacSha96) DeriveKey(protocolKey, usage []byte) ([]byte, error) { + return rfc3961.DeriveKey(protocolKey, usage, e) +} + +// DeriveRandom generates data needed for key generation. +func (e Aes256CtsHmacSha96) DeriveRandom(protocolKey, usage []byte) ([]byte, error) { + return rfc3961.DeriveRandom(protocolKey, usage, e) +} + +// VerifyIntegrity checks the integrity of the plaintext message. +func (e Aes256CtsHmacSha96) VerifyIntegrity(protocolKey, ct, pt []byte, usage uint32) bool { + return rfc3961.VerifyIntegrity(protocolKey, ct, pt, usage, e) +} + +// GetChecksumHash returns a keyed checksum hash of the bytes provided. +func (e Aes256CtsHmacSha96) GetChecksumHash(protocolKey, data []byte, usage uint32) ([]byte, error) { + return common.GetHash(data, protocolKey, common.GetUsageKc(usage), e) +} + +// VerifyChecksum compares the checksum of the message bytes is the same as the checksum provided. +func (e Aes256CtsHmacSha96) VerifyChecksum(protocolKey, data, chksum []byte, usage uint32) bool { + c, err := e.GetChecksumHash(protocolKey, data, usage) + if err != nil { + return false + } + return hmac.Equal(chksum, c) +} diff --git a/github.com/jcmturner/gokrb5/v8/crypto/aes256-cts-hmac-sha384-192.go b/github.com/jcmturner/gokrb5/v8/crypto/aes256-cts-hmac-sha384-192.go new file mode 100644 index 0000000000..6a54475930 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/crypto/aes256-cts-hmac-sha384-192.go @@ -0,0 +1,132 @@ +package crypto + +import ( + "crypto/aes" + "crypto/hmac" + "crypto/sha512" + "hash" + + "github.com/jcmturner/gokrb5/v8/crypto/common" + "github.com/jcmturner/gokrb5/v8/crypto/rfc8009" + "github.com/jcmturner/gokrb5/v8/iana/chksumtype" + "github.com/jcmturner/gokrb5/v8/iana/etypeID" +) + +// RFC https://tools.ietf.org/html/rfc8009 + +// Aes256CtsHmacSha384192 implements Kerberos encryption type aes256-cts-hmac-sha384-192 +type Aes256CtsHmacSha384192 struct { +} + +// GetETypeID returns the EType ID number. +func (e Aes256CtsHmacSha384192) GetETypeID() int32 { + return etypeID.AES256_CTS_HMAC_SHA384_192 +} + +// GetHashID returns the checksum type ID number. +func (e Aes256CtsHmacSha384192) GetHashID() int32 { + return chksumtype.HMAC_SHA384_192_AES256 +} + +// GetKeyByteSize returns the number of bytes for key of this etype. +func (e Aes256CtsHmacSha384192) GetKeyByteSize() int { + return 192 / 8 +} + +// GetKeySeedBitLength returns the number of bits for the seed for key generation. +func (e Aes256CtsHmacSha384192) GetKeySeedBitLength() int { + return e.GetKeyByteSize() * 8 +} + +// GetHashFunc returns the hash function for this etype. +func (e Aes256CtsHmacSha384192) GetHashFunc() func() hash.Hash { + return sha512.New384 +} + +// GetMessageBlockByteSize returns the block size for the etype's messages. +func (e Aes256CtsHmacSha384192) GetMessageBlockByteSize() int { + return 1 +} + +// GetDefaultStringToKeyParams returns the default key derivation parameters in string form. +func (e Aes256CtsHmacSha384192) GetDefaultStringToKeyParams() string { + return "00008000" +} + +// GetConfounderByteSize returns the byte count for confounder to be used during cryptographic operations. +func (e Aes256CtsHmacSha384192) GetConfounderByteSize() int { + return aes.BlockSize +} + +// GetHMACBitLength returns the bit count size of the integrity hash. +func (e Aes256CtsHmacSha384192) GetHMACBitLength() int { + return 192 +} + +// GetCypherBlockBitLength returns the bit count size of the cypher block. +func (e Aes256CtsHmacSha384192) GetCypherBlockBitLength() int { + return aes.BlockSize * 8 +} + +// StringToKey returns a key derived from the string provided. +func (e Aes256CtsHmacSha384192) StringToKey(secret string, salt string, s2kparams string) ([]byte, error) { + saltp := rfc8009.GetSaltP(salt, "aes256-cts-hmac-sha384-192") + return rfc8009.StringToKey(secret, saltp, s2kparams, e) +} + +// RandomToKey returns a key from the bytes provided. +func (e Aes256CtsHmacSha384192) RandomToKey(b []byte) []byte { + return rfc8009.RandomToKey(b) +} + +// EncryptData encrypts the data provided. +func (e Aes256CtsHmacSha384192) EncryptData(key, data []byte) ([]byte, []byte, error) { + return rfc8009.EncryptData(key, data, e) +} + +// EncryptMessage encrypts the message provided and concatenates it with the integrity hash to create an encrypted message. +func (e Aes256CtsHmacSha384192) EncryptMessage(key, message []byte, usage uint32) ([]byte, []byte, error) { + return rfc8009.EncryptMessage(key, message, usage, e) +} + +// DecryptData decrypts the data provided. +func (e Aes256CtsHmacSha384192) DecryptData(key, data []byte) ([]byte, error) { + return rfc8009.DecryptData(key, data, e) +} + +// DecryptMessage decrypts the message provided and verifies the integrity of the message. +func (e Aes256CtsHmacSha384192) DecryptMessage(key, ciphertext []byte, usage uint32) ([]byte, error) { + return rfc8009.DecryptMessage(key, ciphertext, usage, e) +} + +// DeriveKey derives a key from the protocol key based on the usage value. +func (e Aes256CtsHmacSha384192) DeriveKey(protocolKey, usage []byte) ([]byte, error) { + return rfc8009.DeriveKey(protocolKey, usage, e), nil +} + +// DeriveRandom generates data needed for key generation. +func (e Aes256CtsHmacSha384192) DeriveRandom(protocolKey, usage []byte) ([]byte, error) { + return rfc8009.DeriveRandom(protocolKey, usage, e) +} + +// VerifyIntegrity checks the integrity of the ciphertext message. +// As the hash is calculated over the iv concatenated with the AES cipher output not the plaintext the pt value to this +// interface method is not use. Pass any []byte. +func (e Aes256CtsHmacSha384192) VerifyIntegrity(protocolKey, ct, pt []byte, usage uint32) bool { + // We don't need ib just there for the interface + return rfc8009.VerifyIntegrity(protocolKey, ct, usage, e) +} + +// GetChecksumHash returns a keyed checksum hash of the bytes provided. +func (e Aes256CtsHmacSha384192) GetChecksumHash(protocolKey, data []byte, usage uint32) ([]byte, error) { + return common.GetHash(data, protocolKey, common.GetUsageKc(usage), e) +} + +// VerifyChecksum compares the checksum of the message bytes is the same as the checksum provided. +func (e Aes256CtsHmacSha384192) VerifyChecksum(protocolKey, data, chksum []byte, usage uint32) bool { + c, err := e.GetChecksumHash(protocolKey, data, usage) + if err != nil { + return false + } + return hmac.Equal(chksum, c) +} diff --git a/github.com/jcmturner/gokrb5/v8/crypto/common/common.go b/github.com/jcmturner/gokrb5/v8/crypto/common/common.go new file mode 100644 index 0000000000..dab55be758 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/crypto/common/common.go @@ -0,0 +1,132 @@ +// Package common provides encryption methods common across encryption types +package common + +import ( + "bytes" + "crypto/hmac" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + + "github.com/jcmturner/gokrb5/v8/crypto/etype" +) + +// ZeroPad pads bytes with zeros to nearest multiple of message size m. +func ZeroPad(b []byte, m int) ([]byte, error) { + if m <= 0 { + return nil, errors.New("Invalid message block size when padding") + } + if b == nil || len(b) == 0 { + return nil, errors.New("Data not valid to pad: Zero size") + } + if l := len(b) % m; l != 0 { + n := m - l + z := make([]byte, n) + b = append(b, z...) + } + return b, nil +} + +// PKCS7Pad pads bytes according to RFC 2315 to nearest multiple of message size m. +func PKCS7Pad(b []byte, m int) ([]byte, error) { + if m <= 0 { + return nil, errors.New("Invalid message block size when padding") + } + if b == nil || len(b) == 0 { + return nil, errors.New("Data not valid to pad: Zero size") + } + n := m - (len(b) % m) + pb := make([]byte, len(b)+n) + copy(pb, b) + copy(pb[len(b):], bytes.Repeat([]byte{byte(n)}, n)) + return pb, nil +} + +// PKCS7Unpad removes RFC 2315 padding from byes where message size is m. +func PKCS7Unpad(b []byte, m int) ([]byte, error) { + if m <= 0 { + return nil, errors.New("invalid message block size when unpadding") + } + if b == nil || len(b) == 0 { + return nil, errors.New("padded data not valid: Zero size") + } + if len(b)%m != 0 { + return nil, errors.New("padded data not valid: Not multiple of message block size") + } + c := b[len(b)-1] + n := int(c) + if n == 0 || n > len(b) { + return nil, errors.New("padded data not valid: Data may not have been padded") + } + for i := 0; i < n; i++ { + if b[len(b)-n+i] != c { + return nil, errors.New("padded data not valid") + } + } + return b[:len(b)-n], nil +} + +// GetHash generates the keyed hash value according to the etype's hash function. +func GetHash(pt, key []byte, usage []byte, etype etype.EType) ([]byte, error) { + k, err := etype.DeriveKey(key, usage) + if err != nil { + return nil, fmt.Errorf("unable to derive key for checksum: %v", err) + } + mac := hmac.New(etype.GetHashFunc(), k) + p := make([]byte, len(pt)) + copy(p, pt) + mac.Write(p) + return mac.Sum(nil)[:etype.GetHMACBitLength()/8], nil +} + +// GetChecksumHash returns a keyed checksum hash of the bytes provided. +func GetChecksumHash(b, key []byte, usage uint32, etype etype.EType) ([]byte, error) { + return GetHash(b, key, GetUsageKc(usage), etype) +} + +// GetIntegrityHash returns a keyed integrity hash of the bytes provided. +func GetIntegrityHash(b, key []byte, usage uint32, etype etype.EType) ([]byte, error) { + return GetHash(b, key, GetUsageKi(usage), etype) +} + +// VerifyChecksum compares the checksum of the msg bytes is the same as the checksum provided. +func VerifyChecksum(key, chksum, msg []byte, usage uint32, etype etype.EType) bool { + //The encrypted message is a concatenation of the encrypted output and the hash HMAC. + expectedMAC, _ := GetChecksumHash(msg, key, usage, etype) + return hmac.Equal(chksum, expectedMAC) +} + +// GetUsageKc returns the checksum key usage value for the usage number un. +// +// See RFC 3961 5.3 key-derivation function definition. +func GetUsageKc(un uint32) []byte { + return getUsage(un, 0x99) +} + +// GetUsageKe returns the encryption key usage value for the usage number un +// +// See RFC 3961 5.3 key-derivation function definition. +func GetUsageKe(un uint32) []byte { + return getUsage(un, 0xAA) +} + +// GetUsageKi returns the integrity key usage value for the usage number un +// +// See RFC 3961 5.3 key-derivation function definition. +func GetUsageKi(un uint32) []byte { + return getUsage(un, 0x55) +} + +func getUsage(un uint32, o byte) []byte { + var buf bytes.Buffer + binary.Write(&buf, binary.BigEndian, un) + return append(buf.Bytes(), o) +} + +// IterationsToS2Kparams converts the number of iterations as an integer to a string representation. +func IterationsToS2Kparams(i uint32) string { + b := make([]byte, 4, 4) + binary.BigEndian.PutUint32(b, i) + return hex.EncodeToString(b) +} diff --git a/github.com/jcmturner/gokrb5/v8/crypto/crypto.go b/github.com/jcmturner/gokrb5/v8/crypto/crypto.go new file mode 100644 index 0000000000..5c96ddfdf6 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/crypto/crypto.go @@ -0,0 +1,175 @@ +// Package crypto implements cryptographic functions for Kerberos 5 implementation. +package crypto + +import ( + "encoding/hex" + "fmt" + + "github.com/jcmturner/gokrb5/v8/crypto/etype" + "github.com/jcmturner/gokrb5/v8/iana/chksumtype" + "github.com/jcmturner/gokrb5/v8/iana/etypeID" + "github.com/jcmturner/gokrb5/v8/iana/patype" + "github.com/jcmturner/gokrb5/v8/types" +) + +// GetEtype returns an instances of the required etype struct for the etype ID. +func GetEtype(id int32) (etype.EType, error) { + switch id { + case etypeID.AES128_CTS_HMAC_SHA1_96: + var et Aes128CtsHmacSha96 + return et, nil + case etypeID.AES256_CTS_HMAC_SHA1_96: + var et Aes256CtsHmacSha96 + return et, nil + case etypeID.AES128_CTS_HMAC_SHA256_128: + var et Aes128CtsHmacSha256128 + return et, nil + case etypeID.AES256_CTS_HMAC_SHA384_192: + var et Aes256CtsHmacSha384192 + return et, nil + case etypeID.DES3_CBC_SHA1_KD: + var et Des3CbcSha1Kd + return et, nil + case etypeID.RC4_HMAC: + var et RC4HMAC + return et, nil + default: + return nil, fmt.Errorf("unknown or unsupported EType: %d", id) + } +} + +// GetChksumEtype returns an instances of the required etype struct for the checksum ID. +func GetChksumEtype(id int32) (etype.EType, error) { + switch id { + case chksumtype.HMAC_SHA1_96_AES128: + var et Aes128CtsHmacSha96 + return et, nil + case chksumtype.HMAC_SHA1_96_AES256: + var et Aes256CtsHmacSha96 + return et, nil + case chksumtype.HMAC_SHA256_128_AES128: + var et Aes128CtsHmacSha256128 + return et, nil + case chksumtype.HMAC_SHA384_192_AES256: + var et Aes256CtsHmacSha384192 + return et, nil + case chksumtype.HMAC_SHA1_DES3_KD: + var et Des3CbcSha1Kd + return et, nil + case chksumtype.KERB_CHECKSUM_HMAC_MD5: + var et RC4HMAC + return et, nil + //case chksumtype.KERB_CHECKSUM_HMAC_MD5_UNSIGNED: + // var et RC4HMAC + // return et, nil + default: + return nil, fmt.Errorf("unknown or unsupported checksum type: %d", id) + } +} + +// GetKeyFromPassword generates an encryption key from the principal's password. +func GetKeyFromPassword(passwd string, cname types.PrincipalName, realm string, etypeID int32, pas types.PADataSequence) (types.EncryptionKey, etype.EType, error) { + var key types.EncryptionKey + et, err := GetEtype(etypeID) + if err != nil { + return key, et, fmt.Errorf("error getting encryption type: %v", err) + } + sk2p := et.GetDefaultStringToKeyParams() + var salt string + var paID int32 + for _, pa := range pas { + switch pa.PADataType { + case patype.PA_PW_SALT: + if paID > pa.PADataType { + continue + } + salt = string(pa.PADataValue) + case patype.PA_ETYPE_INFO: + if paID > pa.PADataType { + continue + } + var eti types.ETypeInfo + err := eti.Unmarshal(pa.PADataValue) + if err != nil { + return key, et, fmt.Errorf("error unmashaling PA Data to PA-ETYPE-INFO2: %v", err) + } + if etypeID != eti[0].EType { + et, err = GetEtype(eti[0].EType) + if err != nil { + return key, et, fmt.Errorf("error getting encryption type: %v", err) + } + } + salt = string(eti[0].Salt) + case patype.PA_ETYPE_INFO2: + if paID > pa.PADataType { + continue + } + var et2 types.ETypeInfo2 + err := et2.Unmarshal(pa.PADataValue) + if err != nil { + return key, et, fmt.Errorf("error unmashalling PA Data to PA-ETYPE-INFO2: %v", err) + } + if etypeID != et2[0].EType { + et, err = GetEtype(et2[0].EType) + if err != nil { + return key, et, fmt.Errorf("error getting encryption type: %v", err) + } + } + if len(et2[0].S2KParams) == 4 { + sk2p = hex.EncodeToString(et2[0].S2KParams) + } + salt = et2[0].Salt + } + } + if salt == "" { + salt = cname.GetSalt(realm) + } + k, err := et.StringToKey(passwd, salt, sk2p) + if err != nil { + return key, et, fmt.Errorf("error deriving key from string: %+v", err) + } + key = types.EncryptionKey{ + KeyType: etypeID, + KeyValue: k, + } + return key, et, nil +} + +// GetEncryptedData encrypts the data provided and returns and EncryptedData type. +// Pass a usage value of zero to use the key provided directly rather than deriving one. +func GetEncryptedData(plainBytes []byte, key types.EncryptionKey, usage uint32, kvno int) (types.EncryptedData, error) { + var ed types.EncryptedData + et, err := GetEtype(key.KeyType) + if err != nil { + return ed, fmt.Errorf("error getting etype: %v", err) + } + _, b, err := et.EncryptMessage(key.KeyValue, plainBytes, usage) + if err != nil { + return ed, err + } + + ed = types.EncryptedData{ + EType: key.KeyType, + Cipher: b, + KVNO: kvno, + } + return ed, nil +} + +// DecryptEncPart decrypts the EncryptedData. +func DecryptEncPart(ed types.EncryptedData, key types.EncryptionKey, usage uint32) ([]byte, error) { + return DecryptMessage(ed.Cipher, key, usage) +} + +// DecryptMessage decrypts the ciphertext and verifies the integrity. +func DecryptMessage(ciphertext []byte, key types.EncryptionKey, usage uint32) ([]byte, error) { + et, err := GetEtype(key.KeyType) + if err != nil { + return []byte{}, fmt.Errorf("error decrypting: %v", err) + } + b, err := et.DecryptMessage(key.KeyValue, ciphertext, usage) + if err != nil { + return nil, fmt.Errorf("error decrypting: %v", err) + } + return b, nil +} diff --git a/github.com/jcmturner/gokrb5/v8/crypto/des3-cbc-sha1-kd.go b/github.com/jcmturner/gokrb5/v8/crypto/des3-cbc-sha1-kd.go new file mode 100644 index 0000000000..6e650eb6c1 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/crypto/des3-cbc-sha1-kd.go @@ -0,0 +1,139 @@ +package crypto + +import ( + "crypto/des" + "crypto/hmac" + "crypto/sha1" + "errors" + "hash" + + "github.com/jcmturner/gokrb5/v8/crypto/common" + "github.com/jcmturner/gokrb5/v8/crypto/rfc3961" + "github.com/jcmturner/gokrb5/v8/iana/chksumtype" + "github.com/jcmturner/gokrb5/v8/iana/etypeID" +) + +//RFC: 3961 Section 6.3 + +// Des3CbcSha1Kd implements Kerberos encryption type des3-cbc-hmac-sha1-kd +type Des3CbcSha1Kd struct { +} + +// GetETypeID returns the EType ID number. +func (e Des3CbcSha1Kd) GetETypeID() int32 { + return etypeID.DES3_CBC_SHA1_KD +} + +// GetHashID returns the checksum type ID number. +func (e Des3CbcSha1Kd) GetHashID() int32 { + return chksumtype.HMAC_SHA1_DES3_KD +} + +// GetKeyByteSize returns the number of bytes for key of this etype. +func (e Des3CbcSha1Kd) GetKeyByteSize() int { + return 24 +} + +// GetKeySeedBitLength returns the number of bits for the seed for key generation. +func (e Des3CbcSha1Kd) GetKeySeedBitLength() int { + return 21 * 8 +} + +// GetHashFunc returns the hash function for this etype. +func (e Des3CbcSha1Kd) GetHashFunc() func() hash.Hash { + return sha1.New +} + +// GetMessageBlockByteSize returns the block size for the etype's messages. +func (e Des3CbcSha1Kd) GetMessageBlockByteSize() int { + //For traditional CBC mode with padding, it would be the underlying cipher's block size + return des.BlockSize +} + +// GetDefaultStringToKeyParams returns the default key derivation parameters in string form. +func (e Des3CbcSha1Kd) GetDefaultStringToKeyParams() string { + var s string + return s +} + +// GetConfounderByteSize returns the byte count for confounder to be used during cryptographic operations. +func (e Des3CbcSha1Kd) GetConfounderByteSize() int { + return des.BlockSize +} + +// GetHMACBitLength returns the bit count size of the integrity hash. +func (e Des3CbcSha1Kd) GetHMACBitLength() int { + return e.GetHashFunc()().Size() * 8 +} + +// GetCypherBlockBitLength returns the bit count size of the cypher block. +func (e Des3CbcSha1Kd) GetCypherBlockBitLength() int { + return des.BlockSize * 8 +} + +// StringToKey returns a key derived from the string provided. +func (e Des3CbcSha1Kd) StringToKey(secret string, salt string, s2kparams string) ([]byte, error) { + if s2kparams != "" { + return []byte{}, errors.New("s2kparams must be an empty string") + } + return rfc3961.DES3StringToKey(secret, salt, e) +} + +// RandomToKey returns a key from the bytes provided. +func (e Des3CbcSha1Kd) RandomToKey(b []byte) []byte { + return rfc3961.DES3RandomToKey(b) +} + +// DeriveRandom generates data needed for key generation. +func (e Des3CbcSha1Kd) DeriveRandom(protocolKey, usage []byte) ([]byte, error) { + r, err := rfc3961.DeriveRandom(protocolKey, usage, e) + return r, err +} + +// DeriveKey derives a key from the protocol key based on the usage value. +func (e Des3CbcSha1Kd) DeriveKey(protocolKey, usage []byte) ([]byte, error) { + r, err := e.DeriveRandom(protocolKey, usage) + if err != nil { + return nil, err + } + return e.RandomToKey(r), nil +} + +// EncryptData encrypts the data provided. +func (e Des3CbcSha1Kd) EncryptData(key, data []byte) ([]byte, []byte, error) { + return rfc3961.DES3EncryptData(key, data, e) +} + +// EncryptMessage encrypts the message provided and concatenates it with the integrity hash to create an encrypted message. +func (e Des3CbcSha1Kd) EncryptMessage(key, message []byte, usage uint32) ([]byte, []byte, error) { + return rfc3961.DES3EncryptMessage(key, message, usage, e) +} + +// DecryptData decrypts the data provided. +func (e Des3CbcSha1Kd) DecryptData(key, data []byte) ([]byte, error) { + return rfc3961.DES3DecryptData(key, data, e) +} + +// DecryptMessage decrypts the message provided and verifies the integrity of the message. +func (e Des3CbcSha1Kd) DecryptMessage(key, ciphertext []byte, usage uint32) ([]byte, error) { + return rfc3961.DES3DecryptMessage(key, ciphertext, usage, e) +} + +// VerifyIntegrity checks the integrity of the plaintext message. +func (e Des3CbcSha1Kd) VerifyIntegrity(protocolKey, ct, pt []byte, usage uint32) bool { + return rfc3961.VerifyIntegrity(protocolKey, ct, pt, usage, e) +} + +// GetChecksumHash returns a keyed checksum hash of the bytes provided. +func (e Des3CbcSha1Kd) GetChecksumHash(protocolKey, data []byte, usage uint32) ([]byte, error) { + return common.GetHash(data, protocolKey, common.GetUsageKc(usage), e) +} + +// VerifyChecksum compares the checksum of the message bytes is the same as the checksum provided. +func (e Des3CbcSha1Kd) VerifyChecksum(protocolKey, data, chksum []byte, usage uint32) bool { + c, err := e.GetChecksumHash(protocolKey, data, usage) + if err != nil { + return false + } + return hmac.Equal(chksum, c) +} diff --git a/github.com/jcmturner/gokrb5/v8/crypto/etype/etype.go b/github.com/jcmturner/gokrb5/v8/crypto/etype/etype.go new file mode 100644 index 0000000000..ab1496d3f6 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/crypto/etype/etype.go @@ -0,0 +1,29 @@ +// Package etype provides the Kerberos Encryption Type interface +package etype + +import "hash" + +// EType is the interface defining the Encryption Type. +type EType interface { + GetETypeID() int32 + GetHashID() int32 + GetKeyByteSize() int + GetKeySeedBitLength() int + GetDefaultStringToKeyParams() string + StringToKey(string, salt, s2kparams string) ([]byte, error) + RandomToKey(b []byte) []byte + GetHMACBitLength() int + GetMessageBlockByteSize() int + EncryptData(key, data []byte) ([]byte, []byte, error) + EncryptMessage(key, message []byte, usage uint32) ([]byte, []byte, error) + DecryptData(key, data []byte) ([]byte, error) + DecryptMessage(key, ciphertext []byte, usage uint32) ([]byte, error) + GetCypherBlockBitLength() int + GetConfounderByteSize() int + DeriveKey(protocolKey, usage []byte) ([]byte, error) + DeriveRandom(protocolKey, usage []byte) ([]byte, error) + VerifyIntegrity(protocolKey, ct, pt []byte, usage uint32) bool + GetChecksumHash(protocolKey, data []byte, usage uint32) ([]byte, error) + VerifyChecksum(protocolKey, data, chksum []byte, usage uint32) bool + GetHashFunc() func() hash.Hash +} diff --git a/github.com/jcmturner/gokrb5/v8/crypto/rc4-hmac.go b/github.com/jcmturner/gokrb5/v8/crypto/rc4-hmac.go new file mode 100644 index 0000000000..42f84b850b --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/crypto/rc4-hmac.go @@ -0,0 +1,133 @@ +package crypto + +import ( + "bytes" + "crypto/hmac" + "crypto/md5" + "hash" + "io" + + "github.com/jcmturner/gokrb5/v8/crypto/rfc3961" + "github.com/jcmturner/gokrb5/v8/crypto/rfc4757" + "github.com/jcmturner/gokrb5/v8/iana/chksumtype" + "github.com/jcmturner/gokrb5/v8/iana/etypeID" + "golang.org/x/crypto/md4" +) + +// RC4HMAC implements Kerberos encryption type rc4-hmac +type RC4HMAC struct { +} + +// GetETypeID returns the EType ID number. +func (e RC4HMAC) GetETypeID() int32 { + return etypeID.RC4_HMAC +} + +// GetHashID returns the checksum type ID number. +func (e RC4HMAC) GetHashID() int32 { + return chksumtype.KERB_CHECKSUM_HMAC_MD5 +} + +// GetKeyByteSize returns the number of bytes for key of this etype. +func (e RC4HMAC) GetKeyByteSize() int { + return 16 +} + +// GetKeySeedBitLength returns the number of bits for the seed for key generation. +func (e RC4HMAC) GetKeySeedBitLength() int { + return e.GetKeyByteSize() * 8 +} + +// GetHashFunc returns the hash function for this etype. +func (e RC4HMAC) GetHashFunc() func() hash.Hash { + return md5.New +} + +// GetMessageBlockByteSize returns the block size for the etype's messages. +func (e RC4HMAC) GetMessageBlockByteSize() int { + return 1 +} + +// GetDefaultStringToKeyParams returns the default key derivation parameters in string form. +func (e RC4HMAC) GetDefaultStringToKeyParams() string { + return "" +} + +// GetConfounderByteSize returns the byte count for confounder to be used during cryptographic operations. +func (e RC4HMAC) GetConfounderByteSize() int { + return 8 +} + +// GetHMACBitLength returns the bit count size of the integrity hash. +func (e RC4HMAC) GetHMACBitLength() int { + return md5.Size * 8 +} + +// GetCypherBlockBitLength returns the bit count size of the cypher block. +func (e RC4HMAC) GetCypherBlockBitLength() int { + return 8 // doesn't really apply +} + +// StringToKey returns a key derived from the string provided. +func (e RC4HMAC) StringToKey(secret string, salt string, s2kparams string) ([]byte, error) { + return rfc4757.StringToKey(secret) +} + +// RandomToKey returns a key from the bytes provided. +func (e RC4HMAC) RandomToKey(b []byte) []byte { + r := bytes.NewReader(b) + h := md4.New() + io.Copy(h, r) + return h.Sum(nil) +} + +// EncryptData encrypts the data provided. +func (e RC4HMAC) EncryptData(key, data []byte) ([]byte, []byte, error) { + b, err := rfc4757.EncryptData(key, data, e) + return []byte{}, b, err +} + +// EncryptMessage encrypts the message provided and concatenates it with the integrity hash to create an encrypted message. +func (e RC4HMAC) EncryptMessage(key, message []byte, usage uint32) ([]byte, []byte, error) { + b, err := rfc4757.EncryptMessage(key, message, usage, false, e) + return []byte{}, b, err +} + +// DecryptData decrypts the data provided. +func (e RC4HMAC) DecryptData(key, data []byte) ([]byte, error) { + return rfc4757.DecryptData(key, data, e) +} + +// DecryptMessage decrypts the message provided and verifies the integrity of the message. +func (e RC4HMAC) DecryptMessage(key, ciphertext []byte, usage uint32) ([]byte, error) { + return rfc4757.DecryptMessage(key, ciphertext, usage, false, e) +} + +// DeriveKey derives a key from the protocol key based on the usage value. +func (e RC4HMAC) DeriveKey(protocolKey, usage []byte) ([]byte, error) { + return rfc4757.HMAC(protocolKey, usage), nil +} + +// DeriveRandom generates data needed for key generation. +func (e RC4HMAC) DeriveRandom(protocolKey, usage []byte) ([]byte, error) { + return rfc3961.DeriveRandom(protocolKey, usage, e) +} + +// VerifyIntegrity checks the integrity of the plaintext message. +func (e RC4HMAC) VerifyIntegrity(protocolKey, ct, pt []byte, usage uint32) bool { + return rfc4757.VerifyIntegrity(protocolKey, pt, ct, e) +} + +// GetChecksumHash returns a keyed checksum hash of the bytes provided. +func (e RC4HMAC) GetChecksumHash(protocolKey, data []byte, usage uint32) ([]byte, error) { + return rfc4757.Checksum(protocolKey, usage, data) +} + +// VerifyChecksum compares the checksum of the message bytes is the same as the checksum provided. +func (e RC4HMAC) VerifyChecksum(protocolKey, data, chksum []byte, usage uint32) bool { + checksum, err := rfc4757.Checksum(protocolKey, usage, data) + if err != nil { + return false + } + return hmac.Equal(checksum, chksum) +} diff --git a/github.com/jcmturner/gokrb5/v8/crypto/rfc3961/encryption.go b/github.com/jcmturner/gokrb5/v8/crypto/rfc3961/encryption.go new file mode 100644 index 0000000000..1383258c7f --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/crypto/rfc3961/encryption.go @@ -0,0 +1,119 @@ +// Package rfc3961 provides encryption and checksum methods as specified in RFC 3961 +package rfc3961 + +import ( + "crypto/cipher" + "crypto/des" + "crypto/hmac" + "crypto/rand" + "errors" + "fmt" + + "github.com/jcmturner/gokrb5/v8/crypto/common" + "github.com/jcmturner/gokrb5/v8/crypto/etype" +) + +// DES3EncryptData encrypts the data provided using DES3 and methods specific to the etype provided. +func DES3EncryptData(key, data []byte, e etype.EType) ([]byte, []byte, error) { + if len(key) != e.GetKeyByteSize() { + return nil, nil, fmt.Errorf("incorrect keysize: expected: %v actual: %v", e.GetKeyByteSize(), len(key)) + } + data, _ = common.ZeroPad(data, e.GetMessageBlockByteSize()) + + block, err := des.NewTripleDESCipher(key) + if err != nil { + return nil, nil, fmt.Errorf("error creating cipher: %v", err) + } + + //RFC 3961: initial cipher state All bits zero + ivz := make([]byte, des.BlockSize) + + ct := make([]byte, len(data)) + mode := cipher.NewCBCEncrypter(block, ivz) + mode.CryptBlocks(ct, data) + return ct[len(ct)-e.GetMessageBlockByteSize():], ct, nil +} + +// DES3EncryptMessage encrypts the message provided using DES3 and methods specific to the etype provided. +// The encrypted data is concatenated with its integrity hash to create an encrypted message. +func DES3EncryptMessage(key, message []byte, usage uint32, e etype.EType) ([]byte, []byte, error) { + //confounder + c := make([]byte, e.GetConfounderByteSize()) + _, err := rand.Read(c) + if err != nil { + return []byte{}, []byte{}, fmt.Errorf("could not generate random confounder: %v", err) + } + plainBytes := append(c, message...) + plainBytes, _ = common.ZeroPad(plainBytes, e.GetMessageBlockByteSize()) + + // Derive key for encryption from usage + var k []byte + if usage != 0 { + k, err = e.DeriveKey(key, common.GetUsageKe(usage)) + if err != nil { + return []byte{}, []byte{}, fmt.Errorf("error deriving key for encryption: %v", err) + } + } + + iv, b, err := e.EncryptData(k, plainBytes) + if err != nil { + return iv, b, fmt.Errorf("error encrypting data: %v", err) + } + + // Generate and append integrity hash + ih, err := common.GetIntegrityHash(plainBytes, key, usage, e) + if err != nil { + return iv, b, fmt.Errorf("error encrypting data: %v", err) + } + b = append(b, ih...) + return iv, b, nil +} + +// DES3DecryptData decrypts the data provided using DES3 and methods specific to the etype provided. +func DES3DecryptData(key, data []byte, e etype.EType) ([]byte, error) { + if len(key) != e.GetKeyByteSize() { + return []byte{}, fmt.Errorf("incorrect keysize: expected: %v actual: %v", e.GetKeyByteSize(), len(key)) + } + + if len(data) < des.BlockSize || len(data)%des.BlockSize != 0 { + return []byte{}, errors.New("ciphertext is not a multiple of the block size") + } + block, err := des.NewTripleDESCipher(key) + if err != nil { + return []byte{}, fmt.Errorf("error creating cipher: %v", err) + } + pt := make([]byte, len(data)) + ivz := make([]byte, des.BlockSize) + mode := cipher.NewCBCDecrypter(block, ivz) + mode.CryptBlocks(pt, data) + return pt, nil +} + +// DES3DecryptMessage decrypts the message provided using DES3 and methods specific to the etype provided. +// The integrity of the message is also verified. +func DES3DecryptMessage(key, ciphertext []byte, usage uint32, e etype.EType) ([]byte, error) { + //Derive the key + k, err := e.DeriveKey(key, common.GetUsageKe(usage)) + if err != nil { + return nil, fmt.Errorf("error deriving key: %v", err) + } + // Strip off the checksum from the end + b, err := e.DecryptData(k, ciphertext[:len(ciphertext)-e.GetHMACBitLength()/8]) + if err != nil { + return nil, fmt.Errorf("error decrypting: %v", err) + } + //Verify checksum + if !e.VerifyIntegrity(key, ciphertext, b, usage) { + return nil, errors.New("error decrypting: integrity verification failed") + } + //Remove the confounder bytes + return b[e.GetConfounderByteSize():], nil +} + +// VerifyIntegrity verifies the integrity of cipertext bytes ct. +func VerifyIntegrity(key, ct, pt []byte, usage uint32, etype etype.EType) bool { + h := make([]byte, etype.GetHMACBitLength()/8) + copy(h, ct[len(ct)-etype.GetHMACBitLength()/8:]) + expectedMAC, _ := common.GetIntegrityHash(pt, key, usage, etype) + return hmac.Equal(h, expectedMAC) +} diff --git a/github.com/jcmturner/gokrb5/v8/crypto/rfc3961/keyDerivation.go b/github.com/jcmturner/gokrb5/v8/crypto/rfc3961/keyDerivation.go new file mode 100644 index 0000000000..ed9b169c5d --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/crypto/rfc3961/keyDerivation.go @@ -0,0 +1,169 @@ +package rfc3961 + +import ( + "bytes" + + "github.com/jcmturner/gokrb5/v8/crypto/etype" +) + +const ( + prfconstant = "prf" +) + +// DeriveRandom implements the RFC 3961 defined function: DR(Key, Constant) = k-truncate(E(Key, Constant, initial-cipher-state)). +// +// key: base key or protocol key. Likely to be a key from a keytab file. +// +// usage: a constant. +// +// n: block size in bits (not bytes) - note if you use something like aes.BlockSize this is in bytes. +// +// k: key length / key seed length in bits. Eg. for AES256 this value is 256. +// +// e: the encryption etype function to use. +func DeriveRandom(key, usage []byte, e etype.EType) ([]byte, error) { + n := e.GetCypherBlockBitLength() + k := e.GetKeySeedBitLength() + //Ensure the usage constant is at least the size of the cypher block size. Pass it through the nfold algorithm that will "stretch" it if needs be. + nFoldUsage := Nfold(usage, n) + //k-truncate implemented by creating a byte array the size of k (k is in bits hence /8) + out := make([]byte, k/8) + // Keep feeding the output back into the encryption function until it is no longer short than k. + _, K, err := e.EncryptData(key, nFoldUsage) + if err != nil { + return out, err + } + for i := copy(out, K); i < len(out); { + _, K, _ = e.EncryptData(key, K) + i = i + copy(out[i:], K) + } + return out, nil +} + +// DeriveKey derives a key from the protocol key based on the usage and the etype's specific methods. +func DeriveKey(protocolKey, usage []byte, e etype.EType) ([]byte, error) { + r, err := e.DeriveRandom(protocolKey, usage) + if err != nil { + return nil, err + } + return e.RandomToKey(r), nil +} + +// RandomToKey returns a key from the bytes provided according to the definition in RFC 3961. +func RandomToKey(b []byte) []byte { + return b +} + +// DES3RandomToKey returns a key from the bytes provided according to the definition in RFC 3961 for DES3 etypes. +func DES3RandomToKey(b []byte) []byte { + r := fixWeakKey(stretch56Bits(b[:7])) + r2 := fixWeakKey(stretch56Bits(b[7:14])) + r = append(r, r2...) + r3 := fixWeakKey(stretch56Bits(b[14:21])) + r = append(r, r3...) + return r +} + +// DES3StringToKey returns a key derived from the string provided according to the definition in RFC 3961 for DES3 etypes. +func DES3StringToKey(secret, salt string, e etype.EType) ([]byte, error) { + s := secret + salt + tkey := e.RandomToKey(Nfold([]byte(s), e.GetKeySeedBitLength())) + return e.DeriveKey(tkey, []byte("kerberos")) +} + +// PseudoRandom function as defined in RFC 3961 +func PseudoRandom(key, b []byte, e etype.EType) ([]byte, error) { + h := e.GetHashFunc()() + h.Write(b) + tmp := h.Sum(nil)[:e.GetMessageBlockByteSize()] + k, err := e.DeriveKey(key, []byte(prfconstant)) + if err != nil { + return []byte{}, err + } + _, prf, err := e.EncryptData(k, tmp) + if err != nil { + return []byte{}, err + } + return prf, nil +} + +func stretch56Bits(b []byte) []byte { + d := make([]byte, len(b), len(b)) + copy(d, b) + var lb byte + for i, v := range d { + bv, nb := calcEvenParity(v) + d[i] = nb + if bv != 0 { + lb = lb | (1 << uint(i+1)) + } else { + lb = lb &^ (1 << uint(i+1)) + } + } + _, lb = calcEvenParity(lb) + d = append(d, lb) + return d +} + +func calcEvenParity(b byte) (uint8, uint8) { + lowestbit := b & 0x01 + // c counter of 1s in the first 7 bits of the byte + var c int + // Iterate over the highest 7 bits (hence p starts at 1 not zero) and count the 1s. + for p := 1; p < 8; p++ { + v := b & (1 << uint(p)) + if v != 0 { + c++ + } + } + if c%2 == 0 { + //Even number of 1s so set parity to 1 + b = b | 1 + } else { + //Odd number of 1s so set parity to 0 + b = b &^ 1 + } + return lowestbit, b +} + +func fixWeakKey(b []byte) []byte { + if weak(b) { + b[7] ^= 0xF0 + } + return b +} + +func weak(b []byte) bool { + // weak keys from https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-67r1.pdf + weakKeys := [4][]byte{ + {0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + {0xFE, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE, 0xFE}, + {0xE0, 0xE0, 0xE0, 0xE0, 0xF1, 0xF1, 0xF1, 0xF1}, + {0x1F, 0x1F, 0x1F, 0x1F, 0x0E, 0x0E, 0x0E, 0x0E}, + } + semiWeakKeys := [12][]byte{ + {0x01, 0x1F, 0x01, 0x1F, 0x01, 0x0E, 0x01, 0x0E}, + {0x1F, 0x01, 0x1F, 0x01, 0x0E, 0x01, 0x0E, 0x01}, + {0x01, 0xE0, 0x01, 0xE0, 0x01, 0xF1, 0x01, 0xF1}, + {0xE0, 0x01, 0xE0, 0x01, 0xF1, 0x01, 0xF1, 0x01}, + {0x01, 0xFE, 0x01, 0xFE, 0x01, 0xFE, 0x01, 0xFE}, + {0xFE, 0x01, 0xFE, 0x01, 0xFE, 0x01, 0xFE, 0x01}, + {0x1F, 0xE0, 0x1F, 0xE0, 0x0E, 0xF1, 0x0E, 0xF1}, + {0xE0, 0x1F, 0xE0, 0x1F, 0xF1, 0x0E, 0xF1, 0x0E}, + {0x1F, 0xFE, 0x1F, 0xFE, 0x0E, 0xFE, 0x0E, 0xFE}, + {0xFE, 0x1F, 0xFE, 0x1F, 0xFE, 0x0E, 0xFE, 0x0E}, + {0xE0, 0xFE, 0xE0, 0xFE, 0xF1, 0xFE, 0xF1, 0xFE}, + {0xFE, 0xE0, 0xFE, 0xE0, 0xFE, 0xF1, 0xFE, 0xF1}, + } + for _, k := range weakKeys { + if bytes.Equal(b, k) { + return true + } + } + for _, k := range semiWeakKeys { + if bytes.Equal(b, k) { + return true + } + } + return false +} diff --git a/github.com/jcmturner/gokrb5/v8/crypto/rfc3961/nfold.go b/github.com/jcmturner/gokrb5/v8/crypto/rfc3961/nfold.go new file mode 100644 index 0000000000..9536b1e3e3 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/crypto/rfc3961/nfold.go @@ -0,0 +1,107 @@ +package rfc3961 + +// Implementation of the n-fold algorithm as defined in RFC 3961. + +/* Credits +This golang implementation of nfold used the following project for help with implementation detail. +Although their source is in java it was helpful as a reference implementation of the RFC. +You can find the source code of their open source project along with license information below. +We acknowledge and are grateful to these developers for their contributions to open source + +Project: Apache Directory (http://http://directory.apache.org/) +https://svn.apache.org/repos/asf/directory/apacheds/tags/1.5.1/kerberos-shared/src/main/java/org/apache/directory/server/kerberos/shared/crypto/encryption/NFold.java +License: http://www.apache.org/licenses/LICENSE-2.0 +*/ + +// Nfold expands the key to ensure it is not smaller than one cipher block. +// Defined in RFC 3961. +// +// m input bytes that will be "stretched" to the least common multiple of n bits and the bit length of m. +func Nfold(m []byte, n int) []byte { + k := len(m) * 8 + + //Get the lowest common multiple of the two bit sizes + lcm := lcm(n, k) + relicate := lcm / k + var sumBytes []byte + + for i := 0; i < relicate; i++ { + rotation := 13 * i + sumBytes = append(sumBytes, rotateRight(m, rotation)...) + } + + nfold := make([]byte, n/8) + sum := make([]byte, n/8) + for i := 0; i < lcm/n; i++ { + for j := 0; j < n/8; j++ { + sum[j] = sumBytes[j+(i*len(sum))] + } + nfold = onesComplementAddition(nfold, sum) + } + return nfold +} + +func onesComplementAddition(n1, n2 []byte) []byte { + numBits := len(n1) * 8 + out := make([]byte, numBits/8) + carry := 0 + for i := numBits - 1; i > -1; i-- { + n1b := getBit(&n1, i) + n2b := getBit(&n2, i) + s := n1b + n2b + carry + + if s == 0 || s == 1 { + setBit(&out, i, s) + carry = 0 + } else if s == 2 { + carry = 1 + } else if s == 3 { + setBit(&out, i, 1) + carry = 1 + } + } + if carry == 1 { + carryArray := make([]byte, len(n1)) + carryArray[len(carryArray)-1] = 1 + out = onesComplementAddition(out, carryArray) + } + return out +} + +func rotateRight(b []byte, step int) []byte { + out := make([]byte, len(b)) + bitLen := len(b) * 8 + for i := 0; i < bitLen; i++ { + v := getBit(&b, i) + setBit(&out, (i+step)%bitLen, v) + } + return out +} + +func lcm(x, y int) int { + return (x * y) / gcd(x, y) +} + +func gcd(x, y int) int { + for y != 0 { + x, y = y, x%y + } + return x +} + +func getBit(b *[]byte, p int) int { + pByte := p / 8 + pBit := uint(p % 8) + vByte := (*b)[pByte] + vInt := int(vByte >> (8 - (pBit + 1)) & 0x0001) + return vInt +} + +func setBit(b *[]byte, p, v int) { + pByte := p / 8 + pBit := uint(p % 8) + oldByte := (*b)[pByte] + var newByte byte + newByte = byte(v<<(8-(pBit+1))) | oldByte + (*b)[pByte] = newByte +} diff --git a/github.com/jcmturner/gokrb5/v8/crypto/rfc3962/encryption.go b/github.com/jcmturner/gokrb5/v8/crypto/rfc3962/encryption.go new file mode 100644 index 0000000000..5ff89e85b0 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/crypto/rfc3962/encryption.go @@ -0,0 +1,89 @@ +// Package rfc3962 provides encryption and checksum methods as specified in RFC 3962 +package rfc3962 + +import ( + "crypto/rand" + "errors" + "fmt" + + "github.com/jcmturner/aescts/v2" + "github.com/jcmturner/gokrb5/v8/crypto/common" + "github.com/jcmturner/gokrb5/v8/crypto/etype" +) + +// EncryptData encrypts the data provided using methods specific to the etype provided as defined in RFC 3962. +func EncryptData(key, data []byte, e etype.EType) ([]byte, []byte, error) { + if len(key) != e.GetKeyByteSize() { + return []byte{}, []byte{}, fmt.Errorf("incorrect keysize: expected: %v actual: %v", e.GetKeyByteSize(), len(key)) + } + ivz := make([]byte, e.GetCypherBlockBitLength()/8) + return aescts.Encrypt(key, ivz, data) +} + +// EncryptMessage encrypts the message provided using the methods specific to the etype provided as defined in RFC 3962. +// The encrypted data is concatenated with its integrity hash to create an encrypted message. +func EncryptMessage(key, message []byte, usage uint32, e etype.EType) ([]byte, []byte, error) { + if len(key) != e.GetKeyByteSize() { + return []byte{}, []byte{}, fmt.Errorf("incorrect keysize: expected: %v actual: %v", e.GetKeyByteSize(), len(key)) + } + //confounder + c := make([]byte, e.GetConfounderByteSize()) + _, err := rand.Read(c) + if err != nil { + return []byte{}, []byte{}, fmt.Errorf("could not generate random confounder: %v", err) + } + plainBytes := append(c, message...) + + // Derive key for encryption from usage + var k []byte + if usage != 0 { + k, err = e.DeriveKey(key, common.GetUsageKe(usage)) + if err != nil { + return []byte{}, []byte{}, fmt.Errorf("error deriving key for encryption: %v", err) + } + } + + // Encrypt the data + iv, b, err := e.EncryptData(k, plainBytes) + if err != nil { + return iv, b, fmt.Errorf("error encrypting data: %v", err) + } + + // Generate and append integrity hash + ih, err := common.GetIntegrityHash(plainBytes, key, usage, e) + if err != nil { + return iv, b, fmt.Errorf("error encrypting data: %v", err) + } + b = append(b, ih...) + return iv, b, nil +} + +// DecryptData decrypts the data provided using the methods specific to the etype provided as defined in RFC 3962. +func DecryptData(key, data []byte, e etype.EType) ([]byte, error) { + if len(key) != e.GetKeyByteSize() { + return []byte{}, fmt.Errorf("incorrect keysize: expected: %v actual: %v", e.GetKeyByteSize(), len(key)) + } + ivz := make([]byte, e.GetCypherBlockBitLength()/8) + return aescts.Decrypt(key, ivz, data) +} + +// DecryptMessage decrypts the message provided using the methods specific to the etype provided as defined in RFC 3962. +// The integrity of the message is also verified. +func DecryptMessage(key, ciphertext []byte, usage uint32, e etype.EType) ([]byte, error) { + //Derive the key + k, err := e.DeriveKey(key, common.GetUsageKe(usage)) + if err != nil { + return nil, fmt.Errorf("error deriving key: %v", err) + } + // Strip off the checksum from the end + b, err := e.DecryptData(k, ciphertext[:len(ciphertext)-e.GetHMACBitLength()/8]) + if err != nil { + return nil, err + } + //Verify checksum + if !e.VerifyIntegrity(key, ciphertext, b, usage) { + return nil, errors.New("integrity verification failed") + } + //Remove the confounder bytes + return b[e.GetConfounderByteSize():], nil +} diff --git a/github.com/jcmturner/gokrb5/v8/crypto/rfc3962/keyDerivation.go b/github.com/jcmturner/gokrb5/v8/crypto/rfc3962/keyDerivation.go new file mode 100644 index 0000000000..fb402d97b7 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/crypto/rfc3962/keyDerivation.go @@ -0,0 +1,51 @@ +package rfc3962 + +import ( + "encoding/binary" + "encoding/hex" + "errors" + + "github.com/jcmturner/gofork/x/crypto/pbkdf2" + "github.com/jcmturner/gokrb5/v8/crypto/etype" +) + +const ( + s2kParamsZero = 4294967296 +) + +// StringToKey returns a key derived from the string provided according to the definition in RFC 3961. +func StringToKey(secret, salt, s2kparams string, e etype.EType) ([]byte, error) { + i, err := S2KparamsToItertions(s2kparams) + if err != nil { + return nil, err + } + return StringToKeyIter(secret, salt, i, e) +} + +// StringToPBKDF2 generates an encryption key from a pass phrase and salt string using the PBKDF2 function from PKCS #5 v2.0 +func StringToPBKDF2(secret, salt string, iterations int64, e etype.EType) []byte { + return pbkdf2.Key64([]byte(secret), []byte(salt), iterations, int64(e.GetKeyByteSize()), e.GetHashFunc()) +} + +// StringToKeyIter returns a key derived from the string provided according to the definition in RFC 3961. +func StringToKeyIter(secret, salt string, iterations int64, e etype.EType) ([]byte, error) { + tkey := e.RandomToKey(StringToPBKDF2(secret, salt, iterations, e)) + return e.DeriveKey(tkey, []byte("kerberos")) +} + +// S2KparamsToItertions converts the string representation of iterations to an integer +func S2KparamsToItertions(s2kparams string) (int64, error) { + //The s2kparams string should be hex string representing 4 bytes + //The 4 bytes represent a number in big endian order + //If the value is zero then the number of iterations should be 4,294,967,296 (2^32) + var i uint32 + if len(s2kparams) != 8 { + return int64(s2kParamsZero), errors.New("invalid s2kparams length") + } + b, err := hex.DecodeString(s2kparams) + if err != nil { + return int64(s2kParamsZero), errors.New("invalid s2kparams, cannot decode string to bytes") + } + i = binary.BigEndian.Uint32(b) + return int64(i), nil +} diff --git a/github.com/jcmturner/gokrb5/v8/crypto/rfc4757/checksum.go b/github.com/jcmturner/gokrb5/v8/crypto/rfc4757/checksum.go new file mode 100644 index 0000000000..45276e9532 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/crypto/rfc4757/checksum.go @@ -0,0 +1,40 @@ +package rfc4757 + +import ( + "bytes" + "crypto/hmac" + "crypto/md5" + "io" +) + +// Checksum returns a hash of the data in accordance with RFC 4757 +func Checksum(key []byte, usage uint32, data []byte) ([]byte, error) { + // Create hashing key + s := append([]byte(`signaturekey`), byte(0x00)) //includes zero octet at end + mac := hmac.New(md5.New, key) + mac.Write(s) + Ksign := mac.Sum(nil) + + // Format data + tb := UsageToMSMsgType(usage) + p := append(tb, data...) + h := md5.New() + rb := bytes.NewReader(p) + _, err := io.Copy(h, rb) + if err != nil { + return []byte{}, err + } + tmp := h.Sum(nil) + + // Generate HMAC + mac = hmac.New(md5.New, Ksign) + mac.Write(tmp) + return mac.Sum(nil), nil +} + +// HMAC returns a keyed MD5 checksum of the data +func HMAC(key []byte, data []byte) []byte { + mac := hmac.New(md5.New, key) + mac.Write(data) + return mac.Sum(nil) +} diff --git a/github.com/jcmturner/gokrb5/v8/crypto/rfc4757/encryption.go b/github.com/jcmturner/gokrb5/v8/crypto/rfc4757/encryption.go new file mode 100644 index 0000000000..fdebe73668 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/crypto/rfc4757/encryption.go @@ -0,0 +1,80 @@ +// Package rfc4757 provides encryption and checksum methods as specified in RFC 4757 +package rfc4757 + +import ( + "crypto/hmac" + "crypto/rand" + "crypto/rc4" + "errors" + "fmt" + + "github.com/jcmturner/gokrb5/v8/crypto/etype" +) + +// EncryptData encrypts the data provided using methods specific to the etype provided as defined in RFC 4757. +func EncryptData(key, data []byte, e etype.EType) ([]byte, error) { + if len(key) != e.GetKeyByteSize() { + return []byte{}, fmt.Errorf("incorrect keysize: expected: %v actual: %v", e.GetKeyByteSize(), len(key)) + } + rc4Cipher, err := rc4.NewCipher(key) + if err != nil { + return []byte{}, fmt.Errorf("error creating RC4 cipher: %v", err) + } + ed := make([]byte, len(data)) + copy(ed, data) + rc4Cipher.XORKeyStream(ed, ed) + rc4Cipher.Reset() + return ed, nil +} + +// DecryptData decrypts the data provided using the methods specific to the etype provided as defined in RFC 4757. +func DecryptData(key, data []byte, e etype.EType) ([]byte, error) { + return EncryptData(key, data, e) +} + +// EncryptMessage encrypts the message provided using the methods specific to the etype provided as defined in RFC 4757. +// The encrypted data is concatenated with its RC4 header containing integrity checksum and confounder to create an encrypted message. +func EncryptMessage(key, data []byte, usage uint32, export bool, e etype.EType) ([]byte, error) { + confounder := make([]byte, e.GetConfounderByteSize()) // size = 8 + _, err := rand.Read(confounder) + if err != nil { + return []byte{}, fmt.Errorf("error generating confounder: %v", err) + } + k1 := key + k2 := HMAC(k1, UsageToMSMsgType(usage)) + toenc := append(confounder, data...) + chksum := HMAC(k2, toenc) + k3 := HMAC(k2, chksum) + + ed, err := EncryptData(k3, toenc, e) + if err != nil { + return []byte{}, fmt.Errorf("error encrypting data: %v", err) + } + + msg := append(chksum, ed...) + return msg, nil +} + +// DecryptMessage decrypts the message provided using the methods specific to the etype provided as defined in RFC 4757. +// The integrity of the message is also verified. +func DecryptMessage(key, data []byte, usage uint32, export bool, e etype.EType) ([]byte, error) { + checksum := data[:e.GetHMACBitLength()/8] + ct := data[e.GetHMACBitLength()/8:] + _, k2, k3 := deriveKeys(key, checksum, usage, export) + + pt, err := DecryptData(k3, ct, e) + if err != nil { + return []byte{}, fmt.Errorf("error decrypting data: %v", err) + } + + if !VerifyIntegrity(k2, pt, data, e) { + return []byte{}, errors.New("integrity checksum incorrect") + } + return pt[e.GetConfounderByteSize():], nil +} + +// VerifyIntegrity checks the integrity checksum of the data matches that calculated from the decrypted data. +func VerifyIntegrity(key, pt, data []byte, e etype.EType) bool { + chksum := HMAC(key, pt) + return hmac.Equal(chksum, data[:e.GetHMACBitLength()/8]) +} diff --git a/github.com/jcmturner/gokrb5/v8/crypto/rfc4757/keyDerivation.go b/github.com/jcmturner/gokrb5/v8/crypto/rfc4757/keyDerivation.go new file mode 100644 index 0000000000..d1f90c077d --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/crypto/rfc4757/keyDerivation.go @@ -0,0 +1,40 @@ +package rfc4757 + +import ( + "bytes" + "encoding/hex" + "errors" + "fmt" + "io" + + "golang.org/x/crypto/md4" +) + +// StringToKey returns a key derived from the string provided according to the definition in RFC 4757. +func StringToKey(secret string) ([]byte, error) { + b := make([]byte, len(secret)*2, len(secret)*2) + for i, r := range secret { + u := fmt.Sprintf("%04x", r) + c, err := hex.DecodeString(u) + if err != nil { + return []byte{}, errors.New("character could not be encoded") + } + // Swap round the two bytes to make little endian as we put into byte slice + b[2*i] = c[1] + b[2*i+1] = c[0] + } + r := bytes.NewReader(b) + h := md4.New() + _, err := io.Copy(h, r) + if err != nil { + return []byte{}, err + } + return h.Sum(nil), nil +} + +func deriveKeys(key, checksum []byte, usage uint32, export bool) (k1, k2, k3 []byte) { + k1 = key + k2 = HMAC(k1, UsageToMSMsgType(usage)) + k3 = HMAC(k2, checksum) + return +} diff --git a/github.com/jcmturner/gokrb5/v8/crypto/rfc4757/msgtype.go b/github.com/jcmturner/gokrb5/v8/crypto/rfc4757/msgtype.go new file mode 100644 index 0000000000..068588d3ba --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/crypto/rfc4757/msgtype.go @@ -0,0 +1,20 @@ +package rfc4757 + +import "encoding/binary" + +// UsageToMSMsgType converts Kerberos key usage numbers to Microsoft message type encoded as a little-endian four byte slice. +func UsageToMSMsgType(usage uint32) []byte { + // Translate usage numbers to the Microsoft T numbers + switch usage { + case 3: + usage = 8 + case 9: + usage = 8 + case 23: + usage = 13 + } + // Now convert to bytes + tb := make([]byte, 4) // We force an int32 input so we can't go over 4 bytes + binary.PutUvarint(tb, uint64(usage)) + return tb +} diff --git a/github.com/jcmturner/gokrb5/v8/crypto/rfc8009/encryption.go b/github.com/jcmturner/gokrb5/v8/crypto/rfc8009/encryption.go new file mode 100644 index 0000000000..54cff7b462 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/crypto/rfc8009/encryption.go @@ -0,0 +1,125 @@ +// Package rfc8009 provides encryption and checksum methods as specified in RFC 8009 +package rfc8009 + +import ( + "crypto/aes" + "crypto/hmac" + "crypto/rand" + "errors" + "fmt" + + "github.com/jcmturner/aescts/v2" + "github.com/jcmturner/gokrb5/v8/crypto/common" + "github.com/jcmturner/gokrb5/v8/crypto/etype" + "github.com/jcmturner/gokrb5/v8/iana/etypeID" +) + +// EncryptData encrypts the data provided using methods specific to the etype provided as defined in RFC 8009. +func EncryptData(key, data []byte, e etype.EType) ([]byte, []byte, error) { + kl := e.GetKeyByteSize() + if e.GetETypeID() == etypeID.AES256_CTS_HMAC_SHA384_192 { + kl = 32 + } + if len(key) != kl { + return []byte{}, []byte{}, fmt.Errorf("incorrect keysize: expected: %v actual: %v", e.GetKeyByteSize(), len(key)) + } + ivz := make([]byte, aes.BlockSize) + return aescts.Encrypt(key, ivz, data) +} + +// EncryptMessage encrypts the message provided using the methods specific to the etype provided as defined in RFC 8009. +// The encrypted data is concatenated with its integrity hash to create an encrypted message. +func EncryptMessage(key, message []byte, usage uint32, e etype.EType) ([]byte, []byte, error) { + kl := e.GetKeyByteSize() + if e.GetETypeID() == etypeID.AES256_CTS_HMAC_SHA384_192 { + kl = 32 + } + if len(key) != kl { + return []byte{}, []byte{}, fmt.Errorf("incorrect keysize: expected: %v actual: %v", kl, len(key)) + } + if len(key) != e.GetKeyByteSize() { + } + //confounder + c := make([]byte, e.GetConfounderByteSize()) + _, err := rand.Read(c) + if err != nil { + return []byte{}, []byte{}, fmt.Errorf("could not generate random confounder: %v", err) + } + plainBytes := append(c, message...) + + // Derive key for encryption from usage + var k []byte + if usage != 0 { + k, err = e.DeriveKey(key, common.GetUsageKe(usage)) + if err != nil { + return []byte{}, []byte{}, fmt.Errorf("error deriving key for encryption: %v", err) + } + } + + // Encrypt the data + iv, b, err := e.EncryptData(k, plainBytes) + if err != nil { + return iv, b, fmt.Errorf("error encrypting data: %v", err) + } + + ivz := make([]byte, e.GetConfounderByteSize()) + ih, err := GetIntegityHash(ivz, b, key, usage, e) + if err != nil { + return iv, b, fmt.Errorf("error encrypting data: %v", err) + } + b = append(b, ih...) + return iv, b, nil +} + +// DecryptData decrypts the data provided using the methods specific to the etype provided as defined in RFC 8009. +func DecryptData(key, data []byte, e etype.EType) ([]byte, error) { + kl := e.GetKeyByteSize() + if e.GetETypeID() == etypeID.AES256_CTS_HMAC_SHA384_192 { + kl = 32 + } + if len(key) != kl { + return []byte{}, fmt.Errorf("incorrect keysize: expected: %v actual: %v", kl, len(key)) + } + ivz := make([]byte, aes.BlockSize) + return aescts.Decrypt(key, ivz, data) +} + +// DecryptMessage decrypts the message provided using the methods specific to the etype provided as defined in RFC 8009. +// The integrity of the message is also verified. +func DecryptMessage(key, ciphertext []byte, usage uint32, e etype.EType) ([]byte, error) { + //Derive the key + k, err := e.DeriveKey(key, common.GetUsageKe(usage)) + if err != nil { + return nil, fmt.Errorf("error deriving key: %v", err) + } + // Strip off the checksum from the end + b, err := e.DecryptData(k, ciphertext[:len(ciphertext)-e.GetHMACBitLength()/8]) + if err != nil { + return nil, err + } + //Verify checksum + if !e.VerifyIntegrity(key, ciphertext, b, usage) { + return nil, errors.New("integrity verification failed") + } + //Remove the confounder bytes + return b[e.GetConfounderByteSize():], nil +} + +// GetIntegityHash returns a keyed integrity hash of the bytes provided as defined in RFC 8009 +func GetIntegityHash(iv, c, key []byte, usage uint32, e etype.EType) ([]byte, error) { + // Generate and append integrity hash + // Rather than calculating the hash over the confounder and plaintext + // it is calculated over the iv concatenated with the AES cipher output. + ib := append(iv, c...) + return common.GetIntegrityHash(ib, key, usage, e) +} + +// VerifyIntegrity verifies the integrity of cipertext bytes ct. +func VerifyIntegrity(key, ct []byte, usage uint32, etype etype.EType) bool { + h := make([]byte, etype.GetHMACBitLength()/8) + copy(h, ct[len(ct)-etype.GetHMACBitLength()/8:]) + ivz := make([]byte, etype.GetConfounderByteSize()) + ib := append(ivz, ct[:len(ct)-(etype.GetHMACBitLength()/8)]...) + expectedMAC, _ := common.GetIntegrityHash(ib, key, usage, etype) + return hmac.Equal(h, expectedMAC) +} diff --git a/github.com/jcmturner/gokrb5/v8/crypto/rfc8009/keyDerivation.go b/github.com/jcmturner/gokrb5/v8/crypto/rfc8009/keyDerivation.go new file mode 100644 index 0000000000..e94732226c --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/crypto/rfc8009/keyDerivation.go @@ -0,0 +1,135 @@ +package rfc8009 + +import ( + "crypto/hmac" + "encoding/binary" + "encoding/hex" + "errors" + + "github.com/jcmturner/gokrb5/v8/crypto/etype" + "github.com/jcmturner/gokrb5/v8/iana/etypeID" + "golang.org/x/crypto/pbkdf2" +) + +const ( + s2kParamsZero = 32768 +) + +// DeriveRandom for key derivation as defined in RFC 8009 +func DeriveRandom(protocolKey, usage []byte, e etype.EType) ([]byte, error) { + h := e.GetHashFunc()() + return KDF_HMAC_SHA2(protocolKey, []byte("prf"), usage, h.Size(), e), nil +} + +// DeriveKey derives a key from the protocol key based on the usage and the etype's specific methods. +// +// https://tools.ietf.org/html/rfc8009#section-5 +func DeriveKey(protocolKey, label []byte, e etype.EType) []byte { + var context []byte + var kl int + // Key length is longer for aes256-cts-hmac-sha384-192 is it is a Ke or from StringToKey (where label is "kerberos") + if e.GetETypeID() == etypeID.AES256_CTS_HMAC_SHA384_192 { + Swtch: + switch label[len(label)-1] { + case 0x73: + // 0x73 is "s" so label could be kerberos meaning StringToKey so now check if the label is "kerberos" + kerblabel := []byte("kerberos") + if len(label) != len(kerblabel) { + break + } + for i, b := range label { + if b != kerblabel[i] { + kl = e.GetKeySeedBitLength() + break Swtch + } + } + if kl == 0 { + // This is StringToKey + kl = 256 + } + case 0xAA: + // This is a Ke + kl = 256 + } + } + if kl == 0 { + kl = e.GetKeySeedBitLength() + } + return e.RandomToKey(KDF_HMAC_SHA2(protocolKey, label, context, kl, e)) +} + +// RandomToKey returns a key from the bytes provided according to the definition in RFC 8009. +func RandomToKey(b []byte) []byte { + return b +} + +// StringToKey returns a key derived from the string provided according to the definition in RFC 8009. +func StringToKey(secret, salt, s2kparams string, e etype.EType) ([]byte, error) { + i, err := S2KparamsToItertions(s2kparams) + if err != nil { + return nil, err + } + return StringToKeyIter(secret, salt, i, e) +} + +// StringToKeyIter returns a key derived from the string provided according to the definition in RFC 8009. +func StringToKeyIter(secret, salt string, iterations int, e etype.EType) ([]byte, error) { + tkey := e.RandomToKey(StringToPBKDF2(secret, salt, iterations, e)) + return e.DeriveKey(tkey, []byte("kerberos")) +} + +// StringToPBKDF2 generates an encryption key from a pass phrase and salt string using the PBKDF2 function from PKCS #5 v2.0 +func StringToPBKDF2(secret, salt string, iterations int, e etype.EType) []byte { + kl := e.GetKeyByteSize() + if e.GetETypeID() == etypeID.AES256_CTS_HMAC_SHA384_192 { + kl = 32 + } + return pbkdf2.Key([]byte(secret), []byte(salt), iterations, kl, e.GetHashFunc()) +} + +// KDF_HMAC_SHA2 key derivation: https://tools.ietf.org/html/rfc8009#section-3 +func KDF_HMAC_SHA2(protocolKey, label, context []byte, kl int, e etype.EType) []byte { + //k: Length in bits of the key to be outputted, expressed in big-endian binary representation in 4 bytes. + k := make([]byte, 4, 4) + binary.BigEndian.PutUint32(k, uint32(kl)) + + c := make([]byte, 4, 4) + binary.BigEndian.PutUint32(c, uint32(1)) + c = append(c, label...) + c = append(c, byte(0)) + if len(context) > 0 { + c = append(c, context...) + } + c = append(c, k...) + + mac := hmac.New(e.GetHashFunc(), protocolKey) + mac.Write(c) + return mac.Sum(nil)[:(kl / 8)] +} + +// GetSaltP returns the salt value based on the etype name: https://tools.ietf.org/html/rfc8009#section-4 +func GetSaltP(salt, ename string) string { + b := []byte(ename) + b = append(b, byte(0)) + b = append(b, []byte(salt)...) + return string(b) +} + +// S2KparamsToItertions converts the string representation of iterations to an integer for RFC 8009. +func S2KparamsToItertions(s2kparams string) (int, error) { + var i uint32 + if len(s2kparams) != 8 { + return s2kParamsZero, errors.New("Invalid s2kparams length") + } + b, err := hex.DecodeString(s2kparams) + if err != nil { + return s2kParamsZero, errors.New("Invalid s2kparams, cannot decode string to bytes") + } + i = binary.BigEndian.Uint32(b) + //buf := bytes.NewBuffer(b) + //err = binary.Read(buf, binary.BigEndian, &i) + if err != nil { + return s2kParamsZero, errors.New("Invalid s2kparams, cannot convert to big endian int32") + } + return int(i), nil +} diff --git a/github.com/jcmturner/gokrb5/v8/gssapi/MICToken.go b/github.com/jcmturner/gokrb5/v8/gssapi/MICToken.go new file mode 100644 index 0000000000..ab8daa2897 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/gssapi/MICToken.go @@ -0,0 +1,174 @@ +package gssapi + +import ( + "bytes" + "crypto/hmac" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + + "github.com/jcmturner/gokrb5/v8/crypto" + "github.com/jcmturner/gokrb5/v8/iana/keyusage" + "github.com/jcmturner/gokrb5/v8/types" +) + +// RFC 4121, section 4.2.6.1 + +const ( + // MICTokenFlagSentByAcceptor - this flag indicates the sender is the context acceptor. When not set, it indicates the sender is the context initiator + MICTokenFlagSentByAcceptor = 1 << iota + // MICTokenFlagSealed - this flag indicates confidentiality is provided for. It SHALL NOT be set in MIC tokens + MICTokenFlagSealed + // MICTokenFlagAcceptorSubkey - a subkey asserted by the context acceptor is used to protect the message + MICTokenFlagAcceptorSubkey +) + +const ( + micHdrLen = 16 // Length of the MIC Token's header +) + +// MICToken represents a GSS API MIC token, as defined in RFC 4121. +// It contains the header fields, the payload (this is not transmitted) and +// the checksum, and provides the logic for converting to/from bytes plus +// computing and verifying checksums +type MICToken struct { + // const GSS Token ID: 0x0404 + Flags byte // contains three flags: acceptor, sealed, acceptor subkey + // const Filler: 0xFF 0xFF 0xFF 0xFF 0xFF + SndSeqNum uint64 // sender's sequence number. big-endian + Payload []byte // your data! :) + Checksum []byte // checksum of { payload | header } +} + +// Return the 2 bytes identifying a GSS API MIC token +func getGSSMICTokenID() *[2]byte { + return &[2]byte{0x04, 0x04} +} + +// Return the filler bytes used in header +func fillerBytes() *[5]byte { + return &[5]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF} +} + +// Marshal the MICToken into a byte slice. +// The payload should have been set and the checksum computed, otherwise an error is returned. +func (mt *MICToken) Marshal() ([]byte, error) { + if mt.Checksum == nil { + return nil, errors.New("checksum has not been set") + } + + bytes := make([]byte, micHdrLen+len(mt.Checksum)) + copy(bytes[0:micHdrLen], mt.getMICChecksumHeader()[:]) + copy(bytes[micHdrLen:], mt.Checksum) + + return bytes, nil +} + +// SetChecksum uses the passed encryption key and key usage to compute the checksum over the payload and +// the header, and sets the Checksum field of this MICToken. +// If the payload has not been set or the checksum has already been set, an error is returned. +func (mt *MICToken) SetChecksum(key types.EncryptionKey, keyUsage uint32) error { + if mt.Checksum != nil { + return errors.New("checksum has already been computed") + } + checksum, err := mt.checksum(key, keyUsage) + if err != nil { + return err + } + mt.Checksum = checksum + return nil +} + +// Compute and return the checksum of this token, computed using the passed key and key usage. +// Note: This will NOT update the struct's Checksum field. +func (mt *MICToken) checksum(key types.EncryptionKey, keyUsage uint32) ([]byte, error) { + if mt.Payload == nil { + return nil, errors.New("cannot compute checksum with uninitialized payload") + } + d := make([]byte, micHdrLen+len(mt.Payload)) + copy(d[0:], mt.Payload) + copy(d[len(mt.Payload):], mt.getMICChecksumHeader()) + + encType, err := crypto.GetEtype(key.KeyType) + if err != nil { + return nil, err + } + return encType.GetChecksumHash(key.KeyValue, d, keyUsage) +} + +// Build a header suitable for a checksum computation +func (mt *MICToken) getMICChecksumHeader() []byte { + header := make([]byte, micHdrLen) + copy(header[0:2], getGSSMICTokenID()[:]) + header[2] = mt.Flags + copy(header[3:8], fillerBytes()[:]) + binary.BigEndian.PutUint64(header[8:16], mt.SndSeqNum) + return header +} + +// Verify computes the token's checksum with the provided key and usage, +// and compares it to the checksum present in the token. +// In case of any failure, (false, err) is returned, with err an explanatory error. +func (mt *MICToken) Verify(key types.EncryptionKey, keyUsage uint32) (bool, error) { + computed, err := mt.checksum(key, keyUsage) + if err != nil { + return false, err + } + if !hmac.Equal(computed, mt.Checksum) { + return false, fmt.Errorf( + "checksum mismatch. Computed: %s, Contained in token: %s", + hex.EncodeToString(computed), hex.EncodeToString(mt.Checksum)) + } + return true, nil +} + +// Unmarshal bytes into the corresponding MICToken. +// If expectFromAcceptor is true we expect the token to have been emitted by the gss acceptor, +// and will check the according flag, returning an error if the token does not match the expectation. +func (mt *MICToken) Unmarshal(b []byte, expectFromAcceptor bool) error { + if len(b) < micHdrLen { + return errors.New("bytes shorter than header length") + } + if !bytes.Equal(getGSSMICTokenID()[:], b[0:2]) { + return fmt.Errorf("wrong Token ID, Expected %s, was %s", + hex.EncodeToString(getGSSMICTokenID()[:]), + hex.EncodeToString(b[0:2])) + } + flags := b[2] + isFromAcceptor := flags&MICTokenFlagSentByAcceptor != 0 + if isFromAcceptor && !expectFromAcceptor { + return errors.New("unexpected acceptor flag is set: not expecting a token from the acceptor") + } + if !isFromAcceptor && expectFromAcceptor { + return errors.New("unexpected acceptor flag is not set: expecting a token from the acceptor, not in the initiator") + } + if !bytes.Equal(b[3:8], fillerBytes()[:]) { + return fmt.Errorf("unexpected filler bytes: expecting %s, was %s", + hex.EncodeToString(fillerBytes()[:]), + hex.EncodeToString(b[3:8])) + } + + mt.Flags = flags + mt.SndSeqNum = binary.BigEndian.Uint64(b[8:16]) + mt.Checksum = b[micHdrLen:] + return nil +} + +// NewInitiatorMICToken builds a new initiator token (acceptor flag will be set to 0) and computes the authenticated checksum. +// Other flags are set to 0. +// Note that in certain circumstances you may need to provide a sequence number that has been defined earlier. +// This is currently not supported. +func NewInitiatorMICToken(payload []byte, key types.EncryptionKey) (*MICToken, error) { + token := MICToken{ + Flags: 0x00, + SndSeqNum: 0, + Payload: payload, + } + + if err := token.SetChecksum(key, keyusage.GSSAPI_INITIATOR_SIGN); err != nil { + return nil, err + } + + return &token, nil +} diff --git a/github.com/jcmturner/gokrb5/v8/gssapi/README.md b/github.com/jcmturner/gokrb5/v8/gssapi/README.md new file mode 100644 index 0000000000..8fdcf70c32 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/gssapi/README.md @@ -0,0 +1,20 @@ +# Notes on GSS-API Negotiation Mechanism +https://tools.ietf.org/html/rfc4178 + +Client sends an initial negotiation message to the server which specifies the list of mechanisms +the client can support in order of decreasing preference. +This message is generated with the ``NewNegTokenInitKrb5`` method. +The message generated by this function specifies only a kerberos v5 mechanism is supported. + +The RFC states that this message can optionally contain the initial mechanism token +for the preferred mechanism (KRB5 in this case) of the client. The ``NewNegTokenInitKrb5`` +includes this in the message. + +The server side responds to this message with a one of four messages: + +| Message Type/State | Description | +|--------------------|-------------| +| accept-completed | indicates that the initiator-selected mechanism was acceptable to the target, and that the security mechanism token embedded in the first negotiation message was sufficient to complete the authentication | +| accept-incomplete | At least one more message is needed from the client to establish security context. | +| reject | Negotiation is being terminated. | +| request-mic | (this state can only be present in the first reply message from the target) indicates that the MIC token exchange is REQUIRED if per-message integrity services are available | \ No newline at end of file diff --git a/github.com/jcmturner/gokrb5/v8/gssapi/contextFlags.go b/github.com/jcmturner/gokrb5/v8/gssapi/contextFlags.go new file mode 100644 index 0000000000..6634c6d782 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/gssapi/contextFlags.go @@ -0,0 +1,25 @@ +package gssapi + +import "github.com/jcmturner/gofork/encoding/asn1" + +// GSS-API context flags assigned numbers. +const ( + ContextFlagDeleg = 1 + ContextFlagMutual = 2 + ContextFlagReplay = 4 + ContextFlagSequence = 8 + ContextFlagConf = 16 + ContextFlagInteg = 32 + ContextFlagAnon = 64 +) + +// ContextFlags flags for GSSAPI +type ContextFlags asn1.BitString + +// NewContextFlags creates a new ContextFlags instance. +func NewContextFlags() ContextFlags { + var c ContextFlags + c.BitLength = 32 + c.Bytes = make([]byte, 4) + return c +} diff --git a/github.com/jcmturner/gokrb5/v8/gssapi/gssapi.go b/github.com/jcmturner/gokrb5/v8/gssapi/gssapi.go new file mode 100644 index 0000000000..8082231902 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/gssapi/gssapi.go @@ -0,0 +1,202 @@ +// Package gssapi implements Generic Security Services Application Program Interface required for SPNEGO kerberos authentication. +package gssapi + +import ( + "context" + "fmt" + + "github.com/jcmturner/gofork/encoding/asn1" +) + +// GSS-API OID names +const ( + // GSS-API OID names + OIDKRB5 OIDName = "KRB5" // MechType OID for Kerberos 5 + OIDMSLegacyKRB5 OIDName = "MSLegacyKRB5" // MechType OID for Kerberos 5 + OIDSPNEGO OIDName = "SPNEGO" + OIDGSSIAKerb OIDName = "GSSIAKerb" // Indicates the client cannot get a service ticket and asks the server to serve as an intermediate to the target KDC. http://k5wiki.kerberos.org/wiki/Projects/IAKERB#IAKERB_mech +) + +// GSS-API status values +const ( + StatusBadBindings = 1 << iota + StatusBadMech + StatusBadName + StatusBadNameType + StatusBadStatus + StatusBadSig + StatusBadMIC + StatusContextExpired + StatusCredentialsExpired + StatusDefectiveCredential + StatusDefectiveToken + StatusFailure + StatusNoContext + StatusNoCred + StatusBadQOP + StatusUnauthorized + StatusUnavailable + StatusDuplicateElement + StatusNameNotMN + StatusComplete + StatusContinueNeeded + StatusDuplicateToken + StatusOldToken + StatusUnseqToken + StatusGapToken +) + +// ContextToken is an interface for a GSS-API context token. +type ContextToken interface { + Marshal() ([]byte, error) + Unmarshal(b []byte) error + Verify() (bool, Status) + Context() context.Context +} + +/* +CREDENTIAL MANAGEMENT + +GSS_Acquire_cred acquire credentials for use +GSS_Release_cred release credentials after use +GSS_Inquire_cred display information about credentials +GSS_Add_cred construct credentials incrementally +GSS_Inquire_cred_by_mech display per-mechanism credential information + +CONTEXT-LEVEL CALLS + +GSS_Init_sec_context initiate outbound security context +GSS_Accept_sec_context accept inbound security context +GSS_Delete_sec_context flush context when no longer needed +GSS_Process_context_token process received control token on context +GSS_Context_time indicate validity time remaining on context +GSS_Inquire_context display information about context +GSS_Wrap_size_limit determine GSS_Wrap token size limit +GSS_Export_sec_context transfer context to other process +GSS_Import_sec_context import transferred context + +PER-MESSAGE CALLS + +GSS_GetMIC apply integrity check, receive as token separate from message +GSS_VerifyMIC validate integrity check token along with message +GSS_Wrap sign, optionally encrypt, encapsulate +GSS_Unwrap decapsulate, decrypt if needed, validate integrity check + +SUPPORT CALLS + +GSS_Display_status translate status codes to printable form +GSS_Indicate_mechs indicate mech_types supported on local system +GSS_Compare_name compare two names for equality +GSS_Display_name translate name to printable form +GSS_Import_name convert printable name to normalized form +GSS_Release_name free storage of normalized-form name +GSS_Release_buffer free storage of general GSS-allocated object +GSS_Release_OID_set free storage of OID set object +GSS_Create_empty_OID_set create empty OID set +GSS_Add_OID_set_member add member to OID set +GSS_Test_OID_set_member test if OID is member of OID set +GSS_Inquire_names_for_mech indicate name types supported by mechanism +GSS_Inquire_mechs_for_name indicates mechanisms supporting name type +GSS_Canonicalize_name translate name to per-mechanism form +GSS_Export_name externalize per-mechanism name +GSS_Duplicate_name duplicate name object +*/ + +// Mechanism is the GSS-API interface for authentication mechanisms. +type Mechanism interface { + OID() asn1.ObjectIdentifier + AcquireCred() error // acquire credentials for use (eg. AS exchange for KRB5) + InitSecContext() (ContextToken, error) // initiate outbound security context (eg TGS exchange builds AP_REQ to go into ContextToken to send to service) + AcceptSecContext(ct ContextToken) (bool, context.Context, Status) // service verifies the token server side to establish a context + MIC() MICToken // apply integrity check, receive as token separate from message + VerifyMIC(mt MICToken) (bool, error) // validate integrity check token along with message + Wrap(msg []byte) WrapToken // sign, optionally encrypt, encapsulate + Unwrap(wt WrapToken) []byte // decapsulate, decrypt if needed, validate integrity check +} + +// OIDName is the type for defined GSS-API OIDs. +type OIDName string + +// OID returns the OID for the provided OID name. +func (o OIDName) OID() asn1.ObjectIdentifier { + switch o { + case OIDSPNEGO: + return asn1.ObjectIdentifier{1, 3, 6, 1, 5, 5, 2} + case OIDKRB5: + return asn1.ObjectIdentifier{1, 2, 840, 113554, 1, 2, 2} + case OIDMSLegacyKRB5: + return asn1.ObjectIdentifier{1, 2, 840, 48018, 1, 2, 2} + case OIDGSSIAKerb: + return asn1.ObjectIdentifier{1, 3, 6, 1, 5, 2, 5} + } + return asn1.ObjectIdentifier{} +} + +// Status is the GSS-API status and implements the error interface. +type Status struct { + Code int + Message string +} + +// Error returns the Status description. +func (s Status) Error() string { + var str string + switch s.Code { + case StatusBadBindings: + str = "channel binding mismatch" + case StatusBadMech: + str = "unsupported mechanism requested" + case StatusBadName: + str = "invalid name provided" + case StatusBadNameType: + str = "name of unsupported type provided" + case StatusBadStatus: + str = "invalid input status selector" + case StatusBadSig: + str = "token had invalid integrity check" + case StatusBadMIC: + str = "preferred alias for GSS_S_BAD_SIG" + case StatusContextExpired: + str = "specified security context expired" + case StatusCredentialsExpired: + str = "expired credentials detected" + case StatusDefectiveCredential: + str = "defective credential detected" + case StatusDefectiveToken: + str = "defective token detected" + case StatusFailure: + str = "failure, unspecified at GSS-API level" + case StatusNoContext: + str = "no valid security context specified" + case StatusNoCred: + str = "no valid credentials provided" + case StatusBadQOP: + str = "unsupported QOP valu" + case StatusUnauthorized: + str = "operation unauthorized" + case StatusUnavailable: + str = "operation unavailable" + case StatusDuplicateElement: + str = "duplicate credential element requested" + case StatusNameNotMN: + str = "name contains multi-mechanism elements" + case StatusComplete: + str = "normal completion" + case StatusContinueNeeded: + str = "continuation call to routine required" + case StatusDuplicateToken: + str = "duplicate per-message token detected" + case StatusOldToken: + str = "timed-out per-message token detected" + case StatusUnseqToken: + str = "reordered (early) per-message token detected" + case StatusGapToken: + str = "skipped predecessor token(s) detected" + default: + str = "unknown GSS-API error status" + } + if s.Message != "" { + return fmt.Sprintf("%s: %s", str, s.Message) + } + return str +} diff --git a/github.com/jcmturner/gokrb5/v8/gssapi/wrapToken.go b/github.com/jcmturner/gokrb5/v8/gssapi/wrapToken.go new file mode 100644 index 0000000000..c521139504 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/gssapi/wrapToken.go @@ -0,0 +1,193 @@ +package gssapi + +import ( + "bytes" + "crypto/hmac" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + + "github.com/jcmturner/gokrb5/v8/crypto" + "github.com/jcmturner/gokrb5/v8/iana/keyusage" + "github.com/jcmturner/gokrb5/v8/types" +) + +// RFC 4121, section 4.2.6.2 + +const ( + HdrLen = 16 // Length of the Wrap Token's header + FillerByte byte = 0xFF +) + +// WrapToken represents a GSS API Wrap token, as defined in RFC 4121. +// It contains the header fields, the payload and the checksum, and provides +// the logic for converting to/from bytes plus computing and verifying checksums +type WrapToken struct { + // const GSS Token ID: 0x0504 + Flags byte // contains three flags: acceptor, sealed, acceptor subkey + // const Filler: 0xFF + EC uint16 // checksum length. big-endian + RRC uint16 // right rotation count. big-endian + SndSeqNum uint64 // sender's sequence number. big-endian + Payload []byte // your data! :) + CheckSum []byte // authenticated checksum of { payload | header } +} + +// Return the 2 bytes identifying a GSS API Wrap token +func getGssWrapTokenId() *[2]byte { + return &[2]byte{0x05, 0x04} +} + +// Marshal the WrapToken into a byte slice. +// The payload should have been set and the checksum computed, otherwise an error is returned. +func (wt *WrapToken) Marshal() ([]byte, error) { + if wt.CheckSum == nil { + return nil, errors.New("checksum has not been set") + } + if wt.Payload == nil { + return nil, errors.New("payload has not been set") + } + + pldOffset := HdrLen // Offset of the payload in the token + chkSOffset := HdrLen + len(wt.Payload) // Offset of the checksum in the token + + bytes := make([]byte, chkSOffset+int(wt.EC)) + copy(bytes[0:], getGssWrapTokenId()[:]) + bytes[2] = wt.Flags + bytes[3] = FillerByte + binary.BigEndian.PutUint16(bytes[4:6], wt.EC) + binary.BigEndian.PutUint16(bytes[6:8], wt.RRC) + binary.BigEndian.PutUint64(bytes[8:16], wt.SndSeqNum) + copy(bytes[pldOffset:], wt.Payload) + copy(bytes[chkSOffset:], wt.CheckSum) + return bytes, nil +} + +// SetCheckSum uses the passed encryption key and key usage to compute the checksum over the payload and +// the header, and sets the CheckSum field of this WrapToken. +// If the payload has not been set or the checksum has already been set, an error is returned. +func (wt *WrapToken) SetCheckSum(key types.EncryptionKey, keyUsage uint32) error { + if wt.Payload == nil { + return errors.New("payload has not been set") + } + if wt.CheckSum != nil { + return errors.New("checksum has already been computed") + } + chkSum, cErr := wt.computeCheckSum(key, keyUsage) + if cErr != nil { + return cErr + } + wt.CheckSum = chkSum + return nil +} + +// ComputeCheckSum computes and returns the checksum of this token, computed using the passed key and key usage. +// Note: This will NOT update the struct's Checksum field. +func (wt *WrapToken) computeCheckSum(key types.EncryptionKey, keyUsage uint32) ([]byte, error) { + if wt.Payload == nil { + return nil, errors.New("cannot compute checksum with uninitialized payload") + } + // Build a slice containing { payload | header } + checksumMe := make([]byte, HdrLen+len(wt.Payload)) + copy(checksumMe[0:], wt.Payload) + copy(checksumMe[len(wt.Payload):], getChecksumHeader(wt.Flags, wt.SndSeqNum)) + + encType, err := crypto.GetEtype(key.KeyType) + if err != nil { + return nil, err + } + return encType.GetChecksumHash(key.KeyValue, checksumMe, keyUsage) +} + +// Build a header suitable for a checksum computation +func getChecksumHeader(flags byte, senderSeqNum uint64) []byte { + header := make([]byte, 16) + copy(header[0:], []byte{0x05, 0x04, flags, 0xFF, 0x00, 0x00, 0x00, 0x00}) + binary.BigEndian.PutUint64(header[8:], senderSeqNum) + return header +} + +// Verify computes the token's checksum with the provided key and usage, +// and compares it to the checksum present in the token. +// In case of any failure, (false, Err) is returned, with Err an explanatory error. +func (wt *WrapToken) Verify(key types.EncryptionKey, keyUsage uint32) (bool, error) { + computed, cErr := wt.computeCheckSum(key, keyUsage) + if cErr != nil { + return false, cErr + } + if !hmac.Equal(computed, wt.CheckSum) { + return false, fmt.Errorf( + "checksum mismatch. Computed: %s, Contained in token: %s", + hex.EncodeToString(computed), hex.EncodeToString(wt.CheckSum)) + } + return true, nil +} + +// Unmarshal bytes into the corresponding WrapToken. +// If expectFromAcceptor is true, we expect the token to have been emitted by the gss acceptor, +// and will check the according flag, returning an error if the token does not match the expectation. +func (wt *WrapToken) Unmarshal(b []byte, expectFromAcceptor bool) error { + // Check if we can read a whole header + if len(b) < 16 { + return errors.New("bytes shorter than header length") + } + // Is the Token ID correct? + if !bytes.Equal(getGssWrapTokenId()[:], b[0:2]) { + return fmt.Errorf("wrong Token ID. Expected %s, was %s", + hex.EncodeToString(getGssWrapTokenId()[:]), + hex.EncodeToString(b[0:2])) + } + // Check the acceptor flag + flags := b[2] + isFromAcceptor := flags&0x01 == 1 + if isFromAcceptor && !expectFromAcceptor { + return errors.New("unexpected acceptor flag is set: not expecting a token from the acceptor") + } + if !isFromAcceptor && expectFromAcceptor { + return errors.New("expected acceptor flag is not set: expecting a token from the acceptor, not the initiator") + } + // Check the filler byte + if b[3] != FillerByte { + return fmt.Errorf("unexpected filler byte: expecting 0xFF, was %s ", hex.EncodeToString(b[3:4])) + } + checksumL := binary.BigEndian.Uint16(b[4:6]) + // Sanity check on the checksum length + if int(checksumL) > len(b)-HdrLen { + return fmt.Errorf("inconsistent checksum length: %d bytes to parse, checksum length is %d", len(b), checksumL) + } + + wt.Flags = flags + wt.EC = checksumL + wt.RRC = binary.BigEndian.Uint16(b[6:8]) + wt.SndSeqNum = binary.BigEndian.Uint64(b[8:16]) + wt.Payload = b[16 : len(b)-int(checksumL)] + wt.CheckSum = b[len(b)-int(checksumL):] + return nil +} + +// NewInitiatorWrapToken builds a new initiator token (acceptor flag will be set to 0) and computes the authenticated checksum. +// Other flags are set to 0, and the RRC and sequence number are initialized to 0. +// Note that in certain circumstances you may need to provide a sequence number that has been defined earlier. +// This is currently not supported. +func NewInitiatorWrapToken(payload []byte, key types.EncryptionKey) (*WrapToken, error) { + encType, err := crypto.GetEtype(key.KeyType) + if err != nil { + return nil, err + } + + token := WrapToken{ + Flags: 0x00, // all zeroed out (this is a token sent by the initiator) + // Checksum size: length of output of the HMAC function, in bytes. + EC: uint16(encType.GetHMACBitLength() / 8), + RRC: 0, + SndSeqNum: 0, + Payload: payload, + } + + if err := token.SetCheckSum(key, keyusage.GSSAPI_INITIATOR_SEAL); err != nil { + return nil, err + } + + return &token, nil +} diff --git a/github.com/jcmturner/gokrb5/v8/iana/addrtype/constants.go b/github.com/jcmturner/gokrb5/v8/iana/addrtype/constants.go new file mode 100644 index 0000000000..457b89d7ac --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/iana/addrtype/constants.go @@ -0,0 +1,15 @@ +// Package addrtype provides Address type assigned numbers. +package addrtype + +// Address type IDs. +const ( + IPv4 int32 = 2 + Directional int32 = 3 + ChaosNet int32 = 5 + XNS int32 = 6 + ISO int32 = 7 + DECNETPhaseIV int32 = 12 + AppleTalkDDP int32 = 16 + NetBios int32 = 20 + IPv6 int32 = 24 +) diff --git a/github.com/jcmturner/gokrb5/v8/iana/adtype/constants.go b/github.com/jcmturner/gokrb5/v8/iana/adtype/constants.go new file mode 100644 index 0000000000..e805b74666 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/iana/adtype/constants.go @@ -0,0 +1,23 @@ +// Package adtype provides Authenticator type assigned numbers. +package adtype + +// Authenticator type IDs. +const ( + ADIfRelevant int32 = 1 + ADIntendedForServer int32 = 2 + ADIntendedForApplicationClass int32 = 3 + ADKDCIssued int32 = 4 + ADAndOr int32 = 5 + ADMandatoryTicketExtensions int32 = 6 + ADInTicketExtensions int32 = 7 + ADMandatoryForKDC int32 = 8 + OSFDCE int32 = 64 + SESAME int32 = 65 + ADOSFDCEPKICertID int32 = 66 + ADAuthenticationStrength int32 = 70 + ADFXFastArmor int32 = 71 + ADFXFastUsed int32 = 72 + ADWin2KPAC int32 = 128 + ADEtypeNegotiation int32 = 129 + //Reserved values 9-63 +) diff --git a/github.com/jcmturner/gokrb5/v8/iana/asnAppTag/constants.go b/github.com/jcmturner/gokrb5/v8/iana/asnAppTag/constants.go new file mode 100644 index 0000000000..d74cd60e93 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/iana/asnAppTag/constants.go @@ -0,0 +1,24 @@ +// Package asnAppTag provides ASN1 application tag numbers. +package asnAppTag + +// ASN1 application tag numbers. +const ( + Ticket = 1 + Authenticator = 2 + EncTicketPart = 3 + ASREQ = 10 + TGSREQ = 12 + ASREP = 11 + TGSREP = 13 + APREQ = 14 + APREP = 15 + KRBSafe = 20 + KRBPriv = 21 + KRBCred = 22 + EncASRepPart = 25 + EncTGSRepPart = 26 + EncAPRepPart = 27 + EncKrbPrivPart = 28 + EncKrbCredPart = 29 + KRBError = 30 +) diff --git a/github.com/jcmturner/gokrb5/v8/iana/chksumtype/constants.go b/github.com/jcmturner/gokrb5/v8/iana/chksumtype/constants.go new file mode 100644 index 0000000000..93db952dda --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/iana/chksumtype/constants.go @@ -0,0 +1,32 @@ +// Package chksumtype provides Kerberos 5 checksum type assigned numbers. +package chksumtype + +// Checksum type IDs. +const ( + //RESERVED : 0 + CRC32 int32 = 1 + RSA_MD4 int32 = 2 + RSA_MD4_DES int32 = 3 + DES_MAC int32 = 4 + DES_MAC_K int32 = 5 + RSA_MD4_DES_K int32 = 6 + RSA_MD5 int32 = 7 + RSA_MD5_DES int32 = 8 + RSA_MD5_DES3 int32 = 9 + SHA1_ID10 int32 = 10 + //UNASSIGNED : 11 + HMAC_SHA1_DES3_KD int32 = 12 + HMAC_SHA1_DES3 int32 = 13 + SHA1_ID14 int32 = 14 + HMAC_SHA1_96_AES128 int32 = 15 + HMAC_SHA1_96_AES256 int32 = 16 + CMAC_CAMELLIA128 int32 = 17 + CMAC_CAMELLIA256 int32 = 18 + HMAC_SHA256_128_AES128 int32 = 19 + HMAC_SHA384_192_AES256 int32 = 20 + //UNASSIGNED : 21-32770 + GSSAPI int32 = 32771 + //UNASSIGNED : 32772-2147483647 + KERB_CHECKSUM_HMAC_MD5_UNSIGNED uint32 = 4294967158 // 0xFFFFFF76 documentation says this is -138 but in an unsigned int this is 4294967158 + KERB_CHECKSUM_HMAC_MD5 int32 = -138 +) diff --git a/github.com/jcmturner/gokrb5/v8/iana/constants.go b/github.com/jcmturner/gokrb5/v8/iana/constants.go new file mode 100644 index 0000000000..0b8e916d5b --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/iana/constants.go @@ -0,0 +1,5 @@ +// Package iana provides Kerberos 5 assigned numbers. +package iana + +// PVNO is the Protocol Version Number. +const PVNO = 5 diff --git a/github.com/jcmturner/gokrb5/v8/iana/errorcode/constants.go b/github.com/jcmturner/gokrb5/v8/iana/errorcode/constants.go new file mode 100644 index 0000000000..fd756bc5e3 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/iana/errorcode/constants.go @@ -0,0 +1,155 @@ +// Package errorcode provides Kerberos 5 assigned error codes. +package errorcode + +import "fmt" + +// Kerberos error codes. +const ( + KDC_ERR_NONE int32 = 0 //No error + KDC_ERR_NAME_EXP int32 = 1 //Client's entry in database has expired + KDC_ERR_SERVICE_EXP int32 = 2 //Server's entry in database has expired + KDC_ERR_BAD_PVNO int32 = 3 //Requested protocol version number not supported + KDC_ERR_C_OLD_MAST_KVNO int32 = 4 //Client's key encrypted in old master key + KDC_ERR_S_OLD_MAST_KVNO int32 = 5 //Server's key encrypted in old master key + KDC_ERR_C_PRINCIPAL_UNKNOWN int32 = 6 //Client not found in Kerberos database + KDC_ERR_S_PRINCIPAL_UNKNOWN int32 = 7 //Server not found in Kerberos database + KDC_ERR_PRINCIPAL_NOT_UNIQUE int32 = 8 //Multiple principal entries in database + KDC_ERR_NULL_KEY int32 = 9 //The client or server has a null key + KDC_ERR_CANNOT_POSTDATE int32 = 10 //Ticket not eligible for postdating + KDC_ERR_NEVER_VALID int32 = 11 //Requested starttime is later than end time + KDC_ERR_POLICY int32 = 12 //KDC policy rejects request + KDC_ERR_BADOPTION int32 = 13 //KDC cannot accommodate requested option + KDC_ERR_ETYPE_NOSUPP int32 = 14 //KDC has no support for encryption type + KDC_ERR_SUMTYPE_NOSUPP int32 = 15 //KDC has no support for checksum type + KDC_ERR_PADATA_TYPE_NOSUPP int32 = 16 //KDC has no support for padata type + KDC_ERR_TRTYPE_NOSUPP int32 = 17 //KDC has no support for transited type + KDC_ERR_CLIENT_REVOKED int32 = 18 //Clients credentials have been revoked + KDC_ERR_SERVICE_REVOKED int32 = 19 //Credentials for server have been revoked + KDC_ERR_TGT_REVOKED int32 = 20 //TGT has been revoked + KDC_ERR_CLIENT_NOTYET int32 = 21 //Client not yet valid; try again later + KDC_ERR_SERVICE_NOTYET int32 = 22 //Server not yet valid; try again later + KDC_ERR_KEY_EXPIRED int32 = 23 //Password has expired; change password to reset + KDC_ERR_PREAUTH_FAILED int32 = 24 //Pre-authentication information was invalid + KDC_ERR_PREAUTH_REQUIRED int32 = 25 //Additional pre-authentication required + KDC_ERR_SERVER_NOMATCH int32 = 26 //Requested server and ticket don't match + KDC_ERR_MUST_USE_USER2USER int32 = 27 //Server principal valid for user2user only + KDC_ERR_PATH_NOT_ACCEPTED int32 = 28 //KDC Policy rejects transited path + KDC_ERR_SVC_UNAVAILABLE int32 = 29 //A service is not available + KRB_AP_ERR_BAD_INTEGRITY int32 = 31 //Integrity check on decrypted field failed + KRB_AP_ERR_TKT_EXPIRED int32 = 32 //Ticket expired + KRB_AP_ERR_TKT_NYV int32 = 33 //Ticket not yet valid + KRB_AP_ERR_REPEAT int32 = 34 //Request is a replay + KRB_AP_ERR_NOT_US int32 = 35 //The ticket isn't for us + KRB_AP_ERR_BADMATCH int32 = 36 //Ticket and authenticator don't match + KRB_AP_ERR_SKEW int32 = 37 //Clock skew too great + KRB_AP_ERR_BADADDR int32 = 38 //Incorrect net address + KRB_AP_ERR_BADVERSION int32 = 39 //Protocol version mismatch + KRB_AP_ERR_MSG_TYPE int32 = 40 //Invalid msg type + KRB_AP_ERR_MODIFIED int32 = 41 //Message stream modified + KRB_AP_ERR_BADORDER int32 = 42 //Message out of order + KRB_AP_ERR_BADKEYVER int32 = 44 //Specified version of key is not available + KRB_AP_ERR_NOKEY int32 = 45 //Service key not available + KRB_AP_ERR_MUT_FAIL int32 = 46 //Mutual authentication failed + KRB_AP_ERR_BADDIRECTION int32 = 47 //Incorrect message direction + KRB_AP_ERR_METHOD int32 = 48 //Alternative authentication method required + KRB_AP_ERR_BADSEQ int32 = 49 //Incorrect sequence number in message + KRB_AP_ERR_INAPP_CKSUM int32 = 50 //Inappropriate type of checksum in message + KRB_AP_PATH_NOT_ACCEPTED int32 = 51 //Policy rejects transited path + KRB_ERR_RESPONSE_TOO_BIG int32 = 52 //Response too big for UDP; retry with TCP + KRB_ERR_GENERIC int32 = 60 //Generic error (description in e-text) + KRB_ERR_FIELD_TOOLONG int32 = 61 //Field is too long for this implementation + KDC_ERROR_CLIENT_NOT_TRUSTED int32 = 62 //Reserved for PKINIT + KDC_ERROR_KDC_NOT_TRUSTED int32 = 63 //Reserved for PKINIT + KDC_ERROR_INVALID_SIG int32 = 64 //Reserved for PKINIT + KDC_ERR_KEY_TOO_WEAK int32 = 65 //Reserved for PKINIT + KDC_ERR_CERTIFICATE_MISMATCH int32 = 66 //Reserved for PKINIT + KRB_AP_ERR_NO_TGT int32 = 67 //No TGT available to validate USER-TO-USER + KDC_ERR_WRONG_REALM int32 = 68 //Reserved for future use + KRB_AP_ERR_USER_TO_USER_REQUIRED int32 = 69 //Ticket must be for USER-TO-USER + KDC_ERR_CANT_VERIFY_CERTIFICATE int32 = 70 //Reserved for PKINIT + KDC_ERR_INVALID_CERTIFICATE int32 = 71 //Reserved for PKINIT + KDC_ERR_REVOKED_CERTIFICATE int32 = 72 //Reserved for PKINIT + KDC_ERR_REVOCATION_STATUS_UNKNOWN int32 = 73 //Reserved for PKINIT + KDC_ERR_REVOCATION_STATUS_UNAVAILABLE int32 = 74 //Reserved for PKINIT + KDC_ERR_CLIENT_NAME_MISMATCH int32 = 75 //Reserved for PKINIT + KDC_ERR_KDC_NAME_MISMATCH int32 = 76 //Reserved for PKINIT +) + +// Lookup an error code description. +func Lookup(i int32) string { + if s, ok := errorcodeLookup[i]; ok { + return fmt.Sprintf("(%d) %s", i, s) + } + return fmt.Sprintf("Unknown ErrorCode %d", i) +} + +var errorcodeLookup = map[int32]string{ + KDC_ERR_NONE: "KDC_ERR_NONE No error", + KDC_ERR_NAME_EXP: "KDC_ERR_NAME_EXP Client's entry in database has expired", + KDC_ERR_SERVICE_EXP: "KDC_ERR_SERVICE_EXP Server's entry in database has expired", + KDC_ERR_BAD_PVNO: "KDC_ERR_BAD_PVNO Requested protocol version number not supported", + KDC_ERR_C_OLD_MAST_KVNO: "KDC_ERR_C_OLD_MAST_KVNO Client's key encrypted in old master key", + KDC_ERR_S_OLD_MAST_KVNO: "KDC_ERR_S_OLD_MAST_KVNO Server's key encrypted in old master key", + KDC_ERR_C_PRINCIPAL_UNKNOWN: "KDC_ERR_C_PRINCIPAL_UNKNOWN Client not found in Kerberos database", + KDC_ERR_S_PRINCIPAL_UNKNOWN: "KDC_ERR_S_PRINCIPAL_UNKNOWN Server not found in Kerberos database", + KDC_ERR_PRINCIPAL_NOT_UNIQUE: "KDC_ERR_PRINCIPAL_NOT_UNIQUE Multiple principal entries in database", + KDC_ERR_NULL_KEY: "KDC_ERR_NULL_KEY The client or server has a null key", + KDC_ERR_CANNOT_POSTDATE: "KDC_ERR_CANNOT_POSTDATE Ticket not eligible for postdating", + KDC_ERR_NEVER_VALID: "KDC_ERR_NEVER_VALID Requested starttime is later than end time", + KDC_ERR_POLICY: "KDC_ERR_POLICY KDC policy rejects request", + KDC_ERR_BADOPTION: "KDC_ERR_BADOPTION KDC cannot accommodate requested option", + KDC_ERR_ETYPE_NOSUPP: "KDC_ERR_ETYPE_NOSUPP KDC has no support for encryption type", + KDC_ERR_SUMTYPE_NOSUPP: "KDC_ERR_SUMTYPE_NOSUPP KDC has no support for checksum type", + KDC_ERR_PADATA_TYPE_NOSUPP: "KDC_ERR_PADATA_TYPE_NOSUPP KDC has no support for padata type", + KDC_ERR_TRTYPE_NOSUPP: "KDC_ERR_TRTYPE_NOSUPP KDC has no support for transited type", + KDC_ERR_CLIENT_REVOKED: "KDC_ERR_CLIENT_REVOKED Clients credentials have been revoked", + KDC_ERR_SERVICE_REVOKED: "KDC_ERR_SERVICE_REVOKED Credentials for server have been revoked", + KDC_ERR_TGT_REVOKED: "KDC_ERR_TGT_REVOKED TGT has been revoked", + KDC_ERR_CLIENT_NOTYET: "KDC_ERR_CLIENT_NOTYET Client not yet valid; try again later", + KDC_ERR_SERVICE_NOTYET: "KDC_ERR_SERVICE_NOTYET Server not yet valid; try again later", + KDC_ERR_KEY_EXPIRED: "KDC_ERR_KEY_EXPIRED Password has expired; change password to reset", + KDC_ERR_PREAUTH_FAILED: "KDC_ERR_PREAUTH_FAILED Pre-authentication information was invalid", + KDC_ERR_PREAUTH_REQUIRED: "KDC_ERR_PREAUTH_REQUIRED Additional pre-authentication required", + KDC_ERR_SERVER_NOMATCH: "KDC_ERR_SERVER_NOMATCH Requested server and ticket don't match", + KDC_ERR_MUST_USE_USER2USER: "KDC_ERR_MUST_USE_USER2USER Server principal valid for user2user only", + KDC_ERR_PATH_NOT_ACCEPTED: "KDC_ERR_PATH_NOT_ACCEPTED KDC Policy rejects transited path", + KDC_ERR_SVC_UNAVAILABLE: "KDC_ERR_SVC_UNAVAILABLE A service is not available", + KRB_AP_ERR_BAD_INTEGRITY: "KRB_AP_ERR_BAD_INTEGRITY Integrity check on decrypted field failed", + KRB_AP_ERR_TKT_EXPIRED: "KRB_AP_ERR_TKT_EXPIRED Ticket expired", + KRB_AP_ERR_TKT_NYV: "KRB_AP_ERR_TKT_NYV Ticket not yet valid", + KRB_AP_ERR_REPEAT: "KRB_AP_ERR_REPEAT Request is a replay", + KRB_AP_ERR_NOT_US: "KRB_AP_ERR_NOT_US The ticket isn't for us", + KRB_AP_ERR_BADMATCH: "KRB_AP_ERR_BADMATCH Ticket and authenticator don't match", + KRB_AP_ERR_SKEW: "KRB_AP_ERR_SKEW Clock skew too great", + KRB_AP_ERR_BADADDR: "KRB_AP_ERR_BADADDR Incorrect net address", + KRB_AP_ERR_BADVERSION: "KRB_AP_ERR_BADVERSION Protocol version mismatch", + KRB_AP_ERR_MSG_TYPE: "KRB_AP_ERR_MSG_TYPE Invalid msg type", + KRB_AP_ERR_MODIFIED: "KRB_AP_ERR_MODIFIED Message stream modified", + KRB_AP_ERR_BADORDER: "KRB_AP_ERR_BADORDER Message out of order", + KRB_AP_ERR_BADKEYVER: "KRB_AP_ERR_BADKEYVER Specified version of key is not available", + KRB_AP_ERR_NOKEY: "KRB_AP_ERR_NOKEY Service key not available", + KRB_AP_ERR_MUT_FAIL: "KRB_AP_ERR_MUT_FAIL Mutual authentication failed", + KRB_AP_ERR_BADDIRECTION: "KRB_AP_ERR_BADDIRECTION Incorrect message direction", + KRB_AP_ERR_METHOD: "KRB_AP_ERR_METHOD Alternative authentication method required", + KRB_AP_ERR_BADSEQ: "KRB_AP_ERR_BADSEQ Incorrect sequence number in message", + KRB_AP_ERR_INAPP_CKSUM: "KRB_AP_ERR_INAPP_CKSUM Inappropriate type of checksum in message", + KRB_AP_PATH_NOT_ACCEPTED: "KRB_AP_PATH_NOT_ACCEPTED Policy rejects transited path", + KRB_ERR_RESPONSE_TOO_BIG: "KRB_ERR_RESPONSE_TOO_BIG Response too big for UDP; retry with TCP", + KRB_ERR_GENERIC: "KRB_ERR_GENERIC Generic error (description in e-text)", + KRB_ERR_FIELD_TOOLONG: "KRB_ERR_FIELD_TOOLONG Field is too long for this implementation", + KDC_ERROR_CLIENT_NOT_TRUSTED: "KDC_ERROR_CLIENT_NOT_TRUSTED Reserved for PKINIT", + KDC_ERROR_KDC_NOT_TRUSTED: "KDC_ERROR_KDC_NOT_TRUSTED Reserved for PKINIT", + KDC_ERROR_INVALID_SIG: "KDC_ERROR_INVALID_SIG Reserved for PKINIT", + KDC_ERR_KEY_TOO_WEAK: "KDC_ERR_KEY_TOO_WEAK Reserved for PKINIT", + KDC_ERR_CERTIFICATE_MISMATCH: "KDC_ERR_CERTIFICATE_MISMATCH Reserved for PKINIT", + KRB_AP_ERR_NO_TGT: "KRB_AP_ERR_NO_TGT No TGT available to validate USER-TO-USER", + KDC_ERR_WRONG_REALM: "KDC_ERR_WRONG_REALM Reserved for future use", + KRB_AP_ERR_USER_TO_USER_REQUIRED: "KRB_AP_ERR_USER_TO_USER_REQUIRED Ticket must be for USER-TO-USER", + KDC_ERR_CANT_VERIFY_CERTIFICATE: "KDC_ERR_CANT_VERIFY_CERTIFICATE Reserved for PKINIT", + KDC_ERR_INVALID_CERTIFICATE: "KDC_ERR_INVALID_CERTIFICATE Reserved for PKINIT", + KDC_ERR_REVOKED_CERTIFICATE: "KDC_ERR_REVOKED_CERTIFICATE Reserved for PKINIT", + KDC_ERR_REVOCATION_STATUS_UNKNOWN: "KDC_ERR_REVOCATION_STATUS_UNKNOWN Reserved for PKINIT", + KDC_ERR_REVOCATION_STATUS_UNAVAILABLE: "KDC_ERR_REVOCATION_STATUS_UNAVAILABLE Reserved for PKINIT", + KDC_ERR_CLIENT_NAME_MISMATCH: "KDC_ERR_CLIENT_NAME_MISMATCH Reserved for PKINIT", + KDC_ERR_KDC_NAME_MISMATCH: "KDC_ERR_KDC_NAME_MISMATCH Reserved for PKINIT", +} diff --git a/github.com/jcmturner/gokrb5/v8/iana/etypeID/constants.go b/github.com/jcmturner/gokrb5/v8/iana/etypeID/constants.go new file mode 100644 index 0000000000..46a0d748fb --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/iana/etypeID/constants.go @@ -0,0 +1,101 @@ +// Package etypeID provides Kerberos 5 encryption type assigned numbers. +package etypeID + +// Kerberos encryption type assigned numbers. +const ( + //RESERVED : 0 + DES_CBC_CRC int32 = 1 + DES_CBC_MD4 int32 = 2 + DES_CBC_MD5 int32 = 3 + DES_CBC_RAW int32 = 4 + DES3_CBC_MD5 int32 = 5 + DES3_CBC_RAW int32 = 6 + DES3_CBC_SHA1 int32 = 7 + DES_HMAC_SHA1 int32 = 8 + DSAWITHSHA1_CMSOID int32 = 9 + MD5WITHRSAENCRYPTION_CMSOID int32 = 10 + SHA1WITHRSAENCRYPTION_CMSOID int32 = 11 + RC2CBC_ENVOID int32 = 12 + RSAENCRYPTION_ENVOID int32 = 13 + RSAES_OAEP_ENV_OID int32 = 14 + DES_EDE3_CBC_ENV_OID int32 = 15 + DES3_CBC_SHA1_KD int32 = 16 + AES128_CTS_HMAC_SHA1_96 int32 = 17 + AES256_CTS_HMAC_SHA1_96 int32 = 18 + AES128_CTS_HMAC_SHA256_128 int32 = 19 + AES256_CTS_HMAC_SHA384_192 int32 = 20 + //UNASSIGNED : 21-22 + RC4_HMAC int32 = 23 + RC4_HMAC_EXP int32 = 24 + CAMELLIA128_CTS_CMAC int32 = 25 + CAMELLIA256_CTS_CMAC int32 = 26 + //UNASSIGNED : 27-64 + SUBKEY_KEYMATERIAL int32 = 65 + //UNASSIGNED : 66-2147483647 +) + +// ETypesByName is a map of EncType names to their assigned EncType number. +var ETypesByName = map[string]int32{ + "des-cbc-crc": DES_CBC_CRC, + "des-cbc-md4": DES_CBC_MD4, + "des-cbc-md5": DES_CBC_MD5, + "des-cbc-raw": DES_CBC_RAW, + "des3-cbc-md5": DES3_CBC_MD5, + "des3-cbc-raw": DES3_CBC_RAW, + "des3-cbc-sha1": DES3_CBC_SHA1, + "des3-hmac-sha1": DES_HMAC_SHA1, + "des3-cbc-sha1-kd": DES3_CBC_SHA1_KD, + "des-hmac-sha1": DES_HMAC_SHA1, + "dsaWithSHA1-CmsOID": DSAWITHSHA1_CMSOID, + "md5WithRSAEncryption-CmsOID": MD5WITHRSAENCRYPTION_CMSOID, + "sha1WithRSAEncryption-CmsOID": SHA1WITHRSAENCRYPTION_CMSOID, + "rc2CBC-EnvOID": RC2CBC_ENVOID, + "rsaEncryption-EnvOID": RSAENCRYPTION_ENVOID, + "rsaES-OAEP-ENV-OID": RSAES_OAEP_ENV_OID, + "des-ede3-cbc-Env-OID": DES_EDE3_CBC_ENV_OID, + "aes128-cts-hmac-sha1-96": AES128_CTS_HMAC_SHA1_96, + "aes128-cts": AES128_CTS_HMAC_SHA1_96, + "aes128-sha1": AES128_CTS_HMAC_SHA1_96, + "aes256-cts-hmac-sha1-96": AES256_CTS_HMAC_SHA1_96, + "aes256-cts": AES256_CTS_HMAC_SHA1_96, + "aes256-sha1": AES256_CTS_HMAC_SHA1_96, + "aes128-cts-hmac-sha256-128": AES128_CTS_HMAC_SHA256_128, + "aes128-sha2": AES128_CTS_HMAC_SHA256_128, + "aes256-cts-hmac-sha384-192": AES256_CTS_HMAC_SHA384_192, + "aes256-sha2": AES256_CTS_HMAC_SHA384_192, + "arcfour-hmac": RC4_HMAC, + "rc4-hmac": RC4_HMAC, + "arcfour-hmac-md5": RC4_HMAC, + "arcfour-hmac-exp": RC4_HMAC_EXP, + "rc4-hmac-exp": RC4_HMAC_EXP, + "arcfour-hmac-md5-exp": RC4_HMAC_EXP, + "camellia128-cts-cmac": CAMELLIA128_CTS_CMAC, + "camellia128-cts": CAMELLIA128_CTS_CMAC, + "camellia256-cts-cmac": CAMELLIA256_CTS_CMAC, + "camellia256-cts": CAMELLIA256_CTS_CMAC, + "subkey-keymaterial": SUBKEY_KEYMATERIAL, +} + +// EtypeSupported resolves the etype name string to the etype ID. +// If zero is returned the etype is not supported by gokrb5. +func EtypeSupported(etype string) int32 { + // Slice of supported enctype IDs + s := []int32{ + AES128_CTS_HMAC_SHA1_96, + AES256_CTS_HMAC_SHA1_96, + AES128_CTS_HMAC_SHA256_128, + AES256_CTS_HMAC_SHA384_192, + DES3_CBC_SHA1_KD, + RC4_HMAC, + } + id := ETypesByName[etype] + if id == 0 { + return id + } + for _, sid := range s { + if id == sid { + return id + } + } + return 0 +} diff --git a/github.com/jcmturner/gokrb5/v8/iana/flags/constants.go b/github.com/jcmturner/gokrb5/v8/iana/flags/constants.go new file mode 100644 index 0000000000..787801f8f9 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/iana/flags/constants.go @@ -0,0 +1,36 @@ +// Package flags provides Kerberos 5 flag assigned numbers. +package flags + +// Flag values for KRB5 messages and tickets. +const ( + Reserved = 0 + Forwardable = 1 + Forwarded = 2 + Proxiable = 3 + Proxy = 4 + AllowPostDate = 5 + MayPostDate = 5 + PostDated = 6 + Invalid = 7 + Renewable = 8 + Initial = 9 + PreAuthent = 10 + HWAuthent = 11 + OptHardwareAuth = 11 + RequestAnonymous = 12 + TransitedPolicyChecked = 12 + OKAsDelegate = 13 + EncPARep = 15 + Canonicalize = 15 + DisableTransitedCheck = 26 + RenewableOK = 27 + EncTktInSkey = 28 + Renew = 30 + Validate = 31 + + // AP Option Flags + // 0 Reserved for future use. + APOptionUseSessionKey = 1 + APOptionMutualRequired = 2 + // 3-31 Reserved for future use. +) diff --git a/github.com/jcmturner/gokrb5/v8/iana/keyusage/constants.go b/github.com/jcmturner/gokrb5/v8/iana/keyusage/constants.go new file mode 100644 index 0000000000..5b232d1d40 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/iana/keyusage/constants.go @@ -0,0 +1,42 @@ +// Package keyusage provides Kerberos 5 key usage assigned numbers. +package keyusage + +// Key usage numbers. +const ( + AS_REQ_PA_ENC_TIMESTAMP = 1 + KDC_REP_TICKET = 2 + AS_REP_ENCPART = 3 + TGS_REQ_KDC_REQ_BODY_AUTHDATA_SESSION_KEY = 4 + TGS_REQ_KDC_REQ_BODY_AUTHDATA_SUB_KEY = 5 + TGS_REQ_PA_TGS_REQ_AP_REQ_AUTHENTICATOR_CHKSUM = 6 + TGS_REQ_PA_TGS_REQ_AP_REQ_AUTHENTICATOR = 7 + TGS_REP_ENCPART_SESSION_KEY = 8 + TGS_REP_ENCPART_AUTHENTICATOR_SUB_KEY = 9 + AP_REQ_AUTHENTICATOR_CHKSUM = 10 + AP_REQ_AUTHENTICATOR = 11 + AP_REP_ENCPART = 12 + KRB_PRIV_ENCPART = 13 + KRB_CRED_ENCPART = 14 + KRB_SAFE_CHKSUM = 15 + KERB_NON_KERB_SALT = 16 + KERB_NON_KERB_CKSUM_SALT = 17 + //18. Reserved for future use in Kerberos and related protocols. + AD_KDC_ISSUED_CHKSUM = 19 + //20-21. Reserved for future use in Kerberos and related protocols. + GSSAPI_ACCEPTOR_SEAL = 22 + GSSAPI_ACCEPTOR_SIGN = 23 + GSSAPI_INITIATOR_SEAL = 24 + GSSAPI_INITIATOR_SIGN = 25 + KEY_USAGE_FAST_REQ_CHKSUM = 50 + KEY_USAGE_FAST_ENC = 51 + KEY_USAGE_FAST_REP = 52 + KEY_USAGE_FAST_FINISHED = 53 + KEY_USAGE_ENC_CHALLENGE_CLIENT = 54 + KEY_USAGE_ENC_CHALLENGE_KDC = 55 + KEY_USAGE_AS_REQ = 56 + //26-511. Reserved for future use in Kerberos and related protocols. + //512-1023. Reserved for uses internal to a Kerberos implementation. + //1024. Encryption for application use in protocols that do not specify key usage values + //1025. Checksums for application use in protocols that do not specify key usage values + //1026-2047. Reserved for application use. +) diff --git a/github.com/jcmturner/gokrb5/v8/iana/msgtype/constants.go b/github.com/jcmturner/gokrb5/v8/iana/msgtype/constants.go new file mode 100644 index 0000000000..ad21810b67 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/iana/msgtype/constants.go @@ -0,0 +1,18 @@ +// Package msgtype provides Kerberos 5 message type assigned numbers. +package msgtype + +// KRB message type IDs. +const ( + KRB_AS_REQ = 10 //Request for initial authentication + KRB_AS_REP = 11 //Response to KRB_AS_REQ request + KRB_TGS_REQ = 12 //Request for authentication based on TGT + KRB_TGS_REP = 13 //Response to KRB_TGS_REQ request + KRB_AP_REQ = 14 //Application request to server + KRB_AP_REP = 15 //Response to KRB_AP_REQ_MUTUAL + KRB_RESERVED16 = 16 //Reserved for user-to-user krb_tgt_request + KRB_RESERVED17 = 17 //Reserved for user-to-user krb_tgt_reply + KRB_SAFE = 20 // Safe (checksummed) application message + KRB_PRIV = 21 // Private (encrypted) application message + KRB_CRED = 22 //Private (encrypted) message to forward credentials + KRB_ERROR = 30 //Error response +) diff --git a/github.com/jcmturner/gokrb5/v8/iana/nametype/constants.go b/github.com/jcmturner/gokrb5/v8/iana/nametype/constants.go new file mode 100644 index 0000000000..c111a05f92 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/iana/nametype/constants.go @@ -0,0 +1,15 @@ +// Package nametype provides Kerberos 5 principal name type numbers. +package nametype + +// Kerberos name type IDs. +const ( + KRB_NT_UNKNOWN int32 = 0 //Name type not known + KRB_NT_PRINCIPAL int32 = 1 //Just the name of the principal as in DCE, or for users + KRB_NT_SRV_INST int32 = 2 //Service and other unique instance (krbtgt) + KRB_NT_SRV_HST int32 = 3 //Service with host name as instance (telnet, rcommands) + KRB_NT_SRV_XHST int32 = 4 //Service with host as remaining components + KRB_NT_UID int32 = 5 //Unique ID + KRB_NT_X500_PRINCIPAL int32 = 6 //Encoded X.509 Distinguished name [RFC2253] + KRB_NT_SMTP_NAME int32 = 7 //Name in form of SMTP email name (e.g., user@example.com) + KRB_NT_ENTERPRISE int32 = 10 //Enterprise name; may be mapped to principal name +) diff --git a/github.com/jcmturner/gokrb5/v8/iana/patype/constants.go b/github.com/jcmturner/gokrb5/v8/iana/patype/constants.go new file mode 100644 index 0000000000..aa04f63765 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/iana/patype/constants.go @@ -0,0 +1,77 @@ +// Package patype provides Kerberos 5 pre-authentication type assigned numbers. +package patype + +// Kerberos pre-authentication type assigned numbers. +const ( + PA_TGS_REQ int32 = 1 + PA_ENC_TIMESTAMP int32 = 2 + PA_PW_SALT int32 = 3 + //RESERVED : 4 + PA_ENC_UNIX_TIME int32 = 5 + PA_SANDIA_SECUREID int32 = 6 + PA_SESAME int32 = 7 + PA_OSF_DCE int32 = 8 + PA_CYBERSAFE_SECUREID int32 = 9 + PA_AFS3_SALT int32 = 10 + PA_ETYPE_INFO int32 = 11 + PA_SAM_CHALLENGE int32 = 12 + PA_SAM_RESPONSE int32 = 13 + PA_PK_AS_REQ_OLD int32 = 14 + PA_PK_AS_REP_OLD int32 = 15 + PA_PK_AS_REQ int32 = 16 + PA_PK_AS_REP int32 = 17 + PA_PK_OCSP_RESPONSE int32 = 18 + PA_ETYPE_INFO2 int32 = 19 + PA_USE_SPECIFIED_KVNO int32 = 20 + PA_SVR_REFERRAL_INFO int32 = 20 + PA_SAM_REDIRECT int32 = 21 + PA_GET_FROM_TYPED_DATA int32 = 22 + TD_PADATA int32 = 22 + PA_SAM_ETYPE_INFO int32 = 23 + PA_ALT_PRINC int32 = 24 + PA_SERVER_REFERRAL int32 = 25 + //UNASSIGNED : 26-29 + PA_SAM_CHALLENGE2 int32 = 30 + PA_SAM_RESPONSE2 int32 = 31 + //UNASSIGNED : 32-40 + PA_EXTRA_TGT int32 = 41 + //UNASSIGNED : 42-100 + TD_PKINIT_CMS_CERTIFICATES int32 = 101 + TD_KRB_PRINCIPAL int32 = 102 + TD_KRB_REALM int32 = 103 + TD_TRUSTED_CERTIFIERS int32 = 104 + TD_CERTIFICATE_INDEX int32 = 105 + TD_APP_DEFINED_ERROR int32 = 106 + TD_REQ_NONCE int32 = 107 + TD_REQ_SEQ int32 = 108 + TD_DH_PARAMETERS int32 = 109 + //UNASSIGNED : 110 + TD_CMS_DIGEST_ALGORITHMS int32 = 111 + TD_CERT_DIGEST_ALGORITHMS int32 = 112 + //UNASSIGNED : 113-127 + PA_PAC_REQUEST int32 = 128 + PA_FOR_USER int32 = 129 + PA_FOR_X509_USER int32 = 130 + PA_FOR_CHECK_DUPS int32 = 131 + PA_AS_CHECKSUM int32 = 132 + PA_FX_COOKIE int32 = 133 + PA_AUTHENTICATION_SET int32 = 134 + PA_AUTH_SET_SELECTED int32 = 135 + PA_FX_FAST int32 = 136 + PA_FX_ERROR int32 = 137 + PA_ENCRYPTED_CHALLENGE int32 = 138 + //UNASSIGNED : 139-140 + PA_OTP_CHALLENGE int32 = 141 + PA_OTP_REQUEST int32 = 142 + PA_OTP_CONFIRM int32 = 143 + PA_OTP_PIN_CHANGE int32 = 144 + PA_EPAK_AS_REQ int32 = 145 + PA_EPAK_AS_REP int32 = 146 + PA_PKINIT_KX int32 = 147 + PA_PKU2U_NAME int32 = 148 + PA_REQ_ENC_PA_REP int32 = 149 + PA_AS_FRESHNESS int32 = 150 + //UNASSIGNED : 151-164 + PA_SUPPORTED_ETYPES int32 = 165 + PA_EXTENDED_ERROR int32 = 166 +) diff --git a/github.com/jcmturner/gokrb5/v8/kadmin/changepasswddata.go b/github.com/jcmturner/gokrb5/v8/kadmin/changepasswddata.go new file mode 100644 index 0000000000..2d68eda192 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/kadmin/changepasswddata.go @@ -0,0 +1,23 @@ +package kadmin + +import ( + "github.com/jcmturner/gofork/encoding/asn1" + "github.com/jcmturner/gokrb5/v8/types" +) + +// ChangePasswdData is the payload to a password change message. +type ChangePasswdData struct { + NewPasswd []byte `asn1:"explicit,tag:0"` + TargName types.PrincipalName `asn1:"explicit,optional,tag:1"` + TargRealm string `asn1:"generalstring,optional,explicit,tag:2"` +} + +// Marshal ChangePasswdData into a byte slice. +func (c *ChangePasswdData) Marshal() ([]byte, error) { + b, err := asn1.Marshal(*c) + if err != nil { + return []byte{}, err + } + //b = asn1tools.AddASNAppTag(b, asnAppTag.) + return b, nil +} diff --git a/github.com/jcmturner/gokrb5/v8/kadmin/message.go b/github.com/jcmturner/gokrb5/v8/kadmin/message.go new file mode 100644 index 0000000000..d1864c998b --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/kadmin/message.go @@ -0,0 +1,114 @@ +package kadmin + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "math" + + "github.com/jcmturner/gokrb5/v8/messages" + "github.com/jcmturner/gokrb5/v8/types" +) + +const ( + verisonHex = "ff80" +) + +// Request message for changing password. +type Request struct { + APREQ messages.APReq + KRBPriv messages.KRBPriv +} + +// Reply message for a password change. +type Reply struct { + MessageLength int + Version int + APREPLength int + APREP messages.APRep + KRBPriv messages.KRBPriv + KRBError messages.KRBError + IsKRBError bool + ResultCode uint16 + Result string +} + +// Marshal a Request into a byte slice. +func (m *Request) Marshal() (b []byte, err error) { + b = []byte{255, 128} // protocol version number: contains the hex constant 0xff80 (big-endian integer). + ab, e := m.APREQ.Marshal() + if e != nil { + err = fmt.Errorf("error marshaling AP_REQ: %v", e) + return + } + if len(ab) > math.MaxUint16 { + err = errors.New("length of AP_REQ greater then max Uint16 size") + return + } + al := make([]byte, 2) + binary.BigEndian.PutUint16(al, uint16(len(ab))) + b = append(b, al...) + b = append(b, ab...) + pb, e := m.KRBPriv.Marshal() + if e != nil { + err = fmt.Errorf("error marshaling KRB_Priv: %v", e) + return + } + b = append(b, pb...) + if len(b)+2 > math.MaxUint16 { + err = errors.New("length of message greater then max Uint16 size") + return + } + ml := make([]byte, 2) + binary.BigEndian.PutUint16(ml, uint16(len(b)+2)) + b = append(ml, b...) + return +} + +// Unmarshal a byte slice into a Reply. +func (m *Reply) Unmarshal(b []byte) error { + m.MessageLength = int(binary.BigEndian.Uint16(b[0:2])) + m.Version = int(binary.BigEndian.Uint16(b[2:4])) + if m.Version != 1 { + return fmt.Errorf("kadmin reply has incorrect protocol version number: %d", m.Version) + } + m.APREPLength = int(binary.BigEndian.Uint16(b[4:6])) + if m.APREPLength != 0 { + err := m.APREP.Unmarshal(b[6 : 6+m.APREPLength]) + if err != nil { + return err + } + err = m.KRBPriv.Unmarshal(b[6+m.APREPLength : m.MessageLength]) + if err != nil { + return err + } + } else { + m.IsKRBError = true + m.KRBError.Unmarshal(b[6:m.MessageLength]) + m.ResultCode, m.Result = parseResponse(m.KRBError.EData) + } + return nil +} + +func parseResponse(b []byte) (c uint16, s string) { + c = binary.BigEndian.Uint16(b[0:2]) + buf := bytes.NewBuffer(b[2:]) + m := make([]byte, len(b)-2) + binary.Read(buf, binary.BigEndian, &m) + s = string(m) + return +} + +// Decrypt the encrypted part of the KRBError within the change password Reply. +func (m *Reply) Decrypt(key types.EncryptionKey) error { + if m.IsKRBError { + return m.KRBError + } + err := m.KRBPriv.DecryptEncPart(key) + if err != nil { + return err + } + m.ResultCode, m.Result = parseResponse(m.KRBPriv.DecryptedEncPart.UserData) + return nil +} diff --git a/github.com/jcmturner/gokrb5/v8/kadmin/passwd.go b/github.com/jcmturner/gokrb5/v8/kadmin/passwd.go new file mode 100644 index 0000000000..db199bffcb --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/kadmin/passwd.go @@ -0,0 +1,68 @@ +// Package kadmin provides Kerberos administration capabilities. +package kadmin + +import ( + "github.com/jcmturner/gokrb5/v8/crypto" + "github.com/jcmturner/gokrb5/v8/krberror" + "github.com/jcmturner/gokrb5/v8/messages" + "github.com/jcmturner/gokrb5/v8/types" +) + +// ChangePasswdMsg generate a change password request and also return the key needed to decrypt the reply. +func ChangePasswdMsg(cname types.PrincipalName, realm, password string, tkt messages.Ticket, sessionKey types.EncryptionKey) (r Request, k types.EncryptionKey, err error) { + // Create change password data struct and marshal to bytes + chgpasswd := ChangePasswdData{ + NewPasswd: []byte(password), + TargName: cname, + TargRealm: realm, + } + chpwdb, err := chgpasswd.Marshal() + if err != nil { + err = krberror.Errorf(err, krberror.KRBMsgError, "error marshaling change passwd data") + return + } + + // Generate authenticator + auth, err := types.NewAuthenticator(realm, cname) + if err != nil { + err = krberror.Errorf(err, krberror.KRBMsgError, "error generating new authenticator") + return + } + etype, err := crypto.GetEtype(sessionKey.KeyType) + if err != nil { + err = krberror.Errorf(err, krberror.KRBMsgError, "error generating subkey etype") + return + } + err = auth.GenerateSeqNumberAndSubKey(etype.GetETypeID(), etype.GetKeyByteSize()) + if err != nil { + err = krberror.Errorf(err, krberror.KRBMsgError, "error generating subkey") + return + } + k = auth.SubKey + + // Generate AP_REQ + APreq, err := messages.NewAPReq(tkt, sessionKey, auth) + if err != nil { + return + } + + // Form the KRBPriv encpart data + kp := messages.EncKrbPrivPart{ + UserData: chpwdb, + Timestamp: auth.CTime, + Usec: auth.Cusec, + SequenceNumber: auth.SeqNumber, + } + kpriv := messages.NewKRBPriv(kp) + err = kpriv.EncryptEncPart(k) + if err != nil { + err = krberror.Errorf(err, krberror.EncryptingError, "error encrypting change passwd data") + return + } + + r = Request{ + APREQ: APreq, + KRBPriv: kpriv, + } + return +} diff --git a/github.com/jcmturner/gokrb5/v8/keytab/keytab.go b/github.com/jcmturner/gokrb5/v8/keytab/keytab.go new file mode 100644 index 0000000000..dd75f7ff9a --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/keytab/keytab.go @@ -0,0 +1,470 @@ +// Package keytab implements Kerberos keytabs: https://web.mit.edu/kerberos/krb5-devel/doc/formats/keytab_file_format.html. +package keytab + +import ( + "bytes" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "time" + "unsafe" + + "github.com/jcmturner/gokrb5/v8/types" +) + +const ( + keytabFirstByte byte = 05 +) + +// Keytab struct. +type Keytab struct { + version uint8 + Entries []entry +} + +// Keytab entry struct. +type entry struct { + Principal principal + Timestamp time.Time + KVNO8 uint8 + Key types.EncryptionKey + KVNO uint32 +} + +// Keytab entry principal struct. +type principal struct { + NumComponents int16 `json:"-"` + Realm string + Components []string + NameType int32 +} + +// New creates new, empty Keytab type. +func New() *Keytab { + var e []entry + return &Keytab{ + version: 0, + Entries: e, + } +} + +// GetEncryptionKey returns the EncryptionKey from the Keytab for the newest entry with the required kvno, etype and matching principal. +// If the kvno is zero then the latest kvno will be returned. The kvno is also returned for +func (kt *Keytab) GetEncryptionKey(princName types.PrincipalName, realm string, kvno int, etype int32) (types.EncryptionKey, int, error) { + var key types.EncryptionKey + var t time.Time + for _, k := range kt.Entries { + if k.Principal.Realm == realm && len(k.Principal.Components) == len(princName.NameString) && + k.Key.KeyType == etype && + (k.KVNO == uint32(kvno) || kvno == 0) && + k.Timestamp.After(t) { + p := true + for i, n := range k.Principal.Components { + if princName.NameString[i] != n { + p = false + break + } + } + if p { + key = k.Key + kvno = int(k.KVNO) + t = k.Timestamp + } + } + } + if len(key.KeyValue) < 1 { + return key, 0, fmt.Errorf("matching key not found in keytab. Looking for %v realm: %v kvno: %v etype: %v", princName.NameString, realm, kvno, etype) + } + return key, kvno, nil +} + +// Create a new Keytab entry. +func newEntry() entry { + var b []byte + return entry{ + Principal: newPrincipal(), + Timestamp: time.Time{}, + KVNO8: 0, + Key: types.EncryptionKey{ + KeyType: 0, + KeyValue: b, + }, + KVNO: 0, + } +} + +// Create a new principal. +func newPrincipal() principal { + var c []string + return principal{ + NumComponents: 0, + Realm: "", + Components: c, + NameType: 0, + } +} + +// Load a Keytab file into a Keytab type. +func Load(ktPath string) (*Keytab, error) { + kt := new(Keytab) + b, err := ioutil.ReadFile(ktPath) + if err != nil { + return kt, err + } + err = kt.Unmarshal(b) + return kt, err +} + +// Marshal keytab into byte slice +func (kt *Keytab) Marshal() ([]byte, error) { + b := []byte{keytabFirstByte, kt.version} + for _, e := range kt.Entries { + eb, err := e.marshal(int(kt.version)) + if err != nil { + return b, err + } + b = append(b, eb...) + } + return b, nil +} + +// Write the keytab bytes to io.Writer. +// Returns the number of bytes written +func (kt *Keytab) Write(w io.Writer) (int, error) { + b, err := kt.Marshal() + if err != nil { + return 0, fmt.Errorf("error marshaling keytab: %v", err) + } + return w.Write(b) +} + +// Unmarshal byte slice of Keytab data into Keytab type. +func (kt *Keytab) Unmarshal(b []byte) error { + if len(b) < 2 { + return fmt.Errorf("byte array is less than 2 bytes: %d", len(b)) + } + + //The first byte of the file always has the value 5 + if b[0] != keytabFirstByte { + return errors.New("invalid keytab data. First byte does not equal 5") + } + //Get keytab version + //The 2nd byte contains the version number (1 or 2) + kt.version = b[1] + if kt.version != 1 && kt.version != 2 { + return errors.New("invalid keytab data. Keytab version is neither 1 nor 2") + } + //Version 1 of the file format uses native byte order for integer representations. Version 2 always uses big-endian byte order + var endian binary.ByteOrder + endian = binary.BigEndian + if kt.version == 1 && isNativeEndianLittle() { + endian = binary.LittleEndian + } + // n tracks position in the byte array + n := 2 + l, err := readInt32(b, &n, &endian) + if err != nil { + return err + } + for l != 0 { + if l < 0 { + //Zero padded so skip over + l = l * -1 + n = n + int(l) + } else { + if n < 0 { + return fmt.Errorf("%d can't be less than zero", n) + } + if n+int(l) > len(b) { + return fmt.Errorf("%s's length is less than %d", b, n+int(l)) + } + eb := b[n : n+int(l)] + n = n + int(l) + ke := newEntry() + // p keeps track as to where we are in the byte stream + var p int + var err error + parsePrincipal(eb, &p, kt, &ke, &endian) + ke.Timestamp, err = readTimestamp(eb, &p, &endian) + if err != nil { + return err + } + rei8, err := readInt8(eb, &p, &endian) + if err != nil { + return err + } + ke.KVNO8 = uint8(rei8) + rei16, err := readInt16(eb, &p, &endian) + if err != nil { + return err + } + ke.Key.KeyType = int32(rei16) + rei16, err = readInt16(eb, &p, &endian) + if err != nil { + return err + } + kl := int(rei16) + ke.Key.KeyValue, err = readBytes(eb, &p, kl, &endian) + if err != nil { + return err + } + // The 32-bit key version overrides the 8-bit key version. + // If at least 4 bytes are left after the other fields are read and they are non-zero + // this indicates the 32-bit version is present. + if len(eb)-p >= 4 { + // The 32-bit key may be present + ri32, err := readInt32(eb, &p, &endian) + if err != nil { + return err + } + ke.KVNO = uint32(ri32) + } + if ke.KVNO == 0 { + // Handles if the value from the last 4 bytes was zero and also if there are not the 4 bytes present. Makes sense to put the same value here as KVNO8 + ke.KVNO = uint32(ke.KVNO8) + } + // Add the entry to the keytab + kt.Entries = append(kt.Entries, ke) + } + // Check if there are still 4 bytes left to read + // Also check that n is greater than zero + if n < 0 || n > len(b) || len(b[n:]) < 4 { + break + } + // Read the size of the next entry + l, err = readInt32(b, &n, &endian) + if err != nil { + return err + } + } + return nil +} + +func (e entry) marshal(v int) ([]byte, error) { + var b []byte + pb, err := e.Principal.marshal(v) + if err != nil { + return b, err + } + b = append(b, pb...) + + var endian binary.ByteOrder + endian = binary.BigEndian + if v == 1 && isNativeEndianLittle() { + endian = binary.LittleEndian + } + + t := make([]byte, 9) + endian.PutUint32(t[0:4], uint32(e.Timestamp.Unix())) + t[4] = e.KVNO8 + endian.PutUint16(t[5:7], uint16(e.Key.KeyType)) + endian.PutUint16(t[7:9], uint16(len(e.Key.KeyValue))) + b = append(b, t...) + + buf := new(bytes.Buffer) + err = binary.Write(buf, endian, e.Key.KeyValue) + if err != nil { + return b, err + } + b = append(b, buf.Bytes()...) + + t = make([]byte, 4) + endian.PutUint32(t, e.KVNO) + b = append(b, t...) + + // Add the length header + t = make([]byte, 4) + endian.PutUint32(t, uint32(len(b))) + b = append(t, b...) + return b, nil +} + +// Parse the Keytab bytes of a principal into a Keytab entry's principal. +func parsePrincipal(b []byte, p *int, kt *Keytab, ke *entry, e *binary.ByteOrder) error { + var err error + ke.Principal.NumComponents, err = readInt16(b, p, e) + if err != nil { + return err + } + if kt.version == 1 { + //In version 1 the number of components includes the realm. Minus 1 to make consistent with version 2 + ke.Principal.NumComponents-- + } + lenRealm, err := readInt16(b, p, e) + if err != nil { + return err + } + realmB, err := readBytes(b, p, int(lenRealm), e) + if err != nil { + return err + } + ke.Principal.Realm = string(realmB) + for i := 0; i < int(ke.Principal.NumComponents); i++ { + l, err := readInt16(b, p, e) + if err != nil { + return err + } + compB, err := readBytes(b, p, int(l), e) + if err != nil { + return err + } + ke.Principal.Components = append(ke.Principal.Components, string(compB)) + } + if kt.version != 1 { + //Name Type is omitted in version 1 + ke.Principal.NameType, err = readInt32(b, p, e) + if err != nil { + return err + } + } + return nil +} + +func (p principal) marshal(v int) ([]byte, error) { + //var b []byte + b := make([]byte, 2) + var endian binary.ByteOrder + endian = binary.BigEndian + if v == 1 && isNativeEndianLittle() { + endian = binary.LittleEndian + } + endian.PutUint16(b[0:], uint16(p.NumComponents)) + realm, err := marshalString(p.Realm, v) + if err != nil { + return b, err + } + b = append(b, realm...) + for _, c := range p.Components { + cb, err := marshalString(c, v) + if err != nil { + return b, err + } + b = append(b, cb...) + } + if v != 1 { + t := make([]byte, 4) + endian.PutUint32(t, uint32(p.NameType)) + b = append(b, t...) + } + return b, nil +} + +func marshalString(s string, v int) ([]byte, error) { + sb := []byte(s) + b := make([]byte, 2) + var endian binary.ByteOrder + endian = binary.BigEndian + if v == 1 && isNativeEndianLittle() { + endian = binary.LittleEndian + } + endian.PutUint16(b[0:], uint16(len(sb))) + buf := new(bytes.Buffer) + err := binary.Write(buf, endian, sb) + if err != nil { + return b, err + } + b = append(b, buf.Bytes()...) + return b, err +} + +// Read bytes representing a timestamp. +func readTimestamp(b []byte, p *int, e *binary.ByteOrder) (time.Time, error) { + i32, err := readInt32(b, p, e) + if err != nil { + return time.Time{}, err + } + return time.Unix(int64(i32), 0), nil +} + +// Read bytes representing an eight bit integer. +func readInt8(b []byte, p *int, e *binary.ByteOrder) (i int8, err error) { + if *p < 0 { + return 0, fmt.Errorf("%d cannot be less than zero", *p) + } + + if (*p + 1) > len(b) { + return 0, fmt.Errorf("%s's length is less than %d", b, *p+1) + } + buf := bytes.NewBuffer(b[*p : *p+1]) + binary.Read(buf, *e, &i) + *p++ + return +} + +// Read bytes representing a sixteen bit integer. +func readInt16(b []byte, p *int, e *binary.ByteOrder) (i int16, err error) { + if *p < 0 { + return 0, fmt.Errorf("%d cannot be less than zero", *p) + } + + if (*p + 2) > len(b) { + return 0, fmt.Errorf("%s's length is less than %d", b, *p+2) + } + + buf := bytes.NewBuffer(b[*p : *p+2]) + binary.Read(buf, *e, &i) + *p += 2 + return +} + +// Read bytes representing a thirty two bit integer. +func readInt32(b []byte, p *int, e *binary.ByteOrder) (i int32, err error) { + if *p < 0 { + return 0, fmt.Errorf("%d cannot be less than zero", *p) + } + + if (*p + 4) > len(b) { + return 0, fmt.Errorf("%s's length is less than %d", b, *p+4) + } + + buf := bytes.NewBuffer(b[*p : *p+4]) + binary.Read(buf, *e, &i) + *p += 4 + return +} + +func readBytes(b []byte, p *int, s int, e *binary.ByteOrder) ([]byte, error) { + if s < 0 { + return nil, fmt.Errorf("%d cannot be less than zero", s) + } + i := *p + s + if i > len(b) { + return nil, fmt.Errorf("%s's length is greater than %d", b, i) + } + buf := bytes.NewBuffer(b[*p:i]) + r := make([]byte, s) + if err := binary.Read(buf, *e, &r); err != nil { + return nil, err + } + *p += s + return r, nil +} + +func isNativeEndianLittle() bool { + var x = 0x012345678 + var p = unsafe.Pointer(&x) + var bp = (*[4]byte)(p) + + var endian bool + if 0x01 == bp[0] { + endian = false + } else if (0x78 & 0xff) == (bp[0] & 0xff) { + endian = true + } else { + // Default to big endian + endian = false + } + return endian +} + +// JSON return information about the keys held in the keytab in a JSON format. +func (k *Keytab) JSON() (string, error) { + b, err := json.MarshalIndent(k, "", " ") + if err != nil { + return "", err + } + return string(b), nil +} diff --git a/github.com/jcmturner/gokrb5/v8/krberror/error.go b/github.com/jcmturner/gokrb5/v8/krberror/error.go new file mode 100644 index 0000000000..01c6d9904c --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/krberror/error.go @@ -0,0 +1,67 @@ +// Package krberror provides error type and functions for gokrb5. +package krberror + +import ( + "fmt" + "strings" +) + +// Error type descriptions. +const ( + separator = " < " + EncodingError = "Encoding_Error" + NetworkingError = "Networking_Error" + DecryptingError = "Decrypting_Error" + EncryptingError = "Encrypting_Error" + ChksumError = "Checksum_Error" + KRBMsgError = "KRBMessage_Handling_Error" + ConfigError = "Configuration_Error" + KDCError = "KDC_Error" +) + +// Krberror is an error type for gokrb5 +type Krberror struct { + RootCause string + EText []string +} + +// Error function to implement the error interface. +func (e Krberror) Error() string { + return fmt.Sprintf("[Root cause: %s] ", e.RootCause) + strings.Join(e.EText, separator) +} + +// Add another error statement to the error. +func (e *Krberror) Add(et string, s string) { + e.EText = append([]string{fmt.Sprintf("%s: %s", et, s)}, e.EText...) +} + +// New creates a new instance of Krberror. +func New(et, s string) Krberror { + return Krberror{ + RootCause: et, + EText: []string{s}, + } +} + +// Errorf appends to or creates a new Krberror. +func Errorf(err error, et, format string, a ...interface{}) Krberror { + if e, ok := err.(Krberror); ok { + e.Add(et, fmt.Sprintf(format, a...)) + return e + } + return NewErrorf(et, format+": %s", append(a, err)...) +} + +// NewErrorf creates a new Krberror from a formatted string. +func NewErrorf(et, format string, a ...interface{}) Krberror { + var s string + if len(a) > 0 { + s = fmt.Sprintf("%s: %s", et, fmt.Sprintf(format, a...)) + } else { + s = fmt.Sprintf("%s: %s", et, format) + } + return Krberror{ + RootCause: et, + EText: []string{s}, + } +} diff --git a/github.com/jcmturner/gokrb5/v8/messages/APRep.go b/github.com/jcmturner/gokrb5/v8/messages/APRep.go new file mode 100644 index 0000000000..555fb80720 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/messages/APRep.go @@ -0,0 +1,49 @@ +package messages + +import ( + "fmt" + "time" + + "github.com/jcmturner/gofork/encoding/asn1" + "github.com/jcmturner/gokrb5/v8/iana/asnAppTag" + "github.com/jcmturner/gokrb5/v8/iana/msgtype" + "github.com/jcmturner/gokrb5/v8/krberror" + "github.com/jcmturner/gokrb5/v8/types" +) + +// APRep implements RFC 4120 KRB_AP_REP: https://tools.ietf.org/html/rfc4120#section-5.5.2. +type APRep struct { + PVNO int `asn1:"explicit,tag:0"` + MsgType int `asn1:"explicit,tag:1"` + EncPart types.EncryptedData `asn1:"explicit,tag:2"` +} + +// EncAPRepPart is the encrypted part of KRB_AP_REP. +type EncAPRepPart struct { + CTime time.Time `asn1:"generalized,explicit,tag:0"` + Cusec int `asn1:"explicit,tag:1"` + Subkey types.EncryptionKey `asn1:"optional,explicit,tag:2"` + SequenceNumber int64 `asn1:"optional,explicit,tag:3"` +} + +// Unmarshal bytes b into the APRep struct. +func (a *APRep) Unmarshal(b []byte) error { + _, err := asn1.UnmarshalWithParams(b, a, fmt.Sprintf("application,explicit,tag:%v", asnAppTag.APREP)) + if err != nil { + return processUnmarshalReplyError(b, err) + } + expectedMsgType := msgtype.KRB_AP_REP + if a.MsgType != expectedMsgType { + return krberror.NewErrorf(krberror.KRBMsgError, "message ID does not indicate a KRB_AP_REP. Expected: %v; Actual: %v", expectedMsgType, a.MsgType) + } + return nil +} + +// Unmarshal bytes b into the APRep encrypted part struct. +func (a *EncAPRepPart) Unmarshal(b []byte) error { + _, err := asn1.UnmarshalWithParams(b, a, fmt.Sprintf("application,explicit,tag:%v", asnAppTag.EncAPRepPart)) + if err != nil { + return krberror.Errorf(err, krberror.EncodingError, "AP_REP unmarshal error") + } + return nil +} diff --git a/github.com/jcmturner/gokrb5/v8/messages/APReq.go b/github.com/jcmturner/gokrb5/v8/messages/APReq.go new file mode 100644 index 0000000000..1836079763 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/messages/APReq.go @@ -0,0 +1,199 @@ +package messages + +import ( + "fmt" + "time" + + "github.com/jcmturner/gofork/encoding/asn1" + "github.com/jcmturner/gokrb5/v8/asn1tools" + "github.com/jcmturner/gokrb5/v8/crypto" + "github.com/jcmturner/gokrb5/v8/iana" + "github.com/jcmturner/gokrb5/v8/iana/asnAppTag" + "github.com/jcmturner/gokrb5/v8/iana/errorcode" + "github.com/jcmturner/gokrb5/v8/iana/keyusage" + "github.com/jcmturner/gokrb5/v8/iana/msgtype" + "github.com/jcmturner/gokrb5/v8/keytab" + "github.com/jcmturner/gokrb5/v8/krberror" + "github.com/jcmturner/gokrb5/v8/types" +) + +type marshalAPReq struct { + PVNO int `asn1:"explicit,tag:0"` + MsgType int `asn1:"explicit,tag:1"` + APOptions asn1.BitString `asn1:"explicit,tag:2"` + // Ticket needs to be a raw value as it is wrapped in an APPLICATION tag + Ticket asn1.RawValue `asn1:"explicit,tag:3"` + EncryptedAuthenticator types.EncryptedData `asn1:"explicit,tag:4"` +} + +// APReq implements RFC 4120 KRB_AP_REQ: https://tools.ietf.org/html/rfc4120#section-5.5.1. +type APReq struct { + PVNO int `asn1:"explicit,tag:0"` + MsgType int `asn1:"explicit,tag:1"` + APOptions asn1.BitString `asn1:"explicit,tag:2"` + Ticket Ticket `asn1:"explicit,tag:3"` + EncryptedAuthenticator types.EncryptedData `asn1:"explicit,tag:4"` + Authenticator types.Authenticator `asn1:"optional"` +} + +// NewAPReq generates a new KRB_AP_REQ struct. +func NewAPReq(tkt Ticket, sessionKey types.EncryptionKey, auth types.Authenticator) (APReq, error) { + var a APReq + ed, err := encryptAuthenticator(auth, sessionKey, tkt) + if err != nil { + return a, krberror.Errorf(err, krberror.KRBMsgError, "error creating Authenticator for AP_REQ") + } + a = APReq{ + PVNO: iana.PVNO, + MsgType: msgtype.KRB_AP_REQ, + APOptions: types.NewKrbFlags(), + Ticket: tkt, + EncryptedAuthenticator: ed, + } + return a, nil +} + +// Encrypt Authenticator +func encryptAuthenticator(a types.Authenticator, sessionKey types.EncryptionKey, tkt Ticket) (types.EncryptedData, error) { + var ed types.EncryptedData + m, err := a.Marshal() + if err != nil { + return ed, krberror.Errorf(err, krberror.EncodingError, "marshaling error of EncryptedData form of Authenticator") + } + usage := authenticatorKeyUsage(tkt.SName) + ed, err = crypto.GetEncryptedData(m, sessionKey, uint32(usage), tkt.EncPart.KVNO) + if err != nil { + return ed, krberror.Errorf(err, krberror.EncryptingError, "error encrypting Authenticator") + } + return ed, nil +} + +// DecryptAuthenticator decrypts the Authenticator within the AP_REQ. +// sessionKey may simply be the key within the decrypted EncPart of the ticket within the AP_REQ. +func (a *APReq) DecryptAuthenticator(sessionKey types.EncryptionKey) error { + usage := authenticatorKeyUsage(a.Ticket.SName) + ab, e := crypto.DecryptEncPart(a.EncryptedAuthenticator, sessionKey, uint32(usage)) + if e != nil { + return fmt.Errorf("error decrypting authenticator: %v", e) + } + err := a.Authenticator.Unmarshal(ab) + if err != nil { + return fmt.Errorf("error unmarshaling authenticator: %v", err) + } + return nil +} + +func authenticatorKeyUsage(pn types.PrincipalName) int { + if pn.NameString[0] == "krbtgt" { + return keyusage.TGS_REQ_PA_TGS_REQ_AP_REQ_AUTHENTICATOR + } + return keyusage.AP_REQ_AUTHENTICATOR +} + +// Unmarshal bytes b into the APReq struct. +func (a *APReq) Unmarshal(b []byte) error { + var m marshalAPReq + _, err := asn1.UnmarshalWithParams(b, &m, fmt.Sprintf("application,explicit,tag:%v", asnAppTag.APREQ)) + if err != nil { + return krberror.Errorf(err, krberror.EncodingError, "unmarshal error of AP_REQ") + } + if m.MsgType != msgtype.KRB_AP_REQ { + return NewKRBError(types.PrincipalName{}, "", errorcode.KRB_AP_ERR_MSG_TYPE, errorcode.Lookup(errorcode.KRB_AP_ERR_MSG_TYPE)) + } + a.PVNO = m.PVNO + a.MsgType = m.MsgType + a.APOptions = m.APOptions + a.EncryptedAuthenticator = m.EncryptedAuthenticator + a.Ticket, err = unmarshalTicket(m.Ticket.Bytes) + if err != nil { + return krberror.Errorf(err, krberror.EncodingError, "unmarshaling error of Ticket within AP_REQ") + } + return nil +} + +// Marshal APReq struct. +func (a *APReq) Marshal() ([]byte, error) { + m := marshalAPReq{ + PVNO: a.PVNO, + MsgType: a.MsgType, + APOptions: a.APOptions, + EncryptedAuthenticator: a.EncryptedAuthenticator, + } + var b []byte + b, err := a.Ticket.Marshal() + if err != nil { + return b, err + } + m.Ticket = asn1.RawValue{ + Class: asn1.ClassContextSpecific, + IsCompound: true, + Tag: 3, + Bytes: b, + } + mk, err := asn1.Marshal(m) + if err != nil { + return mk, krberror.Errorf(err, krberror.EncodingError, "marshaling error of AP_REQ") + } + mk = asn1tools.AddASNAppTag(mk, asnAppTag.APREQ) + return mk, nil +} + +// Verify an AP_REQ using service's keytab, spn and max acceptable clock skew duration. +// The service ticket encrypted part and authenticator will be decrypted as part of this operation. +func (a *APReq) Verify(kt *keytab.Keytab, d time.Duration, cAddr types.HostAddress, snameOverride *types.PrincipalName) (bool, error) { + // Decrypt ticket's encrypted part with service key + //TODO decrypt with service's session key from its TGT is use-to-user. Need to figure out how to get TGT. + //if types.IsFlagSet(&a.APOptions, flags.APOptionUseSessionKey) { + // err := a.Ticket.Decrypt(tgt.DecryptedEncPart.Key) + // if err != nil { + // return false, krberror.Errorf(err, krberror.DecryptingError, "error decrypting encpart of ticket provided using session key") + // } + //} else { + // err := a.Ticket.DecryptEncPart(*kt, &a.Ticket.SName) + // if err != nil { + // return false, krberror.Errorf(err, krberror.DecryptingError, "error decrypting encpart of service ticket provided") + // } + //} + sname := &a.Ticket.SName + if snameOverride != nil { + sname = snameOverride + } + err := a.Ticket.DecryptEncPart(kt, sname) + if err != nil { + return false, krberror.Errorf(err, krberror.DecryptingError, "error decrypting encpart of service ticket provided") + } + + // Check time validity of ticket + ok, err := a.Ticket.Valid(d) + if err != nil || !ok { + return ok, err + } + + // Check client's address is listed in the client addresses in the ticket + if len(a.Ticket.DecryptedEncPart.CAddr) > 0 { + //If client addresses are present check if any of them match the source IP that sent the APReq + //If there is no match return KRB_AP_ERR_BADADDR error. + if !types.HostAddressesContains(a.Ticket.DecryptedEncPart.CAddr, cAddr) { + return false, NewKRBError(a.Ticket.SName, a.Ticket.Realm, errorcode.KRB_AP_ERR_BADADDR, "client address not within the list contained in the service ticket") + } + } + + // Decrypt authenticator with session key from ticket's encrypted part + err = a.DecryptAuthenticator(a.Ticket.DecryptedEncPart.Key) + if err != nil { + return false, NewKRBError(a.Ticket.SName, a.Ticket.Realm, errorcode.KRB_AP_ERR_BAD_INTEGRITY, "could not decrypt authenticator") + } + + // Check CName in authenticator is the same as that in the ticket + if !a.Authenticator.CName.Equal(a.Ticket.DecryptedEncPart.CName) { + return false, NewKRBError(a.Ticket.SName, a.Ticket.Realm, errorcode.KRB_AP_ERR_BADMATCH, "CName in Authenticator does not match that in service ticket") + } + + // Check the clock skew between the client and the service server + ct := a.Authenticator.CTime.Add(time.Duration(a.Authenticator.Cusec) * time.Microsecond) + t := time.Now().UTC() + if t.Sub(ct) > d || ct.Sub(t) > d { + return false, NewKRBError(a.Ticket.SName, a.Ticket.Realm, errorcode.KRB_AP_ERR_SKEW, fmt.Sprintf("clock skew with client too large. greater than %v seconds", d)) + } + return true, nil +} diff --git a/github.com/jcmturner/gokrb5/v8/messages/KDCRep.go b/github.com/jcmturner/gokrb5/v8/messages/KDCRep.go new file mode 100644 index 0000000000..41d2b2ca6c --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/messages/KDCRep.go @@ -0,0 +1,308 @@ +package messages + +// Reference: https://www.ietf.org/rfc/rfc4120.txt +// Section: 5.4.2 + +import ( + "fmt" + "time" + + "github.com/jcmturner/gofork/encoding/asn1" + "github.com/jcmturner/gokrb5/v8/config" + "github.com/jcmturner/gokrb5/v8/credentials" + "github.com/jcmturner/gokrb5/v8/crypto" + "github.com/jcmturner/gokrb5/v8/iana/asnAppTag" + "github.com/jcmturner/gokrb5/v8/iana/flags" + "github.com/jcmturner/gokrb5/v8/iana/keyusage" + "github.com/jcmturner/gokrb5/v8/iana/msgtype" + "github.com/jcmturner/gokrb5/v8/iana/patype" + "github.com/jcmturner/gokrb5/v8/krberror" + "github.com/jcmturner/gokrb5/v8/types" +) + +type marshalKDCRep struct { + PVNO int `asn1:"explicit,tag:0"` + MsgType int `asn1:"explicit,tag:1"` + PAData types.PADataSequence `asn1:"explicit,optional,tag:2"` + CRealm string `asn1:"generalstring,explicit,tag:3"` + CName types.PrincipalName `asn1:"explicit,tag:4"` + // Ticket needs to be a raw value as it is wrapped in an APPLICATION tag + Ticket asn1.RawValue `asn1:"explicit,tag:5"` + EncPart types.EncryptedData `asn1:"explicit,tag:6"` +} + +// KDCRepFields represents the KRB_KDC_REP fields. +type KDCRepFields struct { + PVNO int + MsgType int + PAData []types.PAData + CRealm string + CName types.PrincipalName + Ticket Ticket + EncPart types.EncryptedData + DecryptedEncPart EncKDCRepPart +} + +// ASRep implements RFC 4120 KRB_AS_REP: https://tools.ietf.org/html/rfc4120#section-5.4.2. +type ASRep struct { + KDCRepFields +} + +// TGSRep implements RFC 4120 KRB_TGS_REP: https://tools.ietf.org/html/rfc4120#section-5.4.2. +type TGSRep struct { + KDCRepFields +} + +// EncKDCRepPart is the encrypted part of KRB_KDC_REP. +type EncKDCRepPart struct { + Key types.EncryptionKey `asn1:"explicit,tag:0"` + LastReqs []LastReq `asn1:"explicit,tag:1"` + Nonce int `asn1:"explicit,tag:2"` + KeyExpiration time.Time `asn1:"generalized,explicit,optional,tag:3"` + Flags asn1.BitString `asn1:"explicit,tag:4"` + AuthTime time.Time `asn1:"generalized,explicit,tag:5"` + StartTime time.Time `asn1:"generalized,explicit,optional,tag:6"` + EndTime time.Time `asn1:"generalized,explicit,tag:7"` + RenewTill time.Time `asn1:"generalized,explicit,optional,tag:8"` + SRealm string `asn1:"generalstring,explicit,tag:9"` + SName types.PrincipalName `asn1:"explicit,tag:10"` + CAddr []types.HostAddress `asn1:"explicit,optional,tag:11"` + EncPAData types.PADataSequence `asn1:"explicit,optional,tag:12"` +} + +// LastReq part of KRB_KDC_REP. +type LastReq struct { + LRType int32 `asn1:"explicit,tag:0"` + LRValue time.Time `asn1:"generalized,explicit,tag:1"` +} + +// Unmarshal bytes b into the ASRep struct. +func (k *ASRep) Unmarshal(b []byte) error { + var m marshalKDCRep + _, err := asn1.UnmarshalWithParams(b, &m, fmt.Sprintf("application,explicit,tag:%v", asnAppTag.ASREP)) + if err != nil { + return processUnmarshalReplyError(b, err) + } + if m.MsgType != msgtype.KRB_AS_REP { + return krberror.NewErrorf(krberror.KRBMsgError, "message ID does not indicate an AS_REP. Expected: %v; Actual: %v", msgtype.KRB_AS_REP, m.MsgType) + } + //Process the raw ticket within + tkt, err := unmarshalTicket(m.Ticket.Bytes) + if err != nil { + return krberror.Errorf(err, krberror.EncodingError, "error unmarshaling Ticket within AS_REP") + } + k.KDCRepFields = KDCRepFields{ + PVNO: m.PVNO, + MsgType: m.MsgType, + PAData: m.PAData, + CRealm: m.CRealm, + CName: m.CName, + Ticket: tkt, + EncPart: m.EncPart, + } + return nil +} + +// Unmarshal bytes b into the TGSRep struct. +func (k *TGSRep) Unmarshal(b []byte) error { + var m marshalKDCRep + _, err := asn1.UnmarshalWithParams(b, &m, fmt.Sprintf("application,explicit,tag:%v", asnAppTag.TGSREP)) + if err != nil { + return processUnmarshalReplyError(b, err) + } + if m.MsgType != msgtype.KRB_TGS_REP { + return krberror.NewErrorf(krberror.KRBMsgError, "message ID does not indicate an TGS_REP. Expected: %v; Actual: %v", msgtype.KRB_TGS_REP, m.MsgType) + } + //Process the raw ticket within + tkt, err := unmarshalTicket(m.Ticket.Bytes) + if err != nil { + return krberror.Errorf(err, krberror.EncodingError, "error unmarshaling Ticket within TGS_REP") + } + k.KDCRepFields = KDCRepFields{ + PVNO: m.PVNO, + MsgType: m.MsgType, + PAData: m.PAData, + CRealm: m.CRealm, + CName: m.CName, + Ticket: tkt, + EncPart: m.EncPart, + } + return nil +} + +// Unmarshal bytes b into encrypted part of KRB_KDC_REP. +func (e *EncKDCRepPart) Unmarshal(b []byte) error { + _, err := asn1.UnmarshalWithParams(b, e, fmt.Sprintf("application,explicit,tag:%v", asnAppTag.EncASRepPart)) + if err != nil { + // Try using tag 26 + // Ref: RFC 4120 - mentions that some implementations use application tag number 26 wether or not the reply is + // a AS-REP or a TGS-REP. + _, err = asn1.UnmarshalWithParams(b, e, fmt.Sprintf("application,explicit,tag:%v", asnAppTag.EncTGSRepPart)) + if err != nil { + return krberror.Errorf(err, krberror.EncodingError, "error unmarshaling encrypted part within KDC_REP") + } + } + return nil +} + +// DecryptEncPart decrypts the encrypted part of an AS_REP. +func (k *ASRep) DecryptEncPart(c *credentials.Credentials) (types.EncryptionKey, error) { + var key types.EncryptionKey + var err error + if c.HasKeytab() { + key, _, err = c.Keytab().GetEncryptionKey(k.CName, k.CRealm, k.EncPart.KVNO, k.EncPart.EType) + if err != nil { + return key, krberror.Errorf(err, krberror.DecryptingError, "error decrypting AS_REP encrypted part") + } + } + if c.HasPassword() { + key, _, err = crypto.GetKeyFromPassword(c.Password(), k.CName, k.CRealm, k.EncPart.EType, k.PAData) + if err != nil { + return key, krberror.Errorf(err, krberror.DecryptingError, "error decrypting AS_REP encrypted part") + } + } + if !c.HasKeytab() && !c.HasPassword() { + return key, krberror.NewErrorf(krberror.DecryptingError, "no secret available in credentials to perform decryption of AS_REP encrypted part") + } + b, err := crypto.DecryptEncPart(k.EncPart, key, keyusage.AS_REP_ENCPART) + if err != nil { + return key, krberror.Errorf(err, krberror.DecryptingError, "error decrypting AS_REP encrypted part") + } + var denc EncKDCRepPart + err = denc.Unmarshal(b) + if err != nil { + return key, krberror.Errorf(err, krberror.EncodingError, "error unmarshaling decrypted encpart of AS_REP") + } + k.DecryptedEncPart = denc + return key, nil +} + +// Verify checks the validity of AS_REP message. +func (k *ASRep) Verify(cfg *config.Config, creds *credentials.Credentials, asReq ASReq) (bool, error) { + //Ref RFC 4120 Section 3.1.5 + if k.CName.NameType != asReq.ReqBody.CName.NameType || k.CName.NameString == nil { + return false, krberror.NewErrorf(krberror.KRBMsgError, "CName in response does not match what was requested. Requested: %+v; Reply: %+v", asReq.ReqBody.CName, k.CName) + } + for i := range k.CName.NameString { + if k.CName.NameString[i] != asReq.ReqBody.CName.NameString[i] { + return false, krberror.NewErrorf(krberror.KRBMsgError, "CName in response does not match what was requested. Requested: %+v; Reply: %+v", asReq.ReqBody.CName, k.CName) + } + } + if k.CRealm != asReq.ReqBody.Realm { + return false, krberror.NewErrorf(krberror.KRBMsgError, "CRealm in response does not match what was requested. Requested: %s; Reply: %s", asReq.ReqBody.Realm, k.CRealm) + } + key, err := k.DecryptEncPart(creds) + if err != nil { + return false, krberror.Errorf(err, krberror.DecryptingError, "error decrypting EncPart of AS_REP") + } + if k.DecryptedEncPart.Nonce != asReq.ReqBody.Nonce { + return false, krberror.NewErrorf(krberror.KRBMsgError, "possible replay attack, nonce in response does not match that in request") + } + if k.DecryptedEncPart.SName.NameType != asReq.ReqBody.SName.NameType || k.DecryptedEncPart.SName.NameString == nil { + return false, krberror.NewErrorf(krberror.KRBMsgError, "SName in response does not match what was requested. Requested: %v; Reply: %v", asReq.ReqBody.SName, k.DecryptedEncPart.SName) + } + for i := range k.CName.NameString { + if k.DecryptedEncPart.SName.NameString[i] != asReq.ReqBody.SName.NameString[i] { + return false, krberror.NewErrorf(krberror.KRBMsgError, "SName in response does not match what was requested. Requested: %+v; Reply: %+v", asReq.ReqBody.SName, k.DecryptedEncPart.SName) + } + } + if k.DecryptedEncPart.SRealm != asReq.ReqBody.Realm { + return false, krberror.NewErrorf(krberror.KRBMsgError, "SRealm in response does not match what was requested. Requested: %s; Reply: %s", asReq.ReqBody.Realm, k.DecryptedEncPart.SRealm) + } + if len(asReq.ReqBody.Addresses) > 0 { + if !types.HostAddressesEqual(k.DecryptedEncPart.CAddr, asReq.ReqBody.Addresses) { + return false, krberror.NewErrorf(krberror.KRBMsgError, "addresses listed in the AS_REP does not match those listed in the AS_REQ") + } + } + t := time.Now().UTC() + if t.Sub(k.DecryptedEncPart.AuthTime) > cfg.LibDefaults.Clockskew || k.DecryptedEncPart.AuthTime.Sub(t) > cfg.LibDefaults.Clockskew { + return false, krberror.NewErrorf(krberror.KRBMsgError, "clock skew with KDC too large. Greater than %v seconds", cfg.LibDefaults.Clockskew.Seconds()) + } + // RFC 6806 https://tools.ietf.org/html/rfc6806.html#section-11 + if asReq.PAData.Contains(patype.PA_REQ_ENC_PA_REP) && types.IsFlagSet(&k.DecryptedEncPart.Flags, flags.EncPARep) { + if len(k.DecryptedEncPart.EncPAData) < 2 || !k.DecryptedEncPart.EncPAData.Contains(patype.PA_FX_FAST) { + return false, krberror.NewErrorf(krberror.KRBMsgError, "KDC did not respond appropriately to FAST negotiation") + } + for _, pa := range k.DecryptedEncPart.EncPAData { + if pa.PADataType == patype.PA_REQ_ENC_PA_REP { + var pafast types.PAReqEncPARep + err := pafast.Unmarshal(pa.PADataValue) + if err != nil { + return false, krberror.Errorf(err, krberror.EncodingError, "KDC FAST negotiation response error, could not unmarshal PA_REQ_ENC_PA_REP") + } + etype, err := crypto.GetChksumEtype(pafast.ChksumType) + if err != nil { + return false, krberror.Errorf(err, krberror.ChksumError, "KDC FAST negotiation response error") + } + ab, _ := asReq.Marshal() + if !etype.VerifyChecksum(key.KeyValue, ab, pafast.Chksum, keyusage.KEY_USAGE_AS_REQ) { + return false, krberror.Errorf(err, krberror.ChksumError, "KDC FAST negotiation response checksum invalid") + } + } + } + } + return true, nil +} + +// DecryptEncPart decrypts the encrypted part of an TGS_REP. +func (k *TGSRep) DecryptEncPart(key types.EncryptionKey) error { + b, err := crypto.DecryptEncPart(k.EncPart, key, keyusage.TGS_REP_ENCPART_SESSION_KEY) + if err != nil { + return krberror.Errorf(err, krberror.DecryptingError, "error decrypting TGS_REP EncPart") + } + var denc EncKDCRepPart + err = denc.Unmarshal(b) + if err != nil { + return krberror.Errorf(err, krberror.EncodingError, "error unmarshaling encrypted part") + } + k.DecryptedEncPart = denc + return nil +} + +// Verify checks the validity of the TGS_REP message. +func (k *TGSRep) Verify(cfg *config.Config, tgsReq TGSReq) (bool, error) { + if k.CName.NameType != tgsReq.ReqBody.CName.NameType || k.CName.NameString == nil { + return false, krberror.NewErrorf(krberror.KRBMsgError, "CName type in response does not match what was requested. Requested: %+v; Reply: %+v", tgsReq.ReqBody.CName, k.CName) + } + for i := range k.CName.NameString { + if k.CName.NameString[i] != tgsReq.ReqBody.CName.NameString[i] { + return false, krberror.NewErrorf(krberror.KRBMsgError, "CName in response does not match what was requested. Requested: %+v; Reply: %+v", tgsReq.ReqBody.CName, k.CName) + } + } + if k.Ticket.Realm != tgsReq.ReqBody.Realm { + return false, krberror.NewErrorf(krberror.KRBMsgError, "realm in response ticket does not match what was requested. Requested: %s; Reply: %s", tgsReq.ReqBody.Realm, k.Ticket.Realm) + } + if k.DecryptedEncPart.Nonce != tgsReq.ReqBody.Nonce { + return false, krberror.NewErrorf(krberror.KRBMsgError, "possible replay attack, nonce in response does not match that in request") + } + //if k.Ticket.SName.NameType != tgsReq.ReqBody.SName.NameType || k.Ticket.SName.NameString == nil { + // return false, krberror.NewErrorf(krberror.KRBMsgError, "SName in response ticket does not match what was requested. Requested: %v; Reply: %v", tgsReq.ReqBody.SName, k.Ticket.SName) + //} + //for i := range k.Ticket.SName.NameString { + // if k.Ticket.SName.NameString[i] != tgsReq.ReqBody.SName.NameString[i] { + // return false, krberror.NewErrorf(krberror.KRBMsgError, "SName in response ticket does not match what was requested. Requested: %+v; Reply: %+v", tgsReq.ReqBody.SName, k.Ticket.SName) + // } + //} + //if k.DecryptedEncPart.SName.NameType != tgsReq.ReqBody.SName.NameType || k.DecryptedEncPart.SName.NameString == nil { + // return false, krberror.NewErrorf(krberror.KRBMsgError, "SName in response does not match what was requested. Requested: %v; Reply: %v", tgsReq.ReqBody.SName, k.DecryptedEncPart.SName) + //} + //for i := range k.DecryptedEncPart.SName.NameString { + // if k.DecryptedEncPart.SName.NameString[i] != tgsReq.ReqBody.SName.NameString[i] { + // return false, krberror.NewErrorf(krberror.KRBMsgError, "SName in response does not match what was requested. Requested: %+v; Reply: %+v", tgsReq.ReqBody.SName, k.DecryptedEncPart.SName) + // } + //} + if k.DecryptedEncPart.SRealm != tgsReq.ReqBody.Realm { + return false, krberror.NewErrorf(krberror.KRBMsgError, "SRealm in response does not match what was requested. Requested: %s; Reply: %s", tgsReq.ReqBody.Realm, k.DecryptedEncPart.SRealm) + } + if len(k.DecryptedEncPart.CAddr) > 0 { + if !types.HostAddressesEqual(k.DecryptedEncPart.CAddr, tgsReq.ReqBody.Addresses) { + return false, krberror.NewErrorf(krberror.KRBMsgError, "addresses listed in the TGS_REP does not match those listed in the TGS_REQ") + } + } + if time.Since(k.DecryptedEncPart.StartTime) > cfg.LibDefaults.Clockskew || k.DecryptedEncPart.StartTime.Sub(time.Now().UTC()) > cfg.LibDefaults.Clockskew { + if time.Since(k.DecryptedEncPart.AuthTime) > cfg.LibDefaults.Clockskew || k.DecryptedEncPart.AuthTime.Sub(time.Now().UTC()) > cfg.LibDefaults.Clockskew { + return false, krberror.NewErrorf(krberror.KRBMsgError, "clock skew with KDC too large. Greater than %v seconds.", cfg.LibDefaults.Clockskew.Seconds()) + } + } + return true, nil +} diff --git a/github.com/jcmturner/gokrb5/v8/messages/KDCReq.go b/github.com/jcmturner/gokrb5/v8/messages/KDCReq.go new file mode 100644 index 0000000000..3745afed2b --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/messages/KDCReq.go @@ -0,0 +1,432 @@ +package messages + +// Reference: https://www.ietf.org/rfc/rfc4120.txt +// Section: 5.4.1 + +import ( + "crypto/rand" + "fmt" + "math" + "math/big" + "time" + + "github.com/jcmturner/gofork/encoding/asn1" + "github.com/jcmturner/gokrb5/v8/asn1tools" + "github.com/jcmturner/gokrb5/v8/config" + "github.com/jcmturner/gokrb5/v8/crypto" + "github.com/jcmturner/gokrb5/v8/iana" + "github.com/jcmturner/gokrb5/v8/iana/asnAppTag" + "github.com/jcmturner/gokrb5/v8/iana/flags" + "github.com/jcmturner/gokrb5/v8/iana/keyusage" + "github.com/jcmturner/gokrb5/v8/iana/msgtype" + "github.com/jcmturner/gokrb5/v8/iana/nametype" + "github.com/jcmturner/gokrb5/v8/iana/patype" + "github.com/jcmturner/gokrb5/v8/krberror" + "github.com/jcmturner/gokrb5/v8/types" +) + +type marshalKDCReq struct { + PVNO int `asn1:"explicit,tag:1"` + MsgType int `asn1:"explicit,tag:2"` + PAData types.PADataSequence `asn1:"explicit,optional,tag:3"` + ReqBody asn1.RawValue `asn1:"explicit,tag:4"` +} + +// KDCReqFields represents the KRB_KDC_REQ fields. +type KDCReqFields struct { + PVNO int + MsgType int + PAData types.PADataSequence + ReqBody KDCReqBody + Renewal bool +} + +// ASReq implements RFC 4120 KRB_AS_REQ: https://tools.ietf.org/html/rfc4120#section-5.4.1. +type ASReq struct { + KDCReqFields +} + +// TGSReq implements RFC 4120 KRB_TGS_REQ: https://tools.ietf.org/html/rfc4120#section-5.4.1. +type TGSReq struct { + KDCReqFields +} + +type marshalKDCReqBody struct { + KDCOptions asn1.BitString `asn1:"explicit,tag:0"` + CName types.PrincipalName `asn1:"explicit,optional,tag:1"` + Realm string `asn1:"generalstring,explicit,tag:2"` + SName types.PrincipalName `asn1:"explicit,optional,tag:3"` + From time.Time `asn1:"generalized,explicit,optional,tag:4"` + Till time.Time `asn1:"generalized,explicit,tag:5"` + RTime time.Time `asn1:"generalized,explicit,optional,tag:6"` + Nonce int `asn1:"explicit,tag:7"` + EType []int32 `asn1:"explicit,tag:8"` + Addresses []types.HostAddress `asn1:"explicit,optional,tag:9"` + EncAuthData types.EncryptedData `asn1:"explicit,optional,tag:10"` + // Ticket needs to be a raw value as it is wrapped in an APPLICATION tag + AdditionalTickets asn1.RawValue `asn1:"explicit,optional,tag:11"` +} + +// KDCReqBody implements the KRB_KDC_REQ request body. +type KDCReqBody struct { + KDCOptions asn1.BitString `asn1:"explicit,tag:0"` + CName types.PrincipalName `asn1:"explicit,optional,tag:1"` + Realm string `asn1:"generalstring,explicit,tag:2"` + SName types.PrincipalName `asn1:"explicit,optional,tag:3"` + From time.Time `asn1:"generalized,explicit,optional,tag:4"` + Till time.Time `asn1:"generalized,explicit,tag:5"` + RTime time.Time `asn1:"generalized,explicit,optional,tag:6"` + Nonce int `asn1:"explicit,tag:7"` + EType []int32 `asn1:"explicit,tag:8"` + Addresses []types.HostAddress `asn1:"explicit,optional,tag:9"` + EncAuthData types.EncryptedData `asn1:"explicit,optional,tag:10"` + AdditionalTickets []Ticket `asn1:"explicit,optional,tag:11"` +} + +// NewASReqForTGT generates a new KRB_AS_REQ struct for a TGT request. +func NewASReqForTGT(realm string, c *config.Config, cname types.PrincipalName) (ASReq, error) { + sname := types.PrincipalName{ + NameType: nametype.KRB_NT_SRV_INST, + NameString: []string{"krbtgt", realm}, + } + return NewASReq(realm, c, cname, sname) +} + +// NewASReqForChgPasswd generates a new KRB_AS_REQ struct for a change password request. +func NewASReqForChgPasswd(realm string, c *config.Config, cname types.PrincipalName) (ASReq, error) { + sname := types.PrincipalName{ + NameType: nametype.KRB_NT_PRINCIPAL, + NameString: []string{"kadmin", "changepw"}, + } + return NewASReq(realm, c, cname, sname) +} + +// NewASReq generates a new KRB_AS_REQ struct for a given SNAME. +func NewASReq(realm string, c *config.Config, cname, sname types.PrincipalName) (ASReq, error) { + nonce, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt32)) + if err != nil { + return ASReq{}, err + } + t := time.Now().UTC() + // Copy the default options to make this thread safe + kopts := types.NewKrbFlags() + copy(kopts.Bytes, c.LibDefaults.KDCDefaultOptions.Bytes) + kopts.BitLength = c.LibDefaults.KDCDefaultOptions.BitLength + a := ASReq{ + KDCReqFields{ + PVNO: iana.PVNO, + MsgType: msgtype.KRB_AS_REQ, + PAData: types.PADataSequence{}, + ReqBody: KDCReqBody{ + KDCOptions: kopts, + Realm: realm, + CName: cname, + SName: sname, + Till: t.Add(c.LibDefaults.TicketLifetime), + Nonce: int(nonce.Int64()), + EType: c.LibDefaults.DefaultTktEnctypeIDs, + }, + }, + } + if c.LibDefaults.Forwardable { + types.SetFlag(&a.ReqBody.KDCOptions, flags.Forwardable) + } + if c.LibDefaults.Canonicalize { + types.SetFlag(&a.ReqBody.KDCOptions, flags.Canonicalize) + } + if c.LibDefaults.Proxiable { + types.SetFlag(&a.ReqBody.KDCOptions, flags.Proxiable) + } + if c.LibDefaults.RenewLifetime != 0 { + types.SetFlag(&a.ReqBody.KDCOptions, flags.Renewable) + a.ReqBody.RTime = t.Add(c.LibDefaults.RenewLifetime) + a.ReqBody.RTime = t.Add(time.Duration(48) * time.Hour) + } + if !c.LibDefaults.NoAddresses { + ha, err := types.LocalHostAddresses() + if err != nil { + return a, fmt.Errorf("could not get local addresses: %v", err) + } + ha = append(ha, types.HostAddressesFromNetIPs(c.LibDefaults.ExtraAddresses)...) + a.ReqBody.Addresses = ha + } + return a, nil +} + +// NewTGSReq generates a new KRB_TGS_REQ struct. +func NewTGSReq(cname types.PrincipalName, kdcRealm string, c *config.Config, tgt Ticket, sessionKey types.EncryptionKey, sname types.PrincipalName, renewal bool) (TGSReq, error) { + a, err := tgsReq(cname, sname, kdcRealm, renewal, c) + if err != nil { + return a, err + } + err = a.setPAData(tgt, sessionKey) + return a, err +} + +// NewUser2UserTGSReq returns a TGS-REQ suitable for user-to-user authentication (https://tools.ietf.org/html/rfc4120#section-3.7) +func NewUser2UserTGSReq(cname types.PrincipalName, kdcRealm string, c *config.Config, clientTGT Ticket, sessionKey types.EncryptionKey, sname types.PrincipalName, renewal bool, verifyingTGT Ticket) (TGSReq, error) { + a, err := tgsReq(cname, sname, kdcRealm, renewal, c) + if err != nil { + return a, err + } + a.ReqBody.AdditionalTickets = []Ticket{verifyingTGT} + types.SetFlag(&a.ReqBody.KDCOptions, flags.EncTktInSkey) + err = a.setPAData(clientTGT, sessionKey) + return a, err +} + +// tgsReq populates the fields for a TGS_REQ +func tgsReq(cname, sname types.PrincipalName, kdcRealm string, renewal bool, c *config.Config) (TGSReq, error) { + nonce, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt32)) + if err != nil { + return TGSReq{}, err + } + t := time.Now().UTC() + k := KDCReqFields{ + PVNO: iana.PVNO, + MsgType: msgtype.KRB_TGS_REQ, + ReqBody: KDCReqBody{ + KDCOptions: types.NewKrbFlags(), + Realm: kdcRealm, + CName: cname, // Add the CName to make validation of the reply easier + SName: sname, + Till: t.Add(c.LibDefaults.TicketLifetime), + Nonce: int(nonce.Int64()), + EType: c.LibDefaults.DefaultTGSEnctypeIDs, + }, + Renewal: renewal, + } + if c.LibDefaults.Forwardable { + types.SetFlag(&k.ReqBody.KDCOptions, flags.Forwardable) + } + if c.LibDefaults.Canonicalize { + types.SetFlag(&k.ReqBody.KDCOptions, flags.Canonicalize) + } + if c.LibDefaults.Proxiable { + types.SetFlag(&k.ReqBody.KDCOptions, flags.Proxiable) + } + if c.LibDefaults.RenewLifetime > time.Duration(0) { + types.SetFlag(&k.ReqBody.KDCOptions, flags.Renewable) + k.ReqBody.RTime = t.Add(c.LibDefaults.RenewLifetime) + } + if !c.LibDefaults.NoAddresses { + ha, err := types.LocalHostAddresses() + if err != nil { + return TGSReq{}, fmt.Errorf("could not get local addresses: %v", err) + } + ha = append(ha, types.HostAddressesFromNetIPs(c.LibDefaults.ExtraAddresses)...) + k.ReqBody.Addresses = ha + } + if renewal { + types.SetFlag(&k.ReqBody.KDCOptions, flags.Renew) + types.SetFlag(&k.ReqBody.KDCOptions, flags.Renewable) + } + return TGSReq{ + k, + }, nil +} + +func (k *TGSReq) setPAData(tgt Ticket, sessionKey types.EncryptionKey) error { + // Marshal the request and calculate checksum + b, err := k.ReqBody.Marshal() + if err != nil { + return krberror.Errorf(err, krberror.EncodingError, "error marshaling TGS_REQ body") + } + etype, err := crypto.GetEtype(sessionKey.KeyType) + if err != nil { + return krberror.Errorf(err, krberror.EncryptingError, "error getting etype to encrypt authenticator") + } + cb, err := etype.GetChecksumHash(sessionKey.KeyValue, b, keyusage.TGS_REQ_PA_TGS_REQ_AP_REQ_AUTHENTICATOR_CHKSUM) + if err != nil { + return krberror.Errorf(err, krberror.ChksumError, "error getting etype checksum hash") + } + + // Form PAData for TGS_REQ + // Create authenticator + auth, err := types.NewAuthenticator(tgt.Realm, k.ReqBody.CName) + if err != nil { + return krberror.Errorf(err, krberror.KRBMsgError, "error generating new authenticator") + } + auth.Cksum = types.Checksum{ + CksumType: etype.GetHashID(), + Checksum: cb, + } + // Create AP_REQ + apReq, err := NewAPReq(tgt, sessionKey, auth) + if err != nil { + return krberror.Errorf(err, krberror.KRBMsgError, "error generating new AP_REQ") + } + apb, err := apReq.Marshal() + if err != nil { + return krberror.Errorf(err, krberror.EncodingError, "error marshaling AP_REQ for pre-authentication data") + } + k.PAData = types.PADataSequence{ + types.PAData{ + PADataType: patype.PA_TGS_REQ, + PADataValue: apb, + }, + } + return nil +} + +// Unmarshal bytes b into the ASReq struct. +func (k *ASReq) Unmarshal(b []byte) error { + var m marshalKDCReq + _, err := asn1.UnmarshalWithParams(b, &m, fmt.Sprintf("application,explicit,tag:%v", asnAppTag.ASREQ)) + if err != nil { + return krberror.Errorf(err, krberror.EncodingError, "error unmarshaling AS_REQ") + } + expectedMsgType := msgtype.KRB_AS_REQ + if m.MsgType != expectedMsgType { + return krberror.NewErrorf(krberror.KRBMsgError, "message ID does not indicate a AS_REQ. Expected: %v; Actual: %v", expectedMsgType, m.MsgType) + } + var reqb KDCReqBody + err = reqb.Unmarshal(m.ReqBody.Bytes) + if err != nil { + return krberror.Errorf(err, krberror.EncodingError, "error processing AS_REQ body") + } + k.MsgType = m.MsgType + k.PAData = m.PAData + k.PVNO = m.PVNO + k.ReqBody = reqb + return nil +} + +// Unmarshal bytes b into the TGSReq struct. +func (k *TGSReq) Unmarshal(b []byte) error { + var m marshalKDCReq + _, err := asn1.UnmarshalWithParams(b, &m, fmt.Sprintf("application,explicit,tag:%v", asnAppTag.TGSREQ)) + if err != nil { + return krberror.Errorf(err, krberror.EncodingError, "error unmarshaling TGS_REQ") + } + expectedMsgType := msgtype.KRB_TGS_REQ + if m.MsgType != expectedMsgType { + return krberror.NewErrorf(krberror.KRBMsgError, "message ID does not indicate a TGS_REQ. Expected: %v; Actual: %v", expectedMsgType, m.MsgType) + } + var reqb KDCReqBody + err = reqb.Unmarshal(m.ReqBody.Bytes) + if err != nil { + return krberror.Errorf(err, krberror.EncodingError, "error processing TGS_REQ body") + } + k.MsgType = m.MsgType + k.PAData = m.PAData + k.PVNO = m.PVNO + k.ReqBody = reqb + return nil +} + +// Unmarshal bytes b into the KRB_KDC_REQ body struct. +func (k *KDCReqBody) Unmarshal(b []byte) error { + var m marshalKDCReqBody + _, err := asn1.Unmarshal(b, &m) + if err != nil { + return krberror.Errorf(err, krberror.EncodingError, "error unmarshaling KDC_REQ body") + } + k.KDCOptions = m.KDCOptions + if len(k.KDCOptions.Bytes) < 4 { + tb := make([]byte, 4-len(k.KDCOptions.Bytes)) + k.KDCOptions.Bytes = append(tb, k.KDCOptions.Bytes...) + k.KDCOptions.BitLength = len(k.KDCOptions.Bytes) * 8 + } + k.CName = m.CName + k.Realm = m.Realm + k.SName = m.SName + k.From = m.From + k.Till = m.Till + k.RTime = m.RTime + k.Nonce = m.Nonce + k.EType = m.EType + k.Addresses = m.Addresses + k.EncAuthData = m.EncAuthData + if len(m.AdditionalTickets.Bytes) > 0 { + k.AdditionalTickets, err = unmarshalTicketsSequence(m.AdditionalTickets) + if err != nil { + return krberror.Errorf(err, krberror.EncodingError, "error unmarshaling additional tickets") + } + } + return nil +} + +// Marshal ASReq struct. +func (k *ASReq) Marshal() ([]byte, error) { + m := marshalKDCReq{ + PVNO: k.PVNO, + MsgType: k.MsgType, + PAData: k.PAData, + } + b, err := k.ReqBody.Marshal() + if err != nil { + var mk []byte + return mk, err + } + m.ReqBody = asn1.RawValue{ + Class: asn1.ClassContextSpecific, + IsCompound: true, + Tag: 4, + Bytes: b, + } + mk, err := asn1.Marshal(m) + if err != nil { + return mk, krberror.Errorf(err, krberror.EncodingError, "error marshaling AS_REQ") + } + mk = asn1tools.AddASNAppTag(mk, asnAppTag.ASREQ) + return mk, nil +} + +// Marshal TGSReq struct. +func (k *TGSReq) Marshal() ([]byte, error) { + m := marshalKDCReq{ + PVNO: k.PVNO, + MsgType: k.MsgType, + PAData: k.PAData, + } + b, err := k.ReqBody.Marshal() + if err != nil { + var mk []byte + return mk, err + } + m.ReqBody = asn1.RawValue{ + Class: asn1.ClassContextSpecific, + IsCompound: true, + Tag: 4, + Bytes: b, + } + mk, err := asn1.Marshal(m) + if err != nil { + return mk, krberror.Errorf(err, krberror.EncodingError, "error marshaling AS_REQ") + } + mk = asn1tools.AddASNAppTag(mk, asnAppTag.TGSREQ) + return mk, nil +} + +// Marshal KRB_KDC_REQ body struct. +func (k *KDCReqBody) Marshal() ([]byte, error) { + var b []byte + m := marshalKDCReqBody{ + KDCOptions: k.KDCOptions, + CName: k.CName, + Realm: k.Realm, + SName: k.SName, + From: k.From, + Till: k.Till, + RTime: k.RTime, + Nonce: k.Nonce, + EType: k.EType, + Addresses: k.Addresses, + EncAuthData: k.EncAuthData, + } + rawtkts, err := MarshalTicketSequence(k.AdditionalTickets) + if err != nil { + return b, krberror.Errorf(err, krberror.EncodingError, "error in marshaling KDC request body additional tickets") + } + //The asn1.rawValue needs the tag setting on it for where it is in the KDCReqBody + rawtkts.Tag = 11 + if len(rawtkts.Bytes) > 0 { + m.AdditionalTickets = rawtkts + } + b, err = asn1.Marshal(m) + if err != nil { + return b, krberror.Errorf(err, krberror.EncodingError, "error in marshaling KDC request body") + } + return b, nil +} diff --git a/github.com/jcmturner/gokrb5/v8/messages/KRBCred.go b/github.com/jcmturner/gokrb5/v8/messages/KRBCred.go new file mode 100644 index 0000000000..536fdb9ec9 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/messages/KRBCred.go @@ -0,0 +1,102 @@ +package messages + +import ( + "fmt" + "time" + + "github.com/jcmturner/gofork/encoding/asn1" + "github.com/jcmturner/gokrb5/v8/crypto" + "github.com/jcmturner/gokrb5/v8/iana/asnAppTag" + "github.com/jcmturner/gokrb5/v8/iana/keyusage" + "github.com/jcmturner/gokrb5/v8/iana/msgtype" + "github.com/jcmturner/gokrb5/v8/krberror" + "github.com/jcmturner/gokrb5/v8/types" +) + +type marshalKRBCred struct { + PVNO int `asn1:"explicit,tag:0"` + MsgType int `asn1:"explicit,tag:1"` + Tickets asn1.RawValue `asn1:"explicit,tag:2"` + EncPart types.EncryptedData `asn1:"explicit,tag:3"` +} + +// KRBCred implements RFC 4120 KRB_CRED: https://tools.ietf.org/html/rfc4120#section-5.8.1. +type KRBCred struct { + PVNO int + MsgType int + Tickets []Ticket + EncPart types.EncryptedData + DecryptedEncPart EncKrbCredPart +} + +// EncKrbCredPart is the encrypted part of KRB_CRED. +type EncKrbCredPart struct { + TicketInfo []KrbCredInfo `asn1:"explicit,tag:0"` + Nouce int `asn1:"optional,explicit,tag:1"` + Timestamp time.Time `asn1:"generalized,optional,explicit,tag:2"` + Usec int `asn1:"optional,explicit,tag:3"` + SAddress types.HostAddress `asn1:"optional,explicit,tag:4"` + RAddress types.HostAddress `asn1:"optional,explicit,tag:5"` +} + +// KrbCredInfo is the KRB_CRED_INFO part of KRB_CRED. +type KrbCredInfo struct { + Key types.EncryptionKey `asn1:"explicit,tag:0"` + PRealm string `asn1:"generalstring,optional,explicit,tag:1"` + PName types.PrincipalName `asn1:"optional,explicit,tag:2"` + Flags asn1.BitString `asn1:"optional,explicit,tag:3"` + AuthTime time.Time `asn1:"generalized,optional,explicit,tag:4"` + StartTime time.Time `asn1:"generalized,optional,explicit,tag:5"` + EndTime time.Time `asn1:"generalized,optional,explicit,tag:6"` + RenewTill time.Time `asn1:"generalized,optional,explicit,tag:7"` + SRealm string `asn1:"optional,explicit,ia5,tag:8"` + SName types.PrincipalName `asn1:"optional,explicit,tag:9"` + CAddr types.HostAddresses `asn1:"optional,explicit,tag:10"` +} + +// Unmarshal bytes b into the KRBCred struct. +func (k *KRBCred) Unmarshal(b []byte) error { + var m marshalKRBCred + _, err := asn1.UnmarshalWithParams(b, &m, fmt.Sprintf("application,explicit,tag:%v", asnAppTag.KRBCred)) + if err != nil { + return processUnmarshalReplyError(b, err) + } + expectedMsgType := msgtype.KRB_CRED + if m.MsgType != expectedMsgType { + return krberror.NewErrorf(krberror.KRBMsgError, "message ID does not indicate a KRB_CRED. Expected: %v; Actual: %v", expectedMsgType, m.MsgType) + } + k.PVNO = m.PVNO + k.MsgType = m.MsgType + k.EncPart = m.EncPart + if len(m.Tickets.Bytes) > 0 { + k.Tickets, err = unmarshalTicketsSequence(m.Tickets) + if err != nil { + return krberror.Errorf(err, krberror.EncodingError, "error unmarshaling tickets within KRB_CRED") + } + } + return nil +} + +// DecryptEncPart decrypts the encrypted part of a KRB_CRED. +func (k *KRBCred) DecryptEncPart(key types.EncryptionKey) error { + b, err := crypto.DecryptEncPart(k.EncPart, key, keyusage.KRB_CRED_ENCPART) + if err != nil { + return krberror.Errorf(err, krberror.DecryptingError, "error decrypting KRB_CRED EncPart") + } + var denc EncKrbCredPart + err = denc.Unmarshal(b) + if err != nil { + return krberror.Errorf(err, krberror.EncodingError, "error unmarshaling encrypted part of KRB_CRED") + } + k.DecryptedEncPart = denc + return nil +} + +// Unmarshal bytes b into the encrypted part of KRB_CRED. +func (k *EncKrbCredPart) Unmarshal(b []byte) error { + _, err := asn1.UnmarshalWithParams(b, k, fmt.Sprintf("application,explicit,tag:%v", asnAppTag.EncKrbCredPart)) + if err != nil { + return krberror.Errorf(err, krberror.EncodingError, "error unmarshaling EncKrbCredPart") + } + return nil +} diff --git a/github.com/jcmturner/gokrb5/v8/messages/KRBError.go b/github.com/jcmturner/gokrb5/v8/messages/KRBError.go new file mode 100644 index 0000000000..4c88949d04 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/messages/KRBError.go @@ -0,0 +1,83 @@ +// Package messages implements Kerberos 5 message types and methods. +package messages + +import ( + "fmt" + "time" + + "github.com/jcmturner/gofork/encoding/asn1" + "github.com/jcmturner/gokrb5/v8/iana" + "github.com/jcmturner/gokrb5/v8/iana/asnAppTag" + "github.com/jcmturner/gokrb5/v8/iana/errorcode" + "github.com/jcmturner/gokrb5/v8/iana/msgtype" + "github.com/jcmturner/gokrb5/v8/krberror" + "github.com/jcmturner/gokrb5/v8/types" +) + +// KRBError implements RFC 4120 KRB_ERROR: https://tools.ietf.org/html/rfc4120#section-5.9.1. +type KRBError struct { + PVNO int `asn1:"explicit,tag:0"` + MsgType int `asn1:"explicit,tag:1"` + CTime time.Time `asn1:"generalized,optional,explicit,tag:2"` + Cusec int `asn1:"optional,explicit,tag:3"` + STime time.Time `asn1:"generalized,explicit,tag:4"` + Susec int `asn1:"explicit,tag:5"` + ErrorCode int32 `asn1:"explicit,tag:6"` + CRealm string `asn1:"generalstring,optional,explicit,tag:7"` + CName types.PrincipalName `asn1:"optional,explicit,tag:8"` + Realm string `asn1:"generalstring,explicit,tag:9"` + SName types.PrincipalName `asn1:"explicit,tag:10"` + EText string `asn1:"generalstring,optional,explicit,tag:11"` + EData []byte `asn1:"optional,explicit,tag:12"` +} + +// NewKRBError creates a new KRBError. +func NewKRBError(sname types.PrincipalName, realm string, code int32, etext string) KRBError { + t := time.Now().UTC() + return KRBError{ + PVNO: iana.PVNO, + MsgType: msgtype.KRB_ERROR, + STime: t, + Susec: int((t.UnixNano() / int64(time.Microsecond)) - (t.Unix() * 1e6)), + ErrorCode: code, + SName: sname, + Realm: realm, + EText: etext, + } +} + +// Unmarshal bytes b into the KRBError struct. +func (k *KRBError) Unmarshal(b []byte) error { + _, err := asn1.UnmarshalWithParams(b, k, fmt.Sprintf("application,explicit,tag:%v", asnAppTag.KRBError)) + if err != nil { + return krberror.Errorf(err, krberror.EncodingError, "KRB_ERROR unmarshal error") + } + expectedMsgType := msgtype.KRB_ERROR + if k.MsgType != expectedMsgType { + return krberror.NewErrorf(krberror.KRBMsgError, "message ID does not indicate a KRB_ERROR. Expected: %v; Actual: %v", expectedMsgType, k.MsgType) + } + return nil +} + +// Error method implementing error interface on KRBError struct. +func (k KRBError) Error() string { + etxt := fmt.Sprintf("KRB Error: %s", errorcode.Lookup(k.ErrorCode)) + if k.EText != "" { + etxt = fmt.Sprintf("%s - %s", etxt, k.EText) + } + return etxt +} + +func processUnmarshalReplyError(b []byte, err error) error { + switch err.(type) { + case asn1.StructuralError: + var krberr KRBError + tmperr := krberr.Unmarshal(b) + if tmperr != nil { + return krberror.Errorf(err, krberror.EncodingError, "failed to unmarshal KDC's reply") + } + return krberr + default: + return krberror.Errorf(err, krberror.EncodingError, "failed to unmarshal KDC's reply") + } +} diff --git a/github.com/jcmturner/gokrb5/v8/messages/KRBPriv.go b/github.com/jcmturner/gokrb5/v8/messages/KRBPriv.go new file mode 100644 index 0000000000..0ca614949f --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/messages/KRBPriv.go @@ -0,0 +1,108 @@ +package messages + +import ( + "fmt" + "time" + + "github.com/jcmturner/gofork/encoding/asn1" + "github.com/jcmturner/gokrb5/v8/asn1tools" + "github.com/jcmturner/gokrb5/v8/crypto" + "github.com/jcmturner/gokrb5/v8/iana" + "github.com/jcmturner/gokrb5/v8/iana/asnAppTag" + "github.com/jcmturner/gokrb5/v8/iana/keyusage" + "github.com/jcmturner/gokrb5/v8/iana/msgtype" + "github.com/jcmturner/gokrb5/v8/krberror" + "github.com/jcmturner/gokrb5/v8/types" +) + +// KRBPriv implements RFC 4120 type: https://tools.ietf.org/html/rfc4120#section-5.7.1. +type KRBPriv struct { + PVNO int `asn1:"explicit,tag:0"` + MsgType int `asn1:"explicit,tag:1"` + EncPart types.EncryptedData `asn1:"explicit,tag:3"` + DecryptedEncPart EncKrbPrivPart `asn1:"optional,omitempty"` // Not part of ASN1 bytes so marked as optional so unmarshalling works +} + +// EncKrbPrivPart is the encrypted part of KRB_PRIV. +type EncKrbPrivPart struct { + UserData []byte `asn1:"explicit,tag:0"` + Timestamp time.Time `asn1:"generalized,optional,explicit,tag:1"` + Usec int `asn1:"optional,explicit,tag:2"` + SequenceNumber int64 `asn1:"optional,explicit,tag:3"` + SAddress types.HostAddress `asn1:"explicit,tag:4"` + RAddress types.HostAddress `asn1:"optional,explicit,tag:5"` +} + +// NewKRBPriv returns a new KRBPriv type. +func NewKRBPriv(part EncKrbPrivPart) KRBPriv { + return KRBPriv{ + PVNO: iana.PVNO, + MsgType: msgtype.KRB_PRIV, + DecryptedEncPart: part, + } +} + +// Unmarshal bytes b into the KRBPriv struct. +func (k *KRBPriv) Unmarshal(b []byte) error { + _, err := asn1.UnmarshalWithParams(b, k, fmt.Sprintf("application,explicit,tag:%v", asnAppTag.KRBPriv)) + if err != nil { + return processUnmarshalReplyError(b, err) + } + expectedMsgType := msgtype.KRB_PRIV + if k.MsgType != expectedMsgType { + return krberror.NewErrorf(krberror.KRBMsgError, "message ID does not indicate a KRB_PRIV. Expected: %v; Actual: %v", expectedMsgType, k.MsgType) + } + return nil +} + +// Unmarshal bytes b into the EncKrbPrivPart struct. +func (k *EncKrbPrivPart) Unmarshal(b []byte) error { + _, err := asn1.UnmarshalWithParams(b, k, fmt.Sprintf("application,explicit,tag:%v", asnAppTag.EncKrbPrivPart)) + if err != nil { + return krberror.Errorf(err, krberror.EncodingError, "KRB_PRIV unmarshal error") + } + return nil +} + +// Marshal the KRBPriv. +func (k *KRBPriv) Marshal() ([]byte, error) { + tk := KRBPriv{ + PVNO: k.PVNO, + MsgType: k.MsgType, + EncPart: k.EncPart, + } + b, err := asn1.Marshal(tk) + if err != nil { + return []byte{}, err + } + b = asn1tools.AddASNAppTag(b, asnAppTag.KRBPriv) + return b, nil +} + +// EncryptEncPart encrypts the DecryptedEncPart within the KRBPriv. +// Use to prepare for marshaling. +func (k *KRBPriv) EncryptEncPart(key types.EncryptionKey) error { + b, err := asn1.Marshal(k.DecryptedEncPart) + if err != nil { + return err + } + b = asn1tools.AddASNAppTag(b, asnAppTag.EncKrbPrivPart) + k.EncPart, err = crypto.GetEncryptedData(b, key, keyusage.KRB_PRIV_ENCPART, 1) + if err != nil { + return err + } + return nil +} + +// DecryptEncPart decrypts the encrypted part of the KRBPriv message. +func (k *KRBPriv) DecryptEncPart(key types.EncryptionKey) error { + b, err := crypto.DecryptEncPart(k.EncPart, key, keyusage.KRB_PRIV_ENCPART) + if err != nil { + return fmt.Errorf("error decrypting KRBPriv EncPart: %v", err) + } + err = k.DecryptedEncPart.Unmarshal(b) + if err != nil { + return fmt.Errorf("error unmarshaling encrypted part: %v", err) + } + return nil +} diff --git a/github.com/jcmturner/gokrb5/v8/messages/KRBSafe.go b/github.com/jcmturner/gokrb5/v8/messages/KRBSafe.go new file mode 100644 index 0000000000..52cd284493 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/messages/KRBSafe.go @@ -0,0 +1,43 @@ +package messages + +import ( + "fmt" + "time" + + "github.com/jcmturner/gofork/encoding/asn1" + "github.com/jcmturner/gokrb5/v8/iana/asnAppTag" + "github.com/jcmturner/gokrb5/v8/iana/msgtype" + "github.com/jcmturner/gokrb5/v8/krberror" + "github.com/jcmturner/gokrb5/v8/types" +) + +// KRBSafe implements RFC 4120 KRB_SAFE: https://tools.ietf.org/html/rfc4120#section-5.6.1. +type KRBSafe struct { + PVNO int `asn1:"explicit,tag:0"` + MsgType int `asn1:"explicit,tag:1"` + SafeBody KRBSafeBody `asn1:"explicit,tag:2"` + Cksum types.Checksum `asn1:"explicit,tag:3"` +} + +// KRBSafeBody implements the KRB_SAFE_BODY of KRB_SAFE. +type KRBSafeBody struct { + UserData []byte `asn1:"explicit,tag:0"` + Timestamp time.Time `asn1:"generalized,optional,explicit,tag:1"` + Usec int `asn1:"optional,explicit,tag:2"` + SequenceNumber int64 `asn1:"optional,explicit,tag:3"` + SAddress types.HostAddress `asn1:"explicit,tag:4"` + RAddress types.HostAddress `asn1:"optional,explicit,tag:5"` +} + +// Unmarshal bytes b into the KRBSafe struct. +func (s *KRBSafe) Unmarshal(b []byte) error { + _, err := asn1.UnmarshalWithParams(b, s, fmt.Sprintf("application,explicit,tag:%v", asnAppTag.KRBSafe)) + if err != nil { + return processUnmarshalReplyError(b, err) + } + expectedMsgType := msgtype.KRB_SAFE + if s.MsgType != expectedMsgType { + return krberror.NewErrorf(krberror.KRBMsgError, "message ID does not indicate a KRB_SAFE. Expected: %v; Actual: %v", expectedMsgType, s.MsgType) + } + return nil +} diff --git a/github.com/jcmturner/gokrb5/v8/messages/Ticket.go b/github.com/jcmturner/gokrb5/v8/messages/Ticket.go new file mode 100644 index 0000000000..58bc97edb5 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/messages/Ticket.go @@ -0,0 +1,265 @@ +package messages + +import ( + "crypto/rand" + "fmt" + "log" + "time" + + "github.com/jcmturner/gofork/encoding/asn1" + "github.com/jcmturner/gokrb5/v8/asn1tools" + "github.com/jcmturner/gokrb5/v8/crypto" + "github.com/jcmturner/gokrb5/v8/iana" + "github.com/jcmturner/gokrb5/v8/iana/adtype" + "github.com/jcmturner/gokrb5/v8/iana/asnAppTag" + "github.com/jcmturner/gokrb5/v8/iana/errorcode" + "github.com/jcmturner/gokrb5/v8/iana/flags" + "github.com/jcmturner/gokrb5/v8/iana/keyusage" + "github.com/jcmturner/gokrb5/v8/keytab" + "github.com/jcmturner/gokrb5/v8/krberror" + "github.com/jcmturner/gokrb5/v8/pac" + "github.com/jcmturner/gokrb5/v8/types" +) + +// Reference: https://www.ietf.org/rfc/rfc4120.txt +// Section: 5.3 + +// Ticket implements the Kerberos ticket. +type Ticket struct { + TktVNO int `asn1:"explicit,tag:0"` + Realm string `asn1:"generalstring,explicit,tag:1"` + SName types.PrincipalName `asn1:"explicit,tag:2"` + EncPart types.EncryptedData `asn1:"explicit,tag:3"` + DecryptedEncPart EncTicketPart `asn1:"optional"` // Not part of ASN1 bytes so marked as optional so unmarshalling works +} + +// EncTicketPart is the encrypted part of the Ticket. +type EncTicketPart struct { + Flags asn1.BitString `asn1:"explicit,tag:0"` + Key types.EncryptionKey `asn1:"explicit,tag:1"` + CRealm string `asn1:"generalstring,explicit,tag:2"` + CName types.PrincipalName `asn1:"explicit,tag:3"` + Transited TransitedEncoding `asn1:"explicit,tag:4"` + AuthTime time.Time `asn1:"generalized,explicit,tag:5"` + StartTime time.Time `asn1:"generalized,explicit,optional,tag:6"` + EndTime time.Time `asn1:"generalized,explicit,tag:7"` + RenewTill time.Time `asn1:"generalized,explicit,optional,tag:8"` + CAddr types.HostAddresses `asn1:"explicit,optional,tag:9"` + AuthorizationData types.AuthorizationData `asn1:"explicit,optional,tag:10"` +} + +// TransitedEncoding part of the ticket's encrypted part. +type TransitedEncoding struct { + TRType int32 `asn1:"explicit,tag:0"` + Contents []byte `asn1:"explicit,tag:1"` +} + +// NewTicket creates a new Ticket instance. +func NewTicket(cname types.PrincipalName, crealm string, sname types.PrincipalName, srealm string, flags asn1.BitString, sktab *keytab.Keytab, eTypeID int32, kvno int, authTime, startTime, endTime, renewTill time.Time) (Ticket, types.EncryptionKey, error) { + etype, err := crypto.GetEtype(eTypeID) + if err != nil { + return Ticket{}, types.EncryptionKey{}, krberror.Errorf(err, krberror.EncryptingError, "error getting etype for new ticket") + } + ks := etype.GetKeyByteSize() + kv := make([]byte, ks, ks) + rand.Read(kv) + sessionKey := types.EncryptionKey{ + KeyType: eTypeID, + KeyValue: kv, + } + etp := EncTicketPart{ + Flags: flags, + Key: sessionKey, + CRealm: crealm, + CName: cname, + Transited: TransitedEncoding{}, + AuthTime: authTime, + StartTime: startTime, + EndTime: endTime, + RenewTill: renewTill, + } + b, err := asn1.Marshal(etp) + if err != nil { + return Ticket{}, types.EncryptionKey{}, krberror.Errorf(err, krberror.EncodingError, "error marshalling ticket encpart") + } + b = asn1tools.AddASNAppTag(b, asnAppTag.EncTicketPart) + skey, _, err := sktab.GetEncryptionKey(sname, srealm, kvno, eTypeID) + if err != nil { + return Ticket{}, types.EncryptionKey{}, krberror.Errorf(err, krberror.EncryptingError, "error getting encryption key for new ticket") + } + ed, err := crypto.GetEncryptedData(b, skey, keyusage.KDC_REP_TICKET, kvno) + if err != nil { + return Ticket{}, types.EncryptionKey{}, krberror.Errorf(err, krberror.EncryptingError, "error encrypting ticket encpart") + } + tkt := Ticket{ + TktVNO: iana.PVNO, + Realm: srealm, + SName: sname, + EncPart: ed, + } + return tkt, sessionKey, nil +} + +// Unmarshal bytes b into a Ticket struct. +func (t *Ticket) Unmarshal(b []byte) error { + _, err := asn1.UnmarshalWithParams(b, t, fmt.Sprintf("application,explicit,tag:%d", asnAppTag.Ticket)) + return err +} + +// Marshal the Ticket. +func (t *Ticket) Marshal() ([]byte, error) { + b, err := asn1.Marshal(*t) + if err != nil { + return nil, err + } + b = asn1tools.AddASNAppTag(b, asnAppTag.Ticket) + return b, nil +} + +// Unmarshal bytes b into the EncTicketPart struct. +func (t *EncTicketPart) Unmarshal(b []byte) error { + _, err := asn1.UnmarshalWithParams(b, t, fmt.Sprintf("application,explicit,tag:%d", asnAppTag.EncTicketPart)) + return err +} + +// unmarshalTicket returns a ticket from the bytes provided. +func unmarshalTicket(b []byte) (t Ticket, err error) { + err = t.Unmarshal(b) + return +} + +// UnmarshalTicketsSequence returns a slice of Tickets from a raw ASN1 value. +func unmarshalTicketsSequence(in asn1.RawValue) ([]Ticket, error) { + //This is a workaround to a asn1 decoding issue in golang - https://github.com/golang/go/issues/17321. It's not pretty I'm afraid + //We pull out raw values from the larger raw value (that is actually the data of the sequence of raw values) and track our position moving along the data. + b := in.Bytes + // Ignore the head of the asn1 stream (1 byte for tag and those for the length) as this is what tells us its a sequence but we're handling it ourselves + p := 1 + asn1tools.GetNumberBytesInLengthHeader(in.Bytes) + var tkts []Ticket + var raw asn1.RawValue + for p < (len(b)) { + _, err := asn1.UnmarshalWithParams(b[p:], &raw, fmt.Sprintf("application,tag:%d", asnAppTag.Ticket)) + if err != nil { + return nil, fmt.Errorf("unmarshaling sequence of tickets failed getting length of ticket: %v", err) + } + t, err := unmarshalTicket(b[p:]) + if err != nil { + return nil, fmt.Errorf("unmarshaling sequence of tickets failed: %v", err) + } + p += len(raw.FullBytes) + tkts = append(tkts, t) + } + MarshalTicketSequence(tkts) + return tkts, nil +} + +// MarshalTicketSequence marshals a slice of Tickets returning an ASN1 raw value containing the ticket sequence. +func MarshalTicketSequence(tkts []Ticket) (asn1.RawValue, error) { + raw := asn1.RawValue{ + Class: 2, + IsCompound: true, + } + if len(tkts) < 1 { + // There are no tickets to marshal + return raw, nil + } + var btkts []byte + for i, t := range tkts { + b, err := t.Marshal() + if err != nil { + return raw, fmt.Errorf("error marshaling ticket number %d in sequence of tickets", i+1) + } + btkts = append(btkts, b...) + } + // The ASN1 wrapping consists of 2 bytes: + // 1st byte -> Identifier Octet - In this case an OCTET STRING (ASN TAG + // 2nd byte -> The length (this will be the size indicated in the input bytes + 2 for the additional bytes we add here. + // Application Tag: + //| Byte: | 8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | + //| Value: | 0 | 1 | 1 | From the RFC spec 4120 | + //| Explanation | Defined by the ASN1 encoding rules for an application tag | A value of 1 indicates a constructed type | The ASN Application tag value | + btkts = append(asn1tools.MarshalLengthBytes(len(btkts)), btkts...) + btkts = append([]byte{byte(32 + asn1.TagSequence)}, btkts...) + raw.Bytes = btkts + // If we need to create the full bytes then identifier octet is "context-specific" = 128 + "constructed" + 32 + the wrapping explicit tag (11) + //fmt.Fprintf(os.Stderr, "mRaw fb: %v\n", raw.FullBytes) + return raw, nil +} + +// DecryptEncPart decrypts the encrypted part of the ticket. +// The sname argument can be used to specify which service principal's key should be used to decrypt the ticket. +// If nil is passed as the sname then the service principal specified within the ticket it used. +func (t *Ticket) DecryptEncPart(keytab *keytab.Keytab, sname *types.PrincipalName) error { + if sname == nil { + sname = &t.SName + } + key, _, err := keytab.GetEncryptionKey(*sname, t.Realm, t.EncPart.KVNO, t.EncPart.EType) + if err != nil { + return NewKRBError(t.SName, t.Realm, errorcode.KRB_AP_ERR_NOKEY, fmt.Sprintf("Could not get key from keytab: %v", err)) + } + return t.Decrypt(key) +} + +// Decrypt decrypts the encrypted part of the ticket using the key provided. +func (t *Ticket) Decrypt(key types.EncryptionKey) error { + b, err := crypto.DecryptEncPart(t.EncPart, key, keyusage.KDC_REP_TICKET) + if err != nil { + return fmt.Errorf("error decrypting Ticket EncPart: %v", err) + } + var denc EncTicketPart + err = denc.Unmarshal(b) + if err != nil { + return fmt.Errorf("error unmarshaling encrypted part: %v", err) + } + t.DecryptedEncPart = denc + return nil +} + +// GetPACType returns a Microsoft PAC that has been extracted from the ticket and processed. +func (t *Ticket) GetPACType(keytab *keytab.Keytab, sname *types.PrincipalName, l *log.Logger) (bool, pac.PACType, error) { + var isPAC bool + for _, ad := range t.DecryptedEncPart.AuthorizationData { + if ad.ADType == adtype.ADIfRelevant { + var ad2 types.AuthorizationData + err := ad2.Unmarshal(ad.ADData) + if err != nil { + l.Printf("PAC authorization data could not be unmarshaled: %v", err) + continue + } + if ad2[0].ADType == adtype.ADWin2KPAC { + isPAC = true + var p pac.PACType + err = p.Unmarshal(ad2[0].ADData) + if err != nil { + return isPAC, p, fmt.Errorf("error unmarshaling PAC: %v", err) + } + if sname == nil { + sname = &t.SName + } + key, _, err := keytab.GetEncryptionKey(*sname, t.Realm, t.EncPart.KVNO, t.EncPart.EType) + if err != nil { + return isPAC, p, NewKRBError(t.SName, t.Realm, errorcode.KRB_AP_ERR_NOKEY, fmt.Sprintf("Could not get key from keytab: %v", err)) + } + err = p.ProcessPACInfoBuffers(key, l) + return isPAC, p, err + } + } + } + return isPAC, pac.PACType{}, nil +} + +// Valid checks it the ticket is currently valid. Max duration passed endtime passed in as argument. +func (t *Ticket) Valid(d time.Duration) (bool, error) { + // Check for future tickets or invalid tickets + time := time.Now().UTC() + if t.DecryptedEncPart.StartTime.Sub(time) > d || types.IsFlagSet(&t.DecryptedEncPart.Flags, flags.Invalid) { + return false, NewKRBError(t.SName, t.Realm, errorcode.KRB_AP_ERR_TKT_NYV, "service ticket provided is not yet valid") + } + + // Check for expired ticket + if time.Sub(t.DecryptedEncPart.EndTime) > d { + return false, NewKRBError(t.SName, t.Realm, errorcode.KRB_AP_ERR_TKT_EXPIRED, "service ticket provided has expired") + } + + return true, nil +} diff --git a/github.com/jcmturner/gokrb5/v8/pac/client_claims.go b/github.com/jcmturner/gokrb5/v8/pac/client_claims.go new file mode 100644 index 0000000000..08e63f4374 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/pac/client_claims.go @@ -0,0 +1,33 @@ +package pac + +import ( + "bytes" + "fmt" + + "github.com/jcmturner/rpc/v2/mstypes" + "github.com/jcmturner/rpc/v2/ndr" +) + +// Claims reference: https://msdn.microsoft.com/en-us/library/hh553895.aspx + +// ClientClaimsInfo implements https://msdn.microsoft.com/en-us/library/hh536365.aspx +type ClientClaimsInfo struct { + ClaimsSetMetadata mstypes.ClaimsSetMetadata + ClaimsSet mstypes.ClaimsSet +} + +// Unmarshal bytes into the ClientClaimsInfo struct +func (k *ClientClaimsInfo) Unmarshal(b []byte) (err error) { + dec := ndr.NewDecoder(bytes.NewReader(b)) + m := new(mstypes.ClaimsSetMetadata) + err = dec.Decode(m) + if err != nil { + err = fmt.Errorf("error unmarshaling ClientClaimsInfo ClaimsSetMetadata: %v", err) + } + k.ClaimsSetMetadata = *m + k.ClaimsSet, err = k.ClaimsSetMetadata.ClaimsSet() + if err != nil { + err = fmt.Errorf("error unmarshaling ClientClaimsInfo ClaimsSet: %v", err) + } + return +} diff --git a/github.com/jcmturner/gokrb5/v8/pac/client_info.go b/github.com/jcmturner/gokrb5/v8/pac/client_info.go new file mode 100644 index 0000000000..ddd957808a --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/pac/client_info.go @@ -0,0 +1,31 @@ +package pac + +import ( + "bytes" + + "github.com/jcmturner/rpc/v2/mstypes" +) + +// ClientInfo implements https://msdn.microsoft.com/en-us/library/cc237951.aspx +type ClientInfo struct { + ClientID mstypes.FileTime // A FILETIME structure in little-endian format that contains the Kerberos initial ticket-granting ticket TGT authentication time + NameLength uint16 // An unsigned 16-bit integer in little-endian format that specifies the length, in bytes, of the Name field. + Name string // An array of 16-bit Unicode characters in little-endian format that contains the client's account name. +} + +// Unmarshal bytes into the ClientInfo struct +func (k *ClientInfo) Unmarshal(b []byte) (err error) { + //The PAC_CLIENT_INFO structure is a simple structure that is not NDR-encoded. + r := mstypes.NewReader(bytes.NewReader(b)) + + k.ClientID, err = r.FileTime() + if err != nil { + return + } + k.NameLength, err = r.Uint16() + if err != nil { + return + } + k.Name, err = r.UTF16String(int(k.NameLength)) + return +} diff --git a/github.com/jcmturner/gokrb5/v8/pac/credentials_info.go b/github.com/jcmturner/gokrb5/v8/pac/credentials_info.go new file mode 100644 index 0000000000..2266ce8f02 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/pac/credentials_info.go @@ -0,0 +1,82 @@ +package pac + +import ( + "bytes" + "errors" + "fmt" + + "github.com/jcmturner/gokrb5/v8/crypto" + "github.com/jcmturner/gokrb5/v8/iana/keyusage" + "github.com/jcmturner/gokrb5/v8/types" + "github.com/jcmturner/rpc/v2/mstypes" + "github.com/jcmturner/rpc/v2/ndr" +) + +// https://msdn.microsoft.com/en-us/library/cc237931.aspx + +// CredentialsInfo implements https://msdn.microsoft.com/en-us/library/cc237953.aspx +type CredentialsInfo struct { + Version uint32 // A 32-bit unsigned integer in little-endian format that defines the version. MUST be 0x00000000. + EType uint32 + PACCredentialDataEncrypted []byte // Key usage number for encryption: KERB_NON_KERB_SALT (16) + PACCredentialData CredentialData +} + +// Unmarshal bytes into the CredentialsInfo struct +func (c *CredentialsInfo) Unmarshal(b []byte, k types.EncryptionKey) (err error) { + //The CredentialsInfo structure is a simple structure that is not NDR-encoded. + r := mstypes.NewReader(bytes.NewReader(b)) + + c.Version, err = r.Uint32() + if err != nil { + return + } + if c.Version != 0 { + err = errors.New("credentials info version is not zero") + return + } + c.EType, err = r.Uint32() + if err != nil { + return + } + c.PACCredentialDataEncrypted, err = r.ReadBytes(len(b) - 8) + + err = c.DecryptEncPart(k) + if err != nil { + err = fmt.Errorf("error decrypting PAC Credentials Data: %v", err) + return + } + return +} + +// DecryptEncPart decrypts the encrypted part of the CredentialsInfo. +func (c *CredentialsInfo) DecryptEncPart(k types.EncryptionKey) error { + if k.KeyType != int32(c.EType) { + return fmt.Errorf("key provided is not the correct type. Type needed: %d, type provided: %d", c.EType, k.KeyType) + } + pt, err := crypto.DecryptMessage(c.PACCredentialDataEncrypted, k, keyusage.KERB_NON_KERB_SALT) + if err != nil { + return err + } + err = c.PACCredentialData.Unmarshal(pt) + if err != nil { + return err + } + return nil +} + +// CredentialData implements https://msdn.microsoft.com/en-us/library/cc237952.aspx +type CredentialData struct { + CredentialCount uint32 + Credentials []SECPKGSupplementalCred // Size is the value of CredentialCount +} + +// Unmarshal converts the bytes provided into a CredentialData type. +func (c *CredentialData) Unmarshal(b []byte) (err error) { + dec := ndr.NewDecoder(bytes.NewReader(b)) + err = dec.Decode(c) + if err != nil { + err = fmt.Errorf("error unmarshaling KerbValidationInfo: %v", err) + } + return +} diff --git a/github.com/jcmturner/gokrb5/v8/pac/device_claims.go b/github.com/jcmturner/gokrb5/v8/pac/device_claims.go new file mode 100644 index 0000000000..0892fadd13 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/pac/device_claims.go @@ -0,0 +1,33 @@ +package pac + +import ( + "bytes" + "fmt" + + "github.com/jcmturner/rpc/v2/mstypes" + "github.com/jcmturner/rpc/v2/ndr" +) + +// Claims reference: https://msdn.microsoft.com/en-us/library/hh553895.aspx + +// DeviceClaimsInfo implements https://msdn.microsoft.com/en-us/library/hh554226.aspx +type DeviceClaimsInfo struct { + ClaimsSetMetadata mstypes.ClaimsSetMetadata + ClaimsSet mstypes.ClaimsSet +} + +// Unmarshal bytes into the ClientClaimsInfo struct +func (k *DeviceClaimsInfo) Unmarshal(b []byte) (err error) { + dec := ndr.NewDecoder(bytes.NewReader(b)) + m := new(mstypes.ClaimsSetMetadata) + err = dec.Decode(m) + if err != nil { + err = fmt.Errorf("error unmarshaling ClientClaimsInfo ClaimsSetMetadata: %v", err) + } + k.ClaimsSetMetadata = *m + k.ClaimsSet, err = k.ClaimsSetMetadata.ClaimsSet() + if err != nil { + err = fmt.Errorf("error unmarshaling ClientClaimsInfo ClaimsSet: %v", err) + } + return +} diff --git a/github.com/jcmturner/gokrb5/v8/pac/device_info.go b/github.com/jcmturner/gokrb5/v8/pac/device_info.go new file mode 100644 index 0000000000..ce82daa580 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/pac/device_info.go @@ -0,0 +1,32 @@ +package pac + +import ( + "bytes" + "fmt" + + "github.com/jcmturner/rpc/v2/mstypes" + "github.com/jcmturner/rpc/v2/ndr" +) + +// DeviceInfo implements https://msdn.microsoft.com/en-us/library/hh536402.aspx +type DeviceInfo struct { + UserID uint32 // A 32-bit unsigned integer that contains the RID of the account. If the UserId member equals 0x00000000, the first group SID in this member is the SID for this account. + PrimaryGroupID uint32 // A 32-bit unsigned integer that contains the RID for the primary group to which this account belongs. + AccountDomainID mstypes.RPCSID `ndr:"pointer"` // A SID structure that contains the SID for the domain of the account.This member is used in conjunction with the UserId, and GroupIds members to create the user and group SIDs for the client. + AccountGroupCount uint32 // A 32-bit unsigned integer that contains the number of groups within the account domain to which the account belongs + AccountGroupIDs []mstypes.GroupMembership `ndr:"pointer,conformant"` // A pointer to a list of GROUP_MEMBERSHIP (section 2.2.2) structures that contains the groups to which the account belongs in the account domain. The number of groups in this list MUST be equal to GroupCount. + SIDCount uint32 // A 32-bit unsigned integer that contains the total number of SIDs present in the ExtraSids member. + ExtraSIDs []mstypes.KerbSidAndAttributes `ndr:"pointer,conformant"` // A pointer to a list of KERB_SID_AND_ATTRIBUTES structures that contain a list of SIDs corresponding to groups not in domains. If the UserId member equals 0x00000000, the first group SID in this member is the SID for this account. + DomainGroupCount uint32 // A 32-bit unsigned integer that contains the number of domains with groups to which the account belongs. + DomainGroup []mstypes.DomainGroupMembership `ndr:"pointer,conformant"` // A pointer to a list of DOMAIN_GROUP_MEMBERSHIP structures (section 2.2.3) that contains the domains to which the account belongs to a group. The number of sets in this list MUST be equal to DomainCount. +} + +// Unmarshal bytes into the DeviceInfo struct +func (k *DeviceInfo) Unmarshal(b []byte) (err error) { + dec := ndr.NewDecoder(bytes.NewReader(b)) + err = dec.Decode(k) + if err != nil { + err = fmt.Errorf("error unmarshaling DeviceInfo: %v", err) + } + return +} diff --git a/github.com/jcmturner/gokrb5/v8/pac/kerb_validation_info.go b/github.com/jcmturner/gokrb5/v8/pac/kerb_validation_info.go new file mode 100644 index 0000000000..dde78614ee --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/pac/kerb_validation_info.go @@ -0,0 +1,110 @@ +// Package pac implements Microsoft Privilege Attribute Certificate (PAC) processing. +package pac + +import ( + "bytes" + "fmt" + + "github.com/jcmturner/rpc/v2/mstypes" + "github.com/jcmturner/rpc/v2/ndr" +) + +// KERB_VALIDATION_INFO flags. +const ( + USERFLAG_GUEST = 31 // Authentication was done via the GUEST account; no password was used. + USERFLAG_NO_ENCRYPTION_AVAILABLE = 30 // No encryption is available. + USERFLAG_LAN_MANAGER_KEY = 28 // LAN Manager key was used for authentication. + USERFLAG_SUB_AUTH = 25 // Sub-authentication used; session key came from the sub-authentication package. + USERFLAG_EXTRA_SIDS = 26 // Indicates that the ExtraSids field is populated and contains additional SIDs. + USERFLAG_MACHINE_ACCOUNT = 24 // Indicates that the account is a machine account. + USERFLAG_DC_NTLM2 = 23 // Indicates that the domain controller understands NTLMv2. + USERFLAG_RESOURCE_GROUPIDS = 22 // Indicates that the ResourceGroupIds field is populated. + USERFLAG_PROFILEPATH = 21 // Indicates that ProfilePath is populated. + USERFLAG_NTLM2_NTCHALLENGERESP = 20 // The NTLMv2 response from the NtChallengeResponseFields ([MS-NLMP] section 2.2.1.3) was used for authentication and session key generation. + USERFLAG_LM2_LMCHALLENGERESP = 19 // The LMv2 response from the LmChallengeResponseFields ([MS-NLMP] section 2.2.1.3) was used for authentication and session key generation. + USERFLAG_AUTH_LMCHALLENGERESP_KEY_NTCHALLENGERESP = 18 // The LMv2 response from the LmChallengeResponseFields ([MS-NLMP] section 2.2.1.3) was used for authentication and the NTLMv2 response from the NtChallengeResponseFields ([MS-NLMP] section 2.2.1.3) was used session key generation. +) + +// KerbValidationInfo implement https://msdn.microsoft.com/en-us/library/cc237948.aspx +type KerbValidationInfo struct { + LogOnTime mstypes.FileTime + LogOffTime mstypes.FileTime + KickOffTime mstypes.FileTime + PasswordLastSet mstypes.FileTime + PasswordCanChange mstypes.FileTime + PasswordMustChange mstypes.FileTime + EffectiveName mstypes.RPCUnicodeString + FullName mstypes.RPCUnicodeString + LogonScript mstypes.RPCUnicodeString + ProfilePath mstypes.RPCUnicodeString + HomeDirectory mstypes.RPCUnicodeString + HomeDirectoryDrive mstypes.RPCUnicodeString + LogonCount uint16 + BadPasswordCount uint16 + UserID uint32 + PrimaryGroupID uint32 + GroupCount uint32 + GroupIDs []mstypes.GroupMembership `ndr:"pointer,conformant"` + UserFlags uint32 + UserSessionKey mstypes.UserSessionKey + LogonServer mstypes.RPCUnicodeString + LogonDomainName mstypes.RPCUnicodeString + LogonDomainID mstypes.RPCSID `ndr:"pointer"` + Reserved1 [2]uint32 // Has 2 elements + UserAccountControl uint32 + SubAuthStatus uint32 + LastSuccessfulILogon mstypes.FileTime + LastFailedILogon mstypes.FileTime + FailedILogonCount uint32 + Reserved3 uint32 + SIDCount uint32 + ExtraSIDs []mstypes.KerbSidAndAttributes `ndr:"pointer,conformant"` + ResourceGroupDomainSID mstypes.RPCSID `ndr:"pointer"` + ResourceGroupCount uint32 + ResourceGroupIDs []mstypes.GroupMembership `ndr:"pointer,conformant"` +} + +// Unmarshal bytes into the DeviceInfo struct +func (k *KerbValidationInfo) Unmarshal(b []byte) (err error) { + dec := ndr.NewDecoder(bytes.NewReader(b)) + err = dec.Decode(k) + if err != nil { + err = fmt.Errorf("error unmarshaling KerbValidationInfo: %v", err) + } + return +} + +// GetGroupMembershipSIDs returns a slice of strings containing the group membership SIDs found in the PAC. +func (k *KerbValidationInfo) GetGroupMembershipSIDs() []string { + var g []string + lSID := k.LogonDomainID.String() + for i := range k.GroupIDs { + g = append(g, fmt.Sprintf("%s-%d", lSID, k.GroupIDs[i].RelativeID)) + } + for _, s := range k.ExtraSIDs { + var exists = false + for _, es := range g { + if es == s.SID.String() { + exists = true + break + } + } + if !exists { + g = append(g, s.SID.String()) + } + } + for _, r := range k.ResourceGroupIDs { + var exists = false + s := fmt.Sprintf("%s-%d", k.ResourceGroupDomainSID.String(), r.RelativeID) + for _, es := range g { + if es == s { + exists = true + break + } + } + if !exists { + g = append(g, s) + } + } + return g +} diff --git a/github.com/jcmturner/gokrb5/v8/pac/pac_type.go b/github.com/jcmturner/gokrb5/v8/pac/pac_type.go new file mode 100644 index 0000000000..fab2ad7cda --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/pac/pac_type.go @@ -0,0 +1,251 @@ +package pac + +import ( + "bytes" + "errors" + "fmt" + "log" + + "github.com/jcmturner/gokrb5/v8/crypto" + "github.com/jcmturner/gokrb5/v8/iana/keyusage" + "github.com/jcmturner/gokrb5/v8/types" + "github.com/jcmturner/rpc/v2/mstypes" +) + +const ( + infoTypeKerbValidationInfo uint32 = 1 + infoTypeCredentials uint32 = 2 + infoTypePACServerSignatureData uint32 = 6 + infoTypePACKDCSignatureData uint32 = 7 + infoTypePACClientInfo uint32 = 10 + infoTypeS4UDelegationInfo uint32 = 11 + infoTypeUPNDNSInfo uint32 = 12 + infoTypePACClientClaimsInfo uint32 = 13 + infoTypePACDeviceInfo uint32 = 14 + infoTypePACDeviceClaimsInfo uint32 = 15 +) + +// PACType implements: https://msdn.microsoft.com/en-us/library/cc237950.aspx +type PACType struct { + CBuffers uint32 + Version uint32 + Buffers []InfoBuffer + Data []byte + KerbValidationInfo *KerbValidationInfo + CredentialsInfo *CredentialsInfo + ServerChecksum *SignatureData + KDCChecksum *SignatureData + ClientInfo *ClientInfo + S4UDelegationInfo *S4UDelegationInfo + UPNDNSInfo *UPNDNSInfo + ClientClaimsInfo *ClientClaimsInfo + DeviceInfo *DeviceInfo + DeviceClaimsInfo *DeviceClaimsInfo + ZeroSigData []byte +} + +// InfoBuffer implements the PAC Info Buffer: https://msdn.microsoft.com/en-us/library/cc237954.aspx +type InfoBuffer struct { + ULType uint32 // A 32-bit unsigned integer in little-endian format that describes the type of data present in the buffer contained at Offset. + CBBufferSize uint32 // A 32-bit unsigned integer in little-endian format that contains the size, in bytes, of the buffer in the PAC located at Offset. + Offset uint64 // A 64-bit unsigned integer in little-endian format that contains the offset to the beginning of the buffer, in bytes, from the beginning of the PACTYPE structure. The data offset MUST be a multiple of eight. The following sections specify the format of each type of element. +} + +// Unmarshal bytes into the PACType struct +func (pac *PACType) Unmarshal(b []byte) (err error) { + pac.Data = b + zb := make([]byte, len(b), len(b)) + copy(zb, b) + pac.ZeroSigData = zb + r := mstypes.NewReader(bytes.NewReader(b)) + pac.CBuffers, err = r.Uint32() + if err != nil { + return + } + pac.Version, err = r.Uint32() + if err != nil { + return + } + buf := make([]InfoBuffer, pac.CBuffers, pac.CBuffers) + for i := range buf { + buf[i].ULType, err = r.Uint32() + if err != nil { + return + } + buf[i].CBBufferSize, err = r.Uint32() + if err != nil { + return + } + buf[i].Offset, err = r.Uint64() + if err != nil { + return + } + } + pac.Buffers = buf + return nil +} + +// ProcessPACInfoBuffers processes the PAC Info Buffers. +// https://msdn.microsoft.com/en-us/library/cc237954.aspx +func (pac *PACType) ProcessPACInfoBuffers(key types.EncryptionKey, l *log.Logger) error { + for _, buf := range pac.Buffers { + p := make([]byte, buf.CBBufferSize, buf.CBBufferSize) + copy(p, pac.Data[int(buf.Offset):int(buf.Offset)+int(buf.CBBufferSize)]) + switch buf.ULType { + case infoTypeKerbValidationInfo: + if pac.KerbValidationInfo != nil { + //Must ignore subsequent buffers of this type + continue + } + var k KerbValidationInfo + err := k.Unmarshal(p) + if err != nil { + return fmt.Errorf("error processing KerbValidationInfo: %v", err) + } + pac.KerbValidationInfo = &k + case infoTypeCredentials: + // Currently PAC parsing is only useful on the service side in gokrb5 + // The CredentialsInfo are only useful when gokrb5 has implemented RFC4556 and only applied on the client side. + // Skipping CredentialsInfo - will be revisited under RFC4556 implementation. + continue + //if pac.CredentialsInfo != nil { + // //Must ignore subsequent buffers of this type + // continue + //} + //var k CredentialsInfo + //err := k.Unmarshal(p, key) // The encryption key used is the AS reply key only available to the client. + //if err != nil { + // return fmt.Errorf("error processing CredentialsInfo: %v", err) + //} + //pac.CredentialsInfo = &k + case infoTypePACServerSignatureData: + if pac.ServerChecksum != nil { + //Must ignore subsequent buffers of this type + continue + } + var k SignatureData + zb, err := k.Unmarshal(p) + copy(pac.ZeroSigData[int(buf.Offset):int(buf.Offset)+int(buf.CBBufferSize)], zb) + if err != nil { + return fmt.Errorf("error processing ServerChecksum: %v", err) + } + pac.ServerChecksum = &k + case infoTypePACKDCSignatureData: + if pac.KDCChecksum != nil { + //Must ignore subsequent buffers of this type + continue + } + var k SignatureData + zb, err := k.Unmarshal(p) + copy(pac.ZeroSigData[int(buf.Offset):int(buf.Offset)+int(buf.CBBufferSize)], zb) + if err != nil { + return fmt.Errorf("error processing KDCChecksum: %v", err) + } + pac.KDCChecksum = &k + case infoTypePACClientInfo: + if pac.ClientInfo != nil { + //Must ignore subsequent buffers of this type + continue + } + var k ClientInfo + err := k.Unmarshal(p) + if err != nil { + return fmt.Errorf("error processing ClientInfo: %v", err) + } + pac.ClientInfo = &k + case infoTypeS4UDelegationInfo: + if pac.S4UDelegationInfo != nil { + //Must ignore subsequent buffers of this type + continue + } + var k S4UDelegationInfo + err := k.Unmarshal(p) + if err != nil { + l.Printf("could not process S4U_DelegationInfo: %v", err) + continue + } + pac.S4UDelegationInfo = &k + case infoTypeUPNDNSInfo: + if pac.UPNDNSInfo != nil { + //Must ignore subsequent buffers of this type + continue + } + var k UPNDNSInfo + err := k.Unmarshal(p) + if err != nil { + l.Printf("could not process UPN_DNSInfo: %v", err) + continue + } + pac.UPNDNSInfo = &k + case infoTypePACClientClaimsInfo: + if pac.ClientClaimsInfo != nil || len(p) < 1 { + //Must ignore subsequent buffers of this type + continue + } + var k ClientClaimsInfo + err := k.Unmarshal(p) + if err != nil { + l.Printf("could not process ClientClaimsInfo: %v", err) + continue + } + pac.ClientClaimsInfo = &k + case infoTypePACDeviceInfo: + if pac.DeviceInfo != nil { + //Must ignore subsequent buffers of this type + continue + } + var k DeviceInfo + err := k.Unmarshal(p) + if err != nil { + l.Printf("could not process DeviceInfo: %v", err) + continue + } + pac.DeviceInfo = &k + case infoTypePACDeviceClaimsInfo: + if pac.DeviceClaimsInfo != nil { + //Must ignore subsequent buffers of this type + continue + } + var k DeviceClaimsInfo + err := k.Unmarshal(p) + if err != nil { + l.Printf("could not process DeviceClaimsInfo: %v", err) + continue + } + pac.DeviceClaimsInfo = &k + } + } + + if ok, err := pac.verify(key); !ok { + return err + } + + return nil +} + +func (pac *PACType) verify(key types.EncryptionKey) (bool, error) { + if pac.KerbValidationInfo == nil { + return false, errors.New("PAC Info Buffers does not contain a KerbValidationInfo") + } + if pac.ServerChecksum == nil { + return false, errors.New("PAC Info Buffers does not contain a ServerChecksum") + } + if pac.KDCChecksum == nil { + return false, errors.New("PAC Info Buffers does not contain a KDCChecksum") + } + if pac.ClientInfo == nil { + return false, errors.New("PAC Info Buffers does not contain a ClientInfo") + } + etype, err := crypto.GetChksumEtype(int32(pac.ServerChecksum.SignatureType)) + if err != nil { + return false, err + } + if ok := etype.VerifyChecksum(key.KeyValue, + pac.ZeroSigData, + pac.ServerChecksum.Signature, + keyusage.KERB_NON_KERB_CKSUM_SALT); !ok { + return false, errors.New("PAC service checksum verification failed") + } + + return true, nil +} diff --git a/github.com/jcmturner/gokrb5/v8/pac/s4u_delegation_info.go b/github.com/jcmturner/gokrb5/v8/pac/s4u_delegation_info.go new file mode 100644 index 0000000000..da837d4b3a --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/pac/s4u_delegation_info.go @@ -0,0 +1,26 @@ +package pac + +import ( + "bytes" + "fmt" + + "github.com/jcmturner/rpc/v2/mstypes" + "github.com/jcmturner/rpc/v2/ndr" +) + +// S4UDelegationInfo implements https://msdn.microsoft.com/en-us/library/cc237944.aspx +type S4UDelegationInfo struct { + S4U2proxyTarget mstypes.RPCUnicodeString // The name of the principal to whom the application can forward the ticket. + TransitedListSize uint32 + S4UTransitedServices []mstypes.RPCUnicodeString `ndr:"pointer,conformant"` // List of all services that have been delegated through by this client and subsequent services or servers.. Size is value of TransitedListSize +} + +// Unmarshal bytes into the S4UDelegationInfo struct +func (k *S4UDelegationInfo) Unmarshal(b []byte) (err error) { + dec := ndr.NewDecoder(bytes.NewReader(b)) + err = dec.Decode(k) + if err != nil { + err = fmt.Errorf("error unmarshaling S4UDelegationInfo: %v", err) + } + return +} diff --git a/github.com/jcmturner/gokrb5/v8/pac/signature_data.go b/github.com/jcmturner/gokrb5/v8/pac/signature_data.go new file mode 100644 index 0000000000..8f6aa58faa --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/pac/signature_data.go @@ -0,0 +1,67 @@ +package pac + +import ( + "bytes" + + "github.com/jcmturner/gokrb5/v8/iana/chksumtype" + "github.com/jcmturner/rpc/v2/mstypes" +) + +/* +https://msdn.microsoft.com/en-us/library/cc237955.aspx + +The Key Usage Value MUST be KERB_NON_KERB_CKSUM_SALT (17) [MS-KILE] (section 3.1.5.9). + +Server Signature (SignatureType = 0x00000006) +https://msdn.microsoft.com/en-us/library/cc237957.aspx + +KDC Signature (SignatureType = 0x00000007) +https://msdn.microsoft.com/en-us/library/dd357117.aspx +*/ + +// SignatureData implements https://msdn.microsoft.com/en-us/library/cc237955.aspx +type SignatureData struct { + SignatureType uint32 // A 32-bit unsigned integer value in little-endian format that defines the cryptographic system used to calculate the checksum. This MUST be one of the following checksum types: KERB_CHECKSUM_HMAC_MD5 (signature size = 16), HMAC_SHA1_96_AES128 (signature size = 12), HMAC_SHA1_96_AES256 (signature size = 12). + Signature []byte // Size depends on the type. See comment above. + RODCIdentifier uint16 // A 16-bit unsigned integer value in little-endian format that contains the first 16 bits of the key version number ([MS-KILE] section 3.1.5.8) when the KDC is an RODC. When the KDC is not an RODC, this field does not exist. +} + +// Unmarshal bytes into the SignatureData struct +func (k *SignatureData) Unmarshal(b []byte) (rb []byte, err error) { + r := mstypes.NewReader(bytes.NewReader(b)) + + k.SignatureType, err = r.Uint32() + if err != nil { + return + } + + var c int + switch k.SignatureType { + case chksumtype.KERB_CHECKSUM_HMAC_MD5_UNSIGNED: + c = 16 + case uint32(chksumtype.HMAC_SHA1_96_AES128): + c = 12 + case uint32(chksumtype.HMAC_SHA1_96_AES256): + c = 12 + } + k.Signature, err = r.ReadBytes(c) + if err != nil { + return + } + + // When the KDC is not an Read Only Domain Controller (RODC), this field does not exist. + if len(b) >= 4+c+2 { + k.RODCIdentifier, err = r.Uint16() + if err != nil { + return + } + } + + // Create bytes with zeroed signature needed for checksum verification + rb = make([]byte, len(b), len(b)) + copy(rb, b) + z := make([]byte, len(b), len(b)) + copy(rb[4:4+c], z) + + return +} diff --git a/github.com/jcmturner/gokrb5/v8/pac/supplemental_cred.go b/github.com/jcmturner/gokrb5/v8/pac/supplemental_cred.go new file mode 100644 index 0000000000..d40679d49b --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/pac/supplemental_cred.go @@ -0,0 +1,87 @@ +package pac + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + + "github.com/jcmturner/rpc/v2/mstypes" + "github.com/jcmturner/rpc/v2/ndr" +) + +const ( + // NTLMSupCredLMOWF indicates that the LM OWF member is present and valid. + NTLMSupCredLMOWF uint32 = 31 + // NTLMSupCredNTOWF indicates that the NT OWF member is present and valid. + NTLMSupCredNTOWF uint32 = 30 +) + +// NTLMSupplementalCred implements https://msdn.microsoft.com/en-us/library/cc237949.aspx +type NTLMSupplementalCred struct { + Version uint32 // A 32-bit unsigned integer that defines the credential version.This field MUST be 0x00000000. + Flags uint32 + LMPassword []byte // A 16-element array of unsigned 8-bit integers that define the LM OWF. The LMPassword member MUST be ignored if the L flag is not set in the Flags member. + NTPassword []byte // A 16-element array of unsigned 8-bit integers that define the NT OWF. The NTPassword member MUST be ignored if the N flag is not set in the Flags member. +} + +// Unmarshal converts the bytes provided into a NTLMSupplementalCred. +func (c *NTLMSupplementalCred) Unmarshal(b []byte) (err error) { + r := mstypes.NewReader(bytes.NewReader(b)) + c.Version, err = r.Uint32() + if err != nil { + return + } + if c.Version != 0 { + err = errors.New("NTLMSupplementalCred version is not zero") + return + } + c.Flags, err = r.Uint32() + if err != nil { + return + } + if isFlagSet(c.Flags, NTLMSupCredLMOWF) { + c.LMPassword, err = r.ReadBytes(16) + if err != nil { + return + } + } + if isFlagSet(c.Flags, NTLMSupCredNTOWF) { + c.NTPassword, err = r.ReadBytes(16) + if err != nil { + return + } + } + return +} + +// isFlagSet tests if a flag is set in the uint32 little endian flag +func isFlagSet(f uint32, i uint32) bool { + //Which byte? + b := int(i / 8) + //Which bit in byte + p := uint(7 - (int(i) - 8*b)) + fb := make([]byte, 4) + binary.LittleEndian.PutUint32(fb, f) + if fb[b]&(1< - no domain specified + // \ + // @ + if strings.Contains(vc[0], `\`) { + u := strings.SplitN(vc[0], `\`, 2) + domain = u[0] + username = u[1] + } else if strings.Contains(vc[0], `@`) { + u := strings.SplitN(vc[0], `@`, 2) + domain = u[1] + username = u[0] + } else { + username = vc[0] + } + return +} diff --git a/github.com/jcmturner/gokrb5/v8/service/cache.go b/github.com/jcmturner/gokrb5/v8/service/cache.go new file mode 100644 index 0000000000..038e594ec0 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/service/cache.go @@ -0,0 +1,128 @@ +// Package service provides server side integrations for Kerberos authentication. +package service + +import ( + "github.com/jcmturner/gokrb5/v8/types" + "sync" + "time" +) + +// Replay cache is required as specified in RFC 4120 section 3.2.3 + +// Cache for tickets received from clients keyed by fully qualified client name. Used to track replay of tickets. +type Cache struct { + entries map[string]clientEntries + mux sync.RWMutex +} + +// clientEntries holds entries of client details sent to the service. +type clientEntries struct { + replayMap map[time.Time]replayCacheEntry + seqNumber int64 + subKey types.EncryptionKey +} + +// Cache entry tracking client time values of tickets sent to the service. +type replayCacheEntry struct { + presentedTime time.Time + sName types.PrincipalName + cTime time.Time // This combines the ticket's CTime and Cusec +} + +func (c *Cache) getClientEntries(cname types.PrincipalName) (clientEntries, bool) { + c.mux.RLock() + defer c.mux.RUnlock() + ce, ok := c.entries[cname.PrincipalNameString()] + return ce, ok +} + +func (c *Cache) getClientEntry(cname types.PrincipalName, t time.Time) (replayCacheEntry, bool) { + if ce, ok := c.getClientEntries(cname); ok { + c.mux.RLock() + defer c.mux.RUnlock() + if e, ok := ce.replayMap[t]; ok { + return e, true + } + } + return replayCacheEntry{}, false +} + +// Instance of the ServiceCache. This needs to be a singleton. +var replayCache Cache +var once sync.Once + +// GetReplayCache returns a pointer to the Cache singleton. +func GetReplayCache(d time.Duration) *Cache { + // Create a singleton of the ReplayCache and start a background thread to regularly clean out old entries + once.Do(func() { + replayCache = Cache{ + entries: make(map[string]clientEntries), + } + go func() { + for { + // TODO consider using a context here. + time.Sleep(d) + replayCache.ClearOldEntries(d) + } + }() + }) + return &replayCache +} + +// AddEntry adds an entry to the Cache. +func (c *Cache) AddEntry(sname types.PrincipalName, a types.Authenticator) { + ct := a.CTime.Add(time.Duration(a.Cusec) * time.Microsecond) + if ce, ok := c.getClientEntries(a.CName); ok { + c.mux.Lock() + defer c.mux.Unlock() + ce.replayMap[ct] = replayCacheEntry{ + presentedTime: time.Now().UTC(), + sName: sname, + cTime: ct, + } + ce.seqNumber = a.SeqNumber + ce.subKey = a.SubKey + } else { + c.mux.Lock() + defer c.mux.Unlock() + c.entries[a.CName.PrincipalNameString()] = clientEntries{ + replayMap: map[time.Time]replayCacheEntry{ + ct: { + presentedTime: time.Now().UTC(), + sName: sname, + cTime: ct, + }, + }, + seqNumber: a.SeqNumber, + subKey: a.SubKey, + } + } +} + +// ClearOldEntries clears entries from the Cache that are older than the duration provided. +func (c *Cache) ClearOldEntries(d time.Duration) { + c.mux.Lock() + defer c.mux.Unlock() + for ke, ce := range c.entries { + for k, e := range ce.replayMap { + if time.Now().UTC().Sub(e.presentedTime) > d { + delete(ce.replayMap, k) + } + } + if len(ce.replayMap) == 0 { + delete(c.entries, ke) + } + } +} + +// IsReplay tests if the Authenticator provided is a replay within the duration defined. If this is not a replay add the entry to the cache for tracking. +func (c *Cache) IsReplay(sname types.PrincipalName, a types.Authenticator) bool { + ct := a.CTime.Add(time.Duration(a.Cusec) * time.Microsecond) + if e, ok := c.getClientEntry(a.CName, ct); ok { + if e.sName.Equal(sname) { + return true + } + } + c.AddEntry(sname, a) + return false +} diff --git a/github.com/jcmturner/gokrb5/v8/service/settings.go b/github.com/jcmturner/gokrb5/v8/service/settings.go new file mode 100644 index 0000000000..a0370ed7af --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/service/settings.go @@ -0,0 +1,163 @@ +package service + +import ( + "log" + "net/http" + "time" + + "github.com/jcmturner/gokrb5/v8/keytab" + "github.com/jcmturner/gokrb5/v8/types" +) + +// Settings defines service side configuration settings. +type Settings struct { + Keytab *keytab.Keytab + ktprinc *types.PrincipalName + sname string + requireHostAddr bool + disablePACDecoding bool + cAddr types.HostAddress + maxClockSkew time.Duration + logger *log.Logger + sessionMgr SessionMgr +} + +// NewSettings creates a new service Settings. +func NewSettings(kt *keytab.Keytab, settings ...func(*Settings)) *Settings { + s := new(Settings) + s.Keytab = kt + for _, set := range settings { + set(s) + } + return s +} + +// RequireHostAddr used to configure service side to required host addresses to be specified in Kerberos tickets. +// +// s := NewSettings(kt, RequireHostAddr(true)) +func RequireHostAddr(b bool) func(*Settings) { + return func(s *Settings) { + s.requireHostAddr = b + } +} + +// RequireHostAddr indicates if the service should require the host address to be included in the ticket. +func (s *Settings) RequireHostAddr() bool { + return s.requireHostAddr +} + +// DecodePAC used to configure service side to enable/disable PAC decoding if the PAC is present. +// Defaults to enabled if not specified. +// +// s := NewSettings(kt, DecodePAC(false)) +func DecodePAC(b bool) func(*Settings) { + return func(s *Settings) { + s.disablePACDecoding = !b + } +} + +// DecodePAC indicates whether the service should decode any PAC information present in the ticket. +func (s *Settings) DecodePAC() bool { + return !s.disablePACDecoding +} + +// ClientAddress used to configure service side with the clients host address to be used during validation. +// +// s := NewSettings(kt, ClientAddress(h)) +func ClientAddress(h types.HostAddress) func(*Settings) { + return func(s *Settings) { + s.cAddr = h + } +} + +// ClientAddress returns the client host address which has been provided to the service. +func (s *Settings) ClientAddress() types.HostAddress { + return s.cAddr +} + +// Logger used to configure service side with a logger. +// +// s := NewSettings(kt, Logger(l)) +func Logger(l *log.Logger) func(*Settings) { + return func(s *Settings) { + s.logger = l + } +} + +// Logger returns the logger instances configured for the service. If none is configured nill will be returned. +func (s *Settings) Logger() *log.Logger { + return s.logger +} + +// KeytabPrincipal used to override the principal name used to find the key in the keytab. +// +// s := NewSettings(kt, KeytabPrincipal("someaccount")) +func KeytabPrincipal(p string) func(*Settings) { + return func(s *Settings) { + pn, _ := types.ParseSPNString(p) + s.ktprinc = &pn + } +} + +// KeytabPrincipal returns the principal name used to find the key in the keytab if it has been overridden. +func (s *Settings) KeytabPrincipal() *types.PrincipalName { + return s.ktprinc +} + +// MaxClockSkew used to configure service side with the maximum acceptable clock skew +// between the service and the issue time of kerberos tickets +// +// s := NewSettings(kt, MaxClockSkew(d)) +func MaxClockSkew(d time.Duration) func(*Settings) { + return func(s *Settings) { + s.maxClockSkew = d + } +} + +// MaxClockSkew returns the maximum acceptable clock skew between the service and the issue time of kerberos tickets. +// If none is defined a duration of 5 minutes is returned. +func (s *Settings) MaxClockSkew() time.Duration { + if s.maxClockSkew.Nanoseconds() == 0 { + return time.Duration(5) * time.Minute + } + return s.maxClockSkew +} + +// SName used provide a specific service name to the service settings. +// +// s := NewSettings(kt, SName("HTTP/some.service.com")) +func SName(sname string) func(*Settings) { + return func(s *Settings) { + s.sname = sname + } +} + +// SName returns the specific service name to the service. +func (s *Settings) SName() string { + return s.sname +} + +// SessionManager configures a session manager to establish sessions with clients to avoid excessive authentication challenges. +// +// s := NewSettings(kt, SessionManager(sm)) +func SessionManager(sm SessionMgr) func(*Settings) { + return func(s *Settings) { + s.sessionMgr = sm + } +} + +// SessionManager returns any configured session manager. +func (s *Settings) SessionManager() SessionMgr { + return s.sessionMgr +} + +// SessionMgr must provide a ways to: +// +// - Create new sessions and in the process add a value to the session under the key provided. +// +// - Get an existing session returning the value in the session under the key provided. +// Return nil bytes and/or error if there is no session. +type SessionMgr interface { + New(w http.ResponseWriter, r *http.Request, k string, v []byte) error + Get(r *http.Request, k string) ([]byte, error) +} diff --git a/github.com/jcmturner/gokrb5/v8/spnego/http.go b/github.com/jcmturner/gokrb5/v8/spnego/http.go new file mode 100644 index 0000000000..4e1d2bda8a --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/spnego/http.go @@ -0,0 +1,373 @@ +package spnego + +import ( + "bytes" + "encoding/base64" + "errors" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "net/http/cookiejar" + "net/url" + "strings" + + "github.com/jcmturner/gofork/encoding/asn1" + "github.com/jcmturner/goidentity/v6" + "github.com/jcmturner/gokrb5/v8/client" + "github.com/jcmturner/gokrb5/v8/credentials" + "github.com/jcmturner/gokrb5/v8/gssapi" + "github.com/jcmturner/gokrb5/v8/keytab" + "github.com/jcmturner/gokrb5/v8/krberror" + "github.com/jcmturner/gokrb5/v8/service" + "github.com/jcmturner/gokrb5/v8/types" +) + +// Client side functionality // + +// Client will negotiate authentication with a server using SPNEGO. +type Client struct { + *http.Client + krb5Client *client.Client + spn string + reqs []*http.Request +} + +type redirectErr struct { + reqTarget *http.Request +} + +func (e redirectErr) Error() string { + return fmt.Sprintf("redirect to %v", e.reqTarget.URL) +} + +type teeReadCloser struct { + io.Reader + io.Closer +} + +// NewClient returns a SPNEGO enabled HTTP client. +// Be careful when passing in the *http.Client if it is beginning reused in multiple calls to this function. +// Ensure reuse of the provided *http.Client is for the same user as a session cookie may have been added to +// http.Client's cookie jar. +// Incorrect reuse of the provided *http.Client could lead to access to the wrong user's session. +func NewClient(krb5Cl *client.Client, httpCl *http.Client, spn string) *Client { + if httpCl == nil { + httpCl = &http.Client{} + } + // Add a cookie jar if there isn't one + if httpCl.Jar == nil { + httpCl.Jar, _ = cookiejar.New(nil) + } + // Add a CheckRedirect function that will execute any functional already defined and then error with a redirectErr + f := httpCl.CheckRedirect + httpCl.CheckRedirect = func(req *http.Request, via []*http.Request) error { + if f != nil { + err := f(req, via) + if err != nil { + return err + } + } + return redirectErr{reqTarget: req} + } + return &Client{ + Client: httpCl, + krb5Client: krb5Cl, + spn: spn, + } +} + +// Do is the SPNEGO enabled HTTP client's equivalent of the http.Client's Do method. +func (c *Client) Do(req *http.Request) (resp *http.Response, err error) { + var body bytes.Buffer + if req.Body != nil { + // Use a tee reader to capture any body sent in case we have to replay it again + teeR := io.TeeReader(req.Body, &body) + teeRC := teeReadCloser{teeR, req.Body} + req.Body = teeRC + } + resp, err = c.Client.Do(req) + if err != nil { + if ue, ok := err.(*url.Error); ok { + if e, ok := ue.Err.(redirectErr); ok { + // Picked up a redirect + e.reqTarget.Header.Del(HTTPHeaderAuthRequest) + c.reqs = append(c.reqs, e.reqTarget) + if len(c.reqs) >= 10 { + return resp, errors.New("stopped after 10 redirects") + } + if req.Body != nil { + // Refresh the body reader so the body can be sent again + e.reqTarget.Body = ioutil.NopCloser(&body) + } + return c.Do(e.reqTarget) + } + } + return resp, err + } + if respUnauthorizedNegotiate(resp) { + err := SetSPNEGOHeader(c.krb5Client, req, c.spn) + if err != nil { + return resp, err + } + if req.Body != nil { + // Refresh the body reader so the body can be sent again + req.Body = ioutil.NopCloser(&body) + } + return c.Do(req) + } + return resp, err +} + +// Get is the SPNEGO enabled HTTP client's equivalent of the http.Client's Get method. +func (c *Client) Get(url string) (resp *http.Response, err error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + return c.Do(req) +} + +// Post is the SPNEGO enabled HTTP client's equivalent of the http.Client's Post method. +func (c *Client) Post(url, contentType string, body io.Reader) (resp *http.Response, err error) { + req, err := http.NewRequest("POST", url, body) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", contentType) + return c.Do(req) +} + +// PostForm is the SPNEGO enabled HTTP client's equivalent of the http.Client's PostForm method. +func (c *Client) PostForm(url string, data url.Values) (resp *http.Response, err error) { + return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode())) +} + +// Head is the SPNEGO enabled HTTP client's equivalent of the http.Client's Head method. +func (c *Client) Head(url string) (resp *http.Response, err error) { + req, err := http.NewRequest("HEAD", url, nil) + if err != nil { + return nil, err + } + return c.Do(req) +} + +func respUnauthorizedNegotiate(resp *http.Response) bool { + if resp.StatusCode == http.StatusUnauthorized { + if resp.Header.Get(HTTPHeaderAuthResponse) == HTTPHeaderAuthResponseValueKey { + return true + } + } + return false +} + +// SetSPNEGOHeader gets the service ticket and sets it as the SPNEGO authorization header on HTTP request object. +// To auto generate the SPN from the request object pass a null string "". +func SetSPNEGOHeader(cl *client.Client, r *http.Request, spn string) error { + if spn == "" { + h := strings.TrimSuffix(strings.SplitN(r.URL.Host, ":", 2)[0], ".") + name, err := net.LookupCNAME(h) + if err == nil { + // Underlyng canonical name should be used for SPN + h = strings.TrimSuffix(name, ".") + } + spn = "HTTP/" + h + r.Host = h + } + cl.Log("using SPN %s", spn) + s := SPNEGOClient(cl, spn) + err := s.AcquireCred() + if err != nil { + return fmt.Errorf("could not acquire client credential: %v", err) + } + st, err := s.InitSecContext() + if err != nil { + return fmt.Errorf("could not initialize context: %v", err) + } + nb, err := st.Marshal() + if err != nil { + return krberror.Errorf(err, krberror.EncodingError, "could not marshal SPNEGO") + } + hs := "Negotiate " + base64.StdEncoding.EncodeToString(nb) + r.Header.Set(HTTPHeaderAuthRequest, hs) + return nil +} + +// Service side functionality // + +const ( + // spnegoNegTokenRespKRBAcceptCompleted - The response on successful authentication always has this header. Capturing as const so we don't have marshaling and encoding overhead. + spnegoNegTokenRespKRBAcceptCompleted = "Negotiate oRQwEqADCgEAoQsGCSqGSIb3EgECAg==" + // spnegoNegTokenRespReject - The response on a failed authentication always has this rejection header. Capturing as const so we don't have marshaling and encoding overhead. + spnegoNegTokenRespReject = "Negotiate oQcwBaADCgEC" + // spnegoNegTokenRespIncompleteKRB5 - Response token specifying incomplete context and KRB5 as the supported mechtype. + spnegoNegTokenRespIncompleteKRB5 = "Negotiate oRQwEqADCgEBoQsGCSqGSIb3EgECAg==" + // sessionCredentials is the session value key holding the credentials jcmturner/goidentity/Identity object. + sessionCredentials = "github.com/jcmturner/gokrb5/v8/sessionCredentials" + // ctxCredentials is the SPNEGO context key holding the credentials jcmturner/goidentity/Identity object. + ctxCredentials = "github.com/jcmturner/gokrb5/v8/ctxCredentials" + // HTTPHeaderAuthRequest is the header that will hold authn/z information. + HTTPHeaderAuthRequest = "Authorization" + // HTTPHeaderAuthResponse is the header that will hold SPNEGO data from the server. + HTTPHeaderAuthResponse = "WWW-Authenticate" + // HTTPHeaderAuthResponseValueKey is the key in the auth header for SPNEGO. + HTTPHeaderAuthResponseValueKey = "Negotiate" + // UnauthorizedMsg is the message returned in the body when authentication fails. + UnauthorizedMsg = "Unauthorised.\n" +) + +// SPNEGOKRB5Authenticate is a Kerberos SPNEGO authentication HTTP handler wrapper. +func SPNEGOKRB5Authenticate(inner http.Handler, kt *keytab.Keytab, settings ...func(*service.Settings)) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Set up the SPNEGO GSS-API mechanism + var spnego *SPNEGO + h, err := types.GetHostAddress(r.RemoteAddr) + if err == nil { + // put in this order so that if the user provides a ClientAddress it will override the one here. + o := append([]func(*service.Settings){service.ClientAddress(h)}, settings...) + spnego = SPNEGOService(kt, o...) + } else { + spnego = SPNEGOService(kt, settings...) + spnego.Log("%s - SPNEGO could not parse client address: %v", r.RemoteAddr, err) + } + + // Check if there is a session manager and if there is an already established session for this client + id, err := getSessionCredentials(spnego, r) + if err == nil && id.Authenticated() { + // There is an established session so bypass auth and serve + spnego.Log("%s - SPNEGO request served under session %s", r.RemoteAddr, id.SessionID()) + inner.ServeHTTP(w, goidentity.AddToHTTPRequestContext(&id, r)) + return + } + + st, err := getAuthorizationNegotiationHeaderAsSPNEGOToken(spnego, r, w) + if st == nil || err != nil { + // response to client and logging handled in function above so just return + return + } + + // Validate the context token + authed, ctx, status := spnego.AcceptSecContext(st) + if status.Code != gssapi.StatusComplete && status.Code != gssapi.StatusContinueNeeded { + spnegoResponseReject(spnego, w, "%s - SPNEGO validation error: %v", r.RemoteAddr, status) + return + } + if status.Code == gssapi.StatusContinueNeeded { + spnegoNegotiateKRB5MechType(spnego, w, "%s - SPNEGO GSS-API continue needed", r.RemoteAddr) + return + } + + if authed { + // Authentication successful; get user's credentials from the context + id := ctx.Value(ctxCredentials).(*credentials.Credentials) + // Create a new session if a session manager has been configured + err = newSession(spnego, r, w, id) + if err != nil { + return + } + spnegoResponseAcceptCompleted(spnego, w, "%s %s@%s - SPNEGO authentication succeeded", r.RemoteAddr, id.UserName(), id.Domain()) + // Add the identity to the context and serve the inner/wrapped handler + inner.ServeHTTP(w, goidentity.AddToHTTPRequestContext(id, r)) + return + } + // If we get to here we have not authenticationed so just reject + spnegoResponseReject(spnego, w, "%s - SPNEGO Kerberos authentication failed", r.RemoteAddr) + return + }) +} + +func getAuthorizationNegotiationHeaderAsSPNEGOToken(spnego *SPNEGO, r *http.Request, w http.ResponseWriter) (*SPNEGOToken, error) { + s := strings.SplitN(r.Header.Get(HTTPHeaderAuthRequest), " ", 2) + if len(s) != 2 || s[0] != HTTPHeaderAuthResponseValueKey { + // No Authorization header set so return 401 with WWW-Authenticate Negotiate header + w.Header().Set(HTTPHeaderAuthResponse, HTTPHeaderAuthResponseValueKey) + http.Error(w, UnauthorizedMsg, http.StatusUnauthorized) + return nil, errors.New("client did not provide a negotiation authorization header") + } + + // Decode the header into an SPNEGO context token + b, err := base64.StdEncoding.DecodeString(s[1]) + if err != nil { + err = fmt.Errorf("error in base64 decoding negotiation header: %v", err) + spnegoNegotiateKRB5MechType(spnego, w, "%s - SPNEGO %v", r.RemoteAddr, err) + return nil, err + } + var st SPNEGOToken + err = st.Unmarshal(b) + if err != nil { + // Check if this is a raw KRB5 context token - issue #347. + var k5t KRB5Token + if k5t.Unmarshal(b) != nil { + err = fmt.Errorf("error in unmarshaling SPNEGO token: %v", err) + spnegoNegotiateKRB5MechType(spnego, w, "%s - SPNEGO %v", r.RemoteAddr, err) + return nil, err + } + // Wrap it into an SPNEGO context token + st.Init = true + st.NegTokenInit = NegTokenInit{ + MechTypes: []asn1.ObjectIdentifier{k5t.OID}, + MechTokenBytes: b, + } + } + return &st, nil +} + +func getSessionCredentials(spnego *SPNEGO, r *http.Request) (credentials.Credentials, error) { + var creds credentials.Credentials + // Check if there is a session manager and if there is an already established session for this client + if sm := spnego.serviceSettings.SessionManager(); sm != nil { + cb, err := sm.Get(r, sessionCredentials) + if err != nil || cb == nil || len(cb) < 1 { + return creds, fmt.Errorf("%s - SPNEGO error getting session and credentials for request: %v", r.RemoteAddr, err) + } + err = creds.Unmarshal(cb) + if err != nil { + return creds, fmt.Errorf("%s - SPNEGO credentials malformed in session: %v", r.RemoteAddr, err) + } + return creds, nil + } + return creds, errors.New("no session manager configured") +} + +func newSession(spnego *SPNEGO, r *http.Request, w http.ResponseWriter, id *credentials.Credentials) error { + if sm := spnego.serviceSettings.SessionManager(); sm != nil { + // create new session + idb, err := id.Marshal() + if err != nil { + spnegoInternalServerError(spnego, w, "SPNEGO could not marshal credentials to add to the session: %v", err) + return err + } + err = sm.New(w, r, sessionCredentials, idb) + if err != nil { + spnegoInternalServerError(spnego, w, "SPNEGO could not create new session: %v", err) + return err + } + spnego.Log("%s %s@%s - SPNEGO new session (%s) created", r.RemoteAddr, id.UserName(), id.Domain(), id.SessionID()) + } + return nil +} + +// Log and respond to client for error conditions + +func spnegoNegotiateKRB5MechType(s *SPNEGO, w http.ResponseWriter, format string, v ...interface{}) { + s.Log(format, v...) + w.Header().Set(HTTPHeaderAuthResponse, spnegoNegTokenRespIncompleteKRB5) + http.Error(w, UnauthorizedMsg, http.StatusUnauthorized) +} + +func spnegoResponseReject(s *SPNEGO, w http.ResponseWriter, format string, v ...interface{}) { + s.Log(format, v...) + w.Header().Set(HTTPHeaderAuthResponse, spnegoNegTokenRespReject) + http.Error(w, UnauthorizedMsg, http.StatusUnauthorized) +} + +func spnegoResponseAcceptCompleted(s *SPNEGO, w http.ResponseWriter, format string, v ...interface{}) { + s.Log(format, v...) + w.Header().Set(HTTPHeaderAuthResponse, spnegoNegTokenRespKRBAcceptCompleted) +} + +func spnegoInternalServerError(s *SPNEGO, w http.ResponseWriter, format string, v ...interface{}) { + s.Log(format, v...) + http.Error(w, "Internal Server Error", http.StatusInternalServerError) +} diff --git a/github.com/jcmturner/gokrb5/v8/spnego/krb5Token.go b/github.com/jcmturner/gokrb5/v8/spnego/krb5Token.go new file mode 100644 index 0000000000..43f8a82c8a --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/spnego/krb5Token.go @@ -0,0 +1,218 @@ +package spnego + +import ( + "context" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + + "github.com/jcmturner/gofork/encoding/asn1" + "github.com/jcmturner/gokrb5/v8/asn1tools" + "github.com/jcmturner/gokrb5/v8/client" + "github.com/jcmturner/gokrb5/v8/credentials" + "github.com/jcmturner/gokrb5/v8/gssapi" + "github.com/jcmturner/gokrb5/v8/iana/chksumtype" + "github.com/jcmturner/gokrb5/v8/iana/msgtype" + "github.com/jcmturner/gokrb5/v8/krberror" + "github.com/jcmturner/gokrb5/v8/messages" + "github.com/jcmturner/gokrb5/v8/service" + "github.com/jcmturner/gokrb5/v8/types" +) + +// GSSAPI KRB5 MechToken IDs. +const ( + TOK_ID_KRB_AP_REQ = "0100" + TOK_ID_KRB_AP_REP = "0200" + TOK_ID_KRB_ERROR = "0300" +) + +// KRB5Token context token implementation for GSSAPI. +type KRB5Token struct { + OID asn1.ObjectIdentifier + tokID []byte + APReq messages.APReq + APRep messages.APRep + KRBError messages.KRBError + settings *service.Settings + context context.Context +} + +// Marshal a KRB5Token into a slice of bytes. +func (m *KRB5Token) Marshal() ([]byte, error) { + // Create the header + b, _ := asn1.Marshal(m.OID) + b = append(b, m.tokID...) + var tb []byte + var err error + switch hex.EncodeToString(m.tokID) { + case TOK_ID_KRB_AP_REQ: + tb, err = m.APReq.Marshal() + if err != nil { + return []byte{}, fmt.Errorf("error marshalling AP_REQ for MechToken: %v", err) + } + case TOK_ID_KRB_AP_REP: + return []byte{}, errors.New("marshal of AP_REP GSSAPI MechToken not supported by gokrb5") + case TOK_ID_KRB_ERROR: + return []byte{}, errors.New("marshal of KRB_ERROR GSSAPI MechToken not supported by gokrb5") + } + if err != nil { + return []byte{}, fmt.Errorf("error mashalling kerberos message within mech token: %v", err) + } + b = append(b, tb...) + return asn1tools.AddASNAppTag(b, 0), nil +} + +// Unmarshal a KRB5Token. +func (m *KRB5Token) Unmarshal(b []byte) error { + var oid asn1.ObjectIdentifier + r, err := asn1.UnmarshalWithParams(b, &oid, fmt.Sprintf("application,explicit,tag:%v", 0)) + if err != nil { + return fmt.Errorf("error unmarshalling KRB5Token OID: %v", err) + } + if !oid.Equal(gssapi.OIDKRB5.OID()) { + return fmt.Errorf("error unmarshalling KRB5Token, OID is %s not %s", oid.String(), gssapi.OIDKRB5.OID().String()) + } + m.OID = oid + if len(r) < 2 { + return fmt.Errorf("krb5token too short") + } + m.tokID = r[0:2] + switch hex.EncodeToString(m.tokID) { + case TOK_ID_KRB_AP_REQ: + var a messages.APReq + err = a.Unmarshal(r[2:]) + if err != nil { + return fmt.Errorf("error unmarshalling KRB5Token AP_REQ: %v", err) + } + m.APReq = a + case TOK_ID_KRB_AP_REP: + var a messages.APRep + err = a.Unmarshal(r[2:]) + if err != nil { + return fmt.Errorf("error unmarshalling KRB5Token AP_REP: %v", err) + } + m.APRep = a + case TOK_ID_KRB_ERROR: + var a messages.KRBError + err = a.Unmarshal(r[2:]) + if err != nil { + return fmt.Errorf("error unmarshalling KRB5Token KRBError: %v", err) + } + m.KRBError = a + } + return nil +} + +// Verify a KRB5Token. +func (m *KRB5Token) Verify() (bool, gssapi.Status) { + switch hex.EncodeToString(m.tokID) { + case TOK_ID_KRB_AP_REQ: + ok, creds, err := service.VerifyAPREQ(&m.APReq, m.settings) + if err != nil { + return false, gssapi.Status{Code: gssapi.StatusDefectiveToken, Message: err.Error()} + } + if !ok { + return false, gssapi.Status{Code: gssapi.StatusDefectiveCredential, Message: "KRB5_AP_REQ token not valid"} + } + m.context = context.Background() + m.context = context.WithValue(m.context, ctxCredentials, creds) + return true, gssapi.Status{Code: gssapi.StatusComplete} + case TOK_ID_KRB_AP_REP: + // Client side + // TODO how to verify the AP_REP - not yet implemented + return false, gssapi.Status{Code: gssapi.StatusFailure, Message: "verifying an AP_REP is not currently supported by gokrb5"} + case TOK_ID_KRB_ERROR: + if m.KRBError.MsgType != msgtype.KRB_ERROR { + return false, gssapi.Status{Code: gssapi.StatusDefectiveToken, Message: "KRB5_Error token not valid"} + } + return true, gssapi.Status{Code: gssapi.StatusUnavailable} + } + return false, gssapi.Status{Code: gssapi.StatusDefectiveToken, Message: "unknown TOK_ID in KRB5 token"} +} + +// IsAPReq tests if the MechToken contains an AP_REQ. +func (m *KRB5Token) IsAPReq() bool { + if hex.EncodeToString(m.tokID) == TOK_ID_KRB_AP_REQ { + return true + } + return false +} + +// IsAPRep tests if the MechToken contains an AP_REP. +func (m *KRB5Token) IsAPRep() bool { + if hex.EncodeToString(m.tokID) == TOK_ID_KRB_AP_REP { + return true + } + return false +} + +// IsKRBError tests if the MechToken contains an KRB_ERROR. +func (m *KRB5Token) IsKRBError() bool { + if hex.EncodeToString(m.tokID) == TOK_ID_KRB_ERROR { + return true + } + return false +} + +// Context returns the KRB5 token's context which will contain any verify user identity information. +func (m *KRB5Token) Context() context.Context { + return m.context +} + +// NewKRB5TokenAPREQ creates a new KRB5 token with AP_REQ +func NewKRB5TokenAPREQ(cl *client.Client, tkt messages.Ticket, sessionKey types.EncryptionKey, GSSAPIFlags []int, APOptions []int) (KRB5Token, error) { + // TODO consider providing the SPN rather than the specific tkt and key and get these from the krb client. + var m KRB5Token + m.OID = gssapi.OIDKRB5.OID() + tb, _ := hex.DecodeString(TOK_ID_KRB_AP_REQ) + m.tokID = tb + + auth, err := krb5TokenAuthenticator(cl.Credentials, GSSAPIFlags) + if err != nil { + return m, err + } + APReq, err := messages.NewAPReq( + tkt, + sessionKey, + auth, + ) + if err != nil { + return m, err + } + for _, o := range APOptions { + types.SetFlag(&APReq.APOptions, o) + } + m.APReq = APReq + return m, nil +} + +// krb5TokenAuthenticator creates a new kerberos authenticator for kerberos MechToken +func krb5TokenAuthenticator(creds *credentials.Credentials, flags []int) (types.Authenticator, error) { + //RFC 4121 Section 4.1.1 + auth, err := types.NewAuthenticator(creds.Domain(), creds.CName()) + if err != nil { + return auth, krberror.Errorf(err, krberror.KRBMsgError, "error generating new authenticator") + } + auth.Cksum = types.Checksum{ + CksumType: chksumtype.GSSAPI, + Checksum: newAuthenticatorChksum(flags), + } + return auth, nil +} + +// Create new authenticator checksum for kerberos MechToken +func newAuthenticatorChksum(flags []int) []byte { + a := make([]byte, 24) + binary.LittleEndian.PutUint32(a[:4], 16) + for _, i := range flags { + if i == gssapi.ContextFlagDeleg { + x := make([]byte, 28-len(a)) + a = append(a, x...) + } + f := binary.LittleEndian.Uint32(a[20:24]) + f |= uint32(i) + binary.LittleEndian.PutUint32(a[20:24], f) + } + return a +} diff --git a/github.com/jcmturner/gokrb5/v8/spnego/negotiationToken.go b/github.com/jcmturner/gokrb5/v8/spnego/negotiationToken.go new file mode 100644 index 0000000000..409ffcf92e --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/spnego/negotiationToken.go @@ -0,0 +1,302 @@ +package spnego + +import ( + "context" + "errors" + "fmt" + + "github.com/jcmturner/gofork/encoding/asn1" + "github.com/jcmturner/gokrb5/v8/client" + "github.com/jcmturner/gokrb5/v8/gssapi" + "github.com/jcmturner/gokrb5/v8/messages" + "github.com/jcmturner/gokrb5/v8/service" + "github.com/jcmturner/gokrb5/v8/types" +) + +// https://msdn.microsoft.com/en-us/library/ms995330.aspx + +// Negotiation state values. +const ( + NegStateAcceptCompleted NegState = 0 + NegStateAcceptIncomplete NegState = 1 + NegStateReject NegState = 2 + NegStateRequestMIC NegState = 3 +) + +// NegState is a type to indicate the SPNEGO negotiation state. +type NegState int + +// NegTokenInit implements Negotiation Token of type Init. +type NegTokenInit struct { + MechTypes []asn1.ObjectIdentifier + ReqFlags gssapi.ContextFlags + MechTokenBytes []byte + MechListMIC []byte + mechToken gssapi.ContextToken + settings *service.Settings +} + +type marshalNegTokenInit struct { + MechTypes []asn1.ObjectIdentifier `asn1:"explicit,tag:0"` + ReqFlags gssapi.ContextFlags `asn1:"explicit,optional,tag:1"` + MechTokenBytes []byte `asn1:"explicit,optional,omitempty,tag:2"` + MechListMIC []byte `asn1:"explicit,optional,omitempty,tag:3"` // This field is not used when negotiating Kerberos tokens +} + +// NegTokenResp implements Negotiation Token of type Resp/Targ +type NegTokenResp struct { + NegState asn1.Enumerated + SupportedMech asn1.ObjectIdentifier + ResponseToken []byte + MechListMIC []byte + mechToken gssapi.ContextToken + settings *service.Settings +} + +type marshalNegTokenResp struct { + NegState asn1.Enumerated `asn1:"explicit,tag:0"` + SupportedMech asn1.ObjectIdentifier `asn1:"explicit,optional,tag:1"` + ResponseToken []byte `asn1:"explicit,optional,omitempty,tag:2"` + MechListMIC []byte `asn1:"explicit,optional,omitempty,tag:3"` // This field is not used when negotiating Kerberos tokens +} + +// NegTokenTarg implements Negotiation Token of type Resp/Targ +type NegTokenTarg NegTokenResp + +// Marshal an Init negotiation token +func (n *NegTokenInit) Marshal() ([]byte, error) { + m := marshalNegTokenInit{ + MechTypes: n.MechTypes, + ReqFlags: n.ReqFlags, + MechTokenBytes: n.MechTokenBytes, + MechListMIC: n.MechListMIC, + } + b, err := asn1.Marshal(m) + if err != nil { + return nil, err + } + nt := asn1.RawValue{ + Tag: 0, + Class: 2, + IsCompound: true, + Bytes: b, + } + nb, err := asn1.Marshal(nt) + if err != nil { + return nil, err + } + return nb, nil +} + +// Unmarshal an Init negotiation token +func (n *NegTokenInit) Unmarshal(b []byte) error { + init, nt, err := UnmarshalNegToken(b) + if err != nil { + return err + } + if !init { + return errors.New("bytes were not that of a NegTokenInit") + } + nInit := nt.(NegTokenInit) + n.MechTokenBytes = nInit.MechTokenBytes + n.MechListMIC = nInit.MechListMIC + n.MechTypes = nInit.MechTypes + n.ReqFlags = nInit.ReqFlags + return nil +} + +// Verify an Init negotiation token +func (n *NegTokenInit) Verify() (bool, gssapi.Status) { + // Check if supported mechanisms are in the MechTypeList + var mtSupported bool + for _, m := range n.MechTypes { + if m.Equal(gssapi.OIDKRB5.OID()) || m.Equal(gssapi.OIDMSLegacyKRB5.OID()) { + if n.mechToken == nil && n.MechTokenBytes == nil { + return false, gssapi.Status{Code: gssapi.StatusContinueNeeded} + } + mtSupported = true + break + } + } + if !mtSupported { + return false, gssapi.Status{Code: gssapi.StatusBadMech, Message: "no supported mechanism specified in negotiation"} + } + // There should be some mechtoken bytes for a KRB5Token (other mech types are not supported) + mt := new(KRB5Token) + mt.settings = n.settings + if n.mechToken == nil { + err := mt.Unmarshal(n.MechTokenBytes) + if err != nil { + return false, gssapi.Status{Code: gssapi.StatusDefectiveToken, Message: err.Error()} + } + n.mechToken = mt + } else { + var ok bool + mt, ok = n.mechToken.(*KRB5Token) + if !ok { + return false, gssapi.Status{Code: gssapi.StatusDefectiveToken, Message: "MechToken is not a KRB5 token as expected"} + } + } + // Verify the mechtoken + return n.mechToken.Verify() +} + +// Context returns the SPNEGO context which will contain any verify user identity information. +func (n *NegTokenInit) Context() context.Context { + if n.mechToken != nil { + mt, ok := n.mechToken.(*KRB5Token) + if !ok { + return nil + } + return mt.Context() + } + return nil +} + +// Marshal a Resp/Targ negotiation token +func (n *NegTokenResp) Marshal() ([]byte, error) { + m := marshalNegTokenResp{ + NegState: n.NegState, + SupportedMech: n.SupportedMech, + ResponseToken: n.ResponseToken, + MechListMIC: n.MechListMIC, + } + b, err := asn1.Marshal(m) + if err != nil { + return nil, err + } + nt := asn1.RawValue{ + Tag: 1, + Class: 2, + IsCompound: true, + Bytes: b, + } + nb, err := asn1.Marshal(nt) + if err != nil { + return nil, err + } + return nb, nil +} + +// Unmarshal a Resp/Targ negotiation token +func (n *NegTokenResp) Unmarshal(b []byte) error { + init, nt, err := UnmarshalNegToken(b) + if err != nil { + return err + } + if init { + return errors.New("bytes were not that of a NegTokenResp") + } + nResp := nt.(NegTokenResp) + n.MechListMIC = nResp.MechListMIC + n.NegState = nResp.NegState + n.ResponseToken = nResp.ResponseToken + n.SupportedMech = nResp.SupportedMech + return nil +} + +// Verify a Resp/Targ negotiation token +func (n *NegTokenResp) Verify() (bool, gssapi.Status) { + if n.SupportedMech.Equal(gssapi.OIDKRB5.OID()) || n.SupportedMech.Equal(gssapi.OIDMSLegacyKRB5.OID()) { + if n.mechToken == nil && n.ResponseToken == nil { + return false, gssapi.Status{Code: gssapi.StatusContinueNeeded} + } + mt := new(KRB5Token) + mt.settings = n.settings + if n.mechToken == nil { + err := mt.Unmarshal(n.ResponseToken) + if err != nil { + return false, gssapi.Status{Code: gssapi.StatusDefectiveToken, Message: err.Error()} + } + n.mechToken = mt + } else { + var ok bool + mt, ok = n.mechToken.(*KRB5Token) + if !ok { + return false, gssapi.Status{Code: gssapi.StatusDefectiveToken, Message: "MechToken is not a KRB5 token as expected"} + } + } + if mt == nil { + return false, gssapi.Status{Code: gssapi.StatusContinueNeeded} + } + // Verify the mechtoken + return mt.Verify() + } + return false, gssapi.Status{Code: gssapi.StatusBadMech, Message: "no supported mechanism specified in negotiation"} +} + +// State returns the negotiation state of the negotiation response. +func (n *NegTokenResp) State() NegState { + return NegState(n.NegState) +} + +// Context returns the SPNEGO context which will contain any verify user identity information. +func (n *NegTokenResp) Context() context.Context { + if n.mechToken != nil { + mt, ok := n.mechToken.(*KRB5Token) + if !ok { + return nil + } + return mt.Context() + } + return nil +} + +// UnmarshalNegToken umarshals and returns either a NegTokenInit or a NegTokenResp. +// +// The boolean indicates if the response is a NegTokenInit. +// If error is nil and the boolean is false the response is a NegTokenResp. +func UnmarshalNegToken(b []byte) (bool, interface{}, error) { + var a asn1.RawValue + _, err := asn1.Unmarshal(b, &a) + if err != nil { + return false, nil, fmt.Errorf("error unmarshalling NegotiationToken: %v", err) + } + switch a.Tag { + case 0: + var n marshalNegTokenInit + _, err = asn1.Unmarshal(a.Bytes, &n) + if err != nil { + return false, nil, fmt.Errorf("error unmarshalling NegotiationToken type %d (Init): %v", a.Tag, err) + } + nt := NegTokenInit{ + MechTypes: n.MechTypes, + ReqFlags: n.ReqFlags, + MechTokenBytes: n.MechTokenBytes, + MechListMIC: n.MechListMIC, + } + return true, nt, nil + case 1: + var n marshalNegTokenResp + _, err = asn1.Unmarshal(a.Bytes, &n) + if err != nil { + return false, nil, fmt.Errorf("error unmarshalling NegotiationToken type %d (Resp/Targ): %v", a.Tag, err) + } + nt := NegTokenResp{ + NegState: n.NegState, + SupportedMech: n.SupportedMech, + ResponseToken: n.ResponseToken, + MechListMIC: n.MechListMIC, + } + return false, nt, nil + default: + return false, nil, errors.New("unknown choice type for NegotiationToken") + } + +} + +// NewNegTokenInitKRB5 creates new Init negotiation token for Kerberos 5 +func NewNegTokenInitKRB5(cl *client.Client, tkt messages.Ticket, sessionKey types.EncryptionKey) (NegTokenInit, error) { + mt, err := NewKRB5TokenAPREQ(cl, tkt, sessionKey, []int{gssapi.ContextFlagInteg, gssapi.ContextFlagConf}, []int{}) + if err != nil { + return NegTokenInit{}, fmt.Errorf("error getting KRB5 token; %v", err) + } + mtb, err := mt.Marshal() + if err != nil { + return NegTokenInit{}, fmt.Errorf("error marshalling KRB5 token; %v", err) + } + return NegTokenInit{ + MechTypes: []asn1.ObjectIdentifier{gssapi.OIDKRB5.OID()}, + MechTokenBytes: mtb, + }, nil +} diff --git a/github.com/jcmturner/gokrb5/v8/spnego/spnego.go b/github.com/jcmturner/gokrb5/v8/spnego/spnego.go new file mode 100644 index 0000000000..ebb818b9e6 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/spnego/spnego.go @@ -0,0 +1,203 @@ +// Package spnego implements the Simple and Protected GSSAPI Negotiation Mechanism for Kerberos authentication. +package spnego + +import ( + "context" + "errors" + "fmt" + + "github.com/jcmturner/gofork/encoding/asn1" + "github.com/jcmturner/gokrb5/v8/asn1tools" + "github.com/jcmturner/gokrb5/v8/client" + "github.com/jcmturner/gokrb5/v8/gssapi" + "github.com/jcmturner/gokrb5/v8/keytab" + "github.com/jcmturner/gokrb5/v8/service" +) + +// SPNEGO implements the GSS-API mechanism for RFC 4178 +type SPNEGO struct { + serviceSettings *service.Settings + client *client.Client + spn string +} + +// SPNEGOClient configures the SPNEGO mechanism suitable for client side use. +func SPNEGOClient(cl *client.Client, spn string) *SPNEGO { + s := new(SPNEGO) + s.client = cl + s.spn = spn + s.serviceSettings = service.NewSettings(nil, service.SName(spn)) + return s +} + +// SPNEGOService configures the SPNEGO mechanism suitable for service side use. +func SPNEGOService(kt *keytab.Keytab, options ...func(*service.Settings)) *SPNEGO { + s := new(SPNEGO) + s.serviceSettings = service.NewSettings(kt, options...) + return s +} + +// OID returns the GSS-API assigned OID for SPNEGO. +func (s *SPNEGO) OID() asn1.ObjectIdentifier { + return gssapi.OIDSPNEGO.OID() +} + +// AcquireCred is the GSS-API method to acquire a client credential via Kerberos for SPNEGO. +func (s *SPNEGO) AcquireCred() error { + return s.client.AffirmLogin() +} + +// InitSecContext is the GSS-API method for the client to a generate a context token to the service via Kerberos. +func (s *SPNEGO) InitSecContext() (gssapi.ContextToken, error) { + tkt, key, err := s.client.GetServiceTicket(s.spn) + if err != nil { + return &SPNEGOToken{}, err + } + negTokenInit, err := NewNegTokenInitKRB5(s.client, tkt, key) + if err != nil { + return &SPNEGOToken{}, fmt.Errorf("could not create NegTokenInit: %v", err) + } + return &SPNEGOToken{ + Init: true, + NegTokenInit: negTokenInit, + settings: s.serviceSettings, + }, nil +} + +// AcceptSecContext is the GSS-API method for the service to verify the context token provided by the client and +// establish a context. +func (s *SPNEGO) AcceptSecContext(ct gssapi.ContextToken) (bool, context.Context, gssapi.Status) { + var ctx context.Context + t, ok := ct.(*SPNEGOToken) + if !ok { + return false, ctx, gssapi.Status{Code: gssapi.StatusDefectiveToken, Message: "context token provided was not an SPNEGO token"} + } + t.settings = s.serviceSettings + var oid asn1.ObjectIdentifier + if t.Init { + oid = t.NegTokenInit.MechTypes[0] + } + if t.Resp { + oid = t.NegTokenResp.SupportedMech + } + if !(oid.Equal(gssapi.OIDKRB5.OID()) || oid.Equal(gssapi.OIDMSLegacyKRB5.OID())) { + return false, ctx, gssapi.Status{Code: gssapi.StatusDefectiveToken, Message: "SPNEGO OID of MechToken is not of type KRB5"} + } + // Flags in the NegInit must be used t.NegTokenInit.ReqFlags + ok, status := t.Verify() + ctx = t.Context() + return ok, ctx, status +} + +// Log will write to the service's logger if it is configured. +func (s *SPNEGO) Log(format string, v ...interface{}) { + if s.serviceSettings.Logger() != nil { + s.serviceSettings.Logger().Output(2, fmt.Sprintf(format, v...)) + } +} + +// SPNEGOToken is a GSS-API context token +type SPNEGOToken struct { + Init bool + Resp bool + NegTokenInit NegTokenInit + NegTokenResp NegTokenResp + settings *service.Settings + context context.Context +} + +// Marshal SPNEGO context token +func (s *SPNEGOToken) Marshal() ([]byte, error) { + var b []byte + if s.Init { + hb, _ := asn1.Marshal(gssapi.OIDSPNEGO.OID()) + tb, err := s.NegTokenInit.Marshal() + if err != nil { + return b, fmt.Errorf("could not marshal NegTokenInit: %v", err) + } + b = append(hb, tb...) + return asn1tools.AddASNAppTag(b, 0), nil + } + if s.Resp { + b, err := s.NegTokenResp.Marshal() + if err != nil { + return b, fmt.Errorf("could not marshal NegTokenResp: %v", err) + } + return b, nil + } + return b, errors.New("SPNEGO cannot be marshalled. It contains neither a NegTokenInit or NegTokenResp") +} + +// Unmarshal SPNEGO context token +func (s *SPNEGOToken) Unmarshal(b []byte) error { + var r []byte + var err error + // We need some data in the array + if len(b) < 1 { + return fmt.Errorf("provided byte array is empty") + } + if b[0] != byte(161) { + // Not a NegTokenResp/Targ could be a NegTokenInit + var oid asn1.ObjectIdentifier + r, err = asn1.UnmarshalWithParams(b, &oid, fmt.Sprintf("application,explicit,tag:%v", 0)) + if err != nil { + return fmt.Errorf("not a valid SPNEGO token: %v", err) + } + // Check the OID is the SPNEGO OID value + SPNEGOOID := gssapi.OIDSPNEGO.OID() + if !oid.Equal(SPNEGOOID) { + return fmt.Errorf("OID %s does not match SPNEGO OID %s", oid.String(), SPNEGOOID.String()) + } + } else { + // Could be a NegTokenResp/Targ + r = b + } + + _, nt, err := UnmarshalNegToken(r) + if err != nil { + return err + } + switch v := nt.(type) { + case NegTokenInit: + s.Init = true + s.NegTokenInit = v + s.NegTokenInit.settings = s.settings + case NegTokenResp: + s.Resp = true + s.NegTokenResp = v + s.NegTokenResp.settings = s.settings + default: + return errors.New("unknown choice type for NegotiationToken") + } + return nil +} + +// Verify the SPNEGOToken +func (s *SPNEGOToken) Verify() (bool, gssapi.Status) { + if (!s.Init && !s.Resp) || (s.Init && s.Resp) { + return false, gssapi.Status{Code: gssapi.StatusDefectiveToken, Message: "invalid SPNEGO token, unclear if NegTokenInit or NegTokenResp"} + } + if s.Init { + s.NegTokenInit.settings = s.settings + ok, status := s.NegTokenInit.Verify() + if ok { + s.context = s.NegTokenInit.Context() + } + return ok, status + } + if s.Resp { + s.NegTokenResp.settings = s.settings + ok, status := s.NegTokenResp.Verify() + if ok { + s.context = s.NegTokenResp.Context() + } + return ok, status + } + // should not be possible to get here + return false, gssapi.Status{Code: gssapi.StatusFailure, Message: "unable to verify SPNEGO token"} +} + +// Context returns the SPNEGO context which will contain any verify user identity information. +func (s *SPNEGOToken) Context() context.Context { + return s.context +} diff --git a/github.com/jcmturner/gokrb5/v8/types/Authenticator.go b/github.com/jcmturner/gokrb5/v8/types/Authenticator.go new file mode 100644 index 0000000000..1fdba78a37 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/types/Authenticator.go @@ -0,0 +1,81 @@ +// Package types provides Kerberos 5 data types. +package types + +import ( + "crypto/rand" + "fmt" + "math" + "math/big" + "time" + + "github.com/jcmturner/gofork/encoding/asn1" + "github.com/jcmturner/gokrb5/v8/asn1tools" + "github.com/jcmturner/gokrb5/v8/iana" + "github.com/jcmturner/gokrb5/v8/iana/asnAppTag" +) + +// Authenticator - A record containing information that can be shown to have been recently generated using the session +// key known only by the client and server. +// https://tools.ietf.org/html/rfc4120#section-5.5.1 +type Authenticator struct { + AVNO int `asn1:"explicit,tag:0"` + CRealm string `asn1:"generalstring,explicit,tag:1"` + CName PrincipalName `asn1:"explicit,tag:2"` + Cksum Checksum `asn1:"explicit,optional,tag:3"` + Cusec int `asn1:"explicit,tag:4"` + CTime time.Time `asn1:"generalized,explicit,tag:5"` + SubKey EncryptionKey `asn1:"explicit,optional,tag:6"` + SeqNumber int64 `asn1:"explicit,optional,tag:7"` + AuthorizationData AuthorizationData `asn1:"explicit,optional,tag:8"` +} + +// NewAuthenticator creates a new Authenticator. +func NewAuthenticator(realm string, cname PrincipalName) (Authenticator, error) { + seq, err := rand.Int(rand.Reader, big.NewInt(math.MaxUint32)) + if err != nil { + return Authenticator{}, err + } + t := time.Now().UTC() + return Authenticator{ + AVNO: iana.PVNO, + CRealm: realm, + CName: cname, + Cksum: Checksum{}, + Cusec: int((t.UnixNano() / int64(time.Microsecond)) - (t.Unix() * 1e6)), + CTime: t, + SeqNumber: seq.Int64(), + }, nil +} + +// GenerateSeqNumberAndSubKey sets the Authenticator's sequence number and subkey. +func (a *Authenticator) GenerateSeqNumberAndSubKey(keyType int32, keySize int) error { + seq, err := rand.Int(rand.Reader, big.NewInt(math.MaxUint32)) + if err != nil { + return err + } + a.SeqNumber = seq.Int64() + //Generate subkey value + sk := make([]byte, keySize, keySize) + rand.Read(sk) + a.SubKey = EncryptionKey{ + KeyType: keyType, + KeyValue: sk, + } + return nil +} + +// Unmarshal bytes into the Authenticator. +func (a *Authenticator) Unmarshal(b []byte) error { + _, err := asn1.UnmarshalWithParams(b, a, fmt.Sprintf("application,explicit,tag:%v", asnAppTag.Authenticator)) + return err +} + +// Marshal the Authenticator. +func (a *Authenticator) Marshal() ([]byte, error) { + b, err := asn1.Marshal(*a) + if err != nil { + return nil, err + } + b = asn1tools.AddASNAppTag(b, asnAppTag.Authenticator) + return b, nil +} diff --git a/github.com/jcmturner/gokrb5/v8/types/AuthorizationData.go b/github.com/jcmturner/gokrb5/v8/types/AuthorizationData.go new file mode 100644 index 0000000000..80c477cef7 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/types/AuthorizationData.go @@ -0,0 +1,55 @@ +package types + +import ( + "github.com/jcmturner/gofork/encoding/asn1" +) + +// Reference: https://www.ietf.org/rfc/rfc4120.txt +// Section: 5.2.6 + +// AuthorizationData implements RFC 4120 type: https://tools.ietf.org/html/rfc4120#section-5.2.6 +type AuthorizationData []AuthorizationDataEntry + +// AuthorizationDataEntry implements RFC 4120 type: https://tools.ietf.org/html/rfc4120#section-5.2.6 +type AuthorizationDataEntry struct { + ADType int32 `asn1:"explicit,tag:0"` + ADData []byte `asn1:"explicit,tag:1"` +} + +// ADIfRelevant implements RFC 4120 type: https://tools.ietf.org/html/rfc4120#section-5.2.6.1 +type ADIfRelevant AuthorizationData + +// ADKDCIssued implements RFC 4120 type: https://tools.ietf.org/html/rfc4120#section-5.2.6.2 +type ADKDCIssued struct { + ADChecksum Checksum `asn1:"explicit,tag:0"` + IRealm string `asn1:"optional,generalstring,explicit,tag:1"` + Isname PrincipalName `asn1:"optional,explicit,tag:2"` + Elements AuthorizationData `asn1:"explicit,tag:3"` +} + +// ADAndOr implements RFC 4120 type: https://tools.ietf.org/html/rfc4120#section-5.2.6.3 +type ADAndOr struct { + ConditionCount int32 `asn1:"explicit,tag:0"` + Elements AuthorizationData `asn1:"explicit,tag:1"` +} + +// ADMandatoryForKDC implements RFC 4120 type: https://tools.ietf.org/html/rfc4120#section-5.2.6.4 +type ADMandatoryForKDC AuthorizationData + +// Unmarshal bytes into the ADKDCIssued. +func (a *ADKDCIssued) Unmarshal(b []byte) error { + _, err := asn1.Unmarshal(b, a) + return err +} + +// Unmarshal bytes into the AuthorizationData. +func (a *AuthorizationData) Unmarshal(b []byte) error { + _, err := asn1.Unmarshal(b, a) + return err +} + +// Unmarshal bytes into the AuthorizationDataEntry. +func (a *AuthorizationDataEntry) Unmarshal(b []byte) error { + _, err := asn1.Unmarshal(b, a) + return err +} diff --git a/github.com/jcmturner/gokrb5/v8/types/Cryptosystem.go b/github.com/jcmturner/gokrb5/v8/types/Cryptosystem.go new file mode 100644 index 0000000000..1f62d885cd --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/types/Cryptosystem.go @@ -0,0 +1,55 @@ +package types + +import ( + "github.com/jcmturner/gofork/encoding/asn1" +) + +// Reference: https://www.ietf.org/rfc/rfc4120.txt +// Section: 5.2.9 + +// EncryptedData implements RFC 4120 type: https://tools.ietf.org/html/rfc4120#section-5.2.9 +type EncryptedData struct { + EType int32 `asn1:"explicit,tag:0"` + KVNO int `asn1:"explicit,optional,tag:1"` + Cipher []byte `asn1:"explicit,tag:2"` +} + +// EncryptionKey implements RFC 4120 type: https://tools.ietf.org/html/rfc4120#section-5.2.9 +// AKA KeyBlock +type EncryptionKey struct { + KeyType int32 `asn1:"explicit,tag:0"` + KeyValue []byte `asn1:"explicit,tag:1" json:"-"` +} + +// Checksum implements RFC 4120 type: https://tools.ietf.org/html/rfc4120#section-5.2.9 +type Checksum struct { + CksumType int32 `asn1:"explicit,tag:0"` + Checksum []byte `asn1:"explicit,tag:1"` +} + +// Unmarshal bytes into the EncryptedData. +func (a *EncryptedData) Unmarshal(b []byte) error { + _, err := asn1.Unmarshal(b, a) + return err +} + +// Marshal the EncryptedData. +func (a *EncryptedData) Marshal() ([]byte, error) { + edb, err := asn1.Marshal(*a) + if err != nil { + return edb, err + } + return edb, nil +} + +// Unmarshal bytes into the EncryptionKey. +func (a *EncryptionKey) Unmarshal(b []byte) error { + _, err := asn1.Unmarshal(b, a) + return err +} + +// Unmarshal bytes into the Checksum. +func (a *Checksum) Unmarshal(b []byte) error { + _, err := asn1.Unmarshal(b, a) + return err +} diff --git a/github.com/jcmturner/gokrb5/v8/types/HostAddress.go b/github.com/jcmturner/gokrb5/v8/types/HostAddress.go new file mode 100644 index 0000000000..895fe80539 --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/types/HostAddress.go @@ -0,0 +1,180 @@ +package types + +// Reference: https://www.ietf.org/rfc/rfc4120.txt +// Section: 5.2.5 + +import ( + "bytes" + "fmt" + "net" + + "github.com/jcmturner/gofork/encoding/asn1" + "github.com/jcmturner/gokrb5/v8/iana/addrtype" +) + +// HostAddresses implements RFC 4120 type: https://tools.ietf.org/html/rfc4120#section-5.2.5 +type HostAddresses []HostAddress + +// HostAddress implements RFC 4120 type: https://tools.ietf.org/html/rfc4120#section-5.2.5 +type HostAddress struct { + AddrType int32 `asn1:"explicit,tag:0"` + Address []byte `asn1:"explicit,tag:1"` +} + +// GetHostAddress returns a HostAddress struct from a string in the format : +func GetHostAddress(s string) (HostAddress, error) { + var h HostAddress + cAddr, _, err := net.SplitHostPort(s) + if err != nil { + return h, fmt.Errorf("invalid format of client address: %v", err) + } + ip := net.ParseIP(cAddr) + var ht int32 + if ip.To4() != nil { + ht = addrtype.IPv4 + ip = ip.To4() + } else if ip.To16() != nil { + ht = addrtype.IPv6 + ip = ip.To16() + } else { + return h, fmt.Errorf("could not determine client's address types: %v", err) + } + h = HostAddress{ + AddrType: ht, + Address: ip, + } + return h, nil +} + +// GetAddress returns a string representation of the HostAddress. +func (h *HostAddress) GetAddress() (string, error) { + var b []byte + _, err := asn1.Unmarshal(h.Address, &b) + return string(b), err +} + +// LocalHostAddresses returns a HostAddresses struct for the local machines interface IP addresses. +func LocalHostAddresses() (ha HostAddresses, err error) { + ifs, err := net.Interfaces() + if err != nil { + return + } + for _, iface := range ifs { + if iface.Flags&net.FlagLoopback != 0 || iface.Flags&net.FlagUp == 0 { + // Interface is either loopback of not up + continue + } + addrs, err := iface.Addrs() + if err != nil { + continue + } + for _, addr := range addrs { + var ip net.IP + switch v := addr.(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP + } + var a HostAddress + if ip.To16() == nil { + //neither IPv4 or IPv6 + continue + } + if ip.To4() != nil { + //Is IPv4 + a.AddrType = addrtype.IPv4 + a.Address = ip.To4() + } else { + a.AddrType = addrtype.IPv6 + a.Address = ip.To16() + } + ha = append(ha, a) + } + } + return ha, nil +} + +// HostAddressesFromNetIPs returns a HostAddresses type from a slice of net.IP +func HostAddressesFromNetIPs(ips []net.IP) (ha HostAddresses) { + for _, ip := range ips { + ha = append(ha, HostAddressFromNetIP(ip)) + } + return ha +} + +// HostAddressFromNetIP returns a HostAddress type from a net.IP +func HostAddressFromNetIP(ip net.IP) HostAddress { + if ip.To4() != nil { + //Is IPv4 + return HostAddress{ + AddrType: addrtype.IPv4, + Address: ip.To4(), + } + } + return HostAddress{ + AddrType: addrtype.IPv6, + Address: ip.To16(), + } +} + +// HostAddressesEqual tests if two HostAddress slices are equal. +func HostAddressesEqual(h, a []HostAddress) bool { + if len(h) != len(a) { + return false + } + for _, e := range a { + var found bool + for _, i := range h { + if e.Equal(i) { + found = true + break + } + } + if !found { + return false + } + } + return true +} + +// HostAddressesContains tests if a HostAddress is contained in a HostAddress slice. +func HostAddressesContains(h []HostAddress, a HostAddress) bool { + for _, e := range h { + if e.Equal(a) { + return true + } + } + return false +} + +// Equal tests if the HostAddress is equal to another HostAddress provided. +func (h *HostAddress) Equal(a HostAddress) bool { + if h.AddrType != a.AddrType { + return false + } + return bytes.Equal(h.Address, a.Address) +} + +// Contains tests if a HostAddress is contained within the HostAddresses struct. +func (h *HostAddresses) Contains(a HostAddress) bool { + for _, e := range *h { + if e.Equal(a) { + return true + } + } + return false +} + +// Equal tests if a HostAddress slice is equal to the HostAddresses struct. +func (h *HostAddresses) Equal(a []HostAddress) bool { + if len(*h) != len(a) { + return false + } + for _, e := range a { + if !h.Contains(e) { + return false + } + } + return true +} diff --git a/github.com/jcmturner/gokrb5/v8/types/KerberosFlags.go b/github.com/jcmturner/gokrb5/v8/types/KerberosFlags.go new file mode 100644 index 0000000000..0f2038340d --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/types/KerberosFlags.go @@ -0,0 +1,68 @@ +package types + +// Reference: https://www.ietf.org/rfc/rfc4120.txt +// Section: 5.2.8 + +import ( + "github.com/jcmturner/gofork/encoding/asn1" +) + +// NewKrbFlags returns an ASN1 BitString struct of the right size for KrbFlags. +func NewKrbFlags() asn1.BitString { + f := asn1.BitString{} + f.Bytes = make([]byte, 4) + f.BitLength = len(f.Bytes) * 8 + return f +} + +// SetFlags sets the flags of an ASN1 BitString. +func SetFlags(f *asn1.BitString, j []int) { + for _, i := range j { + SetFlag(f, i) + } +} + +// SetFlag sets a flag in an ASN1 BitString. +func SetFlag(f *asn1.BitString, i int) { + for l := len(f.Bytes); l < 4; l++ { + (*f).Bytes = append((*f).Bytes, byte(0)) + (*f).BitLength = len((*f).Bytes) * 8 + } + //Which byte? + b := i / 8 + //Which bit in byte + p := uint(7 - (i - 8*b)) + (*f).Bytes[b] = (*f).Bytes[b] | (1 << p) +} + +// UnsetFlags unsets flags in an ASN1 BitString. +func UnsetFlags(f *asn1.BitString, j []int) { + for _, i := range j { + UnsetFlag(f, i) + } +} + +// UnsetFlag unsets a flag in an ASN1 BitString. +func UnsetFlag(f *asn1.BitString, i int) { + for l := len(f.Bytes); l < 4; l++ { + (*f).Bytes = append((*f).Bytes, byte(0)) + (*f).BitLength = len((*f).Bytes) * 8 + } + //Which byte? + b := i / 8 + //Which bit in byte + p := uint(7 - (i - 8*b)) + (*f).Bytes[b] = (*f).Bytes[b] &^ (1 << p) +} + +// IsFlagSet tests if a flag is set in the ASN1 BitString. +func IsFlagSet(f *asn1.BitString, i int) bool { + //Which byte? + b := i / 8 + //Which bit in byte + p := uint(7 - (i - 8*b)) + if (*f).Bytes[b]&(1</@ +// a PrincipalName type will be returned with the name type set to KRB_NT_PRINCIPAL(1) +// and the realm will be returned as a string. If the "@" suffix +// is not included in the SPN then the value of realm string returned will be "" +func ParseSPNString(spn string) (pn PrincipalName, realm string) { + if strings.Contains(spn, "@") { + s := strings.Split(spn, "@") + realm = s[len(s)-1] + spn = strings.TrimSuffix(spn, "@"+realm) + } + pn = NewPrincipalName(nametype.KRB_NT_PRINCIPAL, spn) + return +} diff --git a/github.com/jcmturner/gokrb5/v8/types/TypedData.go b/github.com/jcmturner/gokrb5/v8/types/TypedData.go new file mode 100644 index 0000000000..19e9f4961f --- /dev/null +++ b/github.com/jcmturner/gokrb5/v8/types/TypedData.go @@ -0,0 +1,18 @@ +package types + +import "github.com/jcmturner/gofork/encoding/asn1" + +// TypedData implements RFC 4120 type: https://tools.ietf.org/html/rfc4120#section-5.9.1 +type TypedData struct { + DataType int32 `asn1:"explicit,tag:0"` + DataValue []byte `asn1:"optional,explicit,tag:1"` +} + +// TypedDataSequence implements RFC 4120 type: https://tools.ietf.org/html/rfc4120#section-5.9.1 +type TypedDataSequence []TypedData + +// Unmarshal bytes into the TypedDataSequence. +func (a *TypedDataSequence) Unmarshal(b []byte) error { + _, err := asn1.Unmarshal(b, a) + return err +} diff --git a/github.com/jcmturner/rpc/v2/LICENSE b/github.com/jcmturner/rpc/v2/LICENSE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/github.com/jcmturner/rpc/v2/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/github.com/jcmturner/rpc/v2/mstypes/claims.go b/github.com/jcmturner/rpc/v2/mstypes/claims.go new file mode 100644 index 0000000000..b9f535f5b9 --- /dev/null +++ b/github.com/jcmturner/rpc/v2/mstypes/claims.go @@ -0,0 +1,152 @@ +package mstypes + +import ( + "bytes" + "encoding/hex" + "errors" + "fmt" + + "github.com/jcmturner/rpc/v2/ndr" + "golang.org/x/net/http2/hpack" +) + +// Compression format assigned numbers. https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-xca/a8b7cb0a-92a6-4187-a23b-5e14273b96f8 +const ( + CompressionFormatNone uint16 = 0 + CompressionFormatLZNT1 uint16 = 2 // LZNT1 aka ntfs compression + CompressionFormatXPress uint16 = 3 // plain LZ77 + CompressionFormatXPressHuff uint16 = 4 // LZ77+Huffman - The Huffman variant of the XPRESS compression format uses LZ77-style dictionary compression combined with Huffman coding. +) + +// ClaimsSourceTypeAD https://msdn.microsoft.com/en-us/library/hh553809.aspx +const ClaimsSourceTypeAD uint16 = 1 + +// Claim Type assigned numbers +const ( + ClaimTypeIDInt64 uint16 = 1 + ClaimTypeIDUInt64 uint16 = 2 + ClaimTypeIDString uint16 = 3 + ClaimsTypeIDBoolean uint16 = 6 +) + +// ClaimsBlob implements https://msdn.microsoft.com/en-us/library/hh554119.aspx +type ClaimsBlob struct { + Size uint32 + EncodedBlob EncodedBlob +} + +// EncodedBlob are the bytes of the encoded Claims +type EncodedBlob []byte + +// Size returns the size of the bytes of the encoded Claims +func (b EncodedBlob) Size(c interface{}) int { + cb := c.(ClaimsBlob) + return int(cb.Size) +} + +// ClaimsSetMetadata implements https://msdn.microsoft.com/en-us/library/hh554073.aspx +type ClaimsSetMetadata struct { + ClaimsSetSize uint32 + ClaimsSetBytes []byte `ndr:"pointer,conformant"` + CompressionFormat uint16 // Enum see constants for options + UncompressedClaimsSetSize uint32 + ReservedType uint16 + ReservedFieldSize uint32 + ReservedField []byte `ndr:"pointer,conformant"` +} + +// ClaimsSet reads the ClaimsSet type from the NDR encoded ClaimsSetBytes in the ClaimsSetMetadata +func (m *ClaimsSetMetadata) ClaimsSet() (c ClaimsSet, err error) { + if len(m.ClaimsSetBytes) < 1 { + err = errors.New("no bytes available for ClaimsSet") + return + } + // TODO switch statement to decompress ClaimsSetBytes + switch m.CompressionFormat { + case CompressionFormatLZNT1: + s := hex.EncodeToString(m.ClaimsSetBytes) + err = fmt.Errorf("ClaimsSet compressed, format LZNT1 not currently supported: %s", s) + return + case CompressionFormatXPress: + s := hex.EncodeToString(m.ClaimsSetBytes) + err = fmt.Errorf("ClaimsSet compressed, format XPress not currently supported: %s", s) + return + case CompressionFormatXPressHuff: + var b []byte + buff := bytes.NewBuffer(b) + _, e := hpack.HuffmanDecode(buff, m.ClaimsSetBytes) + if e != nil { + err = fmt.Errorf("error deflating: %v", e) + return + } + m.ClaimsSetBytes = buff.Bytes() + } + dec := ndr.NewDecoder(bytes.NewReader(m.ClaimsSetBytes)) + err = dec.Decode(&c) + return +} + +// ClaimsSet implements https://msdn.microsoft.com/en-us/library/hh554122.aspx +type ClaimsSet struct { + ClaimsArrayCount uint32 + ClaimsArrays []ClaimsArray `ndr:"pointer,conformant"` + ReservedType uint16 + ReservedFieldSize uint32 + ReservedField []byte `ndr:"pointer,conformant"` +} + +// ClaimsArray implements https://msdn.microsoft.com/en-us/library/hh536458.aspx +type ClaimsArray struct { + ClaimsSourceType uint16 + ClaimsCount uint32 + ClaimEntries []ClaimEntry `ndr:"pointer,conformant"` +} + +// ClaimEntry is a NDR union that implements https://msdn.microsoft.com/en-us/library/hh536374.aspx +type ClaimEntry struct { + ID string `ndr:"pointer,conformant,varying"` + Type uint16 `ndr:"unionTag"` + TypeInt64 ClaimTypeInt64 `ndr:"unionField"` + TypeUInt64 ClaimTypeUInt64 `ndr:"unionField"` + TypeString ClaimTypeString `ndr:"unionField"` + TypeBool ClaimTypeBoolean `ndr:"unionField"` +} + +// SwitchFunc is the ClaimEntry union field selection function +func (u ClaimEntry) SwitchFunc(_ interface{}) string { + switch u.Type { + case ClaimTypeIDInt64: + return "TypeInt64" + case ClaimTypeIDUInt64: + return "TypeUInt64" + case ClaimTypeIDString: + return "TypeString" + case ClaimsTypeIDBoolean: + return "TypeBool" + } + return "" +} + +// ClaimTypeInt64 is a claim of type int64 +type ClaimTypeInt64 struct { + ValueCount uint32 + Value []int64 `ndr:"pointer,conformant"` +} + +// ClaimTypeUInt64 is a claim of type uint64 +type ClaimTypeUInt64 struct { + ValueCount uint32 + Value []uint64 `ndr:"pointer,conformant"` +} + +// ClaimTypeString is a claim of type string +type ClaimTypeString struct { + ValueCount uint32 + Value []LPWSTR `ndr:"pointer,conformant"` +} + +// ClaimTypeBoolean is a claim of type bool +type ClaimTypeBoolean struct { + ValueCount uint32 + Value []bool `ndr:"pointer,conformant"` +} diff --git a/github.com/jcmturner/rpc/v2/mstypes/common.go b/github.com/jcmturner/rpc/v2/mstypes/common.go new file mode 100644 index 0000000000..fb6510d1cb --- /dev/null +++ b/github.com/jcmturner/rpc/v2/mstypes/common.go @@ -0,0 +1,12 @@ +// Package mstypes provides implemnations of some Microsoft data types [MS-DTYP] https://msdn.microsoft.com/en-us/library/cc230283.aspx +package mstypes + +// LPWSTR implements https://msdn.microsoft.com/en-us/library/cc230355.aspx +type LPWSTR struct { + Value string `ndr:"pointer,conformant,varying"` +} + +// String returns the string representation of LPWSTR data type. +func (s *LPWSTR) String() string { + return s.Value +} diff --git a/github.com/jcmturner/rpc/v2/mstypes/filetime.go b/github.com/jcmturner/rpc/v2/mstypes/filetime.go new file mode 100644 index 0000000000..5cc952fa9d --- /dev/null +++ b/github.com/jcmturner/rpc/v2/mstypes/filetime.go @@ -0,0 +1,52 @@ +// Package mstypes implements representations of Microsoft types +package mstypes + +import ( + "time" +) + +/* +FILETIME is a windows data structure. +Ref: https://msdn.microsoft.com/en-us/library/windows/desktop/ms724284%28v=vs.85%29.aspx +It contains two parts that are 32bit integers: + dwLowDateTime + dwHighDateTime +We need to combine these two into one 64bit integer. +This gives the number of 100 nano second period from January 1, 1601, Coordinated Universal Time (UTC) +*/ + +const unixEpochDiff = 116444736000000000 + +// FileTime implements the Microsoft FILETIME type https://msdn.microsoft.com/en-us/library/cc230324.aspx +type FileTime struct { + LowDateTime uint32 + HighDateTime uint32 +} + +// Time return a golang Time type from the FileTime +func (ft FileTime) Time() time.Time { + ns := (ft.MSEpoch() - unixEpochDiff) * 100 + return time.Unix(0, int64(ns)).UTC() +} + +// MSEpoch returns the FileTime as a Microsoft epoch, the number of 100 nano second periods elapsed from January 1, 1601 UTC. +func (ft FileTime) MSEpoch() int64 { + return (int64(ft.HighDateTime) << 32) + int64(ft.LowDateTime) +} + +// Unix returns the FileTime as a Unix time, the number of seconds elapsed since January 1, 1970 UTC. +func (ft FileTime) Unix() int64 { + return (ft.MSEpoch() - unixEpochDiff) / 10000000 +} + +// GetFileTime returns a FileTime type from the provided Golang Time type. +func GetFileTime(t time.Time) FileTime { + ns := t.UnixNano() + fp := (ns / 100) + unixEpochDiff + hd := fp >> 32 + ld := fp - (hd << 32) + return FileTime{ + LowDateTime: uint32(ld), + HighDateTime: uint32(hd), + } +} diff --git a/github.com/jcmturner/rpc/v2/mstypes/group_membership.go b/github.com/jcmturner/rpc/v2/mstypes/group_membership.go new file mode 100644 index 0000000000..79151378fc --- /dev/null +++ b/github.com/jcmturner/rpc/v2/mstypes/group_membership.go @@ -0,0 +1,19 @@ +package mstypes + +// GroupMembership implements https://msdn.microsoft.com/en-us/library/cc237945.aspx +// RelativeID : A 32-bit unsigned integer that contains the RID of a particular group. +// The possible values for the Attributes flags are identical to those specified in KERB_SID_AND_ATTRIBUTES +type GroupMembership struct { + RelativeID uint32 + Attributes uint32 +} + +// DomainGroupMembership implements https://msdn.microsoft.com/en-us/library/hh536344.aspx +// DomainId: A SID structure that contains the SID for the domain.This member is used in conjunction with the GroupIds members to create group SIDs for the device. +// GroupCount: A 32-bit unsigned integer that contains the number of groups within the domain to which the account belongs. +// GroupIds: A pointer to a list of GROUP_MEMBERSHIP structures that contain the groups to which the account belongs in the domain. The number of groups in this list MUST be equal to GroupCount. +type DomainGroupMembership struct { + DomainID RPCSID `ndr:"pointer"` + GroupCount uint32 + GroupIDs []GroupMembership `ndr:"pointer,conformant"` // Size is value of GroupCount +} diff --git a/github.com/jcmturner/rpc/v2/mstypes/kerb_sid_and_attributes.go b/github.com/jcmturner/rpc/v2/mstypes/kerb_sid_and_attributes.go new file mode 100644 index 0000000000..61ac39bbf3 --- /dev/null +++ b/github.com/jcmturner/rpc/v2/mstypes/kerb_sid_and_attributes.go @@ -0,0 +1,23 @@ +package mstypes + +// Attributes of a security group membership and can be combined by using the bitwise OR operation. +// They are used by an access check mechanism to specify whether the membership is to be used in an access check decision. +const ( + SEGroupMandatory = 31 + SEGroupEnabledByDefault = 30 + SEGroupEnabled = 29 + SEGroupOwner = 28 + SEGroupResource = 2 + //All other bits MUST be set to zero and MUST be ignored on receipt. +) + +// KerbSidAndAttributes implements https://msdn.microsoft.com/en-us/library/cc237947.aspx +type KerbSidAndAttributes struct { + SID RPCSID `ndr:"pointer"` // A pointer to an RPC_SID structure. + Attributes uint32 +} + +// SetFlag sets a flag in a uint32 attribute value. +func SetFlag(a *uint32, i uint) { + *a = *a | (1 << (31 - i)) +} diff --git a/github.com/jcmturner/rpc/v2/mstypes/reader.go b/github.com/jcmturner/rpc/v2/mstypes/reader.go new file mode 100644 index 0000000000..24495bca0a --- /dev/null +++ b/github.com/jcmturner/rpc/v2/mstypes/reader.go @@ -0,0 +1,109 @@ +package mstypes + +import ( + "bufio" + "encoding/binary" + "fmt" + "io" +) + +// Byte sizes of primitive types +const ( + SizeBool = 1 + SizeChar = 1 + SizeUint8 = 1 + SizeUint16 = 2 + SizeUint32 = 4 + SizeUint64 = 8 + SizeEnum = 2 + SizeSingle = 4 + SizeDouble = 8 + SizePtr = 4 +) + +// Reader reads simple byte stream data into a Go representations +type Reader struct { + r *bufio.Reader // source of the data +} + +// NewReader creates a new instance of a simple Reader. +func NewReader(r io.Reader) *Reader { + reader := new(Reader) + reader.r = bufio.NewReader(r) + return reader +} + +func (r *Reader) Read(p []byte) (n int, err error) { + return r.r.Read(p) +} + +func (r *Reader) Uint8() (uint8, error) { + b, err := r.r.ReadByte() + if err != nil { + return uint8(0), err + } + return uint8(b), nil +} + +func (r *Reader) Uint16() (uint16, error) { + b, err := r.ReadBytes(SizeUint16) + if err != nil { + return uint16(0), err + } + return binary.LittleEndian.Uint16(b), nil +} + +func (r *Reader) Uint32() (uint32, error) { + b, err := r.ReadBytes(SizeUint32) + if err != nil { + return uint32(0), err + } + return binary.LittleEndian.Uint32(b), nil +} + +func (r *Reader) Uint64() (uint64, error) { + b, err := r.ReadBytes(SizeUint64) + if err != nil { + return uint64(0), err + } + return binary.LittleEndian.Uint64(b), nil +} + +func (r *Reader) FileTime() (f FileTime, err error) { + f.LowDateTime, err = r.Uint32() + if err != nil { + return + } + f.HighDateTime, err = r.Uint32() + if err != nil { + return + } + return +} + +// UTF16String returns a string that is UTF16 encoded in a byte slice. n is the number of bytes representing the string +func (r *Reader) UTF16String(n int) (str string, err error) { + //Length divided by 2 as each run is 16bits = 2bytes + s := make([]rune, n/2, n/2) + for i := 0; i < len(s); i++ { + var u uint16 + u, err = r.Uint16() + if err != nil { + return + } + s[i] = rune(u) + } + str = string(s) + return +} + +// readBytes returns a number of bytes from the NDR byte stream. +func (r *Reader) ReadBytes(n int) ([]byte, error) { + //TODO make this take an int64 as input to allow for larger values on all systems? + b := make([]byte, n, n) + m, err := r.r.Read(b) + if err != nil || m != n { + return b, fmt.Errorf("error reading bytes from stream: %v", err) + } + return b, nil +} diff --git a/github.com/jcmturner/rpc/v2/mstypes/rpc_unicode_string.go b/github.com/jcmturner/rpc/v2/mstypes/rpc_unicode_string.go new file mode 100644 index 0000000000..4bf02e0ed6 --- /dev/null +++ b/github.com/jcmturner/rpc/v2/mstypes/rpc_unicode_string.go @@ -0,0 +1,13 @@ +package mstypes + +// RPCUnicodeString implements https://msdn.microsoft.com/en-us/library/cc230365.aspx +type RPCUnicodeString struct { + Length uint16 // The length, in bytes, of the string pointed to by the Buffer member, not including the terminating null character if any. The length MUST be a multiple of 2. The length SHOULD equal the entire size of the Buffer, in which case there is no terminating null character. Any method that accesses this structure MUST use the Length specified instead of relying on the presence or absence of a null character. + MaximumLength uint16 // The maximum size, in bytes, of the string pointed to by Buffer. The size MUST be a multiple of 2. If not, the size MUST be decremented by 1 prior to use. This value MUST not be less than Length. + Value string `ndr:"pointer,conformant,varying"` +} + +// String returns the RPCUnicodeString string value +func (r *RPCUnicodeString) String() string { + return r.Value +} diff --git a/github.com/jcmturner/rpc/v2/mstypes/sid.go b/github.com/jcmturner/rpc/v2/mstypes/sid.go new file mode 100644 index 0000000000..98a9c5ab8b --- /dev/null +++ b/github.com/jcmturner/rpc/v2/mstypes/sid.go @@ -0,0 +1,32 @@ +package mstypes + +import ( + "encoding/binary" + "encoding/hex" + "fmt" +) + +// RPCSID implements https://msdn.microsoft.com/en-us/library/cc230364.aspx +type RPCSID struct { + Revision uint8 // An 8-bit unsigned integer that specifies the revision level of the SID. This value MUST be set to 0x01. + SubAuthorityCount uint8 // An 8-bit unsigned integer that specifies the number of elements in the SubAuthority array. The maximum number of elements allowed is 15. + IdentifierAuthority [6]byte // An RPC_SID_IDENTIFIER_AUTHORITY structure that indicates the authority under which the SID was created. It describes the entity that created the SID. The Identifier Authority value {0,0,0,0,0,5} denotes SIDs created by the NT SID authority. + SubAuthority []uint32 `ndr:"conformant"` // A variable length array of unsigned 32-bit integers that uniquely identifies a principal relative to the IdentifierAuthority. Its length is determined by SubAuthorityCount. +} + +// String returns the string representation of the RPC_SID. +func (s *RPCSID) String() string { + var str string + b := append(make([]byte, 2, 2), s.IdentifierAuthority[:]...) + // For a strange reason this is read big endian: https://msdn.microsoft.com/en-us/library/dd302645.aspx + i := binary.BigEndian.Uint64(b) + if i >= 4294967296 { + str = fmt.Sprintf("S-1-0x%s", hex.EncodeToString(s.IdentifierAuthority[:])) + } else { + str = fmt.Sprintf("S-1-%d", i) + } + for _, sub := range s.SubAuthority { + str = fmt.Sprintf("%s-%d", str, sub) + } + return str +} diff --git a/github.com/jcmturner/rpc/v2/mstypes/user_session_key.go b/github.com/jcmturner/rpc/v2/mstypes/user_session_key.go new file mode 100644 index 0000000000..fcf0a5d9a6 --- /dev/null +++ b/github.com/jcmturner/rpc/v2/mstypes/user_session_key.go @@ -0,0 +1,11 @@ +package mstypes + +// CypherBlock implements https://msdn.microsoft.com/en-us/library/cc237040.aspx +type CypherBlock struct { + Data [8]byte // size = 8 +} + +// UserSessionKey implements https://msdn.microsoft.com/en-us/library/cc237080.aspx +type UserSessionKey struct { + CypherBlock [2]CypherBlock // size = 2 +} diff --git a/github.com/jcmturner/rpc/v2/ndr/arrays.go b/github.com/jcmturner/rpc/v2/ndr/arrays.go new file mode 100644 index 0000000000..5e2def2a82 --- /dev/null +++ b/github.com/jcmturner/rpc/v2/ndr/arrays.go @@ -0,0 +1,413 @@ +package ndr + +import ( + "errors" + "fmt" + "reflect" + "strconv" +) + +// intFromTag returns an int that is a value in a struct tag key/value pair +func intFromTag(tag reflect.StructTag, key string) (int, error) { + ndrTag := parseTags(tag) + d := 1 + if n, ok := ndrTag.Map[key]; ok { + i, err := strconv.Atoi(n) + if err != nil { + return d, fmt.Errorf("invalid dimensions tag [%s]: %v", n, err) + } + d = i + } + return d, nil +} + +// parseDimensions returns the a slice of the size of each dimension and type of the member at the deepest level. +func parseDimensions(v reflect.Value) (l []int, tb reflect.Type) { + if v.Kind() == reflect.Ptr { + v = v.Elem() + } + t := v.Type() + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + if t.Kind() != reflect.Array && t.Kind() != reflect.Slice { + return + } + l = append(l, v.Len()) + if t.Elem().Kind() == reflect.Array || t.Elem().Kind() == reflect.Slice { + // contains array or slice + var m []int + m, tb = parseDimensions(v.Index(0)) + l = append(l, m...) + } else { + tb = t.Elem() + } + return +} + +// sliceDimensions returns the count of dimensions a slice has. +func sliceDimensions(t reflect.Type) (d int, tb reflect.Type) { + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + if t.Kind() == reflect.Slice { + d++ + var n int + n, tb = sliceDimensions(t.Elem()) + d += n + } else { + tb = t + } + return +} + +// makeSubSlices is a deep recursive creation/initialisation of multi-dimensional slices. +// Takes the reflect.Value of the 1st dimension and a slice of the lengths of the sub dimensions +func makeSubSlices(v reflect.Value, l []int) { + ty := v.Type().Elem() + if ty.Kind() != reflect.Slice { + return + } + for i := 0; i < v.Len(); i++ { + s := reflect.MakeSlice(ty, l[0], l[0]) + v.Index(i).Set(s) + // Are there more sub dimensions? + if len(l) > 1 { + makeSubSlices(v.Index(i), l[1:]) + } + } + return +} + +// multiDimensionalIndexPermutations returns all the permutations of the indexes of a multi-dimensional slice. +// The input is a slice of integers that indicates the max size/length of each dimension +func multiDimensionalIndexPermutations(l []int) (ps [][]int) { + z := make([]int, len(l), len(l)) // The zeros permutation + ps = append(ps, z) + // for each dimension, in reverse + for i := len(l) - 1; i >= 0; i-- { + ws := make([][]int, len(ps)) + copy(ws, ps) + //create a permutation for each of the iterations of the current dimension + for j := 1; j <= l[i]-1; j++ { + // For each existing permutation + for _, p := range ws { + np := make([]int, len(p), len(p)) + copy(np, p) + np[i] = j + ps = append(ps, np) + } + } + } + return +} + +// precedingMax reads off the next conformant max value +func (dec *Decoder) precedingMax() uint32 { + m := dec.conformantMax[0] + dec.conformantMax = dec.conformantMax[1:] + return m +} + +// fillFixedArray establishes if the fixed array is uni or multi dimensional and then fills it. +func (dec *Decoder) fillFixedArray(v reflect.Value, tag reflect.StructTag, def *[]deferedPtr) error { + l, t := parseDimensions(v) + if t.Kind() == reflect.String { + tag = reflect.StructTag(subStringArrayTag) + } + if len(l) < 1 { + return errors.New("could not establish dimensions of fixed array") + } + if len(l) == 1 { + err := dec.fillUniDimensionalFixedArray(v, tag, def) + if err != nil { + return fmt.Errorf("could not fill uni-dimensional fixed array: %v", err) + } + return nil + } + // Fixed array is multidimensional + ps := multiDimensionalIndexPermutations(l[:len(l)-1]) + for _, p := range ps { + // Get current multi-dimensional index to fill + a := v + for _, i := range p { + a = a.Index(i) + } + // fill with the last dimension array + err := dec.fillUniDimensionalFixedArray(a, tag, def) + if err != nil { + return fmt.Errorf("could not fill dimension %v of multi-dimensional fixed array: %v", p, err) + } + } + return nil +} + +// readUniDimensionalFixedArray reads an array (not slice) from the byte stream. +func (dec *Decoder) fillUniDimensionalFixedArray(v reflect.Value, tag reflect.StructTag, def *[]deferedPtr) error { + for i := 0; i < v.Len(); i++ { + err := dec.fill(v.Index(i), tag, def) + if err != nil { + return fmt.Errorf("could not fill index %d of fixed array: %v", i, err) + } + } + return nil +} + +// fillConformantArray establishes if the conformant array is uni or multi dimensional and then fills the slice. +func (dec *Decoder) fillConformantArray(v reflect.Value, tag reflect.StructTag, def *[]deferedPtr) error { + d, _ := sliceDimensions(v.Type()) + if d > 1 { + err := dec.fillMultiDimensionalConformantArray(v, d, tag, def) + if err != nil { + return err + } + } else { + err := dec.fillUniDimensionalConformantArray(v, tag, def) + if err != nil { + return err + } + } + return nil +} + +// fillUniDimensionalConformantArray fills the uni-dimensional slice value. +func (dec *Decoder) fillUniDimensionalConformantArray(v reflect.Value, tag reflect.StructTag, def *[]deferedPtr) error { + m := dec.precedingMax() + n := int(m) + a := reflect.MakeSlice(v.Type(), n, n) + for i := 0; i < n; i++ { + err := dec.fill(a.Index(i), tag, def) + if err != nil { + return fmt.Errorf("could not fill index %d of uni-dimensional conformant array: %v", i, err) + } + } + v.Set(a) + return nil +} + +// fillMultiDimensionalConformantArray fills the multi-dimensional slice value provided from conformant array data. +// The number of dimensions must be specified. This must be less than or equal to the dimensions in the slice for this +// method not to panic. +func (dec *Decoder) fillMultiDimensionalConformantArray(v reflect.Value, d int, tag reflect.StructTag, def *[]deferedPtr) error { + // Read the max size of each dimensions from the ndr stream + l := make([]int, d, d) + for i := range l { + l[i] = int(dec.precedingMax()) + } + // Initialise size of slices + // Initialise the size of the 1st dimension + ty := v.Type() + v.Set(reflect.MakeSlice(ty, l[0], l[0])) + // Initialise the size of the other dimensions recursively + makeSubSlices(v, l[1:]) + + // Get all permutations of the indexes and go through each and fill + ps := multiDimensionalIndexPermutations(l) + for _, p := range ps { + // Get current multi-dimensional index to fill + a := v + for _, i := range p { + a = a.Index(i) + } + err := dec.fill(a, tag, def) + if err != nil { + return fmt.Errorf("could not fill index %v of slice: %v", p, err) + } + } + return nil +} + +// fillVaryingArray establishes if the varying array is uni or multi dimensional and then fills the slice. +func (dec *Decoder) fillVaryingArray(v reflect.Value, tag reflect.StructTag, def *[]deferedPtr) error { + d, t := sliceDimensions(v.Type()) + if d > 1 { + err := dec.fillMultiDimensionalVaryingArray(v, t, d, tag, def) + if err != nil { + return err + } + } else { + err := dec.fillUniDimensionalVaryingArray(v, tag, def) + if err != nil { + return err + } + } + return nil +} + +// fillUniDimensionalVaryingArray fills the uni-dimensional slice value. +func (dec *Decoder) fillUniDimensionalVaryingArray(v reflect.Value, tag reflect.StructTag, def *[]deferedPtr) error { + o, err := dec.readUint32() + if err != nil { + return fmt.Errorf("could not read offset of uni-dimensional varying array: %v", err) + } + s, err := dec.readUint32() + if err != nil { + return fmt.Errorf("could not establish actual count of uni-dimensional varying array: %v", err) + } + t := v.Type() + // Total size of the array is the offset in the index being passed plus the actual count of elements being passed. + n := int(s + o) + a := reflect.MakeSlice(t, n, n) + // Populate the array starting at the offset specified + for i := int(o); i < n; i++ { + err := dec.fill(a.Index(i), tag, def) + if err != nil { + return fmt.Errorf("could not fill index %d of uni-dimensional varying array: %v", i, err) + } + } + v.Set(a) + return nil +} + +// fillMultiDimensionalVaryingArray fills the multi-dimensional slice value provided from varying array data. +// The number of dimensions must be specified. This must be less than or equal to the dimensions in the slice for this +// method not to panic. +func (dec *Decoder) fillMultiDimensionalVaryingArray(v reflect.Value, t reflect.Type, d int, tag reflect.StructTag, def *[]deferedPtr) error { + // Read the offset and actual count of each dimensions from the ndr stream + o := make([]int, d, d) + l := make([]int, d, d) + for i := range l { + off, err := dec.readUint32() + if err != nil { + return fmt.Errorf("could not read offset of dimension %d: %v", i+1, err) + } + o[i] = int(off) + s, err := dec.readUint32() + if err != nil { + return fmt.Errorf("could not read size of dimension %d: %v", i+1, err) + } + l[i] = int(s) + int(off) + } + // Initialise size of slices + // Initialise the size of the 1st dimension + ty := v.Type() + v.Set(reflect.MakeSlice(ty, l[0], l[0])) + // Initialise the size of the other dimensions recursively + makeSubSlices(v, l[1:]) + + // Get all permutations of the indexes and go through each and fill + ps := multiDimensionalIndexPermutations(l) + for _, p := range ps { + // Get current multi-dimensional index to fill + a := v + var os bool // should this permutation be skipped due to the offset of any of the dimensions? + for i, j := range p { + if j < o[i] { + os = true + break + } + a = a.Index(j) + } + if os { + // This permutation should be skipped as it is less than the offset for one of the dimensions. + continue + } + err := dec.fill(a, tag, def) + if err != nil { + return fmt.Errorf("could not fill index %v of slice: %v", p, err) + } + } + return nil +} + +// fillConformantVaryingArray establishes if the varying array is uni or multi dimensional and then fills the slice. +func (dec *Decoder) fillConformantVaryingArray(v reflect.Value, tag reflect.StructTag, def *[]deferedPtr) error { + d, t := sliceDimensions(v.Type()) + if d > 1 { + err := dec.fillMultiDimensionalConformantVaryingArray(v, t, d, tag, def) + if err != nil { + return err + } + } else { + err := dec.fillUniDimensionalConformantVaryingArray(v, tag, def) + if err != nil { + return err + } + } + return nil +} + +// fillUniDimensionalConformantVaryingArray fills the uni-dimensional slice value. +func (dec *Decoder) fillUniDimensionalConformantVaryingArray(v reflect.Value, tag reflect.StructTag, def *[]deferedPtr) error { + m := dec.precedingMax() + o, err := dec.readUint32() + if err != nil { + return fmt.Errorf("could not read offset of uni-dimensional conformant varying array: %v", err) + } + s, err := dec.readUint32() + if err != nil { + return fmt.Errorf("could not establish actual count of uni-dimensional conformant varying array: %v", err) + } + if m < o+s { + return errors.New("max count is less than the offset plus actual count") + } + t := v.Type() + n := int(s) + a := reflect.MakeSlice(t, n, n) + for i := int(o); i < n; i++ { + err := dec.fill(a.Index(i), tag, def) + if err != nil { + return fmt.Errorf("could not fill index %d of uni-dimensional conformant varying array: %v", i, err) + } + } + v.Set(a) + return nil +} + +// fillMultiDimensionalConformantVaryingArray fills the multi-dimensional slice value provided from conformant varying array data. +// The number of dimensions must be specified. This must be less than or equal to the dimensions in the slice for this +// method not to panic. +func (dec *Decoder) fillMultiDimensionalConformantVaryingArray(v reflect.Value, t reflect.Type, d int, tag reflect.StructTag, def *[]deferedPtr) error { + // Read the offset and actual count of each dimensions from the ndr stream + m := make([]int, d, d) + for i := range m { + m[i] = int(dec.precedingMax()) + } + o := make([]int, d, d) + l := make([]int, d, d) + for i := range l { + off, err := dec.readUint32() + if err != nil { + return fmt.Errorf("could not read offset of dimension %d: %v", i+1, err) + } + o[i] = int(off) + s, err := dec.readUint32() + if err != nil { + return fmt.Errorf("could not read actual count of dimension %d: %v", i+1, err) + } + if m[i] < int(s)+int(off) { + m[i] = int(s) + int(off) + } + l[i] = int(s) + } + // Initialise size of slices + // Initialise the size of the 1st dimension + ty := v.Type() + v.Set(reflect.MakeSlice(ty, m[0], m[0])) + // Initialise the size of the other dimensions recursively + makeSubSlices(v, m[1:]) + + // Get all permutations of the indexes and go through each and fill + ps := multiDimensionalIndexPermutations(m) + for _, p := range ps { + // Get current multi-dimensional index to fill + a := v + var os bool // should this permutation be skipped due to the offset of any of the dimensions or max is higher than the actual count being passed + for i, j := range p { + if j < o[i] || j >= l[i] { + os = true + break + } + a = a.Index(j) + } + if os { + // This permutation should be skipped as it is less than the offset for one of the dimensions. + continue + } + err := dec.fill(a, tag, def) + if err != nil { + return fmt.Errorf("could not fill index %v of slice: %v", p, err) + } + } + return nil +} diff --git a/github.com/jcmturner/rpc/v2/ndr/decoder.go b/github.com/jcmturner/rpc/v2/ndr/decoder.go new file mode 100644 index 0000000000..6157b4ef9e --- /dev/null +++ b/github.com/jcmturner/rpc/v2/ndr/decoder.go @@ -0,0 +1,393 @@ +// Package ndr provides the ability to unmarshal NDR encoded byte steams into Go data structures +package ndr + +import ( + "bufio" + "fmt" + "io" + "reflect" + "strings" +) + +// Struct tag values +const ( + TagConformant = "conformant" + TagVarying = "varying" + TagPointer = "pointer" + TagPipe = "pipe" +) + +// Decoder unmarshals NDR byte stream data into a Go struct representation +type Decoder struct { + r *bufio.Reader // source of the data + size int // initial size of bytes in buffer + ch CommonHeader // NDR common header + ph PrivateHeader // NDR private header + conformantMax []uint32 // conformant max values that were moved to the beginning of the structure + s interface{} // pointer to the structure being populated + current []string // keeps track of the current field being populated +} + +type deferedPtr struct { + v reflect.Value + tag reflect.StructTag +} + +// NewDecoder creates a new instance of a NDR Decoder. +func NewDecoder(r io.Reader) *Decoder { + dec := new(Decoder) + dec.r = bufio.NewReader(r) + dec.r.Peek(int(commonHeaderBytes)) // For some reason an operation is needed on the buffer to initialise it so Buffered() != 0 + dec.size = dec.r.Buffered() + return dec +} + +// Decode unmarshals the NDR encoded bytes into the pointer of a struct provided. +func (dec *Decoder) Decode(s interface{}) error { + dec.s = s + err := dec.readCommonHeader() + if err != nil { + return err + } + err = dec.readPrivateHeader() + if err != nil { + return err + } + _, err = dec.r.Discard(4) //The next 4 bytes are an RPC unique pointer referent. We just skip these. + if err != nil { + return Errorf("unable to process byte stream: %v", err) + } + + return dec.process(s, reflect.StructTag("")) +} + +func (dec *Decoder) process(s interface{}, tag reflect.StructTag) error { + // Scan for conformant fields as their max counts are moved to the beginning + // http://pubs.opengroup.org/onlinepubs/9629399/chap14.htm#tagfcjh_37 + err := dec.scanConformantArrays(s, tag) + if err != nil { + return err + } + // Recursively fill the struct fields + var localDef []deferedPtr + err = dec.fill(s, tag, &localDef) + if err != nil { + return Errorf("could not decode: %v", err) + } + // Read any deferred referents associated with pointers + for _, p := range localDef { + err = dec.process(p.v, p.tag) + if err != nil { + return fmt.Errorf("could not decode deferred referent: %v", err) + } + } + return nil +} + +// scanConformantArrays scans the structure for embedded conformant fields and captures the maximum element counts for +// dimensions of the array that are moved to the beginning of the structure. +func (dec *Decoder) scanConformantArrays(s interface{}, tag reflect.StructTag) error { + err := dec.conformantScan(s, tag) + if err != nil { + return fmt.Errorf("failed to scan for embedded conformant arrays: %v", err) + } + for i := range dec.conformantMax { + dec.conformantMax[i], err = dec.readUint32() + if err != nil { + return fmt.Errorf("could not read preceding conformant max count index %d: %v", i, err) + } + } + return nil +} + +// conformantScan inspects the structure's fields for whether they are conformant. +func (dec *Decoder) conformantScan(s interface{}, tag reflect.StructTag) error { + ndrTag := parseTags(tag) + if ndrTag.HasValue(TagPointer) { + return nil + } + v := getReflectValue(s) + switch v.Kind() { + case reflect.Struct: + for i := 0; i < v.NumField(); i++ { + err := dec.conformantScan(v.Field(i), v.Type().Field(i).Tag) + if err != nil { + return err + } + } + case reflect.String: + if !ndrTag.HasValue(TagConformant) { + break + } + dec.conformantMax = append(dec.conformantMax, uint32(0)) + case reflect.Slice: + if !ndrTag.HasValue(TagConformant) { + break + } + d, t := sliceDimensions(v.Type()) + for i := 0; i < d; i++ { + dec.conformantMax = append(dec.conformantMax, uint32(0)) + } + // For string arrays there is a common max for the strings within the array. + if t.Kind() == reflect.String { + dec.conformantMax = append(dec.conformantMax, uint32(0)) + } + } + return nil +} + +func (dec *Decoder) isPointer(v reflect.Value, tag reflect.StructTag, def *[]deferedPtr) (bool, error) { + // Pointer so defer filling the referent + ndrTag := parseTags(tag) + if ndrTag.HasValue(TagPointer) { + p, err := dec.readUint32() + if err != nil { + return true, fmt.Errorf("could not read pointer: %v", err) + } + ndrTag.delete(TagPointer) + if p != 0 { + // if pointer is not zero add to the deferred items at end of stream + *def = append(*def, deferedPtr{v, ndrTag.StructTag()}) + } + return true, nil + } + return false, nil +} + +func getReflectValue(s interface{}) (v reflect.Value) { + if r, ok := s.(reflect.Value); ok { + v = r + } else { + if reflect.ValueOf(s).Kind() == reflect.Ptr { + v = reflect.ValueOf(s).Elem() + } + } + return +} + +// fill populates fields with values from the NDR byte stream. +func (dec *Decoder) fill(s interface{}, tag reflect.StructTag, localDef *[]deferedPtr) error { + v := getReflectValue(s) + + //// Pointer so defer filling the referent + ptr, err := dec.isPointer(v, tag, localDef) + if err != nil { + return fmt.Errorf("could not process struct field(%s): %v", strings.Join(dec.current, "/"), err) + } + if ptr { + return nil + } + + // Populate the value from the byte stream + switch v.Kind() { + case reflect.Struct: + dec.current = append(dec.current, v.Type().Name()) //Track the current field being filled + // in case struct is a union, track this and the selected union field for efficiency + var unionTag reflect.Value + var unionField string // field to fill if struct is a union + // Go through each field in the struct and recursively fill + for i := 0; i < v.NumField(); i++ { + fieldName := v.Type().Field(i).Name + dec.current = append(dec.current, fieldName) //Track the current field being filled + //fmt.Fprintf(os.Stderr, "DEBUG Decoding: %s\n", strings.Join(dec.current, "/")) + structTag := v.Type().Field(i).Tag + ndrTag := parseTags(structTag) + + // Union handling + if !unionTag.IsValid() { + // Is this field a union tag? + unionTag = dec.isUnion(v.Field(i), structTag) + } else { + // What is the selected field value of the union if we don't already know + if unionField == "" { + unionField, err = unionSelectedField(v, unionTag) + if err != nil { + return fmt.Errorf("could not determine selected union value field for %s with discriminat"+ + " tag %s: %v", v.Type().Name(), unionTag, err) + } + } + if ndrTag.HasValue(TagUnionField) && fieldName != unionField { + // is a union and this field has not been selected so will skip it. + dec.current = dec.current[:len(dec.current)-1] //This field has been skipped so remove it from the current field tracker + continue + } + } + + // Check if field is a pointer + if v.Field(i).Type().Implements(reflect.TypeOf(new(RawBytes)).Elem()) && + v.Field(i).Type().Kind() == reflect.Slice && v.Field(i).Type().Elem().Kind() == reflect.Uint8 { + //field is for rawbytes + structTag, err = addSizeToTag(v, v.Field(i), structTag) + if err != nil { + return fmt.Errorf("could not get rawbytes field(%s) size: %v", strings.Join(dec.current, "/"), err) + } + ptr, err := dec.isPointer(v.Field(i), structTag, localDef) + if err != nil { + return fmt.Errorf("could not process struct field(%s): %v", strings.Join(dec.current, "/"), err) + } + if !ptr { + err := dec.readRawBytes(v.Field(i), structTag) + if err != nil { + return fmt.Errorf("could not fill raw bytes struct field(%s): %v", strings.Join(dec.current, "/"), err) + } + } + } else { + err := dec.fill(v.Field(i), structTag, localDef) + if err != nil { + return fmt.Errorf("could not fill struct field(%s): %v", strings.Join(dec.current, "/"), err) + } + } + dec.current = dec.current[:len(dec.current)-1] //This field has been filled so remove it from the current field tracker + } + dec.current = dec.current[:len(dec.current)-1] //This field has been filled so remove it from the current field tracker + case reflect.Bool: + i, err := dec.readBool() + if err != nil { + return fmt.Errorf("could not fill %s: %v", v.Type().Name(), err) + } + v.Set(reflect.ValueOf(i)) + case reflect.Uint8: + i, err := dec.readUint8() + if err != nil { + return fmt.Errorf("could not fill %s: %v", v.Type().Name(), err) + } + v.Set(reflect.ValueOf(i)) + case reflect.Uint16: + i, err := dec.readUint16() + if err != nil { + return fmt.Errorf("could not fill %s: %v", v.Type().Name(), err) + } + v.Set(reflect.ValueOf(i)) + case reflect.Uint32: + i, err := dec.readUint32() + if err != nil { + return fmt.Errorf("could not fill %s: %v", v.Type().Name(), err) + } + v.Set(reflect.ValueOf(i)) + case reflect.Uint64: + i, err := dec.readUint64() + if err != nil { + return fmt.Errorf("could not fill %s: %v", v.Type().Name(), err) + } + v.Set(reflect.ValueOf(i)) + case reflect.Int8: + i, err := dec.readInt8() + if err != nil { + return fmt.Errorf("could not fill %s: %v", v.Type().Name(), err) + } + v.Set(reflect.ValueOf(i)) + case reflect.Int16: + i, err := dec.readInt16() + if err != nil { + return fmt.Errorf("could not fill %s: %v", v.Type().Name(), err) + } + v.Set(reflect.ValueOf(i)) + case reflect.Int32: + i, err := dec.readInt32() + if err != nil { + return fmt.Errorf("could not fill %s: %v", v.Type().Name(), err) + } + v.Set(reflect.ValueOf(i)) + case reflect.Int64: + i, err := dec.readInt64() + if err != nil { + return fmt.Errorf("could not fill %s: %v", v.Type().Name(), err) + } + v.Set(reflect.ValueOf(i)) + case reflect.String: + ndrTag := parseTags(tag) + conformant := ndrTag.HasValue(TagConformant) + // strings are always varying so this is assumed without an explicit tag + var s string + var err error + if conformant { + s, err = dec.readConformantVaryingString(localDef) + if err != nil { + return fmt.Errorf("could not fill with conformant varying string: %v", err) + } + } else { + s, err = dec.readVaryingString(localDef) + if err != nil { + return fmt.Errorf("could not fill with varying string: %v", err) + } + } + v.Set(reflect.ValueOf(s)) + case reflect.Float32: + i, err := dec.readFloat32() + if err != nil { + return fmt.Errorf("could not fill %v: %v", v.Type().Name(), err) + } + v.Set(reflect.ValueOf(i)) + case reflect.Float64: + i, err := dec.readFloat64() + if err != nil { + return fmt.Errorf("could not fill %v: %v", v.Type().Name(), err) + } + v.Set(reflect.ValueOf(i)) + case reflect.Array: + err := dec.fillFixedArray(v, tag, localDef) + if err != nil { + return err + } + case reflect.Slice: + if v.Type().Implements(reflect.TypeOf(new(RawBytes)).Elem()) && v.Type().Elem().Kind() == reflect.Uint8 { + //field is for rawbytes + err := dec.readRawBytes(v, tag) + if err != nil { + return fmt.Errorf("could not fill raw bytes struct field(%s): %v", strings.Join(dec.current, "/"), err) + } + break + } + ndrTag := parseTags(tag) + conformant := ndrTag.HasValue(TagConformant) + varying := ndrTag.HasValue(TagVarying) + if ndrTag.HasValue(TagPipe) { + err := dec.fillPipe(v, tag) + if err != nil { + return err + } + break + } + _, t := sliceDimensions(v.Type()) + if t.Kind() == reflect.String && !ndrTag.HasValue(subStringArrayValue) { + // String array + err := dec.readStringsArray(v, tag, localDef) + if err != nil { + return err + } + break + } + // varying is assumed as fixed arrays use the Go array type rather than slice + if conformant && varying { + err := dec.fillConformantVaryingArray(v, tag, localDef) + if err != nil { + return err + } + } else if !conformant && varying { + err := dec.fillVaryingArray(v, tag, localDef) + if err != nil { + return err + } + } else { + //default to conformant and not varying + err := dec.fillConformantArray(v, tag, localDef) + if err != nil { + return err + } + } + default: + return fmt.Errorf("unsupported type") + } + return nil +} + +// readBytes returns a number of bytes from the NDR byte stream. +func (dec *Decoder) readBytes(n int) ([]byte, error) { + //TODO make this take an int64 as input to allow for larger values on all systems? + b := make([]byte, n, n) + m, err := dec.r.Read(b) + if err != nil || m != n { + return b, fmt.Errorf("error reading bytes from stream: %v", err) + } + return b, nil +} diff --git a/github.com/jcmturner/rpc/v2/ndr/error.go b/github.com/jcmturner/rpc/v2/ndr/error.go new file mode 100644 index 0000000000..9971194d0e --- /dev/null +++ b/github.com/jcmturner/rpc/v2/ndr/error.go @@ -0,0 +1,18 @@ +package ndr + +import "fmt" + +// Malformed implements the error interface for malformed NDR encoding errors. +type Malformed struct { + EText string +} + +// Error implements the error interface on the Malformed struct. +func (e Malformed) Error() string { + return fmt.Sprintf("malformed NDR stream: %s", e.EText) +} + +// Errorf formats an error message into a malformed NDR error. +func Errorf(format string, a ...interface{}) Malformed { + return Malformed{EText: fmt.Sprintf(format, a...)} +} diff --git a/github.com/jcmturner/rpc/v2/ndr/header.go b/github.com/jcmturner/rpc/v2/ndr/header.go new file mode 100644 index 0000000000..1970ddb600 --- /dev/null +++ b/github.com/jcmturner/rpc/v2/ndr/header.go @@ -0,0 +1,116 @@ +package ndr + +import ( + "encoding/binary" + "fmt" +) + +/* +Serialization Version 1 +https://msdn.microsoft.com/en-us/library/cc243563.aspx + +Common Header - https://msdn.microsoft.com/en-us/library/cc243890.aspx +8 bytes in total: +- First byte - Version: Must equal 1 +- Second byte - 1st 4 bits: Endianess (0=Big; 1=Little); 2nd 4 bits: Character Encoding (0=ASCII; 1=EBCDIC) +- 3rd - Floating point representation (This does not seem to be the case in examples for Microsoft test sources) +- 4th - Common Header Length: Must equal 8 +- 5th - 8th - Filler: MUST be set to 0xcccccccc on marshaling, and SHOULD be ignored during unmarshaling. + +Private Header - https://msdn.microsoft.com/en-us/library/cc243919.aspx +8 bytes in total: +- First 4 bytes - Indicates the length of a serialized top-level type in the octet stream. It MUST include the padding length and exclude the header itself. +- Second 4 bytes - Filler: MUST be set to 0 (zero) during marshaling, and SHOULD be ignored during unmarshaling. +*/ + +const ( + protocolVersion uint8 = 1 + commonHeaderBytes uint16 = 8 + bigEndian = 0 + littleEndian = 1 + ascii uint8 = 0 + ebcdic uint8 = 1 + ieee uint8 = 0 + vax uint8 = 1 + cray uint8 = 2 + ibm uint8 = 3 +) + +// CommonHeader implements the NDR common header: https://msdn.microsoft.com/en-us/library/cc243889.aspx +type CommonHeader struct { + Version uint8 + Endianness binary.ByteOrder + CharacterEncoding uint8 + FloatRepresentation uint8 + HeaderLength uint16 + Filler []byte +} + +// PrivateHeader implements the NDR private header: https://msdn.microsoft.com/en-us/library/cc243919.aspx +type PrivateHeader struct { + ObjectBufferLength uint32 + Filler []byte +} + +func (dec *Decoder) readCommonHeader() error { + // Version + vb, err := dec.r.ReadByte() + if err != nil { + return Malformed{EText: "could not read first byte of common header for version"} + } + dec.ch.Version = uint8(vb) + if dec.ch.Version != protocolVersion { + return Malformed{EText: fmt.Sprintf("byte stream does not indicate a RPC Type serialization of version %v", protocolVersion)} + } + // Read Endianness & Character Encoding + eb, err := dec.r.ReadByte() + if err != nil { + return Malformed{EText: "could not read second byte of common header for endianness"} + } + endian := int(eb >> 4 & 0xF) + if endian != 0 && endian != 1 { + return Malformed{EText: "common header does not indicate a valid endianness"} + } + dec.ch.CharacterEncoding = uint8(vb & 0xF) + if dec.ch.CharacterEncoding != 0 && dec.ch.CharacterEncoding != 1 { + return Malformed{EText: "common header does not indicate a valid character encoding"} + } + switch endian { + case littleEndian: + dec.ch.Endianness = binary.LittleEndian + case bigEndian: + dec.ch.Endianness = binary.BigEndian + } + // Common header length + lb, err := dec.readBytes(2) + if err != nil { + return Malformed{EText: fmt.Sprintf("could not read common header length: %v", err)} + } + dec.ch.HeaderLength = dec.ch.Endianness.Uint16(lb) + if dec.ch.HeaderLength != commonHeaderBytes { + return Malformed{EText: "common header does not indicate a valid length"} + } + // Filler bytes + dec.ch.Filler, err = dec.readBytes(4) + if err != nil { + return Malformed{EText: fmt.Sprintf("could not read common header filler: %v", err)} + } + return nil +} + +func (dec *Decoder) readPrivateHeader() error { + // The next 8 bytes after the common header comprise the RPC type marshalling private header for constructed types. + err := binary.Read(dec.r, dec.ch.Endianness, &dec.ph.ObjectBufferLength) + if err != nil { + return Malformed{EText: "could not read private header object buffer length"} + } + if dec.ph.ObjectBufferLength%8 != 0 { + return Malformed{EText: "object buffer length not a multiple of 8"} + } + // Filler bytes + dec.ph.Filler, err = dec.readBytes(4) + if err != nil { + return Malformed{EText: fmt.Sprintf("could not read private header filler: %v", err)} + } + return nil +} diff --git a/github.com/jcmturner/rpc/v2/ndr/pipe.go b/github.com/jcmturner/rpc/v2/ndr/pipe.go new file mode 100644 index 0000000000..5fd27da019 --- /dev/null +++ b/github.com/jcmturner/rpc/v2/ndr/pipe.go @@ -0,0 +1,31 @@ +package ndr + +import ( + "fmt" + "reflect" +) + +func (dec *Decoder) fillPipe(v reflect.Value, tag reflect.StructTag) error { + s, err := dec.readUint32() // read element count of first chunk + if err != nil { + return err + } + a := reflect.MakeSlice(v.Type(), 0, 0) + c := reflect.MakeSlice(v.Type(), int(s), int(s)) + for s != 0 { + for i := 0; i < int(s); i++ { + err := dec.fill(c.Index(i), tag, &[]deferedPtr{}) + if err != nil { + return fmt.Errorf("could not fill element %d of pipe: %v", i, err) + } + } + s, err = dec.readUint32() // read element count of first chunk + if err != nil { + return err + } + a = reflect.AppendSlice(a, c) + c = reflect.MakeSlice(v.Type(), int(s), int(s)) + } + v.Set(a) + return nil +} diff --git a/github.com/jcmturner/rpc/v2/ndr/primitives.go b/github.com/jcmturner/rpc/v2/ndr/primitives.go new file mode 100644 index 0000000000..7eb1d1afbf --- /dev/null +++ b/github.com/jcmturner/rpc/v2/ndr/primitives.go @@ -0,0 +1,211 @@ +package ndr + +import ( + "bytes" + "encoding/binary" + "math" +) + +// Byte sizes of primitive types +const ( + SizeBool = 1 + SizeChar = 1 + SizeUint8 = 1 + SizeUint16 = 2 + SizeUint32 = 4 + SizeUint64 = 8 + SizeEnum = 2 + SizeSingle = 4 + SizeDouble = 8 + SizePtr = 4 +) + +// Bool is an NDR Boolean which is a logical quantity that assumes one of two values: TRUE or FALSE. +// NDR represents a Boolean as one octet. +// It represents a value of FALSE as a zero octet, an octet in which every bit is reset. +// It represents a value of TRUE as a non-zero octet, an octet in which one or more bits are set. + +// Char is an NDR character. +// NDR represents a character as one octet. +// Characters have two representation formats: ASCII and EBCDIC. + +// USmall is an unsigned 8 bit integer + +// UShort is an unsigned 16 bit integer + +// ULong is an unsigned 32 bit integer + +// UHyper is an unsigned 64 bit integer + +// Small is an signed 8 bit integer + +// Short is an signed 16 bit integer + +// Long is an signed 32 bit integer + +// Hyper is an signed 64 bit integer + +// Enum is the NDR representation of enumerated types as signed short integers (2 octets) + +// Single is an NDR defined single-precision floating-point data type + +// Double is an NDR defined double-precision floating-point data type + +// readBool reads a byte representing a boolean. +// NDR represents a Boolean as one octet. +// It represents a value of FALSE as a zero octet, an octet in which every bit is reset. +// It represents a value of TRUE as a non-zero octet, an octet in which one or more bits are set. +func (dec *Decoder) readBool() (bool, error) { + i, err := dec.readUint8() + if err != nil { + return false, err + } + if i != 0 { + return true, nil + } + return false, nil +} + +// readChar reads bytes representing a 8bit ASCII integer cast to a rune. +func (dec *Decoder) readChar() (rune, error) { + var r rune + a, err := dec.readUint8() + if err != nil { + return r, err + } + return rune(a), nil +} + +// readUint8 reads bytes representing a 8bit unsigned integer. +func (dec *Decoder) readUint8() (uint8, error) { + b, err := dec.r.ReadByte() + if err != nil { + return uint8(0), err + } + return uint8(b), nil +} + +// readUint16 reads bytes representing a 16bit unsigned integer. +func (dec *Decoder) readUint16() (uint16, error) { + dec.ensureAlignment(SizeUint16) + b, err := dec.readBytes(SizeUint16) + if err != nil { + return uint16(0), err + } + return dec.ch.Endianness.Uint16(b), nil +} + +// readUint32 reads bytes representing a 32bit unsigned integer. +func (dec *Decoder) readUint32() (uint32, error) { + dec.ensureAlignment(SizeUint32) + b, err := dec.readBytes(SizeUint32) + if err != nil { + return uint32(0), err + } + return dec.ch.Endianness.Uint32(b), nil +} + +// readUint32 reads bytes representing a 32bit unsigned integer. +func (dec *Decoder) readUint64() (uint64, error) { + dec.ensureAlignment(SizeUint64) + b, err := dec.readBytes(SizeUint64) + if err != nil { + return uint64(0), err + } + return dec.ch.Endianness.Uint64(b), nil +} + +func (dec *Decoder) readInt8() (int8, error) { + dec.ensureAlignment(SizeUint8) + b, err := dec.readBytes(SizeUint8) + if err != nil { + return 0, err + } + var i int8 + buf := bytes.NewReader(b) + err = binary.Read(buf, dec.ch.Endianness, &i) + if err != nil { + return 0, err + } + return i, nil +} + +func (dec *Decoder) readInt16() (int16, error) { + dec.ensureAlignment(SizeUint16) + b, err := dec.readBytes(SizeUint16) + if err != nil { + return 0, err + } + var i int16 + buf := bytes.NewReader(b) + err = binary.Read(buf, dec.ch.Endianness, &i) + if err != nil { + return 0, err + } + return i, nil +} + +func (dec *Decoder) readInt32() (int32, error) { + dec.ensureAlignment(SizeUint32) + b, err := dec.readBytes(SizeUint32) + if err != nil { + return 0, err + } + var i int32 + buf := bytes.NewReader(b) + err = binary.Read(buf, dec.ch.Endianness, &i) + if err != nil { + return 0, err + } + return i, nil +} + +func (dec *Decoder) readInt64() (int64, error) { + dec.ensureAlignment(SizeUint64) + b, err := dec.readBytes(SizeUint64) + if err != nil { + return 0, err + } + var i int64 + buf := bytes.NewReader(b) + err = binary.Read(buf, dec.ch.Endianness, &i) + if err != nil { + return 0, err + } + return i, nil +} + +// https://en.wikipedia.org/wiki/IEEE_754-1985 +func (dec *Decoder) readFloat32() (f float32, err error) { + dec.ensureAlignment(SizeSingle) + b, err := dec.readBytes(SizeSingle) + if err != nil { + return + } + bits := dec.ch.Endianness.Uint32(b) + f = math.Float32frombits(bits) + return +} + +func (dec *Decoder) readFloat64() (f float64, err error) { + dec.ensureAlignment(SizeDouble) + b, err := dec.readBytes(SizeDouble) + if err != nil { + return + } + bits := dec.ch.Endianness.Uint64(b) + f = math.Float64frombits(bits) + return +} + +// NDR enforces NDR alignment of primitive data; that is, any primitive of size n octets is aligned at a octet stream +// index that is a multiple of n. (In this version of NDR, n is one of {1, 2, 4, 8}.) An octet stream index indicates +// the number of an octet in an octet stream when octets are numbered, beginning with 0, from the first octet in the +// stream. Where necessary, an alignment gap, consisting of octets of unspecified value, precedes the representation +// of a primitive. The gap is of the smallest size sufficient to align the primitive. +func (dec *Decoder) ensureAlignment(n int) { + p := dec.size - dec.r.Buffered() + if s := p % n; s != 0 { + dec.r.Discard(n - s) + } +} diff --git a/github.com/jcmturner/rpc/v2/ndr/rawbytes.go b/github.com/jcmturner/rpc/v2/ndr/rawbytes.go new file mode 100644 index 0000000000..9ee59fb10e --- /dev/null +++ b/github.com/jcmturner/rpc/v2/ndr/rawbytes.go @@ -0,0 +1,61 @@ +package ndr + +import ( + "errors" + "fmt" + "reflect" + "strconv" +) + +// type MyBytes []byte +// implement RawBytes interface + +const ( + sizeMethod = "Size" +) + +// RawBytes interface should be implemented if reading just a number of bytes from the NDR stream +type RawBytes interface { + Size(interface{}) int +} + +func rawBytesSize(parent reflect.Value, v reflect.Value) (int, error) { + sf := v.MethodByName(sizeMethod) + if !sf.IsValid() { + return 0, fmt.Errorf("could not find a method called %s on the implementation of RawBytes", sizeMethod) + } + in := []reflect.Value{parent} + f := sf.Call(in) + if f[0].Kind() != reflect.Int { + return 0, errors.New("the RawBytes size function did not return an integer") + } + return int(f[0].Int()), nil +} + +func addSizeToTag(parent reflect.Value, v reflect.Value, tag reflect.StructTag) (reflect.StructTag, error) { + size, err := rawBytesSize(parent, v) + if err != nil { + return tag, err + } + ndrTag := parseTags(tag) + ndrTag.Map["size"] = strconv.Itoa(size) + return ndrTag.StructTag(), nil +} + +func (dec *Decoder) readRawBytes(v reflect.Value, tag reflect.StructTag) error { + ndrTag := parseTags(tag) + sizeStr, ok := ndrTag.Map["size"] + if !ok { + return errors.New("size tag not available") + } + size, err := strconv.Atoi(sizeStr) + if err != nil { + return fmt.Errorf("size not valid: %v", err) + } + b, err := dec.readBytes(size) + if err != nil { + return err + } + v.Set(reflect.ValueOf(b).Convert(v.Type())) + return nil +} diff --git a/github.com/jcmturner/rpc/v2/ndr/strings.go b/github.com/jcmturner/rpc/v2/ndr/strings.go new file mode 100644 index 0000000000..b7a910b3d1 --- /dev/null +++ b/github.com/jcmturner/rpc/v2/ndr/strings.go @@ -0,0 +1,70 @@ +package ndr + +import ( + "fmt" + "reflect" +) + +const ( + subStringArrayTag = `ndr:"varying,X-subStringArray"` + subStringArrayValue = "X-subStringArray" +) + +func uint16SliceToString(a []uint16) string { + s := make([]rune, len(a), len(a)) + for i := range s { + s[i] = rune(a[i]) + } + if len(s) > 0 { + // Remove any null terminator + if s[len(s)-1] == rune(0) { + s = s[:len(s)-1] + } + } + return string(s) +} + +func (dec *Decoder) readVaryingString(def *[]deferedPtr) (string, error) { + a := new([]uint16) + v := reflect.ValueOf(a) + var t reflect.StructTag + err := dec.fillUniDimensionalVaryingArray(v.Elem(), t, def) + if err != nil { + return "", err + } + s := uint16SliceToString(*a) + return s, nil +} + +func (dec *Decoder) readConformantVaryingString(def *[]deferedPtr) (string, error) { + a := new([]uint16) + v := reflect.ValueOf(a) + var t reflect.StructTag + err := dec.fillUniDimensionalConformantVaryingArray(v.Elem(), t, def) + if err != nil { + return "", err + } + s := uint16SliceToString(*a) + return s, nil +} + +func (dec *Decoder) readStringsArray(v reflect.Value, tag reflect.StructTag, def *[]deferedPtr) error { + d, _ := sliceDimensions(v.Type()) + ndrTag := parseTags(tag) + var m []int + //var ms int + if ndrTag.HasValue(TagConformant) { + for i := 0; i < d; i++ { + m = append(m, int(dec.precedingMax())) + } + //common max size + _ = dec.precedingMax() + //ms = int(n) + } + tag = reflect.StructTag(subStringArrayTag) + err := dec.fillVaryingArray(v, tag, def) + if err != nil { + return fmt.Errorf("could not read string array: %v", err) + } + return nil +} diff --git a/github.com/jcmturner/rpc/v2/ndr/tags.go b/github.com/jcmturner/rpc/v2/ndr/tags.go new file mode 100644 index 0000000000..01657e0a1d --- /dev/null +++ b/github.com/jcmturner/rpc/v2/ndr/tags.go @@ -0,0 +1,69 @@ +package ndr + +import ( + "fmt" + "reflect" + "strings" +) + +const ndrNameSpace = "ndr" + +type tags struct { + Values []string + Map map[string]string +} + +// parse the struct field tags and extract the ndr related ones. +// format of tag ndr:"value,key:value1,value2" +func parseTags(st reflect.StructTag) tags { + s := st.Get(ndrNameSpace) + t := tags{ + Values: []string{}, + Map: make(map[string]string), + } + if s != "" { + ndrTags := strings.Trim(s, `"`) + for _, tag := range strings.Split(ndrTags, ",") { + if strings.Contains(tag, ":") { + m := strings.SplitN(tag, ":", 2) + t.Map[m[0]] = m[1] + } else { + t.Values = append(t.Values, tag) + } + } + } + return t +} + +func appendTag(t reflect.StructTag, s string) reflect.StructTag { + ts := t.Get(ndrNameSpace) + ts = fmt.Sprintf(`%s"%s,%s"`, ndrNameSpace, ts, s) + return reflect.StructTag(ts) +} + +func (t *tags) StructTag() reflect.StructTag { + mv := t.Values + for key, val := range t.Map { + mv = append(mv, key+":"+val) + } + s := ndrNameSpace + ":" + `"` + strings.Join(mv, ",") + `"` + return reflect.StructTag(s) +} + +func (t *tags) delete(s string) { + for i, x := range t.Values { + if x == s { + t.Values = append(t.Values[:i], t.Values[i+1:]...) + } + } + delete(t.Map, s) +} + +func (t *tags) HasValue(s string) bool { + for _, v := range t.Values { + if v == s { + return true + } + } + return false +} diff --git a/github.com/jcmturner/rpc/v2/ndr/union.go b/github.com/jcmturner/rpc/v2/ndr/union.go new file mode 100644 index 0000000000..6a657fa6f6 --- /dev/null +++ b/github.com/jcmturner/rpc/v2/ndr/union.go @@ -0,0 +1,57 @@ +package ndr + +import ( + "errors" + "fmt" + "reflect" +) + +// Union interface must be implemented by structs that will be unmarshaled into from the NDR byte stream union representation. +// The union's discriminating tag will be passed to the SwitchFunc method. +// The discriminating tag field must have the struct tag: `ndr:"unionTag"` +// If the union is encapsulated the discriminating tag field must have the struct tag: `ndr:"encapsulated"` +// The possible value fields that can be selected from must have the struct tag: `ndr:"unionField"` +type Union interface { + SwitchFunc(t interface{}) string +} + +// Union related constants such as struct tag values +const ( + unionSelectionFuncName = "SwitchFunc" + TagEncapsulated = "encapsulated" + TagUnionTag = "unionTag" + TagUnionField = "unionField" +) + +func (dec *Decoder) isUnion(field reflect.Value, tag reflect.StructTag) (r reflect.Value) { + ndrTag := parseTags(tag) + if !ndrTag.HasValue(TagUnionTag) { + return + } + r = field + // For a non-encapsulated union, the discriminant is marshalled into the transmitted data stream twice: once as the + // field or parameter, which is referenced by the switch_is construct, in the procedure argument list; and once as + // the first part of the union representation. + if !ndrTag.HasValue(TagEncapsulated) { + dec.r.Discard(int(r.Type().Size())) + } + return +} + +// unionSelectedField returns the field name of which of the union values to fill +func unionSelectedField(union, discriminant reflect.Value) (string, error) { + if !union.Type().Implements(reflect.TypeOf(new(Union)).Elem()) { + return "", errors.New("struct does not implement union interface") + } + args := []reflect.Value{discriminant} + // Call the SelectFunc of the union struct to find the name of the field to fill with the value selected. + sf := union.MethodByName(unionSelectionFuncName) + if !sf.IsValid() { + return "", fmt.Errorf("could not find a selection function called %s in the unions struct representation", unionSelectionFuncName) + } + f := sf.Call(args) + if f[0].Kind() != reflect.String || f[0].String() == "" { + return "", fmt.Errorf("the union select function did not return a string for the name of the field to fill") + } + return f[0].String(), nil +} diff --git a/github.com/lib/pq/README.md b/github.com/lib/pq/README.md index 16fc31cd26..ecd01939b5 100644 --- a/github.com/lib/pq/README.md +++ b/github.com/lib/pq/README.md @@ -20,6 +20,10 @@ * Notifications: `LISTEN`/`NOTIFY` * pgpass support +## Optional Features + +* GSS (Kerberos) auth (to use, see GoDoc) + ## Tests `go test` is used for testing. See [TESTS.md](TESTS.md) for more details. diff --git a/github.com/lib/pq/auth/kerberos/go.mod b/github.com/lib/pq/auth/kerberos/go.mod new file mode 100644 index 0000000000..d58d6df487 --- /dev/null +++ b/github.com/lib/pq/auth/kerberos/go.mod @@ -0,0 +1,8 @@ +module github.com/lib/pq/auth/kerberos + +go 1.13 + +require ( + github.com/alexbrainman/sspi v0.0.0-20180613141037-e580b900e9f5 + github.com/jcmturner/gokrb5/v8 v8.2.0 +) diff --git a/github.com/lib/pq/auth/kerberos/go.sum b/github.com/lib/pq/auth/kerberos/go.sum new file mode 100644 index 0000000000..138e36f1df --- /dev/null +++ b/github.com/lib/pq/auth/kerberos/go.sum @@ -0,0 +1,40 @@ +github.com/alexbrainman/sspi v0.0.0-20180613141037-e580b900e9f5 h1:P5U+E4x5OkVEKQDklVPmzs71WM56RTTRqV4OrDC//Y4= +github.com/alexbrainman/sspi v0.0.0-20180613141037-e580b900e9f5/go.mod h1:976q2ETgjT2snVCf2ZaBnyBbVoPERGjUz+0sofzEfro= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= +github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/sessions v1.2.0 h1:S7P+1Hm5V/AT9cjEcUD5uDaQSX0OE577aCXgoaKpYbQ= +github.com/gorilla/sessions v1.2.0/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= +github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= +github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= +github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= +github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= +github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= +github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= +github.com/jcmturner/gokrb5/v8 v8.2.0 h1:lzPl/30ZLkTveYsYZPKMcgXc8MbnE6RsTd4F9KgiLtk= +github.com/jcmturner/gokrb5/v8 v8.2.0/go.mod h1:T1hnNppQsBtxW0tCHMHTkAt8n/sABdzZgZdoFrZaZNM= +github.com/jcmturner/rpc/v2 v2.0.2 h1:gMB4IwRXYsWw4Bc6o/az2HJgFUA1ffSh90i26ZJ6Xl0= +github.com/jcmturner/rpc/v2 v2.0.2/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200117160349-530e935923ad h1:Jh8cai0fqIK+f6nG0UgPW5wFk8wmiMhM3AyciDBdtQg= +golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa h1:F+8P+gmewFQYRk6JoLQLwjBCTu3mcIURZfNkVweuRKA= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/github.com/lib/pq/auth/kerberos/krb.go b/github.com/lib/pq/auth/kerberos/krb.go new file mode 100644 index 0000000000..451a7ee1b4 --- /dev/null +++ b/github.com/lib/pq/auth/kerberos/krb.go @@ -0,0 +1,29 @@ +package kerberos + +import ( + "net" + "strings" +) + +/* + * Find the A record associated with a hostname + * In general, hostnames supplied to the driver should be + * canonicalized because the KDC usually only has one + * principal and not one per potential alias of a host. + */ +func canonicalizeHostname(host string) (string, error) { + canon := host + + name, err := net.LookupCNAME(host) + if err != nil { + return "", err + } + + name = strings.TrimSuffix(name, ".") + + if name != "" { + canon = name + } + + return canon, nil +} diff --git a/github.com/lib/pq/auth/kerberos/krb_unix.go b/github.com/lib/pq/auth/kerberos/krb_unix.go new file mode 100644 index 0000000000..7d5ec76a59 --- /dev/null +++ b/github.com/lib/pq/auth/kerberos/krb_unix.go @@ -0,0 +1,127 @@ +// +build !windows + +package kerberos + +import ( + "fmt" + "os" + "os/user" + "strings" + + "github.com/jcmturner/gokrb5/v8/client" + "github.com/jcmturner/gokrb5/v8/config" + "github.com/jcmturner/gokrb5/v8/credentials" + "github.com/jcmturner/gokrb5/v8/spnego" +) + +/* + * UNIX Kerberos support, using jcmturner's pure-go + * implementation + */ + +// GSS implements the pq.GSS interface. +type GSS struct { + cli *client.Client +} + +// NewGSS creates a new GSS provider. +func NewGSS() (*GSS, error) { + g := &GSS{} + err := g.init() + + if err != nil { + return nil, err + } + + return g, nil +} + +func (g *GSS) init() error { + cfgPath, ok := os.LookupEnv("KRB5_CONFIG") + if !ok { + cfgPath = "/etc/krb5.conf" + } + + cfg, err := config.Load(cfgPath) + if err != nil { + return err + } + + u, err := user.Current() + if err != nil { + return err + } + + ccpath := "/tmp/krb5cc_" + u.Uid + + ccname := os.Getenv("KRB5CCNAME") + if strings.HasPrefix(ccname, "FILE:") { + ccpath = strings.SplitN(ccname, ":", 2)[1] + } + + ccache, err := credentials.LoadCCache(ccpath) + if err != nil { + return err + } + + cl, err := client.NewFromCCache(ccache, cfg, client.DisablePAFXFAST(true)) + if err != nil { + return err + } + + cl.Login() + + g.cli = cl + + return nil +} + +// GetInitToken implements the GSS interface. +func (g *GSS) GetInitToken(host string, service string) ([]byte, error) { + + // Resolve the hostname down to an 'A' record, if required (usually, it is) + if g.cli.Config.LibDefaults.DNSCanonicalizeHostname { + var err error + host, err = canonicalizeHostname(host) + if err != nil { + return nil, err + } + } + + spn := service + "/" + host + + return g.GetInitTokenFromSpn(spn) +} + +// GetInitTokenFromSpn implements the GSS interface. +func (g *GSS) GetInitTokenFromSpn(spn string) ([]byte, error) { + s := spnego.SPNEGOClient(g.cli, spn) + + st, err := s.InitSecContext() + if err != nil { + return nil, fmt.Errorf("kerberos error (InitSecContext): %s", err.Error()) + } + + b, err := st.Marshal() + if err != nil { + return nil, fmt.Errorf("kerberos error (Marshaling token): %s", err.Error()) + } + + return b, nil +} + +// Continue implements the GSS interface. +func (g *GSS) Continue(inToken []byte) (done bool, outToken []byte, err error) { + t := &spnego.SPNEGOToken{} + err = t.Unmarshal(inToken) + if err != nil { + return true, nil, fmt.Errorf("kerberos error (Unmarshaling token): %s", err.Error()) + } + + state := t.NegTokenResp.State() + if state != spnego.NegStateAcceptCompleted { + return true, nil, fmt.Errorf("kerberos: expected state 'Completed' - got %d", state) + } + + return true, nil, nil +} diff --git a/github.com/lib/pq/auth/kerberos/krb_windows.go b/github.com/lib/pq/auth/kerberos/krb_windows.go new file mode 100644 index 0000000000..474517aea2 --- /dev/null +++ b/github.com/lib/pq/auth/kerberos/krb_windows.go @@ -0,0 +1,66 @@ +// +build windows + +package kerberos + +import ( + "github.com/alexbrainman/sspi" + "github.com/alexbrainman/sspi/negotiate" +) + +// GSS implements the pq.GSS interface. +type GSS struct { + creds *sspi.Credentials + ctx *negotiate.ClientContext +} + +// NewGSS creates a new GSS provider. +func NewGSS() (*GSS, error) { + g := &GSS{} + err := g.init() + + if err != nil { + return nil, err + } + + return g, nil +} + +func (g *GSS) init() error { + creds, err := negotiate.AcquireCurrentUserCredentials() + if err != nil { + return err + } + + g.creds = creds + return nil +} + +// GetInitToken implements the GSS interface. +func (g *GSS) GetInitToken(host string, service string) ([]byte, error) { + + host, err := canonicalizeHostname(host) + if err != nil { + return nil, err + } + + spn := service + "/" + host + + return g.GetInitTokenFromSpn(spn) +} + +// GetInitTokenFromSpn implements the GSS interface. +func (g *GSS) GetInitTokenFromSpn(spn string) ([]byte, error) { + ctx, token, err := negotiate.NewClientContext(g.creds, spn) + if err != nil { + return nil, err + } + + g.ctx = ctx + + return token, nil +} + +// Continue implements the GSS interface. +func (g *GSS) Continue(inToken []byte) (done bool, outToken []byte, err error) { + return g.ctx.Update(inToken) +} diff --git a/github.com/lib/pq/conn.go b/github.com/lib/pq/conn.go index 77c361341e..b3ab14d3cc 100644 --- a/github.com/lib/pq/conn.go +++ b/github.com/lib/pq/conn.go @@ -155,6 +155,9 @@ type conn struct { // If not nil, notifications will be synchronously sent here notificationHandler func(*Notification) + + // GSSAPI context + gss GSS } // Handle driver-side settings in parsed connection string. @@ -335,10 +338,6 @@ func (c *Connector) open(ctx context.Context) (cn *conn, err error) { func dial(ctx context.Context, d Dialer, o values) (net.Conn, error) { network, address := network(o) - // SSL is not necessary or supported over UNIX domain sockets - if network == "unix" { - o["sslmode"] = "disable" - } // Zero or not specified means wait indefinitely. if timeout, ok := o["connect_timeout"]; ok && timeout != "0" { @@ -1075,7 +1074,10 @@ func isDriverSetting(key string) bool { return true case "binary_parameters": return true - + case "service": + return true + case "spn": + return true default: return false } @@ -1155,6 +1157,59 @@ func (cn *conn) auth(r *readBuf, o values) { if r.int32() != 0 { errorf("unexpected authentication response: %q", t) } + case 7: // GSSAPI, startup + if newGss == nil { + errorf("kerberos error: no GSSAPI provider registered (import github.com/lib/pq/auth/kerberos if you need Kerberos support)") + } + cli, err := newGss() + if err != nil { + errorf("kerberos error: %s", err.Error()) + } + + var token []byte + + if spn, ok := o["spn"]; ok { + // Use the supplied SPN if provided.. + token, err = cli.GetInitTokenFromSpn(spn) + } else { + // Allow the kerberos service name to be overridden + service := "postgres" + if val, ok := o["service"]; ok { + service = val + } + + token, err = cli.GetInitToken(o["host"], service) + } + + if err != nil { + errorf("failed to get Kerberos ticket: %q", err) + } + + w := cn.writeBuf('p') + w.bytes(token) + cn.send(w) + + // Store for GSSAPI continue message + cn.gss = cli + + case 8: // GSSAPI continue + + if cn.gss == nil { + errorf("GSSAPI protocol error") + } + + b := []byte(*r) + + done, tokOut, err := cn.gss.Continue(b) + if err == nil && !done { + w := cn.writeBuf('p') + w.bytes(tokOut) + cn.send(w) + } + + // Errors fall through and read the more detailed message + // from the server.. + case 10: sc := scram.NewClient(sha256.New, o["user"], o["password"]) sc.Step(nil) diff --git a/github.com/lib/pq/connector.go b/github.com/lib/pq/connector.go index 2f8ced6737..d7d4726156 100644 --- a/github.com/lib/pq/connector.go +++ b/github.com/lib/pq/connector.go @@ -27,7 +27,7 @@ func (c *Connector) Connect(ctx context.Context) (driver.Conn, error) { return c.open(ctx) } -// Driver returnst the underlying driver of this Connector. +// Driver returns the underlying driver of this Connector. func (c *Connector) Driver() driver.Driver { return &Driver{} } @@ -106,5 +106,10 @@ func NewConnector(dsn string) (*Connector, error) { o["user"] = u } + // SSL is not necessary or supported over UNIX domain sockets + if network, _ := network(o); network == "unix" { + o["sslmode"] = "disable" + } + return &Connector{opts: o, dialer: defaultDialer{}}, nil } diff --git a/github.com/lib/pq/copy.go b/github.com/lib/pq/copy.go index d3bc1edd86..38d5bb693f 100644 --- a/github.com/lib/pq/copy.go +++ b/github.com/lib/pq/copy.go @@ -49,6 +49,7 @@ type copyin struct { buffer []byte rowData chan []byte done chan bool + driver.Result closed bool @@ -151,6 +152,8 @@ func (ci *copyin) resploop() { switch t { case 'C': // complete + res, _ := ci.cn.parseComplete(r.string()) + ci.setResult(res) case 'N': if n := ci.cn.noticeHandler; n != nil { n(parseError(&r)) @@ -201,6 +204,22 @@ func (ci *copyin) setError(err error) { ci.Unlock() } +func (ci *copyin) setResult(result driver.Result) { + ci.Lock() + ci.Result = result + ci.Unlock() +} + +func (ci *copyin) getResult() driver.Result { + ci.Lock() + result := ci.Result + if result == nil { + return driver.RowsAffected(0) + } + ci.Unlock() + return result +} + func (ci *copyin) NumInput() int { return -1 } @@ -231,7 +250,11 @@ func (ci *copyin) Exec(v []driver.Value) (r driver.Result, err error) { } if len(v) == 0 { - return driver.RowsAffected(0), ci.Close() + if err := ci.Close(); err != nil { + return driver.RowsAffected(0), err + } + + return ci.getResult(), nil } numValues := len(v) diff --git a/github.com/lib/pq/doc.go b/github.com/lib/pq/doc.go index 2a60054e2e..78c670b1d9 100644 --- a/github.com/lib/pq/doc.go +++ b/github.com/lib/pq/doc.go @@ -57,6 +57,8 @@ supported: * sslkey - Key file location. The file must contain PEM encoded data. * sslrootcert - The location of the root certificate file. The file must contain PEM encoded data. + * spn - Configures GSS (Kerberos) SPN. + * service - GSS (Kerberos) service name to use when constructing the SPN (default is `postgres`). Valid values for sslmode are: @@ -241,5 +243,21 @@ bytes by the PostgreSQL server. You can find a complete, working example of Listener usage at https://godoc.org/github.com/lib/pq/example/listen. + +Kerberos Support + + +If you need support for Kerberos authentication, add the following to your main +package: + + import "github.com/lib/pq/auth/kerberos" + + func init() { + pq.RegisterGSSProvider(func() (pq.Gss, error) { return kerberos.NewGSS() }) + } + +This package is in a separate module so that users who don't need Kerberos +don't have to download unnecessary dependencies. + */ package pq diff --git a/github.com/lib/pq/go.mod b/github.com/lib/pq/go.mod index edf0b343fd..b5a5639ab6 100644 --- a/github.com/lib/pq/go.mod +++ b/github.com/lib/pq/go.mod @@ -1 +1,3 @@ module github.com/lib/pq + +go 1.13 diff --git a/github.com/lib/pq/krb.go b/github.com/lib/pq/krb.go new file mode 100644 index 0000000000..408ec01f97 --- /dev/null +++ b/github.com/lib/pq/krb.go @@ -0,0 +1,27 @@ +package pq + +// NewGSSFunc creates a GSS authentication provider, for use with +// RegisterGSSProvider. +type NewGSSFunc func() (GSS, error) + +var newGss NewGSSFunc + +// RegisterGSSProvider registers a GSS authentication provider. For example, if +// you need to use Kerberos to authenticate with your server, add this to your +// main package: +// +// import "github.com/lib/pq/auth/kerberos" +// +// func init() { +// pq.RegisterGSSProvider(func() (pq.GSS, error) { return kerberos.NewGSS() }) +// } +func RegisterGSSProvider(newGssArg NewGSSFunc) { + newGss = newGssArg +} + +// GSS provides GSSAPI authentication (e.g., Kerberos). +type GSS interface { + GetInitToken(host string, service string) ([]byte, error) + GetInitTokenFromSpn(spn string) ([]byte, error) + Continue(inToken []byte) (done bool, outToken []byte, err error) +} diff --git a/modules.txt b/modules.txt index 93e8a2dea6..7616331821 100644 --- a/modules.txt +++ b/modules.txt @@ -81,6 +81,9 @@ github.com/VividCortex/ewma # github.com/abourget/teamcity v0.0.0-00010101000000-000000000000 => github.com/cockroachdb/teamcity v0.0.0-20180905144921-8ca25c33eb11 ## explicit github.com/abourget/teamcity +# github.com/alexbrainman/sspi v0.0.0-20180613141037-e580b900e9f5 +github.com/alexbrainman/sspi +github.com/alexbrainman/sspi/negotiate # github.com/andy-kimball/arenaskl v0.0.0-20200617143215-f701008588b9 ## explicit github.com/andy-kimball/arenaskl @@ -496,7 +499,6 @@ github.com/grpc-ecosystem/grpc-gateway/utilities ## explicit github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc # github.com/hashicorp/go-uuid v1.0.2 -## explicit github.com/hashicorp/go-uuid # github.com/ianlancetaylor/cgosymbolizer v0.0.0-20170921033129-f5072df9c550 ## explicit @@ -540,10 +542,51 @@ github.com/jackc/pgx/v4/internal/sanitize # github.com/jaegertracing/jaeger v1.17.0 ## explicit github.com/jaegertracing/jaeger/model/json +# github.com/jcmturner/aescts/v2 v2.0.0 +github.com/jcmturner/aescts/v2 +# github.com/jcmturner/dnsutils/v2 v2.0.0 +github.com/jcmturner/dnsutils/v2 # github.com/jcmturner/gofork v1.0.0 -## explicit github.com/jcmturner/gofork/encoding/asn1 github.com/jcmturner/gofork/x/crypto/pbkdf2 +# github.com/jcmturner/goidentity/v6 v6.0.1 +github.com/jcmturner/goidentity/v6 +# github.com/jcmturner/gokrb5/v8 v8.2.0 +github.com/jcmturner/gokrb5/v8/asn1tools +github.com/jcmturner/gokrb5/v8/client +github.com/jcmturner/gokrb5/v8/config +github.com/jcmturner/gokrb5/v8/credentials +github.com/jcmturner/gokrb5/v8/crypto +github.com/jcmturner/gokrb5/v8/crypto/common +github.com/jcmturner/gokrb5/v8/crypto/etype +github.com/jcmturner/gokrb5/v8/crypto/rfc3961 +github.com/jcmturner/gokrb5/v8/crypto/rfc3962 +github.com/jcmturner/gokrb5/v8/crypto/rfc4757 +github.com/jcmturner/gokrb5/v8/crypto/rfc8009 +github.com/jcmturner/gokrb5/v8/gssapi +github.com/jcmturner/gokrb5/v8/iana +github.com/jcmturner/gokrb5/v8/iana/addrtype +github.com/jcmturner/gokrb5/v8/iana/adtype +github.com/jcmturner/gokrb5/v8/iana/asnAppTag +github.com/jcmturner/gokrb5/v8/iana/chksumtype +github.com/jcmturner/gokrb5/v8/iana/errorcode +github.com/jcmturner/gokrb5/v8/iana/etypeID +github.com/jcmturner/gokrb5/v8/iana/flags +github.com/jcmturner/gokrb5/v8/iana/keyusage +github.com/jcmturner/gokrb5/v8/iana/msgtype +github.com/jcmturner/gokrb5/v8/iana/nametype +github.com/jcmturner/gokrb5/v8/iana/patype +github.com/jcmturner/gokrb5/v8/kadmin +github.com/jcmturner/gokrb5/v8/keytab +github.com/jcmturner/gokrb5/v8/krberror +github.com/jcmturner/gokrb5/v8/messages +github.com/jcmturner/gokrb5/v8/pac +github.com/jcmturner/gokrb5/v8/service +github.com/jcmturner/gokrb5/v8/spnego +github.com/jcmturner/gokrb5/v8/types +# github.com/jcmturner/rpc/v2 v2.0.2 +github.com/jcmturner/rpc/v2/mstypes +github.com/jcmturner/rpc/v2/ndr # github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af github.com/jmespath/go-jmespath # github.com/jordanlewis/gcassert v0.0.0-20200706043056-bf61eb72ee48 @@ -584,11 +627,14 @@ github.com/kr/text ## explicit github.com/leanovate/gopter github.com/leanovate/gopter/prop -# github.com/lib/pq v1.5.2 +# github.com/lib/pq v1.7.1 ## explicit github.com/lib/pq github.com/lib/pq/oid github.com/lib/pq/scram +# github.com/lib/pq/auth/kerberos v0.0.0-20200720160335-984a6aa1ca46 +## explicit +github.com/lib/pq/auth/kerberos # github.com/lightstep/lightstep-tracer-go v0.15.6 ## explicit github.com/lightstep/lightstep-tracer-go From e4a1a5ff9d472d14de28907e2ece4c24bff01200 Mon Sep 17 00:00:00 2001 From: Oliver Tan Date: Thu, 23 Jul 2020 09:51:20 -0700 Subject: [PATCH 16/49] add pierrre/geohash --- github.com/pierrre/geohash/.gitignore | 1 + github.com/pierrre/geohash/.golangci.yml | 28 ++++ github.com/pierrre/geohash/.travis.yml | 9 + github.com/pierrre/geohash/LICENSE | 7 + github.com/pierrre/geohash/Makefile | 30 ++++ github.com/pierrre/geohash/README.md | 11 ++ github.com/pierrre/geohash/geohash.go | 200 +++++++++++++++++++++++ github.com/pierrre/geohash/go.mod | 13 ++ github.com/pierrre/geohash/go.sum | 14 ++ github.com/pierrre/geohash/util.go | 53 ++++++ modules.txt | 3 + 11 files changed, 369 insertions(+) create mode 100644 github.com/pierrre/geohash/.gitignore create mode 100644 github.com/pierrre/geohash/.golangci.yml create mode 100644 github.com/pierrre/geohash/.travis.yml create mode 100644 github.com/pierrre/geohash/LICENSE create mode 100644 github.com/pierrre/geohash/Makefile create mode 100644 github.com/pierrre/geohash/README.md create mode 100644 github.com/pierrre/geohash/geohash.go create mode 100644 github.com/pierrre/geohash/go.mod create mode 100644 github.com/pierrre/geohash/go.sum create mode 100644 github.com/pierrre/geohash/util.go diff --git a/github.com/pierrre/geohash/.gitignore b/github.com/pierrre/geohash/.gitignore new file mode 100644 index 0000000000..796b96d1c4 --- /dev/null +++ b/github.com/pierrre/geohash/.gitignore @@ -0,0 +1 @@ +/build diff --git a/github.com/pierrre/geohash/.golangci.yml b/github.com/pierrre/geohash/.golangci.yml new file mode 100644 index 0000000000..6f4f966e0c --- /dev/null +++ b/github.com/pierrre/geohash/.golangci.yml @@ -0,0 +1,28 @@ +run: + deadline: "10m" +linters: + disable-all: true + enable: + - "bodyclose" + - "deadcode" + - "errcheck" + - "gocyclo" + - "gofmt" + - "goimports" + - "golint" + - "govet" + - "ineffassign" + - "megacheck" + - "misspell" + - "nakedret" + - "structcheck" + - "unconvert" + - "unparam" + - "varcheck" +linters-settings: + gocyclo: + min-complexity: 10 +issues: + exclude-use-default: false + max-per-linter: 0 + max-same-issues: 0 diff --git a/github.com/pierrre/geohash/.travis.yml b/github.com/pierrre/geohash/.travis.yml new file mode 100644 index 0000000000..4785382a47 --- /dev/null +++ b/github.com/pierrre/geohash/.travis.yml @@ -0,0 +1,9 @@ +dist: trusty +language: go +go: + - 1.12.x + - tip + +install: true + +script: make build test lint diff --git a/github.com/pierrre/geohash/LICENSE b/github.com/pierrre/geohash/LICENSE new file mode 100644 index 0000000000..fbe7514605 --- /dev/null +++ b/github.com/pierrre/geohash/LICENSE @@ -0,0 +1,7 @@ +Copyright (C) 2015 Pierre Durand + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/github.com/pierrre/geohash/Makefile b/github.com/pierrre/geohash/Makefile new file mode 100644 index 0000000000..5fb7b2e2c5 --- /dev/null +++ b/github.com/pierrre/geohash/Makefile @@ -0,0 +1,30 @@ +export GO111MODULE=on + +build: build/geohash + +build/geohash: + go build -o build/geohash ./cmd/geohash + +.PHONY: test +test: + go test ./... + +.PHONY: lint +lint: \ + golangci-lint + +GOLANGCI_LINT_VERSION=v1.17.0 +GOLANGCI_LINT_DIR=$(shell go env GOPATH)/pkg/golangci-lint/$(GOLANGCI_LINT_VERSION) +$(GOLANGCI_LINT_DIR): + curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(GOLANGCI_LINT_DIR) $(GOLANGCI_LINT_VERSION) + +.PHONY: install-golangci-lint +install-golangci-lint: $(GOLANGCI_LINT_DIR) + +.PHONY: golangci-lint +golangci-lint: install-golangci-lint + $(GOLANGCI_LINT_DIR)/golangci-lint run + +.PHONY: clean +clean: + rm -rf build diff --git a/github.com/pierrre/geohash/README.md b/github.com/pierrre/geohash/README.md new file mode 100644 index 0000000000..1c6385f9aa --- /dev/null +++ b/github.com/pierrre/geohash/README.md @@ -0,0 +1,11 @@ +# Geohash +A geohash library for Go (Golang) + +[![GoDoc](https://godoc.org/github.com/pierrre/geohash?status.svg)](https://godoc.org/github.com/pierrre/geohash) +[![Build Status](https://travis-ci.org/pierrre/geohash.svg?branch=master)](https://travis-ci.org/pierrre/geohash) + +## Features +- Encode latitude/longitude to geohash +- Decode geohash to latitude/longitude +- Round a geohash box to a single location +- Commande-line tool (https://godoc.org/github.com/pierrre/geohash/cmd/geohash) diff --git a/github.com/pierrre/geohash/geohash.go b/github.com/pierrre/geohash/geohash.go new file mode 100644 index 0000000000..6f292ceccb --- /dev/null +++ b/github.com/pierrre/geohash/geohash.go @@ -0,0 +1,200 @@ +/* +Package geohash provides an implementation of geohash. + +http://en.wikipedia.com/wiki/Geohash + +http://geohash.org +*/ +package geohash + +import ( + "fmt" + "math" +) + +const ( + base32 = "0123456789bcdefghjkmnpqrstuvwxyz" +) + +var ( + base32Lookup [1 << 8]int + defaultBox = Box{Lat: Range{Min: -90, Max: 90}, Lon: Range{Min: -180, Max: 180}} +) + +func init() { + for i := range base32Lookup { + base32Lookup[i] = -1 + } + for i := 0; i < len(base32); i++ { + base32Lookup[base32[i]] = i + } +} + +// EncodeAuto encodes a location to a geohash using the most suitable precision. +func EncodeAuto(lat, lon float64) string { + var gh string + for precision := 1; precision <= encodeMaxPrecision; precision++ { + gh = Encode(lat, lon, precision) + b, _ := Decode(gh) + p := b.Round() + if p.Lat == lat && p.Lon == lon { + break + } + } + return gh +} + +const encodeMaxPrecision = 32 + +// Encode encodes a location to a geohash. +// +// The maximum supported precision is 32. +func Encode(lat, lon float64, precision int) string { + if precision > encodeMaxPrecision { + precision = encodeMaxPrecision + } + var buf [encodeMaxPrecision]byte + box := defaultBox + even := true + for i := 0; i < precision; i++ { + ci := 0 + for mask := 1 << 4; mask != 0; mask >>= 1 { + var r *Range + var u float64 + if even { + r = &box.Lon + u = lon + } else { + r = &box.Lat + u = lat + } + if mid := r.Mid(); u >= mid { + ci += mask + r.Min = mid + } else { + r.Max = mid + } + even = !even + } + buf[i] = base32[ci] + } + return string(buf[:precision]) +} + +// Decode decode a geohash to a Box. +func Decode(gh string) (Box, error) { + box := defaultBox + even := true + for i := 0; i < len(gh); i++ { + ci := base32Lookup[gh[i]] + if ci == -1 { + return box, fmt.Errorf("geohash decode '%s': invalid character at index %d", gh, i) + } + for mask := 1 << 4; mask != 0; mask >>= 1 { + var r *Range + if even { + r = &box.Lon + } else { + r = &box.Lat + } + if mid := r.Mid(); ci&mask != 0 { + r.Min = mid + } else { + r.Max = mid + } + even = !even + } + } + return box, nil +} + +//Box is a spatial data structure. +// +//It is defined by 2 ranges of latitude/longitude. +type Box struct { + Lat, Lon Range +} + +// Center returns the Box's center as a Point. +func (b Box) Center() Point { + return Point{Lat: b.Lat.Mid(), Lon: b.Lon.Mid()} +} + +// Round returns the Box's approximate location as a Point. +// +// It uses decimal rounding and is in general more useful than Center. +func (b Box) Round() Point { + return Point{Lat: b.Lat.Round(), Lon: b.Lon.Round()} +} + +// Point represents a location (latitude and longitude). +type Point struct { + Lat, Lon float64 +} + +// Range represents a range (min/max) on latitude or longitude. +type Range struct { + Min, Max float64 +} + +// Val returns the difference between Min and Max. +func (r Range) Val() float64 { + return math.Abs(r.Max - r.Min) +} + +// Mid return the middle value between Min and Max. +func (r Range) Mid() float64 { + return (r.Min + r.Max) / 2 +} + +// Round returns the rounded value between Min and Max. +// +// It uses decimal rounding. +func (r Range) Round() float64 { + dec := int(math.Floor(-math.Log10(r.Val()))) + if dec < 0 { + dec = 0 + } + return roundDecimal(r.Mid(), dec) +} + +// Neighbors will contain the geohashes for the neighbors of the supplied +// geohash in each of the cardinal and intercardinal directions. +type Neighbors struct { + North string + NorthEast string + East string + SouthEast string + South string + SouthWest string + West string + NorthWest string +} + +// GetNeighbors returns a struct representing the neighbors of the supplied +// geohash in each of the cardinal and intercardinal directions. +func GetNeighbors(gh string) (Neighbors, error) { + box, err := Decode(gh) + if err != nil { + return Neighbors{}, err + } + latMid := box.Lat.Mid() + lonMid := box.Lon.Mid() + latVal := box.Lat.Val() + lonVal := box.Lon.Val() + precision := len(gh) + encode := func(lat, lon float64) string { + lat, lon = normalize(lat, lon) + return Encode(lat, lon, precision) + } + return Neighbors{ + North: encode(latMid+latVal, lonMid), + NorthEast: encode(latMid+latVal, lonMid+lonVal), + East: encode(latMid, lonMid+lonVal), + SouthEast: encode(latMid-latVal, lonMid+lonVal), + South: encode(latMid-latVal, lonMid), + SouthWest: encode(latMid-latVal, lonMid-lonVal), + West: encode(latMid, lonMid-lonVal), + NorthWest: encode(latMid+latVal, lonMid-lonVal), + }, nil +} diff --git a/github.com/pierrre/geohash/go.mod b/github.com/pierrre/geohash/go.mod new file mode 100644 index 0000000000..ce12ec0cea --- /dev/null +++ b/github.com/pierrre/geohash/go.mod @@ -0,0 +1,13 @@ +module github.com/pierrre/geohash + +go 1.12 + +require ( + github.com/Codefor/geohash v0.0.0-20140723084247-1b41c28e3a9d + github.com/TomiHiltunen/geohash-golang v0.0.0-20150112065804-b3e4e625abfb + github.com/broady/gogeohash v0.0.0-20120525094510-7b2c40d64042 + github.com/fanixk/geohash v0.0.0-20150324002647-c1f9b5fa157a + github.com/mmcloughlin/geohash v0.9.0 + github.com/pierrre/compare v1.0.2 + github.com/the42/cartconvert v0.0.0-20131203171324-aae784c392b8 +) diff --git a/github.com/pierrre/geohash/go.sum b/github.com/pierrre/geohash/go.sum new file mode 100644 index 0000000000..735ef25663 --- /dev/null +++ b/github.com/pierrre/geohash/go.sum @@ -0,0 +1,14 @@ +github.com/Codefor/geohash v0.0.0-20140723084247-1b41c28e3a9d h1:iG9B49Q218F/XxXNRM7k/vWf7MKmLIS8AcJV9cGN4nA= +github.com/Codefor/geohash v0.0.0-20140723084247-1b41c28e3a9d/go.mod h1:RVnhzAX71far8Kc3TQeA0k/dcaEKUnTDSOyet/JCmGI= +github.com/TomiHiltunen/geohash-golang v0.0.0-20150112065804-b3e4e625abfb h1:wumPkzt4zaxO4rHPBrjDK8iZMR41C1qs7njNqlacwQg= +github.com/TomiHiltunen/geohash-golang v0.0.0-20150112065804-b3e4e625abfb/go.mod h1:QiYsIBRQEO+Z4Rz7GoI+dsHVneZNONvhczuA+llOZNM= +github.com/broady/gogeohash v0.0.0-20120525094510-7b2c40d64042 h1:iEdmkrNMLXbM7ecffOAtZJQOQUTE4iMonxrb5opUgE4= +github.com/broady/gogeohash v0.0.0-20120525094510-7b2c40d64042/go.mod h1:f1L9YvXvlt9JTa+A17trQjSMM6bV40f+tHjB+Pi+Fqk= +github.com/fanixk/geohash v0.0.0-20150324002647-c1f9b5fa157a h1:Fyfh/dsHFrC6nkX7H7+nFdTd1wROlX/FxEIWVpKYf1U= +github.com/fanixk/geohash v0.0.0-20150324002647-c1f9b5fa157a/go.mod h1:UgNw+PTmmGN8rV7RvjvnBMsoTU8ZXXnaT3hYsDTBlgQ= +github.com/mmcloughlin/geohash v0.9.0 h1:FihR004p/aE1Sju6gcVq5OLDqGcMnpBY+8moBqIsVOs= +github.com/mmcloughlin/geohash v0.9.0/go.mod h1:oNZxQo5yWJh0eMQEP/8hwQuVx9Z9tjwFUqcTB1SmG0c= +github.com/pierrre/compare v1.0.2 h1:k4IUsHgh+dbcAOIWCfxVa/7G6STjADH2qmhomv+1quc= +github.com/pierrre/compare v1.0.2/go.mod h1:8UvyRHH+9HS8Pczdd2z5x/wvv67krDwVxoOndaIIDVU= +github.com/the42/cartconvert v0.0.0-20131203171324-aae784c392b8 h1:I4DY8wLxJXCrMYzDM6lKCGc3IQwJX0PlTLsd3nQqI3c= +github.com/the42/cartconvert v0.0.0-20131203171324-aae784c392b8/go.mod h1:fWO/msnJVhHqN1yX6OBoxSyfj7TEj1hHiL8bJSQsK30= diff --git a/github.com/pierrre/geohash/util.go b/github.com/pierrre/geohash/util.go new file mode 100644 index 0000000000..cc762098d1 --- /dev/null +++ b/github.com/pierrre/geohash/util.go @@ -0,0 +1,53 @@ +package geohash + +import ( + "math" +) + +func round(val float64) float64 { + if val < 0 { + return math.Ceil(val - 0.5) + } + return math.Floor(val + 0.5) +} + +func roundDecimal(val float64, dec int) float64 { + factor := math.Pow10(dec) + return round(val*factor) / factor +} + +func normalize(lat, lon float64) (float64, float64) { + if lat > 90 || lat < -90 { + lat = center360(lat) + invertLon := true + if lat < -90 { + lat = -180 - lat + } else if lat > 90 { + lat = 180 - lat + } else { + invertLon = false + } + if invertLon { + if lon > 0 { + lon -= 180 + } else { + lon += 180 + } + } + } + if lon > 180 || lon <= -180 { + lon = center360(lon) + } + return lat, lon +} + +func center360(v float64) float64 { + v = math.Mod(v, 360) + if v <= 0 { + v += 360 + } + if v > 180 { + v -= 360 + } + return v +} diff --git a/modules.txt b/modules.txt index 7616331821..183acec842 100644 --- a/modules.txt +++ b/modules.txt @@ -733,6 +733,9 @@ github.com/petermattis/goid ## explicit github.com/pierrec/lz4 github.com/pierrec/lz4/internal/xxh32 +# github.com/pierrre/geohash v1.0.0 +## explicit +github.com/pierrre/geohash # github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 ## explicit github.com/pkg/browser From df563d91a3364fc6cd3e112f34ae563bfb20c526 Mon Sep 17 00:00:00 2001 From: Aditya Maru Date: Fri, 24 Jul 2020 14:01:44 -0400 Subject: [PATCH 17/49] adding KMS support to AWS SDK --- github.com/aws/aws-sdk-go/NOTICE.txt | 2 +- github.com/aws/aws-sdk-go/aws/arn/arn.go | 93 + github.com/aws/aws-sdk-go/aws/awserr/error.go | 23 +- github.com/aws/aws-sdk-go/aws/awserr/types.go | 31 +- .../aws/aws-sdk-go/aws/awsutil/equal.go | 2 +- .../aws/aws-sdk-go/aws/awsutil/path_value.go | 13 +- .../aws/aws-sdk-go/aws/client/client.go | 3 +- .../aws-sdk-go/aws/client/default_retryer.go | 149 +- .../aws/aws-sdk-go/aws/client/logger.go | 18 +- .../aws/client/metadata/client_info.go | 1 + .../aws-sdk-go/aws/client/no_op_retryer.go | 28 + github.com/aws/aws-sdk-go/aws/config.go | 63 +- .../aws/{context.go => context_1_5.go} | 40 +- github.com/aws/aws-sdk-go/aws/context_1_7.go | 9 - github.com/aws/aws-sdk-go/aws/context_1_9.go | 11 + .../aws-sdk-go/aws/context_background_1_5.go | 22 + .../aws-sdk-go/aws/context_background_1_7.go | 20 + .../aws/aws-sdk-go/aws/context_sleep.go | 24 + .../aws/aws-sdk-go/aws/convert_types.go | 531 + .../aws-sdk-go/aws/corehandlers/handlers.go | 64 +- .../credentials/context_background_go1.5.go | 22 + .../credentials/context_background_go1.7.go | 20 + .../aws/credentials/context_go1.5.go | 39 + .../aws/credentials/context_go1.9.go | 13 + .../aws-sdk-go/aws/credentials/credentials.go | 143 +- .../ec2rolecreds/ec2_role_provider.go | 26 +- .../aws/credentials/endpointcreds/provider.go | 30 +- .../aws/credentials/processcreds/provider.go | 3 +- .../shared_credentials_provider.go | 5 +- .../aws/credentials/static_provider.go | 4 +- .../stscreds/assume_role_provider.go | 79 +- .../stscreds/web_identity_provider.go | 135 + github.com/aws/aws-sdk-go/aws/csm/doc.go | 65 +- github.com/aws/aws-sdk-go/aws/csm/enable.go | 34 +- .../aws/aws-sdk-go/aws/csm/metric_chan.go | 11 +- github.com/aws/aws-sdk-go/aws/csm/reporter.go | 32 +- .../aws/aws-sdk-go/aws/ec2metadata/api.go | 156 +- .../aws/aws-sdk-go/aws/ec2metadata/service.go | 121 +- .../aws/ec2metadata/token_provider.go | 92 + .../aws/aws-sdk-go/aws/endpoints/decode.go | 30 +- .../aws/aws-sdk-go/aws/endpoints/defaults.go | 9158 ++++++++-- .../aws/endpoints/dep_service_ids.go | 2 +- .../aws/aws-sdk-go/aws/endpoints/endpoints.go | 127 +- .../aws/endpoints/legacy_regions.go | 24 + .../aws/aws-sdk-go/aws/endpoints/v3model.go | 84 +- .../aws/request/connection_reset_error.go | 17 +- .../request/connection_reset_error_other.go | 11 - .../aws/aws-sdk-go/aws/request/handlers.go | 70 +- .../aws-sdk-go/aws/request/offset_reader.go | 15 +- .../aws/aws-sdk-go/aws/request/request.go | 160 +- .../aws/aws-sdk-go/aws/request/request_1_8.go | 5 +- .../aws/request/request_pagination.go | 8 +- .../aws/aws-sdk-go/aws/request/retryer.go | 201 +- .../aws/session/cabundle_transport.go | 26 + .../aws/session/cabundle_transport_1_5.go | 22 + .../aws/session/cabundle_transport_1_6.go | 23 + .../aws/aws-sdk-go/aws/session/credentials.go | 267 + github.com/aws/aws-sdk-go/aws/session/doc.go | 208 +- .../aws/aws-sdk-go/aws/session/env_config.go | 141 +- .../aws/aws-sdk-go/aws/session/session.go | 438 +- .../aws-sdk-go/aws/session/shared_config.go | 442 +- .../aws-sdk-go/aws/signer/v4/header_rules.go | 5 +- .../aws/signer/v4/request_context_go1.5.go | 13 + .../aws/signer/v4/request_context_go1.7.go | 13 + .../aws/aws-sdk-go/aws/signer/v4/stream.go | 63 + github.com/aws/aws-sdk-go/aws/signer/v4/v4.go | 130 +- github.com/aws/aws-sdk-go/aws/types.go | 77 +- github.com/aws/aws-sdk-go/aws/version.go | 2 +- .../context/background_go1.5.go} | 9 +- .../aws/aws-sdk-go/internal/ini/ini_parser.go | 17 +- .../aws/aws-sdk-go/internal/ini/skipper.go | 6 +- .../aws/aws-sdk-go/internal/sdkio/byte.go | 12 + .../aws/aws-sdk-go/internal/sdkmath/floor.go | 15 + .../internal/sdkmath/floor_go1.9.go | 56 + .../aws/aws-sdk-go/internal/sdkrand/read.go | 11 + .../aws-sdk-go/internal/sdkrand/read_1_5.go | 24 + .../aws-sdk-go/internal/strings/strings.go | 11 + .../internal/sync/singleflight/LICENSE | 27 + .../sync/singleflight/singleflight.go | 120 + .../private/checksum/content_md5.go | 53 + .../private/protocol/eventstream/debug.go | 2 +- .../private/protocol/eventstream/decode.go | 33 +- .../private/protocol/eventstream/encode.go | 72 +- .../eventstream/eventstreamapi/error.go | 55 +- .../eventstreamapi/{api.go => reader.go} | 57 +- .../eventstream/eventstreamapi/shared.go | 23 + .../eventstream/eventstreamapi/signer.go | 123 + .../eventstreamapi/stream_writer.go | 129 + .../eventstream/eventstreamapi/writer.go | 109 + .../private/protocol/eventstream/header.go | 9 + .../protocol/eventstream/header_value.go | 5 + .../private/protocol/eventstream/message.go | 16 +- .../private/protocol/json/jsonutil/build.go | 296 + .../protocol/json/jsonutil/unmarshal.go | 282 + .../private/protocol/jsonrpc/jsonrpc.go | 88 + .../protocol/jsonrpc/unmarshal_error.go | 107 + .../aws-sdk-go/private/protocol/payload.go | 2 +- .../aws-sdk-go/private/protocol/protocol.go | 49 + .../private/protocol/query/build.go | 4 +- .../private/protocol/query/unmarshal.go | 4 +- .../private/protocol/query/unmarshal_error.go | 77 +- .../aws-sdk-go/private/protocol/rest/build.go | 27 +- .../private/protocol/rest/unmarshal.go | 102 +- .../private/protocol/restxml/restxml.go | 12 +- .../aws-sdk-go/private/protocol/timestamp.go | 23 +- .../aws-sdk-go/private/protocol/unmarshal.go | 6 + .../private/protocol/unmarshal_error.go | 65 + .../private/protocol/xml/xmlutil/build.go | 9 + .../private/protocol/xml/xmlutil/sort.go | 32 + .../private/protocol/xml/xmlutil/unmarshal.go | 27 + .../protocol/xml/xmlutil/xml_to_struct.go | 13 +- github.com/aws/aws-sdk-go/service/kms/api.go | 14793 ++++++++++++++++ github.com/aws/aws-sdk-go/service/kms/doc.go | 98 + .../aws/aws-sdk-go/service/kms/errors.go | 367 + .../aws/aws-sdk-go/service/kms/service.go | 103 + github.com/aws/aws-sdk-go/service/s3/api.go | 7790 +++++++- .../aws/aws-sdk-go/service/s3/body_hash.go | 49 +- .../aws-sdk-go/service/s3/bucket_location.go | 3 +- .../aws-sdk-go/service/s3/customizations.go | 19 +- .../aws/aws-sdk-go/service/s3/doc_custom.go | 14 + .../aws/aws-sdk-go/service/s3/endpoint.go | 233 + .../aws-sdk-go/service/s3/endpoint_errors.go | 151 + .../aws/aws-sdk-go/service/s3/errors.go | 10 +- .../service/s3/host_style_bucket.go | 43 +- .../s3/internal/arn/accesspoint_arn.go | 45 + .../aws-sdk-go/service/s3/internal/arn/arn.go | 71 + .../aws-sdk-go/service/s3/s3manager/batch.go | 6 +- .../service/s3/s3manager/bucket_region.go | 71 + .../s3/s3manager/buffered_read_seeker.go | 81 + .../s3manager/default_read_seeker_write_to.go | 7 + .../default_read_seeker_write_to_windows.go | 5 + .../s3/s3manager/default_writer_read_from.go | 7 + .../default_writer_read_from_windows.go | 5 + .../service/s3/s3manager/download.go | 98 +- .../aws-sdk-go/service/s3/s3manager/pool.go | 244 + .../s3/s3manager/read_seeker_write_to.go | 65 + .../aws-sdk-go/service/s3/s3manager/upload.go | 230 +- .../service/s3/s3manager/upload_input.go | 105 +- .../service/s3/s3manager/writer_read_from.go | 75 + .../aws/aws-sdk-go/service/s3/service.go | 10 +- github.com/aws/aws-sdk-go/service/s3/sse.go | 64 +- .../aws-sdk-go/service/s3/statusok_error.go | 18 +- .../aws-sdk-go/service/s3/unmarshal_error.go | 73 +- github.com/aws/aws-sdk-go/service/sts/api.go | 1585 +- .../aws-sdk-go/service/sts/customizations.go | 11 +- github.com/aws/aws-sdk-go/service/sts/doc.go | 76 +- .../aws/aws-sdk-go/service/sts/errors.go | 25 +- .../aws/aws-sdk-go/service/sts/service.go | 9 +- .../service/sts/stsiface/interface.go | 96 + github.com/go-sql-driver/mysql/.travis.yml | 26 +- github.com/go-sql-driver/mysql/AUTHORS | 13 +- github.com/go-sql-driver/mysql/CHANGELOG.md | 39 + .../go-sql-driver/mysql/CONTRIBUTING.md | 23 - github.com/go-sql-driver/mysql/README.md | 35 +- github.com/go-sql-driver/mysql/buffer.go | 48 +- github.com/go-sql-driver/mysql/collations.go | 376 +- github.com/go-sql-driver/mysql/conncheck.go | 54 + .../{appengine.go => conncheck_dummy.go} | 12 +- github.com/go-sql-driver/mysql/connection.go | 10 +- github.com/go-sql-driver/mysql/connector.go | 146 + github.com/go-sql-driver/mysql/driver.go | 152 +- github.com/go-sql-driver/mysql/dsn.go | 237 +- github.com/go-sql-driver/mysql/go.mod | 3 + github.com/go-sql-driver/mysql/nulltime.go | 50 + .../go-sql-driver/mysql/nulltime_go113.go | 31 + .../go-sql-driver/mysql/nulltime_legacy.go | 34 + github.com/go-sql-driver/mysql/packets.go | 46 + github.com/go-sql-driver/mysql/rows.go | 7 + github.com/go-sql-driver/mysql/statement.go | 11 +- github.com/go-sql-driver/mysql/utils.go | 60 +- github.com/jmespath/go-jmespath/.travis.yml | 10 +- github.com/jmespath/go-jmespath/README.md | 82 +- github.com/jmespath/go-jmespath/api.go | 2 +- github.com/jmespath/go-jmespath/go.mod | 5 + github.com/jmespath/go-jmespath/go.sum | 11 + github.com/jmespath/go-jmespath/parser.go | 2 +- .../appengine/cloudsql/cloudsql.go | 62 - .../appengine/cloudsql/cloudsql_classic.go | 17 - .../appengine/cloudsql/cloudsql_vm.go | 16 - modules.txt | 18 +- 180 files changed, 39299 insertions(+), 5272 deletions(-) create mode 100644 github.com/aws/aws-sdk-go/aws/arn/arn.go create mode 100644 github.com/aws/aws-sdk-go/aws/client/no_op_retryer.go rename github.com/aws/aws-sdk-go/aws/{context.go => context_1_5.go} (58%) delete mode 100644 github.com/aws/aws-sdk-go/aws/context_1_7.go create mode 100644 github.com/aws/aws-sdk-go/aws/context_1_9.go create mode 100644 github.com/aws/aws-sdk-go/aws/context_background_1_5.go create mode 100644 github.com/aws/aws-sdk-go/aws/context_background_1_7.go create mode 100644 github.com/aws/aws-sdk-go/aws/context_sleep.go create mode 100644 github.com/aws/aws-sdk-go/aws/credentials/context_background_go1.5.go create mode 100644 github.com/aws/aws-sdk-go/aws/credentials/context_background_go1.7.go create mode 100644 github.com/aws/aws-sdk-go/aws/credentials/context_go1.5.go create mode 100644 github.com/aws/aws-sdk-go/aws/credentials/context_go1.9.go create mode 100644 github.com/aws/aws-sdk-go/aws/credentials/stscreds/web_identity_provider.go create mode 100644 github.com/aws/aws-sdk-go/aws/ec2metadata/token_provider.go create mode 100644 github.com/aws/aws-sdk-go/aws/endpoints/legacy_regions.go delete mode 100644 github.com/aws/aws-sdk-go/aws/request/connection_reset_error_other.go create mode 100644 github.com/aws/aws-sdk-go/aws/session/cabundle_transport.go create mode 100644 github.com/aws/aws-sdk-go/aws/session/cabundle_transport_1_5.go create mode 100644 github.com/aws/aws-sdk-go/aws/session/cabundle_transport_1_6.go create mode 100644 github.com/aws/aws-sdk-go/aws/session/credentials.go create mode 100644 github.com/aws/aws-sdk-go/aws/signer/v4/request_context_go1.5.go create mode 100644 github.com/aws/aws-sdk-go/aws/signer/v4/request_context_go1.7.go create mode 100644 github.com/aws/aws-sdk-go/aws/signer/v4/stream.go rename github.com/aws/aws-sdk-go/{aws/context_1_6.go => internal/context/background_go1.5.go} (86%) create mode 100644 github.com/aws/aws-sdk-go/internal/sdkio/byte.go create mode 100644 github.com/aws/aws-sdk-go/internal/sdkmath/floor.go create mode 100644 github.com/aws/aws-sdk-go/internal/sdkmath/floor_go1.9.go create mode 100644 github.com/aws/aws-sdk-go/internal/sdkrand/read.go create mode 100644 github.com/aws/aws-sdk-go/internal/sdkrand/read_1_5.go create mode 100644 github.com/aws/aws-sdk-go/internal/strings/strings.go create mode 100644 github.com/aws/aws-sdk-go/internal/sync/singleflight/LICENSE create mode 100644 github.com/aws/aws-sdk-go/internal/sync/singleflight/singleflight.go create mode 100644 github.com/aws/aws-sdk-go/private/checksum/content_md5.go rename github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/{api.go => reader.go} (76%) create mode 100644 github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/shared.go create mode 100644 github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/signer.go create mode 100644 github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/stream_writer.go create mode 100644 github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/writer.go create mode 100644 github.com/aws/aws-sdk-go/private/protocol/json/jsonutil/build.go create mode 100644 github.com/aws/aws-sdk-go/private/protocol/json/jsonutil/unmarshal.go create mode 100644 github.com/aws/aws-sdk-go/private/protocol/jsonrpc/jsonrpc.go create mode 100644 github.com/aws/aws-sdk-go/private/protocol/jsonrpc/unmarshal_error.go create mode 100644 github.com/aws/aws-sdk-go/private/protocol/protocol.go create mode 100644 github.com/aws/aws-sdk-go/private/protocol/unmarshal_error.go create mode 100644 github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil/sort.go create mode 100644 github.com/aws/aws-sdk-go/service/kms/api.go create mode 100644 github.com/aws/aws-sdk-go/service/kms/doc.go create mode 100644 github.com/aws/aws-sdk-go/service/kms/errors.go create mode 100644 github.com/aws/aws-sdk-go/service/kms/service.go create mode 100644 github.com/aws/aws-sdk-go/service/s3/endpoint.go create mode 100644 github.com/aws/aws-sdk-go/service/s3/endpoint_errors.go create mode 100644 github.com/aws/aws-sdk-go/service/s3/internal/arn/accesspoint_arn.go create mode 100644 github.com/aws/aws-sdk-go/service/s3/internal/arn/arn.go create mode 100644 github.com/aws/aws-sdk-go/service/s3/s3manager/buffered_read_seeker.go create mode 100644 github.com/aws/aws-sdk-go/service/s3/s3manager/default_read_seeker_write_to.go create mode 100644 github.com/aws/aws-sdk-go/service/s3/s3manager/default_read_seeker_write_to_windows.go create mode 100644 github.com/aws/aws-sdk-go/service/s3/s3manager/default_writer_read_from.go create mode 100644 github.com/aws/aws-sdk-go/service/s3/s3manager/default_writer_read_from_windows.go create mode 100644 github.com/aws/aws-sdk-go/service/s3/s3manager/pool.go create mode 100644 github.com/aws/aws-sdk-go/service/s3/s3manager/read_seeker_write_to.go create mode 100644 github.com/aws/aws-sdk-go/service/s3/s3manager/writer_read_from.go create mode 100644 github.com/aws/aws-sdk-go/service/sts/stsiface/interface.go delete mode 100644 github.com/go-sql-driver/mysql/CONTRIBUTING.md create mode 100644 github.com/go-sql-driver/mysql/conncheck.go rename github.com/go-sql-driver/mysql/{appengine.go => conncheck_dummy.go} (59%) create mode 100644 github.com/go-sql-driver/mysql/connector.go create mode 100644 github.com/go-sql-driver/mysql/go.mod create mode 100644 github.com/go-sql-driver/mysql/nulltime.go create mode 100644 github.com/go-sql-driver/mysql/nulltime_go113.go create mode 100644 github.com/go-sql-driver/mysql/nulltime_legacy.go create mode 100644 github.com/jmespath/go-jmespath/go.mod create mode 100644 github.com/jmespath/go-jmespath/go.sum delete mode 100644 google.golang.org/appengine/cloudsql/cloudsql.go delete mode 100644 google.golang.org/appengine/cloudsql/cloudsql_classic.go delete mode 100644 google.golang.org/appengine/cloudsql/cloudsql_vm.go diff --git a/github.com/aws/aws-sdk-go/NOTICE.txt b/github.com/aws/aws-sdk-go/NOTICE.txt index 5f14d1162e..899129ecc4 100644 --- a/github.com/aws/aws-sdk-go/NOTICE.txt +++ b/github.com/aws/aws-sdk-go/NOTICE.txt @@ -1,3 +1,3 @@ AWS SDK for Go -Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. Copyright 2014-2015 Stripe, Inc. diff --git a/github.com/aws/aws-sdk-go/aws/arn/arn.go b/github.com/aws/aws-sdk-go/aws/arn/arn.go new file mode 100644 index 0000000000..1c49674290 --- /dev/null +++ b/github.com/aws/aws-sdk-go/aws/arn/arn.go @@ -0,0 +1,93 @@ +// Package arn provides a parser for interacting with Amazon Resource Names. +package arn + +import ( + "errors" + "strings" +) + +const ( + arnDelimiter = ":" + arnSections = 6 + arnPrefix = "arn:" + + // zero-indexed + sectionPartition = 1 + sectionService = 2 + sectionRegion = 3 + sectionAccountID = 4 + sectionResource = 5 + + // errors + invalidPrefix = "arn: invalid prefix" + invalidSections = "arn: not enough sections" +) + +// ARN captures the individual fields of an Amazon Resource Name. +// See http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html for more information. +type ARN struct { + // The partition that the resource is in. For standard AWS regions, the partition is "aws". If you have resources in + // other partitions, the partition is "aws-partitionname". For example, the partition for resources in the China + // (Beijing) region is "aws-cn". + Partition string + + // The service namespace that identifies the AWS product (for example, Amazon S3, IAM, or Amazon RDS). For a list of + // namespaces, see + // http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html#genref-aws-service-namespaces. + Service string + + // The region the resource resides in. Note that the ARNs for some resources do not require a region, so this + // component might be omitted. + Region string + + // The ID of the AWS account that owns the resource, without the hyphens. For example, 123456789012. Note that the + // ARNs for some resources don't require an account number, so this component might be omitted. + AccountID string + + // The content of this part of the ARN varies by service. It often includes an indicator of the type of resource — + // for example, an IAM user or Amazon RDS database - followed by a slash (/) or a colon (:), followed by the + // resource name itself. Some services allows paths for resource names, as described in + // http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html#arns-paths. + Resource string +} + +// Parse parses an ARN into its constituent parts. +// +// Some example ARNs: +// arn:aws:elasticbeanstalk:us-east-1:123456789012:environment/My App/MyEnvironment +// arn:aws:iam::123456789012:user/David +// arn:aws:rds:eu-west-1:123456789012:db:mysql-db +// arn:aws:s3:::my_corporate_bucket/exampleobject.png +func Parse(arn string) (ARN, error) { + if !strings.HasPrefix(arn, arnPrefix) { + return ARN{}, errors.New(invalidPrefix) + } + sections := strings.SplitN(arn, arnDelimiter, arnSections) + if len(sections) != arnSections { + return ARN{}, errors.New(invalidSections) + } + return ARN{ + Partition: sections[sectionPartition], + Service: sections[sectionService], + Region: sections[sectionRegion], + AccountID: sections[sectionAccountID], + Resource: sections[sectionResource], + }, nil +} + +// IsARN returns whether the given string is an ARN by looking for +// whether the string starts with "arn:" and contains the correct number +// of sections delimited by colons(:). +func IsARN(arn string) bool { + return strings.HasPrefix(arn, arnPrefix) && strings.Count(arn, ":") >= arnSections-1 +} + +// String returns the canonical representation of the ARN +func (arn ARN) String() string { + return arnPrefix + + arn.Partition + arnDelimiter + + arn.Service + arnDelimiter + + arn.Region + arnDelimiter + + arn.AccountID + arnDelimiter + + arn.Resource +} diff --git a/github.com/aws/aws-sdk-go/aws/awserr/error.go b/github.com/aws/aws-sdk-go/aws/awserr/error.go index 56fdfc2bfc..99849c0e19 100644 --- a/github.com/aws/aws-sdk-go/aws/awserr/error.go +++ b/github.com/aws/aws-sdk-go/aws/awserr/error.go @@ -138,8 +138,27 @@ type RequestFailure interface { RequestID() string } -// NewRequestFailure returns a new request error wrapper for the given Error -// provided. +// NewRequestFailure returns a wrapped error with additional information for +// request status code, and service requestID. +// +// Should be used to wrap all request which involve service requests. Even if +// the request failed without a service response, but had an HTTP status code +// that may be meaningful. func NewRequestFailure(err Error, statusCode int, reqID string) RequestFailure { return newRequestError(err, statusCode, reqID) } + +// UnmarshalError provides the interface for the SDK failing to unmarshal data. +type UnmarshalError interface { + awsError + Bytes() []byte +} + +// NewUnmarshalError returns an initialized UnmarshalError error wrapper adding +// the bytes that fail to unmarshal to the error. +func NewUnmarshalError(err error, msg string, bytes []byte) UnmarshalError { + return &unmarshalError{ + awsError: New("UnmarshalError", msg, err), + bytes: bytes, + } +} diff --git a/github.com/aws/aws-sdk-go/aws/awserr/types.go b/github.com/aws/aws-sdk-go/aws/awserr/types.go index 0202a008f5..9cf7eaf400 100644 --- a/github.com/aws/aws-sdk-go/aws/awserr/types.go +++ b/github.com/aws/aws-sdk-go/aws/awserr/types.go @@ -1,6 +1,9 @@ package awserr -import "fmt" +import ( + "encoding/hex" + "fmt" +) // SprintError returns a string of the formatted error code. // @@ -119,6 +122,7 @@ type requestError struct { awsError statusCode int requestID string + bytes []byte } // newRequestError returns a wrapped error with additional information for @@ -170,6 +174,29 @@ func (r requestError) OrigErrs() []error { return []error{r.OrigErr()} } +type unmarshalError struct { + awsError + bytes []byte +} + +// Error returns the string representation of the error. +// Satisfies the error interface. +func (e unmarshalError) Error() string { + extra := hex.Dump(e.bytes) + return SprintError(e.Code(), e.Message(), extra, e.OrigErr()) +} + +// String returns the string representation of the error. +// Alias for Error to satisfy the stringer interface. +func (e unmarshalError) String() string { + return e.Error() +} + +// Bytes returns the bytes that failed to unmarshal. +func (e unmarshalError) Bytes() []byte { + return e.bytes +} + // An error list that satisfies the golang interface type errorList []error @@ -181,7 +208,7 @@ func (e errorList) Error() string { // How do we want to handle the array size being zero if size := len(e); size > 0 { for i := 0; i < size; i++ { - msg += fmt.Sprintf("%s", e[i].Error()) + msg += e[i].Error() // We check the next index to see if it is within the slice. // If it is, then we append a newline. We do this, because unit tests // could be broken with the additional '\n' diff --git a/github.com/aws/aws-sdk-go/aws/awsutil/equal.go b/github.com/aws/aws-sdk-go/aws/awsutil/equal.go index 59fa4a558a..142a7a01c5 100644 --- a/github.com/aws/aws-sdk-go/aws/awsutil/equal.go +++ b/github.com/aws/aws-sdk-go/aws/awsutil/equal.go @@ -15,7 +15,7 @@ func DeepEqual(a, b interface{}) bool { rb := reflect.Indirect(reflect.ValueOf(b)) if raValid, rbValid := ra.IsValid(), rb.IsValid(); !raValid && !rbValid { - // If the elements are both nil, and of the same type the are equal + // If the elements are both nil, and of the same type they are equal // If they are of different types they are not equal return reflect.TypeOf(a) == reflect.TypeOf(b) } else if raValid != rbValid { diff --git a/github.com/aws/aws-sdk-go/aws/awsutil/path_value.go b/github.com/aws/aws-sdk-go/aws/awsutil/path_value.go index 11c52c3896..a4eb6a7f43 100644 --- a/github.com/aws/aws-sdk-go/aws/awsutil/path_value.go +++ b/github.com/aws/aws-sdk-go/aws/awsutil/path_value.go @@ -70,7 +70,7 @@ func rValuesAtPath(v interface{}, path string, createPath, caseSensitive, nilTer value = value.FieldByNameFunc(func(name string) bool { if c == name { return true - } else if !caseSensitive && strings.ToLower(name) == strings.ToLower(c) { + } else if !caseSensitive && strings.EqualFold(name, c) { return true } return false @@ -185,13 +185,12 @@ func ValuesAtPath(i interface{}, path string) ([]interface{}, error) { // SetValueAtPath sets a value at the case insensitive lexical path inside // of a structure. func SetValueAtPath(i interface{}, path string, v interface{}) { - if rvals := rValuesAtPath(i, path, true, false, v == nil); rvals != nil { - for _, rval := range rvals { - if rval.Kind() == reflect.Ptr && rval.IsNil() { - continue - } - setValue(rval, v) + rvals := rValuesAtPath(i, path, true, false, v == nil) + for _, rval := range rvals { + if rval.Kind() == reflect.Ptr && rval.IsNil() { + continue } + setValue(rval, v) } } diff --git a/github.com/aws/aws-sdk-go/aws/client/client.go b/github.com/aws/aws-sdk-go/aws/client/client.go index 7096053840..03334d6920 100644 --- a/github.com/aws/aws-sdk-go/aws/client/client.go +++ b/github.com/aws/aws-sdk-go/aws/client/client.go @@ -12,6 +12,7 @@ import ( type Config struct { Config *aws.Config Handlers request.Handlers + PartitionID string Endpoint string SigningRegion string SigningName string @@ -64,7 +65,7 @@ func New(cfg aws.Config, info metadata.ClientInfo, handlers request.Handlers, op default: maxRetries := aws.IntValue(cfg.MaxRetries) if cfg.MaxRetries == nil || maxRetries == aws.UseServiceDefaultRetries { - maxRetries = 3 + maxRetries = DefaultRetryerMaxNumRetries } svc.Retryer = DefaultRetryer{NumMaxRetries: maxRetries} } diff --git a/github.com/aws/aws-sdk-go/aws/client/default_retryer.go b/github.com/aws/aws-sdk-go/aws/client/default_retryer.go index a397b0d044..9f6af19dd4 100644 --- a/github.com/aws/aws-sdk-go/aws/client/default_retryer.go +++ b/github.com/aws/aws-sdk-go/aws/client/default_retryer.go @@ -1,6 +1,7 @@ package client import ( + "math" "strconv" "time" @@ -9,82 +10,142 @@ import ( ) // DefaultRetryer implements basic retry logic using exponential backoff for -// most services. If you want to implement custom retry logic, implement the -// request.Retryer interface or create a structure type that composes this -// struct and override the specific methods. For example, to override only -// the MaxRetries method: +// most services. If you want to implement custom retry logic, you can implement the +// request.Retryer interface. // -// type retryer struct { -// client.DefaultRetryer -// } -// -// // This implementation always has 100 max retries -// func (d retryer) MaxRetries() int { return 100 } type DefaultRetryer struct { + // Num max Retries is the number of max retries that will be performed. + // By default, this is zero. NumMaxRetries int + + // MinRetryDelay is the minimum retry delay after which retry will be performed. + // If not set, the value is 0ns. + MinRetryDelay time.Duration + + // MinThrottleRetryDelay is the minimum retry delay when throttled. + // If not set, the value is 0ns. + MinThrottleDelay time.Duration + + // MaxRetryDelay is the maximum retry delay before which retry must be performed. + // If not set, the value is 0ns. + MaxRetryDelay time.Duration + + // MaxThrottleDelay is the maximum retry delay when throttled. + // If not set, the value is 0ns. + MaxThrottleDelay time.Duration } +const ( + // DefaultRetryerMaxNumRetries sets maximum number of retries + DefaultRetryerMaxNumRetries = 3 + + // DefaultRetryerMinRetryDelay sets minimum retry delay + DefaultRetryerMinRetryDelay = 30 * time.Millisecond + + // DefaultRetryerMinThrottleDelay sets minimum delay when throttled + DefaultRetryerMinThrottleDelay = 500 * time.Millisecond + + // DefaultRetryerMaxRetryDelay sets maximum retry delay + DefaultRetryerMaxRetryDelay = 300 * time.Second + + // DefaultRetryerMaxThrottleDelay sets maximum delay when throttled + DefaultRetryerMaxThrottleDelay = 300 * time.Second +) + // MaxRetries returns the number of maximum returns the service will use to make // an individual API request. func (d DefaultRetryer) MaxRetries() int { return d.NumMaxRetries } +// setRetryerDefaults sets the default values of the retryer if not set +func (d *DefaultRetryer) setRetryerDefaults() { + if d.MinRetryDelay == 0 { + d.MinRetryDelay = DefaultRetryerMinRetryDelay + } + if d.MaxRetryDelay == 0 { + d.MaxRetryDelay = DefaultRetryerMaxRetryDelay + } + if d.MinThrottleDelay == 0 { + d.MinThrottleDelay = DefaultRetryerMinThrottleDelay + } + if d.MaxThrottleDelay == 0 { + d.MaxThrottleDelay = DefaultRetryerMaxThrottleDelay + } +} + // RetryRules returns the delay duration before retrying this request again func (d DefaultRetryer) RetryRules(r *request.Request) time.Duration { - // Set the upper limit of delay in retrying at ~five minutes - minTime := 30 - throttle := d.shouldThrottle(r) - if throttle { - if delay, ok := getRetryDelay(r); ok { - return delay - } - minTime = 500 + // if number of max retries is zero, no retries will be performed. + if d.NumMaxRetries == 0 { + return 0 + } + + // Sets default value for retryer members + d.setRetryerDefaults() + + // minDelay is the minimum retryer delay + minDelay := d.MinRetryDelay + + var initialDelay time.Duration + + isThrottle := r.IsErrorThrottle() + if isThrottle { + if delay, ok := getRetryAfterDelay(r); ok { + initialDelay = delay + } + minDelay = d.MinThrottleDelay } retryCount := r.RetryCount - if throttle && retryCount > 8 { - retryCount = 8 - } else if retryCount > 13 { - retryCount = 13 + + // maxDelay the maximum retryer delay + maxDelay := d.MaxRetryDelay + + if isThrottle { + maxDelay = d.MaxThrottleDelay + } + + var delay time.Duration + + // Logic to cap the retry count based on the minDelay provided + actualRetryCount := int(math.Log2(float64(minDelay))) + 1 + if actualRetryCount < 63-retryCount { + delay = time.Duration(1< maxDelay { + delay = getJitterDelay(maxDelay / 2) + } + } else { + delay = getJitterDelay(maxDelay / 2) } + return delay + initialDelay +} - delay := (1 << uint(retryCount)) * (sdkrand.SeededRand.Intn(minTime) + minTime) - return time.Duration(delay) * time.Millisecond +// getJitterDelay returns a jittered delay for retry +func getJitterDelay(duration time.Duration) time.Duration { + return time.Duration(sdkrand.SeededRand.Int63n(int64(duration)) + int64(duration)) } // ShouldRetry returns true if the request should be retried. func (d DefaultRetryer) ShouldRetry(r *request.Request) bool { + + // ShouldRetry returns false if number of max retries is 0. + if d.NumMaxRetries == 0 { + return false + } + // If one of the other handlers already set the retry state // we don't want to override it based on the service's state if r.Retryable != nil { return *r.Retryable } - - if r.HTTPResponse.StatusCode >= 500 && r.HTTPResponse.StatusCode != 501 { - return true - } - return r.IsErrorRetryable() || d.shouldThrottle(r) -} - -// ShouldThrottle returns true if the request should be throttled. -func (d DefaultRetryer) shouldThrottle(r *request.Request) bool { - switch r.HTTPResponse.StatusCode { - case 429: - case 502: - case 503: - case 504: - default: - return r.IsErrorThrottle() - } - - return true + return r.IsErrorRetryable() || r.IsErrorThrottle() } // This will look in the Retry-After header, RFC 7231, for how long // it will wait before attempting another request -func getRetryDelay(r *request.Request) (time.Duration, bool) { +func getRetryAfterDelay(r *request.Request) (time.Duration, bool) { if !canUseRetryAfterHeader(r) { return 0, false } diff --git a/github.com/aws/aws-sdk-go/aws/client/logger.go b/github.com/aws/aws-sdk-go/aws/client/logger.go index ce9fb896d9..8958c32d4e 100644 --- a/github.com/aws/aws-sdk-go/aws/client/logger.go +++ b/github.com/aws/aws-sdk-go/aws/client/logger.go @@ -67,10 +67,14 @@ func logRequest(r *request.Request) { if !bodySeekable { r.SetReaderBody(aws.ReadSeekCloser(r.HTTPRequest.Body)) } - // Reset the request body because dumpRequest will re-wrap the r.HTTPRequest's - // Body as a NoOpCloser and will not be reset after read by the HTTP - // client reader. - r.ResetBody() + // Reset the request body because dumpRequest will re-wrap the + // r.HTTPRequest's Body as a NoOpCloser and will not be reset after + // read by the HTTP client reader. + if err := r.Error; err != nil { + r.Config.Logger.Log(fmt.Sprintf(logReqErrMsg, + r.ClientInfo.ServiceName, r.Operation.Name, err)) + return + } } r.Config.Logger.Log(fmt.Sprintf(logReqMsg, @@ -118,6 +122,12 @@ var LogHTTPResponseHandler = request.NamedHandler{ func logResponse(r *request.Request) { lw := &logWriter{r.Config.Logger, bytes.NewBuffer(nil)} + if r.HTTPResponse == nil { + lw.Logger.Log(fmt.Sprintf(logRespErrMsg, + r.ClientInfo.ServiceName, r.Operation.Name, "request's HTTPResponse is nil")) + return + } + logBody := r.Config.LogLevel.Matches(aws.LogDebugWithHTTPBody) if logBody { r.HTTPResponse.Body = &teeReaderCloser{ diff --git a/github.com/aws/aws-sdk-go/aws/client/metadata/client_info.go b/github.com/aws/aws-sdk-go/aws/client/metadata/client_info.go index 920e9fddf8..0c48f72e08 100644 --- a/github.com/aws/aws-sdk-go/aws/client/metadata/client_info.go +++ b/github.com/aws/aws-sdk-go/aws/client/metadata/client_info.go @@ -5,6 +5,7 @@ type ClientInfo struct { ServiceName string ServiceID string APIVersion string + PartitionID string Endpoint string SigningName string SigningRegion string diff --git a/github.com/aws/aws-sdk-go/aws/client/no_op_retryer.go b/github.com/aws/aws-sdk-go/aws/client/no_op_retryer.go new file mode 100644 index 0000000000..881d575f01 --- /dev/null +++ b/github.com/aws/aws-sdk-go/aws/client/no_op_retryer.go @@ -0,0 +1,28 @@ +package client + +import ( + "time" + + "github.com/aws/aws-sdk-go/aws/request" +) + +// NoOpRetryer provides a retryer that performs no retries. +// It should be used when we do not want retries to be performed. +type NoOpRetryer struct{} + +// MaxRetries returns the number of maximum returns the service will use to make +// an individual API; For NoOpRetryer the MaxRetries will always be zero. +func (d NoOpRetryer) MaxRetries() int { + return 0 +} + +// ShouldRetry will always return false for NoOpRetryer, as it should never retry. +func (d NoOpRetryer) ShouldRetry(_ *request.Request) bool { + return false +} + +// RetryRules returns the delay duration before retrying this request again; +// since NoOpRetryer does not retry, RetryRules always returns 0. +func (d NoOpRetryer) RetryRules(_ *request.Request) time.Duration { + return 0 +} diff --git a/github.com/aws/aws-sdk-go/aws/config.go b/github.com/aws/aws-sdk-go/aws/config.go index 10634d173d..3b809e8478 100644 --- a/github.com/aws/aws-sdk-go/aws/config.go +++ b/github.com/aws/aws-sdk-go/aws/config.go @@ -20,7 +20,7 @@ type RequestRetryer interface{} // A Config provides service configuration for service clients. By default, // all clients will use the defaults.DefaultConfig structure. // -// // Create Session with MaxRetry configuration to be shared by multiple +// // Create Session with MaxRetries configuration to be shared by multiple // // service clients. // sess := session.Must(session.NewSession(&aws.Config{ // MaxRetries: aws.Int(3), @@ -43,7 +43,7 @@ type Config struct { // An optional endpoint URL (hostname only or fully qualified URI) // that overrides the default generated endpoint for a client. Set this - // to `""` to use the default generated endpoint. + // to `nil` or the value to `""` to use the default generated endpoint. // // Note: You must still provide a `Region` value when specifying an // endpoint for a client. @@ -138,7 +138,7 @@ type Config struct { // `ExpectContinueTimeout` for information on adjusting the continue wait // timeout. https://golang.org/pkg/net/http/#Transport // - // You should use this flag to disble 100-Continue if you experience issues + // You should use this flag to disable 100-Continue if you experience issues // with proxies or third party S3 compatible services. S3Disable100Continue *bool @@ -161,6 +161,17 @@ type Config struct { // on GetObject API calls. S3DisableContentMD5Validation *bool + // Set this to `true` to have the S3 service client to use the region specified + // in the ARN, when an ARN is provided as an argument to a bucket parameter. + S3UseARNRegion *bool + + // Set this to `true` to enable the SDK to unmarshal API response header maps to + // normalized lower case map keys. + // + // For example S3's X-Amz-Meta prefixed header will be unmarshaled to lower case + // Metadata member's map keys. The value of the header in the map is unaffected. + LowerCaseHeaderMaps *bool + // Set this to `true` to disable the EC2Metadata client from overriding the // default http.Client's Timeout. This is helpful if you do not want the // EC2Metadata client to create a new http.Client. This options is only @@ -172,7 +183,7 @@ type Config struct { // // Example: // sess := session.Must(session.NewSession(aws.NewConfig() - // .WithEC2MetadataDiableTimeoutOverride(true))) + // .WithEC2MetadataDisableTimeoutOverride(true))) // // svc := s3.New(sess) // @@ -183,7 +194,7 @@ type Config struct { // both IPv4 and IPv6 addressing. // // Setting this for a service which does not support dual stack will fail - // to make requets. It is not recommended to set this value on the session + // to make requests. It is not recommended to set this value on the session // as it will apply to all service clients created with the session. Even // services which don't support dual stack endpoints. // @@ -227,6 +238,7 @@ type Config struct { // EnableEndpointDiscovery will allow for endpoint discovery on operations that // have the definition in its model. By default, endpoint discovery is off. + // To use EndpointDiscovery, Endpoint should be unset or set to an empty string. // // Example: // sess := session.Must(session.NewSession(&aws.Config{ @@ -246,12 +258,18 @@ type Config struct { // Disabling this feature is useful when you want to use local endpoints // for testing that do not support the modeled host prefix pattern. DisableEndpointHostPrefix *bool + + // STSRegionalEndpoint will enable regional or legacy endpoint resolving + STSRegionalEndpoint endpoints.STSRegionalEndpoint + + // S3UsEast1RegionalEndpoint will enable regional or legacy endpoint resolving + S3UsEast1RegionalEndpoint endpoints.S3UsEast1RegionalEndpoint } // NewConfig returns a new Config pointer that can be chained with builder // methods to set multiple configuration values inline without using pointers. // -// // Create Session with MaxRetry configuration to be shared by multiple +// // Create Session with MaxRetries configuration to be shared by multiple // // service clients. // sess := session.Must(session.NewSession(aws.NewConfig(). // WithMaxRetries(3), @@ -379,6 +397,13 @@ func (c *Config) WithS3DisableContentMD5Validation(enable bool) *Config { } +// WithS3UseARNRegion sets a config S3UseARNRegion value and +// returning a Config pointer for chaining +func (c *Config) WithS3UseARNRegion(enable bool) *Config { + c.S3UseARNRegion = &enable + return c +} + // WithUseDualStack sets a config UseDualStack value returning a Config // pointer for chaining. func (c *Config) WithUseDualStack(enable bool) *Config { @@ -420,6 +445,20 @@ func (c *Config) MergeIn(cfgs ...*Config) { } } +// WithSTSRegionalEndpoint will set whether or not to use regional endpoint flag +// when resolving the endpoint for a service +func (c *Config) WithSTSRegionalEndpoint(sre endpoints.STSRegionalEndpoint) *Config { + c.STSRegionalEndpoint = sre + return c +} + +// WithS3UsEast1RegionalEndpoint will set whether or not to use regional endpoint flag +// when resolving the endpoint for a service +func (c *Config) WithS3UsEast1RegionalEndpoint(sre endpoints.S3UsEast1RegionalEndpoint) *Config { + c.S3UsEast1RegionalEndpoint = sre + return c +} + func mergeInConfig(dst *Config, other *Config) { if other == nil { return @@ -493,6 +532,10 @@ func mergeInConfig(dst *Config, other *Config) { dst.S3DisableContentMD5Validation = other.S3DisableContentMD5Validation } + if other.S3UseARNRegion != nil { + dst.S3UseARNRegion = other.S3UseARNRegion + } + if other.UseDualStack != nil { dst.UseDualStack = other.UseDualStack } @@ -520,6 +563,14 @@ func mergeInConfig(dst *Config, other *Config) { if other.DisableEndpointHostPrefix != nil { dst.DisableEndpointHostPrefix = other.DisableEndpointHostPrefix } + + if other.STSRegionalEndpoint != endpoints.UnsetSTSEndpoint { + dst.STSRegionalEndpoint = other.STSRegionalEndpoint + } + + if other.S3UsEast1RegionalEndpoint != endpoints.UnsetS3UsEast1Endpoint { + dst.S3UsEast1RegionalEndpoint = other.S3UsEast1RegionalEndpoint + } } // Copy will return a shallow copy of the Config object. If any additional diff --git a/github.com/aws/aws-sdk-go/aws/context.go b/github.com/aws/aws-sdk-go/aws/context_1_5.go similarity index 58% rename from github.com/aws/aws-sdk-go/aws/context.go rename to github.com/aws/aws-sdk-go/aws/context_1_5.go index 79f426853b..2866f9a7fb 100644 --- a/github.com/aws/aws-sdk-go/aws/context.go +++ b/github.com/aws/aws-sdk-go/aws/context_1_5.go @@ -1,8 +1,8 @@ +// +build !go1.9 + package aws -import ( - "time" -) +import "time" // Context is an copy of the Go v1.7 stdlib's context.Context interface. // It is represented as a SDK interface to enable you to use the "WithContext" @@ -35,37 +35,3 @@ type Context interface { // functions. Value(key interface{}) interface{} } - -// BackgroundContext returns a context that will never be canceled, has no -// values, and no deadline. This context is used by the SDK to provide -// backwards compatibility with non-context API operations and functionality. -// -// Go 1.6 and before: -// This context function is equivalent to context.Background in the Go stdlib. -// -// Go 1.7 and later: -// The context returned will be the value returned by context.Background() -// -// See https://golang.org/pkg/context for more information on Contexts. -func BackgroundContext() Context { - return backgroundCtx -} - -// SleepWithContext will wait for the timer duration to expire, or the context -// is canceled. Which ever happens first. If the context is canceled the Context's -// error will be returned. -// -// Expects Context to always return a non-nil error if the Done channel is closed. -func SleepWithContext(ctx Context, dur time.Duration) error { - t := time.NewTimer(dur) - defer t.Stop() - - select { - case <-t.C: - break - case <-ctx.Done(): - return ctx.Err() - } - - return nil -} diff --git a/github.com/aws/aws-sdk-go/aws/context_1_7.go b/github.com/aws/aws-sdk-go/aws/context_1_7.go deleted file mode 100644 index 064f75c925..0000000000 --- a/github.com/aws/aws-sdk-go/aws/context_1_7.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build go1.7 - -package aws - -import "context" - -var ( - backgroundCtx = context.Background() -) diff --git a/github.com/aws/aws-sdk-go/aws/context_1_9.go b/github.com/aws/aws-sdk-go/aws/context_1_9.go new file mode 100644 index 0000000000..3718b26e10 --- /dev/null +++ b/github.com/aws/aws-sdk-go/aws/context_1_9.go @@ -0,0 +1,11 @@ +// +build go1.9 + +package aws + +import "context" + +// Context is an alias of the Go stdlib's context.Context interface. +// It can be used within the SDK's API operation "WithContext" methods. +// +// See https://golang.org/pkg/context on how to use contexts. +type Context = context.Context diff --git a/github.com/aws/aws-sdk-go/aws/context_background_1_5.go b/github.com/aws/aws-sdk-go/aws/context_background_1_5.go new file mode 100644 index 0000000000..2f9446333a --- /dev/null +++ b/github.com/aws/aws-sdk-go/aws/context_background_1_5.go @@ -0,0 +1,22 @@ +// +build !go1.7 + +package aws + +import ( + "github.com/aws/aws-sdk-go/internal/context" +) + +// BackgroundContext returns a context that will never be canceled, has no +// values, and no deadline. This context is used by the SDK to provide +// backwards compatibility with non-context API operations and functionality. +// +// Go 1.6 and before: +// This context function is equivalent to context.Background in the Go stdlib. +// +// Go 1.7 and later: +// The context returned will be the value returned by context.Background() +// +// See https://golang.org/pkg/context for more information on Contexts. +func BackgroundContext() Context { + return context.BackgroundCtx +} diff --git a/github.com/aws/aws-sdk-go/aws/context_background_1_7.go b/github.com/aws/aws-sdk-go/aws/context_background_1_7.go new file mode 100644 index 0000000000..9c29f29af1 --- /dev/null +++ b/github.com/aws/aws-sdk-go/aws/context_background_1_7.go @@ -0,0 +1,20 @@ +// +build go1.7 + +package aws + +import "context" + +// BackgroundContext returns a context that will never be canceled, has no +// values, and no deadline. This context is used by the SDK to provide +// backwards compatibility with non-context API operations and functionality. +// +// Go 1.6 and before: +// This context function is equivalent to context.Background in the Go stdlib. +// +// Go 1.7 and later: +// The context returned will be the value returned by context.Background() +// +// See https://golang.org/pkg/context for more information on Contexts. +func BackgroundContext() Context { + return context.Background() +} diff --git a/github.com/aws/aws-sdk-go/aws/context_sleep.go b/github.com/aws/aws-sdk-go/aws/context_sleep.go new file mode 100644 index 0000000000..304fd15612 --- /dev/null +++ b/github.com/aws/aws-sdk-go/aws/context_sleep.go @@ -0,0 +1,24 @@ +package aws + +import ( + "time" +) + +// SleepWithContext will wait for the timer duration to expire, or the context +// is canceled. Which ever happens first. If the context is canceled the Context's +// error will be returned. +// +// Expects Context to always return a non-nil error if the Done channel is closed. +func SleepWithContext(ctx Context, dur time.Duration) error { + t := time.NewTimer(dur) + defer t.Stop() + + select { + case <-t.C: + break + case <-ctx.Done(): + return ctx.Err() + } + + return nil +} diff --git a/github.com/aws/aws-sdk-go/aws/convert_types.go b/github.com/aws/aws-sdk-go/aws/convert_types.go index ff5d58e068..4e076c1837 100644 --- a/github.com/aws/aws-sdk-go/aws/convert_types.go +++ b/github.com/aws/aws-sdk-go/aws/convert_types.go @@ -179,6 +179,242 @@ func IntValueMap(src map[string]*int) map[string]int { return dst } +// Uint returns a pointer to the uint value passed in. +func Uint(v uint) *uint { + return &v +} + +// UintValue returns the value of the uint pointer passed in or +// 0 if the pointer is nil. +func UintValue(v *uint) uint { + if v != nil { + return *v + } + return 0 +} + +// UintSlice converts a slice of uint values uinto a slice of +// uint pointers +func UintSlice(src []uint) []*uint { + dst := make([]*uint, len(src)) + for i := 0; i < len(src); i++ { + dst[i] = &(src[i]) + } + return dst +} + +// UintValueSlice converts a slice of uint pointers uinto a slice of +// uint values +func UintValueSlice(src []*uint) []uint { + dst := make([]uint, len(src)) + for i := 0; i < len(src); i++ { + if src[i] != nil { + dst[i] = *(src[i]) + } + } + return dst +} + +// UintMap converts a string map of uint values uinto a string +// map of uint pointers +func UintMap(src map[string]uint) map[string]*uint { + dst := make(map[string]*uint) + for k, val := range src { + v := val + dst[k] = &v + } + return dst +} + +// UintValueMap converts a string map of uint pointers uinto a string +// map of uint values +func UintValueMap(src map[string]*uint) map[string]uint { + dst := make(map[string]uint) + for k, val := range src { + if val != nil { + dst[k] = *val + } + } + return dst +} + +// Int8 returns a pointer to the int8 value passed in. +func Int8(v int8) *int8 { + return &v +} + +// Int8Value returns the value of the int8 pointer passed in or +// 0 if the pointer is nil. +func Int8Value(v *int8) int8 { + if v != nil { + return *v + } + return 0 +} + +// Int8Slice converts a slice of int8 values into a slice of +// int8 pointers +func Int8Slice(src []int8) []*int8 { + dst := make([]*int8, len(src)) + for i := 0; i < len(src); i++ { + dst[i] = &(src[i]) + } + return dst +} + +// Int8ValueSlice converts a slice of int8 pointers into a slice of +// int8 values +func Int8ValueSlice(src []*int8) []int8 { + dst := make([]int8, len(src)) + for i := 0; i < len(src); i++ { + if src[i] != nil { + dst[i] = *(src[i]) + } + } + return dst +} + +// Int8Map converts a string map of int8 values into a string +// map of int8 pointers +func Int8Map(src map[string]int8) map[string]*int8 { + dst := make(map[string]*int8) + for k, val := range src { + v := val + dst[k] = &v + } + return dst +} + +// Int8ValueMap converts a string map of int8 pointers into a string +// map of int8 values +func Int8ValueMap(src map[string]*int8) map[string]int8 { + dst := make(map[string]int8) + for k, val := range src { + if val != nil { + dst[k] = *val + } + } + return dst +} + +// Int16 returns a pointer to the int16 value passed in. +func Int16(v int16) *int16 { + return &v +} + +// Int16Value returns the value of the int16 pointer passed in or +// 0 if the pointer is nil. +func Int16Value(v *int16) int16 { + if v != nil { + return *v + } + return 0 +} + +// Int16Slice converts a slice of int16 values into a slice of +// int16 pointers +func Int16Slice(src []int16) []*int16 { + dst := make([]*int16, len(src)) + for i := 0; i < len(src); i++ { + dst[i] = &(src[i]) + } + return dst +} + +// Int16ValueSlice converts a slice of int16 pointers into a slice of +// int16 values +func Int16ValueSlice(src []*int16) []int16 { + dst := make([]int16, len(src)) + for i := 0; i < len(src); i++ { + if src[i] != nil { + dst[i] = *(src[i]) + } + } + return dst +} + +// Int16Map converts a string map of int16 values into a string +// map of int16 pointers +func Int16Map(src map[string]int16) map[string]*int16 { + dst := make(map[string]*int16) + for k, val := range src { + v := val + dst[k] = &v + } + return dst +} + +// Int16ValueMap converts a string map of int16 pointers into a string +// map of int16 values +func Int16ValueMap(src map[string]*int16) map[string]int16 { + dst := make(map[string]int16) + for k, val := range src { + if val != nil { + dst[k] = *val + } + } + return dst +} + +// Int32 returns a pointer to the int32 value passed in. +func Int32(v int32) *int32 { + return &v +} + +// Int32Value returns the value of the int32 pointer passed in or +// 0 if the pointer is nil. +func Int32Value(v *int32) int32 { + if v != nil { + return *v + } + return 0 +} + +// Int32Slice converts a slice of int32 values into a slice of +// int32 pointers +func Int32Slice(src []int32) []*int32 { + dst := make([]*int32, len(src)) + for i := 0; i < len(src); i++ { + dst[i] = &(src[i]) + } + return dst +} + +// Int32ValueSlice converts a slice of int32 pointers into a slice of +// int32 values +func Int32ValueSlice(src []*int32) []int32 { + dst := make([]int32, len(src)) + for i := 0; i < len(src); i++ { + if src[i] != nil { + dst[i] = *(src[i]) + } + } + return dst +} + +// Int32Map converts a string map of int32 values into a string +// map of int32 pointers +func Int32Map(src map[string]int32) map[string]*int32 { + dst := make(map[string]*int32) + for k, val := range src { + v := val + dst[k] = &v + } + return dst +} + +// Int32ValueMap converts a string map of int32 pointers into a string +// map of int32 values +func Int32ValueMap(src map[string]*int32) map[string]int32 { + dst := make(map[string]int32) + for k, val := range src { + if val != nil { + dst[k] = *val + } + } + return dst +} + // Int64 returns a pointer to the int64 value passed in. func Int64(v int64) *int64 { return &v @@ -238,6 +474,301 @@ func Int64ValueMap(src map[string]*int64) map[string]int64 { return dst } +// Uint8 returns a pointer to the uint8 value passed in. +func Uint8(v uint8) *uint8 { + return &v +} + +// Uint8Value returns the value of the uint8 pointer passed in or +// 0 if the pointer is nil. +func Uint8Value(v *uint8) uint8 { + if v != nil { + return *v + } + return 0 +} + +// Uint8Slice converts a slice of uint8 values into a slice of +// uint8 pointers +func Uint8Slice(src []uint8) []*uint8 { + dst := make([]*uint8, len(src)) + for i := 0; i < len(src); i++ { + dst[i] = &(src[i]) + } + return dst +} + +// Uint8ValueSlice converts a slice of uint8 pointers into a slice of +// uint8 values +func Uint8ValueSlice(src []*uint8) []uint8 { + dst := make([]uint8, len(src)) + for i := 0; i < len(src); i++ { + if src[i] != nil { + dst[i] = *(src[i]) + } + } + return dst +} + +// Uint8Map converts a string map of uint8 values into a string +// map of uint8 pointers +func Uint8Map(src map[string]uint8) map[string]*uint8 { + dst := make(map[string]*uint8) + for k, val := range src { + v := val + dst[k] = &v + } + return dst +} + +// Uint8ValueMap converts a string map of uint8 pointers into a string +// map of uint8 values +func Uint8ValueMap(src map[string]*uint8) map[string]uint8 { + dst := make(map[string]uint8) + for k, val := range src { + if val != nil { + dst[k] = *val + } + } + return dst +} + +// Uint16 returns a pointer to the uint16 value passed in. +func Uint16(v uint16) *uint16 { + return &v +} + +// Uint16Value returns the value of the uint16 pointer passed in or +// 0 if the pointer is nil. +func Uint16Value(v *uint16) uint16 { + if v != nil { + return *v + } + return 0 +} + +// Uint16Slice converts a slice of uint16 values into a slice of +// uint16 pointers +func Uint16Slice(src []uint16) []*uint16 { + dst := make([]*uint16, len(src)) + for i := 0; i < len(src); i++ { + dst[i] = &(src[i]) + } + return dst +} + +// Uint16ValueSlice converts a slice of uint16 pointers into a slice of +// uint16 values +func Uint16ValueSlice(src []*uint16) []uint16 { + dst := make([]uint16, len(src)) + for i := 0; i < len(src); i++ { + if src[i] != nil { + dst[i] = *(src[i]) + } + } + return dst +} + +// Uint16Map converts a string map of uint16 values into a string +// map of uint16 pointers +func Uint16Map(src map[string]uint16) map[string]*uint16 { + dst := make(map[string]*uint16) + for k, val := range src { + v := val + dst[k] = &v + } + return dst +} + +// Uint16ValueMap converts a string map of uint16 pointers into a string +// map of uint16 values +func Uint16ValueMap(src map[string]*uint16) map[string]uint16 { + dst := make(map[string]uint16) + for k, val := range src { + if val != nil { + dst[k] = *val + } + } + return dst +} + +// Uint32 returns a pointer to the uint32 value passed in. +func Uint32(v uint32) *uint32 { + return &v +} + +// Uint32Value returns the value of the uint32 pointer passed in or +// 0 if the pointer is nil. +func Uint32Value(v *uint32) uint32 { + if v != nil { + return *v + } + return 0 +} + +// Uint32Slice converts a slice of uint32 values into a slice of +// uint32 pointers +func Uint32Slice(src []uint32) []*uint32 { + dst := make([]*uint32, len(src)) + for i := 0; i < len(src); i++ { + dst[i] = &(src[i]) + } + return dst +} + +// Uint32ValueSlice converts a slice of uint32 pointers into a slice of +// uint32 values +func Uint32ValueSlice(src []*uint32) []uint32 { + dst := make([]uint32, len(src)) + for i := 0; i < len(src); i++ { + if src[i] != nil { + dst[i] = *(src[i]) + } + } + return dst +} + +// Uint32Map converts a string map of uint32 values into a string +// map of uint32 pointers +func Uint32Map(src map[string]uint32) map[string]*uint32 { + dst := make(map[string]*uint32) + for k, val := range src { + v := val + dst[k] = &v + } + return dst +} + +// Uint32ValueMap converts a string map of uint32 pointers into a string +// map of uint32 values +func Uint32ValueMap(src map[string]*uint32) map[string]uint32 { + dst := make(map[string]uint32) + for k, val := range src { + if val != nil { + dst[k] = *val + } + } + return dst +} + +// Uint64 returns a pointer to the uint64 value passed in. +func Uint64(v uint64) *uint64 { + return &v +} + +// Uint64Value returns the value of the uint64 pointer passed in or +// 0 if the pointer is nil. +func Uint64Value(v *uint64) uint64 { + if v != nil { + return *v + } + return 0 +} + +// Uint64Slice converts a slice of uint64 values into a slice of +// uint64 pointers +func Uint64Slice(src []uint64) []*uint64 { + dst := make([]*uint64, len(src)) + for i := 0; i < len(src); i++ { + dst[i] = &(src[i]) + } + return dst +} + +// Uint64ValueSlice converts a slice of uint64 pointers into a slice of +// uint64 values +func Uint64ValueSlice(src []*uint64) []uint64 { + dst := make([]uint64, len(src)) + for i := 0; i < len(src); i++ { + if src[i] != nil { + dst[i] = *(src[i]) + } + } + return dst +} + +// Uint64Map converts a string map of uint64 values into a string +// map of uint64 pointers +func Uint64Map(src map[string]uint64) map[string]*uint64 { + dst := make(map[string]*uint64) + for k, val := range src { + v := val + dst[k] = &v + } + return dst +} + +// Uint64ValueMap converts a string map of uint64 pointers into a string +// map of uint64 values +func Uint64ValueMap(src map[string]*uint64) map[string]uint64 { + dst := make(map[string]uint64) + for k, val := range src { + if val != nil { + dst[k] = *val + } + } + return dst +} + +// Float32 returns a pointer to the float32 value passed in. +func Float32(v float32) *float32 { + return &v +} + +// Float32Value returns the value of the float32 pointer passed in or +// 0 if the pointer is nil. +func Float32Value(v *float32) float32 { + if v != nil { + return *v + } + return 0 +} + +// Float32Slice converts a slice of float32 values into a slice of +// float32 pointers +func Float32Slice(src []float32) []*float32 { + dst := make([]*float32, len(src)) + for i := 0; i < len(src); i++ { + dst[i] = &(src[i]) + } + return dst +} + +// Float32ValueSlice converts a slice of float32 pointers into a slice of +// float32 values +func Float32ValueSlice(src []*float32) []float32 { + dst := make([]float32, len(src)) + for i := 0; i < len(src); i++ { + if src[i] != nil { + dst[i] = *(src[i]) + } + } + return dst +} + +// Float32Map converts a string map of float32 values into a string +// map of float32 pointers +func Float32Map(src map[string]float32) map[string]*float32 { + dst := make(map[string]*float32) + for k, val := range src { + v := val + dst[k] = &v + } + return dst +} + +// Float32ValueMap converts a string map of float32 pointers into a string +// map of float32 values +func Float32ValueMap(src map[string]*float32) map[string]float32 { + dst := make(map[string]float32) + for k, val := range src { + if val != nil { + dst[k] = *val + } + } + return dst +} + // Float64 returns a pointer to the float64 value passed in. func Float64(v float64) *float64 { return &v diff --git a/github.com/aws/aws-sdk-go/aws/corehandlers/handlers.go b/github.com/aws/aws-sdk-go/aws/corehandlers/handlers.go index f8853d78af..d95a5eb540 100644 --- a/github.com/aws/aws-sdk-go/aws/corehandlers/handlers.go +++ b/github.com/aws/aws-sdk-go/aws/corehandlers/handlers.go @@ -159,9 +159,9 @@ func handleSendError(r *request.Request, err error) { Body: ioutil.NopCloser(bytes.NewReader([]byte{})), } } - // Catch all other request errors. - r.Error = awserr.New("RequestError", "send request failed", err) - r.Retryable = aws.Bool(true) // network errors are retryable + // Catch all request errors, and let the default retrier determine + // if the error is retryable. + r.Error = awserr.New(request.ErrCodeRequestError, "send request failed", err) // Override the error with a context canceled error, if that was canceled. ctx := r.Context() @@ -184,37 +184,39 @@ var ValidateResponseHandler = request.NamedHandler{Name: "core.ValidateResponseH // AfterRetryHandler performs final checks to determine if the request should // be retried and how long to delay. -var AfterRetryHandler = request.NamedHandler{Name: "core.AfterRetryHandler", Fn: func(r *request.Request) { - // If one of the other handlers already set the retry state - // we don't want to override it based on the service's state - if r.Retryable == nil || aws.BoolValue(r.Config.EnforceShouldRetryCheck) { - r.Retryable = aws.Bool(r.ShouldRetry(r)) - } +var AfterRetryHandler = request.NamedHandler{ + Name: "core.AfterRetryHandler", + Fn: func(r *request.Request) { + // If one of the other handlers already set the retry state + // we don't want to override it based on the service's state + if r.Retryable == nil || aws.BoolValue(r.Config.EnforceShouldRetryCheck) { + r.Retryable = aws.Bool(r.ShouldRetry(r)) + } - if r.WillRetry() { - r.RetryDelay = r.RetryRules(r) + if r.WillRetry() { + r.RetryDelay = r.RetryRules(r) - if sleepFn := r.Config.SleepDelay; sleepFn != nil { - // Support SleepDelay for backwards compatibility and testing - sleepFn(r.RetryDelay) - } else if err := aws.SleepWithContext(r.Context(), r.RetryDelay); err != nil { - r.Error = awserr.New(request.CanceledErrorCode, - "request context canceled", err) - r.Retryable = aws.Bool(false) - return - } + if sleepFn := r.Config.SleepDelay; sleepFn != nil { + // Support SleepDelay for backwards compatibility and testing + sleepFn(r.RetryDelay) + } else if err := aws.SleepWithContext(r.Context(), r.RetryDelay); err != nil { + r.Error = awserr.New(request.CanceledErrorCode, + "request context canceled", err) + r.Retryable = aws.Bool(false) + return + } - // when the expired token exception occurs the credentials - // need to be expired locally so that the next request to - // get credentials will trigger a credentials refresh. - if r.IsErrorExpired() { - r.Config.Credentials.Expire() - } + // when the expired token exception occurs the credentials + // need to be expired locally so that the next request to + // get credentials will trigger a credentials refresh. + if r.IsErrorExpired() { + r.Config.Credentials.Expire() + } - r.RetryCount++ - r.Error = nil - } -}} + r.RetryCount++ + r.Error = nil + } + }} // ValidateEndpointHandler is a request handler to validate a request had the // appropriate Region and Endpoint set. Will set r.Error if the endpoint or @@ -223,6 +225,8 @@ var ValidateEndpointHandler = request.NamedHandler{Name: "core.ValidateEndpointH if r.ClientInfo.SigningRegion == "" && aws.StringValue(r.Config.Region) == "" { r.Error = aws.ErrMissingRegion } else if r.ClientInfo.Endpoint == "" { + // Was any endpoint provided by the user, or one was derived by the + // SDK's endpoint resolver? r.Error = aws.ErrMissingEndpoint } }} diff --git a/github.com/aws/aws-sdk-go/aws/credentials/context_background_go1.5.go b/github.com/aws/aws-sdk-go/aws/credentials/context_background_go1.5.go new file mode 100644 index 0000000000..5852b26487 --- /dev/null +++ b/github.com/aws/aws-sdk-go/aws/credentials/context_background_go1.5.go @@ -0,0 +1,22 @@ +// +build !go1.7 + +package credentials + +import ( + "github.com/aws/aws-sdk-go/internal/context" +) + +// backgroundContext returns a context that will never be canceled, has no +// values, and no deadline. This context is used by the SDK to provide +// backwards compatibility with non-context API operations and functionality. +// +// Go 1.6 and before: +// This context function is equivalent to context.Background in the Go stdlib. +// +// Go 1.7 and later: +// The context returned will be the value returned by context.Background() +// +// See https://golang.org/pkg/context for more information on Contexts. +func backgroundContext() Context { + return context.BackgroundCtx +} diff --git a/github.com/aws/aws-sdk-go/aws/credentials/context_background_go1.7.go b/github.com/aws/aws-sdk-go/aws/credentials/context_background_go1.7.go new file mode 100644 index 0000000000..388b215418 --- /dev/null +++ b/github.com/aws/aws-sdk-go/aws/credentials/context_background_go1.7.go @@ -0,0 +1,20 @@ +// +build go1.7 + +package credentials + +import "context" + +// backgroundContext returns a context that will never be canceled, has no +// values, and no deadline. This context is used by the SDK to provide +// backwards compatibility with non-context API operations and functionality. +// +// Go 1.6 and before: +// This context function is equivalent to context.Background in the Go stdlib. +// +// Go 1.7 and later: +// The context returned will be the value returned by context.Background() +// +// See https://golang.org/pkg/context for more information on Contexts. +func backgroundContext() Context { + return context.Background() +} diff --git a/github.com/aws/aws-sdk-go/aws/credentials/context_go1.5.go b/github.com/aws/aws-sdk-go/aws/credentials/context_go1.5.go new file mode 100644 index 0000000000..8152a864ad --- /dev/null +++ b/github.com/aws/aws-sdk-go/aws/credentials/context_go1.5.go @@ -0,0 +1,39 @@ +// +build !go1.9 + +package credentials + +import "time" + +// Context is an copy of the Go v1.7 stdlib's context.Context interface. +// It is represented as a SDK interface to enable you to use the "WithContext" +// API methods with Go v1.6 and a Context type such as golang.org/x/net/context. +// +// This type, aws.Context, and context.Context are equivalent. +// +// See https://golang.org/pkg/context on how to use contexts. +type Context interface { + // Deadline returns the time when work done on behalf of this context + // should be canceled. Deadline returns ok==false when no deadline is + // set. Successive calls to Deadline return the same results. + Deadline() (deadline time.Time, ok bool) + + // Done returns a channel that's closed when work done on behalf of this + // context should be canceled. Done may return nil if this context can + // never be canceled. Successive calls to Done return the same value. + Done() <-chan struct{} + + // Err returns a non-nil error value after Done is closed. Err returns + // Canceled if the context was canceled or DeadlineExceeded if the + // context's deadline passed. No other values for Err are defined. + // After Done is closed, successive calls to Err return the same value. + Err() error + + // Value returns the value associated with this context for key, or nil + // if no value is associated with key. Successive calls to Value with + // the same key returns the same result. + // + // Use context values only for request-scoped data that transits + // processes and API boundaries, not for passing optional parameters to + // functions. + Value(key interface{}) interface{} +} diff --git a/github.com/aws/aws-sdk-go/aws/credentials/context_go1.9.go b/github.com/aws/aws-sdk-go/aws/credentials/context_go1.9.go new file mode 100644 index 0000000000..4356edb3d5 --- /dev/null +++ b/github.com/aws/aws-sdk-go/aws/credentials/context_go1.9.go @@ -0,0 +1,13 @@ +// +build go1.9 + +package credentials + +import "context" + +// Context is an alias of the Go stdlib's context.Context interface. +// It can be used within the SDK's API operation "WithContext" methods. +// +// This type, aws.Context, and context.Context are equivalent. +// +// See https://golang.org/pkg/context on how to use contexts. +type Context = context.Context diff --git a/github.com/aws/aws-sdk-go/aws/credentials/credentials.go b/github.com/aws/aws-sdk-go/aws/credentials/credentials.go index 894bbc7f82..9f8fd92a50 100644 --- a/github.com/aws/aws-sdk-go/aws/credentials/credentials.go +++ b/github.com/aws/aws-sdk-go/aws/credentials/credentials.go @@ -50,9 +50,11 @@ package credentials import ( "fmt" - "github.com/aws/aws-sdk-go/aws/awserr" - "sync" + "sync/atomic" "time" + + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/internal/sync/singleflight" ) // AnonymousCredentials is an empty Credential object that can be used as @@ -83,6 +85,12 @@ type Value struct { ProviderName string } +// HasKeys returns if the credentials Value has both AccessKeyID and +// SecretAccessKey value set. +func (v Value) HasKeys() bool { + return len(v.AccessKeyID) != 0 && len(v.SecretAccessKey) != 0 +} + // A Provider is the interface for any component which will provide credentials // Value. A provider is required to manage its own Expired state, and what to // be expired means. @@ -99,6 +107,13 @@ type Provider interface { IsExpired() bool } +// ProviderWithContext is a Provider that can retrieve credentials with a Context +type ProviderWithContext interface { + Provider + + RetrieveWithContext(Context) (Value, error) +} + // An Expirer is an interface that Providers can implement to expose the expiration // time, if known. If the Provider cannot accurately provide this info, // it should not implement this interface. @@ -190,24 +205,24 @@ func (e *Expiry) ExpiresAt() time.Time { // first instance of the credentials Value. All calls to Get() after that // will return the cached credentials Value until IsExpired() returns true. type Credentials struct { - creds Value - forceRefresh bool - - m sync.RWMutex + creds atomic.Value + sf singleflight.Group provider Provider } // NewCredentials returns a pointer to a new Credentials with the provider set. func NewCredentials(provider Provider) *Credentials { - return &Credentials{ - provider: provider, - forceRefresh: true, + c := &Credentials{ + provider: provider, } + c.creds.Store(Value{}) + return c } -// Get returns the credentials value, or error if the credentials Value failed -// to be retrieved. +// GetWithContext returns the credentials value, or error if the credentials +// Value failed to be retrieved. Will return early if the passed in context is +// canceled. // // Will return the cached credentials Value if it has not expired. If the // credentials Value has expired the Provider's Retrieve() will be called @@ -215,31 +230,56 @@ func NewCredentials(provider Provider) *Credentials { // // If Credentials.Expire() was called the credentials Value will be force // expired, and the next call to Get() will cause them to be refreshed. -func (c *Credentials) Get() (Value, error) { - // Check the cached credentials first with just the read lock. - c.m.RLock() - if !c.isExpired() { - creds := c.creds - c.m.RUnlock() - return creds, nil +// +// Passed in Context is equivalent to aws.Context, and context.Context. +func (c *Credentials) GetWithContext(ctx Context) (Value, error) { + if curCreds := c.creds.Load(); !c.isExpired(curCreds) { + return curCreds.(Value), nil + } + + // Cannot pass context down to the actual retrieve, because the first + // context would cancel the whole group when there is not direct + // association of items in the group. + resCh := c.sf.DoChan("", func() (interface{}, error) { + return c.singleRetrieve(&suppressedContext{ctx}) + }) + select { + case res := <-resCh: + return res.Val.(Value), res.Err + case <-ctx.Done(): + return Value{}, awserr.New("RequestCanceled", + "request context canceled", ctx.Err()) } - c.m.RUnlock() - - // Credentials are expired need to retrieve the credentials taking the full - // lock. - c.m.Lock() - defer c.m.Unlock() - - if c.isExpired() { - creds, err := c.provider.Retrieve() - if err != nil { - return Value{}, err - } - c.creds = creds - c.forceRefresh = false +} + +func (c *Credentials) singleRetrieve(ctx Context) (creds interface{}, err error) { + if curCreds := c.creds.Load(); !c.isExpired(curCreds) { + return curCreds.(Value), nil } - return c.creds, nil + if p, ok := c.provider.(ProviderWithContext); ok { + creds, err = p.RetrieveWithContext(ctx) + } else { + creds, err = c.provider.Retrieve() + } + if err == nil { + c.creds.Store(creds) + } + + return creds, err +} + +// Get returns the credentials value, or error if the credentials Value failed +// to be retrieved. +// +// Will return the cached credentials Value if it has not expired. If the +// credentials Value has expired the Provider's Retrieve() will be called +// to refresh the credentials. +// +// If Credentials.Expire() was called the credentials Value will be force +// expired, and the next call to Get() will cause them to be refreshed. +func (c *Credentials) Get() (Value, error) { + return c.GetWithContext(backgroundContext()) } // Expire expires the credentials and forces them to be retrieved on the @@ -248,10 +288,7 @@ func (c *Credentials) Get() (Value, error) { // This will override the Provider's expired state, and force Credentials // to call the Provider's Retrieve(). func (c *Credentials) Expire() { - c.m.Lock() - defer c.m.Unlock() - - c.forceRefresh = true + c.creds.Store(Value{}) } // IsExpired returns if the credentials are no longer valid, and need @@ -260,33 +297,43 @@ func (c *Credentials) Expire() { // If the Credentials were forced to be expired with Expire() this will // reflect that override. func (c *Credentials) IsExpired() bool { - c.m.RLock() - defer c.m.RUnlock() - - return c.isExpired() + return c.isExpired(c.creds.Load()) } // isExpired helper method wrapping the definition of expired credentials. -func (c *Credentials) isExpired() bool { - return c.forceRefresh || c.provider.IsExpired() +func (c *Credentials) isExpired(creds interface{}) bool { + return creds == nil || creds.(Value) == Value{} || c.provider.IsExpired() } // ExpiresAt provides access to the functionality of the Expirer interface of // the underlying Provider, if it supports that interface. Otherwise, it returns // an error. func (c *Credentials) ExpiresAt() (time.Time, error) { - c.m.RLock() - defer c.m.RUnlock() - expirer, ok := c.provider.(Expirer) if !ok { return time.Time{}, awserr.New("ProviderNotExpirer", - fmt.Sprintf("provider %s does not support ExpiresAt()", c.creds.ProviderName), + fmt.Sprintf("provider %s does not support ExpiresAt()", c.creds.Load().(Value).ProviderName), nil) } - if c.forceRefresh { + if c.creds.Load().(Value) == (Value{}) { // set expiration time to the distant past return time.Time{}, nil } return expirer.ExpiresAt(), nil } + +type suppressedContext struct { + Context +} + +func (s *suppressedContext) Deadline() (deadline time.Time, ok bool) { + return time.Time{}, false +} + +func (s *suppressedContext) Done() <-chan struct{} { + return nil +} + +func (s *suppressedContext) Err() error { + return nil +} diff --git a/github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds/ec2_role_provider.go b/github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds/ec2_role_provider.go index 0ed791be64..92af5b7250 100644 --- a/github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds/ec2_role_provider.go +++ b/github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds/ec2_role_provider.go @@ -7,10 +7,12 @@ import ( "strings" "time" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/client" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/ec2metadata" + "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/internal/sdkuri" ) @@ -86,7 +88,14 @@ func NewCredentialsWithClient(client *ec2metadata.EC2Metadata, options ...func(* // Error will be returned if the request fails, or unable to extract // the desired credentials. func (m *EC2RoleProvider) Retrieve() (credentials.Value, error) { - credsList, err := requestCredList(m.Client) + return m.RetrieveWithContext(aws.BackgroundContext()) +} + +// RetrieveWithContext retrieves credentials from the EC2 service. +// Error will be returned if the request fails, or unable to extract +// the desired credentials. +func (m *EC2RoleProvider) RetrieveWithContext(ctx credentials.Context) (credentials.Value, error) { + credsList, err := requestCredList(ctx, m.Client) if err != nil { return credentials.Value{ProviderName: ProviderName}, err } @@ -96,7 +105,7 @@ func (m *EC2RoleProvider) Retrieve() (credentials.Value, error) { } credsName := credsList[0] - roleCreds, err := requestCred(m.Client, credsName) + roleCreds, err := requestCred(ctx, m.Client, credsName) if err != nil { return credentials.Value{ProviderName: ProviderName}, err } @@ -129,8 +138,8 @@ const iamSecurityCredsPath = "iam/security-credentials/" // requestCredList requests a list of credentials from the EC2 service. // If there are no credentials, or there is an error making or receiving the request -func requestCredList(client *ec2metadata.EC2Metadata) ([]string, error) { - resp, err := client.GetMetadata(iamSecurityCredsPath) +func requestCredList(ctx aws.Context, client *ec2metadata.EC2Metadata) ([]string, error) { + resp, err := client.GetMetadataWithContext(ctx, iamSecurityCredsPath) if err != nil { return nil, awserr.New("EC2RoleRequestError", "no EC2 instance role found", err) } @@ -142,7 +151,8 @@ func requestCredList(client *ec2metadata.EC2Metadata) ([]string, error) { } if err := s.Err(); err != nil { - return nil, awserr.New("SerializationError", "failed to read EC2 instance role from metadata service", err) + return nil, awserr.New(request.ErrCodeSerialization, + "failed to read EC2 instance role from metadata service", err) } return credsList, nil @@ -152,8 +162,8 @@ func requestCredList(client *ec2metadata.EC2Metadata) ([]string, error) { // // If the credentials cannot be found, or there is an error reading the response // and error will be returned. -func requestCred(client *ec2metadata.EC2Metadata, credsName string) (ec2RoleCredRespBody, error) { - resp, err := client.GetMetadata(sdkuri.PathJoin(iamSecurityCredsPath, credsName)) +func requestCred(ctx aws.Context, client *ec2metadata.EC2Metadata, credsName string) (ec2RoleCredRespBody, error) { + resp, err := client.GetMetadataWithContext(ctx, sdkuri.PathJoin(iamSecurityCredsPath, credsName)) if err != nil { return ec2RoleCredRespBody{}, awserr.New("EC2RoleRequestError", @@ -164,7 +174,7 @@ func requestCred(client *ec2metadata.EC2Metadata, credsName string) (ec2RoleCred respCreds := ec2RoleCredRespBody{} if err := json.NewDecoder(strings.NewReader(resp)).Decode(&respCreds); err != nil { return ec2RoleCredRespBody{}, - awserr.New("SerializationError", + awserr.New(request.ErrCodeSerialization, fmt.Sprintf("failed to decode %s EC2 instance role credentials", credsName), err) } diff --git a/github.com/aws/aws-sdk-go/aws/credentials/endpointcreds/provider.go b/github.com/aws/aws-sdk-go/aws/credentials/endpointcreds/provider.go index ace5131382..785f30d8e6 100644 --- a/github.com/aws/aws-sdk-go/aws/credentials/endpointcreds/provider.go +++ b/github.com/aws/aws-sdk-go/aws/credentials/endpointcreds/provider.go @@ -39,6 +39,7 @@ import ( "github.com/aws/aws-sdk-go/aws/client/metadata" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/private/protocol/json/jsonutil" ) // ProviderName is the name of the credentials provider. @@ -97,8 +98,8 @@ func NewProviderClient(cfg aws.Config, handlers request.Handlers, endpoint strin return p } -// NewCredentialsClient returns a Credentials wrapper for retrieving credentials -// from an arbitrary endpoint concurrently. The client will request the +// NewCredentialsClient returns a pointer to a new Credentials object +// wrapping the endpoint credentials Provider. func NewCredentialsClient(cfg aws.Config, handlers request.Handlers, endpoint string, options ...func(*Provider)) *credentials.Credentials { return credentials.NewCredentials(NewProviderClient(cfg, handlers, endpoint, options...)) } @@ -115,7 +116,13 @@ func (p *Provider) IsExpired() bool { // Retrieve will attempt to request the credentials from the endpoint the Provider // was configured for. And error will be returned if the retrieval fails. func (p *Provider) Retrieve() (credentials.Value, error) { - resp, err := p.getCredentials() + return p.RetrieveWithContext(aws.BackgroundContext()) +} + +// RetrieveWithContext will attempt to request the credentials from the endpoint the Provider +// was configured for. And error will be returned if the retrieval fails. +func (p *Provider) RetrieveWithContext(ctx credentials.Context) (credentials.Value, error) { + resp, err := p.getCredentials(ctx) if err != nil { return credentials.Value{ProviderName: ProviderName}, awserr.New("CredentialsEndpointError", "failed to load credentials", err) @@ -147,7 +154,7 @@ type errorOutput struct { Message string `json:"message"` } -func (p *Provider) getCredentials() (*getCredentialsOutput, error) { +func (p *Provider) getCredentials(ctx aws.Context) (*getCredentialsOutput, error) { op := &request.Operation{ Name: "GetCredentials", HTTPMethod: "GET", @@ -155,6 +162,7 @@ func (p *Provider) getCredentials() (*getCredentialsOutput, error) { out := &getCredentialsOutput{} req := p.Client.NewRequest(op, nil, out) + req.SetContext(ctx) req.HTTPRequest.Header.Set("Accept", "application/json") if authToken := p.AuthorizationToken; len(authToken) != 0 { req.HTTPRequest.Header.Set("Authorization", authToken) @@ -174,7 +182,7 @@ func unmarshalHandler(r *request.Request) { out := r.Data.(*getCredentialsOutput) if err := json.NewDecoder(r.HTTPResponse.Body).Decode(&out); err != nil { - r.Error = awserr.New("SerializationError", + r.Error = awserr.New(request.ErrCodeSerialization, "failed to decode endpoint credentials", err, ) @@ -185,11 +193,15 @@ func unmarshalError(r *request.Request) { defer r.HTTPResponse.Body.Close() var errOut errorOutput - if err := json.NewDecoder(r.HTTPResponse.Body).Decode(&errOut); err != nil { - r.Error = awserr.New("SerializationError", - "failed to decode endpoint credentials", - err, + err := jsonutil.UnmarshalJSONError(&errOut, r.HTTPResponse.Body) + if err != nil { + r.Error = awserr.NewRequestFailure( + awserr.New(request.ErrCodeSerialization, + "failed to decode error message", err), + r.HTTPResponse.StatusCode, + r.RequestID, ) + return } // Response body format is not consistent between metadata endpoints. diff --git a/github.com/aws/aws-sdk-go/aws/credentials/processcreds/provider.go b/github.com/aws/aws-sdk-go/aws/credentials/processcreds/provider.go index 1980c8c140..e624836002 100644 --- a/github.com/aws/aws-sdk-go/aws/credentials/processcreds/provider.go +++ b/github.com/aws/aws-sdk-go/aws/credentials/processcreds/provider.go @@ -90,6 +90,7 @@ import ( "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/internal/sdkio" ) const ( @@ -142,7 +143,7 @@ const ( // DefaultBufSize limits buffer size from growing to an enormous // amount due to a faulty process. - DefaultBufSize = 1024 + DefaultBufSize = int(8 * sdkio.KibiByte) // DefaultTimeout default limit on time a process can run. DefaultTimeout = time.Duration(1) * time.Minute diff --git a/github.com/aws/aws-sdk-go/aws/credentials/shared_credentials_provider.go b/github.com/aws/aws-sdk-go/aws/credentials/shared_credentials_provider.go index e155149581..22b5c5d9f3 100644 --- a/github.com/aws/aws-sdk-go/aws/credentials/shared_credentials_provider.go +++ b/github.com/aws/aws-sdk-go/aws/credentials/shared_credentials_provider.go @@ -17,8 +17,9 @@ var ( ErrSharedCredentialsHomeNotFound = awserr.New("UserHomeNotFound", "user home directory not found.", nil) ) -// A SharedCredentialsProvider retrieves credentials from the current user's home -// directory, and keeps track if those credentials are expired. +// A SharedCredentialsProvider retrieves access key pair (access key ID, +// secret access key, and session token if present) credentials from the current +// user's home directory, and keeps track if those credentials are expired. // // Profile ini file example: $HOME/.aws/credentials type SharedCredentialsProvider struct { diff --git a/github.com/aws/aws-sdk-go/aws/credentials/static_provider.go b/github.com/aws/aws-sdk-go/aws/credentials/static_provider.go index 531139e397..cbba1e3d56 100644 --- a/github.com/aws/aws-sdk-go/aws/credentials/static_provider.go +++ b/github.com/aws/aws-sdk-go/aws/credentials/static_provider.go @@ -19,7 +19,9 @@ type StaticProvider struct { } // NewStaticCredentials returns a pointer to a new Credentials object -// wrapping a static credentials value provider. +// wrapping a static credentials value provider. Token is only required +// for temporary security credentials retrieved via STS, otherwise an empty +// string can be passed for this parameter. func NewStaticCredentials(id, secret, token string) *Credentials { return NewCredentials(&StaticProvider{Value: Value{ AccessKeyID: id, diff --git a/github.com/aws/aws-sdk-go/aws/credentials/stscreds/assume_role_provider.go b/github.com/aws/aws-sdk-go/aws/credentials/stscreds/assume_role_provider.go index 4108e433e6..6846ef6f80 100644 --- a/github.com/aws/aws-sdk-go/aws/credentials/stscreds/assume_role_provider.go +++ b/github.com/aws/aws-sdk-go/aws/credentials/stscreds/assume_role_provider.go @@ -80,16 +80,19 @@ package stscreds import ( "fmt" + "os" "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/client" "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/internal/sdkrand" "github.com/aws/aws-sdk-go/service/sts" ) -// StdinTokenProvider will prompt on stdout and read from stdin for a string value. +// StdinTokenProvider will prompt on stderr and read from stdin for a string value. // An error is returned if reading from stdin fails. // // Use this function go read MFA tokens from stdin. The function makes no attempt @@ -102,7 +105,7 @@ import ( // Will wait forever until something is provided on the stdin. func StdinTokenProvider() (string, error) { var v string - fmt.Printf("Assume Role MFA token code: ") + fmt.Fprintf(os.Stderr, "Assume Role MFA token code: ") _, err := fmt.Scanln(&v) return v, err @@ -116,6 +119,10 @@ type AssumeRoler interface { AssumeRole(input *sts.AssumeRoleInput) (*sts.AssumeRoleOutput, error) } +type assumeRolerWithContext interface { + AssumeRoleWithContext(aws.Context, *sts.AssumeRoleInput, ...request.Option) (*sts.AssumeRoleOutput, error) +} + // DefaultDuration is the default amount of time in minutes that the credentials // will be valid for. var DefaultDuration = time.Duration(15) * time.Minute @@ -142,6 +149,13 @@ type AssumeRoleProvider struct { // Session name, if you wish to reuse the credentials elsewhere. RoleSessionName string + // Optional, you can pass tag key-value pairs to your session. These tags are called session tags. + Tags []*sts.Tag + + // A list of keys for session tags that you want to set as transitive. + // If you set a tag key as transitive, the corresponding key and value passes to subsequent sessions in a role chain. + TransitiveTagKeys []*string + // Expiry duration of the STS credentials. Defaults to 15 minutes if not set. Duration time.Duration @@ -155,6 +169,29 @@ type AssumeRoleProvider struct { // size. Policy *string + // The ARNs of IAM managed policies you want to use as managed session policies. + // The policies must exist in the same account as the role. + // + // This parameter is optional. You can provide up to 10 managed policy ARNs. + // However, the plain text that you use for both inline and managed session + // policies can't exceed 2,048 characters. + // + // An AWS conversion compresses the passed session policies and session tags + // into a packed binary format that has a separate limit. Your request can fail + // for this limit even if your plain text meets the other requirements. The + // PackedPolicySize response element indicates by percentage how close the policies + // and tags for your request are to the upper size limit. + // + // Passing policies to this operation returns new temporary credentials. The + // resulting session's permissions are the intersection of the role's identity-based + // policy and the session policies. You can use the role's temporary credentials + // in subsequent AWS API calls to access resources in the account that owns + // the role. You cannot use session policies to grant more permissions than + // those allowed by the identity-based policy of the role that is being assumed. + // For more information, see Session Policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session) + // in the IAM User Guide. + PolicyArns []*sts.PolicyDescriptorType + // The identification number of the MFA device that is associated with the user // who is making the AssumeRole call. Specify this value if the trust policy // of the role being assumed includes a condition that requires MFA authentication. @@ -193,6 +230,18 @@ type AssumeRoleProvider struct { // // If ExpiryWindow is 0 or less it will be ignored. ExpiryWindow time.Duration + + // MaxJitterFrac reduces the effective Duration of each credential requested + // by a random percentage between 0 and MaxJitterFraction. MaxJitterFrac must + // have a value between 0 and 1. Any other value may lead to expected behavior. + // With a MaxJitterFrac value of 0, default) will no jitter will be used. + // + // For example, with a Duration of 30m and a MaxJitterFrac of 0.1, the + // AssumeRole call will be made with an arbitrary Duration between 27m and + // 30m. + // + // MaxJitterFrac should not be negative. + MaxJitterFrac float64 } // NewCredentials returns a pointer to a new Credentials object wrapping the @@ -244,7 +293,11 @@ func NewCredentialsWithClient(svc AssumeRoler, roleARN string, options ...func(* // Retrieve generates a new set of temporary credentials using STS. func (p *AssumeRoleProvider) Retrieve() (credentials.Value, error) { + return p.RetrieveWithContext(aws.BackgroundContext()) +} +// RetrieveWithContext generates a new set of temporary credentials using STS. +func (p *AssumeRoleProvider) RetrieveWithContext(ctx credentials.Context) (credentials.Value, error) { // Apply defaults where parameters are not set. if p.RoleSessionName == "" { // Try to work out a role name that will hopefully end up unique. @@ -254,11 +307,15 @@ func (p *AssumeRoleProvider) Retrieve() (credentials.Value, error) { // Expire as often as AWS permits. p.Duration = DefaultDuration } + jitter := time.Duration(sdkrand.SeededRand.Float64() * p.MaxJitterFrac * float64(p.Duration)) input := &sts.AssumeRoleInput{ - DurationSeconds: aws.Int64(int64(p.Duration / time.Second)), - RoleArn: aws.String(p.RoleARN), - RoleSessionName: aws.String(p.RoleSessionName), - ExternalId: p.ExternalID, + DurationSeconds: aws.Int64(int64((p.Duration - jitter) / time.Second)), + RoleArn: aws.String(p.RoleARN), + RoleSessionName: aws.String(p.RoleSessionName), + ExternalId: p.ExternalID, + Tags: p.Tags, + PolicyArns: p.PolicyArns, + TransitiveTagKeys: p.TransitiveTagKeys, } if p.Policy != nil { input.Policy = p.Policy @@ -281,7 +338,15 @@ func (p *AssumeRoleProvider) Retrieve() (credentials.Value, error) { } } - roleOutput, err := p.Client.AssumeRole(input) + var roleOutput *sts.AssumeRoleOutput + var err error + + if c, ok := p.Client.(assumeRolerWithContext); ok { + roleOutput, err = c.AssumeRoleWithContext(ctx, input) + } else { + roleOutput, err = p.Client.AssumeRole(input) + } + if err != nil { return credentials.Value{ProviderName: ProviderName}, err } diff --git a/github.com/aws/aws-sdk-go/aws/credentials/stscreds/web_identity_provider.go b/github.com/aws/aws-sdk-go/aws/credentials/stscreds/web_identity_provider.go new file mode 100644 index 0000000000..6feb262b2f --- /dev/null +++ b/github.com/aws/aws-sdk-go/aws/credentials/stscreds/web_identity_provider.go @@ -0,0 +1,135 @@ +package stscreds + +import ( + "fmt" + "io/ioutil" + "strconv" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/client" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/service/sts" + "github.com/aws/aws-sdk-go/service/sts/stsiface" +) + +const ( + // ErrCodeWebIdentity will be used as an error code when constructing + // a new error to be returned during session creation or retrieval. + ErrCodeWebIdentity = "WebIdentityErr" + + // WebIdentityProviderName is the web identity provider name + WebIdentityProviderName = "WebIdentityCredentials" +) + +// now is used to return a time.Time object representing +// the current time. This can be used to easily test and +// compare test values. +var now = time.Now + +// TokenFetcher shuold return WebIdentity token bytes or an error +type TokenFetcher interface { + FetchToken(credentials.Context) ([]byte, error) +} + +// FetchTokenPath is a path to a WebIdentity token file +type FetchTokenPath string + +// FetchToken returns a token by reading from the filesystem +func (f FetchTokenPath) FetchToken(ctx credentials.Context) ([]byte, error) { + data, err := ioutil.ReadFile(string(f)) + if err != nil { + errMsg := fmt.Sprintf("unable to read file at %s", f) + return nil, awserr.New(ErrCodeWebIdentity, errMsg, err) + } + return data, nil +} + +// WebIdentityRoleProvider is used to retrieve credentials using +// an OIDC token. +type WebIdentityRoleProvider struct { + credentials.Expiry + PolicyArns []*sts.PolicyDescriptorType + + client stsiface.STSAPI + ExpiryWindow time.Duration + + tokenFetcher TokenFetcher + roleARN string + roleSessionName string +} + +// NewWebIdentityCredentials will return a new set of credentials with a given +// configuration, role arn, and token file path. +func NewWebIdentityCredentials(c client.ConfigProvider, roleARN, roleSessionName, path string) *credentials.Credentials { + svc := sts.New(c) + p := NewWebIdentityRoleProvider(svc, roleARN, roleSessionName, path) + return credentials.NewCredentials(p) +} + +// NewWebIdentityRoleProvider will return a new WebIdentityRoleProvider with the +// provided stsiface.STSAPI +func NewWebIdentityRoleProvider(svc stsiface.STSAPI, roleARN, roleSessionName, path string) *WebIdentityRoleProvider { + return NewWebIdentityRoleProviderWithToken(svc, roleARN, roleSessionName, FetchTokenPath(path)) +} + +// NewWebIdentityRoleProviderWithToken will return a new WebIdentityRoleProvider with the +// provided stsiface.STSAPI and a TokenFetcher +func NewWebIdentityRoleProviderWithToken(svc stsiface.STSAPI, roleARN, roleSessionName string, tokenFetcher TokenFetcher) *WebIdentityRoleProvider { + return &WebIdentityRoleProvider{ + client: svc, + tokenFetcher: tokenFetcher, + roleARN: roleARN, + roleSessionName: roleSessionName, + } +} + +// Retrieve will attempt to assume a role from a token which is located at +// 'WebIdentityTokenFilePath' specified destination and if that is empty an +// error will be returned. +func (p *WebIdentityRoleProvider) Retrieve() (credentials.Value, error) { + return p.RetrieveWithContext(aws.BackgroundContext()) +} + +// RetrieveWithContext will attempt to assume a role from a token which is located at +// 'WebIdentityTokenFilePath' specified destination and if that is empty an +// error will be returned. +func (p *WebIdentityRoleProvider) RetrieveWithContext(ctx credentials.Context) (credentials.Value, error) { + b, err := p.tokenFetcher.FetchToken(ctx) + if err != nil { + return credentials.Value{}, awserr.New(ErrCodeWebIdentity, "failed fetching WebIdentity token: ", err) + } + + sessionName := p.roleSessionName + if len(sessionName) == 0 { + // session name is used to uniquely identify a session. This simply + // uses unix time in nanoseconds to uniquely identify sessions. + sessionName = strconv.FormatInt(now().UnixNano(), 10) + } + req, resp := p.client.AssumeRoleWithWebIdentityRequest(&sts.AssumeRoleWithWebIdentityInput{ + PolicyArns: p.PolicyArns, + RoleArn: &p.roleARN, + RoleSessionName: &sessionName, + WebIdentityToken: aws.String(string(b)), + }) + + req.SetContext(ctx) + + // InvalidIdentityToken error is a temporary error that can occur + // when assuming an Role with a JWT web identity token. + req.RetryErrorCodes = append(req.RetryErrorCodes, sts.ErrCodeInvalidIdentityTokenException) + if err := req.Send(); err != nil { + return credentials.Value{}, awserr.New(ErrCodeWebIdentity, "failed to retrieve credentials", err) + } + + p.SetExpiration(aws.TimeValue(resp.Credentials.Expiration), p.ExpiryWindow) + + value := credentials.Value{ + AccessKeyID: aws.StringValue(resp.Credentials.AccessKeyId), + SecretAccessKey: aws.StringValue(resp.Credentials.SecretAccessKey), + SessionToken: aws.StringValue(resp.Credentials.SessionToken), + ProviderName: WebIdentityProviderName, + } + return value, nil +} diff --git a/github.com/aws/aws-sdk-go/aws/csm/doc.go b/github.com/aws/aws-sdk-go/aws/csm/doc.go index 152d785b36..25a66d1dda 100644 --- a/github.com/aws/aws-sdk-go/aws/csm/doc.go +++ b/github.com/aws/aws-sdk-go/aws/csm/doc.go @@ -1,30 +1,61 @@ -// Package csm provides Client Side Monitoring (CSM) which enables sending metrics -// via UDP connection. Using the Start function will enable the reporting of -// metrics on a given port. If Start is called, with different parameters, again, -// a panic will occur. +// Package csm provides the Client Side Monitoring (CSM) client which enables +// sending metrics via UDP connection to the CSM agent. This package provides +// control options, and configuration for the CSM client. The client can be +// controlled manually, or automatically via the SDK's Session configuration. // -// Pause can be called to pause any metrics publishing on a given port. Sessions -// that have had their handlers modified via InjectHandlers may still be used. -// However, the handlers will act as a no-op meaning no metrics will be published. +// Enabling CSM client via SDK's Session configuration +// +// The CSM client can be enabled automatically via SDK's Session configuration. +// The SDK's session configuration enables the CSM client if the AWS_CSM_PORT +// environment variable is set to a non-empty value. +// +// The configuration options for the CSM client via the SDK's session +// configuration are: +// +// * AWS_CSM_PORT= +// The port number the CSM agent will receive metrics on. +// +// * AWS_CSM_HOST= +// The hostname, or IP address the CSM agent will receive metrics on. +// Without port number. +// +// Manually enabling the CSM client +// +// The CSM client can be started, paused, and resumed manually. The Start +// function will enable the CSM client to publish metrics to the CSM agent. It +// is safe to call Start concurrently, but if Start is called additional times +// with different ClientID or address it will panic. // -// Example: // r, err := csm.Start("clientID", ":31000") // if err != nil { // panic(fmt.Errorf("failed starting CSM: %v", err)) // } // +// When controlling the CSM client manually, you must also inject its request +// handlers into the SDK's Session configuration for the SDK's API clients to +// publish metrics. +// // sess, err := session.NewSession(&aws.Config{}) // if err != nil { // panic(fmt.Errorf("failed loading session: %v", err)) // } // +// // Add CSM client's metric publishing request handlers to the SDK's +// // Session Configuration. // r.InjectHandlers(&sess.Handlers) // -// client := s3.New(sess) -// resp, err := client.GetObject(&s3.GetObjectInput{ -// Bucket: aws.String("bucket"), -// Key: aws.String("key"), -// }) +// Controlling CSM client +// +// Once the CSM client has been enabled the Get function will return a Reporter +// value that you can use to pause and resume the metrics published to the CSM +// agent. If Get function is called before the reporter is enabled with the +// Start function or via SDK's Session configuration nil will be returned. +// +// The Pause method can be called to stop the CSM client publishing metrics to +// the CSM agent. The Continue method will resume metric publishing. +// +// // Get the CSM client Reporter. +// r := csm.Get() // // // Will pause monitoring // r.Pause() @@ -35,12 +66,4 @@ // // // Resume monitoring // r.Continue() -// -// Start returns a Reporter that is used to enable or disable monitoring. If -// access to the Reporter is required later, calling Get will return the Reporter -// singleton. -// -// Example: -// r := csm.Get() -// r.Continue() package csm diff --git a/github.com/aws/aws-sdk-go/aws/csm/enable.go b/github.com/aws/aws-sdk-go/aws/csm/enable.go index 2f0c6eac9a..4b19e2800e 100644 --- a/github.com/aws/aws-sdk-go/aws/csm/enable.go +++ b/github.com/aws/aws-sdk-go/aws/csm/enable.go @@ -2,6 +2,7 @@ package csm import ( "fmt" + "strings" "sync" ) @@ -9,19 +10,40 @@ var ( lock sync.Mutex ) -// Client side metric handler names const ( - APICallMetricHandlerName = "awscsm.SendAPICallMetric" - APICallAttemptMetricHandlerName = "awscsm.SendAPICallAttemptMetric" + // DefaultPort is used when no port is specified. + DefaultPort = "31000" + + // DefaultHost is the host that will be used when none is specified. + DefaultHost = "127.0.0.1" ) -// Start will start the a long running go routine to capture +// AddressWithDefaults returns a CSM address built from the host and port +// values. If the host or port is not set, default values will be used +// instead. If host is "localhost" it will be replaced with "127.0.0.1". +func AddressWithDefaults(host, port string) string { + if len(host) == 0 || strings.EqualFold(host, "localhost") { + host = DefaultHost + } + + if len(port) == 0 { + port = DefaultPort + } + + // Only IP6 host can contain a colon + if strings.Contains(host, ":") { + return "[" + host + "]:" + port + } + + return host + ":" + port +} + +// Start will start a long running go routine to capture // client side metrics. Calling start multiple time will only // start the metric listener once and will panic if a different // client ID or port is passed in. // -// Example: -// r, err := csm.Start("clientID", "127.0.0.1:8094") +// r, err := csm.Start("clientID", "127.0.0.1:31000") // if err != nil { // panic(fmt.Errorf("expected no error, but received %v", err)) // } diff --git a/github.com/aws/aws-sdk-go/aws/csm/metric_chan.go b/github.com/aws/aws-sdk-go/aws/csm/metric_chan.go index 514fc3739a..82a3e345e9 100644 --- a/github.com/aws/aws-sdk-go/aws/csm/metric_chan.go +++ b/github.com/aws/aws-sdk-go/aws/csm/metric_chan.go @@ -16,25 +16,26 @@ var ( type metricChan struct { ch chan metric - paused int64 + paused *int64 } func newMetricChan(size int) metricChan { return metricChan{ - ch: make(chan metric, size), + ch: make(chan metric, size), + paused: new(int64), } } func (ch *metricChan) Pause() { - atomic.StoreInt64(&ch.paused, pausedEnum) + atomic.StoreInt64(ch.paused, pausedEnum) } func (ch *metricChan) Continue() { - atomic.StoreInt64(&ch.paused, runningEnum) + atomic.StoreInt64(ch.paused, runningEnum) } func (ch *metricChan) IsPaused() bool { - v := atomic.LoadInt64(&ch.paused) + v := atomic.LoadInt64(ch.paused) return v == pausedEnum } diff --git a/github.com/aws/aws-sdk-go/aws/csm/reporter.go b/github.com/aws/aws-sdk-go/aws/csm/reporter.go index 0b5571acfb..835bcd49cb 100644 --- a/github.com/aws/aws-sdk-go/aws/csm/reporter.go +++ b/github.com/aws/aws-sdk-go/aws/csm/reporter.go @@ -10,11 +10,6 @@ import ( "github.com/aws/aws-sdk-go/aws/request" ) -const ( - // DefaultPort is used when no port is specified - DefaultPort = "31000" -) - // Reporter will gather metrics of API requests made and // send those metrics to the CSM endpoint. type Reporter struct { @@ -71,7 +66,6 @@ func (rep *Reporter) sendAPICallAttemptMetric(r *request.Request) { XAmzRequestID: aws.String(r.RequestID), - AttemptCount: aws.Int(r.RetryCount + 1), AttemptLatency: aws.Int(int(now.Sub(r.AttemptTime).Nanoseconds() / int64(time.Millisecond))), AccessKey: aws.String(creds.AccessKeyID), } @@ -95,8 +89,8 @@ func getMetricException(err awserr.Error) metricException { code := err.Code() switch code { - case "RequestError", - "SerializationError", + case request.ErrCodeRequestError, + request.ErrCodeSerialization, request.CanceledErrorCode: return sdkException{ requestException{exception: code, message: msg}, @@ -123,7 +117,7 @@ func (rep *Reporter) sendAPICallMetric(r *request.Request) { Type: aws.String("ApiCall"), AttemptCount: aws.Int(r.RetryCount + 1), Region: r.Config.Region, - Latency: aws.Int(int(time.Now().Sub(r.Time) / time.Millisecond)), + Latency: aws.Int(int(time.Since(r.Time) / time.Millisecond)), XAmzRequestID: aws.String(r.RequestID), MaxRetriesExceeded: aws.Int(boolIntValue(r.RetryCount >= r.MaxRetries())), } @@ -190,8 +184,9 @@ func (rep *Reporter) start() { } } -// Pause will pause the metric channel preventing any new metrics from -// being added. +// Pause will pause the metric channel preventing any new metrics from being +// added. It is safe to call concurrently with other calls to Pause, but if +// called concurently with Continue can lead to unexpected state. func (rep *Reporter) Pause() { lock.Lock() defer lock.Unlock() @@ -203,8 +198,9 @@ func (rep *Reporter) Pause() { rep.close() } -// Continue will reopen the metric channel and allow for monitoring -// to be resumed. +// Continue will reopen the metric channel and allow for monitoring to be +// resumed. It is safe to call concurrently with other calls to Continue, but +// if called concurently with Pause can lead to unexpected state. func (rep *Reporter) Continue() { lock.Lock() defer lock.Unlock() @@ -219,10 +215,18 @@ func (rep *Reporter) Continue() { rep.metricsCh.Continue() } +// Client side metric handler names +const ( + APICallMetricHandlerName = "awscsm.SendAPICallMetric" + APICallAttemptMetricHandlerName = "awscsm.SendAPICallAttemptMetric" +) + // InjectHandlers will will enable client side metrics and inject the proper // handlers to handle how metrics are sent. // -// Example: +// InjectHandlers is NOT safe to call concurrently. Calling InjectHandlers +// multiple times may lead to unexpected behavior, (e.g. duplicate metrics). +// // // Start must be called in order to inject the correct handlers // r, err := csm.Start("clientID", "127.0.0.1:8094") // if err != nil { diff --git a/github.com/aws/aws-sdk-go/aws/ec2metadata/api.go b/github.com/aws/aws-sdk-go/aws/ec2metadata/api.go index 88e2fc7073..a716c021cf 100644 --- a/github.com/aws/aws-sdk-go/aws/ec2metadata/api.go +++ b/github.com/aws/aws-sdk-go/aws/ec2metadata/api.go @@ -4,34 +4,87 @@ import ( "encoding/json" "fmt" "net/http" + "strconv" "strings" "time" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/internal/sdkuri" ) +// getToken uses the duration to return a token for EC2 metadata service, +// or an error if the request failed. +func (c *EC2Metadata) getToken(ctx aws.Context, duration time.Duration) (tokenOutput, error) { + op := &request.Operation{ + Name: "GetToken", + HTTPMethod: "PUT", + HTTPPath: "/api/token", + } + + var output tokenOutput + req := c.NewRequest(op, nil, &output) + req.SetContext(ctx) + + // remove the fetch token handler from the request handlers to avoid infinite recursion + req.Handlers.Sign.RemoveByName(fetchTokenHandlerName) + + // Swap the unmarshalMetadataHandler with unmarshalTokenHandler on this request. + req.Handlers.Unmarshal.Swap(unmarshalMetadataHandlerName, unmarshalTokenHandler) + + ttl := strconv.FormatInt(int64(duration/time.Second), 10) + req.HTTPRequest.Header.Set(ttlHeader, ttl) + + err := req.Send() + + // Errors with bad request status should be returned. + if err != nil { + err = awserr.NewRequestFailure( + awserr.New(req.HTTPResponse.Status, http.StatusText(req.HTTPResponse.StatusCode), err), + req.HTTPResponse.StatusCode, req.RequestID) + } + + return output, err +} + // GetMetadata uses the path provided to request information from the EC2 -// instance metdata service. The content will be returned as a string, or +// instance metadata service. The content will be returned as a string, or // error if the request failed. func (c *EC2Metadata) GetMetadata(p string) (string, error) { + return c.GetMetadataWithContext(aws.BackgroundContext(), p) +} + +// GetMetadataWithContext uses the path provided to request information from the EC2 +// instance metadata service. The content will be returned as a string, or +// error if the request failed. +func (c *EC2Metadata) GetMetadataWithContext(ctx aws.Context, p string) (string, error) { op := &request.Operation{ Name: "GetMetadata", HTTPMethod: "GET", HTTPPath: sdkuri.PathJoin("/meta-data", p), } - output := &metadataOutput{} + req := c.NewRequest(op, nil, output) - return output.Content, req.Send() + req.SetContext(ctx) + + err := req.Send() + return output.Content, err } // GetUserData returns the userdata that was configured for the service. If // there is no user-data setup for the EC2 instance a "NotFoundError" error // code will be returned. func (c *EC2Metadata) GetUserData() (string, error) { + return c.GetUserDataWithContext(aws.BackgroundContext()) +} + +// GetUserDataWithContext returns the userdata that was configured for the service. If +// there is no user-data setup for the EC2 instance a "NotFoundError" error +// code will be returned. +func (c *EC2Metadata) GetUserDataWithContext(ctx aws.Context) (string, error) { op := &request.Operation{ Name: "GetUserData", HTTPMethod: "GET", @@ -40,19 +93,23 @@ func (c *EC2Metadata) GetUserData() (string, error) { output := &metadataOutput{} req := c.NewRequest(op, nil, output) - req.Handlers.UnmarshalError.PushBack(func(r *request.Request) { - if r.HTTPResponse.StatusCode == http.StatusNotFound { - r.Error = awserr.New("NotFoundError", "user-data not found", r.Error) - } - }) + req.SetContext(ctx) - return output.Content, req.Send() + err := req.Send() + return output.Content, err } // GetDynamicData uses the path provided to request information from the EC2 // instance metadata service for dynamic data. The content will be returned // as a string, or error if the request failed. func (c *EC2Metadata) GetDynamicData(p string) (string, error) { + return c.GetDynamicDataWithContext(aws.BackgroundContext(), p) +} + +// GetDynamicDataWithContext uses the path provided to request information from the EC2 +// instance metadata service for dynamic data. The content will be returned +// as a string, or error if the request failed. +func (c *EC2Metadata) GetDynamicDataWithContext(ctx aws.Context, p string) (string, error) { op := &request.Operation{ Name: "GetDynamicData", HTTPMethod: "GET", @@ -61,15 +118,24 @@ func (c *EC2Metadata) GetDynamicData(p string) (string, error) { output := &metadataOutput{} req := c.NewRequest(op, nil, output) + req.SetContext(ctx) - return output.Content, req.Send() + err := req.Send() + return output.Content, err } // GetInstanceIdentityDocument retrieves an identity document describing an // instance. Error is returned if the request fails or is unable to parse // the response. func (c *EC2Metadata) GetInstanceIdentityDocument() (EC2InstanceIdentityDocument, error) { - resp, err := c.GetDynamicData("instance-identity/document") + return c.GetInstanceIdentityDocumentWithContext(aws.BackgroundContext()) +} + +// GetInstanceIdentityDocumentWithContext retrieves an identity document describing an +// instance. Error is returned if the request fails or is unable to parse +// the response. +func (c *EC2Metadata) GetInstanceIdentityDocumentWithContext(ctx aws.Context) (EC2InstanceIdentityDocument, error) { + resp, err := c.GetDynamicDataWithContext(ctx, "instance-identity/document") if err != nil { return EC2InstanceIdentityDocument{}, awserr.New("EC2MetadataRequestError", @@ -79,7 +145,7 @@ func (c *EC2Metadata) GetInstanceIdentityDocument() (EC2InstanceIdentityDocument doc := EC2InstanceIdentityDocument{} if err := json.NewDecoder(strings.NewReader(resp)).Decode(&doc); err != nil { return EC2InstanceIdentityDocument{}, - awserr.New("SerializationError", + awserr.New(request.ErrCodeSerialization, "failed to decode EC2 instance identity document", err) } @@ -88,7 +154,12 @@ func (c *EC2Metadata) GetInstanceIdentityDocument() (EC2InstanceIdentityDocument // IAMInfo retrieves IAM info from the metadata API func (c *EC2Metadata) IAMInfo() (EC2IAMInfo, error) { - resp, err := c.GetMetadata("iam/info") + return c.IAMInfoWithContext(aws.BackgroundContext()) +} + +// IAMInfoWithContext retrieves IAM info from the metadata API +func (c *EC2Metadata) IAMInfoWithContext(ctx aws.Context) (EC2IAMInfo, error) { + resp, err := c.GetMetadataWithContext(ctx, "iam/info") if err != nil { return EC2IAMInfo{}, awserr.New("EC2MetadataRequestError", @@ -98,7 +169,7 @@ func (c *EC2Metadata) IAMInfo() (EC2IAMInfo, error) { info := EC2IAMInfo{} if err := json.NewDecoder(strings.NewReader(resp)).Decode(&info); err != nil { return EC2IAMInfo{}, - awserr.New("SerializationError", + awserr.New(request.ErrCodeSerialization, "failed to decode EC2 IAM info", err) } @@ -113,24 +184,36 @@ func (c *EC2Metadata) IAMInfo() (EC2IAMInfo, error) { // Region returns the region the instance is running in. func (c *EC2Metadata) Region() (string, error) { - resp, err := c.GetMetadata("placement/availability-zone") + return c.RegionWithContext(aws.BackgroundContext()) +} + +// RegionWithContext returns the region the instance is running in. +func (c *EC2Metadata) RegionWithContext(ctx aws.Context) (string, error) { + ec2InstanceIdentityDocument, err := c.GetInstanceIdentityDocumentWithContext(ctx) if err != nil { return "", err } - - if len(resp) == 0 { - return "", awserr.New("EC2MetadataError", "invalid Region response", nil) + // extract region from the ec2InstanceIdentityDocument + region := ec2InstanceIdentityDocument.Region + if len(region) == 0 { + return "", awserr.New("EC2MetadataError", "invalid region received for ec2metadata instance", nil) } - - // returns region without the suffix. Eg: us-west-2a becomes us-west-2 - return resp[:len(resp)-1], nil + // returns region + return region, nil } // Available returns if the application has access to the EC2 Metadata service. // Can be used to determine if application is running within an EC2 Instance and // the metadata service is available. func (c *EC2Metadata) Available() bool { - if _, err := c.GetMetadata("instance-id"); err != nil { + return c.AvailableWithContext(aws.BackgroundContext()) +} + +// AvailableWithContext returns if the application has access to the EC2 Metadata service. +// Can be used to determine if application is running within an EC2 Instance and +// the metadata service is available. +func (c *EC2Metadata) AvailableWithContext(ctx aws.Context) bool { + if _, err := c.GetMetadataWithContext(ctx, "instance-id"); err != nil { return false } @@ -149,18 +232,19 @@ type EC2IAMInfo struct { // An EC2InstanceIdentityDocument provides the shape for unmarshaling // an instance identity document type EC2InstanceIdentityDocument struct { - DevpayProductCodes []string `json:"devpayProductCodes"` - AvailabilityZone string `json:"availabilityZone"` - PrivateIP string `json:"privateIp"` - Version string `json:"version"` - Region string `json:"region"` - InstanceID string `json:"instanceId"` - BillingProducts []string `json:"billingProducts"` - InstanceType string `json:"instanceType"` - AccountID string `json:"accountId"` - PendingTime time.Time `json:"pendingTime"` - ImageID string `json:"imageId"` - KernelID string `json:"kernelId"` - RamdiskID string `json:"ramdiskId"` - Architecture string `json:"architecture"` + DevpayProductCodes []string `json:"devpayProductCodes"` + MarketplaceProductCodes []string `json:"marketplaceProductCodes"` + AvailabilityZone string `json:"availabilityZone"` + PrivateIP string `json:"privateIp"` + Version string `json:"version"` + Region string `json:"region"` + InstanceID string `json:"instanceId"` + BillingProducts []string `json:"billingProducts"` + InstanceType string `json:"instanceType"` + AccountID string `json:"accountId"` + PendingTime time.Time `json:"pendingTime"` + ImageID string `json:"imageId"` + KernelID string `json:"kernelId"` + RamdiskID string `json:"ramdiskId"` + Architecture string `json:"architecture"` } diff --git a/github.com/aws/aws-sdk-go/aws/ec2metadata/service.go b/github.com/aws/aws-sdk-go/aws/ec2metadata/service.go index 7d1f66e4e8..b8b2940d74 100644 --- a/github.com/aws/aws-sdk-go/aws/ec2metadata/service.go +++ b/github.com/aws/aws-sdk-go/aws/ec2metadata/service.go @@ -13,6 +13,7 @@ import ( "io" "net/http" "os" + "strconv" "strings" "time" @@ -24,9 +25,25 @@ import ( "github.com/aws/aws-sdk-go/aws/request" ) -// ServiceName is the name of the service. -const ServiceName = "ec2metadata" -const disableServiceEnvVar = "AWS_EC2_METADATA_DISABLED" +const ( + // ServiceName is the name of the service. + ServiceName = "ec2metadata" + disableServiceEnvVar = "AWS_EC2_METADATA_DISABLED" + + // Headers for Token and TTL + ttlHeader = "x-aws-ec2-metadata-token-ttl-seconds" + tokenHeader = "x-aws-ec2-metadata-token" + + // Named Handler constants + fetchTokenHandlerName = "FetchTokenHandler" + unmarshalMetadataHandlerName = "unmarshalMetadataHandler" + unmarshalTokenHandlerName = "unmarshalTokenHandler" + enableTokenProviderHandlerName = "enableTokenProviderHandler" + + // TTL constants + defaultTTL = 21600 * time.Second + ttlExpirationWindow = 30 * time.Second +) // A EC2Metadata is an EC2 Metadata service Client. type EC2Metadata struct { @@ -63,8 +80,10 @@ func NewClient(cfg aws.Config, handlers request.Handlers, endpoint, signingRegio // use a shorter timeout than default because the metadata // service is local if it is running, and to fail faster // if not running on an ec2 instance. - Timeout: 5 * time.Second, + Timeout: 1 * time.Second, } + // max number of retries on the client operation + cfg.MaxRetries = aws.Int(2) } svc := &EC2Metadata{ @@ -80,18 +99,35 @@ func NewClient(cfg aws.Config, handlers request.Handlers, endpoint, signingRegio ), } - svc.Handlers.Unmarshal.PushBack(unmarshalHandler) + // token provider instance + tp := newTokenProvider(svc, defaultTTL) + + // NamedHandler for fetching token + svc.Handlers.Sign.PushBackNamed(request.NamedHandler{ + Name: fetchTokenHandlerName, + Fn: tp.fetchTokenHandler, + }) + // NamedHandler for enabling token provider + svc.Handlers.Complete.PushBackNamed(request.NamedHandler{ + Name: enableTokenProviderHandlerName, + Fn: tp.enableTokenProviderHandler, + }) + + svc.Handlers.Unmarshal.PushBackNamed(unmarshalHandler) svc.Handlers.UnmarshalError.PushBack(unmarshalError) svc.Handlers.Validate.Clear() svc.Handlers.Validate.PushBack(validateEndpointHandler) // Disable the EC2 Metadata service if the environment variable is set. - // This shortcirctes the service's functionality to always fail to send + // This short-circuits the service's functionality to always fail to send // requests. if strings.ToLower(os.Getenv(disableServiceEnvVar)) == "true" { svc.Handlers.Send.SwapNamed(request.NamedHandler{ Name: corehandlers.SendHandler.Name, Fn: func(r *request.Request) { + r.HTTPResponse = &http.Response{ + Header: http.Header{}, + } r.Error = awserr.New( request.CanceledErrorCode, "EC2 IMDS access disabled via "+disableServiceEnvVar+" env var", @@ -104,7 +140,6 @@ func NewClient(cfg aws.Config, handlers request.Handlers, endpoint, signingRegio for _, option := range opts { option(svc.Client) } - return svc } @@ -116,30 +151,74 @@ type metadataOutput struct { Content string } -func unmarshalHandler(r *request.Request) { - defer r.HTTPResponse.Body.Close() - b := &bytes.Buffer{} - if _, err := io.Copy(b, r.HTTPResponse.Body); err != nil { - r.Error = awserr.New("SerializationError", "unable to unmarshal EC2 metadata respose", err) - return - } +type tokenOutput struct { + Token string + TTL time.Duration +} - if data, ok := r.Data.(*metadataOutput); ok { - data.Content = b.String() - } +// unmarshal token handler is used to parse the response of a getToken operation +var unmarshalTokenHandler = request.NamedHandler{ + Name: unmarshalTokenHandlerName, + Fn: func(r *request.Request) { + defer r.HTTPResponse.Body.Close() + var b bytes.Buffer + if _, err := io.Copy(&b, r.HTTPResponse.Body); err != nil { + r.Error = awserr.NewRequestFailure(awserr.New(request.ErrCodeSerialization, + "unable to unmarshal EC2 metadata response", err), r.HTTPResponse.StatusCode, r.RequestID) + return + } + + v := r.HTTPResponse.Header.Get(ttlHeader) + data, ok := r.Data.(*tokenOutput) + if !ok { + return + } + + data.Token = b.String() + // TTL is in seconds + i, err := strconv.ParseInt(v, 10, 64) + if err != nil { + r.Error = awserr.NewRequestFailure(awserr.New(request.ParamFormatErrCode, + "unable to parse EC2 token TTL response", err), r.HTTPResponse.StatusCode, r.RequestID) + return + } + t := time.Duration(i) * time.Second + data.TTL = t + }, +} + +var unmarshalHandler = request.NamedHandler{ + Name: unmarshalMetadataHandlerName, + Fn: func(r *request.Request) { + defer r.HTTPResponse.Body.Close() + var b bytes.Buffer + if _, err := io.Copy(&b, r.HTTPResponse.Body); err != nil { + r.Error = awserr.NewRequestFailure(awserr.New(request.ErrCodeSerialization, + "unable to unmarshal EC2 metadata response", err), r.HTTPResponse.StatusCode, r.RequestID) + return + } + + if data, ok := r.Data.(*metadataOutput); ok { + data.Content = b.String() + } + }, } func unmarshalError(r *request.Request) { defer r.HTTPResponse.Body.Close() - b := &bytes.Buffer{} - if _, err := io.Copy(b, r.HTTPResponse.Body); err != nil { - r.Error = awserr.New("SerializationError", "unable to unmarshal EC2 metadata error respose", err) + var b bytes.Buffer + + if _, err := io.Copy(&b, r.HTTPResponse.Body); err != nil { + r.Error = awserr.NewRequestFailure( + awserr.New(request.ErrCodeSerialization, "unable to unmarshal EC2 metadata error response", err), + r.HTTPResponse.StatusCode, r.RequestID) return } // Response body format is not consistent between metadata endpoints. // Grab the error message as a string and include that as the source error - r.Error = awserr.New("EC2MetadataError", "failed to make EC2Metadata request", errors.New(b.String())) + r.Error = awserr.NewRequestFailure(awserr.New("EC2MetadataError", "failed to make EC2Metadata request", errors.New(b.String())), + r.HTTPResponse.StatusCode, r.RequestID) } func validateEndpointHandler(r *request.Request) { diff --git a/github.com/aws/aws-sdk-go/aws/ec2metadata/token_provider.go b/github.com/aws/aws-sdk-go/aws/ec2metadata/token_provider.go new file mode 100644 index 0000000000..d0a3a020d8 --- /dev/null +++ b/github.com/aws/aws-sdk-go/aws/ec2metadata/token_provider.go @@ -0,0 +1,92 @@ +package ec2metadata + +import ( + "net/http" + "sync/atomic" + "time" + + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/request" +) + +// A tokenProvider struct provides access to EC2Metadata client +// and atomic instance of a token, along with configuredTTL for it. +// tokenProvider also provides an atomic flag to disable the +// fetch token operation. +// The disabled member will use 0 as false, and 1 as true. +type tokenProvider struct { + client *EC2Metadata + token atomic.Value + configuredTTL time.Duration + disabled uint32 +} + +// A ec2Token struct helps use of token in EC2 Metadata service ops +type ec2Token struct { + token string + credentials.Expiry +} + +// newTokenProvider provides a pointer to a tokenProvider instance +func newTokenProvider(c *EC2Metadata, duration time.Duration) *tokenProvider { + return &tokenProvider{client: c, configuredTTL: duration} +} + +// fetchTokenHandler fetches token for EC2Metadata service client by default. +func (t *tokenProvider) fetchTokenHandler(r *request.Request) { + + // short-circuits to insecure data flow if tokenProvider is disabled. + if v := atomic.LoadUint32(&t.disabled); v == 1 { + return + } + + if ec2Token, ok := t.token.Load().(ec2Token); ok && !ec2Token.IsExpired() { + r.HTTPRequest.Header.Set(tokenHeader, ec2Token.token) + return + } + + output, err := t.client.getToken(r.Context(), t.configuredTTL) + + if err != nil { + + // change the disabled flag on token provider to true, + // when error is request timeout error. + if requestFailureError, ok := err.(awserr.RequestFailure); ok { + switch requestFailureError.StatusCode() { + case http.StatusForbidden, http.StatusNotFound, http.StatusMethodNotAllowed: + atomic.StoreUint32(&t.disabled, 1) + case http.StatusBadRequest: + r.Error = requestFailureError + } + + // Check if request timed out while waiting for response + if e, ok := requestFailureError.OrigErr().(awserr.Error); ok { + if e.Code() == request.ErrCodeRequestError { + atomic.StoreUint32(&t.disabled, 1) + } + } + } + return + } + + newToken := ec2Token{ + token: output.Token, + } + newToken.SetExpiration(time.Now().Add(output.TTL), ttlExpirationWindow) + t.token.Store(newToken) + + // Inject token header to the request. + if ec2Token, ok := t.token.Load().(ec2Token); ok { + r.HTTPRequest.Header.Set(tokenHeader, ec2Token.token) + } +} + +// enableTokenProviderHandler enables the token provider +func (t *tokenProvider) enableTokenProviderHandler(r *request.Request) { + // If the error code status is 401, we enable the token provider + if e, ok := r.Error.(awserr.RequestFailure); ok && e != nil && + e.StatusCode() == http.StatusUnauthorized { + atomic.StoreUint32(&t.disabled, 0) + } +} diff --git a/github.com/aws/aws-sdk-go/aws/endpoints/decode.go b/github.com/aws/aws-sdk-go/aws/endpoints/decode.go index 87b9ff3ffe..654fb1ad52 100644 --- a/github.com/aws/aws-sdk-go/aws/endpoints/decode.go +++ b/github.com/aws/aws-sdk-go/aws/endpoints/decode.go @@ -83,6 +83,7 @@ func decodeV3Endpoints(modelDef modelDefinition, opts DecodeModelOptions) (Resol p := &ps[i] custAddEC2Metadata(p) custAddS3DualStack(p) + custRegionalS3(p) custRmIotDataService(p) custFixAppAutoscalingChina(p) custFixAppAutoscalingUsGov(p) @@ -92,7 +93,7 @@ func decodeV3Endpoints(modelDef modelDefinition, opts DecodeModelOptions) (Resol } func custAddS3DualStack(p *partition) { - if p.ID != "aws" { + if !(p.ID == "aws" || p.ID == "aws-cn" || p.ID == "aws-us-gov") { return } @@ -100,6 +101,33 @@ func custAddS3DualStack(p *partition) { custAddDualstack(p, "s3-control") } +func custRegionalS3(p *partition) { + if p.ID != "aws" { + return + } + + service, ok := p.Services["s3"] + if !ok { + return + } + + // If global endpoint already exists no customization needed. + if _, ok := service.Endpoints["aws-global"]; ok { + return + } + + service.PartitionEndpoint = "aws-global" + service.Endpoints["us-east-1"] = endpoint{} + service.Endpoints["aws-global"] = endpoint{ + Hostname: "s3.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + } + + p.Services["s3"] = service +} + func custAddDualstack(p *partition, svcName string) { s, ok := p.Services[svcName] if !ok { diff --git a/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go b/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go index 636f614f16..bc5fb73f91 100644 --- a/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go +++ b/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go @@ -11,21 +11,27 @@ const ( AwsPartitionID = "aws" // AWS Standard partition. AwsCnPartitionID = "aws-cn" // AWS China partition. AwsUsGovPartitionID = "aws-us-gov" // AWS GovCloud (US) partition. + AwsIsoPartitionID = "aws-iso" // AWS ISO (US) partition. + AwsIsoBPartitionID = "aws-iso-b" // AWS ISOB (US) partition. ) // AWS Standard partition's regions. const ( + AfSouth1RegionID = "af-south-1" // Africa (Cape Town). + ApEast1RegionID = "ap-east-1" // Asia Pacific (Hong Kong). ApNortheast1RegionID = "ap-northeast-1" // Asia Pacific (Tokyo). ApNortheast2RegionID = "ap-northeast-2" // Asia Pacific (Seoul). ApSouth1RegionID = "ap-south-1" // Asia Pacific (Mumbai). ApSoutheast1RegionID = "ap-southeast-1" // Asia Pacific (Singapore). ApSoutheast2RegionID = "ap-southeast-2" // Asia Pacific (Sydney). CaCentral1RegionID = "ca-central-1" // Canada (Central). - EuCentral1RegionID = "eu-central-1" // EU (Frankfurt). - EuNorth1RegionID = "eu-north-1" // EU (Stockholm). - EuWest1RegionID = "eu-west-1" // EU (Ireland). - EuWest2RegionID = "eu-west-2" // EU (London). - EuWest3RegionID = "eu-west-3" // EU (Paris). + EuCentral1RegionID = "eu-central-1" // Europe (Frankfurt). + EuNorth1RegionID = "eu-north-1" // Europe (Stockholm). + EuSouth1RegionID = "eu-south-1" // Europe (Milan). + EuWest1RegionID = "eu-west-1" // Europe (Ireland). + EuWest2RegionID = "eu-west-2" // Europe (London). + EuWest3RegionID = "eu-west-3" // Europe (Paris). + MeSouth1RegionID = "me-south-1" // Middle East (Bahrain). SaEast1RegionID = "sa-east-1" // South America (Sao Paulo). UsEast1RegionID = "us-east-1" // US East (N. Virginia). UsEast2RegionID = "us-east-2" // US East (Ohio). @@ -42,11 +48,21 @@ const ( // AWS GovCloud (US) partition's regions. const ( UsGovEast1RegionID = "us-gov-east-1" // AWS GovCloud (US-East). - UsGovWest1RegionID = "us-gov-west-1" // AWS GovCloud (US). + UsGovWest1RegionID = "us-gov-west-1" // AWS GovCloud (US-West). +) + +// AWS ISO (US) partition's regions. +const ( + UsIsoEast1RegionID = "us-iso-east-1" // US ISO East. +) + +// AWS ISOB (US) partition's regions. +const ( + UsIsobEast1RegionID = "us-isob-east-1" // US ISOB East (Ohio). ) // DefaultResolver returns an Endpoint resolver that will be able -// to resolve endpoints for: AWS Standard, AWS China, and AWS GovCloud (US). +// to resolve endpoints for: AWS Standard, AWS China, AWS GovCloud (US), AWS ISO (US), and AWS ISOB (US). // // Use DefaultPartitions() to get the list of the default partitions. func DefaultResolver() Resolver { @@ -54,7 +70,7 @@ func DefaultResolver() Resolver { } // DefaultPartitions returns a list of the partitions the SDK is bundled -// with. The available partitions are: AWS Standard, AWS China, and AWS GovCloud (US). +// with. The available partitions are: AWS Standard, AWS China, AWS GovCloud (US), AWS ISO (US), and AWS ISOB (US). // // partitions := endpoints.DefaultPartitions // for _, p := range partitions { @@ -68,6 +84,8 @@ var defaultPartitions = partitions{ awsPartition, awscnPartition, awsusgovPartition, + awsisoPartition, + awsisobPartition, } // AwsPartition returns the Resolver for AWS Standard. @@ -81,7 +99,7 @@ var awsPartition = partition{ DNSSuffix: "amazonaws.com", RegionRegex: regionRegex{ Regexp: func() *regexp.Regexp { - reg, _ := regexp.Compile("^(us|eu|ap|sa|ca)\\-\\w+\\-\\d+$") + reg, _ := regexp.Compile("^(us|eu|ap|sa|ca|me|af)\\-\\w+\\-\\d+$") return reg }(), }, @@ -91,6 +109,12 @@ var awsPartition = partition{ SignatureVersions: []string{"v4"}, }, Regions: regions{ + "af-south-1": region{ + Description: "Africa (Cape Town)", + }, + "ap-east-1": region{ + Description: "Asia Pacific (Hong Kong)", + }, "ap-northeast-1": region{ Description: "Asia Pacific (Tokyo)", }, @@ -110,19 +134,25 @@ var awsPartition = partition{ Description: "Canada (Central)", }, "eu-central-1": region{ - Description: "EU (Frankfurt)", + Description: "Europe (Frankfurt)", }, "eu-north-1": region{ - Description: "EU (Stockholm)", + Description: "Europe (Stockholm)", + }, + "eu-south-1": region{ + Description: "Europe (Milan)", }, "eu-west-1": region{ - Description: "EU (Ireland)", + Description: "Europe (Ireland)", }, "eu-west-2": region{ - Description: "EU (London)", + Description: "Europe (London)", }, "eu-west-3": region{ - Description: "EU (Paris)", + Description: "Europe (Paris)", + }, + "me-south-1": region{ + Description: "Middle East (Bahrain)", }, "sa-east-1": region{ Description: "South America (Sao Paulo)", @@ -147,9 +177,11 @@ var awsPartition = partition{ "us-east-1": endpoint{}, }, }, - "acm": service{ + "access-analyzer": service{ Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -158,9 +190,11 @@ var awsPartition = partition{ "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, "sa-east-1": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, @@ -168,163 +202,67 @@ var awsPartition = partition{ "us-west-2": endpoint{}, }, }, - "acm-pca": service{ - Defaults: endpoint{ - Protocols: []string{"https"}, - }, - Endpoints: endpoints{ - "ap-northeast-1": endpoint{}, - "ap-southeast-1": endpoint{}, - "ap-southeast-2": endpoint{}, - "ca-central-1": endpoint{}, - "eu-central-1": endpoint{}, - "eu-west-1": endpoint{}, - "eu-west-2": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-2": endpoint{}, - }, - }, - "api.mediatailor": service{ - - Endpoints: endpoints{ - "ap-northeast-1": endpoint{}, - "ap-southeast-1": endpoint{}, - "ap-southeast-2": endpoint{}, - "eu-west-1": endpoint{}, - "us-east-1": endpoint{}, - }, - }, - "api.pricing": service{ - Defaults: endpoint{ - CredentialScope: credentialScope{ - Service: "pricing", - }, - }, - Endpoints: endpoints{ - "ap-south-1": endpoint{}, - "us-east-1": endpoint{}, - }, - }, - "api.sagemaker": service{ - - Endpoints: endpoints{ - "ap-northeast-1": endpoint{}, - "ap-northeast-2": endpoint{}, - "ap-south-1": endpoint{}, - "ap-southeast-1": endpoint{}, - "ap-southeast-2": endpoint{}, - "ca-central-1": endpoint{}, - "eu-central-1": endpoint{}, - "eu-west-1": endpoint{}, - "eu-west-2": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, - }, - }, - "apigateway": service{ + "acm": service{ Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, "ca-central-1": endpoint{}, - "eu-central-1": endpoint{}, - "eu-north-1": endpoint{}, - "eu-west-1": endpoint{}, - "eu-west-2": endpoint{}, - "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, - }, - }, - "application-autoscaling": service{ - Defaults: endpoint{ - Hostname: "autoscaling.{region}.amazonaws.com", - Protocols: []string{"http", "https"}, - CredentialScope: credentialScope{ - Service: "application-autoscaling", + "ca-central-1-fips": endpoint{ + Hostname: "acm-fips.ca-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, }, - }, - Endpoints: endpoints{ - "ap-northeast-1": endpoint{}, - "ap-northeast-2": endpoint{}, - "ap-south-1": endpoint{}, - "ap-southeast-1": endpoint{}, - "ap-southeast-2": endpoint{}, - "ca-central-1": endpoint{}, - "eu-central-1": endpoint{}, - "eu-north-1": endpoint{}, - "eu-west-1": endpoint{}, - "eu-west-2": endpoint{}, - "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, - }, - }, - "appstream2": service{ - Defaults: endpoint{ - Protocols: []string{"https"}, - CredentialScope: credentialScope{ - Service: "appstream", + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-1-fips": endpoint{ + Hostname: "acm-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "us-east-2": endpoint{}, + "us-east-2-fips": endpoint{ + Hostname: "acm-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "us-west-1": endpoint{}, + "us-west-1-fips": endpoint{ + Hostname: "acm-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "us-west-2": endpoint{}, + "us-west-2-fips": endpoint{ + Hostname: "acm-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, }, - }, - Endpoints: endpoints{ - "ap-northeast-1": endpoint{}, - "ap-southeast-1": endpoint{}, - "ap-southeast-2": endpoint{}, - "eu-central-1": endpoint{}, - "eu-west-1": endpoint{}, - "us-east-1": endpoint{}, - "us-west-2": endpoint{}, - }, - }, - "appsync": service{ - - Endpoints: endpoints{ - "ap-northeast-1": endpoint{}, - "ap-northeast-2": endpoint{}, - "ap-south-1": endpoint{}, - "ap-southeast-1": endpoint{}, - "ap-southeast-2": endpoint{}, - "eu-central-1": endpoint{}, - "eu-west-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-2": endpoint{}, - }, - }, - "athena": service{ - - Endpoints: endpoints{ - "ap-northeast-1": endpoint{}, - "ap-northeast-2": endpoint{}, - "ap-south-1": endpoint{}, - "ap-southeast-1": endpoint{}, - "ap-southeast-2": endpoint{}, - "eu-central-1": endpoint{}, - "eu-west-1": endpoint{}, - "eu-west-2": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-2": endpoint{}, }, }, - "autoscaling": service{ + "acm-pca": service{ Defaults: endpoint{ - Protocols: []string{"http", "https"}, + Protocols: []string{"https"}, }, Endpoints: endpoints{ + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -336,39 +274,48 @@ var awsPartition = partition{ "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, + "fips-ca-central-1": endpoint{ + Hostname: "acm-pca-fips.ca-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, + }, + "fips-us-east-1": endpoint{ + Hostname: "acm-pca-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "acm-pca-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "acm-pca-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "acm-pca-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "autoscaling-plans": service{ + "api.detective": service{ Defaults: endpoint{ - Hostname: "autoscaling.{region}.amazonaws.com", - Protocols: []string{"http", "https"}, - CredentialScope: credentialScope{ - Service: "autoscaling-plans", - }, - }, - Endpoints: endpoints{ - "ap-northeast-1": endpoint{}, - "ap-northeast-2": endpoint{}, - "ap-south-1": endpoint{}, - "ap-southeast-1": endpoint{}, - "ap-southeast-2": endpoint{}, - "ca-central-1": endpoint{}, - "eu-central-1": endpoint{}, - "eu-west-1": endpoint{}, - "eu-west-2": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, + Protocols: []string{"https"}, }, - }, - "batch": service{ - Endpoints: endpoints{ "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, @@ -377,6 +324,7 @@ var awsPartition = partition{ "ap-southeast-2": endpoint{}, "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, @@ -387,76 +335,253 @@ var awsPartition = partition{ "us-west-2": endpoint{}, }, }, - "budgets": service{ - PartitionEndpoint: "aws-global", - IsRegionalized: boxedFalse, + "api.ecr": service{ Endpoints: endpoints{ - "aws-global": endpoint{ - Hostname: "budgets.amazonaws.com", + "af-south-1": endpoint{ + Hostname: "api.ecr.af-south-1.amazonaws.com", CredentialScope: credentialScope{ - Region: "us-east-1", + Region: "af-south-1", }, }, - }, - }, - "ce": service{ - PartitionEndpoint: "aws-global", - IsRegionalized: boxedFalse, - - Endpoints: endpoints{ - "aws-global": endpoint{ - Hostname: "ce.us-east-1.amazonaws.com", + "ap-east-1": endpoint{ + Hostname: "api.ecr.ap-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-east-1", + }, + }, + "ap-northeast-1": endpoint{ + Hostname: "api.ecr.ap-northeast-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-northeast-1", + }, + }, + "ap-northeast-2": endpoint{ + Hostname: "api.ecr.ap-northeast-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-northeast-2", + }, + }, + "ap-south-1": endpoint{ + Hostname: "api.ecr.ap-south-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-south-1", + }, + }, + "ap-southeast-1": endpoint{ + Hostname: "api.ecr.ap-southeast-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-southeast-1", + }, + }, + "ap-southeast-2": endpoint{ + Hostname: "api.ecr.ap-southeast-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-southeast-2", + }, + }, + "ca-central-1": endpoint{ + Hostname: "api.ecr.ca-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, + }, + "eu-central-1": endpoint{ + Hostname: "api.ecr.eu-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-central-1", + }, + }, + "eu-north-1": endpoint{ + Hostname: "api.ecr.eu-north-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-north-1", + }, + }, + "eu-south-1": endpoint{ + Hostname: "api.ecr.eu-south-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-south-1", + }, + }, + "eu-west-1": endpoint{ + Hostname: "api.ecr.eu-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-west-1", + }, + }, + "eu-west-2": endpoint{ + Hostname: "api.ecr.eu-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-west-2", + }, + }, + "eu-west-3": endpoint{ + Hostname: "api.ecr.eu-west-3.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-west-3", + }, + }, + "fips-us-east-1": endpoint{ + Hostname: "ecr-fips.us-east-1.amazonaws.com", CredentialScope: credentialScope{ Region: "us-east-1", }, }, - }, - }, - "chime": service{ - PartitionEndpoint: "aws-global", - IsRegionalized: boxedFalse, - Defaults: endpoint{ - SSLCommonName: "service.chime.aws.amazon.com", - Protocols: []string{"https"}, - }, - Endpoints: endpoints{ - "aws-global": endpoint{ - Hostname: "service.chime.aws.amazon.com", - Protocols: []string{"https"}, + "fips-us-east-2": endpoint{ + Hostname: "ecr-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "ecr-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "ecr-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{ + Hostname: "api.ecr.me-south-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "me-south-1", + }, + }, + "sa-east-1": endpoint{ + Hostname: "api.ecr.sa-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "sa-east-1", + }, + }, + "us-east-1": endpoint{ + Hostname: "api.ecr.us-east-1.amazonaws.com", CredentialScope: credentialScope{ Region: "us-east-1", }, }, + "us-east-2": endpoint{ + Hostname: "api.ecr.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "us-west-1": endpoint{ + Hostname: "api.ecr.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "us-west-2": endpoint{ + Hostname: "api.ecr.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, }, }, - "cloud9": service{ + "api.elastic-inference": service{ + + Endpoints: endpoints{ + "ap-northeast-1": endpoint{ + Hostname: "api.elastic-inference.ap-northeast-1.amazonaws.com", + }, + "ap-northeast-2": endpoint{ + Hostname: "api.elastic-inference.ap-northeast-2.amazonaws.com", + }, + "eu-west-1": endpoint{ + Hostname: "api.elastic-inference.eu-west-1.amazonaws.com", + }, + "us-east-1": endpoint{ + Hostname: "api.elastic-inference.us-east-1.amazonaws.com", + }, + "us-east-2": endpoint{ + Hostname: "api.elastic-inference.us-east-2.amazonaws.com", + }, + "us-west-2": endpoint{ + Hostname: "api.elastic-inference.us-west-2.amazonaws.com", + }, + }, + }, + "api.mediatailor": service{ Endpoints: endpoints{ + "ap-northeast-1": endpoint{}, "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "eu-central-1": endpoint{}, "eu-west-1": endpoint{}, "us-east-1": endpoint{}, - "us-east-2": endpoint{}, "us-west-2": endpoint{}, }, }, - "clouddirectory": service{ + "api.pricing": service{ + Defaults: endpoint{ + CredentialScope: credentialScope{ + Service: "pricing", + }, + }, + Endpoints: endpoints{ + "ap-south-1": endpoint{}, + "us-east-1": endpoint{}, + }, + }, + "api.sagemaker": service{ Endpoints: endpoints{ + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-2": endpoint{}, + "us-east-1-fips": endpoint{ + Hostname: "api-fips.sagemaker.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "us-east-2": endpoint{}, + "us-east-2-fips": endpoint{ + Hostname: "api-fips.sagemaker.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "us-west-1": endpoint{}, + "us-west-1-fips": endpoint{ + Hostname: "api-fips.sagemaker.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "us-west-2": endpoint{}, + "us-west-2-fips": endpoint{ + Hostname: "api-fips.sagemaker.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, }, }, - "cloudformation": service{ + "apigateway": service{ Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -465,9 +590,11 @@ var awsPartition = partition{ "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, "sa-east-1": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, @@ -475,42 +602,37 @@ var awsPartition = partition{ "us-west-2": endpoint{}, }, }, - "cloudfront": service{ - PartitionEndpoint: "aws-global", - IsRegionalized: boxedFalse, - - Endpoints: endpoints{ - "aws-global": endpoint{ - Hostname: "cloudfront.amazonaws.com", - Protocols: []string{"http", "https"}, - CredentialScope: credentialScope{ - Region: "us-east-1", - }, - }, + "application-autoscaling": service{ + Defaults: endpoint{ + Protocols: []string{"http", "https"}, }, - }, - "cloudhsm": service{ - Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, "us-west-1": endpoint{}, "us-west-2": endpoint{}, }, }, - "cloudhsmv2": service{ - Defaults: endpoint{ - CredentialScope: credentialScope{ - Service: "cloudhsm", - }, - }, + "appmesh": service{ + Endpoints: endpoints{ + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -522,30 +644,64 @@ var awsPartition = partition{ "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, "us-west-1": endpoint{}, "us-west-2": endpoint{}, }, }, - "cloudsearch": service{ + "appstream2": service{ + Defaults: endpoint{ + Protocols: []string{"https"}, + CredentialScope: credentialScope{ + Service: "appstream", + }, + }, + Endpoints: endpoints{ + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "eu-central-1": endpoint{}, + "eu-west-1": endpoint{}, + "fips": endpoint{ + Hostname: "appstream2-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "us-east-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "appsync": service{ Endpoints: endpoints{ "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, "sa-east-1": endpoint{}, "us-east-1": endpoint{}, + "us-east-2": endpoint{}, "us-west-1": endpoint{}, "us-west-2": endpoint{}, }, }, - "cloudtrail": service{ + "athena": service{ Endpoints: endpoints{ + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -557,6 +713,7 @@ var awsPartition = partition{ "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, "sa-east-1": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, @@ -564,9 +721,13 @@ var awsPartition = partition{ "us-west-2": endpoint{}, }, }, - "codebuild": service{ - + "autoscaling": service{ + Defaults: endpoint{ + Protocols: []string{"http", "https"}, + }, Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -574,43 +735,161 @@ var awsPartition = partition{ "ap-southeast-2": endpoint{}, "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, "sa-east-1": endpoint{}, "us-east-1": endpoint{}, - "us-east-1-fips": endpoint{ - Hostname: "codebuild-fips.us-east-1.amazonaws.com", + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "autoscaling-plans": service{ + Defaults: endpoint{ + Protocols: []string{"http", "https"}, + }, + Endpoints: endpoints{ + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "backup": service{ + + Endpoints: endpoints{ + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "batch": service{ + + Endpoints: endpoints{ + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "fips.batch.us-east-1.amazonaws.com", CredentialScope: credentialScope{ Region: "us-east-1", }, }, - "us-east-2": endpoint{}, - "us-east-2-fips": endpoint{ - Hostname: "codebuild-fips.us-east-2.amazonaws.com", + "fips-us-east-2": endpoint{ + Hostname: "fips.batch.us-east-2.amazonaws.com", CredentialScope: credentialScope{ Region: "us-east-2", }, }, - "us-west-1": endpoint{}, - "us-west-1-fips": endpoint{ - Hostname: "codebuild-fips.us-west-1.amazonaws.com", + "fips-us-west-1": endpoint{ + Hostname: "fips.batch.us-west-1.amazonaws.com", CredentialScope: credentialScope{ Region: "us-west-1", }, }, - "us-west-2": endpoint{}, - "us-west-2-fips": endpoint{ - Hostname: "codebuild-fips.us-west-2.amazonaws.com", + "fips-us-west-2": endpoint{ + Hostname: "fips.batch.us-west-2.amazonaws.com", CredentialScope: credentialScope{ Region: "us-west-2", }, }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "codecommit": service{ + "budgets": service{ + PartitionEndpoint: "aws-global", + IsRegionalized: boxedFalse, + + Endpoints: endpoints{ + "aws-global": endpoint{ + Hostname: "budgets.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + }, + }, + "ce": service{ + PartitionEndpoint: "aws-global", + IsRegionalized: boxedFalse, + + Endpoints: endpoints{ + "aws-global": endpoint{ + Hostname: "ce.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + }, + }, + "chime": service{ + PartitionEndpoint: "aws-global", + IsRegionalized: boxedFalse, + Defaults: endpoint{ + SSLCommonName: "service.chime.aws.amazon.com", + Protocols: []string{"https"}, + }, + Endpoints: endpoints{ + "aws-global": endpoint{ + Hostname: "service.chime.aws.amazon.com", + Protocols: []string{"https"}, + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + }, + }, + "cloud9": service{ Endpoints: endpoints{ + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -618,25 +897,37 @@ var awsPartition = partition{ "ap-southeast-2": endpoint{}, "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, - "fips": endpoint{ - Hostname: "codecommit-fips.ca-central-1.amazonaws.com", - CredentialScope: credentialScope{ - Region: "ca-central-1", - }, - }, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "codedeploy": service{ + "clouddirectory": service{ + + Endpoints: endpoints{ + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "cloudformation": service{ Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -645,97 +936,118 @@ var awsPartition = partition{ "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, "sa-east-1": endpoint{}, "us-east-1": endpoint{}, "us-east-1-fips": endpoint{ - Hostname: "codedeploy-fips.us-east-1.amazonaws.com", + Hostname: "cloudformation-fips.us-east-1.amazonaws.com", CredentialScope: credentialScope{ Region: "us-east-1", }, }, "us-east-2": endpoint{}, "us-east-2-fips": endpoint{ - Hostname: "codedeploy-fips.us-east-2.amazonaws.com", + Hostname: "cloudformation-fips.us-east-2.amazonaws.com", CredentialScope: credentialScope{ Region: "us-east-2", }, }, "us-west-1": endpoint{}, "us-west-1-fips": endpoint{ - Hostname: "codedeploy-fips.us-west-1.amazonaws.com", + Hostname: "cloudformation-fips.us-west-1.amazonaws.com", CredentialScope: credentialScope{ Region: "us-west-1", }, }, "us-west-2": endpoint{}, "us-west-2-fips": endpoint{ - Hostname: "codedeploy-fips.us-west-2.amazonaws.com", + Hostname: "cloudformation-fips.us-west-2.amazonaws.com", CredentialScope: credentialScope{ Region: "us-west-2", }, }, }, }, - "codepipeline": service{ + "cloudfront": service{ + PartitionEndpoint: "aws-global", + IsRegionalized: boxedFalse, + + Endpoints: endpoints{ + "aws-global": endpoint{ + Hostname: "cloudfront.amazonaws.com", + Protocols: []string{"http", "https"}, + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + }, + }, + "cloudhsm": service{ Endpoints: endpoints{ "ap-northeast-1": endpoint{}, - "ap-northeast-2": endpoint{}, - "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, "eu-west-1": endpoint{}, - "eu-west-2": endpoint{}, - "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, "us-west-1": endpoint{}, "us-west-2": endpoint{}, }, }, - "codestar": service{ - + "cloudhsmv2": service{ + Defaults: endpoint{ + CredentialScope: credentialScope{ + Service: "cloudhsm", + }, + }, Endpoints: endpoints{ + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, "us-west-1": endpoint{}, "us-west-2": endpoint{}, }, }, - "cognito-identity": service{ + "cloudsearch": service{ Endpoints: endpoints{ "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, - "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, - "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, "eu-west-1": endpoint{}, - "eu-west-2": endpoint{}, + "sa-east-1": endpoint{}, "us-east-1": endpoint{}, - "us-east-2": endpoint{}, + "us-west-1": endpoint{}, "us-west-2": endpoint{}, }, }, - "cognito-idp": service{ + "cloudtrail": service{ Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -743,45 +1055,62 @@ var awsPartition = partition{ "ap-southeast-2": endpoint{}, "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "cloudtrail-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "cloudtrail-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "cloudtrail-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "cloudtrail-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "cognito-sync": service{ + "codeartifact": service{ Endpoints: endpoints{ "ap-northeast-1": endpoint{}, - "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, "eu-central-1": endpoint{}, - "eu-west-1": endpoint{}, - "eu-west-2": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-2": endpoint{}, - }, - }, - "comprehend": service{ - Defaults: endpoint{ - Protocols: []string{"https"}, - }, - Endpoints: endpoints{ - "ap-southeast-2": endpoint{}, - "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, "us-west-2": endpoint{}, }, }, - "config": service{ + "codebuild": service{ Endpoints: endpoints{ + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -793,68 +1122,42 @@ var awsPartition = partition{ "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, "sa-east-1": endpoint{}, "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, - }, - }, - "cur": service{ - - Endpoints: endpoints{ - "us-east-1": endpoint{}, - }, - }, - "datapipeline": service{ - - Endpoints: endpoints{ - "ap-northeast-1": endpoint{}, - "ap-southeast-2": endpoint{}, - "eu-west-1": endpoint{}, - "us-east-1": endpoint{}, - "us-west-2": endpoint{}, - }, - }, - "datasync": service{ - - Endpoints: endpoints{ - "ap-northeast-1": endpoint{}, - "ap-northeast-2": endpoint{}, - "ap-southeast-1": endpoint{}, - "ap-southeast-2": endpoint{}, - "eu-central-1": endpoint{}, - "eu-west-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, - }, - }, - "dax": service{ - - Endpoints: endpoints{ - "ap-northeast-1": endpoint{}, - "ap-south-1": endpoint{}, - "ap-southeast-1": endpoint{}, - "ap-southeast-2": endpoint{}, - "eu-west-1": endpoint{}, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, - }, - }, - "devicefarm": service{ - - Endpoints: endpoints{ + "us-east-1-fips": endpoint{ + Hostname: "codebuild-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "us-east-2": endpoint{}, + "us-east-2-fips": endpoint{ + Hostname: "codebuild-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "us-west-1": endpoint{}, + "us-west-1-fips": endpoint{ + Hostname: "codebuild-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, "us-west-2": endpoint{}, + "us-west-2-fips": endpoint{ + Hostname: "codebuild-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, }, }, - "directconnect": service{ + "codecommit": service{ Endpoints: endpoints{ + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -866,22 +1169,25 @@ var awsPartition = partition{ "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, - }, - }, - "discovery": service{ - - Endpoints: endpoints{ - "us-west-2": endpoint{}, + "fips": endpoint{ + Hostname: "codecommit-fips.ca-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "dms": service{ + "codedeploy": service{ Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -889,68 +1195,45 @@ var awsPartition = partition{ "ap-southeast-2": endpoint{}, "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, "sa-east-1": endpoint{}, "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, - }, - }, - "docdb": service{ - - Endpoints: endpoints{ - "eu-west-1": endpoint{ - Hostname: "rds.eu-west-1.amazonaws.com", + "us-east-1-fips": endpoint{ + Hostname: "codedeploy-fips.us-east-1.amazonaws.com", CredentialScope: credentialScope{ - Region: "eu-west-1", + Region: "us-east-1", }, }, - "us-east-1": endpoint{ - Hostname: "rds.us-east-1.amazonaws.com", + "us-east-2": endpoint{}, + "us-east-2-fips": endpoint{ + Hostname: "codedeploy-fips.us-east-2.amazonaws.com", CredentialScope: credentialScope{ - Region: "us-east-1", + Region: "us-east-2", }, }, - "us-east-2": endpoint{ - Hostname: "rds.us-east-2.amazonaws.com", + "us-west-1": endpoint{}, + "us-west-1-fips": endpoint{ + Hostname: "codedeploy-fips.us-west-1.amazonaws.com", CredentialScope: credentialScope{ - Region: "us-east-2", + Region: "us-west-1", }, }, - "us-west-2": endpoint{ - Hostname: "rds.us-west-2.amazonaws.com", + "us-west-2": endpoint{}, + "us-west-2-fips": endpoint{ + Hostname: "codedeploy-fips.us-west-2.amazonaws.com", CredentialScope: credentialScope{ Region: "us-west-2", }, }, }, }, - "ds": service{ + "codepipeline": service{ - Endpoints: endpoints{ - "ap-northeast-1": endpoint{}, - "ap-northeast-2": endpoint{}, - "ap-south-1": endpoint{}, - "ap-southeast-1": endpoint{}, - "ap-southeast-2": endpoint{}, - "ca-central-1": endpoint{}, - "eu-central-1": endpoint{}, - "eu-west-1": endpoint{}, - "eu-west-2": endpoint{}, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, - }, - }, - "dynamodb": service{ - Defaults: endpoint{ - Protocols: []string{"http", "https"}, - }, Endpoints: endpoints{ "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, @@ -963,13 +1246,36 @@ var awsPartition = partition{ "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, - "local": endpoint{ - Hostname: "localhost:8000", - Protocols: []string{"http"}, + "fips-ca-central-1": endpoint{ + Hostname: "codepipeline-fips.ca-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, + }, + "fips-us-east-1": endpoint{ + Hostname: "codepipeline-fips.us-east-1.amazonaws.com", CredentialScope: credentialScope{ Region: "us-east-1", }, }, + "fips-us-east-2": endpoint{ + Hostname: "codepipeline-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "codepipeline-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "codepipeline-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, "sa-east-1": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, @@ -977,14 +1283,11 @@ var awsPartition = partition{ "us-west-2": endpoint{}, }, }, - "ec2": service{ - Defaults: endpoint{ - Protocols: []string{"http", "https"}, - }, + "codestar": service{ + Endpoints: endpoints{ "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, - "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, "ca-central-1": endpoint{}, @@ -992,26 +1295,13 @@ var awsPartition = partition{ "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, - "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, "us-west-1": endpoint{}, "us-west-2": endpoint{}, }, }, - "ec2metadata": service{ - PartitionEndpoint: "aws-global", - IsRegionalized: boxedFalse, - - Endpoints: endpoints{ - "aws-global": endpoint{ - Hostname: "169.254.169.254/latest", - Protocols: []string{"http"}, - }, - }, - }, - "ecr": service{ + "codestar-connections": service{ Endpoints: endpoints{ "ap-northeast-1": endpoint{}, @@ -1032,7 +1322,7 @@ var awsPartition = partition{ "us-west-2": endpoint{}, }, }, - "ecs": service{ + "cognito-identity": service{ Endpoints: endpoints{ "ap-northeast-1": endpoint{}, @@ -1042,18 +1332,32 @@ var awsPartition = partition{ "ap-southeast-2": endpoint{}, "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, - "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, - "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "cognito-identity-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "cognito-identity-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "cognito-identity-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-2": endpoint{}, }, }, - "elasticache": service{ + "cognito-idp": service{ Endpoints: endpoints{ "ap-northeast-1": endpoint{}, @@ -1063,24 +1367,32 @@ var awsPartition = partition{ "ap-southeast-2": endpoint{}, "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, - "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, - "eu-west-3": endpoint{}, - "fips": endpoint{ - Hostname: "elasticache-fips.us-west-1.amazonaws.com", + "fips-us-east-1": endpoint{ + Hostname: "cognito-idp-fips.us-east-1.amazonaws.com", CredentialScope: credentialScope{ - Region: "us-west-1", + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "cognito-idp-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "cognito-idp-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", }, }, - "sa-east-1": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, - "us-west-1": endpoint{}, "us-west-2": endpoint{}, }, }, - "elasticbeanstalk": service{ + "cognito-sync": service{ Endpoints: endpoints{ "ap-northeast-1": endpoint{}, @@ -1088,35 +1400,15 @@ var awsPartition = partition{ "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, - "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, - "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, - "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, - "us-west-1": endpoint{}, "us-west-2": endpoint{}, }, }, - "elasticfilesystem": service{ - - Endpoints: endpoints{ - "ap-northeast-1": endpoint{}, - "ap-northeast-2": endpoint{}, - "ap-southeast-1": endpoint{}, - "ap-southeast-2": endpoint{}, - "eu-central-1": endpoint{}, - "eu-west-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, - }, - }, - "elasticloadbalancing": service{ + "comprehend": service{ Defaults: endpoint{ Protocols: []string{"https"}, }, @@ -1128,79 +1420,65 @@ var awsPartition = partition{ "ap-southeast-2": endpoint{}, "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, - "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, - "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, - }, - }, - "elasticmapreduce": service{ - Defaults: endpoint{ - SSLCommonName: "{region}.{service}.{dnsSuffix}", - Protocols: []string{"https"}, - }, - Endpoints: endpoints{ - "ap-northeast-1": endpoint{}, - "ap-northeast-2": endpoint{}, - "ap-south-1": endpoint{}, - "ap-southeast-1": endpoint{}, - "ap-southeast-2": endpoint{}, - "ca-central-1": endpoint{}, - "eu-central-1": endpoint{ - SSLCommonName: "{service}.{region}.{dnsSuffix}", + "fips-us-east-1": endpoint{ + Hostname: "comprehend-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, }, - "eu-north-1": endpoint{}, - "eu-west-1": endpoint{}, - "eu-west-2": endpoint{}, - "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{ - SSLCommonName: "{service}.{region}.{dnsSuffix}", + "fips-us-east-2": endpoint{ + Hostname: "comprehend-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "comprehend-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, }, + "us-east-1": endpoint{}, "us-east-2": endpoint{}, - "us-west-1": endpoint{}, "us-west-2": endpoint{}, }, }, - "elastictranscoder": service{ + "comprehendmedical": service{ Endpoints: endpoints{ - "ap-northeast-1": endpoint{}, - "ap-south-1": endpoint{}, - "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, "eu-west-1": endpoint{}, - "us-east-1": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, - }, - }, - "email": service{ - - Endpoints: endpoints{ - "eu-west-1": endpoint{}, - "us-east-1": endpoint{}, - "us-west-2": endpoint{}, - }, - }, - "entitlement.marketplace": service{ - Defaults: endpoint{ - CredentialScope: credentialScope{ - Service: "aws-marketplace", + "eu-west-2": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "comprehendmedical-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "comprehendmedical-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "comprehendmedical-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, }, - }, - Endpoints: endpoints{ "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-2": endpoint{}, }, }, - "es": service{ + "config": service{ Endpoints: endpoints{ + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -1212,6 +1490,7 @@ var awsPartition = partition{ "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, "sa-east-1": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, @@ -1219,73 +1498,69 @@ var awsPartition = partition{ "us-west-2": endpoint{}, }, }, - "events": service{ + "connect": service{ Endpoints: endpoints{ "ap-northeast-1": endpoint{}, - "ap-northeast-2": endpoint{}, - "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, - "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, - "eu-north-1": endpoint{}, - "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, - "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, "us-west-2": endpoint{}, }, }, - "firehose": service{ + "cur": service{ + + Endpoints: endpoints{ + "us-east-1": endpoint{}, + }, + }, + "data.mediastore": service{ Endpoints: endpoints{ "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, - "ap-south-1": endpoint{}, - "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, - "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, - "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, "us-west-2": endpoint{}, }, }, - "fms": service{ - Defaults: endpoint{ - Protocols: []string{"https"}, - }, + "dataexchange": service{ + Endpoints: endpoints{ "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, "eu-central-1": endpoint{}, "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, + "us-west-1": endpoint{}, "us-west-2": endpoint{}, }, }, - "fsx": service{ + "datapipeline": service{ Endpoints: endpoints{ - "eu-west-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-2": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "eu-west-1": endpoint{}, + "us-east-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "gamelift": service{ + "datasync": service{ Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -1293,28 +1568,57 @@ var awsPartition = partition{ "ap-southeast-2": endpoint{}, "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "fips-ca-central-1": endpoint{ + Hostname: "datasync-fips.ca-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, + }, + "fips-us-east-1": endpoint{ + Hostname: "datasync-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "datasync-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "datasync-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "datasync-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "glacier": service{ - Defaults: endpoint{ - Protocols: []string{"http", "https"}, - }, + "dax": service{ + Endpoints: endpoints{ "ap-northeast-1": endpoint{}, - "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, - "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, - "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, @@ -1325,9 +1629,17 @@ var awsPartition = partition{ "us-west-2": endpoint{}, }, }, - "glue": service{ + "devicefarm": service{ + + Endpoints: endpoints{ + "us-west-2": endpoint{}, + }, + }, + "directconnect": service{ Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -1335,134 +1647,291 @@ var awsPartition = partition{ "ap-southeast-2": endpoint{}, "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "directconnect-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "directconnect-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "directconnect-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "directconnect-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "greengrass": service{ - IsRegionalized: boxedTrue, - Defaults: endpoint{ - Protocols: []string{"https"}, - }, + "discovery": service{ + Endpoints: endpoints{ "ap-northeast-1": endpoint{}, "ap-southeast-2": endpoint{}, "eu-central-1": endpoint{}, - "eu-west-1": endpoint{}, "us-east-1": endpoint{}, "us-west-2": endpoint{}, }, }, - "guardduty": service{ - IsRegionalized: boxedTrue, - Defaults: endpoint{ - Protocols: []string{"https"}, - }, + "dms": service{ + Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, "ca-central-1": endpoint{}, - "eu-central-1": endpoint{}, - "eu-west-1": endpoint{}, - "eu-west-2": endpoint{}, - "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, - }, - }, - "health": service{ - - Endpoints: endpoints{ - "us-east-1": endpoint{}, - }, - }, - "iam": service{ - PartitionEndpoint: "aws-global", - IsRegionalized: boxedFalse, - - Endpoints: endpoints{ - "aws-global": endpoint{ - Hostname: "iam.amazonaws.com", + "dms-fips": endpoint{ + Hostname: "dms-fips.us-west-1.amazonaws.com", CredentialScope: credentialScope{ - Region: "us-east-1", + Region: "us-west-1", }, }, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "importexport": service{ - PartitionEndpoint: "aws-global", - IsRegionalized: boxedFalse, + "docdb": service{ Endpoints: endpoints{ - "aws-global": endpoint{ - Hostname: "importexport.amazonaws.com", - SignatureVersions: []string{"v2", "v4"}, + "ap-northeast-1": endpoint{ + Hostname: "rds.ap-northeast-1.amazonaws.com", CredentialScope: credentialScope{ - Region: "us-east-1", - Service: "IngestionService", + Region: "ap-northeast-1", + }, + }, + "ap-northeast-2": endpoint{ + Hostname: "rds.ap-northeast-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-northeast-2", + }, + }, + "ap-south-1": endpoint{ + Hostname: "rds.ap-south-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-south-1", + }, + }, + "ap-southeast-1": endpoint{ + Hostname: "rds.ap-southeast-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-southeast-1", + }, + }, + "ap-southeast-2": endpoint{ + Hostname: "rds.ap-southeast-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-southeast-2", + }, + }, + "ca-central-1": endpoint{ + Hostname: "rds.ca-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, + }, + "eu-central-1": endpoint{ + Hostname: "rds.eu-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-central-1", + }, + }, + "eu-west-1": endpoint{ + Hostname: "rds.eu-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-west-1", + }, + }, + "eu-west-2": endpoint{ + Hostname: "rds.eu-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-west-2", + }, + }, + "eu-west-3": endpoint{ + Hostname: "rds.eu-west-3.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-west-3", + }, + }, + "us-east-1": endpoint{ + Hostname: "rds.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "us-east-2": endpoint{ + Hostname: "rds.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "us-west-2": endpoint{ + Hostname: "rds.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", }, }, }, }, - "inspector": service{ + "ds": service{ Endpoints: endpoints{ + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "fips-ca-central-1": endpoint{ + Hostname: "ds-fips.ca-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, + }, + "fips-us-east-1": endpoint{ + Hostname: "ds-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "ds-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "ds-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "ds-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "iot": service{ + "dynamodb": service{ Defaults: endpoint{ - CredentialScope: credentialScope{ - Service: "execute-api", - }, + Protocols: []string{"http", "https"}, }, Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, - "eu-central-1": endpoint{}, - "eu-west-1": endpoint{}, - "eu-west-2": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-2": endpoint{}, + "ca-central-1": endpoint{}, + "ca-central-1-fips": endpoint{ + Hostname: "dynamodb-fips.ca-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, + }, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "local": endpoint{ + Hostname: "localhost:8000", + Protocols: []string{"http"}, + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-1-fips": endpoint{ + Hostname: "dynamodb-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "us-east-2": endpoint{}, + "us-east-2-fips": endpoint{ + Hostname: "dynamodb-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "us-west-1": endpoint{}, + "us-west-1-fips": endpoint{ + Hostname: "dynamodb-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "us-west-2": endpoint{}, + "us-west-2-fips": endpoint{ + Hostname: "dynamodb-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, }, }, - "iotanalytics": service{ - - Endpoints: endpoints{ - "ap-northeast-1": endpoint{}, - "eu-central-1": endpoint{}, - "eu-west-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-2": endpoint{}, + "ec2": service{ + Defaults: endpoint{ + Protocols: []string{"http", "https"}, }, - }, - "kinesis": service{ - Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -1471,39 +1940,64 @@ var awsPartition = partition{ "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, - }, - }, - "kinesisanalytics": service{ - - Endpoints: endpoints{ - "eu-central-1": endpoint{}, - "eu-west-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-2": endpoint{}, + "fips-ca-central-1": endpoint{ + Hostname: "ec2-fips.ca-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, + }, + "fips-us-east-1": endpoint{ + Hostname: "ec2-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "ec2-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "ec2-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "ec2-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "kinesisvideo": service{ + "ec2metadata": service{ + PartitionEndpoint: "aws-global", + IsRegionalized: boxedFalse, Endpoints: endpoints{ - "ap-northeast-1": endpoint{}, - "eu-central-1": endpoint{}, - "eu-west-1": endpoint{}, - "us-east-1": endpoint{}, - "us-west-2": endpoint{}, + "aws-global": endpoint{ + Hostname: "169.254.169.254/latest", + Protocols: []string{"http"}, + }, }, }, - "kms": service{ + "ecs": service{ Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -1512,19 +2006,48 @@ var awsPartition = partition{ "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "ecs-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "ecs-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "ecs-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "ecs-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "lambda": service{ - + "eks": service{ + Defaults: endpoint{ + Protocols: []string{"http", "https"}, + }, Endpoints: endpoints{ + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -1536,16 +2059,36 @@ var awsPartition = partition{ "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "fips.eks.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "fips.eks.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "fips.eks.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-2": endpoint{}, }, }, - "lightsail": service{ + "elasticache": service{ Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -1553,17 +2096,30 @@ var awsPartition = partition{ "ap-southeast-2": endpoint{}, "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-2": endpoint{}, + "fips": endpoint{ + Hostname: "elasticache-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "logs": service{ + "elasticbeanstalk": service{ Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -1572,32 +2128,47 @@ var awsPartition = partition{ "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, - }, - }, - "machinelearning": service{ - - Endpoints: endpoints{ - "eu-west-1": endpoint{}, - "us-east-1": endpoint{}, - }, - }, - "marketplacecommerceanalytics": service{ - - Endpoints: endpoints{ - "us-east-1": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "elasticbeanstalk-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "elasticbeanstalk-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "elasticbeanstalk-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "elasticbeanstalk-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "mediaconvert": service{ + "elasticfilesystem": service{ Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -1605,113 +2176,291 @@ var awsPartition = partition{ "ap-southeast-2": endpoint{}, "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "fips-af-south-1": endpoint{ + Hostname: "elasticfilesystem-fips.af-south-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "af-south-1", + }, + }, + "fips-ap-east-1": endpoint{ + Hostname: "elasticfilesystem-fips.ap-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-east-1", + }, + }, + "fips-ap-northeast-1": endpoint{ + Hostname: "elasticfilesystem-fips.ap-northeast-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-northeast-1", + }, + }, + "fips-ap-northeast-2": endpoint{ + Hostname: "elasticfilesystem-fips.ap-northeast-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-northeast-2", + }, + }, + "fips-ap-south-1": endpoint{ + Hostname: "elasticfilesystem-fips.ap-south-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-south-1", + }, + }, + "fips-ap-southeast-1": endpoint{ + Hostname: "elasticfilesystem-fips.ap-southeast-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-southeast-1", + }, + }, + "fips-ap-southeast-2": endpoint{ + Hostname: "elasticfilesystem-fips.ap-southeast-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-southeast-2", + }, + }, + "fips-ca-central-1": endpoint{ + Hostname: "elasticfilesystem-fips.ca-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, + }, + "fips-eu-central-1": endpoint{ + Hostname: "elasticfilesystem-fips.eu-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-central-1", + }, + }, + "fips-eu-north-1": endpoint{ + Hostname: "elasticfilesystem-fips.eu-north-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-north-1", + }, + }, + "fips-eu-south-1": endpoint{ + Hostname: "elasticfilesystem-fips.eu-south-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-south-1", + }, + }, + "fips-eu-west-1": endpoint{ + Hostname: "elasticfilesystem-fips.eu-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-west-1", + }, + }, + "fips-eu-west-2": endpoint{ + Hostname: "elasticfilesystem-fips.eu-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-west-2", + }, + }, + "fips-eu-west-3": endpoint{ + Hostname: "elasticfilesystem-fips.eu-west-3.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-west-3", + }, + }, + "fips-me-south-1": endpoint{ + Hostname: "elasticfilesystem-fips.me-south-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "me-south-1", + }, + }, + "fips-sa-east-1": endpoint{ + Hostname: "elasticfilesystem-fips.sa-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "sa-east-1", + }, + }, + "fips-us-east-1": endpoint{ + Hostname: "elasticfilesystem-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "elasticfilesystem-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "elasticfilesystem-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "elasticfilesystem-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "medialive": service{ - + "elasticloadbalancing": service{ + Defaults: endpoint{ + Protocols: []string{"https"}, + }, Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, "eu-west-1": endpoint{}, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{}, - "us-west-2": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "elasticloadbalancing-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "elasticloadbalancing-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "elasticloadbalancing-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "elasticloadbalancing-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "mediapackage": service{ - + "elasticmapreduce": service{ + Defaults: endpoint{ + SSLCommonName: "{region}.{service}.{dnsSuffix}", + Protocols: []string{"https"}, + }, Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, - "eu-central-1": endpoint{}, - "eu-west-1": endpoint{}, - "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, - }, - }, - "mediastore": service{ - - Endpoints: endpoints{ - "ap-northeast-1": endpoint{}, - "ap-northeast-2": endpoint{}, - "ap-southeast-2": endpoint{}, - "eu-central-1": endpoint{}, - "eu-west-1": endpoint{}, - "us-east-1": endpoint{}, - "us-west-2": endpoint{}, - }, - }, - "metering.marketplace": service{ - Defaults: endpoint{ - CredentialScope: credentialScope{ - Service: "aws-marketplace", + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{ + SSLCommonName: "{service}.{region}.{dnsSuffix}", + }, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "fips-ca-central-1": endpoint{ + Hostname: "elasticmapreduce-fips.ca-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, + }, + "fips-us-east-1": endpoint{ + Hostname: "elasticmapreduce-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "elasticmapreduce-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, }, + "fips-us-west-1": endpoint{ + Hostname: "elasticmapreduce-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "elasticmapreduce-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{ + SSLCommonName: "{service}.{region}.{dnsSuffix}", + }, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, + }, + "elastictranscoder": service{ + Endpoints: endpoints{ "ap-northeast-1": endpoint{}, - "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, - "ca-central-1": endpoint{}, - "eu-central-1": endpoint{}, - "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, - "eu-west-2": endpoint{}, - "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, "us-east-1": endpoint{}, - "us-east-2": endpoint{}, "us-west-1": endpoint{}, "us-west-2": endpoint{}, }, }, - "mgh": service{ - - Endpoints: endpoints{ - "us-west-2": endpoint{}, - }, - }, - "mobileanalytics": service{ + "email": service{ Endpoints: endpoints{ - "us-east-1": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "eu-central-1": endpoint{}, + "eu-west-1": endpoint{}, + "us-east-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "models.lex": service{ + "entitlement.marketplace": service{ Defaults: endpoint{ CredentialScope: credentialScope{ - Service: "lex", + Service: "aws-marketplace", }, }, Endpoints: endpoints{ - "eu-west-1": endpoint{}, "us-east-1": endpoint{}, - "us-west-2": endpoint{}, }, }, - "monitoring": service{ - Defaults: endpoint{ - Protocols: []string{"http", "https"}, - }, + "es": service{ + Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -1720,95 +2469,236 @@ var awsPartition = partition{ "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, + "fips": endpoint{ + Hostname: "es-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "mq": service{ + "events": service{ Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, "eu-west-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "events-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "events-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "events-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "events-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "mturk-requester": service{ - IsRegionalized: boxedFalse, + "firehose": service{ Endpoints: endpoints{ - "sandbox": endpoint{ - Hostname: "mturk-requester-sandbox.us-east-1.amazonaws.com", + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "firehose-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, }, - "us-east-1": endpoint{}, + "fips-us-east-2": endpoint{ + Hostname: "firehose-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "firehose-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "firehose-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "neptune": service{ - + "fms": service{ + Defaults: endpoint{ + Protocols: []string{"https"}, + }, Endpoints: endpoints{ - "ap-southeast-1": endpoint{ - Hostname: "rds.ap-southeast-1.amazonaws.com", + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "fips-ap-northeast-1": endpoint{ + Hostname: "fms-fips.ap-northeast-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-northeast-1", + }, + }, + "fips-ap-northeast-2": endpoint{ + Hostname: "fms-fips.ap-northeast-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-northeast-2", + }, + }, + "fips-ap-south-1": endpoint{ + Hostname: "fms-fips.ap-south-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-south-1", + }, + }, + "fips-ap-southeast-1": endpoint{ + Hostname: "fms-fips.ap-southeast-1.amazonaws.com", CredentialScope: credentialScope{ Region: "ap-southeast-1", }, }, - "ap-southeast-2": endpoint{ - Hostname: "rds.ap-southeast-2.amazonaws.com", + "fips-ap-southeast-2": endpoint{ + Hostname: "fms-fips.ap-southeast-2.amazonaws.com", CredentialScope: credentialScope{ Region: "ap-southeast-2", }, }, - "eu-central-1": endpoint{ - Hostname: "rds.eu-central-1.amazonaws.com", + "fips-ca-central-1": endpoint{ + Hostname: "fms-fips.ca-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, + }, + "fips-eu-central-1": endpoint{ + Hostname: "fms-fips.eu-central-1.amazonaws.com", CredentialScope: credentialScope{ Region: "eu-central-1", }, }, - "eu-west-1": endpoint{ - Hostname: "rds.eu-west-1.amazonaws.com", + "fips-eu-west-1": endpoint{ + Hostname: "fms-fips.eu-west-1.amazonaws.com", CredentialScope: credentialScope{ Region: "eu-west-1", }, }, - "eu-west-2": endpoint{ - Hostname: "rds.eu-west-2.amazonaws.com", + "fips-eu-west-2": endpoint{ + Hostname: "fms-fips.eu-west-2.amazonaws.com", CredentialScope: credentialScope{ Region: "eu-west-2", }, }, - "us-east-1": endpoint{ - Hostname: "rds.us-east-1.amazonaws.com", + "fips-eu-west-3": endpoint{ + Hostname: "fms-fips.eu-west-3.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-west-3", + }, + }, + "fips-sa-east-1": endpoint{ + Hostname: "fms-fips.sa-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "sa-east-1", + }, + }, + "fips-us-east-1": endpoint{ + Hostname: "fms-fips.us-east-1.amazonaws.com", CredentialScope: credentialScope{ Region: "us-east-1", }, }, - "us-east-2": endpoint{ - Hostname: "rds.us-east-2.amazonaws.com", + "fips-us-east-2": endpoint{ + Hostname: "fms-fips.us-east-2.amazonaws.com", CredentialScope: credentialScope{ Region: "us-east-2", }, }, - "us-west-2": endpoint{ - Hostname: "rds.us-west-2.amazonaws.com", + "fips-us-west-1": endpoint{ + Hostname: "fms-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "fms-fips.us-west-2.amazonaws.com", CredentialScope: credentialScope{ Region: "us-west-2", }, }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "opsworks": service{ + "forecast": service{ Endpoints: endpoints{ "ap-northeast-1": endpoint{}, @@ -1816,59 +2706,49 @@ var awsPartition = partition{ "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, - "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, "eu-west-1": endpoint{}, - "eu-west-2": endpoint{}, - "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, - "us-west-1": endpoint{}, "us-west-2": endpoint{}, }, }, - "opsworks-cm": service{ + "forecastquery": service{ Endpoints: endpoints{ "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, "eu-central-1": endpoint{}, "eu-west-1": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, - "us-west-1": endpoint{}, "us-west-2": endpoint{}, }, }, - "organizations": service{ - PartitionEndpoint: "aws-global", - IsRegionalized: boxedFalse, + "fsx": service{ Endpoints: endpoints{ - "aws-global": endpoint{ - Hostname: "organizations.us-east-1.amazonaws.com", - CredentialScope: credentialScope{ - Region: "us-east-1", - }, - }, - }, - }, - "pinpoint": service{ - Defaults: endpoint{ - CredentialScope: credentialScope{ - Service: "mobiletargeting", - }, - }, - Endpoints: endpoints{ - "eu-central-1": endpoint{}, - "eu-west-1": endpoint{}, - "us-east-1": endpoint{}, - "us-west-2": endpoint{}, + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "polly": service{ + "gamelift": service{ Endpoints: endpoints{ "ap-northeast-1": endpoint{}, @@ -1880,7 +2760,6 @@ var awsPartition = partition{ "eu-central-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, - "eu-west-3": endpoint{}, "sa-east-1": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, @@ -1888,9 +2767,13 @@ var awsPartition = partition{ "us-west-2": endpoint{}, }, }, - "rds": service{ - + "glacier": service{ + Defaults: endpoint{ + Protocols: []string{"http", "https"}, + }, Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -1899,21 +2782,52 @@ var awsPartition = partition{ "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{ - SSLCommonName: "{service}.{dnsSuffix}", + "fips-ca-central-1": endpoint{ + Hostname: "glacier-fips.ca-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, }, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "glacier-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "glacier-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "glacier-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "glacier-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "redshift": service{ + "glue": service{ Endpoints: endpoints{ + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -1925,27 +2839,75 @@ var awsPartition = partition{ "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "glue-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "glue-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "glue-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "glue-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "greengrass": service{ + IsRegionalized: boxedTrue, + Defaults: endpoint{ + Protocols: []string{"https"}, + }, + Endpoints: endpoints{ + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "eu-central-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, - "us-west-1": endpoint{}, "us-west-2": endpoint{}, }, }, - "rekognition": service{ + "groundstation": service{ Endpoints: endpoints{ - "ap-northeast-1": endpoint{}, "ap-southeast-2": endpoint{}, + "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, - "us-east-1": endpoint{}, + "me-south-1": endpoint{}, "us-east-2": endpoint{}, "us-west-2": endpoint{}, }, }, - "resource-groups": service{ - + "guardduty": service{ + IsRegionalized: boxedTrue, + Defaults: endpoint{ + Protocols: []string{"https"}, + }, Endpoints: endpoints{ + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -1957,316 +2919,286 @@ var awsPartition = partition{ "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, "sa-east-1": endpoint{}, "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, + "us-east-1-fips": endpoint{ + Hostname: "guardduty-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "us-east-2": endpoint{}, + "us-east-2-fips": endpoint{ + Hostname: "guardduty-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "us-west-1": endpoint{}, + "us-west-1-fips": endpoint{ + Hostname: "guardduty-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "us-west-2": endpoint{}, + "us-west-2-fips": endpoint{ + Hostname: "guardduty-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, }, }, - "robomaker": service{ + "health": service{ Endpoints: endpoints{ - "eu-west-1": endpoint{}, "us-east-1": endpoint{}, + }, + }, + "honeycode": service{ + + Endpoints: endpoints{ "us-west-2": endpoint{}, }, }, - "route53": service{ + "iam": service{ PartitionEndpoint: "aws-global", IsRegionalized: boxedFalse, Endpoints: endpoints{ "aws-global": endpoint{ - Hostname: "route53.amazonaws.com", + Hostname: "iam.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "iam-fips": endpoint{ + Hostname: "iam-fips.amazonaws.com", CredentialScope: credentialScope{ Region: "us-east-1", }, }, }, }, - "route53domains": service{ + "importexport": service{ + PartitionEndpoint: "aws-global", + IsRegionalized: boxedFalse, + + Endpoints: endpoints{ + "aws-global": endpoint{ + Hostname: "importexport.amazonaws.com", + SignatureVersions: []string{"v2", "v4"}, + CredentialScope: credentialScope{ + Region: "us-east-1", + Service: "IngestionService", + }, + }, + }, + }, + "inspector": service{ Endpoints: endpoints{ + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "inspector-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "inspector-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "inspector-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "inspector-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "route53resolver": service{ + "iot": service{ Defaults: endpoint{ - Protocols: []string{"http", "https"}, + CredentialScope: credentialScope{ + Service: "execute-api", + }, }, Endpoints: endpoints{ + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, + "us-west-1": endpoint{}, "us-west-2": endpoint{}, }, }, - "runtime.lex": service{ - Defaults: endpoint{ - CredentialScope: credentialScope{ - Service: "lex", - }, - }, + "iotanalytics": service{ + Endpoints: endpoints{ - "eu-west-1": endpoint{}, - "us-east-1": endpoint{}, - "us-west-2": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "eu-central-1": endpoint{}, + "eu-west-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-2": endpoint{}, }, }, - "runtime.sagemaker": service{ + "iotevents": service{ Endpoints: endpoints{ "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, - "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, - "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, - "us-west-1": endpoint{}, "us-west-2": endpoint{}, }, }, - "s3": service{ - PartitionEndpoint: "us-east-1", - IsRegionalized: boxedTrue, - Defaults: endpoint{ - Protocols: []string{"http", "https"}, - SignatureVersions: []string{"s3v4"}, + "ioteventsdata": service{ - HasDualStack: boxedTrue, - DualStackHostname: "{service}.dualstack.{region}.{dnsSuffix}", - }, Endpoints: endpoints{ "ap-northeast-1": endpoint{ - Hostname: "s3.ap-northeast-1.amazonaws.com", - SignatureVersions: []string{"s3", "s3v4"}, - }, - "ap-northeast-2": endpoint{}, - "ap-south-1": endpoint{}, - "ap-southeast-1": endpoint{ - Hostname: "s3.ap-southeast-1.amazonaws.com", - SignatureVersions: []string{"s3", "s3v4"}, - }, - "ap-southeast-2": endpoint{ - Hostname: "s3.ap-southeast-2.amazonaws.com", - SignatureVersions: []string{"s3", "s3v4"}, - }, - "ca-central-1": endpoint{}, - "eu-central-1": endpoint{}, - "eu-north-1": endpoint{}, - "eu-west-1": endpoint{ - Hostname: "s3.eu-west-1.amazonaws.com", - SignatureVersions: []string{"s3", "s3v4"}, - }, - "eu-west-2": endpoint{}, - "eu-west-3": endpoint{}, - "s3-external-1": endpoint{ - Hostname: "s3-external-1.amazonaws.com", - SignatureVersions: []string{"s3", "s3v4"}, - CredentialScope: credentialScope{ - Region: "us-east-1", - }, - }, - "sa-east-1": endpoint{ - Hostname: "s3.sa-east-1.amazonaws.com", - SignatureVersions: []string{"s3", "s3v4"}, - }, - "us-east-1": endpoint{ - Hostname: "s3.amazonaws.com", - SignatureVersions: []string{"s3", "s3v4"}, - }, - "us-east-2": endpoint{}, - "us-west-1": endpoint{ - Hostname: "s3.us-west-1.amazonaws.com", - SignatureVersions: []string{"s3", "s3v4"}, - }, - "us-west-2": endpoint{ - Hostname: "s3.us-west-2.amazonaws.com", - SignatureVersions: []string{"s3", "s3v4"}, - }, - }, - }, - "s3-control": service{ - Defaults: endpoint{ - Protocols: []string{"https"}, - SignatureVersions: []string{"s3v4"}, - - HasDualStack: boxedTrue, - DualStackHostname: "{service}.dualstack.{region}.{dnsSuffix}", - }, - Endpoints: endpoints{ - "ap-northeast-1": endpoint{ - Hostname: "s3-control.ap-northeast-1.amazonaws.com", - SignatureVersions: []string{"s3v4"}, + Hostname: "data.iotevents.ap-northeast-1.amazonaws.com", CredentialScope: credentialScope{ Region: "ap-northeast-1", }, }, "ap-northeast-2": endpoint{ - Hostname: "s3-control.ap-northeast-2.amazonaws.com", - SignatureVersions: []string{"s3v4"}, + Hostname: "data.iotevents.ap-northeast-2.amazonaws.com", CredentialScope: credentialScope{ Region: "ap-northeast-2", }, }, - "ap-south-1": endpoint{ - Hostname: "s3-control.ap-south-1.amazonaws.com", - SignatureVersions: []string{"s3v4"}, - CredentialScope: credentialScope{ - Region: "ap-south-1", - }, - }, "ap-southeast-1": endpoint{ - Hostname: "s3-control.ap-southeast-1.amazonaws.com", - SignatureVersions: []string{"s3v4"}, + Hostname: "data.iotevents.ap-southeast-1.amazonaws.com", CredentialScope: credentialScope{ Region: "ap-southeast-1", }, }, "ap-southeast-2": endpoint{ - Hostname: "s3-control.ap-southeast-2.amazonaws.com", - SignatureVersions: []string{"s3v4"}, + Hostname: "data.iotevents.ap-southeast-2.amazonaws.com", CredentialScope: credentialScope{ Region: "ap-southeast-2", }, }, - "ca-central-1": endpoint{ - Hostname: "s3-control.ca-central-1.amazonaws.com", - SignatureVersions: []string{"s3v4"}, - CredentialScope: credentialScope{ - Region: "ca-central-1", - }, - }, "eu-central-1": endpoint{ - Hostname: "s3-control.eu-central-1.amazonaws.com", - SignatureVersions: []string{"s3v4"}, + Hostname: "data.iotevents.eu-central-1.amazonaws.com", CredentialScope: credentialScope{ Region: "eu-central-1", }, }, - "eu-north-1": endpoint{ - Hostname: "s3-control.eu-north-1.amazonaws.com", - SignatureVersions: []string{"s3v4"}, - CredentialScope: credentialScope{ - Region: "eu-north-1", - }, - }, "eu-west-1": endpoint{ - Hostname: "s3-control.eu-west-1.amazonaws.com", - SignatureVersions: []string{"s3v4"}, + Hostname: "data.iotevents.eu-west-1.amazonaws.com", CredentialScope: credentialScope{ Region: "eu-west-1", }, }, "eu-west-2": endpoint{ - Hostname: "s3-control.eu-west-2.amazonaws.com", - SignatureVersions: []string{"s3v4"}, + Hostname: "data.iotevents.eu-west-2.amazonaws.com", CredentialScope: credentialScope{ Region: "eu-west-2", }, }, - "eu-west-3": endpoint{ - Hostname: "s3-control.eu-west-3.amazonaws.com", - SignatureVersions: []string{"s3v4"}, - CredentialScope: credentialScope{ - Region: "eu-west-3", - }, - }, - "sa-east-1": endpoint{ - Hostname: "s3-control.sa-east-1.amazonaws.com", - SignatureVersions: []string{"s3v4"}, - CredentialScope: credentialScope{ - Region: "sa-east-1", - }, - }, "us-east-1": endpoint{ - Hostname: "s3-control.us-east-1.amazonaws.com", - SignatureVersions: []string{"s3v4"}, - CredentialScope: credentialScope{ - Region: "us-east-1", - }, - }, - "us-east-1-fips": endpoint{ - Hostname: "s3-control-fips.us-east-1.amazonaws.com", - SignatureVersions: []string{"s3v4"}, + Hostname: "data.iotevents.us-east-1.amazonaws.com", CredentialScope: credentialScope{ Region: "us-east-1", }, }, "us-east-2": endpoint{ - Hostname: "s3-control.us-east-2.amazonaws.com", - SignatureVersions: []string{"s3v4"}, - CredentialScope: credentialScope{ - Region: "us-east-2", - }, - }, - "us-east-2-fips": endpoint{ - Hostname: "s3-control-fips.us-east-2.amazonaws.com", - SignatureVersions: []string{"s3v4"}, + Hostname: "data.iotevents.us-east-2.amazonaws.com", CredentialScope: credentialScope{ Region: "us-east-2", }, }, - "us-west-1": endpoint{ - Hostname: "s3-control.us-west-1.amazonaws.com", - SignatureVersions: []string{"s3v4"}, - CredentialScope: credentialScope{ - Region: "us-west-1", - }, - }, - "us-west-1-fips": endpoint{ - Hostname: "s3-control-fips.us-west-1.amazonaws.com", - SignatureVersions: []string{"s3v4"}, - CredentialScope: credentialScope{ - Region: "us-west-1", - }, - }, "us-west-2": endpoint{ - Hostname: "s3-control.us-west-2.amazonaws.com", - SignatureVersions: []string{"s3v4"}, - CredentialScope: credentialScope{ - Region: "us-west-2", - }, - }, - "us-west-2-fips": endpoint{ - Hostname: "s3-control-fips.us-west-2.amazonaws.com", - SignatureVersions: []string{"s3v4"}, + Hostname: "data.iotevents.us-west-2.amazonaws.com", CredentialScope: credentialScope{ Region: "us-west-2", }, }, }, }, - "sdb": service{ - Defaults: endpoint{ - Protocols: []string{"http", "https"}, - SignatureVersions: []string{"v2"}, - }, + "iotsecuredtunneling": service{ + Endpoints: endpoints{ + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, "sa-east-1": endpoint{}, - "us-east-1": endpoint{ - Hostname: "sdb.amazonaws.com", + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "iotthingsgraph": service{ + Defaults: endpoint{ + CredentialScope: credentialScope{ + Service: "iotthingsgraph", }, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, + }, + Endpoints: endpoints{ + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-southeast-2": endpoint{}, + "eu-west-1": endpoint{}, + "us-east-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "secretsmanager": service{ + "kafka": service{ Endpoints: endpoints{ + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -2274,92 +3206,93 @@ var awsPartition = partition{ "ap-southeast-2": endpoint{}, "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, "sa-east-1": endpoint{}, "us-east-1": endpoint{}, - "us-east-1-fips": endpoint{ - Hostname: "secretsmanager-fips.us-east-1.amazonaws.com", + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "kinesis": service{ + + Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "kinesis-fips.us-east-1.amazonaws.com", CredentialScope: credentialScope{ Region: "us-east-1", }, }, - "us-east-2": endpoint{}, - "us-east-2-fips": endpoint{ - Hostname: "secretsmanager-fips.us-east-2.amazonaws.com", + "fips-us-east-2": endpoint{ + Hostname: "kinesis-fips.us-east-2.amazonaws.com", CredentialScope: credentialScope{ Region: "us-east-2", }, }, - "us-west-1": endpoint{}, - "us-west-1-fips": endpoint{ - Hostname: "secretsmanager-fips.us-west-1.amazonaws.com", + "fips-us-west-1": endpoint{ + Hostname: "kinesis-fips.us-west-1.amazonaws.com", CredentialScope: credentialScope{ Region: "us-west-1", }, }, - "us-west-2": endpoint{}, - "us-west-2-fips": endpoint{ - Hostname: "secretsmanager-fips.us-west-2.amazonaws.com", + "fips-us-west-2": endpoint{ + Hostname: "kinesis-fips.us-west-2.amazonaws.com", CredentialScope: credentialScope{ Region: "us-west-2", }, }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "serverlessrepo": service{ - Defaults: endpoint{ - Protocols: []string{"https"}, + "kinesisanalytics": service{ + + Endpoints: endpoints{ + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-2": endpoint{}, }, + }, + "kinesisvideo": service{ + Endpoints: endpoints{ - "ap-northeast-1": endpoint{ - Protocols: []string{"https"}, - }, - "ap-northeast-2": endpoint{ - Protocols: []string{"https"}, - }, - "ap-south-1": endpoint{ - Protocols: []string{"https"}, - }, - "ap-southeast-1": endpoint{ - Protocols: []string{"https"}, - }, - "ap-southeast-2": endpoint{ - Protocols: []string{"https"}, - }, - "ca-central-1": endpoint{ - Protocols: []string{"https"}, - }, - "eu-central-1": endpoint{ - Protocols: []string{"https"}, - }, - "eu-west-1": endpoint{ - Protocols: []string{"https"}, - }, - "eu-west-2": endpoint{ - Protocols: []string{"https"}, - }, - "sa-east-1": endpoint{ - Protocols: []string{"https"}, - }, - "us-east-1": endpoint{ - Protocols: []string{"https"}, - }, - "us-east-2": endpoint{ - Protocols: []string{"https"}, - }, - "us-west-1": endpoint{ - Protocols: []string{"https"}, - }, - "us-west-2": endpoint{ - Protocols: []string{"https"}, - }, - }, - }, - "servicecatalog": service{ - - Endpoints: endpoints{ + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -2372,38 +3305,15 @@ var awsPartition = partition{ "eu-west-3": endpoint{}, "sa-east-1": endpoint{}, "us-east-1": endpoint{}, - "us-east-1-fips": endpoint{ - Hostname: "servicecatalog-fips.us-east-1.amazonaws.com", - CredentialScope: credentialScope{ - Region: "us-east-1", - }, - }, - "us-east-2": endpoint{}, - "us-east-2-fips": endpoint{ - Hostname: "servicecatalog-fips.us-east-2.amazonaws.com", - CredentialScope: credentialScope{ - Region: "us-east-2", - }, - }, - "us-west-1": endpoint{}, - "us-west-1-fips": endpoint{ - Hostname: "servicecatalog-fips.us-west-1.amazonaws.com", - CredentialScope: credentialScope{ - Region: "us-west-1", - }, - }, - "us-west-2": endpoint{}, - "us-west-2-fips": endpoint{ - Hostname: "servicecatalog-fips.us-west-2.amazonaws.com", - CredentialScope: credentialScope{ - Region: "us-west-2", - }, - }, + "us-east-2": endpoint{}, + "us-west-2": endpoint{}, }, }, - "servicediscovery": service{ + "kms": service{ Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -2411,9 +3321,12 @@ var awsPartition = partition{ "ap-southeast-2": endpoint{}, "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, "sa-east-1": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, @@ -2421,17 +3334,7 @@ var awsPartition = partition{ "us-west-2": endpoint{}, }, }, - "shield": service{ - IsRegionalized: boxedFalse, - Defaults: endpoint{ - SSLCommonName: "shield.us-east-1.amazonaws.com", - Protocols: []string{"https"}, - }, - Endpoints: endpoints{ - "us-east-1": endpoint{}, - }, - }, - "sms": service{ + "lakeformation": service{ Endpoints: endpoints{ "ap-northeast-1": endpoint{}, @@ -2441,6 +3344,7 @@ var awsPartition = partition{ "ap-southeast-2": endpoint{}, "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, @@ -2451,30 +3355,60 @@ var awsPartition = partition{ "us-west-2": endpoint{}, }, }, - "snowball": service{ + "lambda": service{ Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "lambda-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "lambda-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "lambda-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "lambda-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "sns": service{ - Defaults: endpoint{ - Protocols: []string{"http", "https"}, - }, + "license-manager": service{ + Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -2483,22 +3417,65 @@ var awsPartition = partition{ "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "license-manager-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "license-manager-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "license-manager-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "license-manager-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "lightsail": service{ + + Endpoints: endpoints{ + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, - "us-west-1": endpoint{}, "us-west-2": endpoint{}, }, }, - "sqs": service{ - Defaults: endpoint{ - SSLCommonName: "{region}.queue.{dnsSuffix}", - Protocols: []string{"http", "https"}, - }, + "logs": service{ + Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -2507,45 +3484,72 @@ var awsPartition = partition{ "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, "fips-us-east-1": endpoint{ - Hostname: "sqs-fips.us-east-1.amazonaws.com", + Hostname: "logs-fips.us-east-1.amazonaws.com", CredentialScope: credentialScope{ Region: "us-east-1", }, }, "fips-us-east-2": endpoint{ - Hostname: "sqs-fips.us-east-2.amazonaws.com", + Hostname: "logs-fips.us-east-2.amazonaws.com", CredentialScope: credentialScope{ Region: "us-east-2", }, }, "fips-us-west-1": endpoint{ - Hostname: "sqs-fips.us-west-1.amazonaws.com", + Hostname: "logs-fips.us-west-1.amazonaws.com", CredentialScope: credentialScope{ Region: "us-west-1", }, }, "fips-us-west-2": endpoint{ - Hostname: "sqs-fips.us-west-2.amazonaws.com", + Hostname: "logs-fips.us-west-2.amazonaws.com", CredentialScope: credentialScope{ Region: "us-west-2", }, }, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{ - SSLCommonName: "queue.{dnsSuffix}", + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "machinelearning": service{ + + Endpoints: endpoints{ + "eu-west-1": endpoint{}, + "us-east-1": endpoint{}, + }, + }, + "macie": service{ + + Endpoints: endpoints{ + "fips-us-east-1": endpoint{ + Hostname: "macie-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, }, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, + "fips-us-west-2": endpoint{ + Hostname: "macie-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "us-east-1": endpoint{}, "us-west-2": endpoint{}, }, }, - "ssm": service{ + "macie2": service{ Endpoints: endpoints{ + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -2557,43 +3561,62 @@ var awsPartition = partition{ "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "macie2-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "macie2-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "macie2-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "macie2-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "states": service{ + "managedblockchain": service{ Endpoints: endpoints{ "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, - "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, - "ap-southeast-2": endpoint{}, - "ca-central-1": endpoint{}, - "eu-central-1": endpoint{}, - "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, - "eu-west-2": endpoint{}, - "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, }, }, - "storagegateway": service{ + "marketplacecommerceanalytics": service{ + + Endpoints: endpoints{ + "us-east-1": endpoint{}, + }, + }, + "mediaconnect": service{ Endpoints: endpoints{ + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, - "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, @@ -2606,13 +3629,8 @@ var awsPartition = partition{ "us-west-2": endpoint{}, }, }, - "streams.dynamodb": service{ - Defaults: endpoint{ - Protocols: []string{"http", "https"}, - CredentialScope: credentialScope{ - Service: "dynamodb", - }, - }, + "mediaconvert": service{ + Endpoints: endpoints{ "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, @@ -2625,84 +3643,44 @@ var awsPartition = partition{ "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, - "local": endpoint{ - Hostname: "localhost:8000", - Protocols: []string{"http"}, + "fips-ca-central-1": endpoint{ + Hostname: "mediaconvert-fips.ca-central-1.amazonaws.com", CredentialScope: credentialScope{ - Region: "us-east-1", + Region: "ca-central-1", }, }, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, - }, - }, - "sts": service{ - PartitionEndpoint: "aws-global", - Defaults: endpoint{ - Hostname: "sts.amazonaws.com", - CredentialScope: credentialScope{ - Region: "us-east-1", - }, - }, - Endpoints: endpoints{ - "ap-northeast-1": endpoint{}, - "ap-northeast-2": endpoint{ - Hostname: "sts.ap-northeast-2.amazonaws.com", - CredentialScope: credentialScope{ - Region: "ap-northeast-2", - }, - }, - "ap-south-1": endpoint{}, - "ap-southeast-1": endpoint{}, - "ap-southeast-2": endpoint{}, - "aws-global": endpoint{}, - "ca-central-1": endpoint{}, - "eu-central-1": endpoint{}, - "eu-north-1": endpoint{}, - "eu-west-1": endpoint{}, - "eu-west-2": endpoint{}, - "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-1-fips": endpoint{ - Hostname: "sts-fips.us-east-1.amazonaws.com", + "fips-us-east-1": endpoint{ + Hostname: "mediaconvert-fips.us-east-1.amazonaws.com", CredentialScope: credentialScope{ Region: "us-east-1", }, }, - "us-east-2": endpoint{}, - "us-east-2-fips": endpoint{ - Hostname: "sts-fips.us-east-2.amazonaws.com", + "fips-us-east-2": endpoint{ + Hostname: "mediaconvert-fips.us-east-2.amazonaws.com", CredentialScope: credentialScope{ Region: "us-east-2", }, }, - "us-west-1": endpoint{}, - "us-west-1-fips": endpoint{ - Hostname: "sts-fips.us-west-1.amazonaws.com", + "fips-us-west-1": endpoint{ + Hostname: "mediaconvert-fips.us-west-1.amazonaws.com", CredentialScope: credentialScope{ Region: "us-west-1", }, }, - "us-west-2": endpoint{}, - "us-west-2-fips": endpoint{ - Hostname: "sts-fips.us-west-2.amazonaws.com", + "fips-us-west-2": endpoint{ + Hostname: "mediaconvert-fips.us-west-2.amazonaws.com", CredentialScope: credentialScope{ Region: "us-west-2", }, }, - }, - }, - "support": service{ - - Endpoints: endpoints{ + "sa-east-1": endpoint{}, "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - "swf": service{ + "medialive": service{ Endpoints: endpoints{ "ap-northeast-1": endpoint{}, @@ -2710,7 +3688,6 @@ var awsPartition = partition{ "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, - "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, @@ -2719,11 +3696,10 @@ var awsPartition = partition{ "sa-east-1": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, - "us-west-1": endpoint{}, "us-west-2": endpoint{}, }, }, - "tagging": service{ + "mediapackage": service{ Endpoints: endpoints{ "ap-northeast-1": endpoint{}, @@ -2731,7 +3707,6 @@ var awsPartition = partition{ "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, - "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, @@ -2739,124 +3714,93 @@ var awsPartition = partition{ "eu-west-3": endpoint{}, "sa-east-1": endpoint{}, "us-east-1": endpoint{}, - "us-east-2": endpoint{}, "us-west-1": endpoint{}, "us-west-2": endpoint{}, }, }, - "transfer": service{ + "mediastore": service{ Endpoints: endpoints{ "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, - "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, - "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, - "eu-west-3": endpoint{}, "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, "us-west-2": endpoint{}, }, }, - "translate": service{ + "metering.marketplace": service{ Defaults: endpoint{ - Protocols: []string{"https"}, - }, - Endpoints: endpoints{ - "eu-west-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-1-fips": endpoint{ - Hostname: "translate-fips.us-east-1.amazonaws.com", - CredentialScope: credentialScope{ - Region: "us-east-1", - }, - }, - "us-east-2": endpoint{}, - "us-east-2-fips": endpoint{ - Hostname: "translate-fips.us-east-2.amazonaws.com", - CredentialScope: credentialScope{ - Region: "us-east-2", - }, - }, - "us-west-2": endpoint{}, - "us-west-2-fips": endpoint{ - Hostname: "translate-fips.us-west-2.amazonaws.com", - CredentialScope: credentialScope{ - Region: "us-west-2", - }, - }, - }, - }, - "waf": service{ - PartitionEndpoint: "aws-global", - IsRegionalized: boxedFalse, - - Endpoints: endpoints{ - "aws-global": endpoint{ - Hostname: "waf.amazonaws.com", - CredentialScope: credentialScope{ - Region: "us-east-1", - }, + CredentialScope: credentialScope{ + Service: "aws-marketplace", }, }, - }, - "waf-regional": service{ - Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, "us-west-1": endpoint{}, "us-west-2": endpoint{}, }, }, - "workdocs": service{ + "mgh": service{ Endpoints: endpoints{ "ap-northeast-1": endpoint{}, - "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, - "eu-west-1": endpoint{}, + "eu-central-1": endpoint{}, "us-east-1": endpoint{}, "us-west-2": endpoint{}, }, }, - "workmail": service{ - Defaults: endpoint{ - Protocols: []string{"https"}, - }, + "mobileanalytics": service{ + Endpoints: endpoints{ - "eu-west-1": endpoint{}, "us-east-1": endpoint{}, - "us-west-2": endpoint{}, }, }, - "workspaces": service{ - + "models.lex": service{ + Defaults: endpoint{ + CredentialScope: credentialScope{ + Service: "lex", + }, + }, Endpoints: endpoints{ "ap-northeast-1": endpoint{}, - "ap-northeast-2": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, - "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, - "sa-east-1": endpoint{}, "us-east-1": endpoint{}, "us-west-2": endpoint{}, }, }, - "xray": service{ - + "monitoring": service{ + Defaults: endpoint{ + Protocols: []string{"http", "https"}, + }, Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -2865,970 +3809,5480 @@ var awsPartition = partition{ "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "monitoring-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "monitoring-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "monitoring-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "monitoring-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, - }, -} - -// AwsCnPartition returns the Resolver for AWS China. -func AwsCnPartition() Partition { - return awscnPartition.Partition() -} + "mq": service{ -var awscnPartition = partition{ - ID: "aws-cn", - Name: "AWS China", - DNSSuffix: "amazonaws.com.cn", - RegionRegex: regionRegex{ - Regexp: func() *regexp.Regexp { - reg, _ := regexp.Compile("^cn\\-\\w+\\-\\d+$") - return reg - }(), - }, - Defaults: endpoint{ - Hostname: "{service}.{region}.{dnsSuffix}", - Protocols: []string{"https"}, - SignatureVersions: []string{"v4"}, - }, - Regions: regions{ - "cn-north-1": region{ - Description: "China (Beijing)", - }, - "cn-northwest-1": region{ - Description: "China (Ningxia)", + Endpoints: endpoints{ + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "mq-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "mq-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "mq-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "mq-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, }, - }, - Services: services{ - "apigateway": service{ + "mturk-requester": service{ + IsRegionalized: boxedFalse, Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "sandbox": endpoint{ + Hostname: "mturk-requester-sandbox.us-east-1.amazonaws.com", + }, + "us-east-1": endpoint{}, }, }, - "application-autoscaling": service{ - Defaults: endpoint{ - Hostname: "autoscaling.{region}.amazonaws.com.cn", - Protocols: []string{"http", "https"}, + "neptune": service{ + + Endpoints: endpoints{ + "ap-northeast-1": endpoint{ + Hostname: "rds.ap-northeast-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-northeast-1", + }, + }, + "ap-northeast-2": endpoint{ + Hostname: "rds.ap-northeast-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-northeast-2", + }, + }, + "ap-south-1": endpoint{ + Hostname: "rds.ap-south-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-south-1", + }, + }, + "ap-southeast-1": endpoint{ + Hostname: "rds.ap-southeast-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-southeast-1", + }, + }, + "ap-southeast-2": endpoint{ + Hostname: "rds.ap-southeast-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-southeast-2", + }, + }, + "ca-central-1": endpoint{ + Hostname: "rds.ca-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, + }, + "eu-central-1": endpoint{ + Hostname: "rds.eu-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-central-1", + }, + }, + "eu-north-1": endpoint{ + Hostname: "rds.eu-north-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-north-1", + }, + }, + "eu-west-1": endpoint{ + Hostname: "rds.eu-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-west-1", + }, + }, + "eu-west-2": endpoint{ + Hostname: "rds.eu-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-west-2", + }, + }, + "eu-west-3": endpoint{ + Hostname: "rds.eu-west-3.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-west-3", + }, + }, + "me-south-1": endpoint{ + Hostname: "rds.me-south-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "me-south-1", + }, + }, + "us-east-1": endpoint{ + Hostname: "rds.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "us-east-2": endpoint{ + Hostname: "rds.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "us-west-1": endpoint{ + Hostname: "rds.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "us-west-2": endpoint{ + Hostname: "rds.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + }, + }, + "oidc": service{ + + Endpoints: endpoints{ + "ap-southeast-1": endpoint{ + Hostname: "oidc.ap-southeast-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-southeast-1", + }, + }, + "ap-southeast-2": endpoint{ + Hostname: "oidc.ap-southeast-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-southeast-2", + }, + }, + "ca-central-1": endpoint{ + Hostname: "oidc.ca-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, + }, + "eu-central-1": endpoint{ + Hostname: "oidc.eu-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-central-1", + }, + }, + "eu-west-1": endpoint{ + Hostname: "oidc.eu-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-west-1", + }, + }, + "eu-west-2": endpoint{ + Hostname: "oidc.eu-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-west-2", + }, + }, + "us-east-1": endpoint{ + Hostname: "oidc.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "us-east-2": endpoint{ + Hostname: "oidc.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "us-west-2": endpoint{ + Hostname: "oidc.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + }, + }, + "opsworks": service{ + + Endpoints: endpoints{ + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "opsworks-cm": service{ + + Endpoints: endpoints{ + "ap-northeast-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "eu-central-1": endpoint{}, + "eu-west-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "organizations": service{ + PartitionEndpoint: "aws-global", + IsRegionalized: boxedFalse, + + Endpoints: endpoints{ + "aws-global": endpoint{ + Hostname: "organizations.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-aws-global": endpoint{ + Hostname: "organizations-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + }, + }, + "outposts": service{ + + Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "fips-ca-central-1": endpoint{ + Hostname: "outposts-fips.ca-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, + }, + "fips-us-east-1": endpoint{ + Hostname: "outposts-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "outposts-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "outposts-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "outposts-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "pinpoint": service{ + Defaults: endpoint{ + CredentialScope: credentialScope{ + Service: "mobiletargeting", + }, + }, + Endpoints: endpoints{ + "ap-south-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "eu-central-1": endpoint{}, + "eu-west-1": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "pinpoint-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "pinpoint-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "us-east-1": endpoint{ + Hostname: "pinpoint.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "us-west-2": endpoint{ + Hostname: "pinpoint.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + }, + }, + "polly": service{ + + Endpoints: endpoints{ + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "polly-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "polly-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "polly-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "polly-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "portal.sso": service{ + + Endpoints: endpoints{ + "ap-southeast-1": endpoint{ + Hostname: "portal.sso.ap-southeast-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-southeast-1", + }, + }, + "ap-southeast-2": endpoint{ + Hostname: "portal.sso.ap-southeast-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-southeast-2", + }, + }, + "ca-central-1": endpoint{ + Hostname: "portal.sso.ca-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, + }, + "eu-central-1": endpoint{ + Hostname: "portal.sso.eu-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-central-1", + }, + }, + "eu-west-1": endpoint{ + Hostname: "portal.sso.eu-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-west-1", + }, + }, + "eu-west-2": endpoint{ + Hostname: "portal.sso.eu-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-west-2", + }, + }, + "us-east-1": endpoint{ + Hostname: "portal.sso.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "us-east-2": endpoint{ + Hostname: "portal.sso.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "us-west-2": endpoint{ + Hostname: "portal.sso.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + }, + }, + "projects.iot1click": service{ + + Endpoints: endpoints{ + "ap-northeast-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "qldb": service{ + + Endpoints: endpoints{ + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "eu-central-1": endpoint{}, + "eu-west-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "ram": service{ + + Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "rds": service{ + + Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, + "rds-fips.ca-central-1": endpoint{ + Hostname: "rds-fips.ca-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, + }, + "rds-fips.us-east-1": endpoint{ + Hostname: "rds-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "rds-fips.us-east-2": endpoint{ + Hostname: "rds-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "rds-fips.us-west-1": endpoint{ + Hostname: "rds-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "rds-fips.us-west-2": endpoint{ + Hostname: "rds-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{ + SSLCommonName: "{service}.{dnsSuffix}", + }, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "redshift": service{ + + Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "fips-ca-central-1": endpoint{ + Hostname: "redshift-fips.ca-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, + }, + "fips-us-east-1": endpoint{ + Hostname: "redshift-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "redshift-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "redshift-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "redshift-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "rekognition": service{ + + Endpoints: endpoints{ + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "eu-central-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "rekognition-fips.us-east-1": endpoint{ + Hostname: "rekognition-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "rekognition-fips.us-east-2": endpoint{ + Hostname: "rekognition-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "rekognition-fips.us-west-1": endpoint{ + Hostname: "rekognition-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "rekognition-fips.us-west-2": endpoint{ + Hostname: "rekognition-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "resource-groups": service{ + + Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "resource-groups-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "resource-groups-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "resource-groups-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "resource-groups-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "robomaker": service{ + + Endpoints: endpoints{ + "ap-northeast-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-west-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "route53": service{ + PartitionEndpoint: "aws-global", + IsRegionalized: boxedFalse, + + Endpoints: endpoints{ + "aws-global": endpoint{ + Hostname: "route53.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + }, + }, + "route53domains": service{ + + Endpoints: endpoints{ + "us-east-1": endpoint{}, + }, + }, + "route53resolver": service{ + Defaults: endpoint{ + Protocols: []string{"https"}, + }, + Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "runtime.lex": service{ + Defaults: endpoint{ + CredentialScope: credentialScope{ + Service: "lex", + }, + }, + Endpoints: endpoints{ + "ap-northeast-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "eu-central-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "us-east-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "runtime.sagemaker": service{ + + Endpoints: endpoints{ + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-1-fips": endpoint{ + Hostname: "runtime-fips.sagemaker.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "us-east-2": endpoint{}, + "us-east-2-fips": endpoint{ + Hostname: "runtime-fips.sagemaker.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "us-west-1": endpoint{}, + "us-west-1-fips": endpoint{ + Hostname: "runtime-fips.sagemaker.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "us-west-2": endpoint{}, + "us-west-2-fips": endpoint{ + Hostname: "runtime-fips.sagemaker.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + }, + }, + "s3": service{ + PartitionEndpoint: "aws-global", + IsRegionalized: boxedTrue, + Defaults: endpoint{ + Protocols: []string{"http", "https"}, + SignatureVersions: []string{"s3v4"}, + + HasDualStack: boxedTrue, + DualStackHostname: "{service}.dualstack.{region}.{dnsSuffix}", + }, + Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{ + Hostname: "s3.ap-northeast-1.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{ + Hostname: "s3.ap-southeast-1.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + "ap-southeast-2": endpoint{ + Hostname: "s3.ap-southeast-2.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + "aws-global": endpoint{ + Hostname: "s3.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, + "eu-west-1": endpoint{ + Hostname: "s3.eu-west-1.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, + "s3-external-1": endpoint{ + Hostname: "s3-external-1.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "sa-east-1": endpoint{ + Hostname: "s3.sa-east-1.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + "us-east-1": endpoint{ + Hostname: "s3.us-east-1.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + "us-east-2": endpoint{}, + "us-west-1": endpoint{ + Hostname: "s3.us-west-1.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + "us-west-2": endpoint{ + Hostname: "s3.us-west-2.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + }, + }, + "s3-control": service{ + Defaults: endpoint{ + Protocols: []string{"https"}, + SignatureVersions: []string{"s3v4"}, + + HasDualStack: boxedTrue, + DualStackHostname: "{service}.dualstack.{region}.{dnsSuffix}", + }, + Endpoints: endpoints{ + "ap-northeast-1": endpoint{ + Hostname: "s3-control.ap-northeast-1.amazonaws.com", + SignatureVersions: []string{"s3v4"}, + CredentialScope: credentialScope{ + Region: "ap-northeast-1", + }, + }, + "ap-northeast-2": endpoint{ + Hostname: "s3-control.ap-northeast-2.amazonaws.com", + SignatureVersions: []string{"s3v4"}, + CredentialScope: credentialScope{ + Region: "ap-northeast-2", + }, + }, + "ap-south-1": endpoint{ + Hostname: "s3-control.ap-south-1.amazonaws.com", + SignatureVersions: []string{"s3v4"}, + CredentialScope: credentialScope{ + Region: "ap-south-1", + }, + }, + "ap-southeast-1": endpoint{ + Hostname: "s3-control.ap-southeast-1.amazonaws.com", + SignatureVersions: []string{"s3v4"}, + CredentialScope: credentialScope{ + Region: "ap-southeast-1", + }, + }, + "ap-southeast-2": endpoint{ + Hostname: "s3-control.ap-southeast-2.amazonaws.com", + SignatureVersions: []string{"s3v4"}, + CredentialScope: credentialScope{ + Region: "ap-southeast-2", + }, + }, + "ca-central-1": endpoint{ + Hostname: "s3-control.ca-central-1.amazonaws.com", + SignatureVersions: []string{"s3v4"}, + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, + }, + "eu-central-1": endpoint{ + Hostname: "s3-control.eu-central-1.amazonaws.com", + SignatureVersions: []string{"s3v4"}, + CredentialScope: credentialScope{ + Region: "eu-central-1", + }, + }, + "eu-north-1": endpoint{ + Hostname: "s3-control.eu-north-1.amazonaws.com", + SignatureVersions: []string{"s3v4"}, + CredentialScope: credentialScope{ + Region: "eu-north-1", + }, + }, + "eu-west-1": endpoint{ + Hostname: "s3-control.eu-west-1.amazonaws.com", + SignatureVersions: []string{"s3v4"}, + CredentialScope: credentialScope{ + Region: "eu-west-1", + }, + }, + "eu-west-2": endpoint{ + Hostname: "s3-control.eu-west-2.amazonaws.com", + SignatureVersions: []string{"s3v4"}, + CredentialScope: credentialScope{ + Region: "eu-west-2", + }, + }, + "eu-west-3": endpoint{ + Hostname: "s3-control.eu-west-3.amazonaws.com", + SignatureVersions: []string{"s3v4"}, + CredentialScope: credentialScope{ + Region: "eu-west-3", + }, + }, + "sa-east-1": endpoint{ + Hostname: "s3-control.sa-east-1.amazonaws.com", + SignatureVersions: []string{"s3v4"}, + CredentialScope: credentialScope{ + Region: "sa-east-1", + }, + }, + "us-east-1": endpoint{ + Hostname: "s3-control.us-east-1.amazonaws.com", + SignatureVersions: []string{"s3v4"}, + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "us-east-1-fips": endpoint{ + Hostname: "s3-control-fips.us-east-1.amazonaws.com", + SignatureVersions: []string{"s3v4"}, + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "us-east-2": endpoint{ + Hostname: "s3-control.us-east-2.amazonaws.com", + SignatureVersions: []string{"s3v4"}, + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "us-east-2-fips": endpoint{ + Hostname: "s3-control-fips.us-east-2.amazonaws.com", + SignatureVersions: []string{"s3v4"}, + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "us-west-1": endpoint{ + Hostname: "s3-control.us-west-1.amazonaws.com", + SignatureVersions: []string{"s3v4"}, + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "us-west-1-fips": endpoint{ + Hostname: "s3-control-fips.us-west-1.amazonaws.com", + SignatureVersions: []string{"s3v4"}, + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "us-west-2": endpoint{ + Hostname: "s3-control.us-west-2.amazonaws.com", + SignatureVersions: []string{"s3v4"}, + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "us-west-2-fips": endpoint{ + Hostname: "s3-control-fips.us-west-2.amazonaws.com", + SignatureVersions: []string{"s3v4"}, + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + }, + }, + "savingsplans": service{ + PartitionEndpoint: "aws-global", + IsRegionalized: boxedFalse, + + Endpoints: endpoints{ + "aws-global": endpoint{ + Hostname: "savingsplans.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + }, + }, + "schemas": service{ + + Endpoints: endpoints{ + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "sdb": service{ + Defaults: endpoint{ + Protocols: []string{"http", "https"}, + SignatureVersions: []string{"v2"}, + }, + Endpoints: endpoints{ + "ap-northeast-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "eu-west-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{ + Hostname: "sdb.amazonaws.com", + }, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "secretsmanager": service{ + + Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-1-fips": endpoint{ + Hostname: "secretsmanager-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "us-east-2": endpoint{}, + "us-east-2-fips": endpoint{ + Hostname: "secretsmanager-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "us-west-1": endpoint{}, + "us-west-1-fips": endpoint{ + Hostname: "secretsmanager-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "us-west-2": endpoint{}, + "us-west-2-fips": endpoint{ + Hostname: "secretsmanager-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + }, + }, + "securityhub": service{ + + Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "securityhub-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "securityhub-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "securityhub-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "securityhub-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "serverlessrepo": service{ + Defaults: endpoint{ + Protocols: []string{"https"}, + }, + Endpoints: endpoints{ + "ap-east-1": endpoint{ + Protocols: []string{"https"}, + }, + "ap-northeast-1": endpoint{ + Protocols: []string{"https"}, + }, + "ap-northeast-2": endpoint{ + Protocols: []string{"https"}, + }, + "ap-south-1": endpoint{ + Protocols: []string{"https"}, + }, + "ap-southeast-1": endpoint{ + Protocols: []string{"https"}, + }, + "ap-southeast-2": endpoint{ + Protocols: []string{"https"}, + }, + "ca-central-1": endpoint{ + Protocols: []string{"https"}, + }, + "eu-central-1": endpoint{ + Protocols: []string{"https"}, + }, + "eu-north-1": endpoint{ + Protocols: []string{"https"}, + }, + "eu-west-1": endpoint{ + Protocols: []string{"https"}, + }, + "eu-west-2": endpoint{ + Protocols: []string{"https"}, + }, + "eu-west-3": endpoint{ + Protocols: []string{"https"}, + }, + "me-south-1": endpoint{ + Protocols: []string{"https"}, + }, + "sa-east-1": endpoint{ + Protocols: []string{"https"}, + }, + "us-east-1": endpoint{ + Protocols: []string{"https"}, + }, + "us-east-2": endpoint{ + Protocols: []string{"https"}, + }, + "us-west-1": endpoint{ + Protocols: []string{"https"}, + }, + "us-west-2": endpoint{ + Protocols: []string{"https"}, + }, + }, + }, + "servicecatalog": service{ + + Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-1-fips": endpoint{ + Hostname: "servicecatalog-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "us-east-2": endpoint{}, + "us-east-2-fips": endpoint{ + Hostname: "servicecatalog-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "us-west-1": endpoint{}, + "us-west-1-fips": endpoint{ + Hostname: "servicecatalog-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "us-west-2": endpoint{}, + "us-west-2-fips": endpoint{ + Hostname: "servicecatalog-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + }, + }, + "servicediscovery": service{ + + Endpoints: endpoints{ + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "session.qldb": service{ + + Endpoints: endpoints{ + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "eu-central-1": endpoint{}, + "eu-west-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "shield": service{ + PartitionEndpoint: "aws-global", + IsRegionalized: boxedFalse, + Defaults: endpoint{ + SSLCommonName: "shield.us-east-1.amazonaws.com", + Protocols: []string{"https"}, + }, + Endpoints: endpoints{ + "aws-global": endpoint{ + Hostname: "shield.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-aws-global": endpoint{ + Hostname: "shield-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + }, + }, + "sms": service{ + + Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "sms-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "sms-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "sms-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "sms-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "snowball": service{ + + Endpoints: endpoints{ + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "fips-ap-northeast-1": endpoint{ + Hostname: "snowball-fips.ap-northeast-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-northeast-1", + }, + }, + "fips-ap-northeast-2": endpoint{ + Hostname: "snowball-fips.ap-northeast-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-northeast-2", + }, + }, + "fips-ap-south-1": endpoint{ + Hostname: "snowball-fips.ap-south-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-south-1", + }, + }, + "fips-ap-southeast-1": endpoint{ + Hostname: "snowball-fips.ap-southeast-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-southeast-1", + }, + }, + "fips-ap-southeast-2": endpoint{ + Hostname: "snowball-fips.ap-southeast-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-southeast-2", + }, + }, + "fips-ca-central-1": endpoint{ + Hostname: "snowball-fips.ca-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, + }, + "fips-eu-central-1": endpoint{ + Hostname: "snowball-fips.eu-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-central-1", + }, + }, + "fips-eu-west-1": endpoint{ + Hostname: "snowball-fips.eu-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-west-1", + }, + }, + "fips-eu-west-2": endpoint{ + Hostname: "snowball-fips.eu-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-west-2", + }, + }, + "fips-eu-west-3": endpoint{ + Hostname: "snowball-fips.eu-west-3.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-west-3", + }, + }, + "fips-sa-east-1": endpoint{ + Hostname: "snowball-fips.sa-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "sa-east-1", + }, + }, + "fips-us-east-1": endpoint{ + Hostname: "snowball-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "snowball-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "snowball-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "snowball-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "sns": service{ + Defaults: endpoint{ + Protocols: []string{"http", "https"}, + }, + Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "sns-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "sns-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "sns-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "sns-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "sqs": service{ + Defaults: endpoint{ + SSLCommonName: "{region}.queue.{dnsSuffix}", + Protocols: []string{"http", "https"}, + }, + Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "sqs-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "sqs-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "sqs-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "sqs-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{ + SSLCommonName: "queue.{dnsSuffix}", + }, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "ssm": service{ + + Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "ssm-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "ssm-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "ssm-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "ssm-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "ssm-facade-fips-us-east-1": endpoint{ + Hostname: "ssm-facade-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "ssm-facade-fips-us-east-2": endpoint{ + Hostname: "ssm-facade-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "ssm-facade-fips-us-west-1": endpoint{ + Hostname: "ssm-facade-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "ssm-facade-fips-us-west-2": endpoint{ + Hostname: "ssm-facade-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "states": service{ + + Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "states-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "states-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "states-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "states-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "storagegateway": service{ + + Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "streams.dynamodb": service{ + Defaults: endpoint{ + Protocols: []string{"http", "https"}, + CredentialScope: credentialScope{ + Service: "dynamodb", + }, + }, + Endpoints: endpoints{ + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "ca-central-1-fips": endpoint{ + Hostname: "dynamodb-fips.ca-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, + }, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "local": endpoint{ + Hostname: "localhost:8000", + Protocols: []string{"http"}, + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-1-fips": endpoint{ + Hostname: "dynamodb-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "us-east-2": endpoint{}, + "us-east-2-fips": endpoint{ + Hostname: "dynamodb-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "us-west-1": endpoint{}, + "us-west-1-fips": endpoint{ + Hostname: "dynamodb-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "us-west-2": endpoint{}, + "us-west-2-fips": endpoint{ + Hostname: "dynamodb-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + }, + }, + "sts": service{ + PartitionEndpoint: "aws-global", + + Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "aws-global": endpoint{ + Hostname: "sts.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-1-fips": endpoint{ + Hostname: "sts-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "us-east-2": endpoint{}, + "us-east-2-fips": endpoint{ + Hostname: "sts-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "us-west-1": endpoint{}, + "us-west-1-fips": endpoint{ + Hostname: "sts-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "us-west-2": endpoint{}, + "us-west-2-fips": endpoint{ + Hostname: "sts-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + }, + }, + "support": service{ + PartitionEndpoint: "aws-global", + + Endpoints: endpoints{ + "aws-global": endpoint{ + Hostname: "support.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + }, + }, + "swf": service{ + + Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "swf-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "swf-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "swf-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "swf-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "tagging": service{ + + Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "transcribe": service{ + Defaults: endpoint{ + Protocols: []string{"https"}, + }, + Endpoints: endpoints{ + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "fips.transcribe.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "fips.transcribe.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "fips.transcribe.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "fips.transcribe.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "transcribestreaming": service{ + + Endpoints: endpoints{ + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-west-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "transfer": service{ + + Endpoints: endpoints{ + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "translate": service{ + Defaults: endpoint{ + Protocols: []string{"https"}, + }, + Endpoints: endpoints{ + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "us-east-1": endpoint{}, + "us-east-1-fips": endpoint{ + Hostname: "translate-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "us-east-2": endpoint{}, + "us-east-2-fips": endpoint{ + Hostname: "translate-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + "us-west-2-fips": endpoint{ + Hostname: "translate-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + }, + }, + "waf": service{ + PartitionEndpoint: "aws-global", + IsRegionalized: boxedFalse, + + Endpoints: endpoints{ + "aws-fips": endpoint{ + Hostname: "waf-fips.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "aws-global": endpoint{ + Hostname: "waf.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + }, + }, + "waf-regional": service{ + + Endpoints: endpoints{ + "ap-east-1": endpoint{ + Hostname: "waf-regional.ap-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-east-1", + }, + }, + "ap-northeast-1": endpoint{ + Hostname: "waf-regional.ap-northeast-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-northeast-1", + }, + }, + "ap-northeast-2": endpoint{ + Hostname: "waf-regional.ap-northeast-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-northeast-2", + }, + }, + "ap-south-1": endpoint{ + Hostname: "waf-regional.ap-south-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-south-1", + }, + }, + "ap-southeast-1": endpoint{ + Hostname: "waf-regional.ap-southeast-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-southeast-1", + }, + }, + "ap-southeast-2": endpoint{ + Hostname: "waf-regional.ap-southeast-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-southeast-2", + }, + }, + "ca-central-1": endpoint{ + Hostname: "waf-regional.ca-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, + }, + "eu-central-1": endpoint{ + Hostname: "waf-regional.eu-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-central-1", + }, + }, + "eu-north-1": endpoint{ + Hostname: "waf-regional.eu-north-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-north-1", + }, + }, + "eu-west-1": endpoint{ + Hostname: "waf-regional.eu-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-west-1", + }, + }, + "eu-west-2": endpoint{ + Hostname: "waf-regional.eu-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-west-2", + }, + }, + "eu-west-3": endpoint{ + Hostname: "waf-regional.eu-west-3.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-west-3", + }, + }, + "fips-ap-east-1": endpoint{ + Hostname: "waf-regional-fips.ap-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-east-1", + }, + }, + "fips-ap-northeast-1": endpoint{ + Hostname: "waf-regional-fips.ap-northeast-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-northeast-1", + }, + }, + "fips-ap-northeast-2": endpoint{ + Hostname: "waf-regional-fips.ap-northeast-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-northeast-2", + }, + }, + "fips-ap-south-1": endpoint{ + Hostname: "waf-regional-fips.ap-south-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-south-1", + }, + }, + "fips-ap-southeast-1": endpoint{ + Hostname: "waf-regional-fips.ap-southeast-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-southeast-1", + }, + }, + "fips-ap-southeast-2": endpoint{ + Hostname: "waf-regional-fips.ap-southeast-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ap-southeast-2", + }, + }, + "fips-ca-central-1": endpoint{ + Hostname: "waf-regional-fips.ca-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, + }, + "fips-eu-central-1": endpoint{ + Hostname: "waf-regional-fips.eu-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-central-1", + }, + }, + "fips-eu-north-1": endpoint{ + Hostname: "waf-regional-fips.eu-north-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-north-1", + }, + }, + "fips-eu-west-1": endpoint{ + Hostname: "waf-regional-fips.eu-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-west-1", + }, + }, + "fips-eu-west-2": endpoint{ + Hostname: "waf-regional-fips.eu-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-west-2", + }, + }, + "fips-eu-west-3": endpoint{ + Hostname: "waf-regional-fips.eu-west-3.amazonaws.com", + CredentialScope: credentialScope{ + Region: "eu-west-3", + }, + }, + "fips-me-south-1": endpoint{ + Hostname: "waf-regional-fips.me-south-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "me-south-1", + }, + }, + "fips-sa-east-1": endpoint{ + Hostname: "waf-regional-fips.sa-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "sa-east-1", + }, + }, + "fips-us-east-1": endpoint{ + Hostname: "waf-regional-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "waf-regional-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "waf-regional-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "waf-regional-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{ + Hostname: "waf-regional.me-south-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "me-south-1", + }, + }, + "sa-east-1": endpoint{ + Hostname: "waf-regional.sa-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "sa-east-1", + }, + }, + "us-east-1": endpoint{ + Hostname: "waf-regional.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "us-east-2": endpoint{ + Hostname: "waf-regional.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "us-west-1": endpoint{ + Hostname: "waf-regional.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "us-west-2": endpoint{ + Hostname: "waf-regional.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + }, + }, + "workdocs": service{ + + Endpoints: endpoints{ + "ap-northeast-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "eu-west-1": endpoint{}, + "fips-us-east-1": endpoint{ + Hostname: "workdocs-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "workdocs-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "us-east-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "workmail": service{ + Defaults: endpoint{ + Protocols: []string{"https"}, + }, + Endpoints: endpoints{ + "eu-west-1": endpoint{}, + "us-east-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "workspaces": service{ + + Endpoints: endpoints{ + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + "xray": service{ + + Endpoints: endpoints{ + "af-south-1": endpoint{}, + "ap-east-1": endpoint{}, + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-south-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, + }, +} + +// AwsCnPartition returns the Resolver for AWS China. +func AwsCnPartition() Partition { + return awscnPartition.Partition() +} + +var awscnPartition = partition{ + ID: "aws-cn", + Name: "AWS China", + DNSSuffix: "amazonaws.com.cn", + RegionRegex: regionRegex{ + Regexp: func() *regexp.Regexp { + reg, _ := regexp.Compile("^cn\\-\\w+\\-\\d+$") + return reg + }(), + }, + Defaults: endpoint{ + Hostname: "{service}.{region}.{dnsSuffix}", + Protocols: []string{"https"}, + SignatureVersions: []string{"v4"}, + }, + Regions: regions{ + "cn-north-1": region{ + Description: "China (Beijing)", + }, + "cn-northwest-1": region{ + Description: "China (Ningxia)", + }, + }, + Services: services{ + "acm": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "api.ecr": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{ + Hostname: "api.ecr.cn-north-1.amazonaws.com.cn", + CredentialScope: credentialScope{ + Region: "cn-north-1", + }, + }, + "cn-northwest-1": endpoint{ + Hostname: "api.ecr.cn-northwest-1.amazonaws.com.cn", + CredentialScope: credentialScope{ + Region: "cn-northwest-1", + }, + }, + }, + }, + "api.sagemaker": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "apigateway": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "application-autoscaling": service{ + Defaults: endpoint{ + Protocols: []string{"http", "https"}, + }, + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "appsync": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + }, + }, + "athena": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "autoscaling": service{ + Defaults: endpoint{ + Protocols: []string{"http", "https"}, + }, + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "autoscaling-plans": service{ + Defaults: endpoint{ + Protocols: []string{"http", "https"}, + }, + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "backup": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "batch": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "budgets": service{ + PartitionEndpoint: "aws-cn-global", + IsRegionalized: boxedFalse, + + Endpoints: endpoints{ + "aws-cn-global": endpoint{ + Hostname: "budgets.amazonaws.com.cn", + CredentialScope: credentialScope{ + Region: "cn-northwest-1", + }, + }, + }, + }, + "ce": service{ + PartitionEndpoint: "aws-cn-global", + IsRegionalized: boxedFalse, + + Endpoints: endpoints{ + "aws-cn-global": endpoint{ + Hostname: "ce.cn-northwest-1.amazonaws.com.cn", + CredentialScope: credentialScope{ + Region: "cn-northwest-1", + }, + }, + }, + }, + "cloudformation": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "cloudfront": service{ + PartitionEndpoint: "aws-cn-global", + IsRegionalized: boxedFalse, + + Endpoints: endpoints{ + "aws-cn-global": endpoint{ + Hostname: "cloudfront.cn-northwest-1.amazonaws.com.cn", + Protocols: []string{"http", "https"}, + CredentialScope: credentialScope{ + Region: "cn-northwest-1", + }, + }, + }, + }, + "cloudtrail": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "codebuild": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "codecommit": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "codedeploy": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "cognito-identity": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + }, + }, + "config": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "cur": service{ + + Endpoints: endpoints{ + "cn-northwest-1": endpoint{}, + }, + }, + "dax": service{ + + Endpoints: endpoints{ + "cn-northwest-1": endpoint{}, + }, + }, + "directconnect": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "dms": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "ds": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "dynamodb": service{ + Defaults: endpoint{ + Protocols: []string{"http", "https"}, + }, + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "ec2": service{ + Defaults: endpoint{ + Protocols: []string{"http", "https"}, + }, + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "ec2metadata": service{ + PartitionEndpoint: "aws-global", + IsRegionalized: boxedFalse, + + Endpoints: endpoints{ + "aws-global": endpoint{ + Hostname: "169.254.169.254/latest", + Protocols: []string{"http"}, + }, + }, + }, + "ecs": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "eks": service{ + Defaults: endpoint{ + Protocols: []string{"http", "https"}, + }, + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "elasticache": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "elasticbeanstalk": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "elasticfilesystem": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + "fips-cn-north-1": endpoint{ + Hostname: "elasticfilesystem-fips.cn-north-1.amazonaws.com.cn", + CredentialScope: credentialScope{ + Region: "cn-north-1", + }, + }, + "fips-cn-northwest-1": endpoint{ + Hostname: "elasticfilesystem-fips.cn-northwest-1.amazonaws.com.cn", + CredentialScope: credentialScope{ + Region: "cn-northwest-1", + }, + }, + }, + }, + "elasticloadbalancing": service{ + Defaults: endpoint{ + Protocols: []string{"https"}, + }, + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "elasticmapreduce": service{ + Defaults: endpoint{ + Protocols: []string{"https"}, + }, + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "es": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "events": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "firehose": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "gamelift": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + }, + }, + "glacier": service{ + Defaults: endpoint{ + Protocols: []string{"http", "https"}, + }, + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "glue": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "greengrass": service{ + IsRegionalized: boxedTrue, + Defaults: endpoint{ + Protocols: []string{"https"}, + }, + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + }, + }, + "health": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "iam": service{ + PartitionEndpoint: "aws-cn-global", + IsRegionalized: boxedFalse, + + Endpoints: endpoints{ + "aws-cn-global": endpoint{ + Hostname: "iam.cn-north-1.amazonaws.com.cn", + CredentialScope: credentialScope{ + Region: "cn-north-1", + }, + }, + }, + }, + "iot": service{ + Defaults: endpoint{ + CredentialScope: credentialScope{ + Service: "execute-api", + }, + }, + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "iotsecuredtunneling": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "kafka": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "kinesis": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "kinesisanalytics": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "kms": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "lambda": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "license-manager": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "logs": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "mediaconvert": service{ + + Endpoints: endpoints{ + "cn-northwest-1": endpoint{ + Hostname: "subscribe.mediaconvert.cn-northwest-1.amazonaws.com.cn", + CredentialScope: credentialScope{ + Region: "cn-northwest-1", + }, + }, + }, + }, + "monitoring": service{ + Defaults: endpoint{ + Protocols: []string{"http", "https"}, + }, + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "neptune": service{ + + Endpoints: endpoints{ + "cn-northwest-1": endpoint{ + Hostname: "rds.cn-northwest-1.amazonaws.com.cn", + CredentialScope: credentialScope{ + Region: "cn-northwest-1", + }, + }, + }, + }, + "organizations": service{ + PartitionEndpoint: "aws-cn-global", + IsRegionalized: boxedFalse, + + Endpoints: endpoints{ + "aws-cn-global": endpoint{ + Hostname: "organizations.cn-northwest-1.amazonaws.com.cn", + CredentialScope: credentialScope{ + Region: "cn-northwest-1", + }, + }, + "fips-aws-cn-global": endpoint{ + Hostname: "organizations.cn-northwest-1.amazonaws.com.cn", + CredentialScope: credentialScope{ + Region: "cn-northwest-1", + }, + }, + }, + }, + "polly": service{ + + Endpoints: endpoints{ + "cn-northwest-1": endpoint{}, + }, + }, + "rds": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "redshift": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "route53": service{ + PartitionEndpoint: "aws-cn-global", + IsRegionalized: boxedFalse, + + Endpoints: endpoints{ + "aws-cn-global": endpoint{ + Hostname: "route53.amazonaws.com.cn", + CredentialScope: credentialScope{ + Region: "cn-northwest-1", + }, + }, + }, + }, + "runtime.sagemaker": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "s3": service{ + Defaults: endpoint{ + Protocols: []string{"http", "https"}, + SignatureVersions: []string{"s3v4"}, + + HasDualStack: boxedTrue, + DualStackHostname: "{service}.dualstack.{region}.{dnsSuffix}", + }, + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "s3-control": service{ + Defaults: endpoint{ + Protocols: []string{"https"}, + SignatureVersions: []string{"s3v4"}, + + HasDualStack: boxedTrue, + DualStackHostname: "{service}.dualstack.{region}.{dnsSuffix}", + }, + Endpoints: endpoints{ + "cn-north-1": endpoint{ + Hostname: "s3-control.cn-north-1.amazonaws.com.cn", + SignatureVersions: []string{"s3v4"}, + CredentialScope: credentialScope{ + Region: "cn-north-1", + }, + }, + "cn-northwest-1": endpoint{ + Hostname: "s3-control.cn-northwest-1.amazonaws.com.cn", + SignatureVersions: []string{"s3v4"}, + CredentialScope: credentialScope{ + Region: "cn-northwest-1", + }, + }, + }, + }, + "secretsmanager": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "serverlessrepo": service{ + Defaults: endpoint{ + Protocols: []string{"https"}, + }, + Endpoints: endpoints{ + "cn-north-1": endpoint{ + Protocols: []string{"https"}, + }, + "cn-northwest-1": endpoint{ + Protocols: []string{"https"}, + }, + }, + }, + "sms": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "snowball": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + "fips-cn-north-1": endpoint{ + Hostname: "snowball-fips.cn-north-1.amazonaws.com.cn", + CredentialScope: credentialScope{ + Region: "cn-north-1", + }, + }, + "fips-cn-northwest-1": endpoint{ + Hostname: "snowball-fips.cn-northwest-1.amazonaws.com.cn", + CredentialScope: credentialScope{ + Region: "cn-northwest-1", + }, + }, + }, + }, + "sns": service{ + Defaults: endpoint{ + Protocols: []string{"http", "https"}, + }, + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "sqs": service{ + Defaults: endpoint{ + SSLCommonName: "{region}.queue.{dnsSuffix}", + Protocols: []string{"http", "https"}, + }, + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "ssm": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "states": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "storagegateway": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "streams.dynamodb": service{ + Defaults: endpoint{ + Protocols: []string{"http", "https"}, + CredentialScope: credentialScope{ + Service: "dynamodb", + }, + }, + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "sts": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "support": service{ + PartitionEndpoint: "aws-cn-global", + + Endpoints: endpoints{ + "aws-cn-global": endpoint{ + Hostname: "support.cn-north-1.amazonaws.com.cn", + CredentialScope: credentialScope{ + Region: "cn-north-1", + }, + }, + }, + }, + "swf": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "tagging": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + "transcribe": service{ + Defaults: endpoint{ + Protocols: []string{"https"}, + }, + Endpoints: endpoints{ + "cn-north-1": endpoint{ + Hostname: "cn.transcribe.cn-north-1.amazonaws.com.cn", + CredentialScope: credentialScope{ + Region: "cn-north-1", + }, + }, + "cn-northwest-1": endpoint{ + Hostname: "cn.transcribe.cn-northwest-1.amazonaws.com.cn", + CredentialScope: credentialScope{ + Region: "cn-northwest-1", + }, + }, + }, + }, + "workspaces": service{ + + Endpoints: endpoints{ + "cn-northwest-1": endpoint{}, + }, + }, + "xray": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, + }, +} + +// AwsUsGovPartition returns the Resolver for AWS GovCloud (US). +func AwsUsGovPartition() Partition { + return awsusgovPartition.Partition() +} + +var awsusgovPartition = partition{ + ID: "aws-us-gov", + Name: "AWS GovCloud (US)", + DNSSuffix: "amazonaws.com", + RegionRegex: regionRegex{ + Regexp: func() *regexp.Regexp { + reg, _ := regexp.Compile("^us\\-gov\\-\\w+\\-\\d+$") + return reg + }(), + }, + Defaults: endpoint{ + Hostname: "{service}.{region}.{dnsSuffix}", + Protocols: []string{"https"}, + SignatureVersions: []string{"v4"}, + }, + Regions: regions{ + "us-gov-east-1": region{ + Description: "AWS GovCloud (US-East)", + }, + "us-gov-west-1": region{ + Description: "AWS GovCloud (US-West)", + }, + }, + Services: services{ + "access-analyzer": service{ + + Endpoints: endpoints{ + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "acm": service{ + + Endpoints: endpoints{ + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "acm-pca": service{ + Defaults: endpoint{ + Protocols: []string{"https"}, + }, + Endpoints: endpoints{ + "fips-us-gov-east-1": endpoint{ + Hostname: "acm-pca.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "fips-us-gov-west-1": endpoint{ + Hostname: "acm-pca.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "api.ecr": service{ + + Endpoints: endpoints{ + "fips-us-gov-east-1": endpoint{ + Hostname: "ecr-fips.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "fips-us-gov-west-1": endpoint{ + Hostname: "ecr-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{ + Hostname: "api.ecr.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "us-gov-west-1": endpoint{ + Hostname: "api.ecr.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + }, + }, + "api.sagemaker": service{ + + Endpoints: endpoints{ + "us-gov-west-1": endpoint{}, + "us-gov-west-1-fips": endpoint{ + Hostname: "api-fips.sagemaker.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-west-1-fips-secondary": endpoint{ + Hostname: "api.sagemaker.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + }, + }, + "apigateway": service{ + + Endpoints: endpoints{ + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "application-autoscaling": service{ + Defaults: endpoint{ + Hostname: "autoscaling.{region}.amazonaws.com", + Protocols: []string{"http", "https"}, + CredentialScope: credentialScope{ + Service: "application-autoscaling", + }, + }, + Endpoints: endpoints{ + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "appstream2": service{ + Defaults: endpoint{ + Protocols: []string{"https"}, + CredentialScope: credentialScope{ + Service: "appstream", + }, + }, + Endpoints: endpoints{ + "fips": endpoint{ + Hostname: "appstream2-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-west-1": endpoint{}, + }, + }, + "athena": service{ + + Endpoints: endpoints{ + "fips-us-gov-east-1": endpoint{ + Hostname: "athena-fips.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "fips-us-gov-west-1": endpoint{ + Hostname: "athena-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "autoscaling": service{ + + Endpoints: endpoints{ + "us-gov-east-1": endpoint{ + Protocols: []string{"http", "https"}, + }, + "us-gov-west-1": endpoint{ + Protocols: []string{"http", "https"}, + }, + }, + }, + "autoscaling-plans": service{ + Defaults: endpoint{ + Protocols: []string{"http", "https"}, + }, + Endpoints: endpoints{ + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "backup": service{ + + Endpoints: endpoints{ + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "batch": service{ + + Endpoints: endpoints{ + "fips-us-gov-east-1": endpoint{ + Hostname: "batch.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "fips-us-gov-west-1": endpoint{ + Hostname: "batch.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "clouddirectory": service{ + + Endpoints: endpoints{ + "us-gov-west-1": endpoint{}, + }, + }, + "cloudformation": service{ + + Endpoints: endpoints{ + "us-gov-east-1": endpoint{ + Hostname: "cloudformation.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "us-gov-west-1": endpoint{ + Hostname: "cloudformation.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + }, + }, + "cloudhsm": service{ + + Endpoints: endpoints{ + "us-gov-west-1": endpoint{}, + }, + }, + "cloudhsmv2": service{ + Defaults: endpoint{ + CredentialScope: credentialScope{ + Service: "cloudhsm", + }, + }, + Endpoints: endpoints{ + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "cloudtrail": service{ + + Endpoints: endpoints{ + "us-gov-east-1": endpoint{ + Hostname: "cloudtrail.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "us-gov-west-1": endpoint{ + Hostname: "cloudtrail.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + }, + }, + "codebuild": service{ + + Endpoints: endpoints{ + "us-gov-east-1": endpoint{}, + "us-gov-east-1-fips": endpoint{ + Hostname: "codebuild-fips.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "us-gov-west-1": endpoint{}, + "us-gov-west-1-fips": endpoint{ + Hostname: "codebuild-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + }, + }, + "codecommit": service{ + + Endpoints: endpoints{ + "fips": endpoint{ + Hostname: "codecommit-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "codedeploy": service{ + + Endpoints: endpoints{ + "us-gov-east-1": endpoint{}, + "us-gov-east-1-fips": endpoint{ + Hostname: "codedeploy-fips.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "us-gov-west-1": endpoint{}, + "us-gov-west-1-fips": endpoint{ + Hostname: "codedeploy-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + }, + }, + "codepipeline": service{ + + Endpoints: endpoints{ + "fips-us-gov-west-1": endpoint{ + Hostname: "codepipeline-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-west-1": endpoint{}, + }, + }, + "cognito-identity": service{ + + Endpoints: endpoints{ + "us-gov-west-1": endpoint{}, + }, + }, + "cognito-idp": service{ + + Endpoints: endpoints{ + "fips-us-gov-west-1": endpoint{ + Hostname: "cognito-idp-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-west-1": endpoint{}, + }, + }, + "comprehend": service{ + Defaults: endpoint{ + Protocols: []string{"https"}, + }, + Endpoints: endpoints{ + "fips-us-gov-west-1": endpoint{ + Hostname: "comprehend-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-west-1": endpoint{}, + }, + }, + "comprehendmedical": service{ + + Endpoints: endpoints{ + "fips-us-gov-west-1": endpoint{ + Hostname: "comprehendmedical-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-west-1": endpoint{}, + }, + }, + "config": service{ + + Endpoints: endpoints{ + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "datasync": service{ + + Endpoints: endpoints{ + "fips-us-gov-east-1": endpoint{ + Hostname: "datasync-fips.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "fips-us-gov-west-1": endpoint{ + Hostname: "datasync-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "directconnect": service{ + + Endpoints: endpoints{ + "us-gov-east-1": endpoint{ + Hostname: "directconnect.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "us-gov-west-1": endpoint{ + Hostname: "directconnect.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + }, + }, + "dms": service{ + + Endpoints: endpoints{ + "dms-fips": endpoint{ + Hostname: "dms.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "docdb": service{ + + Endpoints: endpoints{ + "us-gov-west-1": endpoint{ + Hostname: "rds.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + }, + }, + "ds": service{ + + Endpoints: endpoints{ + "fips-us-gov-east-1": endpoint{ + Hostname: "ds-fips.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "fips-us-gov-west-1": endpoint{ + Hostname: "ds-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "dynamodb": service{ + + Endpoints: endpoints{ + "us-gov-east-1": endpoint{}, + "us-gov-east-1-fips": endpoint{ + Hostname: "dynamodb.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "us-gov-west-1": endpoint{}, + "us-gov-west-1-fips": endpoint{ + Hostname: "dynamodb.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + }, + }, + "ec2": service{ + + Endpoints: endpoints{ + "us-gov-east-1": endpoint{ + Hostname: "ec2.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "us-gov-west-1": endpoint{ + Hostname: "ec2.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + }, + }, + "ec2metadata": service{ + PartitionEndpoint: "aws-global", + IsRegionalized: boxedFalse, + + Endpoints: endpoints{ + "aws-global": endpoint{ + Hostname: "169.254.169.254/latest", + Protocols: []string{"http"}, + }, + }, + }, + "ecs": service{ + + Endpoints: endpoints{ + "fips-us-gov-east-1": endpoint{ + Hostname: "ecs-fips.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "fips-us-gov-west-1": endpoint{ + Hostname: "ecs-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "eks": service{ + Defaults: endpoint{ + Protocols: []string{"http", "https"}, + }, + Endpoints: endpoints{ + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "elasticache": service{ + + Endpoints: endpoints{ + "fips": endpoint{ + Hostname: "elasticache.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "elasticbeanstalk": service{ + + Endpoints: endpoints{ + "us-gov-east-1": endpoint{ + Hostname: "elasticbeanstalk.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "us-gov-west-1": endpoint{ + Hostname: "elasticbeanstalk.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + }, + }, + "elasticfilesystem": service{ + + Endpoints: endpoints{ + "fips-us-gov-east-1": endpoint{ + Hostname: "elasticfilesystem-fips.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "fips-us-gov-west-1": endpoint{ + Hostname: "elasticfilesystem-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "elasticloadbalancing": service{ + + Endpoints: endpoints{ + "fips-us-gov-east-1": endpoint{ + Hostname: "elasticloadbalancing-fips.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "fips-us-gov-west-1": endpoint{ + Hostname: "elasticloadbalancing-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{ + Protocols: []string{"http", "https"}, + }, + }, + }, + "elasticmapreduce": service{ + + Endpoints: endpoints{ + "fips-us-gov-east-1": endpoint{ + Hostname: "elasticmapreduce.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "fips-us-gov-west-1": endpoint{ + Hostname: "elasticmapreduce.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{ + Protocols: []string{"https"}, + }, + }, + }, + "email": service{ + + Endpoints: endpoints{ + "fips-us-gov-west-1": endpoint{ + Hostname: "email-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-west-1": endpoint{}, + }, + }, + "es": service{ + + Endpoints: endpoints{ + "fips": endpoint{ + Hostname: "es-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "events": service{ + + Endpoints: endpoints{ + "us-gov-east-1": endpoint{ + Hostname: "events.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "us-gov-west-1": endpoint{ + Hostname: "events.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + }, + }, + "firehose": service{ + + Endpoints: endpoints{ + "fips-us-gov-east-1": endpoint{ + Hostname: "firehose-fips.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "fips-us-gov-west-1": endpoint{ + Hostname: "firehose-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "glacier": service{ + + Endpoints: endpoints{ + "us-gov-east-1": endpoint{ + Hostname: "glacier.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "us-gov-west-1": endpoint{ + Hostname: "glacier.us-gov-west-1.amazonaws.com", + Protocols: []string{"http", "https"}, + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + }, + }, + "glue": service{ + + Endpoints: endpoints{ + "fips-us-gov-east-1": endpoint{ + Hostname: "glue-fips.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "fips-us-gov-west-1": endpoint{ + Hostname: "glue-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "greengrass": service{ + IsRegionalized: boxedTrue, + Defaults: endpoint{ + Protocols: []string{"https"}, + }, + Endpoints: endpoints{ + "us-gov-west-1": endpoint{ + Hostname: "greengrass.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + }, + }, + "guardduty": service{ + IsRegionalized: boxedTrue, + Defaults: endpoint{ + Protocols: []string{"https"}, + }, + Endpoints: endpoints{ + "us-gov-west-1": endpoint{}, + "us-gov-west-1-fips": endpoint{ + Hostname: "guardduty.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + }, + }, + "health": service{ + + Endpoints: endpoints{ + "us-gov-west-1": endpoint{}, + }, + }, + "iam": service{ + PartitionEndpoint: "aws-us-gov-global", + IsRegionalized: boxedFalse, + + Endpoints: endpoints{ + "aws-us-gov-global": endpoint{ + Hostname: "iam.us-gov.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "iam-govcloud-fips": endpoint{ + Hostname: "iam.us-gov.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + }, + }, + "inspector": service{ + + Endpoints: endpoints{ + "fips-us-gov-east-1": endpoint{ + Hostname: "inspector-fips.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "fips-us-gov-west-1": endpoint{ + Hostname: "inspector-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "iot": service{ + Defaults: endpoint{ + CredentialScope: credentialScope{ + Service: "execute-api", + }, + }, + Endpoints: endpoints{ + "us-gov-west-1": endpoint{}, + }, + }, + "iotsecuredtunneling": service{ + + Endpoints: endpoints{ + "us-gov-west-1": endpoint{}, + }, + }, + "kafka": service{ + + Endpoints: endpoints{ + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "kinesis": service{ + + Endpoints: endpoints{ + "fips-us-gov-east-1": endpoint{ + Hostname: "kinesis-fips.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "fips-us-gov-west-1": endpoint{ + Hostname: "kinesis-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "kinesisanalytics": service{ + + Endpoints: endpoints{ + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "kms": service{ + + Endpoints: endpoints{ + "ProdFips": endpoint{ + Hostname: "kms-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "lambda": service{ + + Endpoints: endpoints{ + "fips-us-gov-east-1": endpoint{ + Hostname: "lambda-fips.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "fips-us-gov-west-1": endpoint{ + Hostname: "lambda-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "license-manager": service{ + + Endpoints: endpoints{ + "fips-us-gov-east-1": endpoint{ + Hostname: "license-manager-fips.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "fips-us-gov-west-1": endpoint{ + Hostname: "license-manager-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "logs": service{ + + Endpoints: endpoints{ + "us-gov-east-1": endpoint{ + Hostname: "logs.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "us-gov-west-1": endpoint{ + Hostname: "logs.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + }, + }, + "mediaconvert": service{ + + Endpoints: endpoints{ + "us-gov-west-1": endpoint{ + Hostname: "mediaconvert.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + }, + }, + "metering.marketplace": service{ + Defaults: endpoint{ CredentialScope: credentialScope{ - Service: "application-autoscaling", + Service: "aws-marketplace", + }, + }, + Endpoints: endpoints{ + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "monitoring": service{ + + Endpoints: endpoints{ + "fips-us-gov-east-1": endpoint{ + Hostname: "monitoring.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "fips-us-gov-west-1": endpoint{ + Hostname: "monitoring.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "neptune": service{ + + Endpoints: endpoints{ + "us-gov-east-1": endpoint{ + Hostname: "rds.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "us-gov-west-1": endpoint{ + Hostname: "rds.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + }, + }, + "organizations": service{ + PartitionEndpoint: "aws-us-gov-global", + IsRegionalized: boxedFalse, + + Endpoints: endpoints{ + "aws-us-gov-global": endpoint{ + Hostname: "organizations.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "fips-aws-us-gov-global": endpoint{ + Hostname: "organizations.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + }, + }, + "outposts": service{ + + Endpoints: endpoints{ + "us-gov-east-1": endpoint{ + Hostname: "outposts.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "us-gov-west-1": endpoint{ + Hostname: "outposts.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + }, + }, + "pinpoint": service{ + Defaults: endpoint{ + CredentialScope: credentialScope{ + Service: "mobiletargeting", + }, + }, + Endpoints: endpoints{ + "us-gov-west-1": endpoint{}, + }, + }, + "polly": service{ + + Endpoints: endpoints{ + "fips-us-gov-west-1": endpoint{ + Hostname: "polly-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-west-1": endpoint{}, + }, + }, + "ram": service{ + + Endpoints: endpoints{ + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "rds": service{ + + Endpoints: endpoints{ + "rds.us-gov-east-1": endpoint{ + Hostname: "rds.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "rds.us-gov-west-1": endpoint{ + Hostname: "rds.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "redshift": service{ + + Endpoints: endpoints{ + "us-gov-east-1": endpoint{ + Hostname: "redshift.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "us-gov-west-1": endpoint{ + Hostname: "redshift.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + }, + }, + "rekognition": service{ + + Endpoints: endpoints{ + "rekognition-fips.us-gov-west-1": endpoint{ + Hostname: "rekognition-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-west-1": endpoint{}, + }, + }, + "resource-groups": service{ + + Endpoints: endpoints{ + "fips-us-gov-east-1": endpoint{ + Hostname: "resource-groups.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "fips-us-gov-west-1": endpoint{ + Hostname: "resource-groups.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, + }, + }, + "route53": service{ + PartitionEndpoint: "aws-us-gov-global", + IsRegionalized: boxedFalse, + + Endpoints: endpoints{ + "aws-us-gov-global": endpoint{ + Hostname: "route53.us-gov.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, }, }, + }, + "route53resolver": service{ + Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, }, }, - "autoscaling": service{ + "runtime.sagemaker": service{ + + Endpoints: endpoints{ + "us-gov-west-1": endpoint{}, + }, + }, + "s3": service{ Defaults: endpoint{ - Protocols: []string{"http", "https"}, + SignatureVersions: []string{"s3", "s3v4"}, + + HasDualStack: boxedTrue, + DualStackHostname: "{service}.dualstack.{region}.{dnsSuffix}", }, Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "fips-us-gov-west-1": endpoint{ + Hostname: "s3-fips-us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{ + Hostname: "s3.us-gov-east-1.amazonaws.com", + Protocols: []string{"http", "https"}, + }, + "us-gov-west-1": endpoint{ + Hostname: "s3.us-gov-west-1.amazonaws.com", + Protocols: []string{"http", "https"}, + }, }, }, - "cloudformation": service{ + "s3-control": service{ + Defaults: endpoint{ + Protocols: []string{"https"}, + SignatureVersions: []string{"s3v4"}, + HasDualStack: boxedTrue, + DualStackHostname: "{service}.dualstack.{region}.{dnsSuffix}", + }, Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "us-gov-east-1": endpoint{ + Hostname: "s3-control.us-gov-east-1.amazonaws.com", + SignatureVersions: []string{"s3v4"}, + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "us-gov-east-1-fips": endpoint{ + Hostname: "s3-control-fips.us-gov-east-1.amazonaws.com", + SignatureVersions: []string{"s3v4"}, + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "us-gov-west-1": endpoint{ + Hostname: "s3-control.us-gov-west-1.amazonaws.com", + SignatureVersions: []string{"s3v4"}, + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-west-1-fips": endpoint{ + Hostname: "s3-control-fips.us-gov-west-1.amazonaws.com", + SignatureVersions: []string{"s3v4"}, + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, }, }, - "cloudtrail": service{ + "secretsmanager": service{ Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "us-gov-east-1": endpoint{}, + "us-gov-east-1-fips": endpoint{ + Hostname: "secretsmanager-fips.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "us-gov-west-1": endpoint{}, + "us-gov-west-1-fips": endpoint{ + Hostname: "secretsmanager-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, }, }, - "codebuild": service{ + "securityhub": service{ Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "fips-us-gov-east-1": endpoint{ + Hostname: "securityhub-fips.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "fips-us-gov-west-1": endpoint{ + Hostname: "securityhub-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, }, }, - "codedeploy": service{ + "serverlessrepo": service{ + Defaults: endpoint{ + Protocols: []string{"https"}, + }, + Endpoints: endpoints{ + "us-gov-east-1": endpoint{ + Hostname: "serverlessrepo.us-gov-east-1.amazonaws.com", + Protocols: []string{"https"}, + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "us-gov-west-1": endpoint{ + Hostname: "serverlessrepo.us-gov-west-1.amazonaws.com", + Protocols: []string{"https"}, + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + }, + }, + "servicecatalog": service{ Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "us-gov-east-1": endpoint{}, + "us-gov-east-1-fips": endpoint{ + Hostname: "servicecatalog-fips.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "us-gov-west-1": endpoint{}, + "us-gov-west-1-fips": endpoint{ + Hostname: "servicecatalog-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, }, }, - "cognito-identity": service{ + "sms": service{ Endpoints: endpoints{ - "cn-north-1": endpoint{}, + "fips-us-gov-east-1": endpoint{ + Hostname: "sms-fips.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "fips-us-gov-west-1": endpoint{ + Hostname: "sms-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, }, }, - "config": service{ + "snowball": service{ Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "fips-us-gov-east-1": endpoint{ + Hostname: "snowball-fips.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "fips-us-gov-west-1": endpoint{ + Hostname: "snowball-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, }, }, - "directconnect": service{ + "sns": service{ Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "us-gov-east-1": endpoint{ + Hostname: "sns.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "us-gov-west-1": endpoint{ + Hostname: "sns.us-gov-west-1.amazonaws.com", + Protocols: []string{"http", "https"}, + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, }, }, - "dms": service{ + "sqs": service{ Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "us-gov-east-1": endpoint{ + Hostname: "sqs.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "us-gov-west-1": endpoint{ + Hostname: "sqs.us-gov-west-1.amazonaws.com", + SSLCommonName: "{region}.queue.{dnsSuffix}", + Protocols: []string{"http", "https"}, + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, }, }, - "ds": service{ + "ssm": service{ Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "fips-us-gov-east-1": endpoint{ + Hostname: "ssm.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "fips-us-gov-west-1": endpoint{ + Hostname: "ssm.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "ssm-facade-fips-us-gov-east-1": endpoint{ + Hostname: "ssm-facade.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "ssm-facade-fips-us-gov-west-1": endpoint{ + Hostname: "ssm-facade.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, }, }, - "dynamodb": service{ - Defaults: endpoint{ - Protocols: []string{"http", "https"}, + "states": service{ + + Endpoints: endpoints{ + "fips-us-gov-east-1": endpoint{ + Hostname: "states-fips.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "fips-us-gov-west-1": endpoint{ + Hostname: "states.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, }, + }, + "storagegateway": service{ + Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "fips": endpoint{ + Hostname: "storagegateway-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, }, }, - "ec2": service{ + "streams.dynamodb": service{ Defaults: endpoint{ - Protocols: []string{"http", "https"}, - }, - Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + CredentialScope: credentialScope{ + Service: "dynamodb", + }, }, - }, - "ec2metadata": service{ - PartitionEndpoint: "aws-global", - IsRegionalized: boxedFalse, - Endpoints: endpoints{ - "aws-global": endpoint{ - Hostname: "169.254.169.254/latest", - Protocols: []string{"http"}, + "us-gov-east-1": endpoint{}, + "us-gov-east-1-fips": endpoint{ + Hostname: "dynamodb.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "us-gov-west-1": endpoint{}, + "us-gov-west-1-fips": endpoint{ + Hostname: "dynamodb.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, }, }, }, - "ecr": service{ + "sts": service{ Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "us-gov-east-1": endpoint{}, + "us-gov-east-1-fips": endpoint{ + Hostname: "sts.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "us-gov-west-1": endpoint{}, + "us-gov-west-1-fips": endpoint{ + Hostname: "sts.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, }, }, - "ecs": service{ + "support": service{ + PartitionEndpoint: "aws-us-gov-global", Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "aws-us-gov-global": endpoint{ + Hostname: "support.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "fips-us-gov-west-1": endpoint{ + Hostname: "support.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, }, }, - "elasticache": service{ + "swf": service{ Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "us-gov-east-1": endpoint{ + Hostname: "swf.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "us-gov-west-1": endpoint{ + Hostname: "swf.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, }, }, - "elasticbeanstalk": service{ + "tagging": service{ Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, }, }, - "elasticloadbalancing": service{ + "transcribe": service{ Defaults: endpoint{ Protocols: []string{"https"}, }, Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "fips-us-gov-east-1": endpoint{ + Hostname: "fips.transcribe.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, + "fips-us-gov-west-1": endpoint{ + Hostname: "fips.transcribe.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, }, }, - "elasticmapreduce": service{ + "translate": service{ Defaults: endpoint{ Protocols: []string{"https"}, }, Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "us-gov-west-1": endpoint{}, + "us-gov-west-1-fips": endpoint{ + Hostname: "translate-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, }, }, - "es": service{ + "waf-regional": service{ Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "fips-us-gov-west-1": endpoint{ + Hostname: "waf-regional-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, + "us-gov-west-1": endpoint{ + Hostname: "waf-regional.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + }, }, }, - "events": service{ + "workspaces": service{ Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "us-gov-west-1": endpoint{}, }, }, - "glacier": service{ - Defaults: endpoint{ - Protocols: []string{"http", "https"}, - }, + "xray": service{ + Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "us-gov-east-1": endpoint{}, + "us-gov-west-1": endpoint{}, }, }, - "iam": service{ - PartitionEndpoint: "aws-cn-global", - IsRegionalized: boxedFalse, + }, +} + +// AwsIsoPartition returns the Resolver for AWS ISO (US). +func AwsIsoPartition() Partition { + return awsisoPartition.Partition() +} + +var awsisoPartition = partition{ + ID: "aws-iso", + Name: "AWS ISO (US)", + DNSSuffix: "c2s.ic.gov", + RegionRegex: regionRegex{ + Regexp: func() *regexp.Regexp { + reg, _ := regexp.Compile("^us\\-iso\\-\\w+\\-\\d+$") + return reg + }(), + }, + Defaults: endpoint{ + Hostname: "{service}.{region}.{dnsSuffix}", + Protocols: []string{"https"}, + SignatureVersions: []string{"v4"}, + }, + Regions: regions{ + "us-iso-east-1": region{ + Description: "US ISO East", + }, + }, + Services: services{ + "api.ecr": service{ Endpoints: endpoints{ - "aws-cn-global": endpoint{ - Hostname: "iam.cn-north-1.amazonaws.com.cn", + "us-iso-east-1": endpoint{ + Hostname: "api.ecr.us-iso-east-1.c2s.ic.gov", CredentialScope: credentialScope{ - Region: "cn-north-1", + Region: "us-iso-east-1", }, }, }, }, - "iot": service{ - Defaults: endpoint{ - CredentialScope: credentialScope{ - Service: "execute-api", - }, - }, - Endpoints: endpoints{ - "cn-north-1": endpoint{}, - }, - }, - "kinesis": service{ - - Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, - }, - }, - "lambda": service{ + "api.sagemaker": service{ Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "us-iso-east-1": endpoint{}, }, }, - "logs": service{ + "apigateway": service{ Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "us-iso-east-1": endpoint{}, }, }, - "monitoring": service{ + "application-autoscaling": service{ Defaults: endpoint{ Protocols: []string{"http", "https"}, }, Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "us-iso-east-1": endpoint{}, }, }, - "polly": service{ + "autoscaling": service{ Endpoints: endpoints{ - "cn-northwest-1": endpoint{}, + "us-iso-east-1": endpoint{ + Protocols: []string{"http", "https"}, + }, }, }, - "rds": service{ + "cloudformation": service{ Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "us-iso-east-1": endpoint{}, }, }, - "redshift": service{ + "cloudtrail": service{ Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "us-iso-east-1": endpoint{}, }, }, - "s3": service{ - Defaults: endpoint{ - Protocols: []string{"http", "https"}, - SignatureVersions: []string{"s3v4"}, - }, + "codedeploy": service{ + Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "us-iso-east-1": endpoint{}, }, }, - "s3-control": service{ + "comprehend": service{ Defaults: endpoint{ - Protocols: []string{"https"}, - SignatureVersions: []string{"s3v4"}, + Protocols: []string{"https"}, }, Endpoints: endpoints{ - "cn-north-1": endpoint{ - Hostname: "s3-control.cn-north-1.amazonaws.com.cn", - SignatureVersions: []string{"s3v4"}, - CredentialScope: credentialScope{ - Region: "cn-north-1", - }, - }, - "cn-northwest-1": endpoint{ - Hostname: "s3-control.cn-northwest-1.amazonaws.com.cn", - SignatureVersions: []string{"s3v4"}, - CredentialScope: credentialScope{ - Region: "cn-northwest-1", - }, - }, + "us-iso-east-1": endpoint{}, }, }, - "sms": service{ + "config": service{ Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "us-iso-east-1": endpoint{}, }, }, - "snowball": service{ + "datapipeline": service{ Endpoints: endpoints{ - "cn-north-1": endpoint{}, + "us-iso-east-1": endpoint{}, }, }, - "sns": service{ - Defaults: endpoint{ - Protocols: []string{"http", "https"}, + "directconnect": service{ + + Endpoints: endpoints{ + "us-iso-east-1": endpoint{}, }, + }, + "dms": service{ + Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "dms-fips": endpoint{ + Hostname: "dms.us-iso-east-1.c2s.ic.gov", + CredentialScope: credentialScope{ + Region: "us-iso-east-1", + }, + }, + "us-iso-east-1": endpoint{}, }, }, - "sqs": service{ - Defaults: endpoint{ - SSLCommonName: "{region}.queue.{dnsSuffix}", - Protocols: []string{"http", "https"}, - }, + "ds": service{ + Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "us-iso-east-1": endpoint{}, }, }, - "ssm": service{ + "dynamodb": service{ Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "us-iso-east-1": endpoint{ + Protocols: []string{"http", "https"}, + }, }, }, - "storagegateway": service{ + "ec2": service{ Endpoints: endpoints{ - "cn-north-1": endpoint{}, + "us-iso-east-1": endpoint{}, }, }, - "streams.dynamodb": service{ - Defaults: endpoint{ - Protocols: []string{"http", "https"}, - CredentialScope: credentialScope{ - Service: "dynamodb", + "ec2metadata": service{ + PartitionEndpoint: "aws-global", + IsRegionalized: boxedFalse, + + Endpoints: endpoints{ + "aws-global": endpoint{ + Hostname: "169.254.169.254/latest", + Protocols: []string{"http"}, }, }, + }, + "ecs": service{ + Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "us-iso-east-1": endpoint{}, }, }, - "sts": service{ + "elasticache": service{ Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "us-iso-east-1": endpoint{}, }, }, - "swf": service{ + "elasticloadbalancing": service{ Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "us-iso-east-1": endpoint{ + Protocols: []string{"http", "https"}, + }, }, }, - "tagging": service{ + "elasticmapreduce": service{ Endpoints: endpoints{ - "cn-north-1": endpoint{}, - "cn-northwest-1": endpoint{}, + "us-iso-east-1": endpoint{ + Protocols: []string{"https"}, + }, }, }, - }, -} - -// AwsUsGovPartition returns the Resolver for AWS GovCloud (US). -func AwsUsGovPartition() Partition { - return awsusgovPartition.Partition() -} + "es": service{ -var awsusgovPartition = partition{ - ID: "aws-us-gov", - Name: "AWS GovCloud (US)", - DNSSuffix: "amazonaws.com", - RegionRegex: regionRegex{ - Regexp: func() *regexp.Regexp { - reg, _ := regexp.Compile("^us\\-gov\\-\\w+\\-\\d+$") - return reg - }(), - }, - Defaults: endpoint{ - Hostname: "{service}.{region}.{dnsSuffix}", - Protocols: []string{"https"}, - SignatureVersions: []string{"v4"}, - }, - Regions: regions{ - "us-gov-east-1": region{ - Description: "AWS GovCloud (US-East)", - }, - "us-gov-west-1": region{ - Description: "AWS GovCloud (US)", + Endpoints: endpoints{ + "us-iso-east-1": endpoint{}, + }, }, - }, - Services: services{ - "acm": service{ + "events": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "us-iso-east-1": endpoint{}, }, }, - "api.sagemaker": service{ + "glacier": service{ Endpoints: endpoints{ - "us-gov-west-1": endpoint{}, + "us-iso-east-1": endpoint{ + Protocols: []string{"http", "https"}, + }, }, }, - "apigateway": service{ + "health": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "us-iso-east-1": endpoint{}, }, }, - "application-autoscaling": service{ - Defaults: endpoint{ - Hostname: "autoscaling.{region}.amazonaws.com", - CredentialScope: credentialScope{ - Service: "application-autoscaling", + "iam": service{ + PartitionEndpoint: "aws-iso-global", + IsRegionalized: boxedFalse, + + Endpoints: endpoints{ + "aws-iso-global": endpoint{ + Hostname: "iam.us-iso-east-1.c2s.ic.gov", + CredentialScope: credentialScope{ + Region: "us-iso-east-1", + }, }, }, + }, + "kinesis": service{ + Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "us-iso-east-1": endpoint{}, }, }, - "autoscaling": service{ + "kms": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{ - Protocols: []string{"http", "https"}, + "ProdFips": endpoint{ + Hostname: "kms-fips.us-iso-east-1.c2s.ic.gov", + CredentialScope: credentialScope{ + Region: "us-iso-east-1", + }, }, + "us-iso-east-1": endpoint{}, }, }, - "clouddirectory": service{ + "lambda": service{ Endpoints: endpoints{ - "us-gov-west-1": endpoint{}, + "us-iso-east-1": endpoint{}, }, }, - "cloudformation": service{ + "logs": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "us-iso-east-1": endpoint{}, }, }, - "cloudhsm": service{ + "monitoring": service{ Endpoints: endpoints{ - "us-gov-west-1": endpoint{}, + "us-iso-east-1": endpoint{}, }, }, - "cloudhsmv2": service{ - Defaults: endpoint{ - CredentialScope: credentialScope{ - Service: "cloudhsm", - }, - }, + "rds": service{ + Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "us-iso-east-1": endpoint{}, }, }, - "cloudtrail": service{ + "redshift": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "us-iso-east-1": endpoint{}, }, }, - "codedeploy": service{ + "route53": service{ + PartitionEndpoint: "aws-iso-global", + IsRegionalized: boxedFalse, Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-east-1-fips": endpoint{ - Hostname: "codedeploy-fips.us-gov-east-1.amazonaws.com", - CredentialScope: credentialScope{ - Region: "us-gov-east-1", - }, - }, - "us-gov-west-1": endpoint{}, - "us-gov-west-1-fips": endpoint{ - Hostname: "codedeploy-fips.us-gov-west-1.amazonaws.com", + "aws-iso-global": endpoint{ + Hostname: "route53.c2s.ic.gov", CredentialScope: credentialScope{ - Region: "us-gov-west-1", + Region: "us-iso-east-1", }, }, }, }, - "config": service{ + "runtime.sagemaker": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "us-iso-east-1": endpoint{}, }, }, - "directconnect": service{ + "s3": service{ + Defaults: endpoint{ + SignatureVersions: []string{"s3v4"}, + }, + Endpoints: endpoints{ + "us-iso-east-1": endpoint{ + Protocols: []string{"http", "https"}, + SignatureVersions: []string{"s3v4"}, + }, + }, + }, + "snowball": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "us-iso-east-1": endpoint{}, }, }, - "dms": service{ + "sns": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "us-iso-east-1": endpoint{ + Protocols: []string{"http", "https"}, + }, }, }, - "ds": service{ + "sqs": service{ Endpoints: endpoints{ - "us-gov-west-1": endpoint{}, + "us-iso-east-1": endpoint{ + Protocols: []string{"http", "https"}, + }, }, }, - "dynamodb": service{ + "states": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, - "us-gov-west-1-fips": endpoint{ - Hostname: "dynamodb.us-gov-west-1.amazonaws.com", - CredentialScope: credentialScope{ - Region: "us-gov-west-1", - }, + "us-iso-east-1": endpoint{}, + }, + }, + "streams.dynamodb": service{ + Defaults: endpoint{ + Protocols: []string{"http", "https"}, + CredentialScope: credentialScope{ + Service: "dynamodb", + }, + }, + Endpoints: endpoints{ + "us-iso-east-1": endpoint{ + Protocols: []string{"http", "https"}, }, }, }, - "ec2": service{ + "sts": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "us-iso-east-1": endpoint{}, }, }, - "ec2metadata": service{ - PartitionEndpoint: "aws-global", - IsRegionalized: boxedFalse, + "support": service{ + PartitionEndpoint: "aws-iso-global", Endpoints: endpoints{ - "aws-global": endpoint{ - Hostname: "169.254.169.254/latest", - Protocols: []string{"http"}, + "aws-iso-global": endpoint{ + Hostname: "support.us-iso-east-1.c2s.ic.gov", + CredentialScope: credentialScope{ + Region: "us-iso-east-1", + }, }, }, }, - "ecr": service{ + "swf": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "us-iso-east-1": endpoint{}, }, }, - "ecs": service{ + "transcribestreaming": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "us-iso-east-1": endpoint{}, }, }, - "elasticache": service{ + "workspaces": service{ Endpoints: endpoints{ - "fips": endpoint{ - Hostname: "elasticache-fips.us-gov-west-1.amazonaws.com", - CredentialScope: credentialScope{ - Region: "us-gov-west-1", - }, - }, - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "us-iso-east-1": endpoint{}, }, }, - "elasticbeanstalk": service{ + }, +} +// AwsIsoBPartition returns the Resolver for AWS ISOB (US). +func AwsIsoBPartition() Partition { + return awsisobPartition.Partition() +} + +var awsisobPartition = partition{ + ID: "aws-iso-b", + Name: "AWS ISOB (US)", + DNSSuffix: "sc2s.sgov.gov", + RegionRegex: regionRegex{ + Regexp: func() *regexp.Regexp { + reg, _ := regexp.Compile("^us\\-isob\\-\\w+\\-\\d+$") + return reg + }(), + }, + Defaults: endpoint{ + Hostname: "{service}.{region}.{dnsSuffix}", + Protocols: []string{"https"}, + SignatureVersions: []string{"v4"}, + }, + Regions: regions{ + "us-isob-east-1": region{ + Description: "US ISOB East (Ohio)", + }, + }, + Services: services{ + "application-autoscaling": service{ + Defaults: endpoint{ + Protocols: []string{"http", "https"}, + }, Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "us-isob-east-1": endpoint{}, }, }, - "elasticfilesystem": service{ - + "autoscaling": service{ + Defaults: endpoint{ + Protocols: []string{"http", "https"}, + }, Endpoints: endpoints{ - "us-gov-west-1": endpoint{}, + "us-isob-east-1": endpoint{}, }, }, - "elasticloadbalancing": service{ + "cloudformation": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{ - Protocols: []string{"http", "https"}, - }, + "us-isob-east-1": endpoint{}, }, }, - "elasticmapreduce": service{ + "cloudtrail": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{ - Protocols: []string{"https"}, - }, + "us-isob-east-1": endpoint{}, }, }, - "es": service{ + "config": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "us-isob-east-1": endpoint{}, }, }, - "events": service{ + "directconnect": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "us-isob-east-1": endpoint{}, }, }, - "glacier": service{ + "dms": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{ - Protocols: []string{"http", "https"}, + "dms-fips": endpoint{ + Hostname: "dms.us-isob-east-1.sc2s.sgov.gov", + CredentialScope: credentialScope{ + Region: "us-isob-east-1", + }, }, + "us-isob-east-1": endpoint{}, }, }, - "guardduty": service{ - IsRegionalized: boxedTrue, + "dynamodb": service{ Defaults: endpoint{ - Protocols: []string{"https"}, + Protocols: []string{"http", "https"}, }, Endpoints: endpoints{ - "us-gov-west-1": endpoint{}, + "us-isob-east-1": endpoint{}, }, }, - "iam": service{ - PartitionEndpoint: "aws-us-gov-global", + "ec2": service{ + Defaults: endpoint{ + Protocols: []string{"http", "https"}, + }, + Endpoints: endpoints{ + "us-isob-east-1": endpoint{}, + }, + }, + "ec2metadata": service{ + PartitionEndpoint: "aws-global", IsRegionalized: boxedFalse, Endpoints: endpoints{ - "aws-us-gov-global": endpoint{ - Hostname: "iam.us-gov.amazonaws.com", - CredentialScope: credentialScope{ - Region: "us-gov-west-1", - }, + "aws-global": endpoint{ + Hostname: "169.254.169.254/latest", + Protocols: []string{"http"}, }, }, }, - "inspector": service{ + "elasticache": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "us-isob-east-1": endpoint{}, }, }, - "iot": service{ - Defaults: endpoint{ - CredentialScope: credentialScope{ - Service: "execute-api", - }, - }, + "elasticloadbalancing": service{ + Endpoints: endpoints{ - "us-gov-west-1": endpoint{}, + "us-isob-east-1": endpoint{ + Protocols: []string{"https"}, + }, }, }, - "kinesis": service{ + "elasticmapreduce": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "us-isob-east-1": endpoint{}, }, }, - "kms": service{ + "events": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "us-isob-east-1": endpoint{}, }, }, - "lambda": service{ + "glacier": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "us-isob-east-1": endpoint{}, }, }, - "logs": service{ + "health": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "us-isob-east-1": endpoint{}, }, }, - "mediaconvert": service{ + "iam": service{ + PartitionEndpoint: "aws-iso-b-global", + IsRegionalized: boxedFalse, Endpoints: endpoints{ - "us-gov-west-1": endpoint{}, - }, - }, - "metering.marketplace": service{ - Defaults: endpoint{ - CredentialScope: credentialScope{ - Service: "aws-marketplace", + "aws-iso-b-global": endpoint{ + Hostname: "iam.us-isob-east-1.sc2s.sgov.gov", + CredentialScope: credentialScope{ + Region: "us-isob-east-1", + }, }, }, + }, + "kinesis": service{ + Endpoints: endpoints{ - "us-gov-west-1": endpoint{}, + "us-isob-east-1": endpoint{}, }, }, - "monitoring": service{ + "kms": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "ProdFips": endpoint{ + Hostname: "kms-fips.us-isob-east-1.sc2s.sgov.gov", + CredentialScope: credentialScope{ + Region: "us-isob-east-1", + }, + }, + "us-isob-east-1": endpoint{}, }, }, - "polly": service{ + "lambda": service{ Endpoints: endpoints{ - "us-gov-west-1": endpoint{}, + "us-isob-east-1": endpoint{}, }, }, - "rds": service{ + "license-manager": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "us-isob-east-1": endpoint{}, }, }, - "redshift": service{ + "logs": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "us-isob-east-1": endpoint{}, }, }, - "rekognition": service{ + "monitoring": service{ Endpoints: endpoints{ - "us-gov-west-1": endpoint{}, + "us-isob-east-1": endpoint{}, }, }, - "runtime.sagemaker": service{ + "rds": service{ Endpoints: endpoints{ - "us-gov-west-1": endpoint{}, + "us-isob-east-1": endpoint{}, }, }, - "s3": service{ - Defaults: endpoint{ - SignatureVersions: []string{"s3", "s3v4"}, - }, + "redshift": service{ + Endpoints: endpoints{ - "fips-us-gov-west-1": endpoint{ - Hostname: "s3-fips-us-gov-west-1.amazonaws.com", - CredentialScope: credentialScope{ - Region: "us-gov-west-1", - }, - }, - "us-gov-east-1": endpoint{ - Hostname: "s3.us-gov-east-1.amazonaws.com", - Protocols: []string{"http", "https"}, - }, - "us-gov-west-1": endpoint{ - Hostname: "s3.us-gov-west-1.amazonaws.com", - Protocols: []string{"http", "https"}, - }, + "us-isob-east-1": endpoint{}, }, }, - "s3-control": service{ + "s3": service{ Defaults: endpoint{ - Protocols: []string{"https"}, + Protocols: []string{"http", "https"}, SignatureVersions: []string{"s3v4"}, }, Endpoints: endpoints{ - "us-gov-east-1": endpoint{ - Hostname: "s3-control.us-gov-east-1.amazonaws.com", - SignatureVersions: []string{"s3v4"}, - CredentialScope: credentialScope{ - Region: "us-gov-east-1", - }, - }, - "us-gov-east-1-fips": endpoint{ - Hostname: "s3-control-fips.us-gov-east-1.amazonaws.com", - SignatureVersions: []string{"s3v4"}, - CredentialScope: credentialScope{ - Region: "us-gov-east-1", - }, - }, - "us-gov-west-1": endpoint{ - Hostname: "s3-control.us-gov-west-1.amazonaws.com", - SignatureVersions: []string{"s3v4"}, - CredentialScope: credentialScope{ - Region: "us-gov-west-1", - }, - }, - "us-gov-west-1-fips": endpoint{ - Hostname: "s3-control-fips.us-gov-west-1.amazonaws.com", - SignatureVersions: []string{"s3v4"}, - CredentialScope: credentialScope{ - Region: "us-gov-west-1", - }, - }, - }, - }, - "sms": service{ - - Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "us-isob-east-1": endpoint{}, }, }, "snowball": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "us-isob-east-1": endpoint{}, }, }, "sns": service{ - + Defaults: endpoint{ + Protocols: []string{"http", "https"}, + }, Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{ - Protocols: []string{"http", "https"}, - }, + "us-isob-east-1": endpoint{}, }, }, "sqs": service{ - + Defaults: endpoint{ + SSLCommonName: "{region}.queue.{dnsSuffix}", + Protocols: []string{"http", "https"}, + }, Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{ - SSLCommonName: "{region}.queue.{dnsSuffix}", - Protocols: []string{"http", "https"}, - }, + "us-isob-east-1": endpoint{}, }, }, "ssm": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "us-isob-east-1": endpoint{}, }, }, "states": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, - }, - }, - "storagegateway": service{ - - Endpoints: endpoints{ - "us-gov-west-1": endpoint{}, + "us-isob-east-1": endpoint{}, }, }, "streams.dynamodb": service{ Defaults: endpoint{ + Protocols: []string{"http", "https"}, CredentialScope: credentialScope{ Service: "dynamodb", }, }, Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, - "us-gov-west-1-fips": endpoint{ - Hostname: "dynamodb.us-gov-west-1.amazonaws.com", - CredentialScope: credentialScope{ - Region: "us-gov-west-1", - }, - }, + "us-isob-east-1": endpoint{}, }, }, "sts": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "us-isob-east-1": endpoint{}, }, }, - "swf": service{ + "support": service{ + PartitionEndpoint: "aws-iso-b-global", Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, + "aws-iso-b-global": endpoint{ + Hostname: "support.us-isob-east-1.sc2s.sgov.gov", + CredentialScope: credentialScope{ + Region: "us-isob-east-1", + }, + }, }, }, - "tagging": service{ + "swf": service{ Endpoints: endpoints{ - "us-gov-east-1": endpoint{}, - "us-gov-west-1": endpoint{}, - }, - }, - "translate": service{ - Defaults: endpoint{ - Protocols: []string{"https"}, - }, - Endpoints: endpoints{ - "us-gov-west-1": endpoint{}, - "us-gov-west-1-fips": endpoint{ - Hostname: "translate-fips.us-gov-west-1.amazonaws.com", - CredentialScope: credentialScope{ - Region: "us-gov-west-1", - }, - }, + "us-isob-east-1": endpoint{}, }, }, }, diff --git a/github.com/aws/aws-sdk-go/aws/endpoints/dep_service_ids.go b/github.com/aws/aws-sdk-go/aws/endpoints/dep_service_ids.go index 000dd79eec..ca8fc828e1 100644 --- a/github.com/aws/aws-sdk-go/aws/endpoints/dep_service_ids.go +++ b/github.com/aws/aws-sdk-go/aws/endpoints/dep_service_ids.go @@ -2,7 +2,7 @@ package endpoints // Service identifiers // -// Deprecated: Use client package's EndpointID value instead of these +// Deprecated: Use client package's EndpointsID value instead of these // ServiceIDs. These IDs are not maintained, and are out of date. const ( A4bServiceID = "a4b" // A4b. diff --git a/github.com/aws/aws-sdk-go/aws/endpoints/endpoints.go b/github.com/aws/aws-sdk-go/aws/endpoints/endpoints.go index f82babf6f9..ca956e5f12 100644 --- a/github.com/aws/aws-sdk-go/aws/endpoints/endpoints.go +++ b/github.com/aws/aws-sdk-go/aws/endpoints/endpoints.go @@ -3,6 +3,7 @@ package endpoints import ( "fmt" "regexp" + "strings" "github.com/aws/aws-sdk-go/aws/awserr" ) @@ -46,6 +47,108 @@ type Options struct { // // This option is ignored if StrictMatching is enabled. ResolveUnknownService bool + + // STS Regional Endpoint flag helps with resolving the STS endpoint + STSRegionalEndpoint STSRegionalEndpoint + + // S3 Regional Endpoint flag helps with resolving the S3 endpoint + S3UsEast1RegionalEndpoint S3UsEast1RegionalEndpoint +} + +// STSRegionalEndpoint is an enum for the states of the STS Regional Endpoint +// options. +type STSRegionalEndpoint int + +func (e STSRegionalEndpoint) String() string { + switch e { + case LegacySTSEndpoint: + return "legacy" + case RegionalSTSEndpoint: + return "regional" + case UnsetSTSEndpoint: + return "" + default: + return "unknown" + } +} + +const ( + + // UnsetSTSEndpoint represents that STS Regional Endpoint flag is not specified. + UnsetSTSEndpoint STSRegionalEndpoint = iota + + // LegacySTSEndpoint represents when STS Regional Endpoint flag is specified + // to use legacy endpoints. + LegacySTSEndpoint + + // RegionalSTSEndpoint represents when STS Regional Endpoint flag is specified + // to use regional endpoints. + RegionalSTSEndpoint +) + +// GetSTSRegionalEndpoint function returns the STSRegionalEndpointFlag based +// on the input string provided in env config or shared config by the user. +// +// `legacy`, `regional` are the only case-insensitive valid strings for +// resolving the STS regional Endpoint flag. +func GetSTSRegionalEndpoint(s string) (STSRegionalEndpoint, error) { + switch { + case strings.EqualFold(s, "legacy"): + return LegacySTSEndpoint, nil + case strings.EqualFold(s, "regional"): + return RegionalSTSEndpoint, nil + default: + return UnsetSTSEndpoint, fmt.Errorf("unable to resolve the value of STSRegionalEndpoint for %v", s) + } +} + +// S3UsEast1RegionalEndpoint is an enum for the states of the S3 us-east-1 +// Regional Endpoint options. +type S3UsEast1RegionalEndpoint int + +func (e S3UsEast1RegionalEndpoint) String() string { + switch e { + case LegacyS3UsEast1Endpoint: + return "legacy" + case RegionalS3UsEast1Endpoint: + return "regional" + case UnsetS3UsEast1Endpoint: + return "" + default: + return "unknown" + } +} + +const ( + + // UnsetS3UsEast1Endpoint represents that S3 Regional Endpoint flag is not + // specified. + UnsetS3UsEast1Endpoint S3UsEast1RegionalEndpoint = iota + + // LegacyS3UsEast1Endpoint represents when S3 Regional Endpoint flag is + // specified to use legacy endpoints. + LegacyS3UsEast1Endpoint + + // RegionalS3UsEast1Endpoint represents when S3 Regional Endpoint flag is + // specified to use regional endpoints. + RegionalS3UsEast1Endpoint +) + +// GetS3UsEast1RegionalEndpoint function returns the S3UsEast1RegionalEndpointFlag based +// on the input string provided in env config or shared config by the user. +// +// `legacy`, `regional` are the only case-insensitive valid strings for +// resolving the S3 regional Endpoint flag. +func GetS3UsEast1RegionalEndpoint(s string) (S3UsEast1RegionalEndpoint, error) { + switch { + case strings.EqualFold(s, "legacy"): + return LegacyS3UsEast1Endpoint, nil + case strings.EqualFold(s, "regional"): + return RegionalS3UsEast1Endpoint, nil + default: + return UnsetS3UsEast1Endpoint, + fmt.Errorf("unable to resolve the value of S3UsEast1RegionalEndpoint for %v", s) + } } // Set combines all of the option functions together. @@ -79,6 +182,12 @@ func ResolveUnknownServiceOption(o *Options) { o.ResolveUnknownService = true } +// STSRegionalEndpointOption enables the STS endpoint resolver behavior to resolve +// STS endpoint to their regional endpoint, instead of the global endpoint. +func STSRegionalEndpointOption(o *Options) { + o.STSRegionalEndpoint = RegionalSTSEndpoint +} + // A Resolver provides the interface for functionality to resolve endpoints. // The build in Partition and DefaultResolver return value satisfy this interface. type Resolver interface { @@ -170,10 +279,13 @@ func PartitionForRegion(ps []Partition, regionID string) (Partition, bool) { // A Partition provides the ability to enumerate the partition's regions // and services. type Partition struct { - id string - p *partition + id, dnsSuffix string + p *partition } +// DNSSuffix returns the base domain name of the partition. +func (p Partition) DNSSuffix() string { return p.dnsSuffix } + // ID returns the identifier of the partition. func (p Partition) ID() string { return p.id } @@ -191,7 +303,7 @@ func (p Partition) ID() string { return p.id } // require the provided service and region to be known by the partition. // If the endpoint cannot be strictly resolved an error will be returned. This // mode is useful to ensure the endpoint resolved is valid. Without -// StrictMatching enabled the endpoint returned my look valid but may not work. +// StrictMatching enabled the endpoint returned may look valid but may not work. // StrictMatching requires the SDK to be updated if you want to take advantage // of new regions and services expansions. // @@ -205,7 +317,7 @@ func (p Partition) EndpointFor(service, region string, opts ...func(*Options)) ( // Regions returns a map of Regions indexed by their ID. This is useful for // enumerating over the regions in a partition. func (p Partition) Regions() map[string]Region { - rs := map[string]Region{} + rs := make(map[string]Region, len(p.p.Regions)) for id, r := range p.p.Regions { rs[id] = Region{ id: id, @@ -220,7 +332,7 @@ func (p Partition) Regions() map[string]Region { // Services returns a map of Service indexed by their ID. This is useful for // enumerating over the services in a partition. func (p Partition) Services() map[string]Service { - ss := map[string]Service{} + ss := make(map[string]Service, len(p.p.Services)) for id := range p.p.Services { ss[id] = Service{ id: id, @@ -307,7 +419,7 @@ func (s Service) Regions() map[string]Region { // A region is the AWS region the service exists in. Whereas a Endpoint is // an URL that can be resolved to a instance of a service. func (s Service) Endpoints() map[string]Endpoint { - es := map[string]Endpoint{} + es := make(map[string]Endpoint, len(s.p.Services[s.id].Endpoints)) for id := range s.p.Services[s.id].Endpoints { es[id] = Endpoint{ id: id, @@ -347,6 +459,9 @@ type ResolvedEndpoint struct { // The endpoint URL URL string + // The endpoint partition + PartitionID string + // The region that should be used for signing requests. SigningRegion string diff --git a/github.com/aws/aws-sdk-go/aws/endpoints/legacy_regions.go b/github.com/aws/aws-sdk-go/aws/endpoints/legacy_regions.go new file mode 100644 index 0000000000..df75e899ad --- /dev/null +++ b/github.com/aws/aws-sdk-go/aws/endpoints/legacy_regions.go @@ -0,0 +1,24 @@ +package endpoints + +var legacyGlobalRegions = map[string]map[string]struct{}{ + "sts": { + "ap-northeast-1": {}, + "ap-south-1": {}, + "ap-southeast-1": {}, + "ap-southeast-2": {}, + "ca-central-1": {}, + "eu-central-1": {}, + "eu-north-1": {}, + "eu-west-1": {}, + "eu-west-2": {}, + "eu-west-3": {}, + "sa-east-1": {}, + "us-east-1": {}, + "us-east-2": {}, + "us-west-1": {}, + "us-west-2": {}, + }, + "s3": { + "us-east-1": {}, + }, +} diff --git a/github.com/aws/aws-sdk-go/aws/endpoints/v3model.go b/github.com/aws/aws-sdk-go/aws/endpoints/v3model.go index ff6f76db6e..773613722f 100644 --- a/github.com/aws/aws-sdk-go/aws/endpoints/v3model.go +++ b/github.com/aws/aws-sdk-go/aws/endpoints/v3model.go @@ -7,6 +7,8 @@ import ( "strings" ) +var regionValidationRegex = regexp.MustCompile(`^[[:alnum:]]([[:alnum:]\-]*[[:alnum:]])?$`) + type partitions []partition func (ps partitions) EndpointFor(service, region string, opts ...func(*Options)) (ResolvedEndpoint, error) { @@ -54,8 +56,9 @@ type partition struct { func (p partition) Partition() Partition { return Partition{ - id: p.ID, - p: &p, + dnsSuffix: p.DNSSuffix, + id: p.ID, + p: &p, } } @@ -74,24 +77,56 @@ func (p partition) canResolveEndpoint(service, region string, strictMatch bool) return p.RegionRegex.MatchString(region) } +func allowLegacyEmptyRegion(service string) bool { + legacy := map[string]struct{}{ + "budgets": {}, + "ce": {}, + "chime": {}, + "cloudfront": {}, + "ec2metadata": {}, + "iam": {}, + "importexport": {}, + "organizations": {}, + "route53": {}, + "sts": {}, + "support": {}, + "waf": {}, + } + + _, allowed := legacy[service] + return allowed +} + func (p partition) EndpointFor(service, region string, opts ...func(*Options)) (resolved ResolvedEndpoint, err error) { var opt Options opt.Set(opts...) s, hasService := p.Services[service] - if !(hasService || opt.ResolveUnknownService) { + if len(service) == 0 || !(hasService || opt.ResolveUnknownService) { // Only return error if the resolver will not fallback to creating // endpoint based on service endpoint ID passed in. return resolved, NewUnknownServiceError(p.ID, service, serviceList(p.Services)) } + if len(region) == 0 && allowLegacyEmptyRegion(service) && len(s.PartitionEndpoint) != 0 { + region = s.PartitionEndpoint + } + + if (service == "sts" && opt.STSRegionalEndpoint != RegionalSTSEndpoint) || + (service == "s3" && opt.S3UsEast1RegionalEndpoint != RegionalS3UsEast1Endpoint) { + if _, ok := legacyGlobalRegions[service][region]; ok { + region = "aws-global" + } + } + e, hasEndpoint := s.endpointForRegion(region) - if !hasEndpoint && opt.StrictMatching { + if len(region) == 0 || (!hasEndpoint && opt.StrictMatching) { return resolved, NewUnknownEndpointError(p.ID, service, region, endpointList(s.Endpoints)) } defs := []endpoint{p.Defaults, s.Defaults} - return e.resolve(service, region, p.DNSSuffix, defs, opt), nil + + return e.resolve(service, p.ID, region, p.DNSSuffix, defs, opt) } func serviceList(ss services) []string { @@ -200,7 +235,7 @@ func getByPriority(s []string, p []string, def string) string { return s[0] } -func (e endpoint) resolve(service, region, dnsSuffix string, defs []endpoint, opts Options) ResolvedEndpoint { +func (e endpoint) resolve(service, partitionID, region, dnsSuffix string, defs []endpoint, opts Options) (ResolvedEndpoint, error) { var merged endpoint for _, def := range defs { merged.mergeIn(def) @@ -208,11 +243,27 @@ func (e endpoint) resolve(service, region, dnsSuffix string, defs []endpoint, op merged.mergeIn(e) e = merged - hostname := e.Hostname + signingRegion := e.CredentialScope.Region + if len(signingRegion) == 0 { + signingRegion = region + } + signingName := e.CredentialScope.Service + var signingNameDerived bool + if len(signingName) == 0 { + signingName = service + signingNameDerived = true + } + + hostname := e.Hostname // Offset the hostname for dualstack if enabled if opts.UseDualStack && e.HasDualStack == boxedTrue { hostname = e.DualStackHostname + region = signingRegion + } + + if !validateInputRegion(region) { + return ResolvedEndpoint{}, fmt.Errorf("invalid region identifier format provided") } u := strings.Replace(hostname, "{service}", service, 1) @@ -222,25 +273,14 @@ func (e endpoint) resolve(service, region, dnsSuffix string, defs []endpoint, op scheme := getEndpointScheme(e.Protocols, opts.DisableSSL) u = fmt.Sprintf("%s://%s", scheme, u) - signingRegion := e.CredentialScope.Region - if len(signingRegion) == 0 { - signingRegion = region - } - - signingName := e.CredentialScope.Service - var signingNameDerived bool - if len(signingName) == 0 { - signingName = service - signingNameDerived = true - } - return ResolvedEndpoint{ URL: u, + PartitionID: partitionID, SigningRegion: signingRegion, SigningName: signingName, SigningNameDerived: signingNameDerived, SigningMethod: getByPriority(e.SignatureVersions, signerPriority, defaultSigner), - } + }, nil } func getEndpointScheme(protocols []string, disableSSL bool) string { @@ -305,3 +345,7 @@ const ( boxedFalse boxedTrue ) + +func validateInputRegion(region string) bool { + return regionValidationRegex.MatchString(region) +} diff --git a/github.com/aws/aws-sdk-go/aws/request/connection_reset_error.go b/github.com/aws/aws-sdk-go/aws/request/connection_reset_error.go index 271da432ce..d9b37f4d32 100644 --- a/github.com/aws/aws-sdk-go/aws/request/connection_reset_error.go +++ b/github.com/aws/aws-sdk-go/aws/request/connection_reset_error.go @@ -1,18 +1,17 @@ -// +build !appengine,!plan9 - package request import ( - "net" - "os" - "syscall" + "strings" ) func isErrConnectionReset(err error) bool { - if opErr, ok := err.(*net.OpError); ok { - if sysErr, ok := opErr.Err.(*os.SyscallError); ok { - return sysErr.Err == syscall.ECONNRESET - } + if strings.Contains(err.Error(), "read: connection reset") { + return false + } + + if strings.Contains(err.Error(), "connection reset") || + strings.Contains(err.Error(), "broken pipe") { + return true } return false diff --git a/github.com/aws/aws-sdk-go/aws/request/connection_reset_error_other.go b/github.com/aws/aws-sdk-go/aws/request/connection_reset_error_other.go deleted file mode 100644 index daf9eca437..0000000000 --- a/github.com/aws/aws-sdk-go/aws/request/connection_reset_error_other.go +++ /dev/null @@ -1,11 +0,0 @@ -// +build appengine plan9 - -package request - -import ( - "strings" -) - -func isErrConnectionReset(err error) bool { - return strings.Contains(err.Error(), "connection reset") -} diff --git a/github.com/aws/aws-sdk-go/aws/request/handlers.go b/github.com/aws/aws-sdk-go/aws/request/handlers.go index 8ef8548a96..e819ab6c0e 100644 --- a/github.com/aws/aws-sdk-go/aws/request/handlers.go +++ b/github.com/aws/aws-sdk-go/aws/request/handlers.go @@ -10,6 +10,7 @@ import ( type Handlers struct { Validate HandlerList Build HandlerList + BuildStream HandlerList Sign HandlerList Send HandlerList ValidateResponse HandlerList @@ -23,11 +24,12 @@ type Handlers struct { Complete HandlerList } -// Copy returns of this handler's lists. +// Copy returns a copy of this handler's lists. func (h *Handlers) Copy() Handlers { return Handlers{ Validate: h.Validate.copy(), Build: h.Build.copy(), + BuildStream: h.BuildStream.copy(), Sign: h.Sign.copy(), Send: h.Send.copy(), ValidateResponse: h.ValidateResponse.copy(), @@ -42,10 +44,11 @@ func (h *Handlers) Copy() Handlers { } } -// Clear removes callback functions for all handlers +// Clear removes callback functions for all handlers. func (h *Handlers) Clear() { h.Validate.Clear() h.Build.Clear() + h.BuildStream.Clear() h.Send.Clear() h.Sign.Clear() h.Unmarshal.Clear() @@ -59,6 +62,54 @@ func (h *Handlers) Clear() { h.Complete.Clear() } +// IsEmpty returns if there are no handlers in any of the handlerlists. +func (h *Handlers) IsEmpty() bool { + if h.Validate.Len() != 0 { + return false + } + if h.Build.Len() != 0 { + return false + } + if h.BuildStream.Len() != 0 { + return false + } + if h.Send.Len() != 0 { + return false + } + if h.Sign.Len() != 0 { + return false + } + if h.Unmarshal.Len() != 0 { + return false + } + if h.UnmarshalStream.Len() != 0 { + return false + } + if h.UnmarshalMeta.Len() != 0 { + return false + } + if h.UnmarshalError.Len() != 0 { + return false + } + if h.ValidateResponse.Len() != 0 { + return false + } + if h.Retry.Len() != 0 { + return false + } + if h.AfterRetry.Len() != 0 { + return false + } + if h.CompleteAttempt.Len() != 0 { + return false + } + if h.Complete.Len() != 0 { + return false + } + + return true +} + // A HandlerListRunItem represents an entry in the HandlerList which // is being run. type HandlerListRunItem struct { @@ -275,3 +326,18 @@ func MakeAddToUserAgentFreeFormHandler(s string) func(*Request) { AddToUserAgent(r, s) } } + +// WithSetRequestHeaders updates the operation request's HTTP header to contain +// the header key value pairs provided. If the header key already exists in the +// request's HTTP header set, the existing value(s) will be replaced. +func WithSetRequestHeaders(h map[string]string) Option { + return withRequestHeader(h).SetRequestHeaders +} + +type withRequestHeader map[string]string + +func (h withRequestHeader) SetRequestHeaders(r *Request) { + for k, v := range h { + r.HTTPRequest.Header[k] = []string{v} + } +} diff --git a/github.com/aws/aws-sdk-go/aws/request/offset_reader.go b/github.com/aws/aws-sdk-go/aws/request/offset_reader.go index b0c2ef4fe6..9370fa50c3 100644 --- a/github.com/aws/aws-sdk-go/aws/request/offset_reader.go +++ b/github.com/aws/aws-sdk-go/aws/request/offset_reader.go @@ -15,12 +15,15 @@ type offsetReader struct { closed bool } -func newOffsetReader(buf io.ReadSeeker, offset int64) *offsetReader { +func newOffsetReader(buf io.ReadSeeker, offset int64) (*offsetReader, error) { reader := &offsetReader{} - buf.Seek(offset, sdkio.SeekStart) + _, err := buf.Seek(offset, sdkio.SeekStart) + if err != nil { + return nil, err + } reader.buf = buf - return reader + return reader, nil } // Close will close the instance of the offset reader's access to @@ -54,7 +57,9 @@ func (o *offsetReader) Seek(offset int64, whence int) (int64, error) { // CloseAndCopy will return a new offsetReader with a copy of the old buffer // and close the old buffer. -func (o *offsetReader) CloseAndCopy(offset int64) *offsetReader { - o.Close() +func (o *offsetReader) CloseAndCopy(offset int64) (*offsetReader, error) { + if err := o.Close(); err != nil { + return nil, err + } return newOffsetReader(o.buf, offset) } diff --git a/github.com/aws/aws-sdk-go/aws/request/request.go b/github.com/aws/aws-sdk-go/aws/request/request.go index b716684650..d597c6ead5 100644 --- a/github.com/aws/aws-sdk-go/aws/request/request.go +++ b/github.com/aws/aws-sdk-go/aws/request/request.go @@ -4,7 +4,6 @@ import ( "bytes" "fmt" "io" - "net" "net/http" "net/url" "reflect" @@ -37,6 +36,10 @@ const ( // API request that was canceled. Requests given a aws.Context may // return this error when canceled. CanceledErrorCode = "RequestCanceled" + + // ErrCodeRequestError is an error preventing the SDK from continuing to + // process the request. + ErrCodeRequestError = "RequestError" ) // A Request is the service request to be made. @@ -52,6 +55,7 @@ type Request struct { HTTPRequest *http.Request HTTPResponse *http.Response Body io.ReadSeeker + streamingBody io.ReadCloser BodyStart int64 // offset from beginning of Body that the request body starts Params interface{} Error error @@ -65,6 +69,15 @@ type Request struct { LastSignedAt time.Time DisableFollowRedirects bool + // Additional API error codes that should be retried. IsErrorRetryable + // will consider these codes in addition to its built in cases. + RetryErrorCodes []string + + // Additional API error codes that should be retried with throttle backoff + // delay. IsErrorThrottle will consider these codes in addition to its + // built in cases. + ThrottleErrorCodes []string + // A value greater than 0 instructs the request to be signed as Presigned URL // You should not set this field directly. Instead use Request's // Presign or PresignRequest methods. @@ -91,8 +104,12 @@ type Operation struct { BeforePresignFn func(r *Request) error } -// New returns a new Request pointer for the service API -// operation and parameters. +// New returns a new Request pointer for the service API operation and +// parameters. +// +// A Retryer should be provided to direct how the request is retried. If +// Retryer is nil, a default no retry value will be used. You can use +// NoOpRetryer in the Client package to disable retry behavior directly. // // Params is any value of input parameters to be the request payload. // Data is pointer value to an object which the request's response @@ -100,6 +117,10 @@ type Operation struct { func New(cfg aws.Config, clientInfo metadata.ClientInfo, handlers Handlers, retryer Retryer, operation *Operation, params interface{}, data interface{}) *Request { + if retryer == nil { + retryer = noOpRetryer{} + } + method := operation.HTTPMethod if method == "" { method = "POST" @@ -114,8 +135,6 @@ func New(cfg aws.Config, clientInfo metadata.ClientInfo, handlers Handlers, err = awserr.New("InvalidEndpointURL", "invalid endpoint uri", err) } - SanitizeHostForHeader(httpReq) - r := &Request{ Config: cfg, ClientInfo: clientInfo, @@ -232,6 +251,10 @@ func (r *Request) WillRetry() bool { return r.Error != nil && aws.BoolValue(r.Retryable) && r.RetryCount < r.MaxRetries() } +func fmtAttemptCount(retryCount, maxRetries int) string { + return fmt.Sprintf("attempt %v/%v", retryCount, maxRetries) +} + // ParamsFilled returns if the request's parameters have been populated // and the parameters are valid. False is returned if no parameters are // provided or invalid. @@ -260,10 +283,28 @@ func (r *Request) SetStringBody(s string) { // SetReaderBody will set the request's body reader. func (r *Request) SetReaderBody(reader io.ReadSeeker) { r.Body = reader - r.BodyStart, _ = reader.Seek(0, sdkio.SeekCurrent) // Get the Bodies current offset. + + if aws.IsReaderSeekable(reader) { + var err error + // Get the Bodies current offset so retries will start from the same + // initial position. + r.BodyStart, err = reader.Seek(0, sdkio.SeekCurrent) + if err != nil { + r.Error = awserr.New(ErrCodeSerialization, + "failed to determine start of request body", err) + return + } + } r.ResetBody() } +// SetStreamingBody set the reader to be used for the request that will stream +// bytes to the server. Request's Body must not be set to any reader. +func (r *Request) SetStreamingBody(reader io.ReadCloser) { + r.streamingBody = reader + r.SetReaderBody(aws.ReadSeekCloser(reader)) +} + // Presign returns the request's signed URL. Error will be returned // if the signing fails. The expire parameter is only used for presigned Amazon // S3 API requests. All other AWS services will use a fixed expiration @@ -331,16 +372,15 @@ func getPresignedURL(r *Request, expire time.Duration) (string, http.Header, err return r.HTTPRequest.URL.String(), r.SignedHeaderVals, nil } -func debugLogReqError(r *Request, stage string, retrying bool, err error) { +const ( + notRetrying = "not retrying" +) + +func debugLogReqError(r *Request, stage, retryStr string, err error) { if !r.Config.LogLevel.Matches(aws.LogDebugWithRequestErrors) { return } - retryStr := "not retrying" - if retrying { - retryStr = "will retry" - } - r.Config.Logger.Log(fmt.Sprintf("DEBUG: %s %s/%s failed, %s, error %v", stage, r.ClientInfo.ServiceName, r.Operation.Name, retryStr, err)) } @@ -359,12 +399,12 @@ func (r *Request) Build() error { if !r.built { r.Handlers.Validate.Run(r) if r.Error != nil { - debugLogReqError(r, "Validate Request", false, r.Error) + debugLogReqError(r, "Validate Request", notRetrying, r.Error) return r.Error } r.Handlers.Build.Run(r) if r.Error != nil { - debugLogReqError(r, "Build Request", false, r.Error) + debugLogReqError(r, "Build Request", notRetrying, r.Error) return r.Error } r.built = true @@ -380,20 +420,30 @@ func (r *Request) Build() error { func (r *Request) Sign() error { r.Build() if r.Error != nil { - debugLogReqError(r, "Build Request", false, r.Error) + debugLogReqError(r, "Build Request", notRetrying, r.Error) return r.Error } + SanitizeHostForHeader(r.HTTPRequest) + r.Handlers.Sign.Run(r) return r.Error } -func (r *Request) getNextRequestBody() (io.ReadCloser, error) { +func (r *Request) getNextRequestBody() (body io.ReadCloser, err error) { + if r.streamingBody != nil { + return r.streamingBody, nil + } + if r.safeBody != nil { r.safeBody.Close() } - r.safeBody = newOffsetReader(r.Body, r.BodyStart) + r.safeBody, err = newOffsetReader(r.Body, r.BodyStart) + if err != nil { + return nil, awserr.New(ErrCodeSerialization, + "failed to get next request body reader", err) + } // Go 1.8 tightened and clarified the rules code needs to use when building // requests with the http package. Go 1.8 removed the automatic detection @@ -410,10 +460,10 @@ func (r *Request) getNextRequestBody() (io.ReadCloser, error) { // Related golang/go#18257 l, err := aws.SeekerLen(r.Body) if err != nil { - return nil, awserr.New(ErrCodeSerialization, "failed to compute request body size", err) + return nil, awserr.New(ErrCodeSerialization, + "failed to compute request body size", err) } - var body io.ReadCloser if l == 0 { body = NoBody } else if l > 0 { @@ -474,29 +524,28 @@ func (r *Request) Send() error { r.AttemptTime = time.Now() if err := r.Sign(); err != nil { - debugLogReqError(r, "Sign Request", false, err) + debugLogReqError(r, "Sign Request", notRetrying, err) return err } if err := r.sendRequest(); err == nil { return nil - } else if !shouldRetryCancel(r) { - return err - } else { - r.Handlers.Retry.Run(r) - r.Handlers.AfterRetry.Run(r) + } + r.Handlers.Retry.Run(r) + r.Handlers.AfterRetry.Run(r) - if r.Error != nil || !aws.BoolValue(r.Retryable) { - return r.Error - } + if r.Error != nil || !aws.BoolValue(r.Retryable) { + return r.Error + } - r.prepareRetry() - continue + if err := r.prepareRetry(); err != nil { + r.Error = err + return err } } } -func (r *Request) prepareRetry() { +func (r *Request) prepareRetry() error { if r.Config.LogLevel.Matches(aws.LogDebugWithRequestRetries) { r.Config.Logger.Log(fmt.Sprintf("DEBUG: Retrying Request %s/%s, attempt %d", r.ClientInfo.ServiceName, r.Operation.Name, r.RetryCount)) @@ -507,12 +556,19 @@ func (r *Request) prepareRetry() { // the request's body even though the Client's Do returned. r.HTTPRequest = copyHTTPRequest(r.HTTPRequest, nil) r.ResetBody() + if err := r.Error; err != nil { + return awserr.New(ErrCodeSerialization, + "failed to prepare body for retry", err) + + } // Closing response body to ensure that no response body is leaked // between retry attempts. if r.HTTPResponse != nil && r.HTTPResponse.Body != nil { r.HTTPResponse.Body.Close() } + + return nil } func (r *Request) sendRequest() (sendErr error) { @@ -521,7 +577,9 @@ func (r *Request) sendRequest() (sendErr error) { r.Retryable = nil r.Handlers.Send.Run(r) if r.Error != nil { - debugLogReqError(r, "Send Request", r.WillRetry(), r.Error) + debugLogReqError(r, "Send Request", + fmtAttemptCount(r.RetryCount, r.MaxRetries()), + r.Error) return r.Error } @@ -529,13 +587,17 @@ func (r *Request) sendRequest() (sendErr error) { r.Handlers.ValidateResponse.Run(r) if r.Error != nil { r.Handlers.UnmarshalError.Run(r) - debugLogReqError(r, "Validate Response", r.WillRetry(), r.Error) + debugLogReqError(r, "Validate Response", + fmtAttemptCount(r.RetryCount, r.MaxRetries()), + r.Error) return r.Error } r.Handlers.Unmarshal.Run(r) if r.Error != nil { - debugLogReqError(r, "Unmarshal Response", r.WillRetry(), r.Error) + debugLogReqError(r, "Unmarshal Response", + fmtAttemptCount(r.RetryCount, r.MaxRetries()), + r.Error) return r.Error } @@ -562,32 +624,6 @@ func AddToUserAgent(r *Request, s string) { r.HTTPRequest.Header.Set("User-Agent", s) } -func shouldRetryCancel(r *Request) bool { - awsErr, ok := r.Error.(awserr.Error) - timeoutErr := false - errStr := r.Error.Error() - if ok { - if awsErr.Code() == CanceledErrorCode { - return false - } - err := awsErr.OrigErr() - netErr, netOK := err.(net.Error) - timeoutErr = netOK && netErr.Temporary() - if urlErr, ok := err.(*url.Error); !timeoutErr && ok { - errStr = urlErr.Err.Error() - } - } - - // There can be two types of canceled errors here. - // The first being a net.Error and the other being an error. - // If the request was timed out, we want to continue the retry - // process. Otherwise, return the canceled error. - return timeoutErr || - (errStr != "net/http: request canceled" && - errStr != "net/http: request canceled while waiting for connection") - -} - // SanitizeHostForHeader removes default port from host and updates request.Host func SanitizeHostForHeader(r *http.Request) { host := getHost(r) @@ -603,6 +639,10 @@ func getHost(r *http.Request) string { return r.Host } + if r.URL == nil { + return "" + } + return r.URL.Host } diff --git a/github.com/aws/aws-sdk-go/aws/request/request_1_8.go b/github.com/aws/aws-sdk-go/aws/request/request_1_8.go index 7c6a8000f6..de1292f45a 100644 --- a/github.com/aws/aws-sdk-go/aws/request/request_1_8.go +++ b/github.com/aws/aws-sdk-go/aws/request/request_1_8.go @@ -4,6 +4,8 @@ package request import ( "net/http" + + "github.com/aws/aws-sdk-go/aws/awserr" ) // NoBody is a http.NoBody reader instructing Go HTTP client to not include @@ -24,7 +26,8 @@ var NoBody = http.NoBody func (r *Request) ResetBody() { body, err := r.getNextRequestBody() if err != nil { - r.Error = err + r.Error = awserr.New(ErrCodeSerialization, + "failed to reset request body", err) return } diff --git a/github.com/aws/aws-sdk-go/aws/request/request_pagination.go b/github.com/aws/aws-sdk-go/aws/request/request_pagination.go index a633ed5acf..64784e16f3 100644 --- a/github.com/aws/aws-sdk-go/aws/request/request_pagination.go +++ b/github.com/aws/aws-sdk-go/aws/request/request_pagination.go @@ -17,11 +17,13 @@ import ( // does the pagination between API operations, and Paginator defines the // configuration that will be used per page request. // -// cont := true -// for p.Next() && cont { +// for p.Next() { // data := p.Page().(*s3.ListObjectsOutput) // // process the page's data +// // ... +// // break out of loop to stop fetching additional pages // } +// // return p.Err() // // See service client API operation Pages methods for examples how the SDK will @@ -146,7 +148,7 @@ func (r *Request) nextPageTokens() []interface{} { return nil } case bool: - if v == false { + if !v { return nil } } diff --git a/github.com/aws/aws-sdk-go/aws/request/retryer.go b/github.com/aws/aws-sdk-go/aws/request/retryer.go index 7bc5da7826..752ae47f84 100644 --- a/github.com/aws/aws-sdk-go/aws/request/retryer.go +++ b/github.com/aws/aws-sdk-go/aws/request/retryer.go @@ -1,32 +1,81 @@ package request import ( + "net" + "net/url" + "strings" "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" ) -// Retryer is an interface to control retry logic for a given service. -// The default implementation used by most services is the client.DefaultRetryer -// structure, which contains basic retry logic using exponential backoff. +// Retryer provides the interface drive the SDK's request retry behavior. The +// Retryer implementation is responsible for implementing exponential backoff, +// and determine if a request API error should be retried. +// +// client.DefaultRetryer is the SDK's default implementation of the Retryer. It +// uses the which uses the Request.IsErrorRetryable and Request.IsErrorThrottle +// methods to determine if the request is retried. type Retryer interface { + // RetryRules return the retry delay that should be used by the SDK before + // making another request attempt for the failed request. RetryRules(*Request) time.Duration + + // ShouldRetry returns if the failed request is retryable. + // + // Implementations may consider request attempt count when determining if a + // request is retryable, but the SDK will use MaxRetries to limit the + // number of attempts a request are made. ShouldRetry(*Request) bool + + // MaxRetries is the number of times a request may be retried before + // failing. MaxRetries() int } -// WithRetryer sets a config Retryer value to the given Config returning it -// for chaining. +// WithRetryer sets a Retryer value to the given Config returning the Config +// value for chaining. The value must not be nil. func WithRetryer(cfg *aws.Config, retryer Retryer) *aws.Config { + if retryer == nil { + if cfg.Logger != nil { + cfg.Logger.Log("ERROR: Request.WithRetryer called with nil retryer. Replacing with retry disabled Retryer.") + } + retryer = noOpRetryer{} + } cfg.Retryer = retryer return cfg + +} + +// noOpRetryer is a internal no op retryer used when a request is created +// without a retryer. +// +// Provides a retryer that performs no retries. +// It should be used when we do not want retries to be performed. +type noOpRetryer struct{} + +// MaxRetries returns the number of maximum returns the service will use to make +// an individual API; For NoOpRetryer the MaxRetries will always be zero. +func (d noOpRetryer) MaxRetries() int { + return 0 +} + +// ShouldRetry will always return false for NoOpRetryer, as it should never retry. +func (d noOpRetryer) ShouldRetry(_ *Request) bool { + return false +} + +// RetryRules returns the delay duration before retrying this request again; +// since NoOpRetryer does not retry, RetryRules always returns 0. +func (d noOpRetryer) RetryRules(_ *Request) time.Duration { + return 0 } // retryableCodes is a collection of service response codes which are retry-able // without any further action. var retryableCodes = map[string]struct{}{ - "RequestError": {}, + ErrCodeRequestError: {}, "RequestTimeout": {}, ErrCodeResponseTimeout: {}, "RequestTimeoutException": {}, // Glacier's flavor of RequestTimeout @@ -34,13 +83,16 @@ var retryableCodes = map[string]struct{}{ var throttleCodes = map[string]struct{}{ "ProvisionedThroughputExceededException": {}, + "ThrottledException": {}, // SNS, XRay, ResourceGroupsTagging API "Throttling": {}, "ThrottlingException": {}, "RequestLimitExceeded": {}, "RequestThrottled": {}, + "RequestThrottledException": {}, "TooManyRequestsException": {}, // Lambda functions "PriorRequestNotComplete": {}, // Route53 "TransactionInProgressException": {}, + "EC2ThrottledException": {}, // EC2 } // credsExpiredCodes is a collection of error codes which signify the credentials @@ -75,10 +127,6 @@ var validParentCodes = map[string]struct{}{ ErrCodeRead: {}, } -type temporaryError interface { - Temporary() bool -} - func isNestedErrorRetryable(parentErr awserr.Error) bool { if parentErr == nil { return false @@ -97,7 +145,7 @@ func isNestedErrorRetryable(parentErr awserr.Error) bool { return isCodeRetryable(aerr.Code()) } - if t, ok := err.(temporaryError); ok { + if t, ok := err.(temporary); ok { return t.Temporary() || isErrConnectionReset(err) } @@ -107,32 +155,90 @@ func isNestedErrorRetryable(parentErr awserr.Error) bool { // IsErrorRetryable returns whether the error is retryable, based on its Code. // Returns false if error is nil. func IsErrorRetryable(err error) bool { - if err != nil { - if aerr, ok := err.(awserr.Error); ok { - return isCodeRetryable(aerr.Code()) || isNestedErrorRetryable(aerr) + if err == nil { + return false + } + return shouldRetryError(err) +} + +type temporary interface { + Temporary() bool +} + +func shouldRetryError(origErr error) bool { + switch err := origErr.(type) { + case awserr.Error: + if err.Code() == CanceledErrorCode { + return false } + if isNestedErrorRetryable(err) { + return true + } + + origErr := err.OrigErr() + var shouldRetry bool + if origErr != nil { + shouldRetry = shouldRetryError(origErr) + if err.Code() == ErrCodeRequestError && !shouldRetry { + return false + } + } + if isCodeRetryable(err.Code()) { + return true + } + return shouldRetry + + case *url.Error: + if strings.Contains(err.Error(), "connection refused") { + // Refused connections should be retried as the service may not yet + // be running on the port. Go TCP dial considers refused + // connections as not temporary. + return true + } + // *url.Error only implements Temporary after golang 1.6 but since + // url.Error only wraps the error: + return shouldRetryError(err.Err) + + case temporary: + if netErr, ok := err.(*net.OpError); ok && netErr.Op == "dial" { + return true + } + // If the error is temporary, we want to allow continuation of the + // retry process + return err.Temporary() || isErrConnectionReset(origErr) + + case nil: + // `awserr.Error.OrigErr()` can be nil, meaning there was an error but + // because we don't know the cause, it is marked as retryable. See + // TestRequest4xxUnretryable for an example. + return true + + default: + switch err.Error() { + case "net/http: request canceled", + "net/http: request canceled while waiting for connection": + // known 1.5 error case when an http request is cancelled + return false + } + // here we don't know the error; so we allow a retry. + return true } - return false } // IsErrorThrottle returns whether the error is to be throttled based on its code. // Returns false if error is nil. func IsErrorThrottle(err error) bool { - if err != nil { - if aerr, ok := err.(awserr.Error); ok { - return isCodeThrottle(aerr.Code()) - } + if aerr, ok := err.(awserr.Error); ok && aerr != nil { + return isCodeThrottle(aerr.Code()) } return false } -// IsErrorExpiredCreds returns whether the error code is a credential expiry error. -// Returns false if error is nil. +// IsErrorExpiredCreds returns whether the error code is a credential expiry +// error. Returns false if error is nil. func IsErrorExpiredCreds(err error) bool { - if err != nil { - if aerr, ok := err.(awserr.Error); ok { - return isCodeExpiredCreds(aerr.Code()) - } + if aerr, ok := err.(awserr.Error); ok && aerr != nil { + return isCodeExpiredCreds(aerr.Code()) } return false } @@ -142,17 +248,58 @@ func IsErrorExpiredCreds(err error) bool { // // Alias for the utility function IsErrorRetryable func (r *Request) IsErrorRetryable() bool { + if isErrCode(r.Error, r.RetryErrorCodes) { + return true + } + + // HTTP response status code 501 should not be retried. + // 501 represents Not Implemented which means the request method is not + // supported by the server and cannot be handled. + if r.HTTPResponse != nil { + // HTTP response status code 500 represents internal server error and + // should be retried without any throttle. + if r.HTTPResponse.StatusCode == 500 { + return true + } + } return IsErrorRetryable(r.Error) } -// IsErrorThrottle returns whether the error is to be throttled based on its code. -// Returns false if the request has no Error set +// IsErrorThrottle returns whether the error is to be throttled based on its +// code. Returns false if the request has no Error set. // // Alias for the utility function IsErrorThrottle func (r *Request) IsErrorThrottle() bool { + if isErrCode(r.Error, r.ThrottleErrorCodes) { + return true + } + + if r.HTTPResponse != nil { + switch r.HTTPResponse.StatusCode { + case + 429, // error caused due to too many requests + 502, // Bad Gateway error should be throttled + 503, // caused when service is unavailable + 504: // error occurred due to gateway timeout + return true + } + } + return IsErrorThrottle(r.Error) } +func isErrCode(err error, codes []string) bool { + if aerr, ok := err.(awserr.Error); ok && aerr != nil { + for _, code := range codes { + if code == aerr.Code() { + return true + } + } + } + + return false +} + // IsErrorExpired returns whether the error code is a credential expiry error. // Returns false if the request has no Error set. // diff --git a/github.com/aws/aws-sdk-go/aws/session/cabundle_transport.go b/github.com/aws/aws-sdk-go/aws/session/cabundle_transport.go new file mode 100644 index 0000000000..ea9ebb6f6a --- /dev/null +++ b/github.com/aws/aws-sdk-go/aws/session/cabundle_transport.go @@ -0,0 +1,26 @@ +// +build go1.7 + +package session + +import ( + "net" + "net/http" + "time" +) + +// Transport that should be used when a custom CA bundle is specified with the +// SDK. +func getCABundleTransport() *http.Transport { + return &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } +} diff --git a/github.com/aws/aws-sdk-go/aws/session/cabundle_transport_1_5.go b/github.com/aws/aws-sdk-go/aws/session/cabundle_transport_1_5.go new file mode 100644 index 0000000000..fec39dfc12 --- /dev/null +++ b/github.com/aws/aws-sdk-go/aws/session/cabundle_transport_1_5.go @@ -0,0 +1,22 @@ +// +build !go1.6,go1.5 + +package session + +import ( + "net" + "net/http" + "time" +) + +// Transport that should be used when a custom CA bundle is specified with the +// SDK. +func getCABundleTransport() *http.Transport { + return &http.Transport{ + Proxy: http.ProxyFromEnvironment, + Dial: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).Dial, + TLSHandshakeTimeout: 10 * time.Second, + } +} diff --git a/github.com/aws/aws-sdk-go/aws/session/cabundle_transport_1_6.go b/github.com/aws/aws-sdk-go/aws/session/cabundle_transport_1_6.go new file mode 100644 index 0000000000..1c5a5391e6 --- /dev/null +++ b/github.com/aws/aws-sdk-go/aws/session/cabundle_transport_1_6.go @@ -0,0 +1,23 @@ +// +build !go1.7,go1.6 + +package session + +import ( + "net" + "net/http" + "time" +) + +// Transport that should be used when a custom CA bundle is specified with the +// SDK. +func getCABundleTransport() *http.Transport { + return &http.Transport{ + Proxy: http.ProxyFromEnvironment, + Dial: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).Dial, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } +} diff --git a/github.com/aws/aws-sdk-go/aws/session/credentials.go b/github.com/aws/aws-sdk-go/aws/session/credentials.go new file mode 100644 index 0000000000..fe6dac1f47 --- /dev/null +++ b/github.com/aws/aws-sdk-go/aws/session/credentials.go @@ -0,0 +1,267 @@ +package session + +import ( + "fmt" + "os" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/credentials/processcreds" + "github.com/aws/aws-sdk-go/aws/credentials/stscreds" + "github.com/aws/aws-sdk-go/aws/defaults" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/internal/shareddefaults" +) + +func resolveCredentials(cfg *aws.Config, + envCfg envConfig, sharedCfg sharedConfig, + handlers request.Handlers, + sessOpts Options, +) (*credentials.Credentials, error) { + + switch { + case len(sessOpts.Profile) != 0: + // User explicitly provided an Profile in the session's configuration + // so load that profile from shared config first. + // Github(aws/aws-sdk-go#2727) + return resolveCredsFromProfile(cfg, envCfg, sharedCfg, handlers, sessOpts) + + case envCfg.Creds.HasKeys(): + // Environment credentials + return credentials.NewStaticCredentialsFromCreds(envCfg.Creds), nil + + case len(envCfg.WebIdentityTokenFilePath) != 0: + // Web identity token from environment, RoleARN required to also be + // set. + return assumeWebIdentity(cfg, handlers, + envCfg.WebIdentityTokenFilePath, + envCfg.RoleARN, + envCfg.RoleSessionName, + ) + + default: + // Fallback to the "default" credential resolution chain. + return resolveCredsFromProfile(cfg, envCfg, sharedCfg, handlers, sessOpts) + } +} + +// WebIdentityEmptyRoleARNErr will occur if 'AWS_WEB_IDENTITY_TOKEN_FILE' was set but +// 'AWS_ROLE_ARN' was not set. +var WebIdentityEmptyRoleARNErr = awserr.New(stscreds.ErrCodeWebIdentity, "role ARN is not set", nil) + +// WebIdentityEmptyTokenFilePathErr will occur if 'AWS_ROLE_ARN' was set but +// 'AWS_WEB_IDENTITY_TOKEN_FILE' was not set. +var WebIdentityEmptyTokenFilePathErr = awserr.New(stscreds.ErrCodeWebIdentity, "token file path is not set", nil) + +func assumeWebIdentity(cfg *aws.Config, handlers request.Handlers, + filepath string, + roleARN, sessionName string, +) (*credentials.Credentials, error) { + + if len(filepath) == 0 { + return nil, WebIdentityEmptyTokenFilePathErr + } + + if len(roleARN) == 0 { + return nil, WebIdentityEmptyRoleARNErr + } + + creds := stscreds.NewWebIdentityCredentials( + &Session{ + Config: cfg, + Handlers: handlers.Copy(), + }, + roleARN, + sessionName, + filepath, + ) + + return creds, nil +} + +func resolveCredsFromProfile(cfg *aws.Config, + envCfg envConfig, sharedCfg sharedConfig, + handlers request.Handlers, + sessOpts Options, +) (creds *credentials.Credentials, err error) { + + switch { + case sharedCfg.SourceProfile != nil: + // Assume IAM role with credentials source from a different profile. + creds, err = resolveCredsFromProfile(cfg, envCfg, + *sharedCfg.SourceProfile, handlers, sessOpts, + ) + + case sharedCfg.Creds.HasKeys(): + // Static Credentials from Shared Config/Credentials file. + creds = credentials.NewStaticCredentialsFromCreds( + sharedCfg.Creds, + ) + + case len(sharedCfg.CredentialProcess) != 0: + // Get credentials from CredentialProcess + creds = processcreds.NewCredentials(sharedCfg.CredentialProcess) + + case len(sharedCfg.CredentialSource) != 0: + creds, err = resolveCredsFromSource(cfg, envCfg, + sharedCfg, handlers, sessOpts, + ) + + case len(sharedCfg.WebIdentityTokenFile) != 0: + // Credentials from Assume Web Identity token require an IAM Role, and + // that roll will be assumed. May be wrapped with another assume role + // via SourceProfile. + return assumeWebIdentity(cfg, handlers, + sharedCfg.WebIdentityTokenFile, + sharedCfg.RoleARN, + sharedCfg.RoleSessionName, + ) + + default: + // Fallback to default credentials provider, include mock errors for + // the credential chain so user can identify why credentials failed to + // be retrieved. + creds = credentials.NewCredentials(&credentials.ChainProvider{ + VerboseErrors: aws.BoolValue(cfg.CredentialsChainVerboseErrors), + Providers: []credentials.Provider{ + &credProviderError{ + Err: awserr.New("EnvAccessKeyNotFound", + "failed to find credentials in the environment.", nil), + }, + &credProviderError{ + Err: awserr.New("SharedCredsLoad", + fmt.Sprintf("failed to load profile, %s.", envCfg.Profile), nil), + }, + defaults.RemoteCredProvider(*cfg, handlers), + }, + }) + } + if err != nil { + return nil, err + } + + if len(sharedCfg.RoleARN) > 0 { + cfgCp := *cfg + cfgCp.Credentials = creds + return credsFromAssumeRole(cfgCp, handlers, sharedCfg, sessOpts) + } + + return creds, nil +} + +// valid credential source values +const ( + credSourceEc2Metadata = "Ec2InstanceMetadata" + credSourceEnvironment = "Environment" + credSourceECSContainer = "EcsContainer" +) + +func resolveCredsFromSource(cfg *aws.Config, + envCfg envConfig, sharedCfg sharedConfig, + handlers request.Handlers, + sessOpts Options, +) (creds *credentials.Credentials, err error) { + + switch sharedCfg.CredentialSource { + case credSourceEc2Metadata: + p := defaults.RemoteCredProvider(*cfg, handlers) + creds = credentials.NewCredentials(p) + + case credSourceEnvironment: + creds = credentials.NewStaticCredentialsFromCreds(envCfg.Creds) + + case credSourceECSContainer: + if len(os.Getenv(shareddefaults.ECSCredsProviderEnvVar)) == 0 { + return nil, ErrSharedConfigECSContainerEnvVarEmpty + } + + p := defaults.RemoteCredProvider(*cfg, handlers) + creds = credentials.NewCredentials(p) + + default: + return nil, ErrSharedConfigInvalidCredSource + } + + return creds, nil +} + +func credsFromAssumeRole(cfg aws.Config, + handlers request.Handlers, + sharedCfg sharedConfig, + sessOpts Options, +) (*credentials.Credentials, error) { + + if len(sharedCfg.MFASerial) != 0 && sessOpts.AssumeRoleTokenProvider == nil { + // AssumeRole Token provider is required if doing Assume Role + // with MFA. + return nil, AssumeRoleTokenProviderNotSetError{} + } + + return stscreds.NewCredentials( + &Session{ + Config: &cfg, + Handlers: handlers.Copy(), + }, + sharedCfg.RoleARN, + func(opt *stscreds.AssumeRoleProvider) { + opt.RoleSessionName = sharedCfg.RoleSessionName + + if sessOpts.AssumeRoleDuration == 0 && + sharedCfg.AssumeRoleDuration != nil && + *sharedCfg.AssumeRoleDuration/time.Minute > 15 { + opt.Duration = *sharedCfg.AssumeRoleDuration + } else if sessOpts.AssumeRoleDuration != 0 { + opt.Duration = sessOpts.AssumeRoleDuration + } + + // Assume role with external ID + if len(sharedCfg.ExternalID) > 0 { + opt.ExternalID = aws.String(sharedCfg.ExternalID) + } + + // Assume role with MFA + if len(sharedCfg.MFASerial) > 0 { + opt.SerialNumber = aws.String(sharedCfg.MFASerial) + opt.TokenProvider = sessOpts.AssumeRoleTokenProvider + } + }, + ), nil +} + +// AssumeRoleTokenProviderNotSetError is an error returned when creating a +// session when the MFAToken option is not set when shared config is configured +// load assume a role with an MFA token. +type AssumeRoleTokenProviderNotSetError struct{} + +// Code is the short id of the error. +func (e AssumeRoleTokenProviderNotSetError) Code() string { + return "AssumeRoleTokenProviderNotSetError" +} + +// Message is the description of the error +func (e AssumeRoleTokenProviderNotSetError) Message() string { + return fmt.Sprintf("assume role with MFA enabled, but AssumeRoleTokenProvider session option not set.") +} + +// OrigErr is the underlying error that caused the failure. +func (e AssumeRoleTokenProviderNotSetError) OrigErr() error { + return nil +} + +// Error satisfies the error interface. +func (e AssumeRoleTokenProviderNotSetError) Error() string { + return awserr.SprintError(e.Code(), e.Message(), "", nil) +} + +type credProviderError struct { + Err error +} + +func (c credProviderError) Retrieve() (credentials.Value, error) { + return credentials.Value{}, c.Err +} +func (c credProviderError) IsExpired() bool { + return true +} diff --git a/github.com/aws/aws-sdk-go/aws/session/doc.go b/github.com/aws/aws-sdk-go/aws/session/doc.go index 38a7b05a62..7ec66e7e58 100644 --- a/github.com/aws/aws-sdk-go/aws/session/doc.go +++ b/github.com/aws/aws-sdk-go/aws/session/doc.go @@ -1,97 +1,93 @@ /* -Package session provides configuration for the SDK's service clients. - -Sessions can be shared across all service clients that share the same base -configuration. The Session is built from the SDK's default configuration and -request handlers. - -Sessions should be cached when possible, because creating a new Session will -load all configuration values from the environment, and config files each time -the Session is created. Sharing the Session value across all of your service -clients will ensure the configuration is loaded the fewest number of times possible. - -Concurrency +Package session provides configuration for the SDK's service clients. Sessions +can be shared across service clients that share the same base configuration. Sessions are safe to use concurrently as long as the Session is not being -modified. The SDK will not modify the Session once the Session has been created. -Creating service clients concurrently from a shared Session is safe. - -Sessions from Shared Config - -Sessions can be created using the method above that will only load the -additional config if the AWS_SDK_LOAD_CONFIG environment variable is set. -Alternatively you can explicitly create a Session with shared config enabled. -To do this you can use NewSessionWithOptions to configure how the Session will -be created. Using the NewSessionWithOptions with SharedConfigState set to -SharedConfigEnable will create the session as if the AWS_SDK_LOAD_CONFIG -environment variable was set. +modified. Sessions should be cached when possible, because creating a new +Session will load all configuration values from the environment, and config +files each time the Session is created. Sharing the Session value across all of +your service clients will ensure the configuration is loaded the fewest number +of times possible. -Creating Sessions - -When creating Sessions optional aws.Config values can be passed in that will -override the default, or loaded config values the Session is being created -with. This allows you to provide additional, or case based, configuration -as needed. +Sessions options from Shared Config By default NewSession will only load credentials from the shared credentials file (~/.aws/credentials). If the AWS_SDK_LOAD_CONFIG environment variable is set to a truthy value the Session will be created from the configuration values from the shared config (~/.aws/config) and shared credentials -(~/.aws/credentials) files. See the section Sessions from Shared Config for -more information. +(~/.aws/credentials) files. Using the NewSessionWithOptions with +SharedConfigState set to SharedConfigEnable will create the session as if the +AWS_SDK_LOAD_CONFIG environment variable was set. -Create a Session with the default config and request handlers. With credentials -region, and profile loaded from the environment and shared config automatically. -Requires the AWS_PROFILE to be set, or "default" is used. +Credential and config loading order - // Create Session - sess := session.Must(session.NewSession()) +The Session will attempt to load configuration and credentials from the +environment, configuration files, and other credential sources. The order +configuration is loaded in is: - // Create a Session with a custom region - sess := session.Must(session.NewSession(&aws.Config{ - Region: aws.String("us-east-1"), - })) + * Environment Variables + * Shared Credentials file + * Shared Configuration file (if SharedConfig is enabled) + * EC2 Instance Metadata (credentials only) - // Create a S3 client instance from a session - sess := session.Must(session.NewSession()) +The Environment variables for credentials will have precedence over shared +config even if SharedConfig is enabled. To override this behavior, and use +shared config credentials instead specify the session.Options.Profile, (e.g. +when using credential_source=Environment to assume a role). + + sess, err := session.NewSessionWithOptions(session.Options{ + Profile: "myProfile", + }) - svc := s3.New(sess) +Creating Sessions -Create Session With Option Overrides +Creating a Session without additional options will load credentials region, and +profile loaded from the environment and shared config automatically. See, +"Environment Variables" section for information on environment variables used +by Session. -In addition to NewSession, Sessions can be created using NewSessionWithOptions. -This func allows you to control and override how the Session will be created -through code instead of being driven by environment variables only. + // Create Session + sess, err := session.NewSession() -Use NewSessionWithOptions when you want to provide the config profile, or -override the shared config state (AWS_SDK_LOAD_CONFIG). + +When creating Sessions optional aws.Config values can be passed in that will +override the default, or loaded, config values the Session is being created +with. This allows you to provide additional, or case based, configuration +as needed. + + // Create a Session with a custom region + sess, err := session.NewSession(&aws.Config{ + Region: aws.String("us-west-2"), + }) + +Use NewSessionWithOptions to provide additional configuration driving how the +Session's configuration will be loaded. Such as, specifying shared config +profile, or override the shared config state, (AWS_SDK_LOAD_CONFIG). // Equivalent to session.NewSession() - sess := session.Must(session.NewSessionWithOptions(session.Options{ + sess, err := session.NewSessionWithOptions(session.Options{ // Options - })) + }) - // Specify profile to load for the session's config - sess := session.Must(session.NewSessionWithOptions(session.Options{ - Profile: "profile_name", - })) + sess, err := session.NewSessionWithOptions(session.Options{ + // Specify profile to load for the session's config + Profile: "profile_name", - // Specify profile for config and region for requests - sess := session.Must(session.NewSessionWithOptions(session.Options{ - Config: aws.Config{Region: aws.String("us-east-1")}, - Profile: "profile_name", - })) + // Provide SDK Config options, such as Region. + Config: aws.Config{ + Region: aws.String("us-west-2"), + }, - // Force enable Shared Config support - sess := session.Must(session.NewSessionWithOptions(session.Options{ + // Force enable Shared Config support SharedConfigState: session.SharedConfigEnable, - })) + }) Adding Handlers -You can add handlers to a session for processing HTTP requests. All service -clients that use the session inherit the handlers. For example, the following -handler logs every request and its payload made by a service client: +You can add handlers to a session to decorate API operation, (e.g. adding HTTP +headers). All clients that use the Session receive a copy of the Session's +handlers. For example, the following request handler added to the Session logs +every requests made. // Create a session, and add additional handlers for all service // clients created with the Session to inherit. Adds logging handler. @@ -99,22 +95,15 @@ handler logs every request and its payload made by a service client: sess.Handlers.Send.PushFront(func(r *request.Request) { // Log every request made and its payload - logger.Printf("Request: %s/%s, Payload: %s", + logger.Printf("Request: %s/%s, Params: %s", r.ClientInfo.ServiceName, r.Operation, r.Params) }) -Deprecated "New" function - -The New session function has been deprecated because it does not provide good -way to return errors that occur when loading the configuration files and values. -Because of this, NewSession was created so errors can be retrieved when -creating a session fails. - Shared Config Fields -By default the SDK will only load the shared credentials file's (~/.aws/credentials) -credentials values, and all other config is provided by the environment variables, -SDK defaults, and user provided aws.Config values. +By default the SDK will only load the shared credentials file's +(~/.aws/credentials) credentials values, and all other config is provided by +the environment variables, SDK defaults, and user provided aws.Config values. If the AWS_SDK_LOAD_CONFIG environment variable is set, or SharedConfigEnable option is used to create the Session the full shared config values will be @@ -125,24 +114,31 @@ files have the same format. If both config files are present the configuration from both files will be read. The Session will be created from configuration values from the shared -credentials file (~/.aws/credentials) over those in the shared config file (~/.aws/config). +credentials file (~/.aws/credentials) over those in the shared config file +(~/.aws/config). -Credentials are the values the SDK should use for authenticating requests with -AWS Services. They are from a configuration file will need to include both -aws_access_key_id and aws_secret_access_key must be provided together in the -same file to be considered valid. The values will be ignored if not a complete -group. aws_session_token is an optional field that can be provided if both of -the other two fields are also provided. +Credentials are the values the SDK uses to authenticating requests with AWS +Services. When specified in a file, both aws_access_key_id and +aws_secret_access_key must be provided together in the same file to be +considered valid. They will be ignored if both are not present. +aws_session_token is an optional field that can be provided in addition to the +other two fields. aws_access_key_id = AKID aws_secret_access_key = SECRET aws_session_token = TOKEN -Assume Role values allow you to configure the SDK to assume an IAM role using -a set of credentials provided in a config file via the source_profile field. -Both "role_arn" and "source_profile" are required. The SDK supports assuming -a role with MFA token if the session option AssumeRoleTokenProvider -is set. + ; region only supported if SharedConfigEnabled. + region = us-east-1 + +Assume Role configuration + +The role_arn field allows you to configure the SDK to assume an IAM role using +a set of credentials from another source. Such as when paired with static +credentials, "profile_source", "credential_process", or "credential_source" +fields. If "role_arn" is provided, a source of credentials must also be +specified, such as "source_profile", "credential_source", or +"credential_process". role_arn = arn:aws:iam:::role/ source_profile = profile_with_creds @@ -150,40 +146,16 @@ is set. mfa_serial = role_session_name = session_name -Region is the region the SDK should use for looking up AWS service endpoints -and signing requests. - - region = us-east-1 - -Assume Role with MFA token -To create a session with support for assuming an IAM role with MFA set the -session option AssumeRoleTokenProvider to a function that will prompt for the -MFA token code when the SDK assumes the role and refreshes the role's credentials. -This allows you to configure the SDK via the shared config to assumea role -with MFA tokens. - -In order for the SDK to assume a role with MFA the SharedConfigState -session option must be set to SharedConfigEnable, or AWS_SDK_LOAD_CONFIG -environment variable set. - -The shared configuration instructs the SDK to assume an IAM role with MFA -when the mfa_serial configuration field is set in the shared config -(~/.aws/config) or shared credentials (~/.aws/credentials) file. - -If mfa_serial is set in the configuration, the SDK will assume the role, and -the AssumeRoleTokenProvider session option is not set an an error will -be returned when creating the session. +The SDK supports assuming a role with MFA token. If "mfa_serial" is set, you +must also set the Session Option.AssumeRoleTokenProvider. The Session will fail +to load if the AssumeRoleTokenProvider is not specified. sess := session.Must(session.NewSessionWithOptions(session.Options{ AssumeRoleTokenProvider: stscreds.StdinTokenProvider, })) - // Create service client value configured for credentials - // from assumed role. - svc := s3.New(sess) - -To setup assume role outside of a session see the stscreds.AssumeRoleProvider +To setup Assume Role outside of a session see the stscreds.AssumeRoleProvider documentation. Environment Variables diff --git a/github.com/aws/aws-sdk-go/aws/session/env_config.go b/github.com/aws/aws-sdk-go/aws/session/env_config.go index e3959b959e..c1e0e9c954 100644 --- a/github.com/aws/aws-sdk-go/aws/session/env_config.go +++ b/github.com/aws/aws-sdk-go/aws/session/env_config.go @@ -1,12 +1,15 @@ package session import ( + "fmt" "os" "strconv" + "strings" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/defaults" + "github.com/aws/aws-sdk-go/aws/endpoints" ) // EnvProviderName provides a name of the provider when config is loaded from environment. @@ -99,21 +102,61 @@ type envConfig struct { CustomCABundle string csmEnabled string - CSMEnabled bool + CSMEnabled *bool CSMPort string + CSMHost string CSMClientID string - enableEndpointDiscovery string // Enables endpoint discovery via environment variables. // // AWS_ENABLE_ENDPOINT_DISCOVERY=true EnableEndpointDiscovery *bool + enableEndpointDiscovery string + + // Specifies the WebIdentity token the SDK should use to assume a role + // with. + // + // AWS_WEB_IDENTITY_TOKEN_FILE=file_path + WebIdentityTokenFilePath string + + // Specifies the IAM role arn to use when assuming an role. + // + // AWS_ROLE_ARN=role_arn + RoleARN string + + // Specifies the IAM role session name to use when assuming a role. + // + // AWS_ROLE_SESSION_NAME=session_name + RoleSessionName string + + // Specifies the STS Regional Endpoint flag for the SDK to resolve the endpoint + // for a service. + // + // AWS_STS_REGIONAL_ENDPOINTS=regional + // This can take value as `regional` or `legacy` + STSRegionalEndpoint endpoints.STSRegionalEndpoint + + // Specifies the S3 Regional Endpoint flag for the SDK to resolve the + // endpoint for a service. + // + // AWS_S3_US_EAST_1_REGIONAL_ENDPOINT=regional + // This can take value as `regional` or `legacy` + S3UsEast1RegionalEndpoint endpoints.S3UsEast1RegionalEndpoint + + // Specifies if the S3 service should allow ARNs to direct the region + // the client's requests are sent to. + // + // AWS_S3_USE_ARN_REGION=true + S3UseARNRegion bool } var ( csmEnabledEnvKey = []string{ "AWS_CSM_ENABLED", } + csmHostEnvKey = []string{ + "AWS_CSM_HOST", + } csmPortEnvKey = []string{ "AWS_CSM_PORT", } @@ -150,6 +193,24 @@ var ( sharedConfigFileEnvKey = []string{ "AWS_CONFIG_FILE", } + webIdentityTokenFilePathEnvKey = []string{ + "AWS_WEB_IDENTITY_TOKEN_FILE", + } + roleARNEnvKey = []string{ + "AWS_ROLE_ARN", + } + roleSessionNameEnvKey = []string{ + "AWS_ROLE_SESSION_NAME", + } + stsRegionalEndpointKey = []string{ + "AWS_STS_REGIONAL_ENDPOINTS", + } + s3UsEast1RegionalEndpoint = []string{ + "AWS_S3_US_EAST_1_REGIONAL_ENDPOINT", + } + s3UseARNRegionEnvKey = []string{ + "AWS_S3_USE_ARN_REGION", + } ) // loadEnvConfig retrieves the SDK's environment configuration. @@ -158,7 +219,7 @@ var ( // If the environment variable `AWS_SDK_LOAD_CONFIG` is set to a truthy value // the shared SDK config will be loaded in addition to the SDK's specific // configuration values. -func loadEnvConfig() envConfig { +func loadEnvConfig() (envConfig, error) { enableSharedConfig, _ := strconv.ParseBool(os.Getenv("AWS_SDK_LOAD_CONFIG")) return envConfigLoad(enableSharedConfig) } @@ -169,30 +230,42 @@ func loadEnvConfig() envConfig { // Loads the shared configuration in addition to the SDK's specific configuration. // This will load the same values as `loadEnvConfig` if the `AWS_SDK_LOAD_CONFIG` // environment variable is set. -func loadSharedEnvConfig() envConfig { +func loadSharedEnvConfig() (envConfig, error) { return envConfigLoad(true) } -func envConfigLoad(enableSharedConfig bool) envConfig { +func envConfigLoad(enableSharedConfig bool) (envConfig, error) { cfg := envConfig{} cfg.EnableSharedConfig = enableSharedConfig - setFromEnvVal(&cfg.Creds.AccessKeyID, credAccessEnvKey) - setFromEnvVal(&cfg.Creds.SecretAccessKey, credSecretEnvKey) - setFromEnvVal(&cfg.Creds.SessionToken, credSessionEnvKey) + // Static environment credentials + var creds credentials.Value + setFromEnvVal(&creds.AccessKeyID, credAccessEnvKey) + setFromEnvVal(&creds.SecretAccessKey, credSecretEnvKey) + setFromEnvVal(&creds.SessionToken, credSessionEnvKey) + if creds.HasKeys() { + // Require logical grouping of credentials + creds.ProviderName = EnvProviderName + cfg.Creds = creds + } + + // Role Metadata + setFromEnvVal(&cfg.RoleARN, roleARNEnvKey) + setFromEnvVal(&cfg.RoleSessionName, roleSessionNameEnvKey) + + // Web identity environment variables + setFromEnvVal(&cfg.WebIdentityTokenFilePath, webIdentityTokenFilePathEnvKey) // CSM environment variables setFromEnvVal(&cfg.csmEnabled, csmEnabledEnvKey) + setFromEnvVal(&cfg.CSMHost, csmHostEnvKey) setFromEnvVal(&cfg.CSMPort, csmPortEnvKey) setFromEnvVal(&cfg.CSMClientID, csmClientIDEnvKey) - cfg.CSMEnabled = len(cfg.csmEnabled) > 0 - // Require logical grouping of credentials - if len(cfg.Creds.AccessKeyID) == 0 || len(cfg.Creds.SecretAccessKey) == 0 { - cfg.Creds = credentials.Value{} - } else { - cfg.Creds.ProviderName = EnvProviderName + if len(cfg.csmEnabled) != 0 { + v, _ := strconv.ParseBool(cfg.csmEnabled) + cfg.CSMEnabled = &v } regionKeys := regionEnvKeys @@ -223,12 +296,48 @@ func envConfigLoad(enableSharedConfig bool) envConfig { cfg.CustomCABundle = os.Getenv("AWS_CA_BUNDLE") - return cfg + var err error + // STS Regional Endpoint variable + for _, k := range stsRegionalEndpointKey { + if v := os.Getenv(k); len(v) != 0 { + cfg.STSRegionalEndpoint, err = endpoints.GetSTSRegionalEndpoint(v) + if err != nil { + return cfg, fmt.Errorf("failed to load, %v from env config, %v", k, err) + } + } + } + + // S3 Regional Endpoint variable + for _, k := range s3UsEast1RegionalEndpoint { + if v := os.Getenv(k); len(v) != 0 { + cfg.S3UsEast1RegionalEndpoint, err = endpoints.GetS3UsEast1RegionalEndpoint(v) + if err != nil { + return cfg, fmt.Errorf("failed to load, %v from env config, %v", k, err) + } + } + } + + var s3UseARNRegion string + setFromEnvVal(&s3UseARNRegion, s3UseARNRegionEnvKey) + if len(s3UseARNRegion) != 0 { + switch { + case strings.EqualFold(s3UseARNRegion, "false"): + cfg.S3UseARNRegion = false + case strings.EqualFold(s3UseARNRegion, "true"): + cfg.S3UseARNRegion = true + default: + return envConfig{}, fmt.Errorf( + "invalid value for environment variable, %s=%s, need true or false", + s3UseARNRegionEnvKey[0], s3UseARNRegion) + } + } + + return cfg, nil } func setFromEnvVal(dst *string, keys []string) { for _, k := range keys { - if v := os.Getenv(k); len(v) > 0 { + if v := os.Getenv(k); len(v) != 0 { *dst = v break } diff --git a/github.com/aws/aws-sdk-go/aws/session/session.go b/github.com/aws/aws-sdk-go/aws/session/session.go index 9bdbafd65c..0ff4996051 100644 --- a/github.com/aws/aws-sdk-go/aws/session/session.go +++ b/github.com/aws/aws-sdk-go/aws/session/session.go @@ -8,19 +8,17 @@ import ( "io/ioutil" "net/http" "os" + "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/client" "github.com/aws/aws-sdk-go/aws/corehandlers" "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/credentials/processcreds" - "github.com/aws/aws-sdk-go/aws/credentials/stscreds" "github.com/aws/aws-sdk-go/aws/csm" "github.com/aws/aws-sdk-go/aws/defaults" "github.com/aws/aws-sdk-go/aws/endpoints" "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/internal/shareddefaults" ) const ( @@ -75,7 +73,7 @@ type Session struct { // func is called instead of waiting to receive an error until a request is made. func New(cfgs ...*aws.Config) *Session { // load initial config from environment - envCfg := loadEnvConfig() + envCfg, envErr := loadEnvConfig() if envCfg.EnableSharedConfig { var cfg aws.Config @@ -95,19 +93,28 @@ func New(cfgs ...*aws.Config) *Session { // Session creation failed, need to report the error and prevent // any requests from succeeding. s = &Session{Config: defaults.Config()} - s.Config.MergeIn(cfgs...) - s.Config.Logger.Log("ERROR:", msg, "Error:", err) - s.Handlers.Validate.PushBack(func(r *request.Request) { - r.Error = err - }) + s.logDeprecatedNewSessionError(msg, err, cfgs) } return s } s := deprecatedNewSession(cfgs...) - if envCfg.CSMEnabled { - enableCSM(&s.Handlers, envCfg.CSMClientID, envCfg.CSMPort, s.Config.Logger) + if envErr != nil { + msg := "failed to load env config" + s.logDeprecatedNewSessionError(msg, envErr, cfgs) + } + + if csmCfg, err := loadCSMConfig(envCfg, []string{}); err != nil { + if l := s.Config.Logger; l != nil { + l.Log(fmt.Sprintf("ERROR: failed to load CSM configuration, %v", err)) + } + } else if csmCfg.Enabled { + err := enableCSM(&s.Handlers, csmCfg, s.Config.Logger) + if err != nil { + msg := "failed to enable CSM" + s.logDeprecatedNewSessionError(msg, err, cfgs) + } } return s @@ -126,7 +133,7 @@ func New(cfgs ...*aws.Config) *Session { // to be built with retrieving credentials with AssumeRole set in the config. // // See the NewSessionWithOptions func for information on how to override or -// control through code how the Session will be created. Such as specifying the +// control through code how the Session will be created, such as specifying the // config profile, and controlling if shared config is enabled or not. func NewSession(cfgs ...*aws.Config) (*Session, error) { opts := Options{} @@ -210,6 +217,12 @@ type Options struct { // the config enables assume role wit MFA via the mfa_serial field. AssumeRoleTokenProvider func() (string, error) + // When the SDK's shared config is configured to assume a role this option + // may be provided to set the expiry duration of the STS credentials. + // Defaults to 15 minutes if not set as documented in the + // stscreds.AssumeRoleProvider. + AssumeRoleDuration time.Duration + // Reader for a custom Credentials Authority (CA) bundle in PEM format that // the SDK will use instead of the default system's root CA bundle. Use this // only if you want to replace the CA bundle the SDK uses for TLS requests. @@ -224,6 +237,12 @@ type Options struct { // to also enable this feature. CustomCABundle session option field has priority // over the AWS_CA_BUNDLE environment variable, and will be used if both are set. CustomCABundle io.Reader + + // The handlers that the session and all API clients will be created with. + // This must be a complete set of handlers. Use the defaults.Handlers() + // function to initialize this value before changing the handlers to be + // used by the SDK. + Handlers request.Handlers } // NewSessionWithOptions returns a new Session created from SDK defaults, config files, @@ -257,13 +276,20 @@ type Options struct { // })) func NewSessionWithOptions(opts Options) (*Session, error) { var envCfg envConfig + var err error if opts.SharedConfigState == SharedConfigEnable { - envCfg = loadSharedEnvConfig() + envCfg, err = loadSharedEnvConfig() + if err != nil { + return nil, fmt.Errorf("failed to load shared config, %v", err) + } } else { - envCfg = loadEnvConfig() + envCfg, err = loadEnvConfig() + if err != nil { + return nil, fmt.Errorf("failed to load environment config, %v", err) + } } - if len(opts.Profile) > 0 { + if len(opts.Profile) != 0 { envCfg.Profile = opts.Profile } @@ -329,27 +355,33 @@ func deprecatedNewSession(cfgs ...*aws.Config) *Session { return s } -func enableCSM(handlers *request.Handlers, clientID string, port string, logger aws.Logger) { - logger.Log("Enabling CSM") - if len(port) == 0 { - port = csm.DefaultPort +func enableCSM(handlers *request.Handlers, cfg csmConfig, logger aws.Logger) error { + if logger != nil { + logger.Log("Enabling CSM") } - r, err := csm.Start(clientID, "127.0.0.1:"+port) + r, err := csm.Start(cfg.ClientID, csm.AddressWithDefaults(cfg.Host, cfg.Port)) if err != nil { - return + return err } r.InjectHandlers(handlers) + + return nil } func newSession(opts Options, envCfg envConfig, cfgs ...*aws.Config) (*Session, error) { cfg := defaults.Config() - handlers := defaults.Handlers() + + handlers := opts.Handlers + if handlers.IsEmpty() { + handlers = defaults.Handlers() + } // Get a merged version of the user provided config to determine if // credentials were. userCfg := &aws.Config{} userCfg.MergeIn(cfgs...) + cfg.MergeIn(userCfg) // Ordered config files will be loaded in with later files overwriting // previous config file values. @@ -366,9 +398,17 @@ func newSession(opts Options, envCfg envConfig, cfgs ...*aws.Config) (*Session, } // Load additional config from file(s) - sharedCfg, err := loadSharedConfig(envCfg.Profile, cfgFiles) + sharedCfg, err := loadSharedConfig(envCfg.Profile, cfgFiles, envCfg.EnableSharedConfig) if err != nil { - return nil, err + if len(envCfg.Profile) == 0 && !envCfg.EnableSharedConfig && (envCfg.Creds.HasKeys() || userCfg.Credentials != nil) { + // Special case where the user has not explicitly specified an AWS_PROFILE, + // or session.Options.profile, shared config is not enabled, and the + // environment has credentials, allow the shared config file to fail to + // load since the user has already provided credentials, and nothing else + // is required to be read file. Github(aws/aws-sdk-go#2455) + } else if _, ok := err.(SharedConfigProfileNotExistsError); !ok { + return nil, err + } } if err := mergeConfigSrcs(cfg, userCfg, envCfg, sharedCfg, handlers, opts); err != nil { @@ -381,8 +421,16 @@ func newSession(opts Options, envCfg envConfig, cfgs ...*aws.Config) (*Session, } initHandlers(s) - if envCfg.CSMEnabled { - enableCSM(&s.Handlers, envCfg.CSMClientID, envCfg.CSMPort, s.Config.Logger) + + if csmCfg, err := loadCSMConfig(envCfg, cfgFiles); err != nil { + if l := s.Config.Logger; l != nil { + l.Log(fmt.Sprintf("ERROR: failed to load CSM configuration, %v", err)) + } + } else if csmCfg.Enabled { + err = enableCSM(&s.Handlers, csmCfg, s.Config.Logger) + if err != nil { + return nil, err + } } // Setup HTTP client with custom cert bundle if enabled @@ -395,6 +443,46 @@ func newSession(opts Options, envCfg envConfig, cfgs ...*aws.Config) (*Session, return s, nil } +type csmConfig struct { + Enabled bool + Host string + Port string + ClientID string +} + +var csmProfileName = "aws_csm" + +func loadCSMConfig(envCfg envConfig, cfgFiles []string) (csmConfig, error) { + if envCfg.CSMEnabled != nil { + if *envCfg.CSMEnabled { + return csmConfig{ + Enabled: true, + ClientID: envCfg.CSMClientID, + Host: envCfg.CSMHost, + Port: envCfg.CSMPort, + }, nil + } + return csmConfig{}, nil + } + + sharedCfg, err := loadSharedConfig(csmProfileName, cfgFiles, false) + if err != nil { + if _, ok := err.(SharedConfigProfileNotExistsError); !ok { + return csmConfig{}, err + } + } + if sharedCfg.CSMEnabled != nil && *sharedCfg.CSMEnabled == true { + return csmConfig{ + Enabled: true, + ClientID: sharedCfg.CSMClientID, + Host: sharedCfg.CSMHost, + Port: sharedCfg.CSMPort, + }, nil + } + + return csmConfig{}, nil +} + func loadCustomCABundle(s *Session, bundle io.Reader) error { var t *http.Transport switch v := s.Config.HTTPClient.Transport.(type) { @@ -407,7 +495,10 @@ func loadCustomCABundle(s *Session, bundle io.Reader) error { } } if t == nil { - t = &http.Transport{} + // Nil transport implies `http.DefaultTransport` should be used. Since + // the SDK cannot modify, nor copy the `DefaultTransport` specifying + // the values the next closest behavior. + t = getCABundleTransport() } p, err := loadCertPool(bundle) @@ -440,9 +531,11 @@ func loadCertPool(r io.Reader) (*x509.CertPool, error) { return p, nil } -func mergeConfigSrcs(cfg, userCfg *aws.Config, envCfg envConfig, sharedCfg sharedConfig, handlers request.Handlers, sessOpts Options) error { - // Merge in user provided configuration - cfg.MergeIn(userCfg) +func mergeConfigSrcs(cfg, userCfg *aws.Config, + envCfg envConfig, sharedCfg sharedConfig, + handlers request.Handlers, + sessOpts Options, +) error { // Region if not already set by user if len(aws.StringValue(cfg.Region)) == 0 { @@ -461,162 +554,59 @@ func mergeConfigSrcs(cfg, userCfg *aws.Config, envCfg envConfig, sharedCfg share } } - // Configure credentials if not already set + // Regional Endpoint flag for STS endpoint resolving + mergeSTSRegionalEndpointConfig(cfg, []endpoints.STSRegionalEndpoint{ + userCfg.STSRegionalEndpoint, + envCfg.STSRegionalEndpoint, + sharedCfg.STSRegionalEndpoint, + endpoints.LegacySTSEndpoint, + }) + + // Regional Endpoint flag for S3 endpoint resolving + mergeS3UsEast1RegionalEndpointConfig(cfg, []endpoints.S3UsEast1RegionalEndpoint{ + userCfg.S3UsEast1RegionalEndpoint, + envCfg.S3UsEast1RegionalEndpoint, + sharedCfg.S3UsEast1RegionalEndpoint, + endpoints.LegacyS3UsEast1Endpoint, + }) + + // Configure credentials if not already set by the user when creating the + // Session. if cfg.Credentials == credentials.AnonymousCredentials && userCfg.Credentials == nil { - - // inspect the profile to see if a credential source has been specified. - if envCfg.EnableSharedConfig && len(sharedCfg.AssumeRole.CredentialSource) > 0 { - - // if both credential_source and source_profile have been set, return an error - // as this is undefined behavior. - if len(sharedCfg.AssumeRole.SourceProfile) > 0 { - return ErrSharedConfigSourceCollision - } - - // valid credential source values - const ( - credSourceEc2Metadata = "Ec2InstanceMetadata" - credSourceEnvironment = "Environment" - credSourceECSContainer = "EcsContainer" - ) - - switch sharedCfg.AssumeRole.CredentialSource { - case credSourceEc2Metadata: - cfgCp := *cfg - p := defaults.RemoteCredProvider(cfgCp, handlers) - cfgCp.Credentials = credentials.NewCredentials(p) - - if len(sharedCfg.AssumeRole.MFASerial) > 0 && sessOpts.AssumeRoleTokenProvider == nil { - // AssumeRole Token provider is required if doing Assume Role - // with MFA. - return AssumeRoleTokenProviderNotSetError{} - } - - cfg.Credentials = assumeRoleCredentials(cfgCp, handlers, sharedCfg, sessOpts) - case credSourceEnvironment: - cfg.Credentials = credentials.NewStaticCredentialsFromCreds( - envCfg.Creds, - ) - case credSourceECSContainer: - if len(os.Getenv(shareddefaults.ECSCredsProviderEnvVar)) == 0 { - return ErrSharedConfigECSContainerEnvVarEmpty - } - - cfgCp := *cfg - p := defaults.RemoteCredProvider(cfgCp, handlers) - creds := credentials.NewCredentials(p) - - cfg.Credentials = creds - default: - return ErrSharedConfigInvalidCredSource - } - - return nil - } - - if len(envCfg.Creds.AccessKeyID) > 0 { - cfg.Credentials = credentials.NewStaticCredentialsFromCreds( - envCfg.Creds, - ) - } else if envCfg.EnableSharedConfig && len(sharedCfg.AssumeRole.RoleARN) > 0 && sharedCfg.AssumeRoleSource != nil { - cfgCp := *cfg - cfgCp.Credentials = credentials.NewStaticCredentialsFromCreds( - sharedCfg.AssumeRoleSource.Creds, - ) - - if len(sharedCfg.AssumeRole.MFASerial) > 0 && sessOpts.AssumeRoleTokenProvider == nil { - // AssumeRole Token provider is required if doing Assume Role - // with MFA. - return AssumeRoleTokenProviderNotSetError{} - } - - cfg.Credentials = assumeRoleCredentials(cfgCp, handlers, sharedCfg, sessOpts) - } else if len(sharedCfg.Creds.AccessKeyID) > 0 { - cfg.Credentials = credentials.NewStaticCredentialsFromCreds( - sharedCfg.Creds, - ) - } else if len(sharedCfg.CredentialProcess) > 0 { - cfg.Credentials = processcreds.NewCredentials( - sharedCfg.CredentialProcess, - ) - } else { - // Fallback to default credentials provider, include mock errors - // for the credential chain so user can identify why credentials - // failed to be retrieved. - cfg.Credentials = credentials.NewCredentials(&credentials.ChainProvider{ - VerboseErrors: aws.BoolValue(cfg.CredentialsChainVerboseErrors), - Providers: []credentials.Provider{ - &credProviderError{Err: awserr.New("EnvAccessKeyNotFound", "failed to find credentials in the environment.", nil)}, - &credProviderError{Err: awserr.New("SharedCredsLoad", fmt.Sprintf("failed to load profile, %s.", envCfg.Profile), nil)}, - defaults.RemoteCredProvider(*cfg, handlers), - }, - }) + creds, err := resolveCredentials(cfg, envCfg, sharedCfg, handlers, sessOpts) + if err != nil { + return err } + cfg.Credentials = creds } - return nil -} - -func assumeRoleCredentials(cfg aws.Config, handlers request.Handlers, sharedCfg sharedConfig, sessOpts Options) *credentials.Credentials { - return stscreds.NewCredentials( - &Session{ - Config: &cfg, - Handlers: handlers.Copy(), - }, - sharedCfg.AssumeRole.RoleARN, - func(opt *stscreds.AssumeRoleProvider) { - opt.RoleSessionName = sharedCfg.AssumeRole.RoleSessionName - - // Assume role with external ID - if len(sharedCfg.AssumeRole.ExternalID) > 0 { - opt.ExternalID = aws.String(sharedCfg.AssumeRole.ExternalID) - } - - // Assume role with MFA - if len(sharedCfg.AssumeRole.MFASerial) > 0 { - opt.SerialNumber = aws.String(sharedCfg.AssumeRole.MFASerial) - opt.TokenProvider = sessOpts.AssumeRoleTokenProvider - } - }, - ) -} - -// AssumeRoleTokenProviderNotSetError is an error returned when creating a session when the -// MFAToken option is not set when shared config is configured load assume a -// role with an MFA token. -type AssumeRoleTokenProviderNotSetError struct{} - -// Code is the short id of the error. -func (e AssumeRoleTokenProviderNotSetError) Code() string { - return "AssumeRoleTokenProviderNotSetError" -} - -// Message is the description of the error -func (e AssumeRoleTokenProviderNotSetError) Message() string { - return fmt.Sprintf("assume role with MFA enabled, but AssumeRoleTokenProvider session option not set.") -} + cfg.S3UseARNRegion = userCfg.S3UseARNRegion + if cfg.S3UseARNRegion == nil { + cfg.S3UseARNRegion = &envCfg.S3UseARNRegion + } + if cfg.S3UseARNRegion == nil { + cfg.S3UseARNRegion = &sharedCfg.S3UseARNRegion + } -// OrigErr is the underlying error that caused the failure. -func (e AssumeRoleTokenProviderNotSetError) OrigErr() error { return nil } -// Error satisfies the error interface. -func (e AssumeRoleTokenProviderNotSetError) Error() string { - return awserr.SprintError(e.Code(), e.Message(), "", nil) -} - -type credProviderError struct { - Err error +func mergeSTSRegionalEndpointConfig(cfg *aws.Config, values []endpoints.STSRegionalEndpoint) { + for _, v := range values { + if v != endpoints.UnsetSTSEndpoint { + cfg.STSRegionalEndpoint = v + break + } + } } -var emptyCreds = credentials.Value{} - -func (c credProviderError) Retrieve() (credentials.Value, error) { - return credentials.Value{}, c.Err -} -func (c credProviderError) IsExpired() bool { - return true +func mergeS3UsEast1RegionalEndpointConfig(cfg *aws.Config, values []endpoints.S3UsEast1RegionalEndpoint) { + for _, v := range values { + if v != endpoints.UnsetS3UsEast1Endpoint { + cfg.S3UsEast1RegionalEndpoint = v + break + } + } } func initHandlers(s *Session) { @@ -627,7 +617,7 @@ func initHandlers(s *Session) { } } -// Copy creates and returns a copy of the current Session, coping the config +// Copy creates and returns a copy of the current Session, copying the config // and handlers. If any additional configs are provided they will be merged // on top of the Session's copied config. // @@ -647,47 +637,67 @@ func (s *Session) Copy(cfgs ...*aws.Config) *Session { // ClientConfig satisfies the client.ConfigProvider interface and is used to // configure the service client instances. Passing the Session to the service // client's constructor (New) will use this method to configure the client. -func (s *Session) ClientConfig(serviceName string, cfgs ...*aws.Config) client.Config { - // Backwards compatibility, the error will be eaten if user calls ClientConfig - // directly. All SDK services will use ClientconfigWithError. - cfg, _ := s.clientConfigWithErr(serviceName, cfgs...) - - return cfg -} - -func (s *Session) clientConfigWithErr(serviceName string, cfgs ...*aws.Config) (client.Config, error) { +func (s *Session) ClientConfig(service string, cfgs ...*aws.Config) client.Config { s = s.Copy(cfgs...) - var resolved endpoints.ResolvedEndpoint - var err error - region := aws.StringValue(s.Config.Region) - - if endpoint := aws.StringValue(s.Config.Endpoint); len(endpoint) != 0 { - resolved.URL = endpoints.AddScheme(endpoint, aws.BoolValue(s.Config.DisableSSL)) - resolved.SigningRegion = region - } else { - resolved, err = s.Config.EndpointResolver.EndpointFor( - serviceName, region, - func(opt *endpoints.Options) { - opt.DisableSSL = aws.BoolValue(s.Config.DisableSSL) - opt.UseDualStack = aws.BoolValue(s.Config.UseDualStack) - - // Support the condition where the service is modeled but its - // endpoint metadata is not available. - opt.ResolveUnknownService = true - }, - ) + resolved, err := s.resolveEndpoint(service, region, s.Config) + if err != nil { + s.Handlers.Validate.PushBack(func(r *request.Request) { + if len(r.ClientInfo.Endpoint) != 0 { + // Error occurred while resolving endpoint, but the request + // being invoked has had an endpoint specified after the client + // was created. + return + } + r.Error = err + }) } return client.Config{ Config: s.Config, Handlers: s.Handlers, + PartitionID: resolved.PartitionID, Endpoint: resolved.URL, SigningRegion: resolved.SigningRegion, SigningNameDerived: resolved.SigningNameDerived, SigningName: resolved.SigningName, - }, err + } +} + +func (s *Session) resolveEndpoint(service, region string, cfg *aws.Config) (endpoints.ResolvedEndpoint, error) { + + if ep := aws.StringValue(cfg.Endpoint); len(ep) != 0 { + return endpoints.ResolvedEndpoint{ + URL: endpoints.AddScheme(ep, aws.BoolValue(cfg.DisableSSL)), + SigningRegion: region, + }, nil + } + + resolved, err := cfg.EndpointResolver.EndpointFor(service, region, + func(opt *endpoints.Options) { + opt.DisableSSL = aws.BoolValue(cfg.DisableSSL) + opt.UseDualStack = aws.BoolValue(cfg.UseDualStack) + // Support for STSRegionalEndpoint where the STSRegionalEndpoint is + // provided in envConfig or sharedConfig with envConfig getting + // precedence. + opt.STSRegionalEndpoint = cfg.STSRegionalEndpoint + + // Support for S3UsEast1RegionalEndpoint where the S3UsEast1RegionalEndpoint is + // provided in envConfig or sharedConfig with envConfig getting + // precedence. + opt.S3UsEast1RegionalEndpoint = cfg.S3UsEast1RegionalEndpoint + + // Support the condition where the service is modeled but its + // endpoint metadata is not available. + opt.ResolveUnknownService = true + }, + ) + if err != nil { + return endpoints.ResolvedEndpoint{}, err + } + + return resolved, nil } // ClientConfigNoResolveEndpoint is the same as ClientConfig with the exception @@ -697,12 +707,9 @@ func (s *Session) ClientConfigNoResolveEndpoint(cfgs ...*aws.Config) client.Conf s = s.Copy(cfgs...) var resolved endpoints.ResolvedEndpoint - - region := aws.StringValue(s.Config.Region) - if ep := aws.StringValue(s.Config.Endpoint); len(ep) > 0 { resolved.URL = endpoints.AddScheme(ep, aws.BoolValue(s.Config.DisableSSL)) - resolved.SigningRegion = region + resolved.SigningRegion = aws.StringValue(s.Config.Region) } return client.Config{ @@ -714,3 +721,14 @@ func (s *Session) ClientConfigNoResolveEndpoint(cfgs ...*aws.Config) client.Conf SigningName: resolved.SigningName, } } + +// logDeprecatedNewSessionError function enables error handling for session +func (s *Session) logDeprecatedNewSessionError(msg string, err error, cfgs []*aws.Config) { + // Session creation failed, need to report the error and prevent + // any requests from succeeding. + s.Config.MergeIn(cfgs...) + s.Config.Logger.Log("ERROR:", msg, "Error:", err) + s.Handlers.Validate.PushBack(func(r *request.Request) { + r.Error = err + }) +} diff --git a/github.com/aws/aws-sdk-go/aws/session/shared_config.go b/github.com/aws/aws-sdk-go/aws/session/shared_config.go index 7cb44021b3..680805a38a 100644 --- a/github.com/aws/aws-sdk-go/aws/session/shared_config.go +++ b/github.com/aws/aws-sdk-go/aws/session/shared_config.go @@ -2,10 +2,11 @@ package session import ( "fmt" + "time" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/credentials" - + "github.com/aws/aws-sdk-go/aws/endpoints" "github.com/aws/aws-sdk-go/internal/ini" ) @@ -16,57 +17,75 @@ const ( sessionTokenKey = `aws_session_token` // optional // Assume Role Credentials group - roleArnKey = `role_arn` // group required - sourceProfileKey = `source_profile` // group required (or credential_source) - credentialSourceKey = `credential_source` // group required (or source_profile) - externalIDKey = `external_id` // optional - mfaSerialKey = `mfa_serial` // optional - roleSessionNameKey = `role_session_name` // optional + roleArnKey = `role_arn` // group required + sourceProfileKey = `source_profile` // group required (or credential_source) + credentialSourceKey = `credential_source` // group required (or source_profile) + externalIDKey = `external_id` // optional + mfaSerialKey = `mfa_serial` // optional + roleSessionNameKey = `role_session_name` // optional + roleDurationSecondsKey = "duration_seconds" // optional + + // CSM options + csmEnabledKey = `csm_enabled` + csmHostKey = `csm_host` + csmPortKey = `csm_port` + csmClientIDKey = `csm_client_id` // Additional Config fields regionKey = `region` // endpoint discovery group enableEndpointDiscoveryKey = `endpoint_discovery_enabled` // optional + // External Credential Process - credentialProcessKey = `credential_process` + credentialProcessKey = `credential_process` // optional + + // Web Identity Token File + webIdentityTokenFileKey = `web_identity_token_file` // optional + + // Additional config fields for regional or legacy endpoints + stsRegionalEndpointSharedKey = `sts_regional_endpoints` + + // Additional config fields for regional or legacy endpoints + s3UsEast1RegionalSharedKey = `s3_us_east_1_regional_endpoint` // DefaultSharedConfigProfile is the default profile to be used when // loading configuration from the config files if another profile name // is not provided. DefaultSharedConfigProfile = `default` -) -type assumeRoleConfig struct { - RoleARN string - SourceProfile string - CredentialSource string - ExternalID string - MFASerial string - RoleSessionName string -} + // S3 ARN Region Usage + s3UseARNRegionKey = "s3_use_arn_region" +) // sharedConfig represents the configuration fields of the SDK config files. type sharedConfig struct { - // Credentials values from the config file. Both aws_access_key_id - // and aws_secret_access_key must be provided together in the same file - // to be considered valid. The values will be ignored if not a complete group. - // aws_session_token is an optional field that can be provided if both of the - // other two fields are also provided. + // Credentials values from the config file. Both aws_access_key_id and + // aws_secret_access_key must be provided together in the same file to be + // considered valid. The values will be ignored if not a complete group. + // aws_session_token is an optional field that can be provided if both of + // the other two fields are also provided. // // aws_access_key_id // aws_secret_access_key // aws_session_token Creds credentials.Value - AssumeRole assumeRoleConfig - AssumeRoleSource *sharedConfig + CredentialSource string + CredentialProcess string + WebIdentityTokenFile string + + RoleARN string + RoleSessionName string + ExternalID string + MFASerial string + AssumeRoleDuration *time.Duration - // An external process to request credentials - CredentialProcess string + SourceProfileName string + SourceProfile *sharedConfig - // Region is the region the SDK should use for looking up AWS service endpoints - // and signing requests. + // Region is the region the SDK should use for looking up AWS service + // endpoints and signing requests. // // region Region string @@ -76,6 +95,30 @@ type sharedConfig struct { // // endpoint_discovery_enabled = true EnableEndpointDiscovery *bool + + // CSM Options + CSMEnabled *bool + CSMHost string + CSMPort string + CSMClientID string + + // Specifies the Regional Endpoint flag for the SDK to resolve the endpoint for a service + // + // sts_regional_endpoints = regional + // This can take value as `LegacySTSEndpoint` or `RegionalSTSEndpoint` + STSRegionalEndpoint endpoints.STSRegionalEndpoint + + // Specifies the Regional Endpoint flag for the SDK to resolve the endpoint for a service + // + // s3_us_east_1_regional_endpoint = regional + // This can take value as `LegacyS3UsEast1Endpoint` or `RegionalS3UsEast1Endpoint` + S3UsEast1RegionalEndpoint endpoints.S3UsEast1RegionalEndpoint + + // Specifies if the S3 service should allow ARNs to direct the region + // the client's requests are sent to. + // + // s3_use_arn_region=true + S3UseARNRegion bool } type sharedConfigFile struct { @@ -83,17 +126,18 @@ type sharedConfigFile struct { IniData ini.Sections } -// loadSharedConfig retrieves the configuration from the list of files -// using the profile provided. The order the files are listed will determine +// loadSharedConfig retrieves the configuration from the list of files using +// the profile provided. The order the files are listed will determine // precedence. Values in subsequent files will overwrite values defined in // earlier files. // // For example, given two files A and B. Both define credentials. If the order -// of the files are A then B, B's credential values will be used instead of A's. +// of the files are A then B, B's credential values will be used instead of +// A's. // // See sharedConfig.setFromFile for information how the config files // will be loaded. -func loadSharedConfig(profile string, filenames []string) (sharedConfig, error) { +func loadSharedConfig(profile string, filenames []string, exOpts bool) (sharedConfig, error) { if len(profile) == 0 { profile = DefaultSharedConfigProfile } @@ -104,16 +148,11 @@ func loadSharedConfig(profile string, filenames []string) (sharedConfig, error) } cfg := sharedConfig{} - if err = cfg.setFromIniFiles(profile, files); err != nil { + profiles := map[string]struct{}{} + if err = cfg.setFromIniFiles(profiles, profile, files, exOpts); err != nil { return sharedConfig{}, err } - if len(cfg.AssumeRole.SourceProfile) > 0 { - if err := cfg.setAssumeRoleSource(profile, files); err != nil { - return sharedConfig{}, err - } - } - return cfg, nil } @@ -137,60 +176,88 @@ func loadSharedConfigIniFiles(filenames []string) ([]sharedConfigFile, error) { return files, nil } -func (cfg *sharedConfig) setAssumeRoleSource(origProfile string, files []sharedConfigFile) error { - var assumeRoleSrc sharedConfig - - if len(cfg.AssumeRole.CredentialSource) > 0 { - // setAssumeRoleSource is only called when source_profile is found. - // If both source_profile and credential_source are set, then - // ErrSharedConfigSourceCollision will be returned - return ErrSharedConfigSourceCollision +func (cfg *sharedConfig) setFromIniFiles(profiles map[string]struct{}, profile string, files []sharedConfigFile, exOpts bool) error { + // Trim files from the list that don't exist. + var skippedFiles int + var profileNotFoundErr error + for _, f := range files { + if err := cfg.setFromIniFile(profile, f, exOpts); err != nil { + if _, ok := err.(SharedConfigProfileNotExistsError); ok { + // Ignore profiles not defined in individual files. + profileNotFoundErr = err + skippedFiles++ + continue + } + return err + } + } + if skippedFiles == len(files) { + // If all files were skipped because the profile is not found, return + // the original profile not found error. + return profileNotFoundErr } - // Multiple level assume role chains are not support - if cfg.AssumeRole.SourceProfile == origProfile { - assumeRoleSrc = *cfg - assumeRoleSrc.AssumeRole = assumeRoleConfig{} + if _, ok := profiles[profile]; ok { + // if this is the second instance of the profile the Assume Role + // options must be cleared because they are only valid for the + // first reference of a profile. The self linked instance of the + // profile only have credential provider options. + cfg.clearAssumeRoleOptions() } else { - err := assumeRoleSrc.setFromIniFiles(cfg.AssumeRole.SourceProfile, files) - if err != nil { + // First time a profile has been seen, It must either be a assume role + // or credentials. Assert if the credential type requires a role ARN, + // the ARN is also set. + if err := cfg.validateCredentialsRequireARN(profile); err != nil { return err } } + profiles[profile] = struct{}{} - if len(assumeRoleSrc.Creds.AccessKeyID) == 0 { - return SharedConfigAssumeRoleError{RoleARN: cfg.AssumeRole.RoleARN} + if err := cfg.validateCredentialType(); err != nil { + return err } - cfg.AssumeRoleSource = &assumeRoleSrc + // Link source profiles for assume roles + if len(cfg.SourceProfileName) != 0 { + // Linked profile via source_profile ignore credential provider + // options, the source profile must provide the credentials. + cfg.clearCredentialOptions() - return nil -} - -func (cfg *sharedConfig) setFromIniFiles(profile string, files []sharedConfigFile) error { - // Trim files from the list that don't exist. - for _, f := range files { - if err := cfg.setFromIniFile(profile, f); err != nil { + srcCfg := &sharedConfig{} + err := srcCfg.setFromIniFiles(profiles, cfg.SourceProfileName, files, exOpts) + if err != nil { + // SourceProfile that doesn't exist is an error in configuration. if _, ok := err.(SharedConfigProfileNotExistsError); ok { - // Ignore proviles missings - continue + err = SharedConfigAssumeRoleError{ + RoleARN: cfg.RoleARN, + SourceProfile: cfg.SourceProfileName, + } } return err } + + if !srcCfg.hasCredentials() { + return SharedConfigAssumeRoleError{ + RoleARN: cfg.RoleARN, + SourceProfile: cfg.SourceProfileName, + } + } + + cfg.SourceProfile = srcCfg } return nil } -// setFromFile loads the configuration from the file using -// the profile provided. A sharedConfig pointer type value is used so that -// multiple config file loadings can be chained. +// setFromFile loads the configuration from the file using the profile +// provided. A sharedConfig pointer type value is used so that multiple config +// file loadings can be chained. // // Only loads complete logically grouped values, and will not set fields in cfg -// for incomplete grouped values in the config. Such as credentials. For example -// if a config file only includes aws_access_key_id but no aws_secret_access_key -// the aws_access_key_id will be ignored. -func (cfg *sharedConfig) setFromIniFile(profile string, file sharedConfigFile) error { +// for incomplete grouped values in the config. Such as credentials. For +// example if a config file only includes aws_access_key_id but no +// aws_secret_access_key the aws_access_key_id will be ignored. +func (cfg *sharedConfig) setFromIniFile(profile string, file sharedConfigFile, exOpts bool) error { section, ok := file.IniData.GetSection(profile) if !ok { // Fallback to to alternate profile name: profile @@ -200,53 +267,176 @@ func (cfg *sharedConfig) setFromIniFile(profile string, file sharedConfigFile) e } } - // Shared Credentials - akid := section.String(accessKeyIDKey) - secret := section.String(secretAccessKey) - if len(akid) > 0 && len(secret) > 0 { - cfg.Creds = credentials.Value{ - AccessKeyID: akid, - SecretAccessKey: secret, - SessionToken: section.String(sessionTokenKey), - ProviderName: fmt.Sprintf("SharedConfigCredentials: %s", file.Filename), + if exOpts { + // Assume Role Parameters + updateString(&cfg.RoleARN, section, roleArnKey) + updateString(&cfg.ExternalID, section, externalIDKey) + updateString(&cfg.MFASerial, section, mfaSerialKey) + updateString(&cfg.RoleSessionName, section, roleSessionNameKey) + updateString(&cfg.SourceProfileName, section, sourceProfileKey) + updateString(&cfg.CredentialSource, section, credentialSourceKey) + updateString(&cfg.Region, section, regionKey) + + if section.Has(roleDurationSecondsKey) { + d := time.Duration(section.Int(roleDurationSecondsKey)) * time.Second + cfg.AssumeRoleDuration = &d } - } - // Assume Role - roleArn := section.String(roleArnKey) - srcProfile := section.String(sourceProfileKey) - credentialSource := section.String(credentialSourceKey) - hasSource := len(srcProfile) > 0 || len(credentialSource) > 0 - if len(roleArn) > 0 && hasSource { - cfg.AssumeRole = assumeRoleConfig{ - RoleARN: roleArn, - SourceProfile: srcProfile, - CredentialSource: credentialSource, - ExternalID: section.String(externalIDKey), - MFASerial: section.String(mfaSerialKey), - RoleSessionName: section.String(roleSessionNameKey), + if v := section.String(stsRegionalEndpointSharedKey); len(v) != 0 { + sre, err := endpoints.GetSTSRegionalEndpoint(v) + if err != nil { + return fmt.Errorf("failed to load %s from shared config, %s, %v", + stsRegionalEndpointSharedKey, file.Filename, err) + } + cfg.STSRegionalEndpoint = sre } - } - // `credential_process` - if credProc := section.String(credentialProcessKey); len(credProc) > 0 { - cfg.CredentialProcess = credProc + if v := section.String(s3UsEast1RegionalSharedKey); len(v) != 0 { + sre, err := endpoints.GetS3UsEast1RegionalEndpoint(v) + if err != nil { + return fmt.Errorf("failed to load %s from shared config, %s, %v", + s3UsEast1RegionalSharedKey, file.Filename, err) + } + cfg.S3UsEast1RegionalEndpoint = sre + } } - // Region - if v := section.String(regionKey); len(v) > 0 { - cfg.Region = v + updateString(&cfg.CredentialProcess, section, credentialProcessKey) + updateString(&cfg.WebIdentityTokenFile, section, webIdentityTokenFileKey) + + // Shared Credentials + creds := credentials.Value{ + AccessKeyID: section.String(accessKeyIDKey), + SecretAccessKey: section.String(secretAccessKey), + SessionToken: section.String(sessionTokenKey), + ProviderName: fmt.Sprintf("SharedConfigCredentials: %s", file.Filename), + } + if creds.HasKeys() { + cfg.Creds = creds } // Endpoint discovery - if section.Has(enableEndpointDiscoveryKey) { - v := section.Bool(enableEndpointDiscoveryKey) - cfg.EnableEndpointDiscovery = &v + updateBoolPtr(&cfg.EnableEndpointDiscovery, section, enableEndpointDiscoveryKey) + + // CSM options + updateBoolPtr(&cfg.CSMEnabled, section, csmEnabledKey) + updateString(&cfg.CSMHost, section, csmHostKey) + updateString(&cfg.CSMPort, section, csmPortKey) + updateString(&cfg.CSMClientID, section, csmClientIDKey) + + updateBool(&cfg.S3UseARNRegion, section, s3UseARNRegionKey) + + return nil +} + +func (cfg *sharedConfig) validateCredentialsRequireARN(profile string) error { + var credSource string + + switch { + case len(cfg.SourceProfileName) != 0: + credSource = sourceProfileKey + case len(cfg.CredentialSource) != 0: + credSource = credentialSourceKey + case len(cfg.WebIdentityTokenFile) != 0: + credSource = webIdentityTokenFileKey + } + + if len(credSource) != 0 && len(cfg.RoleARN) == 0 { + return CredentialRequiresARNError{ + Type: credSource, + Profile: profile, + } + } + + return nil +} + +func (cfg *sharedConfig) validateCredentialType() error { + // Only one or no credential type can be defined. + if !oneOrNone( + len(cfg.SourceProfileName) != 0, + len(cfg.CredentialSource) != 0, + len(cfg.CredentialProcess) != 0, + len(cfg.WebIdentityTokenFile) != 0, + ) { + return ErrSharedConfigSourceCollision } return nil } +func (cfg *sharedConfig) hasCredentials() bool { + switch { + case len(cfg.SourceProfileName) != 0: + case len(cfg.CredentialSource) != 0: + case len(cfg.CredentialProcess) != 0: + case len(cfg.WebIdentityTokenFile) != 0: + case cfg.Creds.HasKeys(): + default: + return false + } + + return true +} + +func (cfg *sharedConfig) clearCredentialOptions() { + cfg.CredentialSource = "" + cfg.CredentialProcess = "" + cfg.WebIdentityTokenFile = "" + cfg.Creds = credentials.Value{} +} + +func (cfg *sharedConfig) clearAssumeRoleOptions() { + cfg.RoleARN = "" + cfg.ExternalID = "" + cfg.MFASerial = "" + cfg.RoleSessionName = "" + cfg.SourceProfileName = "" +} + +func oneOrNone(bs ...bool) bool { + var count int + + for _, b := range bs { + if b { + count++ + if count > 1 { + return false + } + } + } + + return true +} + +// updateString will only update the dst with the value in the section key, key +// is present in the section. +func updateString(dst *string, section ini.Section, key string) { + if !section.Has(key) { + return + } + *dst = section.String(key) +} + +// updateBool will only update the dst with the value in the section key, key +// is present in the section. +func updateBool(dst *bool, section ini.Section, key string) { + if !section.Has(key) { + return + } + *dst = section.Bool(key) +} + +// updateBoolPtr will only update the dst with the value in the section key, +// key is present in the section. +func updateBoolPtr(dst **bool, section ini.Section, key string) { + if !section.Has(key) { + return + } + *dst = new(bool) + **dst = section.Bool(key) +} + // SharedConfigLoadError is an error for the shared config file failed to load. type SharedConfigLoadError struct { Filename string @@ -304,7 +494,8 @@ func (e SharedConfigProfileNotExistsError) Error() string { // profile contains assume role information, but that information is invalid // or not complete. type SharedConfigAssumeRoleError struct { - RoleARN string + RoleARN string + SourceProfile string } // Code is the short id of the error. @@ -314,8 +505,10 @@ func (e SharedConfigAssumeRoleError) Code() string { // Message is the description of the error func (e SharedConfigAssumeRoleError) Message() string { - return fmt.Sprintf("failed to load assume role for %s, source profile has no shared credentials", - e.RoleARN) + return fmt.Sprintf( + "failed to load assume role for %s, source profile %s has no shared credentials", + e.RoleARN, e.SourceProfile, + ) } // OrigErr is the underlying error that caused the failure. @@ -327,3 +520,36 @@ func (e SharedConfigAssumeRoleError) OrigErr() error { func (e SharedConfigAssumeRoleError) Error() string { return awserr.SprintError(e.Code(), e.Message(), "", nil) } + +// CredentialRequiresARNError provides the error for shared config credentials +// that are incorrectly configured in the shared config or credentials file. +type CredentialRequiresARNError struct { + // type of credentials that were configured. + Type string + + // Profile name the credentials were in. + Profile string +} + +// Code is the short id of the error. +func (e CredentialRequiresARNError) Code() string { + return "CredentialRequiresARNError" +} + +// Message is the description of the error +func (e CredentialRequiresARNError) Message() string { + return fmt.Sprintf( + "credential type %s requires role_arn, profile %s", + e.Type, e.Profile, + ) +} + +// OrigErr is the underlying error that caused the failure. +func (e CredentialRequiresARNError) OrigErr() error { + return nil +} + +// Error satisfies the error interface. +func (e CredentialRequiresARNError) Error() string { + return awserr.SprintError(e.Code(), e.Message(), "", nil) +} diff --git a/github.com/aws/aws-sdk-go/aws/signer/v4/header_rules.go b/github.com/aws/aws-sdk-go/aws/signer/v4/header_rules.go index 244c86da05..07ea799fbd 100644 --- a/github.com/aws/aws-sdk-go/aws/signer/v4/header_rules.go +++ b/github.com/aws/aws-sdk-go/aws/signer/v4/header_rules.go @@ -1,8 +1,7 @@ package v4 import ( - "net/http" - "strings" + "github.com/aws/aws-sdk-go/internal/strings" ) // validator houses a set of rule needed for validation of a @@ -61,7 +60,7 @@ type patterns []string // been found func (p patterns) IsValid(value string) bool { for _, pattern := range p { - if strings.HasPrefix(http.CanonicalHeaderKey(value), pattern) { + if strings.HasPrefixFold(value, pattern) { return true } } diff --git a/github.com/aws/aws-sdk-go/aws/signer/v4/request_context_go1.5.go b/github.com/aws/aws-sdk-go/aws/signer/v4/request_context_go1.5.go new file mode 100644 index 0000000000..f35fc860b3 --- /dev/null +++ b/github.com/aws/aws-sdk-go/aws/signer/v4/request_context_go1.5.go @@ -0,0 +1,13 @@ +// +build !go1.7 + +package v4 + +import ( + "net/http" + + "github.com/aws/aws-sdk-go/aws" +) + +func requestContext(r *http.Request) aws.Context { + return aws.BackgroundContext() +} diff --git a/github.com/aws/aws-sdk-go/aws/signer/v4/request_context_go1.7.go b/github.com/aws/aws-sdk-go/aws/signer/v4/request_context_go1.7.go new file mode 100644 index 0000000000..fed5c859ca --- /dev/null +++ b/github.com/aws/aws-sdk-go/aws/signer/v4/request_context_go1.7.go @@ -0,0 +1,13 @@ +// +build go1.7 + +package v4 + +import ( + "net/http" + + "github.com/aws/aws-sdk-go/aws" +) + +func requestContext(r *http.Request) aws.Context { + return r.Context() +} diff --git a/github.com/aws/aws-sdk-go/aws/signer/v4/stream.go b/github.com/aws/aws-sdk-go/aws/signer/v4/stream.go new file mode 100644 index 0000000000..02cbd97e23 --- /dev/null +++ b/github.com/aws/aws-sdk-go/aws/signer/v4/stream.go @@ -0,0 +1,63 @@ +package v4 + +import ( + "encoding/hex" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws/credentials" +) + +type credentialValueProvider interface { + Get() (credentials.Value, error) +} + +// StreamSigner implements signing of event stream encoded payloads +type StreamSigner struct { + region string + service string + + credentials credentialValueProvider + + prevSig []byte +} + +// NewStreamSigner creates a SigV4 signer used to sign Event Stream encoded messages +func NewStreamSigner(region, service string, seedSignature []byte, credentials *credentials.Credentials) *StreamSigner { + return &StreamSigner{ + region: region, + service: service, + credentials: credentials, + prevSig: seedSignature, + } +} + +// GetSignature takes an event stream encoded headers and payload and returns a signature +func (s *StreamSigner) GetSignature(headers, payload []byte, date time.Time) ([]byte, error) { + credValue, err := s.credentials.Get() + if err != nil { + return nil, err + } + + sigKey := deriveSigningKey(s.region, s.service, credValue.SecretAccessKey, date) + + keyPath := buildSigningScope(s.region, s.service, date) + + stringToSign := buildEventStreamStringToSign(headers, payload, s.prevSig, keyPath, date) + + signature := hmacSHA256(sigKey, []byte(stringToSign)) + s.prevSig = signature + + return signature, nil +} + +func buildEventStreamStringToSign(headers, payload, prevSig []byte, scope string, date time.Time) string { + return strings.Join([]string{ + "AWS4-HMAC-SHA256-PAYLOAD", + formatTime(date), + scope, + hex.EncodeToString(prevSig), + hex.EncodeToString(hashSHA256(headers)), + hex.EncodeToString(hashSHA256(payload)), + }, "\n") +} diff --git a/github.com/aws/aws-sdk-go/aws/signer/v4/v4.go b/github.com/aws/aws-sdk-go/aws/signer/v4/v4.go index 523db79f8d..d71f7b3f4f 100644 --- a/github.com/aws/aws-sdk-go/aws/signer/v4/v4.go +++ b/github.com/aws/aws-sdk-go/aws/signer/v4/v4.go @@ -76,9 +76,14 @@ import ( ) const ( + authorizationHeader = "Authorization" + authHeaderSignatureElem = "Signature=" + signatureQueryKey = "X-Amz-Signature" + authHeaderPrefix = "AWS4-HMAC-SHA256" timeFormat = "20060102T150405Z" shortTimeFormat = "20060102" + awsV4Request = "aws4_request" // emptyStringSHA256 is a SHA256 of an empty string emptyStringSHA256 = `e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855` @@ -87,9 +92,9 @@ const ( var ignoredHeaders = rules{ blacklist{ mapRule{ - "Authorization": struct{}{}, - "User-Agent": struct{}{}, - "X-Amzn-Trace-Id": struct{}{}, + authorizationHeader: struct{}{}, + "User-Agent": struct{}{}, + "X-Amzn-Trace-Id": struct{}{}, }, }, } @@ -229,11 +234,9 @@ type signingCtx struct { DisableURIPathEscaping bool - credValues credentials.Value - isPresign bool - formattedTime string - formattedShortTime string - unsignedPayload bool + credValues credentials.Value + isPresign bool + unsignedPayload bool bodyDigest string signedHeaders string @@ -337,7 +340,7 @@ func (v4 Signer) signWithBody(r *http.Request, body io.ReadSeeker, service, regi } var err error - ctx.credValues, err = v4.Credentials.Get() + ctx.credValues, err = v4.Credentials.GetWithContext(requestContext(r)) if err != nil { return http.Header{}, err } @@ -532,39 +535,56 @@ func (ctx *signingCtx) build(disableHeaderHoisting bool) error { ctx.buildSignature() // depends on string to sign if ctx.isPresign { - ctx.Request.URL.RawQuery += "&X-Amz-Signature=" + ctx.signature + ctx.Request.URL.RawQuery += "&" + signatureQueryKey + "=" + ctx.signature } else { parts := []string{ authHeaderPrefix + " Credential=" + ctx.credValues.AccessKeyID + "/" + ctx.credentialString, "SignedHeaders=" + ctx.signedHeaders, - "Signature=" + ctx.signature, + authHeaderSignatureElem + ctx.signature, } - ctx.Request.Header.Set("Authorization", strings.Join(parts, ", ")) + ctx.Request.Header.Set(authorizationHeader, strings.Join(parts, ", ")) } return nil } -func (ctx *signingCtx) buildTime() { - ctx.formattedTime = ctx.Time.UTC().Format(timeFormat) - ctx.formattedShortTime = ctx.Time.UTC().Format(shortTimeFormat) +// GetSignedRequestSignature attempts to extract the signature of the request. +// Returning an error if the request is unsigned, or unable to extract the +// signature. +func GetSignedRequestSignature(r *http.Request) ([]byte, error) { + + if auth := r.Header.Get(authorizationHeader); len(auth) != 0 { + ps := strings.Split(auth, ", ") + for _, p := range ps { + if idx := strings.Index(p, authHeaderSignatureElem); idx >= 0 { + sig := p[len(authHeaderSignatureElem):] + if len(sig) == 0 { + return nil, fmt.Errorf("invalid request signature authorization header") + } + return hex.DecodeString(sig) + } + } + } + if sig := r.URL.Query().Get("X-Amz-Signature"); len(sig) != 0 { + return hex.DecodeString(sig) + } + + return nil, fmt.Errorf("request not signed") +} + +func (ctx *signingCtx) buildTime() { if ctx.isPresign { duration := int64(ctx.ExpireTime / time.Second) - ctx.Query.Set("X-Amz-Date", ctx.formattedTime) + ctx.Query.Set("X-Amz-Date", formatTime(ctx.Time)) ctx.Query.Set("X-Amz-Expires", strconv.FormatInt(duration, 10)) } else { - ctx.Request.Header.Set("X-Amz-Date", ctx.formattedTime) + ctx.Request.Header.Set("X-Amz-Date", formatTime(ctx.Time)) } } func (ctx *signingCtx) buildCredentialString() { - ctx.credentialString = strings.Join([]string{ - ctx.formattedShortTime, - ctx.Region, - ctx.ServiceName, - "aws4_request", - }, "/") + ctx.credentialString = buildSigningScope(ctx.Region, ctx.ServiceName, ctx.Time) if ctx.isPresign { ctx.Query.Set("X-Amz-Credential", ctx.credValues.AccessKeyID+"/"+ctx.credentialString) @@ -588,8 +608,7 @@ func (ctx *signingCtx) buildCanonicalHeaders(r rule, header http.Header) { var headers []string headers = append(headers, "host") for k, v := range header { - canonicalKey := http.CanonicalHeaderKey(k) - if !r.IsValid(canonicalKey) { + if !r.IsValid(k) { continue // ignored header } if ctx.SignedHeaderVals == nil { @@ -653,19 +672,15 @@ func (ctx *signingCtx) buildCanonicalString() { func (ctx *signingCtx) buildStringToSign() { ctx.stringToSign = strings.Join([]string{ authHeaderPrefix, - ctx.formattedTime, + formatTime(ctx.Time), ctx.credentialString, - hex.EncodeToString(makeSha256([]byte(ctx.canonicalString))), + hex.EncodeToString(hashSHA256([]byte(ctx.canonicalString))), }, "\n") } func (ctx *signingCtx) buildSignature() { - secret := ctx.credValues.SecretAccessKey - date := makeHmac([]byte("AWS4"+secret), []byte(ctx.formattedShortTime)) - region := makeHmac(date, []byte(ctx.Region)) - service := makeHmac(region, []byte(ctx.ServiceName)) - credentials := makeHmac(service, []byte("aws4_request")) - signature := makeHmac(credentials, []byte(ctx.stringToSign)) + creds := deriveSigningKey(ctx.Region, ctx.ServiceName, ctx.credValues.SecretAccessKey, ctx.Time) + signature := hmacSHA256(creds, []byte(ctx.stringToSign)) ctx.signature = hex.EncodeToString(signature) } @@ -687,7 +702,11 @@ func (ctx *signingCtx) buildBodyDigest() error { if !aws.IsReaderSeekable(ctx.Body) { return fmt.Errorf("cannot use unseekable request body %T, for signed request with body", ctx.Body) } - hash = hex.EncodeToString(makeSha256Reader(ctx.Body)) + hashBytes, err := makeSha256Reader(ctx.Body) + if err != nil { + return err + } + hash = hex.EncodeToString(hashBytes) } if includeSHA256Header { @@ -722,22 +741,28 @@ func (ctx *signingCtx) removePresign() { ctx.Query.Del("X-Amz-SignedHeaders") } -func makeHmac(key []byte, data []byte) []byte { +func hmacSHA256(key []byte, data []byte) []byte { hash := hmac.New(sha256.New, key) hash.Write(data) return hash.Sum(nil) } -func makeSha256(data []byte) []byte { +func hashSHA256(data []byte) []byte { hash := sha256.New() hash.Write(data) return hash.Sum(nil) } -func makeSha256Reader(reader io.ReadSeeker) []byte { +func makeSha256Reader(reader io.ReadSeeker) (hashBytes []byte, err error) { hash := sha256.New() - start, _ := reader.Seek(0, sdkio.SeekCurrent) - defer reader.Seek(start, sdkio.SeekStart) + start, err := reader.Seek(0, sdkio.SeekCurrent) + if err != nil { + return nil, err + } + defer func() { + // ensure error is return if unable to seek back to start of payload. + _, err = reader.Seek(start, sdkio.SeekStart) + }() // Use CopyN to avoid allocating the 32KB buffer in io.Copy for bodies // smaller than 32KB. Fall back to io.Copy if we fail to determine the size. @@ -748,7 +773,7 @@ func makeSha256Reader(reader io.ReadSeeker) []byte { io.CopyN(hash, reader, size) } - return hash.Sum(nil) + return hash.Sum(nil), nil } const doubleSpace = " " @@ -794,3 +819,28 @@ func stripExcessSpaces(vals []string) { vals[i] = string(buf[:m]) } } + +func buildSigningScope(region, service string, dt time.Time) string { + return strings.Join([]string{ + formatShortTime(dt), + region, + service, + awsV4Request, + }, "/") +} + +func deriveSigningKey(region, service, secretKey string, dt time.Time) []byte { + kDate := hmacSHA256([]byte("AWS4"+secretKey), []byte(formatShortTime(dt))) + kRegion := hmacSHA256(kDate, []byte(region)) + kService := hmacSHA256(kRegion, []byte(service)) + signingKey := hmacSHA256(kService, []byte(awsV4Request)) + return signingKey +} + +func formatShortTime(dt time.Time) string { + return dt.UTC().Format(shortTimeFormat) +} + +func formatTime(dt time.Time) string { + return dt.UTC().Format(timeFormat) +} diff --git a/github.com/aws/aws-sdk-go/aws/types.go b/github.com/aws/aws-sdk-go/aws/types.go index 8b6f23425a..98751ee84f 100644 --- a/github.com/aws/aws-sdk-go/aws/types.go +++ b/github.com/aws/aws-sdk-go/aws/types.go @@ -2,18 +2,24 @@ package aws import ( "io" + "strings" "sync" "github.com/aws/aws-sdk-go/internal/sdkio" ) -// ReadSeekCloser wraps a io.Reader returning a ReaderSeekerCloser. Should -// only be used with an io.Reader that is also an io.Seeker. Doing so may -// cause request signature errors, or request body's not sent for GET, HEAD -// and DELETE HTTP methods. +// ReadSeekCloser wraps a io.Reader returning a ReaderSeekerCloser. Allows the +// SDK to accept an io.Reader that is not also an io.Seeker for unsigned +// streaming payload API operations. // -// Deprecated: Should only be used with io.ReadSeeker. If using for -// S3 PutObject to stream content use s3manager.Uploader instead. +// A ReadSeekCloser wrapping an nonseekable io.Reader used in an API +// operation's input will prevent that operation being retried in the case of +// network errors, and cause operation requests to fail if the operation +// requires payload signing. +// +// Note: If using With S3 PutObject to stream an object upload The SDK's S3 +// Upload manager (s3manager.Uploader) provides support for streaming with the +// ability to retry network errors. func ReadSeekCloser(r io.Reader) ReaderSeekerCloser { return ReaderSeekerCloser{r} } @@ -43,7 +49,8 @@ func IsReaderSeekable(r io.Reader) bool { // Read reads from the reader up to size of p. The number of bytes read, and // error if it occurred will be returned. // -// If the reader is not an io.Reader zero bytes read, and nil error will be returned. +// If the reader is not an io.Reader zero bytes read, and nil error will be +// returned. // // Performs the same functionality as io.Reader Read func (r ReaderSeekerCloser) Read(p []byte) (int, error) { @@ -199,3 +206,59 @@ func (b *WriteAtBuffer) Bytes() []byte { defer b.m.Unlock() return b.buf } + +// MultiCloser is a utility to close multiple io.Closers within a single +// statement. +type MultiCloser []io.Closer + +// Close closes all of the io.Closers making up the MultiClosers. Any +// errors that occur while closing will be returned in the order they +// occur. +func (m MultiCloser) Close() error { + var errs errors + for _, c := range m { + err := c.Close() + if err != nil { + errs = append(errs, err) + } + } + if len(errs) != 0 { + return errs + } + + return nil +} + +type errors []error + +func (es errors) Error() string { + var parts []string + for _, e := range es { + parts = append(parts, e.Error()) + } + + return strings.Join(parts, "\n") +} + +// CopySeekableBody copies the seekable body to an io.Writer +func CopySeekableBody(dst io.Writer, src io.ReadSeeker) (int64, error) { + curPos, err := src.Seek(0, sdkio.SeekCurrent) + if err != nil { + return 0, err + } + + // copy errors may be assumed to be from the body. + n, err := io.Copy(dst, src) + if err != nil { + return n, err + } + + // seek back to the first position after reading to reset + // the body for transmission. + _, err = src.Seek(curPos, sdkio.SeekStart) + if err != nil { + return n, err + } + + return n, nil +} diff --git a/github.com/aws/aws-sdk-go/aws/version.go b/github.com/aws/aws-sdk-go/aws/version.go index f8869e4378..ab2e5269e0 100644 --- a/github.com/aws/aws-sdk-go/aws/version.go +++ b/github.com/aws/aws-sdk-go/aws/version.go @@ -5,4 +5,4 @@ package aws const SDKName = "aws-sdk-go" // SDKVersion is the version of this SDK -const SDKVersion = "1.16.19" +const SDKVersion = "1.33.8" diff --git a/github.com/aws/aws-sdk-go/aws/context_1_6.go b/github.com/aws/aws-sdk-go/internal/context/background_go1.5.go similarity index 86% rename from github.com/aws/aws-sdk-go/aws/context_1_6.go rename to github.com/aws/aws-sdk-go/internal/context/background_go1.5.go index 8fdda53033..876dcb3fde 100644 --- a/github.com/aws/aws-sdk-go/aws/context_1_6.go +++ b/github.com/aws/aws-sdk-go/internal/context/background_go1.5.go @@ -1,6 +1,6 @@ // +build !go1.7 -package aws +package context import "time" @@ -30,12 +30,11 @@ func (*emptyCtx) Value(key interface{}) interface{} { func (e *emptyCtx) String() string { switch e { - case backgroundCtx: + case BackgroundCtx: return "aws.BackgroundContext" } return "unknown empty Context" } -var ( - backgroundCtx = new(emptyCtx) -) +// BackgroundCtx is the common base context. +var BackgroundCtx = new(emptyCtx) diff --git a/github.com/aws/aws-sdk-go/internal/ini/ini_parser.go b/github.com/aws/aws-sdk-go/internal/ini/ini_parser.go index f99703372c..cf9fad81e7 100644 --- a/github.com/aws/aws-sdk-go/internal/ini/ini_parser.go +++ b/github.com/aws/aws-sdk-go/internal/ini/ini_parser.go @@ -162,7 +162,7 @@ loop: if len(tokens) == 0 { break loop } - + // if should skip is true, we skip the tokens until should skip is set to false. step = SkipTokenState } @@ -218,7 +218,7 @@ loop: // S -> equal_expr' expr_stmt' switch k.Kind { case ASTKindEqualExpr: - // assiging a value to some key + // assigning a value to some key k.AppendChild(newExpression(tok)) stack.Push(newExprStatement(k)) case ASTKindExpr: @@ -250,6 +250,13 @@ loop: if !runeCompare(tok.Raw(), openBrace) { return nil, NewParseError("expected '['") } + // If OpenScopeState is not at the start, we must mark the previous ast as complete + // + // for example: if previous ast was a skip statement; + // we should mark it as complete before we create a new statement + if k.Kind != ASTKindStart { + stack.MarkComplete(k) + } stmt := newStatement() stack.Push(stmt) @@ -304,7 +311,9 @@ loop: stmt := newCommentStatement(tok) stack.Push(stmt) default: - return nil, NewParseError(fmt.Sprintf("invalid state with ASTKind %v and TokenType %v", k, tok)) + return nil, NewParseError( + fmt.Sprintf("invalid state with ASTKind %v and TokenType %v", + k, tok.Type())) } if len(tokens) > 0 { @@ -314,7 +323,7 @@ loop: // this occurs when a statement has not been completed if stack.top > 1 { - return nil, NewParseError(fmt.Sprintf("incomplete expression: %v", stack.container)) + return nil, NewParseError(fmt.Sprintf("incomplete ini expression")) } // returns a sublist which excludes the start symbol diff --git a/github.com/aws/aws-sdk-go/internal/ini/skipper.go b/github.com/aws/aws-sdk-go/internal/ini/skipper.go index 6bb6964475..da7a4049cf 100644 --- a/github.com/aws/aws-sdk-go/internal/ini/skipper.go +++ b/github.com/aws/aws-sdk-go/internal/ini/skipper.go @@ -22,24 +22,24 @@ func newSkipper() skipper { } func (s *skipper) ShouldSkip(tok Token) bool { + // should skip state will be modified only if previous token was new line (NL); + // and the current token is not WhiteSpace (WS). if s.shouldSkip && s.prevTok.Type() == TokenNL && tok.Type() != TokenWS { - s.Continue() return false } s.prevTok = tok - return s.shouldSkip } func (s *skipper) Skip() { s.shouldSkip = true - s.prevTok = emptyToken } func (s *skipper) Continue() { s.shouldSkip = false + // empty token is assigned as we return to default state, when should skip is false s.prevTok = emptyToken } diff --git a/github.com/aws/aws-sdk-go/internal/sdkio/byte.go b/github.com/aws/aws-sdk-go/internal/sdkio/byte.go new file mode 100644 index 0000000000..6c443988bb --- /dev/null +++ b/github.com/aws/aws-sdk-go/internal/sdkio/byte.go @@ -0,0 +1,12 @@ +package sdkio + +const ( + // Byte is 8 bits + Byte int64 = 1 + // KibiByte (KiB) is 1024 Bytes + KibiByte = Byte * 1024 + // MebiByte (MiB) is 1024 KiB + MebiByte = KibiByte * 1024 + // GibiByte (GiB) is 1024 MiB + GibiByte = MebiByte * 1024 +) diff --git a/github.com/aws/aws-sdk-go/internal/sdkmath/floor.go b/github.com/aws/aws-sdk-go/internal/sdkmath/floor.go new file mode 100644 index 0000000000..44898eed0f --- /dev/null +++ b/github.com/aws/aws-sdk-go/internal/sdkmath/floor.go @@ -0,0 +1,15 @@ +// +build go1.10 + +package sdkmath + +import "math" + +// Round returns the nearest integer, rounding half away from zero. +// +// Special cases are: +// Round(±0) = ±0 +// Round(±Inf) = ±Inf +// Round(NaN) = NaN +func Round(x float64) float64 { + return math.Round(x) +} diff --git a/github.com/aws/aws-sdk-go/internal/sdkmath/floor_go1.9.go b/github.com/aws/aws-sdk-go/internal/sdkmath/floor_go1.9.go new file mode 100644 index 0000000000..810ec7f08b --- /dev/null +++ b/github.com/aws/aws-sdk-go/internal/sdkmath/floor_go1.9.go @@ -0,0 +1,56 @@ +// +build !go1.10 + +package sdkmath + +import "math" + +// Copied from the Go standard library's (Go 1.12) math/floor.go for use in +// Go version prior to Go 1.10. +const ( + uvone = 0x3FF0000000000000 + mask = 0x7FF + shift = 64 - 11 - 1 + bias = 1023 + signMask = 1 << 63 + fracMask = 1<= 0.5 { + // return t + Copysign(1, x) + // } + // return t + // } + bits := math.Float64bits(x) + e := uint(bits>>shift) & mask + if e < bias { + // Round abs(x) < 1 including denormals. + bits &= signMask // +-0 + if e == bias-1 { + bits |= uvone // +-1 + } + } else if e < bias+shift { + // Round any abs(x) >= 1 containing a fractional component [0,1). + // + // Numbers with larger exponents are returned unchanged since they + // must be either an integer, infinity, or NaN. + const half = 1 << (shift - 1) + e -= bias + bits += half >> e + bits &^= fracMask >> e + } + return math.Float64frombits(bits) +} diff --git a/github.com/aws/aws-sdk-go/internal/sdkrand/read.go b/github.com/aws/aws-sdk-go/internal/sdkrand/read.go new file mode 100644 index 0000000000..f4651da2da --- /dev/null +++ b/github.com/aws/aws-sdk-go/internal/sdkrand/read.go @@ -0,0 +1,11 @@ +// +build go1.6 + +package sdkrand + +import "math/rand" + +// Read provides the stub for math.Rand.Read method support for go version's +// 1.6 and greater. +func Read(r *rand.Rand, p []byte) (int, error) { + return r.Read(p) +} diff --git a/github.com/aws/aws-sdk-go/internal/sdkrand/read_1_5.go b/github.com/aws/aws-sdk-go/internal/sdkrand/read_1_5.go new file mode 100644 index 0000000000..b1d93a33d4 --- /dev/null +++ b/github.com/aws/aws-sdk-go/internal/sdkrand/read_1_5.go @@ -0,0 +1,24 @@ +// +build !go1.6 + +package sdkrand + +import "math/rand" + +// Read backfills Go 1.6's math.Rand.Reader for Go 1.5 +func Read(r *rand.Rand, p []byte) (n int, err error) { + // Copy of Go standard libraries math package's read function not added to + // standard library until Go 1.6. + var pos int8 + var val int64 + for n = 0; n < len(p); n++ { + if pos == 0 { + val = r.Int63() + pos = 7 + } + p[n] = byte(val) + val >>= 8 + pos-- + } + + return n, err +} diff --git a/github.com/aws/aws-sdk-go/internal/strings/strings.go b/github.com/aws/aws-sdk-go/internal/strings/strings.go new file mode 100644 index 0000000000..d008ae27cb --- /dev/null +++ b/github.com/aws/aws-sdk-go/internal/strings/strings.go @@ -0,0 +1,11 @@ +package strings + +import ( + "strings" +) + +// HasPrefixFold tests whether the string s begins with prefix, interpreted as UTF-8 strings, +// under Unicode case-folding. +func HasPrefixFold(s, prefix string) bool { + return len(s) >= len(prefix) && strings.EqualFold(s[0:len(prefix)], prefix) +} diff --git a/github.com/aws/aws-sdk-go/internal/sync/singleflight/LICENSE b/github.com/aws/aws-sdk-go/internal/sync/singleflight/LICENSE new file mode 100644 index 0000000000..6a66aea5ea --- /dev/null +++ b/github.com/aws/aws-sdk-go/internal/sync/singleflight/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/github.com/aws/aws-sdk-go/internal/sync/singleflight/singleflight.go b/github.com/aws/aws-sdk-go/internal/sync/singleflight/singleflight.go new file mode 100644 index 0000000000..14ad0c5891 --- /dev/null +++ b/github.com/aws/aws-sdk-go/internal/sync/singleflight/singleflight.go @@ -0,0 +1,120 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package singleflight provides a duplicate function call suppression +// mechanism. +package singleflight + +import "sync" + +// call is an in-flight or completed singleflight.Do call +type call struct { + wg sync.WaitGroup + + // These fields are written once before the WaitGroup is done + // and are only read after the WaitGroup is done. + val interface{} + err error + + // forgotten indicates whether Forget was called with this call's key + // while the call was still in flight. + forgotten bool + + // These fields are read and written with the singleflight + // mutex held before the WaitGroup is done, and are read but + // not written after the WaitGroup is done. + dups int + chans []chan<- Result +} + +// Group represents a class of work and forms a namespace in +// which units of work can be executed with duplicate suppression. +type Group struct { + mu sync.Mutex // protects m + m map[string]*call // lazily initialized +} + +// Result holds the results of Do, so they can be passed +// on a channel. +type Result struct { + Val interface{} + Err error + Shared bool +} + +// Do executes and returns the results of the given function, making +// sure that only one execution is in-flight for a given key at a +// time. If a duplicate comes in, the duplicate caller waits for the +// original to complete and receives the same results. +// The return value shared indicates whether v was given to multiple callers. +func (g *Group) Do(key string, fn func() (interface{}, error)) (v interface{}, err error, shared bool) { + g.mu.Lock() + if g.m == nil { + g.m = make(map[string]*call) + } + if c, ok := g.m[key]; ok { + c.dups++ + g.mu.Unlock() + c.wg.Wait() + return c.val, c.err, true + } + c := new(call) + c.wg.Add(1) + g.m[key] = c + g.mu.Unlock() + + g.doCall(c, key, fn) + return c.val, c.err, c.dups > 0 +} + +// DoChan is like Do but returns a channel that will receive the +// results when they are ready. +func (g *Group) DoChan(key string, fn func() (interface{}, error)) <-chan Result { + ch := make(chan Result, 1) + g.mu.Lock() + if g.m == nil { + g.m = make(map[string]*call) + } + if c, ok := g.m[key]; ok { + c.dups++ + c.chans = append(c.chans, ch) + g.mu.Unlock() + return ch + } + c := &call{chans: []chan<- Result{ch}} + c.wg.Add(1) + g.m[key] = c + g.mu.Unlock() + + go g.doCall(c, key, fn) + + return ch +} + +// doCall handles the single call for a key. +func (g *Group) doCall(c *call, key string, fn func() (interface{}, error)) { + c.val, c.err = fn() + c.wg.Done() + + g.mu.Lock() + if !c.forgotten { + delete(g.m, key) + } + for _, ch := range c.chans { + ch <- Result{c.val, c.err, c.dups > 0} + } + g.mu.Unlock() +} + +// Forget tells the singleflight to forget about a key. Future calls +// to Do for this key will call the function rather than waiting for +// an earlier call to complete. +func (g *Group) Forget(key string) { + g.mu.Lock() + if c, ok := g.m[key]; ok { + c.forgotten = true + } + delete(g.m, key) + g.mu.Unlock() +} diff --git a/github.com/aws/aws-sdk-go/private/checksum/content_md5.go b/github.com/aws/aws-sdk-go/private/checksum/content_md5.go new file mode 100644 index 0000000000..e045f38d83 --- /dev/null +++ b/github.com/aws/aws-sdk-go/private/checksum/content_md5.go @@ -0,0 +1,53 @@ +package checksum + +import ( + "crypto/md5" + "encoding/base64" + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/request" +) + +const contentMD5Header = "Content-Md5" + +// AddBodyContentMD5Handler computes and sets the HTTP Content-MD5 header for requests that +// require it. +func AddBodyContentMD5Handler(r *request.Request) { + // if Content-MD5 header is already present, return + if v := r.HTTPRequest.Header.Get(contentMD5Header); len(v) != 0 { + return + } + + // if S3DisableContentMD5Validation flag is set, return + if aws.BoolValue(r.Config.S3DisableContentMD5Validation) { + return + } + + // if request is presigned, return + if r.IsPresigned() { + return + } + + // if body is not seekable, return + if !aws.IsReaderSeekable(r.Body) { + if r.Config.Logger != nil { + r.Config.Logger.Log(fmt.Sprintf( + "Unable to compute Content-MD5 for unseekable body, S3.%s", + r.Operation.Name)) + } + return + } + + h := md5.New() + + if _, err := aws.CopySeekableBody(h, r.Body); err != nil { + r.Error = awserr.New("ContentMD5", "failed to compute body MD5", err) + return + } + + // encode the md5 checksum in base64 and set the request header. + v := base64.StdEncoding.EncodeToString(h.Sum(nil)) + r.HTTPRequest.Header.Set(contentMD5Header, v) +} diff --git a/github.com/aws/aws-sdk-go/private/protocol/eventstream/debug.go b/github.com/aws/aws-sdk-go/private/protocol/eventstream/debug.go index ecc7bf82fa..151054971a 100644 --- a/github.com/aws/aws-sdk-go/private/protocol/eventstream/debug.go +++ b/github.com/aws/aws-sdk-go/private/protocol/eventstream/debug.go @@ -101,7 +101,7 @@ func (hs *decodedHeaders) UnmarshalJSON(b []byte) error { } headers.Set(h.Name, value) } - (*hs) = decodedHeaders(headers) + *hs = decodedHeaders(headers) return nil } diff --git a/github.com/aws/aws-sdk-go/private/protocol/eventstream/decode.go b/github.com/aws/aws-sdk-go/private/protocol/eventstream/decode.go index 4b972b2d66..4743393918 100644 --- a/github.com/aws/aws-sdk-go/private/protocol/eventstream/decode.go +++ b/github.com/aws/aws-sdk-go/private/protocol/eventstream/decode.go @@ -21,10 +21,24 @@ type Decoder struct { // NewDecoder initializes and returns a Decoder for decoding event // stream messages from the reader provided. -func NewDecoder(r io.Reader) *Decoder { - return &Decoder{ +func NewDecoder(r io.Reader, opts ...func(*Decoder)) *Decoder { + d := &Decoder{ r: r, } + + for _, opt := range opts { + opt(d) + } + + return d +} + +// DecodeWithLogger adds a logger to be used by the decoder when decoding +// stream events. +func DecodeWithLogger(logger aws.Logger) func(*Decoder) { + return func(d *Decoder) { + d.logger = logger + } } // Decode attempts to decode a single message from the event stream reader. @@ -40,6 +54,15 @@ func (d *Decoder) Decode(payloadBuf []byte) (m Message, err error) { }() } + m, err = Decode(reader, payloadBuf) + + return m, err +} + +// Decode attempts to decode a single message from the event stream reader. +// Will return the event stream message, or error if Decode fails to read +// the message from the reader. +func Decode(reader io.Reader, payloadBuf []byte) (m Message, err error) { crc := crc32.New(crc32IEEETable) hashReader := io.TeeReader(reader, crc) @@ -72,12 +95,6 @@ func (d *Decoder) Decode(payloadBuf []byte) (m Message, err error) { return m, nil } -// UseLogger specifies the Logger that that the decoder should use to log the -// message decode to. -func (d *Decoder) UseLogger(logger aws.Logger) { - d.logger = logger -} - func logMessageDecode(logger aws.Logger, msgBuf *bytes.Buffer, msg Message, decodeErr error) { w := bytes.NewBuffer(nil) defer func() { logger.Log(w.String()) }() diff --git a/github.com/aws/aws-sdk-go/private/protocol/eventstream/encode.go b/github.com/aws/aws-sdk-go/private/protocol/eventstream/encode.go index 150a60981d..ffade3bc0c 100644 --- a/github.com/aws/aws-sdk-go/private/protocol/eventstream/encode.go +++ b/github.com/aws/aws-sdk-go/private/protocol/eventstream/encode.go @@ -3,61 +3,107 @@ package eventstream import ( "bytes" "encoding/binary" + "encoding/hex" + "encoding/json" + "fmt" "hash" "hash/crc32" "io" + + "github.com/aws/aws-sdk-go/aws" ) // Encoder provides EventStream message encoding. type Encoder struct { - w io.Writer + w io.Writer + logger aws.Logger headersBuf *bytes.Buffer } // NewEncoder initializes and returns an Encoder to encode Event Stream // messages to an io.Writer. -func NewEncoder(w io.Writer) *Encoder { - return &Encoder{ +func NewEncoder(w io.Writer, opts ...func(*Encoder)) *Encoder { + e := &Encoder{ w: w, headersBuf: bytes.NewBuffer(nil), } + + for _, opt := range opts { + opt(e) + } + + return e +} + +// EncodeWithLogger adds a logger to be used by the encode when decoding +// stream events. +func EncodeWithLogger(logger aws.Logger) func(*Encoder) { + return func(d *Encoder) { + d.logger = logger + } } // Encode encodes a single EventStream message to the io.Writer the Encoder // was created with. An error is returned if writing the message fails. -func (e *Encoder) Encode(msg Message) error { +func (e *Encoder) Encode(msg Message) (err error) { e.headersBuf.Reset() - err := encodeHeaders(e.headersBuf, msg.Headers) - if err != nil { + writer := e.w + if e.logger != nil { + encodeMsgBuf := bytes.NewBuffer(nil) + writer = io.MultiWriter(writer, encodeMsgBuf) + defer func() { + logMessageEncode(e.logger, encodeMsgBuf, msg, err) + }() + } + + if err = EncodeHeaders(e.headersBuf, msg.Headers); err != nil { return err } crc := crc32.New(crc32IEEETable) - hashWriter := io.MultiWriter(e.w, crc) + hashWriter := io.MultiWriter(writer, crc) headersLen := uint32(e.headersBuf.Len()) payloadLen := uint32(len(msg.Payload)) - if err := encodePrelude(hashWriter, crc, headersLen, payloadLen); err != nil { + if err = encodePrelude(hashWriter, crc, headersLen, payloadLen); err != nil { return err } if headersLen > 0 { - if _, err := io.Copy(hashWriter, e.headersBuf); err != nil { + if _, err = io.Copy(hashWriter, e.headersBuf); err != nil { return err } } if payloadLen > 0 { - if _, err := hashWriter.Write(msg.Payload); err != nil { + if _, err = hashWriter.Write(msg.Payload); err != nil { return err } } msgCRC := crc.Sum32() - return binary.Write(e.w, binary.BigEndian, msgCRC) + return binary.Write(writer, binary.BigEndian, msgCRC) +} + +func logMessageEncode(logger aws.Logger, msgBuf *bytes.Buffer, msg Message, encodeErr error) { + w := bytes.NewBuffer(nil) + defer func() { logger.Log(w.String()) }() + + fmt.Fprintf(w, "Message to encode:\n") + encoder := json.NewEncoder(w) + if err := encoder.Encode(msg); err != nil { + fmt.Fprintf(w, "Failed to get encoded message, %v\n", err) + } + + if encodeErr != nil { + fmt.Fprintf(w, "Encode error: %v\n", encodeErr) + return + } + + fmt.Fprintf(w, "Raw message:\n%s\n", hex.Dump(msgBuf.Bytes())) } func encodePrelude(w io.Writer, crc hash.Hash32, headersLen, payloadLen uint32) error { @@ -86,7 +132,9 @@ func encodePrelude(w io.Writer, crc hash.Hash32, headersLen, payloadLen uint32) return nil } -func encodeHeaders(w io.Writer, headers Headers) error { +// EncodeHeaders writes the header values to the writer encoded in the event +// stream format. Returns an error if a header fails to encode. +func EncodeHeaders(w io.Writer, headers Headers) error { for _, h := range headers { hn := headerName{ Len: uint8(len(h.Name)), diff --git a/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/error.go b/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/error.go index 5ea5a988b6..34c2e89d53 100644 --- a/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/error.go +++ b/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/error.go @@ -1,6 +1,9 @@ package eventstreamapi -import "fmt" +import ( + "fmt" + "sync" +) type messageError struct { code string @@ -22,3 +25,53 @@ func (e messageError) Error() string { func (e messageError) OrigErr() error { return nil } + +// OnceError wraps the behavior of recording an error +// once and signal on a channel when this has occurred. +// Signaling is done by closing of the channel. +// +// Type is safe for concurrent usage. +type OnceError struct { + mu sync.RWMutex + err error + ch chan struct{} +} + +// NewOnceError return a new OnceError +func NewOnceError() *OnceError { + return &OnceError{ + ch: make(chan struct{}, 1), + } +} + +// Err acquires a read-lock and returns an +// error if one has been set. +func (e *OnceError) Err() error { + e.mu.RLock() + err := e.err + e.mu.RUnlock() + + return err +} + +// SetError acquires a write-lock and will set +// the underlying error value if one has not been set. +func (e *OnceError) SetError(err error) { + if err == nil { + return + } + + e.mu.Lock() + if e.err == nil { + e.err = err + close(e.ch) + } + e.mu.Unlock() +} + +// ErrorSet returns a channel that will be used to signal +// that an error has been set. This channel will be closed +// when the error value has been set for OnceError. +func (e *OnceError) ErrorSet() <-chan struct{} { + return e.ch +} diff --git a/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/api.go b/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/reader.go similarity index 76% rename from github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/api.go rename to github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/reader.go index 97937c8e59..0e4aa42f3e 100644 --- a/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/api.go +++ b/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/reader.go @@ -2,9 +2,7 @@ package eventstreamapi import ( "fmt" - "io" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/private/protocol" "github.com/aws/aws-sdk-go/private/protocol/eventstream" ) @@ -15,27 +13,8 @@ type Unmarshaler interface { UnmarshalEvent(protocol.PayloadUnmarshaler, eventstream.Message) error } -// EventStream headers with specific meaning to async API functionality. -const ( - MessageTypeHeader = `:message-type` // Identifies type of message. - EventMessageType = `event` - ErrorMessageType = `error` - ExceptionMessageType = `exception` - - // Message Events - EventTypeHeader = `:event-type` // Identifies message event type e.g. "Stats". - - // Message Error - ErrorCodeHeader = `:error-code` - ErrorMessageHeader = `:error-message` - - // Message Exception - ExceptionTypeHeader = `:exception-type` -) - // EventReader provides reading from the EventStream of an reader. type EventReader struct { - reader io.ReadCloser decoder *eventstream.Decoder unmarshalerForEventType func(string) (Unmarshaler, error) @@ -47,27 +26,18 @@ type EventReader struct { // NewEventReader returns a EventReader built from the reader and unmarshaler // provided. Use ReadStream method to start reading from the EventStream. func NewEventReader( - reader io.ReadCloser, + decoder *eventstream.Decoder, payloadUnmarshaler protocol.PayloadUnmarshaler, unmarshalerForEventType func(string) (Unmarshaler, error), ) *EventReader { return &EventReader{ - reader: reader, - decoder: eventstream.NewDecoder(reader), + decoder: decoder, payloadUnmarshaler: payloadUnmarshaler, unmarshalerForEventType: unmarshalerForEventType, payloadBuf: make([]byte, 10*1024), } } -// UseLogger instructs the EventReader to use the logger and log level -// specified. -func (r *EventReader) UseLogger(logger aws.Logger, logLevel aws.LogLevelType) { - if logger != nil && logLevel.Matches(aws.LogDebugWithEventStreamBody) { - r.decoder.UseLogger(logger) - } -} - // ReadEvent attempts to read a message from the EventStream and return the // unmarshaled event value that the message is for. // @@ -95,15 +65,27 @@ func (r *EventReader) ReadEvent() (event interface{}, err error) { case EventMessageType: return r.unmarshalEventMessage(msg) case ExceptionMessageType: - err = r.unmarshalEventException(msg) - return nil, err + return nil, r.unmarshalEventException(msg) case ErrorMessageType: return nil, r.unmarshalErrorMessage(msg) default: - return nil, fmt.Errorf("unknown eventstream message type, %v", typ) + return nil, &UnknownMessageTypeError{ + Type: typ, Message: msg.Clone(), + } } } +// UnknownMessageTypeError provides an error when a message is received from +// the stream, but the reader is unable to determine what kind of message it is. +type UnknownMessageTypeError struct { + Type string + Message eventstream.Message +} + +func (e *UnknownMessageTypeError) Error() string { + return "unknown eventstream message type, " + e.Type +} + func (r *EventReader) unmarshalEventMessage( msg eventstream.Message, ) (event interface{}, err error) { @@ -174,11 +156,6 @@ func (r *EventReader) unmarshalErrorMessage(msg eventstream.Message) (err error) return msgErr } -// Close closes the EventReader's EventStream reader. -func (r *EventReader) Close() error { - return r.reader.Close() -} - // GetHeaderString returns the value of the header as a string. If the header // is not set or the value is not a string an error will be returned. func GetHeaderString(msg eventstream.Message, headerName string) (string, error) { diff --git a/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/shared.go b/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/shared.go new file mode 100644 index 0000000000..e46b8acc20 --- /dev/null +++ b/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/shared.go @@ -0,0 +1,23 @@ +package eventstreamapi + +// EventStream headers with specific meaning to async API functionality. +const ( + ChunkSignatureHeader = `:chunk-signature` // chunk signature for message + DateHeader = `:date` // Date header for signature + + // Message header and values + MessageTypeHeader = `:message-type` // Identifies type of message. + EventMessageType = `event` + ErrorMessageType = `error` + ExceptionMessageType = `exception` + + // Message Events + EventTypeHeader = `:event-type` // Identifies message event type e.g. "Stats". + + // Message Error + ErrorCodeHeader = `:error-code` + ErrorMessageHeader = `:error-message` + + // Message Exception + ExceptionTypeHeader = `:exception-type` +) diff --git a/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/signer.go b/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/signer.go new file mode 100644 index 0000000000..3a7ba5cd57 --- /dev/null +++ b/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/signer.go @@ -0,0 +1,123 @@ +package eventstreamapi + +import ( + "bytes" + "strings" + "time" + + "github.com/aws/aws-sdk-go/private/protocol/eventstream" +) + +var timeNow = time.Now + +// StreamSigner defines an interface for the implementation of signing of event stream payloads +type StreamSigner interface { + GetSignature(headers, payload []byte, date time.Time) ([]byte, error) +} + +// SignEncoder envelopes event stream messages +// into an event stream message payload with included +// signature headers using the provided signer and encoder. +type SignEncoder struct { + signer StreamSigner + encoder Encoder + bufEncoder *BufferEncoder + + closeErr error + closed bool +} + +// NewSignEncoder returns a new SignEncoder using the provided stream signer and +// event stream encoder. +func NewSignEncoder(signer StreamSigner, encoder Encoder) *SignEncoder { + // TODO: Need to pass down logging + + return &SignEncoder{ + signer: signer, + encoder: encoder, + bufEncoder: NewBufferEncoder(), + } +} + +// Close encodes a final event stream signing envelope with an empty event stream +// payload. This final end-frame is used to mark the conclusion of the stream. +func (s *SignEncoder) Close() error { + if s.closed { + return s.closeErr + } + + if err := s.encode([]byte{}); err != nil { + if strings.Contains(err.Error(), "on closed pipe") { + return nil + } + + s.closeErr = err + s.closed = true + return s.closeErr + } + + return nil +} + +// Encode takes the provided message and add envelopes the message +// with the required signature. +func (s *SignEncoder) Encode(msg eventstream.Message) error { + payload, err := s.bufEncoder.Encode(msg) + if err != nil { + return err + } + + return s.encode(payload) +} + +func (s SignEncoder) encode(payload []byte) error { + date := timeNow() + + var msg eventstream.Message + msg.Headers.Set(DateHeader, eventstream.TimestampValue(date)) + msg.Payload = payload + + var headers bytes.Buffer + if err := eventstream.EncodeHeaders(&headers, msg.Headers); err != nil { + return err + } + + sig, err := s.signer.GetSignature(headers.Bytes(), msg.Payload, date) + if err != nil { + return err + } + + msg.Headers.Set(ChunkSignatureHeader, eventstream.BytesValue(sig)) + + return s.encoder.Encode(msg) +} + +// BufferEncoder is a utility that provides a buffered +// event stream encoder +type BufferEncoder struct { + encoder Encoder + buffer *bytes.Buffer +} + +// NewBufferEncoder returns a new BufferEncoder initialized +// with a 1024 byte buffer. +func NewBufferEncoder() *BufferEncoder { + buf := bytes.NewBuffer(make([]byte, 1024)) + return &BufferEncoder{ + encoder: eventstream.NewEncoder(buf), + buffer: buf, + } +} + +// Encode returns the encoded message as a byte slice. +// The returned byte slice will be modified on the next encode call +// and should not be held onto. +func (e *BufferEncoder) Encode(msg eventstream.Message) ([]byte, error) { + e.buffer.Reset() + + if err := e.encoder.Encode(msg); err != nil { + return nil, err + } + + return e.buffer.Bytes(), nil +} diff --git a/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/stream_writer.go b/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/stream_writer.go new file mode 100644 index 0000000000..433bb1630a --- /dev/null +++ b/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/stream_writer.go @@ -0,0 +1,129 @@ +package eventstreamapi + +import ( + "fmt" + "io" + "sync" + + "github.com/aws/aws-sdk-go/aws" +) + +// StreamWriter provides concurrent safe writing to an event stream. +type StreamWriter struct { + eventWriter *EventWriter + stream chan eventWriteAsyncReport + + done chan struct{} + closeOnce sync.Once + err *OnceError + + streamCloser io.Closer +} + +// NewStreamWriter returns a StreamWriter for the event writer, and stream +// closer provided. +func NewStreamWriter(eventWriter *EventWriter, streamCloser io.Closer) *StreamWriter { + w := &StreamWriter{ + eventWriter: eventWriter, + streamCloser: streamCloser, + stream: make(chan eventWriteAsyncReport), + done: make(chan struct{}), + err: NewOnceError(), + } + go w.writeStream() + + return w +} + +// Close terminates the writers ability to write new events to the stream. Any +// future call to Send will fail with an error. +func (w *StreamWriter) Close() error { + w.closeOnce.Do(w.safeClose) + return w.Err() +} + +func (w *StreamWriter) safeClose() { + close(w.done) +} + +// ErrorSet returns a channel which will be closed +// if an error occurs. +func (w *StreamWriter) ErrorSet() <-chan struct{} { + return w.err.ErrorSet() +} + +// Err returns any error that occurred while attempting to write an event to the +// stream. +func (w *StreamWriter) Err() error { + return w.err.Err() +} + +// Send writes a single event to the stream returning an error if the write +// failed. +// +// Send may be called concurrently. Events will be written to the stream +// safely. +func (w *StreamWriter) Send(ctx aws.Context, event Marshaler) error { + if err := w.Err(); err != nil { + return err + } + + resultCh := make(chan error) + wrapped := eventWriteAsyncReport{ + Event: event, + Result: resultCh, + } + + select { + case w.stream <- wrapped: + case <-ctx.Done(): + return ctx.Err() + case <-w.done: + return fmt.Errorf("stream closed, unable to send event") + } + + select { + case err := <-resultCh: + return err + case <-ctx.Done(): + return ctx.Err() + case <-w.done: + return fmt.Errorf("stream closed, unable to send event") + } +} + +func (w *StreamWriter) writeStream() { + defer w.Close() + + for { + select { + case wrapper := <-w.stream: + err := w.eventWriter.WriteEvent(wrapper.Event) + wrapper.ReportResult(w.done, err) + if err != nil { + w.err.SetError(err) + return + } + + case <-w.done: + if err := w.streamCloser.Close(); err != nil { + w.err.SetError(err) + } + return + } + } +} + +type eventWriteAsyncReport struct { + Event Marshaler + Result chan<- error +} + +func (e eventWriteAsyncReport) ReportResult(cancel <-chan struct{}, err error) bool { + select { + case e.Result <- err: + return true + case <-cancel: + return false + } +} diff --git a/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/writer.go b/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/writer.go new file mode 100644 index 0000000000..10a3823dfa --- /dev/null +++ b/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/writer.go @@ -0,0 +1,109 @@ +package eventstreamapi + +import ( + "github.com/aws/aws-sdk-go/private/protocol" + "github.com/aws/aws-sdk-go/private/protocol/eventstream" +) + +// Marshaler provides a marshaling interface for event types to event stream +// messages. +type Marshaler interface { + MarshalEvent(protocol.PayloadMarshaler) (eventstream.Message, error) +} + +// Encoder is an stream encoder that will encode an event stream message for +// the transport. +type Encoder interface { + Encode(eventstream.Message) error +} + +// EventWriter provides a wrapper around the underlying event stream encoder +// for an io.WriteCloser. +type EventWriter struct { + encoder Encoder + payloadMarshaler protocol.PayloadMarshaler + eventTypeFor func(Marshaler) (string, error) +} + +// NewEventWriter returns a new event stream writer, that will write to the +// writer provided. Use the WriteEvent method to write an event to the stream. +func NewEventWriter(encoder Encoder, pm protocol.PayloadMarshaler, eventTypeFor func(Marshaler) (string, error), +) *EventWriter { + return &EventWriter{ + encoder: encoder, + payloadMarshaler: pm, + eventTypeFor: eventTypeFor, + } +} + +// WriteEvent writes an event to the stream. Returns an error if the event +// fails to marshal into a message, or writing to the underlying writer fails. +func (w *EventWriter) WriteEvent(event Marshaler) error { + msg, err := w.marshal(event) + if err != nil { + return err + } + + return w.encoder.Encode(msg) +} + +func (w *EventWriter) marshal(event Marshaler) (eventstream.Message, error) { + eventType, err := w.eventTypeFor(event) + if err != nil { + return eventstream.Message{}, err + } + + msg, err := event.MarshalEvent(w.payloadMarshaler) + if err != nil { + return eventstream.Message{}, err + } + + msg.Headers.Set(EventTypeHeader, eventstream.StringValue(eventType)) + return msg, nil +} + +//type EventEncoder struct { +// encoder Encoder +// ppayloadMarshaler protocol.PayloadMarshaler +// eventTypeFor func(Marshaler) (string, error) +//} +// +//func (e EventEncoder) Encode(event Marshaler) error { +// msg, err := e.marshal(event) +// if err != nil { +// return err +// } +// +// return w.encoder.Encode(msg) +//} +// +//func (e EventEncoder) marshal(event Marshaler) (eventstream.Message, error) { +// eventType, err := w.eventTypeFor(event) +// if err != nil { +// return eventstream.Message{}, err +// } +// +// msg, err := event.MarshalEvent(w.payloadMarshaler) +// if err != nil { +// return eventstream.Message{}, err +// } +// +// msg.Headers.Set(EventTypeHeader, eventstream.StringValue(eventType)) +// return msg, nil +//} +// +//func (w *EventWriter) marshal(event Marshaler) (eventstream.Message, error) { +// eventType, err := w.eventTypeFor(event) +// if err != nil { +// return eventstream.Message{}, err +// } +// +// msg, err := event.MarshalEvent(w.payloadMarshaler) +// if err != nil { +// return eventstream.Message{}, err +// } +// +// msg.Headers.Set(EventTypeHeader, eventstream.StringValue(eventType)) +// return msg, nil +//} +// diff --git a/github.com/aws/aws-sdk-go/private/protocol/eventstream/header.go b/github.com/aws/aws-sdk-go/private/protocol/eventstream/header.go index 3b44dde2f3..f6f8c5674e 100644 --- a/github.com/aws/aws-sdk-go/private/protocol/eventstream/header.go +++ b/github.com/aws/aws-sdk-go/private/protocol/eventstream/header.go @@ -52,6 +52,15 @@ func (hs *Headers) Del(name string) { } } +// Clone returns a deep copy of the headers +func (hs Headers) Clone() Headers { + o := make(Headers, 0, len(hs)) + for _, h := range hs { + o.Set(h.Name, h.Value) + } + return o +} + func decodeHeaders(r io.Reader) (Headers, error) { hs := Headers{} diff --git a/github.com/aws/aws-sdk-go/private/protocol/eventstream/header_value.go b/github.com/aws/aws-sdk-go/private/protocol/eventstream/header_value.go index e3fc0766a9..9f509d8f6d 100644 --- a/github.com/aws/aws-sdk-go/private/protocol/eventstream/header_value.go +++ b/github.com/aws/aws-sdk-go/private/protocol/eventstream/header_value.go @@ -461,6 +461,11 @@ func (v *TimestampValue) decode(r io.Reader) error { return nil } +// MarshalJSON implements the json.Marshaler interface +func (v TimestampValue) MarshalJSON() ([]byte, error) { + return []byte(v.String()), nil +} + func timeFromEpochMilli(t int64) time.Time { secs := t / 1e3 msec := t % 1e3 diff --git a/github.com/aws/aws-sdk-go/private/protocol/eventstream/message.go b/github.com/aws/aws-sdk-go/private/protocol/eventstream/message.go index 2dc012a66e..f7427da039 100644 --- a/github.com/aws/aws-sdk-go/private/protocol/eventstream/message.go +++ b/github.com/aws/aws-sdk-go/private/protocol/eventstream/message.go @@ -27,7 +27,7 @@ func (m *Message) rawMessage() (rawMessage, error) { if len(m.Headers) > 0 { var headers bytes.Buffer - if err := encodeHeaders(&headers, m.Headers); err != nil { + if err := EncodeHeaders(&headers, m.Headers); err != nil { return rawMessage{}, err } raw.Headers = headers.Bytes() @@ -57,6 +57,20 @@ func (m *Message) rawMessage() (rawMessage, error) { return raw, nil } +// Clone returns a deep copy of the message. +func (m Message) Clone() Message { + var payload []byte + if m.Payload != nil { + payload = make([]byte, len(m.Payload)) + copy(payload, m.Payload) + } + + return Message{ + Headers: m.Headers.Clone(), + Payload: payload, + } +} + type messagePrelude struct { Length uint32 HeadersLen uint32 diff --git a/github.com/aws/aws-sdk-go/private/protocol/json/jsonutil/build.go b/github.com/aws/aws-sdk-go/private/protocol/json/jsonutil/build.go new file mode 100644 index 0000000000..864fb6704b --- /dev/null +++ b/github.com/aws/aws-sdk-go/private/protocol/json/jsonutil/build.go @@ -0,0 +1,296 @@ +// Package jsonutil provides JSON serialization of AWS requests and responses. +package jsonutil + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "math" + "reflect" + "sort" + "strconv" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/private/protocol" +) + +var timeType = reflect.ValueOf(time.Time{}).Type() +var byteSliceType = reflect.ValueOf([]byte{}).Type() + +// BuildJSON builds a JSON string for a given object v. +func BuildJSON(v interface{}) ([]byte, error) { + var buf bytes.Buffer + + err := buildAny(reflect.ValueOf(v), &buf, "") + return buf.Bytes(), err +} + +func buildAny(value reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) error { + origVal := value + value = reflect.Indirect(value) + if !value.IsValid() { + return nil + } + + vtype := value.Type() + + t := tag.Get("type") + if t == "" { + switch vtype.Kind() { + case reflect.Struct: + // also it can't be a time object + if value.Type() != timeType { + t = "structure" + } + case reflect.Slice: + // also it can't be a byte slice + if _, ok := value.Interface().([]byte); !ok { + t = "list" + } + case reflect.Map: + // cannot be a JSONValue map + if _, ok := value.Interface().(aws.JSONValue); !ok { + t = "map" + } + } + } + + switch t { + case "structure": + if field, ok := vtype.FieldByName("_"); ok { + tag = field.Tag + } + return buildStruct(value, buf, tag) + case "list": + return buildList(value, buf, tag) + case "map": + return buildMap(value, buf, tag) + default: + return buildScalar(origVal, buf, tag) + } +} + +func buildStruct(value reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) error { + if !value.IsValid() { + return nil + } + + // unwrap payloads + if payload := tag.Get("payload"); payload != "" { + field, _ := value.Type().FieldByName(payload) + tag = field.Tag + value = elemOf(value.FieldByName(payload)) + + if !value.IsValid() { + return nil + } + } + + buf.WriteByte('{') + + t := value.Type() + first := true + for i := 0; i < t.NumField(); i++ { + member := value.Field(i) + + // This allocates the most memory. + // Additionally, we cannot skip nil fields due to + // idempotency auto filling. + field := t.Field(i) + + if field.PkgPath != "" { + continue // ignore unexported fields + } + if field.Tag.Get("json") == "-" { + continue + } + if field.Tag.Get("location") != "" { + continue // ignore non-body elements + } + if field.Tag.Get("ignore") != "" { + continue + } + + if protocol.CanSetIdempotencyToken(member, field) { + token := protocol.GetIdempotencyToken() + member = reflect.ValueOf(&token) + } + + if (member.Kind() == reflect.Ptr || member.Kind() == reflect.Slice || member.Kind() == reflect.Map) && member.IsNil() { + continue // ignore unset fields + } + + if first { + first = false + } else { + buf.WriteByte(',') + } + + // figure out what this field is called + name := field.Name + if locName := field.Tag.Get("locationName"); locName != "" { + name = locName + } + + writeString(name, buf) + buf.WriteString(`:`) + + err := buildAny(member, buf, field.Tag) + if err != nil { + return err + } + + } + + buf.WriteString("}") + + return nil +} + +func buildList(value reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) error { + buf.WriteString("[") + + for i := 0; i < value.Len(); i++ { + buildAny(value.Index(i), buf, "") + + if i < value.Len()-1 { + buf.WriteString(",") + } + } + + buf.WriteString("]") + + return nil +} + +type sortedValues []reflect.Value + +func (sv sortedValues) Len() int { return len(sv) } +func (sv sortedValues) Swap(i, j int) { sv[i], sv[j] = sv[j], sv[i] } +func (sv sortedValues) Less(i, j int) bool { return sv[i].String() < sv[j].String() } + +func buildMap(value reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) error { + buf.WriteString("{") + + sv := sortedValues(value.MapKeys()) + sort.Sort(sv) + + for i, k := range sv { + if i > 0 { + buf.WriteByte(',') + } + + writeString(k.String(), buf) + buf.WriteString(`:`) + + buildAny(value.MapIndex(k), buf, "") + } + + buf.WriteString("}") + + return nil +} + +func buildScalar(v reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) error { + // prevents allocation on the heap. + scratch := [64]byte{} + switch value := reflect.Indirect(v); value.Kind() { + case reflect.String: + writeString(value.String(), buf) + case reflect.Bool: + if value.Bool() { + buf.WriteString("true") + } else { + buf.WriteString("false") + } + case reflect.Int64: + buf.Write(strconv.AppendInt(scratch[:0], value.Int(), 10)) + case reflect.Float64: + f := value.Float() + if math.IsInf(f, 0) || math.IsNaN(f) { + return &json.UnsupportedValueError{Value: v, Str: strconv.FormatFloat(f, 'f', -1, 64)} + } + buf.Write(strconv.AppendFloat(scratch[:0], f, 'f', -1, 64)) + default: + switch converted := value.Interface().(type) { + case time.Time: + format := tag.Get("timestampFormat") + if len(format) == 0 { + format = protocol.UnixTimeFormatName + } + + ts := protocol.FormatTime(format, converted) + if format != protocol.UnixTimeFormatName { + ts = `"` + ts + `"` + } + + buf.WriteString(ts) + case []byte: + if !value.IsNil() { + buf.WriteByte('"') + if len(converted) < 1024 { + // for small buffers, using Encode directly is much faster. + dst := make([]byte, base64.StdEncoding.EncodedLen(len(converted))) + base64.StdEncoding.Encode(dst, converted) + buf.Write(dst) + } else { + // for large buffers, avoid unnecessary extra temporary + // buffer space. + enc := base64.NewEncoder(base64.StdEncoding, buf) + enc.Write(converted) + enc.Close() + } + buf.WriteByte('"') + } + case aws.JSONValue: + str, err := protocol.EncodeJSONValue(converted, protocol.QuotedEscape) + if err != nil { + return fmt.Errorf("unable to encode JSONValue, %v", err) + } + buf.WriteString(str) + default: + return fmt.Errorf("unsupported JSON value %v (%s)", value.Interface(), value.Type()) + } + } + return nil +} + +var hex = "0123456789abcdef" + +func writeString(s string, buf *bytes.Buffer) { + buf.WriteByte('"') + for i := 0; i < len(s); i++ { + if s[i] == '"' { + buf.WriteString(`\"`) + } else if s[i] == '\\' { + buf.WriteString(`\\`) + } else if s[i] == '\b' { + buf.WriteString(`\b`) + } else if s[i] == '\f' { + buf.WriteString(`\f`) + } else if s[i] == '\r' { + buf.WriteString(`\r`) + } else if s[i] == '\t' { + buf.WriteString(`\t`) + } else if s[i] == '\n' { + buf.WriteString(`\n`) + } else if s[i] < 32 { + buf.WriteString("\\u00") + buf.WriteByte(hex[s[i]>>4]) + buf.WriteByte(hex[s[i]&0xF]) + } else { + buf.WriteByte(s[i]) + } + } + buf.WriteByte('"') +} + +// Returns the reflection element of a value, if it is a pointer. +func elemOf(value reflect.Value) reflect.Value { + for value.Kind() == reflect.Ptr { + value = value.Elem() + } + return value +} diff --git a/github.com/aws/aws-sdk-go/private/protocol/json/jsonutil/unmarshal.go b/github.com/aws/aws-sdk-go/private/protocol/json/jsonutil/unmarshal.go new file mode 100644 index 0000000000..5e9499699b --- /dev/null +++ b/github.com/aws/aws-sdk-go/private/protocol/json/jsonutil/unmarshal.go @@ -0,0 +1,282 @@ +package jsonutil + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "reflect" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/private/protocol" +) + +// UnmarshalJSONError unmarshal's the reader's JSON document into the passed in +// type. The value to unmarshal the json document into must be a pointer to the +// type. +func UnmarshalJSONError(v interface{}, stream io.Reader) error { + var errBuf bytes.Buffer + body := io.TeeReader(stream, &errBuf) + + err := json.NewDecoder(body).Decode(v) + if err != nil { + msg := "failed decoding error message" + if err == io.EOF { + msg = "error message missing" + err = nil + } + return awserr.NewUnmarshalError(err, msg, errBuf.Bytes()) + } + + return nil +} + +// UnmarshalJSON reads a stream and unmarshals the results in object v. +func UnmarshalJSON(v interface{}, stream io.Reader) error { + var out interface{} + + err := json.NewDecoder(stream).Decode(&out) + if err == io.EOF { + return nil + } else if err != nil { + return err + } + + return unmarshaler{}.unmarshalAny(reflect.ValueOf(v), out, "") +} + +// UnmarshalJSONCaseInsensitive reads a stream and unmarshals the result into the +// object v. Ignores casing for structure members. +func UnmarshalJSONCaseInsensitive(v interface{}, stream io.Reader) error { + var out interface{} + + err := json.NewDecoder(stream).Decode(&out) + if err == io.EOF { + return nil + } else if err != nil { + return err + } + + return unmarshaler{ + caseInsensitive: true, + }.unmarshalAny(reflect.ValueOf(v), out, "") +} + +type unmarshaler struct { + caseInsensitive bool +} + +func (u unmarshaler) unmarshalAny(value reflect.Value, data interface{}, tag reflect.StructTag) error { + vtype := value.Type() + if vtype.Kind() == reflect.Ptr { + vtype = vtype.Elem() // check kind of actual element type + } + + t := tag.Get("type") + if t == "" { + switch vtype.Kind() { + case reflect.Struct: + // also it can't be a time object + if _, ok := value.Interface().(*time.Time); !ok { + t = "structure" + } + case reflect.Slice: + // also it can't be a byte slice + if _, ok := value.Interface().([]byte); !ok { + t = "list" + } + case reflect.Map: + // cannot be a JSONValue map + if _, ok := value.Interface().(aws.JSONValue); !ok { + t = "map" + } + } + } + + switch t { + case "structure": + if field, ok := vtype.FieldByName("_"); ok { + tag = field.Tag + } + return u.unmarshalStruct(value, data, tag) + case "list": + return u.unmarshalList(value, data, tag) + case "map": + return u.unmarshalMap(value, data, tag) + default: + return u.unmarshalScalar(value, data, tag) + } +} + +func (u unmarshaler) unmarshalStruct(value reflect.Value, data interface{}, tag reflect.StructTag) error { + if data == nil { + return nil + } + mapData, ok := data.(map[string]interface{}) + if !ok { + return fmt.Errorf("JSON value is not a structure (%#v)", data) + } + + t := value.Type() + if value.Kind() == reflect.Ptr { + if value.IsNil() { // create the structure if it's nil + s := reflect.New(value.Type().Elem()) + value.Set(s) + value = s + } + + value = value.Elem() + t = t.Elem() + } + + // unwrap any payloads + if payload := tag.Get("payload"); payload != "" { + field, _ := t.FieldByName(payload) + return u.unmarshalAny(value.FieldByName(payload), data, field.Tag) + } + + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + if field.PkgPath != "" { + continue // ignore unexported fields + } + + // figure out what this field is called + name := field.Name + if locName := field.Tag.Get("locationName"); locName != "" { + name = locName + } + if u.caseInsensitive { + if _, ok := mapData[name]; !ok { + // Fallback to uncased name search if the exact name didn't match. + for kn, v := range mapData { + if strings.EqualFold(kn, name) { + mapData[name] = v + } + } + } + } + + member := value.FieldByIndex(field.Index) + err := u.unmarshalAny(member, mapData[name], field.Tag) + if err != nil { + return err + } + } + return nil +} + +func (u unmarshaler) unmarshalList(value reflect.Value, data interface{}, tag reflect.StructTag) error { + if data == nil { + return nil + } + listData, ok := data.([]interface{}) + if !ok { + return fmt.Errorf("JSON value is not a list (%#v)", data) + } + + if value.IsNil() { + l := len(listData) + value.Set(reflect.MakeSlice(value.Type(), l, l)) + } + + for i, c := range listData { + err := u.unmarshalAny(value.Index(i), c, "") + if err != nil { + return err + } + } + + return nil +} + +func (u unmarshaler) unmarshalMap(value reflect.Value, data interface{}, tag reflect.StructTag) error { + if data == nil { + return nil + } + mapData, ok := data.(map[string]interface{}) + if !ok { + return fmt.Errorf("JSON value is not a map (%#v)", data) + } + + if value.IsNil() { + value.Set(reflect.MakeMap(value.Type())) + } + + for k, v := range mapData { + kvalue := reflect.ValueOf(k) + vvalue := reflect.New(value.Type().Elem()).Elem() + + u.unmarshalAny(vvalue, v, "") + value.SetMapIndex(kvalue, vvalue) + } + + return nil +} + +func (u unmarshaler) unmarshalScalar(value reflect.Value, data interface{}, tag reflect.StructTag) error { + + switch d := data.(type) { + case nil: + return nil // nothing to do here + case string: + switch value.Interface().(type) { + case *string: + value.Set(reflect.ValueOf(&d)) + case []byte: + b, err := base64.StdEncoding.DecodeString(d) + if err != nil { + return err + } + value.Set(reflect.ValueOf(b)) + case *time.Time: + format := tag.Get("timestampFormat") + if len(format) == 0 { + format = protocol.ISO8601TimeFormatName + } + + t, err := protocol.ParseTime(format, d) + if err != nil { + return err + } + value.Set(reflect.ValueOf(&t)) + case aws.JSONValue: + // No need to use escaping as the value is a non-quoted string. + v, err := protocol.DecodeJSONValue(d, protocol.NoEscape) + if err != nil { + return err + } + value.Set(reflect.ValueOf(v)) + default: + return fmt.Errorf("unsupported value: %v (%s)", value.Interface(), value.Type()) + } + case float64: + switch value.Interface().(type) { + case *int64: + di := int64(d) + value.Set(reflect.ValueOf(&di)) + case *float64: + value.Set(reflect.ValueOf(&d)) + case *time.Time: + // Time unmarshaled from a float64 can only be epoch seconds + t := time.Unix(int64(d), 0).UTC() + value.Set(reflect.ValueOf(&t)) + default: + return fmt.Errorf("unsupported value: %v (%s)", value.Interface(), value.Type()) + } + case bool: + switch value.Interface().(type) { + case *bool: + value.Set(reflect.ValueOf(&d)) + default: + return fmt.Errorf("unsupported value: %v (%s)", value.Interface(), value.Type()) + } + default: + return fmt.Errorf("unsupported JSON value (%v)", data) + } + return nil +} diff --git a/github.com/aws/aws-sdk-go/private/protocol/jsonrpc/jsonrpc.go b/github.com/aws/aws-sdk-go/private/protocol/jsonrpc/jsonrpc.go new file mode 100644 index 0000000000..a029217e4c --- /dev/null +++ b/github.com/aws/aws-sdk-go/private/protocol/jsonrpc/jsonrpc.go @@ -0,0 +1,88 @@ +// Package jsonrpc provides JSON RPC utilities for serialization of AWS +// requests and responses. +package jsonrpc + +//go:generate go run -tags codegen ../../../private/model/cli/gen-protocol-tests ../../../models/protocol_tests/input/json.json build_test.go +//go:generate go run -tags codegen ../../../private/model/cli/gen-protocol-tests ../../../models/protocol_tests/output/json.json unmarshal_test.go + +import ( + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/private/protocol/json/jsonutil" + "github.com/aws/aws-sdk-go/private/protocol/rest" +) + +var emptyJSON = []byte("{}") + +// BuildHandler is a named request handler for building jsonrpc protocol +// requests +var BuildHandler = request.NamedHandler{ + Name: "awssdk.jsonrpc.Build", + Fn: Build, +} + +// UnmarshalHandler is a named request handler for unmarshaling jsonrpc +// protocol requests +var UnmarshalHandler = request.NamedHandler{ + Name: "awssdk.jsonrpc.Unmarshal", + Fn: Unmarshal, +} + +// UnmarshalMetaHandler is a named request handler for unmarshaling jsonrpc +// protocol request metadata +var UnmarshalMetaHandler = request.NamedHandler{ + Name: "awssdk.jsonrpc.UnmarshalMeta", + Fn: UnmarshalMeta, +} + +// Build builds a JSON payload for a JSON RPC request. +func Build(req *request.Request) { + var buf []byte + var err error + if req.ParamsFilled() { + buf, err = jsonutil.BuildJSON(req.Params) + if err != nil { + req.Error = awserr.New(request.ErrCodeSerialization, "failed encoding JSON RPC request", err) + return + } + } else { + buf = emptyJSON + } + + if req.ClientInfo.TargetPrefix != "" || string(buf) != "{}" { + req.SetBufferBody(buf) + } + + if req.ClientInfo.TargetPrefix != "" { + target := req.ClientInfo.TargetPrefix + "." + req.Operation.Name + req.HTTPRequest.Header.Add("X-Amz-Target", target) + } + + // Only set the content type if one is not already specified and an + // JSONVersion is specified. + if ct, v := req.HTTPRequest.Header.Get("Content-Type"), req.ClientInfo.JSONVersion; len(ct) == 0 && len(v) != 0 { + jsonVersion := req.ClientInfo.JSONVersion + req.HTTPRequest.Header.Set("Content-Type", "application/x-amz-json-"+jsonVersion) + } +} + +// Unmarshal unmarshals a response for a JSON RPC service. +func Unmarshal(req *request.Request) { + defer req.HTTPResponse.Body.Close() + if req.DataFilled() { + err := jsonutil.UnmarshalJSON(req.Data, req.HTTPResponse.Body) + if err != nil { + req.Error = awserr.NewRequestFailure( + awserr.New(request.ErrCodeSerialization, "failed decoding JSON RPC response", err), + req.HTTPResponse.StatusCode, + req.RequestID, + ) + } + } + return +} + +// UnmarshalMeta unmarshals headers from a response for a JSON RPC service. +func UnmarshalMeta(req *request.Request) { + rest.UnmarshalMeta(req) +} diff --git a/github.com/aws/aws-sdk-go/private/protocol/jsonrpc/unmarshal_error.go b/github.com/aws/aws-sdk-go/private/protocol/jsonrpc/unmarshal_error.go new file mode 100644 index 0000000000..c0c52e2db0 --- /dev/null +++ b/github.com/aws/aws-sdk-go/private/protocol/jsonrpc/unmarshal_error.go @@ -0,0 +1,107 @@ +package jsonrpc + +import ( + "bytes" + "io" + "io/ioutil" + "net/http" + "strings" + + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/private/protocol" + "github.com/aws/aws-sdk-go/private/protocol/json/jsonutil" +) + +// UnmarshalTypedError provides unmarshaling errors API response errors +// for both typed and untyped errors. +type UnmarshalTypedError struct { + exceptions map[string]func(protocol.ResponseMetadata) error +} + +// NewUnmarshalTypedError returns an UnmarshalTypedError initialized for the +// set of exception names to the error unmarshalers +func NewUnmarshalTypedError(exceptions map[string]func(protocol.ResponseMetadata) error) *UnmarshalTypedError { + return &UnmarshalTypedError{ + exceptions: exceptions, + } +} + +// UnmarshalError attempts to unmarshal the HTTP response error as a known +// error type. If unable to unmarshal the error type, the generic SDK error +// type will be used. +func (u *UnmarshalTypedError) UnmarshalError( + resp *http.Response, + respMeta protocol.ResponseMetadata, +) (error, error) { + + var buf bytes.Buffer + var jsonErr jsonErrorResponse + teeReader := io.TeeReader(resp.Body, &buf) + err := jsonutil.UnmarshalJSONError(&jsonErr, teeReader) + if err != nil { + return nil, err + } + body := ioutil.NopCloser(&buf) + + // Code may be separated by hash(#), with the last element being the code + // used by the SDK. + codeParts := strings.SplitN(jsonErr.Code, "#", 2) + code := codeParts[len(codeParts)-1] + msg := jsonErr.Message + + if fn, ok := u.exceptions[code]; ok { + // If exception code is know, use associated constructor to get a value + // for the exception that the JSON body can be unmarshaled into. + v := fn(respMeta) + err := jsonutil.UnmarshalJSONCaseInsensitive(v, body) + if err != nil { + return nil, err + } + + return v, nil + } + + // fallback to unmodeled generic exceptions + return awserr.NewRequestFailure( + awserr.New(code, msg, nil), + respMeta.StatusCode, + respMeta.RequestID, + ), nil +} + +// UnmarshalErrorHandler is a named request handler for unmarshaling jsonrpc +// protocol request errors +var UnmarshalErrorHandler = request.NamedHandler{ + Name: "awssdk.jsonrpc.UnmarshalError", + Fn: UnmarshalError, +} + +// UnmarshalError unmarshals an error response for a JSON RPC service. +func UnmarshalError(req *request.Request) { + defer req.HTTPResponse.Body.Close() + + var jsonErr jsonErrorResponse + err := jsonutil.UnmarshalJSONError(&jsonErr, req.HTTPResponse.Body) + if err != nil { + req.Error = awserr.NewRequestFailure( + awserr.New(request.ErrCodeSerialization, + "failed to unmarshal error message", err), + req.HTTPResponse.StatusCode, + req.RequestID, + ) + return + } + + codes := strings.SplitN(jsonErr.Code, "#", 2) + req.Error = awserr.NewRequestFailure( + awserr.New(codes[len(codes)-1], jsonErr.Message, nil), + req.HTTPResponse.StatusCode, + req.RequestID, + ) +} + +type jsonErrorResponse struct { + Code string `json:"__type"` + Message string `json:"message"` +} diff --git a/github.com/aws/aws-sdk-go/private/protocol/payload.go b/github.com/aws/aws-sdk-go/private/protocol/payload.go index e21614a125..0ea0647a57 100644 --- a/github.com/aws/aws-sdk-go/private/protocol/payload.go +++ b/github.com/aws/aws-sdk-go/private/protocol/payload.go @@ -64,7 +64,7 @@ func (h HandlerPayloadMarshal) MarshalPayload(w io.Writer, v interface{}) error metadata.ClientInfo{}, request.Handlers{}, nil, - &request.Operation{HTTPMethod: "GET"}, + &request.Operation{HTTPMethod: "PUT"}, v, nil, ) diff --git a/github.com/aws/aws-sdk-go/private/protocol/protocol.go b/github.com/aws/aws-sdk-go/private/protocol/protocol.go new file mode 100644 index 0000000000..9d521dcb95 --- /dev/null +++ b/github.com/aws/aws-sdk-go/private/protocol/protocol.go @@ -0,0 +1,49 @@ +package protocol + +import ( + "fmt" + "strings" + + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/request" +) + +// RequireHTTPMinProtocol request handler is used to enforce that +// the target endpoint supports the given major and minor HTTP protocol version. +type RequireHTTPMinProtocol struct { + Major, Minor int +} + +// Handler will mark the request.Request with an error if the +// target endpoint did not connect with the required HTTP protocol +// major and minor version. +func (p RequireHTTPMinProtocol) Handler(r *request.Request) { + if r.Error != nil || r.HTTPResponse == nil { + return + } + + if !strings.HasPrefix(r.HTTPResponse.Proto, "HTTP") { + r.Error = newMinHTTPProtoError(p.Major, p.Minor, r) + } + + if r.HTTPResponse.ProtoMajor < p.Major || r.HTTPResponse.ProtoMinor < p.Minor { + r.Error = newMinHTTPProtoError(p.Major, p.Minor, r) + } +} + +// ErrCodeMinimumHTTPProtocolError error code is returned when the target endpoint +// did not match the required HTTP major and minor protocol version. +const ErrCodeMinimumHTTPProtocolError = "MinimumHTTPProtocolError" + +func newMinHTTPProtoError(major, minor int, r *request.Request) error { + return awserr.NewRequestFailure( + awserr.New("MinimumHTTPProtocolError", + fmt.Sprintf( + "operation requires minimum HTTP protocol of HTTP/%d.%d, but was %s", + major, minor, r.HTTPResponse.Proto, + ), + nil, + ), + r.HTTPResponse.StatusCode, r.RequestID, + ) +} diff --git a/github.com/aws/aws-sdk-go/private/protocol/query/build.go b/github.com/aws/aws-sdk-go/private/protocol/query/build.go index 60e5b09d54..d40346a779 100644 --- a/github.com/aws/aws-sdk-go/private/protocol/query/build.go +++ b/github.com/aws/aws-sdk-go/private/protocol/query/build.go @@ -1,7 +1,7 @@ // Package query provides serialization of AWS query requests, and responses. package query -//go:generate go run -tags codegen ../../../models/protocol_tests/generate.go ../../../models/protocol_tests/input/query.json build_test.go +//go:generate go run -tags codegen ../../../private/model/cli/gen-protocol-tests ../../../models/protocol_tests/input/query.json build_test.go import ( "net/url" @@ -21,7 +21,7 @@ func Build(r *request.Request) { "Version": {r.ClientInfo.APIVersion}, } if err := queryutil.Parse(body, r.Params, false); err != nil { - r.Error = awserr.New("SerializationError", "failed encoding Query request", err) + r.Error = awserr.New(request.ErrCodeSerialization, "failed encoding Query request", err) return } diff --git a/github.com/aws/aws-sdk-go/private/protocol/query/unmarshal.go b/github.com/aws/aws-sdk-go/private/protocol/query/unmarshal.go index 3495c73070..9231e95d16 100644 --- a/github.com/aws/aws-sdk-go/private/protocol/query/unmarshal.go +++ b/github.com/aws/aws-sdk-go/private/protocol/query/unmarshal.go @@ -1,6 +1,6 @@ package query -//go:generate go run -tags codegen ../../../models/protocol_tests/generate.go ../../../models/protocol_tests/output/query.json unmarshal_test.go +//go:generate go run -tags codegen ../../../private/model/cli/gen-protocol-tests ../../../models/protocol_tests/output/query.json unmarshal_test.go import ( "encoding/xml" @@ -24,7 +24,7 @@ func Unmarshal(r *request.Request) { err := xmlutil.UnmarshalXML(r.Data, decoder, r.Operation.Name+"Result") if err != nil { r.Error = awserr.NewRequestFailure( - awserr.New("SerializationError", "failed decoding Query response", err), + awserr.New(request.ErrCodeSerialization, "failed decoding Query response", err), r.HTTPResponse.StatusCode, r.RequestID, ) diff --git a/github.com/aws/aws-sdk-go/private/protocol/query/unmarshal_error.go b/github.com/aws/aws-sdk-go/private/protocol/query/unmarshal_error.go index 46d354e826..831b0110c5 100644 --- a/github.com/aws/aws-sdk-go/private/protocol/query/unmarshal_error.go +++ b/github.com/aws/aws-sdk-go/private/protocol/query/unmarshal_error.go @@ -2,73 +2,68 @@ package query import ( "encoding/xml" - "io/ioutil" + "fmt" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil" ) +// UnmarshalErrorHandler is a name request handler to unmarshal request errors +var UnmarshalErrorHandler = request.NamedHandler{Name: "awssdk.query.UnmarshalError", Fn: UnmarshalError} + type xmlErrorResponse struct { - XMLName xml.Name `xml:"ErrorResponse"` - Code string `xml:"Error>Code"` - Message string `xml:"Error>Message"` - RequestID string `xml:"RequestId"` + Code string `xml:"Error>Code"` + Message string `xml:"Error>Message"` + RequestID string `xml:"RequestId"` } -type xmlServiceUnavailableResponse struct { - XMLName xml.Name `xml:"ServiceUnavailableException"` +type xmlResponseError struct { + xmlErrorResponse } -// UnmarshalErrorHandler is a name request handler to unmarshal request errors -var UnmarshalErrorHandler = request.NamedHandler{Name: "awssdk.query.UnmarshalError", Fn: UnmarshalError} +func (e *xmlResponseError) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + const svcUnavailableTagName = "ServiceUnavailableException" + const errorResponseTagName = "ErrorResponse" + + switch start.Name.Local { + case svcUnavailableTagName: + e.Code = svcUnavailableTagName + e.Message = "service is unavailable" + return d.Skip() + + case errorResponseTagName: + return d.DecodeElement(&e.xmlErrorResponse, &start) + + default: + return fmt.Errorf("unknown error response tag, %v", start) + } +} // UnmarshalError unmarshals an error response for an AWS Query service. func UnmarshalError(r *request.Request) { defer r.HTTPResponse.Body.Close() - bodyBytes, err := ioutil.ReadAll(r.HTTPResponse.Body) + var respErr xmlResponseError + err := xmlutil.UnmarshalXMLError(&respErr, r.HTTPResponse.Body) if err != nil { r.Error = awserr.NewRequestFailure( - awserr.New("SerializationError", "failed to read from query HTTP response body", err), + awserr.New(request.ErrCodeSerialization, + "failed to unmarshal error message", err), r.HTTPResponse.StatusCode, r.RequestID, ) return } - // First check for specific error - resp := xmlErrorResponse{} - decodeErr := xml.Unmarshal(bodyBytes, &resp) - if decodeErr == nil { - reqID := resp.RequestID - if reqID == "" { - reqID = r.RequestID - } - r.Error = awserr.NewRequestFailure( - awserr.New(resp.Code, resp.Message, nil), - r.HTTPResponse.StatusCode, - reqID, - ) - return - } - - // Check for unhandled error - servUnavailResp := xmlServiceUnavailableResponse{} - unavailErr := xml.Unmarshal(bodyBytes, &servUnavailResp) - if unavailErr == nil { - r.Error = awserr.NewRequestFailure( - awserr.New("ServiceUnavailableException", "service is unavailable", nil), - r.HTTPResponse.StatusCode, - r.RequestID, - ) - return + reqID := respErr.RequestID + if len(reqID) == 0 { + reqID = r.RequestID } - // Failed to retrieve any error message from the response body r.Error = awserr.NewRequestFailure( - awserr.New("SerializationError", - "failed to decode query XML error response", decodeErr), + awserr.New(respErr.Code, respErr.Message, nil), r.HTTPResponse.StatusCode, - r.RequestID, + reqID, ) } diff --git a/github.com/aws/aws-sdk-go/private/protocol/rest/build.go b/github.com/aws/aws-sdk-go/private/protocol/rest/build.go index b34f5258a4..1301b149d3 100644 --- a/github.com/aws/aws-sdk-go/private/protocol/rest/build.go +++ b/github.com/aws/aws-sdk-go/private/protocol/rest/build.go @@ -25,6 +25,8 @@ var noEscape [256]bool var errValueNotSet = fmt.Errorf("value not set") +var byteSliceType = reflect.TypeOf([]byte{}) + func init() { for i := 0; i < len(noEscape); i++ { // AWS expects every character except these to be escaped @@ -94,6 +96,14 @@ func buildLocationElements(r *request.Request, v reflect.Value, buildGETQuery bo continue } + // Support the ability to customize values to be marshaled as a + // blob even though they were modeled as a string. Required for S3 + // API operations like SSECustomerKey is modeled as stirng but + // required to be base64 encoded in request. + if field.Tag.Get("marshal-as") == "blob" { + m = m.Convert(byteSliceType) + } + var err error switch field.Tag.Get("location") { case "headers": // header maps @@ -137,7 +147,7 @@ func buildBody(r *request.Request, v reflect.Value) { case string: r.SetStringBody(reader) default: - r.Error = awserr.New("SerializationError", + r.Error = awserr.New(request.ErrCodeSerialization, "failed to encode REST request", fmt.Errorf("unknown payload type %s", payload.Type())) } @@ -152,9 +162,12 @@ func buildHeader(header *http.Header, v reflect.Value, name string, tag reflect. if err == errValueNotSet { return nil } else if err != nil { - return awserr.New("SerializationError", "failed to encode REST request", err) + return awserr.New(request.ErrCodeSerialization, "failed to encode REST request", err) } + name = strings.TrimSpace(name) + str = strings.TrimSpace(str) + header.Add(name, str) return nil @@ -167,11 +180,13 @@ func buildHeaderMap(header *http.Header, v reflect.Value, tag reflect.StructTag) if err == errValueNotSet { continue } else if err != nil { - return awserr.New("SerializationError", "failed to encode REST request", err) + return awserr.New(request.ErrCodeSerialization, "failed to encode REST request", err) } + keyStr := strings.TrimSpace(key.String()) + str = strings.TrimSpace(str) - header.Add(prefix+key.String(), str) + header.Add(prefix+keyStr, str) } return nil } @@ -181,7 +196,7 @@ func buildURI(u *url.URL, v reflect.Value, name string, tag reflect.StructTag) e if err == errValueNotSet { return nil } else if err != nil { - return awserr.New("SerializationError", "failed to encode REST request", err) + return awserr.New(request.ErrCodeSerialization, "failed to encode REST request", err) } u.Path = strings.Replace(u.Path, "{"+name+"}", value, -1) @@ -214,7 +229,7 @@ func buildQueryString(query url.Values, v reflect.Value, name string, tag reflec if err == errValueNotSet { return nil } else if err != nil { - return awserr.New("SerializationError", "failed to encode REST request", err) + return awserr.New(request.ErrCodeSerialization, "failed to encode REST request", err) } query.Set(name, str) } diff --git a/github.com/aws/aws-sdk-go/private/protocol/rest/unmarshal.go b/github.com/aws/aws-sdk-go/private/protocol/rest/unmarshal.go index 33fd53b126..92f8b4d9a4 100644 --- a/github.com/aws/aws-sdk-go/private/protocol/rest/unmarshal.go +++ b/github.com/aws/aws-sdk-go/private/protocol/rest/unmarshal.go @@ -15,6 +15,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/request" + awsStrings "github.com/aws/aws-sdk-go/internal/strings" "github.com/aws/aws-sdk-go/private/protocol" ) @@ -28,7 +29,9 @@ var UnmarshalMetaHandler = request.NamedHandler{Name: "awssdk.rest.UnmarshalMeta func Unmarshal(r *request.Request) { if r.DataFilled() { v := reflect.Indirect(reflect.ValueOf(r.Data)) - unmarshalBody(r, v) + if err := unmarshalBody(r, v); err != nil { + r.Error = err + } } } @@ -40,12 +43,21 @@ func UnmarshalMeta(r *request.Request) { r.RequestID = r.HTTPResponse.Header.Get("X-Amz-Request-Id") } if r.DataFilled() { - v := reflect.Indirect(reflect.ValueOf(r.Data)) - unmarshalLocationElements(r, v) + if err := UnmarshalResponse(r.HTTPResponse, r.Data, aws.BoolValue(r.Config.LowerCaseHeaderMaps)); err != nil { + r.Error = err + } } } -func unmarshalBody(r *request.Request, v reflect.Value) { +// UnmarshalResponse attempts to unmarshal the REST response headers to +// the data type passed in. The type must be a pointer. An error is returned +// with any error unmarshaling the response into the target datatype. +func UnmarshalResponse(resp *http.Response, data interface{}, lowerCaseHeaderMaps bool) error { + v := reflect.Indirect(reflect.ValueOf(data)) + return unmarshalLocationElements(resp, v, lowerCaseHeaderMaps) +} + +func unmarshalBody(r *request.Request, v reflect.Value) error { if field, ok := v.Type().FieldByName("_"); ok { if payloadName := field.Tag.Get("payload"); payloadName != "" { pfield, _ := v.Type().FieldByName(payloadName) @@ -57,35 +69,38 @@ func unmarshalBody(r *request.Request, v reflect.Value) { defer r.HTTPResponse.Body.Close() b, err := ioutil.ReadAll(r.HTTPResponse.Body) if err != nil { - r.Error = awserr.New("SerializationError", "failed to decode REST response", err) - } else { - payload.Set(reflect.ValueOf(b)) + return awserr.New(request.ErrCodeSerialization, "failed to decode REST response", err) } + + payload.Set(reflect.ValueOf(b)) + case *string: defer r.HTTPResponse.Body.Close() b, err := ioutil.ReadAll(r.HTTPResponse.Body) if err != nil { - r.Error = awserr.New("SerializationError", "failed to decode REST response", err) - } else { - str := string(b) - payload.Set(reflect.ValueOf(&str)) + return awserr.New(request.ErrCodeSerialization, "failed to decode REST response", err) } + + str := string(b) + payload.Set(reflect.ValueOf(&str)) + default: switch payload.Type().String() { case "io.ReadCloser": payload.Set(reflect.ValueOf(r.HTTPResponse.Body)) + case "io.ReadSeeker": b, err := ioutil.ReadAll(r.HTTPResponse.Body) if err != nil { - r.Error = awserr.New("SerializationError", + return awserr.New(request.ErrCodeSerialization, "failed to read response body", err) - return } payload.Set(reflect.ValueOf(ioutil.NopCloser(bytes.NewReader(b)))) + default: io.Copy(ioutil.Discard, r.HTTPResponse.Body) - defer r.HTTPResponse.Body.Close() - r.Error = awserr.New("SerializationError", + r.HTTPResponse.Body.Close() + return awserr.New(request.ErrCodeSerialization, "failed to decode REST response", fmt.Errorf("unknown payload type %s", payload.Type())) } @@ -94,9 +109,11 @@ func unmarshalBody(r *request.Request, v reflect.Value) { } } } + + return nil } -func unmarshalLocationElements(r *request.Request, v reflect.Value) { +func unmarshalLocationElements(resp *http.Response, v reflect.Value, lowerCaseHeaderMaps bool) error { for i := 0; i < v.NumField(); i++ { m, field := v.Field(i), v.Type().Field(i) if n := field.Name; n[0:1] == strings.ToLower(n[0:1]) { @@ -111,26 +128,25 @@ func unmarshalLocationElements(r *request.Request, v reflect.Value) { switch field.Tag.Get("location") { case "statusCode": - unmarshalStatusCode(m, r.HTTPResponse.StatusCode) + unmarshalStatusCode(m, resp.StatusCode) + case "header": - err := unmarshalHeader(m, r.HTTPResponse.Header.Get(name), field.Tag) + err := unmarshalHeader(m, resp.Header.Get(name), field.Tag) if err != nil { - r.Error = awserr.New("SerializationError", "failed to decode REST response", err) - break + return awserr.New(request.ErrCodeSerialization, "failed to decode REST response", err) } + case "headers": prefix := field.Tag.Get("locationName") - err := unmarshalHeaderMap(m, r.HTTPResponse.Header, prefix) + err := unmarshalHeaderMap(m, resp.Header, prefix, lowerCaseHeaderMaps) if err != nil { - r.Error = awserr.New("SerializationError", "failed to decode REST response", err) - break + awserr.New(request.ErrCodeSerialization, "failed to decode REST response", err) } } } - if r.Error != nil { - return - } } + + return nil } func unmarshalStatusCode(v reflect.Value, statusCode int) { @@ -145,29 +161,45 @@ func unmarshalStatusCode(v reflect.Value, statusCode int) { } } -func unmarshalHeaderMap(r reflect.Value, headers http.Header, prefix string) error { +func unmarshalHeaderMap(r reflect.Value, headers http.Header, prefix string, normalize bool) error { + if len(headers) == 0 { + return nil + } switch r.Interface().(type) { case map[string]*string: // we only support string map value types out := map[string]*string{} for k, v := range headers { - k = http.CanonicalHeaderKey(k) - if strings.HasPrefix(strings.ToLower(k), strings.ToLower(prefix)) { + if awsStrings.HasPrefixFold(k, prefix) { + if normalize == true { + k = strings.ToLower(k) + } else { + k = http.CanonicalHeaderKey(k) + } out[k[len(prefix):]] = &v[0] } } - r.Set(reflect.ValueOf(out)) + if len(out) != 0 { + r.Set(reflect.ValueOf(out)) + } + } return nil } func unmarshalHeader(v reflect.Value, header string, tag reflect.StructTag) error { - isJSONValue := tag.Get("type") == "jsonvalue" - if isJSONValue { + switch tag.Get("type") { + case "jsonvalue": if len(header) == 0 { return nil } - } else if !v.IsValid() || (header == "" && v.Elem().Kind() != reflect.String) { - return nil + case "blob": + if len(header) == 0 { + return nil + } + default: + if !v.IsValid() || (header == "" && v.Elem().Kind() != reflect.String) { + return nil + } } switch v.Interface().(type) { @@ -178,7 +210,7 @@ func unmarshalHeader(v reflect.Value, header string, tag reflect.StructTag) erro if err != nil { return err } - v.Set(reflect.ValueOf(&b)) + v.Set(reflect.ValueOf(b)) case *bool: b, err := strconv.ParseBool(header) if err != nil { diff --git a/github.com/aws/aws-sdk-go/private/protocol/restxml/restxml.go b/github.com/aws/aws-sdk-go/private/protocol/restxml/restxml.go index b0f4e24566..b1ae364871 100644 --- a/github.com/aws/aws-sdk-go/private/protocol/restxml/restxml.go +++ b/github.com/aws/aws-sdk-go/private/protocol/restxml/restxml.go @@ -2,8 +2,8 @@ // requests and responses. package restxml -//go:generate go run -tags codegen ../../../models/protocol_tests/generate.go ../../../models/protocol_tests/input/rest-xml.json build_test.go -//go:generate go run -tags codegen ../../../models/protocol_tests/generate.go ../../../models/protocol_tests/output/rest-xml.json unmarshal_test.go +//go:generate go run -tags codegen ../../../private/model/cli/gen-protocol-tests ../../../models/protocol_tests/input/rest-xml.json build_test.go +//go:generate go run -tags codegen ../../../private/model/cli/gen-protocol-tests ../../../models/protocol_tests/output/rest-xml.json unmarshal_test.go import ( "bytes" @@ -37,8 +37,9 @@ func Build(r *request.Request) { err := xmlutil.BuildXML(r.Params, xml.NewEncoder(&buf)) if err != nil { r.Error = awserr.NewRequestFailure( - awserr.New("SerializationError", "failed to encode rest XML request", err), - r.HTTPResponse.StatusCode, + awserr.New(request.ErrCodeSerialization, + "failed to encode rest XML request", err), + 0, r.RequestID, ) return @@ -55,7 +56,8 @@ func Unmarshal(r *request.Request) { err := xmlutil.UnmarshalXML(r.Data, decoder, "") if err != nil { r.Error = awserr.NewRequestFailure( - awserr.New("SerializationError", "failed to decode REST XML response", err), + awserr.New(request.ErrCodeSerialization, + "failed to decode REST XML response", err), r.HTTPResponse.StatusCode, r.RequestID, ) diff --git a/github.com/aws/aws-sdk-go/private/protocol/timestamp.go b/github.com/aws/aws-sdk-go/private/protocol/timestamp.go index b7ed6c6f81..d2f6dae532 100644 --- a/github.com/aws/aws-sdk-go/private/protocol/timestamp.go +++ b/github.com/aws/aws-sdk-go/private/protocol/timestamp.go @@ -1,8 +1,11 @@ package protocol import ( + "math" "strconv" "time" + + "github.com/aws/aws-sdk-go/internal/sdkmath" ) // Names of time formats supported by the SDK @@ -13,12 +16,19 @@ const ( ) // Time formats supported by the SDK +// Output time is intended to not contain decimals const ( // RFC 7231#section-7.1.1.1 timetamp format. e.g Tue, 29 Apr 2014 18:30:38 GMT RFC822TimeFormat = "Mon, 2 Jan 2006 15:04:05 GMT" + // This format is used for output time without seconds precision + RFC822OutputTimeFormat = "Mon, 02 Jan 2006 15:04:05 GMT" + // RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z - ISO8601TimeFormat = "2006-01-02T15:04:05Z" + ISO8601TimeFormat = "2006-01-02T15:04:05.999999999Z" + + // This format is used for output time without seconds precision + ISO8601OutputTimeFormat = "2006-01-02T15:04:05Z" ) // IsKnownTimestampFormat returns if the timestamp format name @@ -42,11 +52,12 @@ func FormatTime(name string, t time.Time) string { switch name { case RFC822TimeFormatName: - return t.Format(RFC822TimeFormat) + return t.Format(RFC822OutputTimeFormat) case ISO8601TimeFormatName: - return t.Format(ISO8601TimeFormat) + return t.Format(ISO8601OutputTimeFormat) case UnixTimeFormatName: - return strconv.FormatInt(t.Unix(), 10) + ms := t.UnixNano() / int64(time.Millisecond) + return strconv.FormatFloat(float64(ms)/1e3, 'f', -1, 64) default: panic("unknown timestamp format name, " + name) } @@ -62,10 +73,12 @@ func ParseTime(formatName, value string) (time.Time, error) { return time.Parse(ISO8601TimeFormat, value) case UnixTimeFormatName: v, err := strconv.ParseFloat(value, 64) + _, dec := math.Modf(v) + dec = sdkmath.Round(dec*1e3) / 1e3 //Rounds 0.1229999 to 0.123 if err != nil { return time.Time{}, err } - return time.Unix(int64(v), 0), nil + return time.Unix(int64(v), int64(dec*(1e9))), nil default: panic("unknown timestamp format name, " + formatName) } diff --git a/github.com/aws/aws-sdk-go/private/protocol/unmarshal.go b/github.com/aws/aws-sdk-go/private/protocol/unmarshal.go index da1a68111d..f614ef898b 100644 --- a/github.com/aws/aws-sdk-go/private/protocol/unmarshal.go +++ b/github.com/aws/aws-sdk-go/private/protocol/unmarshal.go @@ -19,3 +19,9 @@ func UnmarshalDiscardBody(r *request.Request) { io.Copy(ioutil.Discard, r.HTTPResponse.Body) r.HTTPResponse.Body.Close() } + +// ResponseMetadata provides the SDK response metadata attributes. +type ResponseMetadata struct { + StatusCode int + RequestID string +} diff --git a/github.com/aws/aws-sdk-go/private/protocol/unmarshal_error.go b/github.com/aws/aws-sdk-go/private/protocol/unmarshal_error.go new file mode 100644 index 0000000000..cc857f136c --- /dev/null +++ b/github.com/aws/aws-sdk-go/private/protocol/unmarshal_error.go @@ -0,0 +1,65 @@ +package protocol + +import ( + "net/http" + + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/request" +) + +// UnmarshalErrorHandler provides unmarshaling errors API response errors for +// both typed and untyped errors. +type UnmarshalErrorHandler struct { + unmarshaler ErrorUnmarshaler +} + +// ErrorUnmarshaler is an abstract interface for concrete implementations to +// unmarshal protocol specific response errors. +type ErrorUnmarshaler interface { + UnmarshalError(*http.Response, ResponseMetadata) (error, error) +} + +// NewUnmarshalErrorHandler returns an UnmarshalErrorHandler +// initialized for the set of exception names to the error unmarshalers +func NewUnmarshalErrorHandler(unmarshaler ErrorUnmarshaler) *UnmarshalErrorHandler { + return &UnmarshalErrorHandler{ + unmarshaler: unmarshaler, + } +} + +// UnmarshalErrorHandlerName is the name of the named handler. +const UnmarshalErrorHandlerName = "awssdk.protocol.UnmarshalError" + +// NamedHandler returns a NamedHandler for the unmarshaler using the set of +// errors the unmarshaler was initialized for. +func (u *UnmarshalErrorHandler) NamedHandler() request.NamedHandler { + return request.NamedHandler{ + Name: UnmarshalErrorHandlerName, + Fn: u.UnmarshalError, + } +} + +// UnmarshalError will attempt to unmarshal the API response's error message +// into either a generic SDK error type, or a typed error corresponding to the +// errors exception name. +func (u *UnmarshalErrorHandler) UnmarshalError(r *request.Request) { + defer r.HTTPResponse.Body.Close() + + respMeta := ResponseMetadata{ + StatusCode: r.HTTPResponse.StatusCode, + RequestID: r.RequestID, + } + + v, err := u.unmarshaler.UnmarshalError(r.HTTPResponse, respMeta) + if err != nil { + r.Error = awserr.NewRequestFailure( + awserr.New(request.ErrCodeSerialization, + "failed to unmarshal response error", err), + respMeta.StatusCode, + respMeta.RequestID, + ) + return + } + + r.Error = v +} diff --git a/github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil/build.go b/github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil/build.go index cf981fe951..09ad951595 100644 --- a/github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil/build.go +++ b/github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil/build.go @@ -8,6 +8,7 @@ import ( "reflect" "sort" "strconv" + "strings" "time" "github.com/aws/aws-sdk-go/private/protocol" @@ -60,6 +61,14 @@ func (b *xmlBuilder) buildValue(value reflect.Value, current *XMLNode, tag refle return nil } + xml := tag.Get("xml") + if len(xml) != 0 { + name := strings.SplitAfterN(xml, ",", 2)[0] + if name == "-" { + return nil + } + } + t := tag.Get("type") if t == "" { switch value.Kind() { diff --git a/github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil/sort.go b/github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil/sort.go new file mode 100644 index 0000000000..c1a511851f --- /dev/null +++ b/github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil/sort.go @@ -0,0 +1,32 @@ +package xmlutil + +import ( + "encoding/xml" + "strings" +) + +type xmlAttrSlice []xml.Attr + +func (x xmlAttrSlice) Len() int { + return len(x) +} + +func (x xmlAttrSlice) Less(i, j int) bool { + spaceI, spaceJ := x[i].Name.Space, x[j].Name.Space + localI, localJ := x[i].Name.Local, x[j].Name.Local + valueI, valueJ := x[i].Value, x[j].Value + + spaceCmp := strings.Compare(spaceI, spaceJ) + localCmp := strings.Compare(localI, localJ) + valueCmp := strings.Compare(valueI, valueJ) + + if spaceCmp == -1 || (spaceCmp == 0 && (localCmp == -1 || (localCmp == 0 && valueCmp == -1))) { + return true + } + + return false +} + +func (x xmlAttrSlice) Swap(i, j int) { + x[i], x[j] = x[j], x[i] +} diff --git a/github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil/unmarshal.go b/github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil/unmarshal.go index ff1ef6830b..107c053f8a 100644 --- a/github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil/unmarshal.go +++ b/github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil/unmarshal.go @@ -1,6 +1,7 @@ package xmlutil import ( + "bytes" "encoding/base64" "encoding/xml" "fmt" @@ -10,9 +11,27 @@ import ( "strings" "time" + "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/private/protocol" ) +// UnmarshalXMLError unmarshals the XML error from the stream into the value +// type specified. The value must be a pointer. If the message fails to +// unmarshal, the message content will be included in the returned error as a +// awserr.UnmarshalError. +func UnmarshalXMLError(v interface{}, stream io.Reader) error { + var errBuf bytes.Buffer + body := io.TeeReader(stream, &errBuf) + + err := xml.NewDecoder(body).Decode(v) + if err != nil && err != io.EOF { + return awserr.NewUnmarshalError(err, + "failed to unmarshal error message", errBuf.Bytes()) + } + + return nil +} + // UnmarshalXML deserializes an xml.Decoder into the container v. V // needs to match the shape of the XML expected to be decoded. // If the shape doesn't match unmarshaling will fail. @@ -45,6 +64,14 @@ func UnmarshalXML(v interface{}, d *xml.Decoder, wrapper string) error { // parse deserializes any value from the XMLNode. The type tag is used to infer the type, or reflect // will be used to determine the type from r. func parse(r reflect.Value, node *XMLNode, tag reflect.StructTag) error { + xml := tag.Get("xml") + if len(xml) != 0 { + name := strings.SplitAfterN(xml, ",", 2)[0] + if name == "-" { + return nil + } + } + rtype := r.Type() if rtype.Kind() == reflect.Ptr { rtype = rtype.Elem() // check kind of actual element type diff --git a/github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil/xml_to_struct.go b/github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil/xml_to_struct.go index 515ce15215..42f71648ee 100644 --- a/github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil/xml_to_struct.go +++ b/github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil/xml_to_struct.go @@ -119,7 +119,18 @@ func (n *XMLNode) findElem(name string) (string, bool) { // StructToXML writes an XMLNode to a xml.Encoder as tokens. func StructToXML(e *xml.Encoder, node *XMLNode, sorted bool) error { - e.EncodeToken(xml.StartElement{Name: node.Name, Attr: node.Attr}) + // Sort Attributes + attrs := node.Attr + if sorted { + sortedAttrs := make([]xml.Attr, len(attrs)) + for _, k := range node.Attr { + sortedAttrs = append(sortedAttrs, k) + } + sort.Sort(xmlAttrSlice(sortedAttrs)) + attrs = sortedAttrs + } + + e.EncodeToken(xml.StartElement{Name: node.Name, Attr: attrs}) if node.Text != "" { e.EncodeToken(xml.CharData([]byte(node.Text))) diff --git a/github.com/aws/aws-sdk-go/service/kms/api.go b/github.com/aws/aws-sdk-go/service/kms/api.go new file mode 100644 index 0000000000..8793c2d808 --- /dev/null +++ b/github.com/aws/aws-sdk-go/service/kms/api.go @@ -0,0 +1,14793 @@ +// Code generated by private/model/cli/gen-api/main.go. DO NOT EDIT. + +package kms + +import ( + "fmt" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awsutil" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/private/protocol" + "github.com/aws/aws-sdk-go/private/protocol/jsonrpc" +) + +const opCancelKeyDeletion = "CancelKeyDeletion" + +// CancelKeyDeletionRequest generates a "aws/request.Request" representing the +// client's request for the CancelKeyDeletion operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See CancelKeyDeletion for more information on using the CancelKeyDeletion +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the CancelKeyDeletionRequest method. +// req, resp := client.CancelKeyDeletionRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/CancelKeyDeletion +func (c *KMS) CancelKeyDeletionRequest(input *CancelKeyDeletionInput) (req *request.Request, output *CancelKeyDeletionOutput) { + op := &request.Operation{ + Name: opCancelKeyDeletion, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &CancelKeyDeletionInput{} + } + + output = &CancelKeyDeletionOutput{} + req = c.newRequest(op, input, output) + return +} + +// CancelKeyDeletion API operation for AWS Key Management Service. +// +// Cancels the deletion of a customer master key (CMK). When this operation +// succeeds, the key state of the CMK is Disabled. To enable the CMK, use EnableKey. +// You cannot perform this operation on a CMK in a different AWS account. +// +// For more information about scheduling and canceling deletion of a CMK, see +// Deleting Customer Master Keys (https://docs.aws.amazon.com/kms/latest/developerguide/deleting-keys.html) +// in the AWS Key Management Service Developer Guide. +// +// The CMK that you use for this operation must be in a compatible key state. +// For details, see How Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation CancelKeyDeletion for usage and error information. +// +// Returned Error Types: +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * InvalidArnException +// The request was rejected because a specified ARN, or an ARN in a key policy, +// is not valid. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/CancelKeyDeletion +func (c *KMS) CancelKeyDeletion(input *CancelKeyDeletionInput) (*CancelKeyDeletionOutput, error) { + req, out := c.CancelKeyDeletionRequest(input) + return out, req.Send() +} + +// CancelKeyDeletionWithContext is the same as CancelKeyDeletion with the addition of +// the ability to pass a context and additional request options. +// +// See CancelKeyDeletion for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) CancelKeyDeletionWithContext(ctx aws.Context, input *CancelKeyDeletionInput, opts ...request.Option) (*CancelKeyDeletionOutput, error) { + req, out := c.CancelKeyDeletionRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opConnectCustomKeyStore = "ConnectCustomKeyStore" + +// ConnectCustomKeyStoreRequest generates a "aws/request.Request" representing the +// client's request for the ConnectCustomKeyStore operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See ConnectCustomKeyStore for more information on using the ConnectCustomKeyStore +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the ConnectCustomKeyStoreRequest method. +// req, resp := client.ConnectCustomKeyStoreRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/ConnectCustomKeyStore +func (c *KMS) ConnectCustomKeyStoreRequest(input *ConnectCustomKeyStoreInput) (req *request.Request, output *ConnectCustomKeyStoreOutput) { + op := &request.Operation{ + Name: opConnectCustomKeyStore, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &ConnectCustomKeyStoreInput{} + } + + output = &ConnectCustomKeyStoreOutput{} + req = c.newRequest(op, input, output) + req.Handlers.Unmarshal.Swap(jsonrpc.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + return +} + +// ConnectCustomKeyStore API operation for AWS Key Management Service. +// +// Connects or reconnects a custom key store (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html) +// to its associated AWS CloudHSM cluster. +// +// The custom key store must be connected before you can create customer master +// keys (CMKs) in the key store or use the CMKs it contains. You can disconnect +// and reconnect a custom key store at any time. +// +// To connect a custom key store, its associated AWS CloudHSM cluster must have +// at least one active HSM. To get the number of active HSMs in a cluster, use +// the DescribeClusters (https://docs.aws.amazon.com/cloudhsm/latest/APIReference/API_DescribeClusters.html) +// operation. To add HSMs to the cluster, use the CreateHsm (https://docs.aws.amazon.com/cloudhsm/latest/APIReference/API_CreateHsm.html) +// operation. Also, the kmsuser crypto user (https://docs.aws.amazon.com/kms/latest/developerguide/key-store-concepts.html#concept-kmsuser) +// (CU) must not be logged into the cluster. This prevents AWS KMS from using +// this account to log in. +// +// The connection process can take an extended amount of time to complete; up +// to 20 minutes. This operation starts the connection process, but it does +// not wait for it to complete. When it succeeds, this operation quickly returns +// an HTTP 200 response and a JSON object with no properties. However, this +// response does not indicate that the custom key store is connected. To get +// the connection state of the custom key store, use the DescribeCustomKeyStores +// operation. +// +// During the connection process, AWS KMS finds the AWS CloudHSM cluster that +// is associated with the custom key store, creates the connection infrastructure, +// connects to the cluster, logs into the AWS CloudHSM client as the kmsuser +// CU, and rotates its password. +// +// The ConnectCustomKeyStore operation might fail for various reasons. To find +// the reason, use the DescribeCustomKeyStores operation and see the ConnectionErrorCode +// in the response. For help interpreting the ConnectionErrorCode, see CustomKeyStoresListEntry. +// +// To fix the failure, use the DisconnectCustomKeyStore operation to disconnect +// the custom key store, correct the error, use the UpdateCustomKeyStore operation +// if necessary, and then use ConnectCustomKeyStore again. +// +// If you are having trouble connecting or disconnecting a custom key store, +// see Troubleshooting a Custom Key Store (https://docs.aws.amazon.com/kms/latest/developerguide/fix-keystore.html) +// in the AWS Key Management Service Developer Guide. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation ConnectCustomKeyStore for usage and error information. +// +// Returned Error Types: +// * CloudHsmClusterNotActiveException +// The request was rejected because the AWS CloudHSM cluster that is associated +// with the custom key store is not active. Initialize and activate the cluster +// and try the command again. For detailed instructions, see Getting Started +// (https://docs.aws.amazon.com/cloudhsm/latest/userguide/getting-started.html) +// in the AWS CloudHSM User Guide. +// +// * CustomKeyStoreInvalidStateException +// The request was rejected because of the ConnectionState of the custom key +// store. To get the ConnectionState of a custom key store, use the DescribeCustomKeyStores +// operation. +// +// This exception is thrown under the following conditions: +// +// * You requested the CreateKey or GenerateRandom operation in a custom +// key store that is not connected. These operations are valid only when +// the custom key store ConnectionState is CONNECTED. +// +// * You requested the UpdateCustomKeyStore or DeleteCustomKeyStore operation +// on a custom key store that is not disconnected. This operation is valid +// only when the custom key store ConnectionState is DISCONNECTED. +// +// * You requested the ConnectCustomKeyStore operation on a custom key store +// with a ConnectionState of DISCONNECTING or FAILED. This operation is valid +// for all other ConnectionState values. +// +// * CustomKeyStoreNotFoundException +// The request was rejected because AWS KMS cannot find a custom key store with +// the specified key store name or ID. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * CloudHsmClusterInvalidConfigurationException +// The request was rejected because the associated AWS CloudHSM cluster did +// not meet the configuration requirements for a custom key store. +// +// * The cluster must be configured with private subnets in at least two +// different Availability Zones in the Region. +// +// * The security group for the cluster (https://docs.aws.amazon.com/cloudhsm/latest/userguide/configure-sg.html) +// (cloudhsm-cluster--sg) must include inbound rules and outbound +// rules that allow TCP traffic on ports 2223-2225. The Source in the inbound +// rules and the Destination in the outbound rules must match the security +// group ID. These rules are set by default when you create the cluster. +// Do not delete or change them. To get information about a particular security +// group, use the DescribeSecurityGroups (https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeSecurityGroups.html) +// operation. +// +// * The cluster must contain at least as many HSMs as the operation requires. +// To add HSMs, use the AWS CloudHSM CreateHsm (https://docs.aws.amazon.com/cloudhsm/latest/APIReference/API_CreateHsm.html) +// operation. For the CreateCustomKeyStore, UpdateCustomKeyStore, and CreateKey +// operations, the AWS CloudHSM cluster must have at least two active HSMs, +// each in a different Availability Zone. For the ConnectCustomKeyStore operation, +// the AWS CloudHSM must contain at least one active HSM. +// +// For information about the requirements for an AWS CloudHSM cluster that is +// associated with a custom key store, see Assemble the Prerequisites (https://docs.aws.amazon.com/kms/latest/developerguide/create-keystore.html#before-keystore) +// in the AWS Key Management Service Developer Guide. For information about +// creating a private subnet for an AWS CloudHSM cluster, see Create a Private +// Subnet (https://docs.aws.amazon.com/cloudhsm/latest/userguide/create-subnets.html) +// in the AWS CloudHSM User Guide. For information about cluster security groups, +// see Configure a Default Security Group (https://docs.aws.amazon.com/cloudhsm/latest/userguide/configure-sg.html) +// in the AWS CloudHSM User Guide . +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/ConnectCustomKeyStore +func (c *KMS) ConnectCustomKeyStore(input *ConnectCustomKeyStoreInput) (*ConnectCustomKeyStoreOutput, error) { + req, out := c.ConnectCustomKeyStoreRequest(input) + return out, req.Send() +} + +// ConnectCustomKeyStoreWithContext is the same as ConnectCustomKeyStore with the addition of +// the ability to pass a context and additional request options. +// +// See ConnectCustomKeyStore for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) ConnectCustomKeyStoreWithContext(ctx aws.Context, input *ConnectCustomKeyStoreInput, opts ...request.Option) (*ConnectCustomKeyStoreOutput, error) { + req, out := c.ConnectCustomKeyStoreRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opCreateAlias = "CreateAlias" + +// CreateAliasRequest generates a "aws/request.Request" representing the +// client's request for the CreateAlias operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See CreateAlias for more information on using the CreateAlias +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the CreateAliasRequest method. +// req, resp := client.CreateAliasRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/CreateAlias +func (c *KMS) CreateAliasRequest(input *CreateAliasInput) (req *request.Request, output *CreateAliasOutput) { + op := &request.Operation{ + Name: opCreateAlias, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &CreateAliasInput{} + } + + output = &CreateAliasOutput{} + req = c.newRequest(op, input, output) + req.Handlers.Unmarshal.Swap(jsonrpc.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + return +} + +// CreateAlias API operation for AWS Key Management Service. +// +// Creates a display name for a customer managed customer master key (CMK). +// You can use an alias to identify a CMK in cryptographic operations (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#cryptographic-operations), +// such as Encrypt and GenerateDataKey. You can change the CMK associated with +// the alias at any time. +// +// Aliases are easier to remember than key IDs. They can also help to simplify +// your applications. For example, if you use an alias in your code, you can +// change the CMK your code uses by associating a given alias with a different +// CMK. +// +// To run the same code in multiple AWS regions, use an alias in your code, +// such as alias/ApplicationKey. Then, in each AWS Region, create an alias/ApplicationKey +// alias that is associated with a CMK in that Region. When you run your code, +// it uses the alias/ApplicationKey CMK for that AWS Region without any Region-specific +// code. +// +// This operation does not return a response. To get the alias that you created, +// use the ListAliases operation. +// +// To use aliases successfully, be aware of the following information. +// +// * Each alias points to only one CMK at a time, although a single CMK can +// have multiple aliases. The alias and its associated CMK must be in the +// same AWS account and Region. +// +// * You can associate an alias with any customer managed CMK in the same +// AWS account and Region. However, you do not have permission to associate +// an alias with an AWS managed CMK (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#aws-managed-cmk) +// or an AWS owned CMK (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#aws-owned-cmk). +// +// * To change the CMK associated with an alias, use the UpdateAlias operation. +// The current CMK and the new CMK must be the same type (both symmetric +// or both asymmetric) and they must have the same key usage (ENCRYPT_DECRYPT +// or SIGN_VERIFY). This restriction prevents cryptographic errors in code +// that uses aliases. +// +// * The alias name must begin with alias/ followed by a name, such as alias/ExampleAlias. +// It can contain only alphanumeric characters, forward slashes (/), underscores +// (_), and dashes (-). The alias name cannot begin with alias/aws/. The +// alias/aws/ prefix is reserved for AWS managed CMKs (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#aws-managed-cmk). +// +// * The alias name must be unique within an AWS Region. However, you can +// use the same alias name in multiple Regions of the same AWS account. Each +// instance of the alias is associated with a CMK in its Region. +// +// * After you create an alias, you cannot change its alias name. However, +// you can use the DeleteAlias operation to delete the alias and then create +// a new alias with the desired name. +// +// * You can use an alias name or alias ARN to identify a CMK in AWS KMS +// cryptographic operations (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#cryptographic-operations) +// and in the DescribeKey operation. However, you cannot use alias names +// or alias ARNs in API operations that manage CMKs, such as DisableKey or +// GetKeyPolicy. For information about the valid CMK identifiers for each +// AWS KMS API operation, see the descriptions of the KeyId parameter in +// the API operation documentation. +// +// Because an alias is not a property of a CMK, you can delete and change the +// aliases of a CMK without affecting the CMK. Also, aliases do not appear in +// the response from the DescribeKey operation. To get the aliases and alias +// ARNs of CMKs in each AWS account and Region, use the ListAliases operation. +// +// The CMK that you use for this operation must be in a compatible key state. +// For details, see How Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation CreateAlias for usage and error information. +// +// Returned Error Types: +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * AlreadyExistsException +// The request was rejected because it attempted to create a resource that already +// exists. +// +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * InvalidAliasNameException +// The request was rejected because the specified alias name is not valid. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * LimitExceededException +// The request was rejected because a quota was exceeded. For more information, +// see Quotas (https://docs.aws.amazon.com/kms/latest/developerguide/limits.html) +// in the AWS Key Management Service Developer Guide. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/CreateAlias +func (c *KMS) CreateAlias(input *CreateAliasInput) (*CreateAliasOutput, error) { + req, out := c.CreateAliasRequest(input) + return out, req.Send() +} + +// CreateAliasWithContext is the same as CreateAlias with the addition of +// the ability to pass a context and additional request options. +// +// See CreateAlias for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) CreateAliasWithContext(ctx aws.Context, input *CreateAliasInput, opts ...request.Option) (*CreateAliasOutput, error) { + req, out := c.CreateAliasRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opCreateCustomKeyStore = "CreateCustomKeyStore" + +// CreateCustomKeyStoreRequest generates a "aws/request.Request" representing the +// client's request for the CreateCustomKeyStore operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See CreateCustomKeyStore for more information on using the CreateCustomKeyStore +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the CreateCustomKeyStoreRequest method. +// req, resp := client.CreateCustomKeyStoreRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/CreateCustomKeyStore +func (c *KMS) CreateCustomKeyStoreRequest(input *CreateCustomKeyStoreInput) (req *request.Request, output *CreateCustomKeyStoreOutput) { + op := &request.Operation{ + Name: opCreateCustomKeyStore, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &CreateCustomKeyStoreInput{} + } + + output = &CreateCustomKeyStoreOutput{} + req = c.newRequest(op, input, output) + return +} + +// CreateCustomKeyStore API operation for AWS Key Management Service. +// +// Creates a custom key store (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html) +// that is associated with an AWS CloudHSM cluster (https://docs.aws.amazon.com/cloudhsm/latest/userguide/clusters.html) +// that you own and manage. +// +// This operation is part of the Custom Key Store feature (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html) +// feature in AWS KMS, which combines the convenience and extensive integration +// of AWS KMS with the isolation and control of a single-tenant key store. +// +// Before you create the custom key store, you must assemble the required elements, +// including an AWS CloudHSM cluster that fulfills the requirements for a custom +// key store. For details about the required elements, see Assemble the Prerequisites +// (https://docs.aws.amazon.com/kms/latest/developerguide/create-keystore.html#before-keystore) +// in the AWS Key Management Service Developer Guide. +// +// When the operation completes successfully, it returns the ID of the new custom +// key store. Before you can use your new custom key store, you need to use +// the ConnectCustomKeyStore operation to connect the new key store to its AWS +// CloudHSM cluster. Even if you are not going to use your custom key store +// immediately, you might want to connect it to verify that all settings are +// correct and then disconnect it until you are ready to use it. +// +// For help with failures, see Troubleshooting a Custom Key Store (https://docs.aws.amazon.com/kms/latest/developerguide/fix-keystore.html) +// in the AWS Key Management Service Developer Guide. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation CreateCustomKeyStore for usage and error information. +// +// Returned Error Types: +// * CloudHsmClusterInUseException +// The request was rejected because the specified AWS CloudHSM cluster is already +// associated with a custom key store or it shares a backup history with a cluster +// that is associated with a custom key store. Each custom key store must be +// associated with a different AWS CloudHSM cluster. +// +// Clusters that share a backup history have the same cluster certificate. To +// view the cluster certificate of a cluster, use the DescribeClusters (https://docs.aws.amazon.com/cloudhsm/latest/APIReference/API_DescribeClusters.html) +// operation. +// +// * CustomKeyStoreNameInUseException +// The request was rejected because the specified custom key store name is already +// assigned to another custom key store in the account. Try again with a custom +// key store name that is unique in the account. +// +// * CloudHsmClusterNotFoundException +// The request was rejected because AWS KMS cannot find the AWS CloudHSM cluster +// with the specified cluster ID. Retry the request with a different cluster +// ID. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * CloudHsmClusterNotActiveException +// The request was rejected because the AWS CloudHSM cluster that is associated +// with the custom key store is not active. Initialize and activate the cluster +// and try the command again. For detailed instructions, see Getting Started +// (https://docs.aws.amazon.com/cloudhsm/latest/userguide/getting-started.html) +// in the AWS CloudHSM User Guide. +// +// * IncorrectTrustAnchorException +// The request was rejected because the trust anchor certificate in the request +// is not the trust anchor certificate for the specified AWS CloudHSM cluster. +// +// When you initialize the cluster (https://docs.aws.amazon.com/cloudhsm/latest/userguide/initialize-cluster.html#sign-csr), +// you create the trust anchor certificate and save it in the customerCA.crt +// file. +// +// * CloudHsmClusterInvalidConfigurationException +// The request was rejected because the associated AWS CloudHSM cluster did +// not meet the configuration requirements for a custom key store. +// +// * The cluster must be configured with private subnets in at least two +// different Availability Zones in the Region. +// +// * The security group for the cluster (https://docs.aws.amazon.com/cloudhsm/latest/userguide/configure-sg.html) +// (cloudhsm-cluster--sg) must include inbound rules and outbound +// rules that allow TCP traffic on ports 2223-2225. The Source in the inbound +// rules and the Destination in the outbound rules must match the security +// group ID. These rules are set by default when you create the cluster. +// Do not delete or change them. To get information about a particular security +// group, use the DescribeSecurityGroups (https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeSecurityGroups.html) +// operation. +// +// * The cluster must contain at least as many HSMs as the operation requires. +// To add HSMs, use the AWS CloudHSM CreateHsm (https://docs.aws.amazon.com/cloudhsm/latest/APIReference/API_CreateHsm.html) +// operation. For the CreateCustomKeyStore, UpdateCustomKeyStore, and CreateKey +// operations, the AWS CloudHSM cluster must have at least two active HSMs, +// each in a different Availability Zone. For the ConnectCustomKeyStore operation, +// the AWS CloudHSM must contain at least one active HSM. +// +// For information about the requirements for an AWS CloudHSM cluster that is +// associated with a custom key store, see Assemble the Prerequisites (https://docs.aws.amazon.com/kms/latest/developerguide/create-keystore.html#before-keystore) +// in the AWS Key Management Service Developer Guide. For information about +// creating a private subnet for an AWS CloudHSM cluster, see Create a Private +// Subnet (https://docs.aws.amazon.com/cloudhsm/latest/userguide/create-subnets.html) +// in the AWS CloudHSM User Guide. For information about cluster security groups, +// see Configure a Default Security Group (https://docs.aws.amazon.com/cloudhsm/latest/userguide/configure-sg.html) +// in the AWS CloudHSM User Guide . +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/CreateCustomKeyStore +func (c *KMS) CreateCustomKeyStore(input *CreateCustomKeyStoreInput) (*CreateCustomKeyStoreOutput, error) { + req, out := c.CreateCustomKeyStoreRequest(input) + return out, req.Send() +} + +// CreateCustomKeyStoreWithContext is the same as CreateCustomKeyStore with the addition of +// the ability to pass a context and additional request options. +// +// See CreateCustomKeyStore for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) CreateCustomKeyStoreWithContext(ctx aws.Context, input *CreateCustomKeyStoreInput, opts ...request.Option) (*CreateCustomKeyStoreOutput, error) { + req, out := c.CreateCustomKeyStoreRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opCreateGrant = "CreateGrant" + +// CreateGrantRequest generates a "aws/request.Request" representing the +// client's request for the CreateGrant operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See CreateGrant for more information on using the CreateGrant +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the CreateGrantRequest method. +// req, resp := client.CreateGrantRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/CreateGrant +func (c *KMS) CreateGrantRequest(input *CreateGrantInput) (req *request.Request, output *CreateGrantOutput) { + op := &request.Operation{ + Name: opCreateGrant, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &CreateGrantInput{} + } + + output = &CreateGrantOutput{} + req = c.newRequest(op, input, output) + return +} + +// CreateGrant API operation for AWS Key Management Service. +// +// Adds a grant to a customer master key (CMK). The grant allows the grantee +// principal to use the CMK when the conditions specified in the grant are met. +// When setting permissions, grants are an alternative to key policies. +// +// To create a grant that allows a cryptographic operation (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#cryptographic-operations) +// only when the request includes a particular encryption context (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#encrypt_context), +// use the Constraints parameter. For details, see GrantConstraints. +// +// You can create grants on symmetric and asymmetric CMKs. However, if the grant +// allows an operation that the CMK does not support, CreateGrant fails with +// a ValidationException. +// +// * Grants for symmetric CMKs cannot allow operations that are not supported +// for symmetric CMKs, including Sign, Verify, and GetPublicKey. (There are +// limited exceptions to this rule for legacy operations, but you should +// not create a grant for an operation that AWS KMS does not support.) +// +// * Grants for asymmetric CMKs cannot allow operations that are not supported +// for asymmetric CMKs, including operations that generate data keys (https://docs.aws.amazon.com/kms/latest/APIReference/API_GenerateDataKey) +// or data key pairs (https://docs.aws.amazon.com/kms/latest/APIReference/API_GenerateDataKeyPair), +// or operations related to automatic key rotation (https://docs.aws.amazon.com/kms/latest/developerguide/rotate-keys.html), +// imported key material (https://docs.aws.amazon.com/kms/latest/developerguide/importing-keys.html), +// or CMKs in custom key stores (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html). +// +// * Grants for asymmetric CMKs with a KeyUsage of ENCRYPT_DECRYPT cannot +// allow the Sign or Verify operations. Grants for asymmetric CMKs with a +// KeyUsage of SIGN_VERIFY cannot allow the Encrypt or Decrypt operations. +// +// * Grants for asymmetric CMKs cannot include an encryption context grant +// constraint. An encryption context is not supported on asymmetric CMKs. +// +// For information about symmetric and asymmetric CMKs, see Using Symmetric +// and Asymmetric CMKs (https://docs.aws.amazon.com/kms/latest/developerguide/symmetric-asymmetric.html) +// in the AWS Key Management Service Developer Guide. +// +// To perform this operation on a CMK in a different AWS account, specify the +// key ARN in the value of the KeyId parameter. For more information about grants, +// see Grants (https://docs.aws.amazon.com/kms/latest/developerguide/grants.html) +// in the AWS Key Management Service Developer Guide . +// +// The CMK that you use for this operation must be in a compatible key state. +// For details, see How Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation CreateGrant for usage and error information. +// +// Returned Error Types: +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * DisabledException +// The request was rejected because the specified CMK is not enabled. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InvalidArnException +// The request was rejected because a specified ARN, or an ARN in a key policy, +// is not valid. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidGrantTokenException +// The request was rejected because the specified grant token is not valid. +// +// * LimitExceededException +// The request was rejected because a quota was exceeded. For more information, +// see Quotas (https://docs.aws.amazon.com/kms/latest/developerguide/limits.html) +// in the AWS Key Management Service Developer Guide. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/CreateGrant +func (c *KMS) CreateGrant(input *CreateGrantInput) (*CreateGrantOutput, error) { + req, out := c.CreateGrantRequest(input) + return out, req.Send() +} + +// CreateGrantWithContext is the same as CreateGrant with the addition of +// the ability to pass a context and additional request options. +// +// See CreateGrant for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) CreateGrantWithContext(ctx aws.Context, input *CreateGrantInput, opts ...request.Option) (*CreateGrantOutput, error) { + req, out := c.CreateGrantRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opCreateKey = "CreateKey" + +// CreateKeyRequest generates a "aws/request.Request" representing the +// client's request for the CreateKey operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See CreateKey for more information on using the CreateKey +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the CreateKeyRequest method. +// req, resp := client.CreateKeyRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/CreateKey +func (c *KMS) CreateKeyRequest(input *CreateKeyInput) (req *request.Request, output *CreateKeyOutput) { + op := &request.Operation{ + Name: opCreateKey, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &CreateKeyInput{} + } + + output = &CreateKeyOutput{} + req = c.newRequest(op, input, output) + return +} + +// CreateKey API operation for AWS Key Management Service. +// +// Creates a unique customer managed customer master key (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#master-keys) +// (CMK) in your AWS account and Region. You cannot use this operation to create +// a CMK in a different AWS account. +// +// You can use the CreateKey operation to create symmetric or asymmetric CMKs. +// +// * Symmetric CMKs contain a 256-bit symmetric key that never leaves AWS +// KMS unencrypted. To use the CMK, you must call AWS KMS. You can use a +// symmetric CMK to encrypt and decrypt small amounts of data, but they are +// typically used to generate data keys (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#data-keys) +// and data keys pairs (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#data-key-pairs). +// For details, see GenerateDataKey and GenerateDataKeyPair. +// +// * Asymmetric CMKs can contain an RSA key pair or an Elliptic Curve (ECC) +// key pair. The private key in an asymmetric CMK never leaves AWS KMS unencrypted. +// However, you can use the GetPublicKey operation to download the public +// key so it can be used outside of AWS KMS. CMKs with RSA key pairs can +// be used to encrypt or decrypt data or sign and verify messages (but not +// both). CMKs with ECC key pairs can be used only to sign and verify messages. +// +// For information about symmetric and asymmetric CMKs, see Using Symmetric +// and Asymmetric CMKs (https://docs.aws.amazon.com/kms/latest/developerguide/symmetric-asymmetric.html) +// in the AWS Key Management Service Developer Guide. +// +// To create different types of CMKs, use the following guidance: +// +// Asymmetric CMKs +// +// To create an asymmetric CMK, use the CustomerMasterKeySpec parameter to specify +// the type of key material in the CMK. Then, use the KeyUsage parameter to +// determine whether the CMK will be used to encrypt and decrypt or sign and +// verify. You can't change these properties after the CMK is created. +// +// Symmetric CMKs +// +// When creating a symmetric CMK, you don't need to specify the CustomerMasterKeySpec +// or KeyUsage parameters. The default value for CustomerMasterKeySpec, SYMMETRIC_DEFAULT, +// and the default value for KeyUsage, ENCRYPT_DECRYPT, are the only valid values +// for symmetric CMKs. +// +// Imported Key Material +// +// To import your own key material, begin by creating a symmetric CMK with no +// key material. To do this, use the Origin parameter of CreateKey with a value +// of EXTERNAL. Next, use GetParametersForImport operation to get a public key +// and import token, and use the public key to encrypt your key material. Then, +// use ImportKeyMaterial with your import token to import the key material. +// For step-by-step instructions, see Importing Key Material (https://docs.aws.amazon.com/kms/latest/developerguide/importing-keys.html) +// in the AWS Key Management Service Developer Guide . You cannot import the +// key material into an asymmetric CMK. +// +// Custom Key Stores +// +// To create a symmetric CMK in a custom key store (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html), +// use the CustomKeyStoreId parameter to specify the custom key store. You must +// also use the Origin parameter with a value of AWS_CLOUDHSM. The AWS CloudHSM +// cluster that is associated with the custom key store must have at least two +// active HSMs in different Availability Zones in the AWS Region. +// +// You cannot create an asymmetric CMK in a custom key store. For information +// about custom key stores in AWS KMS see Using Custom Key Stores (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html) +// in the AWS Key Management Service Developer Guide . +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation CreateKey for usage and error information. +// +// Returned Error Types: +// * MalformedPolicyDocumentException +// The request was rejected because the specified policy is not syntactically +// or semantically correct. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InvalidArnException +// The request was rejected because a specified ARN, or an ARN in a key policy, +// is not valid. +// +// * UnsupportedOperationException +// The request was rejected because a specified parameter is not supported or +// a specified resource is not valid for this operation. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * LimitExceededException +// The request was rejected because a quota was exceeded. For more information, +// see Quotas (https://docs.aws.amazon.com/kms/latest/developerguide/limits.html) +// in the AWS Key Management Service Developer Guide. +// +// * TagException +// The request was rejected because one or more tags are not valid. +// +// * CustomKeyStoreNotFoundException +// The request was rejected because AWS KMS cannot find a custom key store with +// the specified key store name or ID. +// +// * CustomKeyStoreInvalidStateException +// The request was rejected because of the ConnectionState of the custom key +// store. To get the ConnectionState of a custom key store, use the DescribeCustomKeyStores +// operation. +// +// This exception is thrown under the following conditions: +// +// * You requested the CreateKey or GenerateRandom operation in a custom +// key store that is not connected. These operations are valid only when +// the custom key store ConnectionState is CONNECTED. +// +// * You requested the UpdateCustomKeyStore or DeleteCustomKeyStore operation +// on a custom key store that is not disconnected. This operation is valid +// only when the custom key store ConnectionState is DISCONNECTED. +// +// * You requested the ConnectCustomKeyStore operation on a custom key store +// with a ConnectionState of DISCONNECTING or FAILED. This operation is valid +// for all other ConnectionState values. +// +// * CloudHsmClusterInvalidConfigurationException +// The request was rejected because the associated AWS CloudHSM cluster did +// not meet the configuration requirements for a custom key store. +// +// * The cluster must be configured with private subnets in at least two +// different Availability Zones in the Region. +// +// * The security group for the cluster (https://docs.aws.amazon.com/cloudhsm/latest/userguide/configure-sg.html) +// (cloudhsm-cluster--sg) must include inbound rules and outbound +// rules that allow TCP traffic on ports 2223-2225. The Source in the inbound +// rules and the Destination in the outbound rules must match the security +// group ID. These rules are set by default when you create the cluster. +// Do not delete or change them. To get information about a particular security +// group, use the DescribeSecurityGroups (https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeSecurityGroups.html) +// operation. +// +// * The cluster must contain at least as many HSMs as the operation requires. +// To add HSMs, use the AWS CloudHSM CreateHsm (https://docs.aws.amazon.com/cloudhsm/latest/APIReference/API_CreateHsm.html) +// operation. For the CreateCustomKeyStore, UpdateCustomKeyStore, and CreateKey +// operations, the AWS CloudHSM cluster must have at least two active HSMs, +// each in a different Availability Zone. For the ConnectCustomKeyStore operation, +// the AWS CloudHSM must contain at least one active HSM. +// +// For information about the requirements for an AWS CloudHSM cluster that is +// associated with a custom key store, see Assemble the Prerequisites (https://docs.aws.amazon.com/kms/latest/developerguide/create-keystore.html#before-keystore) +// in the AWS Key Management Service Developer Guide. For information about +// creating a private subnet for an AWS CloudHSM cluster, see Create a Private +// Subnet (https://docs.aws.amazon.com/cloudhsm/latest/userguide/create-subnets.html) +// in the AWS CloudHSM User Guide. For information about cluster security groups, +// see Configure a Default Security Group (https://docs.aws.amazon.com/cloudhsm/latest/userguide/configure-sg.html) +// in the AWS CloudHSM User Guide . +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/CreateKey +func (c *KMS) CreateKey(input *CreateKeyInput) (*CreateKeyOutput, error) { + req, out := c.CreateKeyRequest(input) + return out, req.Send() +} + +// CreateKeyWithContext is the same as CreateKey with the addition of +// the ability to pass a context and additional request options. +// +// See CreateKey for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) CreateKeyWithContext(ctx aws.Context, input *CreateKeyInput, opts ...request.Option) (*CreateKeyOutput, error) { + req, out := c.CreateKeyRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opDecrypt = "Decrypt" + +// DecryptRequest generates a "aws/request.Request" representing the +// client's request for the Decrypt operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See Decrypt for more information on using the Decrypt +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the DecryptRequest method. +// req, resp := client.DecryptRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/Decrypt +func (c *KMS) DecryptRequest(input *DecryptInput) (req *request.Request, output *DecryptOutput) { + op := &request.Operation{ + Name: opDecrypt, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &DecryptInput{} + } + + output = &DecryptOutput{} + req = c.newRequest(op, input, output) + return +} + +// Decrypt API operation for AWS Key Management Service. +// +// Decrypts ciphertext that was encrypted by a AWS KMS customer master key (CMK) +// using any of the following operations: +// +// * Encrypt +// +// * GenerateDataKey +// +// * GenerateDataKeyPair +// +// * GenerateDataKeyWithoutPlaintext +// +// * GenerateDataKeyPairWithoutPlaintext +// +// You can use this operation to decrypt ciphertext that was encrypted under +// a symmetric or asymmetric CMK. When the CMK is asymmetric, you must specify +// the CMK and the encryption algorithm that was used to encrypt the ciphertext. +// For information about symmetric and asymmetric CMKs, see Using Symmetric +// and Asymmetric CMKs (https://docs.aws.amazon.com/kms/latest/developerguide/symmetric-asymmetric.html) +// in the AWS Key Management Service Developer Guide. +// +// The Decrypt operation also decrypts ciphertext that was encrypted outside +// of AWS KMS by the public key in an AWS KMS asymmetric CMK. However, it cannot +// decrypt ciphertext produced by other libraries, such as the AWS Encryption +// SDK (https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/) +// or Amazon S3 client-side encryption (https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingClientSideEncryption.html). +// These libraries return a ciphertext format that is incompatible with AWS +// KMS. +// +// If the ciphertext was encrypted under a symmetric CMK, you do not need to +// specify the CMK or the encryption algorithm. AWS KMS can get this information +// from metadata that it adds to the symmetric ciphertext blob. However, if +// you prefer, you can specify the KeyId to ensure that a particular CMK is +// used to decrypt the ciphertext. If you specify a different CMK than the one +// used to encrypt the ciphertext, the Decrypt operation fails. +// +// Whenever possible, use key policies to give users permission to call the +// Decrypt operation on a particular CMK, instead of using IAM policies. Otherwise, +// you might create an IAM user policy that gives the user Decrypt permission +// on all CMKs. This user could decrypt ciphertext that was encrypted by CMKs +// in other accounts if the key policy for the cross-account CMK permits it. +// If you must use an IAM policy for Decrypt permissions, limit the user to +// particular CMKs or particular trusted accounts. +// +// The CMK that you use for this operation must be in a compatible key state. +// For details, see How Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation Decrypt for usage and error information. +// +// Returned Error Types: +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * DisabledException +// The request was rejected because the specified CMK is not enabled. +// +// * InvalidCiphertextException +// From the Decrypt or ReEncrypt operation, the request was rejected because +// the specified ciphertext, or additional authenticated data incorporated into +// the ciphertext, such as the encryption context, is corrupted, missing, or +// otherwise invalid. +// +// From the ImportKeyMaterial operation, the request was rejected because AWS +// KMS could not decrypt the encrypted (wrapped) key material. +// +// * KeyUnavailableException +// The request was rejected because the specified CMK was not available. You +// can retry the request. +// +// * IncorrectKeyException +// The request was rejected because the specified CMK cannot decrypt the data. +// The KeyId in a Decrypt request and the SourceKeyId in a ReEncrypt request +// must identify the same CMK that was used to encrypt the ciphertext. +// +// * InvalidKeyUsageException +// The request was rejected for one of the following reasons: +// +// * The KeyUsage value of the CMK is incompatible with the API operation. +// +// * The encryption algorithm or signing algorithm specified for the operation +// is incompatible with the type of key material in the CMK (CustomerMasterKeySpec). +// +// For encrypting, decrypting, re-encrypting, and generating data keys, the +// KeyUsage must be ENCRYPT_DECRYPT. For signing and verifying, the KeyUsage +// must be SIGN_VERIFY. To find the KeyUsage of a CMK, use the DescribeKey operation. +// +// To find the encryption or signing algorithms supported for a particular CMK, +// use the DescribeKey operation. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InvalidGrantTokenException +// The request was rejected because the specified grant token is not valid. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/Decrypt +func (c *KMS) Decrypt(input *DecryptInput) (*DecryptOutput, error) { + req, out := c.DecryptRequest(input) + return out, req.Send() +} + +// DecryptWithContext is the same as Decrypt with the addition of +// the ability to pass a context and additional request options. +// +// See Decrypt for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) DecryptWithContext(ctx aws.Context, input *DecryptInput, opts ...request.Option) (*DecryptOutput, error) { + req, out := c.DecryptRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opDeleteAlias = "DeleteAlias" + +// DeleteAliasRequest generates a "aws/request.Request" representing the +// client's request for the DeleteAlias operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See DeleteAlias for more information on using the DeleteAlias +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the DeleteAliasRequest method. +// req, resp := client.DeleteAliasRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/DeleteAlias +func (c *KMS) DeleteAliasRequest(input *DeleteAliasInput) (req *request.Request, output *DeleteAliasOutput) { + op := &request.Operation{ + Name: opDeleteAlias, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &DeleteAliasInput{} + } + + output = &DeleteAliasOutput{} + req = c.newRequest(op, input, output) + req.Handlers.Unmarshal.Swap(jsonrpc.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + return +} + +// DeleteAlias API operation for AWS Key Management Service. +// +// Deletes the specified alias. You cannot perform this operation on an alias +// in a different AWS account. +// +// Because an alias is not a property of a CMK, you can delete and change the +// aliases of a CMK without affecting the CMK. Also, aliases do not appear in +// the response from the DescribeKey operation. To get the aliases of all CMKs, +// use the ListAliases operation. +// +// Each CMK can have multiple aliases. To change the alias of a CMK, use DeleteAlias +// to delete the current alias and CreateAlias to create a new alias. To associate +// an existing alias with a different customer master key (CMK), call UpdateAlias. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation DeleteAlias for usage and error information. +// +// Returned Error Types: +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/DeleteAlias +func (c *KMS) DeleteAlias(input *DeleteAliasInput) (*DeleteAliasOutput, error) { + req, out := c.DeleteAliasRequest(input) + return out, req.Send() +} + +// DeleteAliasWithContext is the same as DeleteAlias with the addition of +// the ability to pass a context and additional request options. +// +// See DeleteAlias for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) DeleteAliasWithContext(ctx aws.Context, input *DeleteAliasInput, opts ...request.Option) (*DeleteAliasOutput, error) { + req, out := c.DeleteAliasRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opDeleteCustomKeyStore = "DeleteCustomKeyStore" + +// DeleteCustomKeyStoreRequest generates a "aws/request.Request" representing the +// client's request for the DeleteCustomKeyStore operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See DeleteCustomKeyStore for more information on using the DeleteCustomKeyStore +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the DeleteCustomKeyStoreRequest method. +// req, resp := client.DeleteCustomKeyStoreRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/DeleteCustomKeyStore +func (c *KMS) DeleteCustomKeyStoreRequest(input *DeleteCustomKeyStoreInput) (req *request.Request, output *DeleteCustomKeyStoreOutput) { + op := &request.Operation{ + Name: opDeleteCustomKeyStore, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &DeleteCustomKeyStoreInput{} + } + + output = &DeleteCustomKeyStoreOutput{} + req = c.newRequest(op, input, output) + req.Handlers.Unmarshal.Swap(jsonrpc.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + return +} + +// DeleteCustomKeyStore API operation for AWS Key Management Service. +// +// Deletes a custom key store (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html). +// This operation does not delete the AWS CloudHSM cluster that is associated +// with the custom key store, or affect any users or keys in the cluster. +// +// The custom key store that you delete cannot contain any AWS KMS customer +// master keys (CMKs) (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#master_keys). +// Before deleting the key store, verify that you will never need to use any +// of the CMKs in the key store for any cryptographic operations (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#cryptographic-operations). +// Then, use ScheduleKeyDeletion to delete the AWS KMS customer master keys +// (CMKs) from the key store. When the scheduled waiting period expires, the +// ScheduleKeyDeletion operation deletes the CMKs. Then it makes a best effort +// to delete the key material from the associated cluster. However, you might +// need to manually delete the orphaned key material (https://docs.aws.amazon.com/kms/latest/developerguide/fix-keystore.html#fix-keystore-orphaned-key) +// from the cluster and its backups. +// +// After all CMKs are deleted from AWS KMS, use DisconnectCustomKeyStore to +// disconnect the key store from AWS KMS. Then, you can delete the custom key +// store. +// +// Instead of deleting the custom key store, consider using DisconnectCustomKeyStore +// to disconnect it from AWS KMS. While the key store is disconnected, you cannot +// create or use the CMKs in the key store. But, you do not need to delete CMKs +// and you can reconnect a disconnected custom key store at any time. +// +// If the operation succeeds, it returns a JSON object with no properties. +// +// This operation is part of the Custom Key Store feature (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html) +// feature in AWS KMS, which combines the convenience and extensive integration +// of AWS KMS with the isolation and control of a single-tenant key store. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation DeleteCustomKeyStore for usage and error information. +// +// Returned Error Types: +// * CustomKeyStoreHasCMKsException +// The request was rejected because the custom key store contains AWS KMS customer +// master keys (CMKs). After verifying that you do not need to use the CMKs, +// use the ScheduleKeyDeletion operation to delete the CMKs. After they are +// deleted, you can delete the custom key store. +// +// * CustomKeyStoreInvalidStateException +// The request was rejected because of the ConnectionState of the custom key +// store. To get the ConnectionState of a custom key store, use the DescribeCustomKeyStores +// operation. +// +// This exception is thrown under the following conditions: +// +// * You requested the CreateKey or GenerateRandom operation in a custom +// key store that is not connected. These operations are valid only when +// the custom key store ConnectionState is CONNECTED. +// +// * You requested the UpdateCustomKeyStore or DeleteCustomKeyStore operation +// on a custom key store that is not disconnected. This operation is valid +// only when the custom key store ConnectionState is DISCONNECTED. +// +// * You requested the ConnectCustomKeyStore operation on a custom key store +// with a ConnectionState of DISCONNECTING or FAILED. This operation is valid +// for all other ConnectionState values. +// +// * CustomKeyStoreNotFoundException +// The request was rejected because AWS KMS cannot find a custom key store with +// the specified key store name or ID. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/DeleteCustomKeyStore +func (c *KMS) DeleteCustomKeyStore(input *DeleteCustomKeyStoreInput) (*DeleteCustomKeyStoreOutput, error) { + req, out := c.DeleteCustomKeyStoreRequest(input) + return out, req.Send() +} + +// DeleteCustomKeyStoreWithContext is the same as DeleteCustomKeyStore with the addition of +// the ability to pass a context and additional request options. +// +// See DeleteCustomKeyStore for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) DeleteCustomKeyStoreWithContext(ctx aws.Context, input *DeleteCustomKeyStoreInput, opts ...request.Option) (*DeleteCustomKeyStoreOutput, error) { + req, out := c.DeleteCustomKeyStoreRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opDeleteImportedKeyMaterial = "DeleteImportedKeyMaterial" + +// DeleteImportedKeyMaterialRequest generates a "aws/request.Request" representing the +// client's request for the DeleteImportedKeyMaterial operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See DeleteImportedKeyMaterial for more information on using the DeleteImportedKeyMaterial +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the DeleteImportedKeyMaterialRequest method. +// req, resp := client.DeleteImportedKeyMaterialRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/DeleteImportedKeyMaterial +func (c *KMS) DeleteImportedKeyMaterialRequest(input *DeleteImportedKeyMaterialInput) (req *request.Request, output *DeleteImportedKeyMaterialOutput) { + op := &request.Operation{ + Name: opDeleteImportedKeyMaterial, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &DeleteImportedKeyMaterialInput{} + } + + output = &DeleteImportedKeyMaterialOutput{} + req = c.newRequest(op, input, output) + req.Handlers.Unmarshal.Swap(jsonrpc.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + return +} + +// DeleteImportedKeyMaterial API operation for AWS Key Management Service. +// +// Deletes key material that you previously imported. This operation makes the +// specified customer master key (CMK) unusable. For more information about +// importing key material into AWS KMS, see Importing Key Material (https://docs.aws.amazon.com/kms/latest/developerguide/importing-keys.html) +// in the AWS Key Management Service Developer Guide. You cannot perform this +// operation on a CMK in a different AWS account. +// +// When the specified CMK is in the PendingDeletion state, this operation does +// not change the CMK's state. Otherwise, it changes the CMK's state to PendingImport. +// +// After you delete key material, you can use ImportKeyMaterial to reimport +// the same key material into the CMK. +// +// The CMK that you use for this operation must be in a compatible key state. +// For details, see How Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation DeleteImportedKeyMaterial for usage and error information. +// +// Returned Error Types: +// * InvalidArnException +// The request was rejected because a specified ARN, or an ARN in a key policy, +// is not valid. +// +// * UnsupportedOperationException +// The request was rejected because a specified parameter is not supported or +// a specified resource is not valid for this operation. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/DeleteImportedKeyMaterial +func (c *KMS) DeleteImportedKeyMaterial(input *DeleteImportedKeyMaterialInput) (*DeleteImportedKeyMaterialOutput, error) { + req, out := c.DeleteImportedKeyMaterialRequest(input) + return out, req.Send() +} + +// DeleteImportedKeyMaterialWithContext is the same as DeleteImportedKeyMaterial with the addition of +// the ability to pass a context and additional request options. +// +// See DeleteImportedKeyMaterial for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) DeleteImportedKeyMaterialWithContext(ctx aws.Context, input *DeleteImportedKeyMaterialInput, opts ...request.Option) (*DeleteImportedKeyMaterialOutput, error) { + req, out := c.DeleteImportedKeyMaterialRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opDescribeCustomKeyStores = "DescribeCustomKeyStores" + +// DescribeCustomKeyStoresRequest generates a "aws/request.Request" representing the +// client's request for the DescribeCustomKeyStores operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See DescribeCustomKeyStores for more information on using the DescribeCustomKeyStores +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the DescribeCustomKeyStoresRequest method. +// req, resp := client.DescribeCustomKeyStoresRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/DescribeCustomKeyStores +func (c *KMS) DescribeCustomKeyStoresRequest(input *DescribeCustomKeyStoresInput) (req *request.Request, output *DescribeCustomKeyStoresOutput) { + op := &request.Operation{ + Name: opDescribeCustomKeyStores, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &DescribeCustomKeyStoresInput{} + } + + output = &DescribeCustomKeyStoresOutput{} + req = c.newRequest(op, input, output) + return +} + +// DescribeCustomKeyStores API operation for AWS Key Management Service. +// +// Gets information about custom key stores (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html) +// in the account and region. +// +// This operation is part of the Custom Key Store feature (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html) +// feature in AWS KMS, which combines the convenience and extensive integration +// of AWS KMS with the isolation and control of a single-tenant key store. +// +// By default, this operation returns information about all custom key stores +// in the account and region. To get only information about a particular custom +// key store, use either the CustomKeyStoreName or CustomKeyStoreId parameter +// (but not both). +// +// To determine whether the custom key store is connected to its AWS CloudHSM +// cluster, use the ConnectionState element in the response. If an attempt to +// connect the custom key store failed, the ConnectionState value is FAILED +// and the ConnectionErrorCode element in the response indicates the cause of +// the failure. For help interpreting the ConnectionErrorCode, see CustomKeyStoresListEntry. +// +// Custom key stores have a DISCONNECTED connection state if the key store has +// never been connected or you use the DisconnectCustomKeyStore operation to +// disconnect it. If your custom key store state is CONNECTED but you are having +// trouble using it, make sure that its associated AWS CloudHSM cluster is active +// and contains the minimum number of HSMs required for the operation, if any. +// +// For help repairing your custom key store, see the Troubleshooting Custom +// Key Stores (https://docs.aws.amazon.com/kms/latest/developerguide/fix-keystore.html) +// topic in the AWS Key Management Service Developer Guide. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation DescribeCustomKeyStores for usage and error information. +// +// Returned Error Types: +// * CustomKeyStoreNotFoundException +// The request was rejected because AWS KMS cannot find a custom key store with +// the specified key store name or ID. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/DescribeCustomKeyStores +func (c *KMS) DescribeCustomKeyStores(input *DescribeCustomKeyStoresInput) (*DescribeCustomKeyStoresOutput, error) { + req, out := c.DescribeCustomKeyStoresRequest(input) + return out, req.Send() +} + +// DescribeCustomKeyStoresWithContext is the same as DescribeCustomKeyStores with the addition of +// the ability to pass a context and additional request options. +// +// See DescribeCustomKeyStores for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) DescribeCustomKeyStoresWithContext(ctx aws.Context, input *DescribeCustomKeyStoresInput, opts ...request.Option) (*DescribeCustomKeyStoresOutput, error) { + req, out := c.DescribeCustomKeyStoresRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opDescribeKey = "DescribeKey" + +// DescribeKeyRequest generates a "aws/request.Request" representing the +// client's request for the DescribeKey operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See DescribeKey for more information on using the DescribeKey +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the DescribeKeyRequest method. +// req, resp := client.DescribeKeyRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/DescribeKey +func (c *KMS) DescribeKeyRequest(input *DescribeKeyInput) (req *request.Request, output *DescribeKeyOutput) { + op := &request.Operation{ + Name: opDescribeKey, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &DescribeKeyInput{} + } + + output = &DescribeKeyOutput{} + req = c.newRequest(op, input, output) + return +} + +// DescribeKey API operation for AWS Key Management Service. +// +// Provides detailed information about a customer master key (CMK). You can +// run DescribeKey on a customer managed CMK (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#customer-cmk) +// or an AWS managed CMK (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#aws-managed-cmk). +// +// This detailed information includes the key ARN, creation date (and deletion +// date, if applicable), the key state, and the origin and expiration date (if +// any) of the key material. For CMKs in custom key stores, it includes information +// about the custom key store, such as the key store ID and the AWS CloudHSM +// cluster ID. It includes fields, like KeySpec, that help you distinguish symmetric +// from asymmetric CMKs. It also provides information that is particularly important +// to asymmetric CMKs, such as the key usage (encryption or signing) and the +// encryption algorithms or signing algorithms that the CMK supports. +// +// DescribeKey does not return the following information: +// +// * Aliases associated with the CMK. To get this information, use ListAliases. +// +// * Whether automatic key rotation is enabled on the CMK. To get this information, +// use GetKeyRotationStatus. Also, some key states prevent a CMK from being +// automatically rotated. For details, see How Automatic Key Rotation Works +// (https://docs.aws.amazon.com/kms/latest/developerguide/rotate-keys.html#rotate-keys-how-it-works) +// in AWS Key Management Service Developer Guide. +// +// * Tags on the CMK. To get this information, use ListResourceTags. +// +// * Key policies and grants on the CMK. To get this information, use GetKeyPolicy +// and ListGrants. +// +// If you call the DescribeKey operation on a predefined AWS alias, that is, +// an AWS alias with no key ID, AWS KMS creates an AWS managed CMK (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#master_keys). +// Then, it associates the alias with the new CMK, and returns the KeyId and +// Arn of the new CMK in the response. +// +// To perform this operation on a CMK in a different AWS account, specify the +// key ARN or alias ARN in the value of the KeyId parameter. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation DescribeKey for usage and error information. +// +// Returned Error Types: +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * InvalidArnException +// The request was rejected because a specified ARN, or an ARN in a key policy, +// is not valid. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/DescribeKey +func (c *KMS) DescribeKey(input *DescribeKeyInput) (*DescribeKeyOutput, error) { + req, out := c.DescribeKeyRequest(input) + return out, req.Send() +} + +// DescribeKeyWithContext is the same as DescribeKey with the addition of +// the ability to pass a context and additional request options. +// +// See DescribeKey for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) DescribeKeyWithContext(ctx aws.Context, input *DescribeKeyInput, opts ...request.Option) (*DescribeKeyOutput, error) { + req, out := c.DescribeKeyRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opDisableKey = "DisableKey" + +// DisableKeyRequest generates a "aws/request.Request" representing the +// client's request for the DisableKey operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See DisableKey for more information on using the DisableKey +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the DisableKeyRequest method. +// req, resp := client.DisableKeyRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/DisableKey +func (c *KMS) DisableKeyRequest(input *DisableKeyInput) (req *request.Request, output *DisableKeyOutput) { + op := &request.Operation{ + Name: opDisableKey, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &DisableKeyInput{} + } + + output = &DisableKeyOutput{} + req = c.newRequest(op, input, output) + req.Handlers.Unmarshal.Swap(jsonrpc.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + return +} + +// DisableKey API operation for AWS Key Management Service. +// +// Sets the state of a customer master key (CMK) to disabled, thereby preventing +// its use for cryptographic operations (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#cryptographic-operations). +// You cannot perform this operation on a CMK in a different AWS account. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects the Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// The CMK that you use for this operation must be in a compatible key state. +// For details, see How Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation DisableKey for usage and error information. +// +// Returned Error Types: +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * InvalidArnException +// The request was rejected because a specified ARN, or an ARN in a key policy, +// is not valid. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/DisableKey +func (c *KMS) DisableKey(input *DisableKeyInput) (*DisableKeyOutput, error) { + req, out := c.DisableKeyRequest(input) + return out, req.Send() +} + +// DisableKeyWithContext is the same as DisableKey with the addition of +// the ability to pass a context and additional request options. +// +// See DisableKey for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) DisableKeyWithContext(ctx aws.Context, input *DisableKeyInput, opts ...request.Option) (*DisableKeyOutput, error) { + req, out := c.DisableKeyRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opDisableKeyRotation = "DisableKeyRotation" + +// DisableKeyRotationRequest generates a "aws/request.Request" representing the +// client's request for the DisableKeyRotation operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See DisableKeyRotation for more information on using the DisableKeyRotation +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the DisableKeyRotationRequest method. +// req, resp := client.DisableKeyRotationRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/DisableKeyRotation +func (c *KMS) DisableKeyRotationRequest(input *DisableKeyRotationInput) (req *request.Request, output *DisableKeyRotationOutput) { + op := &request.Operation{ + Name: opDisableKeyRotation, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &DisableKeyRotationInput{} + } + + output = &DisableKeyRotationOutput{} + req = c.newRequest(op, input, output) + req.Handlers.Unmarshal.Swap(jsonrpc.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + return +} + +// DisableKeyRotation API operation for AWS Key Management Service. +// +// Disables automatic rotation of the key material (https://docs.aws.amazon.com/kms/latest/developerguide/rotate-keys.html) +// for the specified symmetric customer master key (CMK). +// +// You cannot enable automatic rotation of asymmetric CMKs, CMKs with imported +// key material, or CMKs in a custom key store (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html). +// You cannot perform this operation on a CMK in a different AWS account. +// +// The CMK that you use for this operation must be in a compatible key state. +// For details, see How Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation DisableKeyRotation for usage and error information. +// +// Returned Error Types: +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * DisabledException +// The request was rejected because the specified CMK is not enabled. +// +// * InvalidArnException +// The request was rejected because a specified ARN, or an ARN in a key policy, +// is not valid. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// * UnsupportedOperationException +// The request was rejected because a specified parameter is not supported or +// a specified resource is not valid for this operation. +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/DisableKeyRotation +func (c *KMS) DisableKeyRotation(input *DisableKeyRotationInput) (*DisableKeyRotationOutput, error) { + req, out := c.DisableKeyRotationRequest(input) + return out, req.Send() +} + +// DisableKeyRotationWithContext is the same as DisableKeyRotation with the addition of +// the ability to pass a context and additional request options. +// +// See DisableKeyRotation for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) DisableKeyRotationWithContext(ctx aws.Context, input *DisableKeyRotationInput, opts ...request.Option) (*DisableKeyRotationOutput, error) { + req, out := c.DisableKeyRotationRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opDisconnectCustomKeyStore = "DisconnectCustomKeyStore" + +// DisconnectCustomKeyStoreRequest generates a "aws/request.Request" representing the +// client's request for the DisconnectCustomKeyStore operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See DisconnectCustomKeyStore for more information on using the DisconnectCustomKeyStore +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the DisconnectCustomKeyStoreRequest method. +// req, resp := client.DisconnectCustomKeyStoreRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/DisconnectCustomKeyStore +func (c *KMS) DisconnectCustomKeyStoreRequest(input *DisconnectCustomKeyStoreInput) (req *request.Request, output *DisconnectCustomKeyStoreOutput) { + op := &request.Operation{ + Name: opDisconnectCustomKeyStore, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &DisconnectCustomKeyStoreInput{} + } + + output = &DisconnectCustomKeyStoreOutput{} + req = c.newRequest(op, input, output) + req.Handlers.Unmarshal.Swap(jsonrpc.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + return +} + +// DisconnectCustomKeyStore API operation for AWS Key Management Service. +// +// Disconnects the custom key store (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html) +// from its associated AWS CloudHSM cluster. While a custom key store is disconnected, +// you can manage the custom key store and its customer master keys (CMKs), +// but you cannot create or use CMKs in the custom key store. You can reconnect +// the custom key store at any time. +// +// While a custom key store is disconnected, all attempts to create customer +// master keys (CMKs) in the custom key store or to use existing CMKs in cryptographic +// operations (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#cryptographic-operations) +// will fail. This action can prevent users from storing and accessing sensitive +// data. +// +// To find the connection state of a custom key store, use the DescribeCustomKeyStores +// operation. To reconnect a custom key store, use the ConnectCustomKeyStore +// operation. +// +// If the operation succeeds, it returns a JSON object with no properties. +// +// This operation is part of the Custom Key Store feature (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html) +// feature in AWS KMS, which combines the convenience and extensive integration +// of AWS KMS with the isolation and control of a single-tenant key store. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation DisconnectCustomKeyStore for usage and error information. +// +// Returned Error Types: +// * CustomKeyStoreInvalidStateException +// The request was rejected because of the ConnectionState of the custom key +// store. To get the ConnectionState of a custom key store, use the DescribeCustomKeyStores +// operation. +// +// This exception is thrown under the following conditions: +// +// * You requested the CreateKey or GenerateRandom operation in a custom +// key store that is not connected. These operations are valid only when +// the custom key store ConnectionState is CONNECTED. +// +// * You requested the UpdateCustomKeyStore or DeleteCustomKeyStore operation +// on a custom key store that is not disconnected. This operation is valid +// only when the custom key store ConnectionState is DISCONNECTED. +// +// * You requested the ConnectCustomKeyStore operation on a custom key store +// with a ConnectionState of DISCONNECTING or FAILED. This operation is valid +// for all other ConnectionState values. +// +// * CustomKeyStoreNotFoundException +// The request was rejected because AWS KMS cannot find a custom key store with +// the specified key store name or ID. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/DisconnectCustomKeyStore +func (c *KMS) DisconnectCustomKeyStore(input *DisconnectCustomKeyStoreInput) (*DisconnectCustomKeyStoreOutput, error) { + req, out := c.DisconnectCustomKeyStoreRequest(input) + return out, req.Send() +} + +// DisconnectCustomKeyStoreWithContext is the same as DisconnectCustomKeyStore with the addition of +// the ability to pass a context and additional request options. +// +// See DisconnectCustomKeyStore for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) DisconnectCustomKeyStoreWithContext(ctx aws.Context, input *DisconnectCustomKeyStoreInput, opts ...request.Option) (*DisconnectCustomKeyStoreOutput, error) { + req, out := c.DisconnectCustomKeyStoreRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opEnableKey = "EnableKey" + +// EnableKeyRequest generates a "aws/request.Request" representing the +// client's request for the EnableKey operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See EnableKey for more information on using the EnableKey +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the EnableKeyRequest method. +// req, resp := client.EnableKeyRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/EnableKey +func (c *KMS) EnableKeyRequest(input *EnableKeyInput) (req *request.Request, output *EnableKeyOutput) { + op := &request.Operation{ + Name: opEnableKey, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &EnableKeyInput{} + } + + output = &EnableKeyOutput{} + req = c.newRequest(op, input, output) + req.Handlers.Unmarshal.Swap(jsonrpc.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + return +} + +// EnableKey API operation for AWS Key Management Service. +// +// Sets the key state of a customer master key (CMK) to enabled. This allows +// you to use the CMK for cryptographic operations (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#cryptographic-operations). +// You cannot perform this operation on a CMK in a different AWS account. +// +// The CMK that you use for this operation must be in a compatible key state. +// For details, see How Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation EnableKey for usage and error information. +// +// Returned Error Types: +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * InvalidArnException +// The request was rejected because a specified ARN, or an ARN in a key policy, +// is not valid. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * LimitExceededException +// The request was rejected because a quota was exceeded. For more information, +// see Quotas (https://docs.aws.amazon.com/kms/latest/developerguide/limits.html) +// in the AWS Key Management Service Developer Guide. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/EnableKey +func (c *KMS) EnableKey(input *EnableKeyInput) (*EnableKeyOutput, error) { + req, out := c.EnableKeyRequest(input) + return out, req.Send() +} + +// EnableKeyWithContext is the same as EnableKey with the addition of +// the ability to pass a context and additional request options. +// +// See EnableKey for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) EnableKeyWithContext(ctx aws.Context, input *EnableKeyInput, opts ...request.Option) (*EnableKeyOutput, error) { + req, out := c.EnableKeyRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opEnableKeyRotation = "EnableKeyRotation" + +// EnableKeyRotationRequest generates a "aws/request.Request" representing the +// client's request for the EnableKeyRotation operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See EnableKeyRotation for more information on using the EnableKeyRotation +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the EnableKeyRotationRequest method. +// req, resp := client.EnableKeyRotationRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/EnableKeyRotation +func (c *KMS) EnableKeyRotationRequest(input *EnableKeyRotationInput) (req *request.Request, output *EnableKeyRotationOutput) { + op := &request.Operation{ + Name: opEnableKeyRotation, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &EnableKeyRotationInput{} + } + + output = &EnableKeyRotationOutput{} + req = c.newRequest(op, input, output) + req.Handlers.Unmarshal.Swap(jsonrpc.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + return +} + +// EnableKeyRotation API operation for AWS Key Management Service. +// +// Enables automatic rotation of the key material (https://docs.aws.amazon.com/kms/latest/developerguide/rotate-keys.html) +// for the specified symmetric customer master key (CMK). You cannot perform +// this operation on a CMK in a different AWS account. +// +// You cannot enable automatic rotation of asymmetric CMKs, CMKs with imported +// key material, or CMKs in a custom key store (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html). +// +// The CMK that you use for this operation must be in a compatible key state. +// For details, see How Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation EnableKeyRotation for usage and error information. +// +// Returned Error Types: +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * DisabledException +// The request was rejected because the specified CMK is not enabled. +// +// * InvalidArnException +// The request was rejected because a specified ARN, or an ARN in a key policy, +// is not valid. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// * UnsupportedOperationException +// The request was rejected because a specified parameter is not supported or +// a specified resource is not valid for this operation. +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/EnableKeyRotation +func (c *KMS) EnableKeyRotation(input *EnableKeyRotationInput) (*EnableKeyRotationOutput, error) { + req, out := c.EnableKeyRotationRequest(input) + return out, req.Send() +} + +// EnableKeyRotationWithContext is the same as EnableKeyRotation with the addition of +// the ability to pass a context and additional request options. +// +// See EnableKeyRotation for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) EnableKeyRotationWithContext(ctx aws.Context, input *EnableKeyRotationInput, opts ...request.Option) (*EnableKeyRotationOutput, error) { + req, out := c.EnableKeyRotationRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opEncrypt = "Encrypt" + +// EncryptRequest generates a "aws/request.Request" representing the +// client's request for the Encrypt operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See Encrypt for more information on using the Encrypt +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the EncryptRequest method. +// req, resp := client.EncryptRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/Encrypt +func (c *KMS) EncryptRequest(input *EncryptInput) (req *request.Request, output *EncryptOutput) { + op := &request.Operation{ + Name: opEncrypt, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &EncryptInput{} + } + + output = &EncryptOutput{} + req = c.newRequest(op, input, output) + return +} + +// Encrypt API operation for AWS Key Management Service. +// +// Encrypts plaintext into ciphertext by using a customer master key (CMK). +// The Encrypt operation has two primary use cases: +// +// * You can encrypt small amounts of arbitrary data, such as a personal +// identifier or database password, or other sensitive information. +// +// * You can use the Encrypt operation to move encrypted data from one AWS +// Region to another. For example, in Region A, generate a data key and use +// the plaintext key to encrypt your data. Then, in Region A, use the Encrypt +// operation to encrypt the plaintext data key under a CMK in Region B. Now, +// you can move the encrypted data and the encrypted data key to Region B. +// When necessary, you can decrypt the encrypted data key and the encrypted +// data entirely within in Region B. +// +// You don't need to use the Encrypt operation to encrypt a data key. The GenerateDataKey +// and GenerateDataKeyPair operations return a plaintext data key and an encrypted +// copy of that data key. +// +// When you encrypt data, you must specify a symmetric or asymmetric CMK to +// use in the encryption operation. The CMK must have a KeyUsage value of ENCRYPT_DECRYPT. +// To find the KeyUsage of a CMK, use the DescribeKey operation. +// +// If you use a symmetric CMK, you can use an encryption context to add additional +// security to your encryption operation. If you specify an EncryptionContext +// when encrypting data, you must specify the same encryption context (a case-sensitive +// exact match) when decrypting the data. Otherwise, the request to decrypt +// fails with an InvalidCiphertextException. For more information, see Encryption +// Context (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#encrypt_context) +// in the AWS Key Management Service Developer Guide. +// +// If you specify an asymmetric CMK, you must also specify the encryption algorithm. +// The algorithm must be compatible with the CMK type. +// +// When you use an asymmetric CMK to encrypt or reencrypt data, be sure to record +// the CMK and encryption algorithm that you choose. You will be required to +// provide the same CMK and encryption algorithm when you decrypt the data. +// If the CMK and algorithm do not match the values used to encrypt the data, +// the decrypt operation fails. +// +// You are not required to supply the CMK ID and encryption algorithm when you +// decrypt with symmetric CMKs because AWS KMS stores this information in the +// ciphertext blob. AWS KMS cannot store metadata in ciphertext generated with +// asymmetric keys. The standard format for asymmetric key ciphertext does not +// include configurable fields. +// +// The maximum size of the data that you can encrypt varies with the type of +// CMK and the encryption algorithm that you choose. +// +// * Symmetric CMKs SYMMETRIC_DEFAULT: 4096 bytes +// +// * RSA_2048 RSAES_OAEP_SHA_1: 214 bytes RSAES_OAEP_SHA_256: 190 bytes +// +// * RSA_3072 RSAES_OAEP_SHA_1: 342 bytes RSAES_OAEP_SHA_256: 318 bytes +// +// * RSA_4096 RSAES_OAEP_SHA_1: 470 bytes RSAES_OAEP_SHA_256: 446 bytes +// +// The CMK that you use for this operation must be in a compatible key state. +// For details, see How Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide. +// +// To perform this operation on a CMK in a different AWS account, specify the +// key ARN or alias ARN in the value of the KeyId parameter. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation Encrypt for usage and error information. +// +// Returned Error Types: +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * DisabledException +// The request was rejected because the specified CMK is not enabled. +// +// * KeyUnavailableException +// The request was rejected because the specified CMK was not available. You +// can retry the request. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InvalidKeyUsageException +// The request was rejected for one of the following reasons: +// +// * The KeyUsage value of the CMK is incompatible with the API operation. +// +// * The encryption algorithm or signing algorithm specified for the operation +// is incompatible with the type of key material in the CMK (CustomerMasterKeySpec). +// +// For encrypting, decrypting, re-encrypting, and generating data keys, the +// KeyUsage must be ENCRYPT_DECRYPT. For signing and verifying, the KeyUsage +// must be SIGN_VERIFY. To find the KeyUsage of a CMK, use the DescribeKey operation. +// +// To find the encryption or signing algorithms supported for a particular CMK, +// use the DescribeKey operation. +// +// * InvalidGrantTokenException +// The request was rejected because the specified grant token is not valid. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/Encrypt +func (c *KMS) Encrypt(input *EncryptInput) (*EncryptOutput, error) { + req, out := c.EncryptRequest(input) + return out, req.Send() +} + +// EncryptWithContext is the same as Encrypt with the addition of +// the ability to pass a context and additional request options. +// +// See Encrypt for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) EncryptWithContext(ctx aws.Context, input *EncryptInput, opts ...request.Option) (*EncryptOutput, error) { + req, out := c.EncryptRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opGenerateDataKey = "GenerateDataKey" + +// GenerateDataKeyRequest generates a "aws/request.Request" representing the +// client's request for the GenerateDataKey operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See GenerateDataKey for more information on using the GenerateDataKey +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the GenerateDataKeyRequest method. +// req, resp := client.GenerateDataKeyRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/GenerateDataKey +func (c *KMS) GenerateDataKeyRequest(input *GenerateDataKeyInput) (req *request.Request, output *GenerateDataKeyOutput) { + op := &request.Operation{ + Name: opGenerateDataKey, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &GenerateDataKeyInput{} + } + + output = &GenerateDataKeyOutput{} + req = c.newRequest(op, input, output) + return +} + +// GenerateDataKey API operation for AWS Key Management Service. +// +// Generates a unique symmetric data key for client-side encryption. This operation +// returns a plaintext copy of the data key and a copy that is encrypted under +// a customer master key (CMK) that you specify. You can use the plaintext key +// to encrypt your data outside of AWS KMS and store the encrypted data key +// with the encrypted data. +// +// GenerateDataKey returns a unique data key for each request. The bytes in +// the plaintext key are not related to the caller or the CMK. +// +// To generate a data key, specify the symmetric CMK that will be used to encrypt +// the data key. You cannot use an asymmetric CMK to generate data keys. To +// get the type of your CMK, use the DescribeKey operation. You must also specify +// the length of the data key. Use either the KeySpec or NumberOfBytes parameters +// (but not both). For 128-bit and 256-bit data keys, use the KeySpec parameter. +// +// To get only an encrypted copy of the data key, use GenerateDataKeyWithoutPlaintext. +// To generate an asymmetric data key pair, use the GenerateDataKeyPair or GenerateDataKeyPairWithoutPlaintext +// operation. To get a cryptographically secure random byte string, use GenerateRandom. +// +// You can use the optional encryption context to add additional security to +// the encryption operation. If you specify an EncryptionContext, you must specify +// the same encryption context (a case-sensitive exact match) when decrypting +// the encrypted data key. Otherwise, the request to decrypt fails with an InvalidCiphertextException. +// For more information, see Encryption Context (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#encrypt_context) +// in the AWS Key Management Service Developer Guide. +// +// The CMK that you use for this operation must be in a compatible key state. +// For details, see How Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide. +// +// How to use your data key +// +// We recommend that you use the following pattern to encrypt data locally in +// your application. You can write your own code or use a client-side encryption +// library, such as the AWS Encryption SDK (https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/), +// the Amazon DynamoDB Encryption Client (https://docs.aws.amazon.com/dynamodb-encryption-client/latest/devguide/), +// or Amazon S3 client-side encryption (https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingClientSideEncryption.html) +// to do these tasks for you. +// +// To encrypt data outside of AWS KMS: +// +// Use the GenerateDataKey operation to get a data key. +// +// Use the plaintext data key (in the Plaintext field of the response) to encrypt +// your data outside of AWS KMS. Then erase the plaintext data key from memory. +// +// Store the encrypted data key (in the CiphertextBlob field of the response) +// with the encrypted data. +// +// To decrypt data outside of AWS KMS: +// +// Use the Decrypt operation to decrypt the encrypted data key. The operation +// returns a plaintext copy of the data key. +// +// Use the plaintext data key to decrypt data outside of AWS KMS, then erase +// the plaintext data key from memory. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation GenerateDataKey for usage and error information. +// +// Returned Error Types: +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * DisabledException +// The request was rejected because the specified CMK is not enabled. +// +// * KeyUnavailableException +// The request was rejected because the specified CMK was not available. You +// can retry the request. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InvalidKeyUsageException +// The request was rejected for one of the following reasons: +// +// * The KeyUsage value of the CMK is incompatible with the API operation. +// +// * The encryption algorithm or signing algorithm specified for the operation +// is incompatible with the type of key material in the CMK (CustomerMasterKeySpec). +// +// For encrypting, decrypting, re-encrypting, and generating data keys, the +// KeyUsage must be ENCRYPT_DECRYPT. For signing and verifying, the KeyUsage +// must be SIGN_VERIFY. To find the KeyUsage of a CMK, use the DescribeKey operation. +// +// To find the encryption or signing algorithms supported for a particular CMK, +// use the DescribeKey operation. +// +// * InvalidGrantTokenException +// The request was rejected because the specified grant token is not valid. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/GenerateDataKey +func (c *KMS) GenerateDataKey(input *GenerateDataKeyInput) (*GenerateDataKeyOutput, error) { + req, out := c.GenerateDataKeyRequest(input) + return out, req.Send() +} + +// GenerateDataKeyWithContext is the same as GenerateDataKey with the addition of +// the ability to pass a context and additional request options. +// +// See GenerateDataKey for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) GenerateDataKeyWithContext(ctx aws.Context, input *GenerateDataKeyInput, opts ...request.Option) (*GenerateDataKeyOutput, error) { + req, out := c.GenerateDataKeyRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opGenerateDataKeyPair = "GenerateDataKeyPair" + +// GenerateDataKeyPairRequest generates a "aws/request.Request" representing the +// client's request for the GenerateDataKeyPair operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See GenerateDataKeyPair for more information on using the GenerateDataKeyPair +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the GenerateDataKeyPairRequest method. +// req, resp := client.GenerateDataKeyPairRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/GenerateDataKeyPair +func (c *KMS) GenerateDataKeyPairRequest(input *GenerateDataKeyPairInput) (req *request.Request, output *GenerateDataKeyPairOutput) { + op := &request.Operation{ + Name: opGenerateDataKeyPair, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &GenerateDataKeyPairInput{} + } + + output = &GenerateDataKeyPairOutput{} + req = c.newRequest(op, input, output) + return +} + +// GenerateDataKeyPair API operation for AWS Key Management Service. +// +// Generates a unique asymmetric data key pair. The GenerateDataKeyPair operation +// returns a plaintext public key, a plaintext private key, and a copy of the +// private key that is encrypted under the symmetric CMK you specify. You can +// use the data key pair to perform asymmetric cryptography outside of AWS KMS. +// +// GenerateDataKeyPair returns a unique data key pair for each request. The +// bytes in the keys are not related to the caller or the CMK that is used to +// encrypt the private key. +// +// You can use the public key that GenerateDataKeyPair returns to encrypt data +// or verify a signature outside of AWS KMS. Then, store the encrypted private +// key with the data. When you are ready to decrypt data or sign a message, +// you can use the Decrypt operation to decrypt the encrypted private key. +// +// To generate a data key pair, you must specify a symmetric customer master +// key (CMK) to encrypt the private key in a data key pair. You cannot use an +// asymmetric CMK or a CMK in a custom key store. To get the type and origin +// of your CMK, use the DescribeKey operation. +// +// If you are using the data key pair to encrypt data, or for any operation +// where you don't immediately need a private key, consider using the GenerateDataKeyPairWithoutPlaintext +// operation. GenerateDataKeyPairWithoutPlaintext returns a plaintext public +// key and an encrypted private key, but omits the plaintext private key that +// you need only to decrypt ciphertext or sign a message. Later, when you need +// to decrypt the data or sign a message, use the Decrypt operation to decrypt +// the encrypted private key in the data key pair. +// +// You can use the optional encryption context to add additional security to +// the encryption operation. If you specify an EncryptionContext, you must specify +// the same encryption context (a case-sensitive exact match) when decrypting +// the encrypted data key. Otherwise, the request to decrypt fails with an InvalidCiphertextException. +// For more information, see Encryption Context (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#encrypt_context) +// in the AWS Key Management Service Developer Guide. +// +// The CMK that you use for this operation must be in a compatible key state. +// For details, see How Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation GenerateDataKeyPair for usage and error information. +// +// Returned Error Types: +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * DisabledException +// The request was rejected because the specified CMK is not enabled. +// +// * KeyUnavailableException +// The request was rejected because the specified CMK was not available. You +// can retry the request. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InvalidKeyUsageException +// The request was rejected for one of the following reasons: +// +// * The KeyUsage value of the CMK is incompatible with the API operation. +// +// * The encryption algorithm or signing algorithm specified for the operation +// is incompatible with the type of key material in the CMK (CustomerMasterKeySpec). +// +// For encrypting, decrypting, re-encrypting, and generating data keys, the +// KeyUsage must be ENCRYPT_DECRYPT. For signing and verifying, the KeyUsage +// must be SIGN_VERIFY. To find the KeyUsage of a CMK, use the DescribeKey operation. +// +// To find the encryption or signing algorithms supported for a particular CMK, +// use the DescribeKey operation. +// +// * InvalidGrantTokenException +// The request was rejected because the specified grant token is not valid. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// * UnsupportedOperationException +// The request was rejected because a specified parameter is not supported or +// a specified resource is not valid for this operation. +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/GenerateDataKeyPair +func (c *KMS) GenerateDataKeyPair(input *GenerateDataKeyPairInput) (*GenerateDataKeyPairOutput, error) { + req, out := c.GenerateDataKeyPairRequest(input) + return out, req.Send() +} + +// GenerateDataKeyPairWithContext is the same as GenerateDataKeyPair with the addition of +// the ability to pass a context and additional request options. +// +// See GenerateDataKeyPair for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) GenerateDataKeyPairWithContext(ctx aws.Context, input *GenerateDataKeyPairInput, opts ...request.Option) (*GenerateDataKeyPairOutput, error) { + req, out := c.GenerateDataKeyPairRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opGenerateDataKeyPairWithoutPlaintext = "GenerateDataKeyPairWithoutPlaintext" + +// GenerateDataKeyPairWithoutPlaintextRequest generates a "aws/request.Request" representing the +// client's request for the GenerateDataKeyPairWithoutPlaintext operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See GenerateDataKeyPairWithoutPlaintext for more information on using the GenerateDataKeyPairWithoutPlaintext +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the GenerateDataKeyPairWithoutPlaintextRequest method. +// req, resp := client.GenerateDataKeyPairWithoutPlaintextRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/GenerateDataKeyPairWithoutPlaintext +func (c *KMS) GenerateDataKeyPairWithoutPlaintextRequest(input *GenerateDataKeyPairWithoutPlaintextInput) (req *request.Request, output *GenerateDataKeyPairWithoutPlaintextOutput) { + op := &request.Operation{ + Name: opGenerateDataKeyPairWithoutPlaintext, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &GenerateDataKeyPairWithoutPlaintextInput{} + } + + output = &GenerateDataKeyPairWithoutPlaintextOutput{} + req = c.newRequest(op, input, output) + return +} + +// GenerateDataKeyPairWithoutPlaintext API operation for AWS Key Management Service. +// +// Generates a unique asymmetric data key pair. The GenerateDataKeyPairWithoutPlaintext +// operation returns a plaintext public key and a copy of the private key that +// is encrypted under the symmetric CMK you specify. Unlike GenerateDataKeyPair, +// this operation does not return a plaintext private key. +// +// To generate a data key pair, you must specify a symmetric customer master +// key (CMK) to encrypt the private key in the data key pair. You cannot use +// an asymmetric CMK or a CMK in a custom key store. To get the type and origin +// of your CMK, use the KeySpec field in the DescribeKey response. +// +// You can use the public key that GenerateDataKeyPairWithoutPlaintext returns +// to encrypt data or verify a signature outside of AWS KMS. Then, store the +// encrypted private key with the data. When you are ready to decrypt data or +// sign a message, you can use the Decrypt operation to decrypt the encrypted +// private key. +// +// GenerateDataKeyPairWithoutPlaintext returns a unique data key pair for each +// request. The bytes in the key are not related to the caller or CMK that is +// used to encrypt the private key. +// +// You can use the optional encryption context to add additional security to +// the encryption operation. If you specify an EncryptionContext, you must specify +// the same encryption context (a case-sensitive exact match) when decrypting +// the encrypted data key. Otherwise, the request to decrypt fails with an InvalidCiphertextException. +// For more information, see Encryption Context (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#encrypt_context) +// in the AWS Key Management Service Developer Guide. +// +// The CMK that you use for this operation must be in a compatible key state. +// For details, see How Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation GenerateDataKeyPairWithoutPlaintext for usage and error information. +// +// Returned Error Types: +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * DisabledException +// The request was rejected because the specified CMK is not enabled. +// +// * KeyUnavailableException +// The request was rejected because the specified CMK was not available. You +// can retry the request. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InvalidKeyUsageException +// The request was rejected for one of the following reasons: +// +// * The KeyUsage value of the CMK is incompatible with the API operation. +// +// * The encryption algorithm or signing algorithm specified for the operation +// is incompatible with the type of key material in the CMK (CustomerMasterKeySpec). +// +// For encrypting, decrypting, re-encrypting, and generating data keys, the +// KeyUsage must be ENCRYPT_DECRYPT. For signing and verifying, the KeyUsage +// must be SIGN_VERIFY. To find the KeyUsage of a CMK, use the DescribeKey operation. +// +// To find the encryption or signing algorithms supported for a particular CMK, +// use the DescribeKey operation. +// +// * InvalidGrantTokenException +// The request was rejected because the specified grant token is not valid. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// * UnsupportedOperationException +// The request was rejected because a specified parameter is not supported or +// a specified resource is not valid for this operation. +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/GenerateDataKeyPairWithoutPlaintext +func (c *KMS) GenerateDataKeyPairWithoutPlaintext(input *GenerateDataKeyPairWithoutPlaintextInput) (*GenerateDataKeyPairWithoutPlaintextOutput, error) { + req, out := c.GenerateDataKeyPairWithoutPlaintextRequest(input) + return out, req.Send() +} + +// GenerateDataKeyPairWithoutPlaintextWithContext is the same as GenerateDataKeyPairWithoutPlaintext with the addition of +// the ability to pass a context and additional request options. +// +// See GenerateDataKeyPairWithoutPlaintext for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) GenerateDataKeyPairWithoutPlaintextWithContext(ctx aws.Context, input *GenerateDataKeyPairWithoutPlaintextInput, opts ...request.Option) (*GenerateDataKeyPairWithoutPlaintextOutput, error) { + req, out := c.GenerateDataKeyPairWithoutPlaintextRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opGenerateDataKeyWithoutPlaintext = "GenerateDataKeyWithoutPlaintext" + +// GenerateDataKeyWithoutPlaintextRequest generates a "aws/request.Request" representing the +// client's request for the GenerateDataKeyWithoutPlaintext operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See GenerateDataKeyWithoutPlaintext for more information on using the GenerateDataKeyWithoutPlaintext +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the GenerateDataKeyWithoutPlaintextRequest method. +// req, resp := client.GenerateDataKeyWithoutPlaintextRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/GenerateDataKeyWithoutPlaintext +func (c *KMS) GenerateDataKeyWithoutPlaintextRequest(input *GenerateDataKeyWithoutPlaintextInput) (req *request.Request, output *GenerateDataKeyWithoutPlaintextOutput) { + op := &request.Operation{ + Name: opGenerateDataKeyWithoutPlaintext, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &GenerateDataKeyWithoutPlaintextInput{} + } + + output = &GenerateDataKeyWithoutPlaintextOutput{} + req = c.newRequest(op, input, output) + return +} + +// GenerateDataKeyWithoutPlaintext API operation for AWS Key Management Service. +// +// Generates a unique symmetric data key. This operation returns a data key +// that is encrypted under a customer master key (CMK) that you specify. To +// request an asymmetric data key pair, use the GenerateDataKeyPair or GenerateDataKeyPairWithoutPlaintext +// operations. +// +// GenerateDataKeyWithoutPlaintext is identical to the GenerateDataKey operation +// except that returns only the encrypted copy of the data key. This operation +// is useful for systems that need to encrypt data at some point, but not immediately. +// When you need to encrypt the data, you call the Decrypt operation on the +// encrypted copy of the key. +// +// It's also useful in distributed systems with different levels of trust. For +// example, you might store encrypted data in containers. One component of your +// system creates new containers and stores an encrypted data key with each +// container. Then, a different component puts the data into the containers. +// That component first decrypts the data key, uses the plaintext data key to +// encrypt data, puts the encrypted data into the container, and then destroys +// the plaintext data key. In this system, the component that creates the containers +// never sees the plaintext data key. +// +// GenerateDataKeyWithoutPlaintext returns a unique data key for each request. +// The bytes in the keys are not related to the caller or CMK that is used to +// encrypt the private key. +// +// To generate a data key, you must specify the symmetric customer master key +// (CMK) that is used to encrypt the data key. You cannot use an asymmetric +// CMK to generate a data key. To get the type of your CMK, use the DescribeKey +// operation. +// +// If the operation succeeds, you will find the encrypted copy of the data key +// in the CiphertextBlob field. +// +// You can use the optional encryption context to add additional security to +// the encryption operation. If you specify an EncryptionContext, you must specify +// the same encryption context (a case-sensitive exact match) when decrypting +// the encrypted data key. Otherwise, the request to decrypt fails with an InvalidCiphertextException. +// For more information, see Encryption Context (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#encrypt_context) +// in the AWS Key Management Service Developer Guide. +// +// The CMK that you use for this operation must be in a compatible key state. +// For details, see How Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation GenerateDataKeyWithoutPlaintext for usage and error information. +// +// Returned Error Types: +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * DisabledException +// The request was rejected because the specified CMK is not enabled. +// +// * KeyUnavailableException +// The request was rejected because the specified CMK was not available. You +// can retry the request. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InvalidKeyUsageException +// The request was rejected for one of the following reasons: +// +// * The KeyUsage value of the CMK is incompatible with the API operation. +// +// * The encryption algorithm or signing algorithm specified for the operation +// is incompatible with the type of key material in the CMK (CustomerMasterKeySpec). +// +// For encrypting, decrypting, re-encrypting, and generating data keys, the +// KeyUsage must be ENCRYPT_DECRYPT. For signing and verifying, the KeyUsage +// must be SIGN_VERIFY. To find the KeyUsage of a CMK, use the DescribeKey operation. +// +// To find the encryption or signing algorithms supported for a particular CMK, +// use the DescribeKey operation. +// +// * InvalidGrantTokenException +// The request was rejected because the specified grant token is not valid. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/GenerateDataKeyWithoutPlaintext +func (c *KMS) GenerateDataKeyWithoutPlaintext(input *GenerateDataKeyWithoutPlaintextInput) (*GenerateDataKeyWithoutPlaintextOutput, error) { + req, out := c.GenerateDataKeyWithoutPlaintextRequest(input) + return out, req.Send() +} + +// GenerateDataKeyWithoutPlaintextWithContext is the same as GenerateDataKeyWithoutPlaintext with the addition of +// the ability to pass a context and additional request options. +// +// See GenerateDataKeyWithoutPlaintext for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) GenerateDataKeyWithoutPlaintextWithContext(ctx aws.Context, input *GenerateDataKeyWithoutPlaintextInput, opts ...request.Option) (*GenerateDataKeyWithoutPlaintextOutput, error) { + req, out := c.GenerateDataKeyWithoutPlaintextRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opGenerateRandom = "GenerateRandom" + +// GenerateRandomRequest generates a "aws/request.Request" representing the +// client's request for the GenerateRandom operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See GenerateRandom for more information on using the GenerateRandom +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the GenerateRandomRequest method. +// req, resp := client.GenerateRandomRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/GenerateRandom +func (c *KMS) GenerateRandomRequest(input *GenerateRandomInput) (req *request.Request, output *GenerateRandomOutput) { + op := &request.Operation{ + Name: opGenerateRandom, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &GenerateRandomInput{} + } + + output = &GenerateRandomOutput{} + req = c.newRequest(op, input, output) + return +} + +// GenerateRandom API operation for AWS Key Management Service. +// +// Returns a random byte string that is cryptographically secure. +// +// By default, the random byte string is generated in AWS KMS. To generate the +// byte string in the AWS CloudHSM cluster that is associated with a custom +// key store (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html), +// specify the custom key store ID. +// +// For more information about entropy and random number generation, see the +// AWS Key Management Service Cryptographic Details (https://d0.awsstatic.com/whitepapers/KMS-Cryptographic-Details.pdf) +// whitepaper. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation GenerateRandom for usage and error information. +// +// Returned Error Types: +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * CustomKeyStoreNotFoundException +// The request was rejected because AWS KMS cannot find a custom key store with +// the specified key store name or ID. +// +// * CustomKeyStoreInvalidStateException +// The request was rejected because of the ConnectionState of the custom key +// store. To get the ConnectionState of a custom key store, use the DescribeCustomKeyStores +// operation. +// +// This exception is thrown under the following conditions: +// +// * You requested the CreateKey or GenerateRandom operation in a custom +// key store that is not connected. These operations are valid only when +// the custom key store ConnectionState is CONNECTED. +// +// * You requested the UpdateCustomKeyStore or DeleteCustomKeyStore operation +// on a custom key store that is not disconnected. This operation is valid +// only when the custom key store ConnectionState is DISCONNECTED. +// +// * You requested the ConnectCustomKeyStore operation on a custom key store +// with a ConnectionState of DISCONNECTING or FAILED. This operation is valid +// for all other ConnectionState values. +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/GenerateRandom +func (c *KMS) GenerateRandom(input *GenerateRandomInput) (*GenerateRandomOutput, error) { + req, out := c.GenerateRandomRequest(input) + return out, req.Send() +} + +// GenerateRandomWithContext is the same as GenerateRandom with the addition of +// the ability to pass a context and additional request options. +// +// See GenerateRandom for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) GenerateRandomWithContext(ctx aws.Context, input *GenerateRandomInput, opts ...request.Option) (*GenerateRandomOutput, error) { + req, out := c.GenerateRandomRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opGetKeyPolicy = "GetKeyPolicy" + +// GetKeyPolicyRequest generates a "aws/request.Request" representing the +// client's request for the GetKeyPolicy operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See GetKeyPolicy for more information on using the GetKeyPolicy +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the GetKeyPolicyRequest method. +// req, resp := client.GetKeyPolicyRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/GetKeyPolicy +func (c *KMS) GetKeyPolicyRequest(input *GetKeyPolicyInput) (req *request.Request, output *GetKeyPolicyOutput) { + op := &request.Operation{ + Name: opGetKeyPolicy, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &GetKeyPolicyInput{} + } + + output = &GetKeyPolicyOutput{} + req = c.newRequest(op, input, output) + return +} + +// GetKeyPolicy API operation for AWS Key Management Service. +// +// Gets a key policy attached to the specified customer master key (CMK). You +// cannot perform this operation on a CMK in a different AWS account. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation GetKeyPolicy for usage and error information. +// +// Returned Error Types: +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * InvalidArnException +// The request was rejected because a specified ARN, or an ARN in a key policy, +// is not valid. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/GetKeyPolicy +func (c *KMS) GetKeyPolicy(input *GetKeyPolicyInput) (*GetKeyPolicyOutput, error) { + req, out := c.GetKeyPolicyRequest(input) + return out, req.Send() +} + +// GetKeyPolicyWithContext is the same as GetKeyPolicy with the addition of +// the ability to pass a context and additional request options. +// +// See GetKeyPolicy for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) GetKeyPolicyWithContext(ctx aws.Context, input *GetKeyPolicyInput, opts ...request.Option) (*GetKeyPolicyOutput, error) { + req, out := c.GetKeyPolicyRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opGetKeyRotationStatus = "GetKeyRotationStatus" + +// GetKeyRotationStatusRequest generates a "aws/request.Request" representing the +// client's request for the GetKeyRotationStatus operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See GetKeyRotationStatus for more information on using the GetKeyRotationStatus +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the GetKeyRotationStatusRequest method. +// req, resp := client.GetKeyRotationStatusRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/GetKeyRotationStatus +func (c *KMS) GetKeyRotationStatusRequest(input *GetKeyRotationStatusInput) (req *request.Request, output *GetKeyRotationStatusOutput) { + op := &request.Operation{ + Name: opGetKeyRotationStatus, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &GetKeyRotationStatusInput{} + } + + output = &GetKeyRotationStatusOutput{} + req = c.newRequest(op, input, output) + return +} + +// GetKeyRotationStatus API operation for AWS Key Management Service. +// +// Gets a Boolean value that indicates whether automatic rotation of the key +// material (https://docs.aws.amazon.com/kms/latest/developerguide/rotate-keys.html) +// is enabled for the specified customer master key (CMK). +// +// You cannot enable automatic rotation of asymmetric CMKs, CMKs with imported +// key material, or CMKs in a custom key store (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html). +// The key rotation status for these CMKs is always false. +// +// The CMK that you use for this operation must be in a compatible key state. +// For details, see How Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide. +// +// * Disabled: The key rotation status does not change when you disable a +// CMK. However, while the CMK is disabled, AWS KMS does not rotate the backing +// key. +// +// * Pending deletion: While a CMK is pending deletion, its key rotation +// status is false and AWS KMS does not rotate the backing key. If you cancel +// the deletion, the original key rotation status is restored. +// +// To perform this operation on a CMK in a different AWS account, specify the +// key ARN in the value of the KeyId parameter. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation GetKeyRotationStatus for usage and error information. +// +// Returned Error Types: +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * InvalidArnException +// The request was rejected because a specified ARN, or an ARN in a key policy, +// is not valid. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// * UnsupportedOperationException +// The request was rejected because a specified parameter is not supported or +// a specified resource is not valid for this operation. +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/GetKeyRotationStatus +func (c *KMS) GetKeyRotationStatus(input *GetKeyRotationStatusInput) (*GetKeyRotationStatusOutput, error) { + req, out := c.GetKeyRotationStatusRequest(input) + return out, req.Send() +} + +// GetKeyRotationStatusWithContext is the same as GetKeyRotationStatus with the addition of +// the ability to pass a context and additional request options. +// +// See GetKeyRotationStatus for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) GetKeyRotationStatusWithContext(ctx aws.Context, input *GetKeyRotationStatusInput, opts ...request.Option) (*GetKeyRotationStatusOutput, error) { + req, out := c.GetKeyRotationStatusRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opGetParametersForImport = "GetParametersForImport" + +// GetParametersForImportRequest generates a "aws/request.Request" representing the +// client's request for the GetParametersForImport operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See GetParametersForImport for more information on using the GetParametersForImport +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the GetParametersForImportRequest method. +// req, resp := client.GetParametersForImportRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/GetParametersForImport +func (c *KMS) GetParametersForImportRequest(input *GetParametersForImportInput) (req *request.Request, output *GetParametersForImportOutput) { + op := &request.Operation{ + Name: opGetParametersForImport, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &GetParametersForImportInput{} + } + + output = &GetParametersForImportOutput{} + req = c.newRequest(op, input, output) + return +} + +// GetParametersForImport API operation for AWS Key Management Service. +// +// Returns the items you need to import key material into a symmetric, customer +// managed customer master key (CMK). For more information about importing key +// material into AWS KMS, see Importing Key Material (https://docs.aws.amazon.com/kms/latest/developerguide/importing-keys.html) +// in the AWS Key Management Service Developer Guide. +// +// This operation returns a public key and an import token. Use the public key +// to encrypt the symmetric key material. Store the import token to send with +// a subsequent ImportKeyMaterial request. +// +// You must specify the key ID of the symmetric CMK into which you will import +// key material. This CMK's Origin must be EXTERNAL. You must also specify the +// wrapping algorithm and type of wrapping key (public key) that you will use +// to encrypt the key material. You cannot perform this operation on an asymmetric +// CMK or on any CMK in a different AWS account. +// +// To import key material, you must use the public key and import token from +// the same response. These items are valid for 24 hours. The expiration date +// and time appear in the GetParametersForImport response. You cannot use an +// expired token in an ImportKeyMaterial request. If your key and token expire, +// send another GetParametersForImport request. +// +// The CMK that you use for this operation must be in a compatible key state. +// For details, see How Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation GetParametersForImport for usage and error information. +// +// Returned Error Types: +// * InvalidArnException +// The request was rejected because a specified ARN, or an ARN in a key policy, +// is not valid. +// +// * UnsupportedOperationException +// The request was rejected because a specified parameter is not supported or +// a specified resource is not valid for this operation. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/GetParametersForImport +func (c *KMS) GetParametersForImport(input *GetParametersForImportInput) (*GetParametersForImportOutput, error) { + req, out := c.GetParametersForImportRequest(input) + return out, req.Send() +} + +// GetParametersForImportWithContext is the same as GetParametersForImport with the addition of +// the ability to pass a context and additional request options. +// +// See GetParametersForImport for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) GetParametersForImportWithContext(ctx aws.Context, input *GetParametersForImportInput, opts ...request.Option) (*GetParametersForImportOutput, error) { + req, out := c.GetParametersForImportRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opGetPublicKey = "GetPublicKey" + +// GetPublicKeyRequest generates a "aws/request.Request" representing the +// client's request for the GetPublicKey operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See GetPublicKey for more information on using the GetPublicKey +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the GetPublicKeyRequest method. +// req, resp := client.GetPublicKeyRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/GetPublicKey +func (c *KMS) GetPublicKeyRequest(input *GetPublicKeyInput) (req *request.Request, output *GetPublicKeyOutput) { + op := &request.Operation{ + Name: opGetPublicKey, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &GetPublicKeyInput{} + } + + output = &GetPublicKeyOutput{} + req = c.newRequest(op, input, output) + return +} + +// GetPublicKey API operation for AWS Key Management Service. +// +// Returns the public key of an asymmetric CMK. Unlike the private key of a +// asymmetric CMK, which never leaves AWS KMS unencrypted, callers with kms:GetPublicKey +// permission can download the public key of an asymmetric CMK. You can share +// the public key to allow others to encrypt messages and verify signatures +// outside of AWS KMS. For information about symmetric and asymmetric CMKs, +// see Using Symmetric and Asymmetric CMKs (https://docs.aws.amazon.com/kms/latest/developerguide/symmetric-asymmetric.html) +// in the AWS Key Management Service Developer Guide. +// +// You do not need to download the public key. Instead, you can use the public +// key within AWS KMS by calling the Encrypt, ReEncrypt, or Verify operations +// with the identifier of an asymmetric CMK. When you use the public key within +// AWS KMS, you benefit from the authentication, authorization, and logging +// that are part of every AWS KMS operation. You also reduce of risk of encrypting +// data that cannot be decrypted. These features are not effective outside of +// AWS KMS. For details, see Special Considerations for Downloading Public Keys +// (https://docs.aws.amazon.com/kms/latest/developerguide/download-public-key.html#download-public-key-considerations). +// +// To help you use the public key safely outside of AWS KMS, GetPublicKey returns +// important information about the public key in the response, including: +// +// * CustomerMasterKeySpec (https://docs.aws.amazon.com/kms/latest/APIReference/API_GetPublicKey.html#KMS-GetPublicKey-response-CustomerMasterKeySpec): +// The type of key material in the public key, such as RSA_4096 or ECC_NIST_P521. +// +// * KeyUsage (https://docs.aws.amazon.com/kms/latest/APIReference/API_GetPublicKey.html#KMS-GetPublicKey-response-KeyUsage): +// Whether the key is used for encryption or signing. +// +// * EncryptionAlgorithms (https://docs.aws.amazon.com/kms/latest/APIReference/API_GetPublicKey.html#KMS-GetPublicKey-response-EncryptionAlgorithms) +// or SigningAlgorithms (https://docs.aws.amazon.com/kms/latest/APIReference/API_GetPublicKey.html#KMS-GetPublicKey-response-SigningAlgorithms): +// A list of the encryption algorithms or the signing algorithms for the +// key. +// +// Although AWS KMS cannot enforce these restrictions on external operations, +// it is crucial that you use this information to prevent the public key from +// being used improperly. For example, you can prevent a public signing key +// from being used encrypt data, or prevent a public key from being used with +// an encryption algorithm that is not supported by AWS KMS. You can also avoid +// errors, such as using the wrong signing algorithm in a verification operation. +// +// The CMK that you use for this operation must be in a compatible key state. +// For details, see How Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation GetPublicKey for usage and error information. +// +// Returned Error Types: +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * DisabledException +// The request was rejected because the specified CMK is not enabled. +// +// * KeyUnavailableException +// The request was rejected because the specified CMK was not available. You +// can retry the request. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * UnsupportedOperationException +// The request was rejected because a specified parameter is not supported or +// a specified resource is not valid for this operation. +// +// * InvalidArnException +// The request was rejected because a specified ARN, or an ARN in a key policy, +// is not valid. +// +// * InvalidGrantTokenException +// The request was rejected because the specified grant token is not valid. +// +// * InvalidKeyUsageException +// The request was rejected for one of the following reasons: +// +// * The KeyUsage value of the CMK is incompatible with the API operation. +// +// * The encryption algorithm or signing algorithm specified for the operation +// is incompatible with the type of key material in the CMK (CustomerMasterKeySpec). +// +// For encrypting, decrypting, re-encrypting, and generating data keys, the +// KeyUsage must be ENCRYPT_DECRYPT. For signing and verifying, the KeyUsage +// must be SIGN_VERIFY. To find the KeyUsage of a CMK, use the DescribeKey operation. +// +// To find the encryption or signing algorithms supported for a particular CMK, +// use the DescribeKey operation. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/GetPublicKey +func (c *KMS) GetPublicKey(input *GetPublicKeyInput) (*GetPublicKeyOutput, error) { + req, out := c.GetPublicKeyRequest(input) + return out, req.Send() +} + +// GetPublicKeyWithContext is the same as GetPublicKey with the addition of +// the ability to pass a context and additional request options. +// +// See GetPublicKey for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) GetPublicKeyWithContext(ctx aws.Context, input *GetPublicKeyInput, opts ...request.Option) (*GetPublicKeyOutput, error) { + req, out := c.GetPublicKeyRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opImportKeyMaterial = "ImportKeyMaterial" + +// ImportKeyMaterialRequest generates a "aws/request.Request" representing the +// client's request for the ImportKeyMaterial operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See ImportKeyMaterial for more information on using the ImportKeyMaterial +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the ImportKeyMaterialRequest method. +// req, resp := client.ImportKeyMaterialRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/ImportKeyMaterial +func (c *KMS) ImportKeyMaterialRequest(input *ImportKeyMaterialInput) (req *request.Request, output *ImportKeyMaterialOutput) { + op := &request.Operation{ + Name: opImportKeyMaterial, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &ImportKeyMaterialInput{} + } + + output = &ImportKeyMaterialOutput{} + req = c.newRequest(op, input, output) + req.Handlers.Unmarshal.Swap(jsonrpc.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + return +} + +// ImportKeyMaterial API operation for AWS Key Management Service. +// +// Imports key material into an existing symmetric AWS KMS customer master key +// (CMK) that was created without key material. After you successfully import +// key material into a CMK, you can reimport the same key material (https://docs.aws.amazon.com/kms/latest/developerguide/importing-keys.html#reimport-key-material) +// into that CMK, but you cannot import different key material. +// +// You cannot perform this operation on an asymmetric CMK or on any CMK in a +// different AWS account. For more information about creating CMKs with no key +// material and then importing key material, see Importing Key Material (https://docs.aws.amazon.com/kms/latest/developerguide/importing-keys.html) +// in the AWS Key Management Service Developer Guide. +// +// Before using this operation, call GetParametersForImport. Its response includes +// a public key and an import token. Use the public key to encrypt the key material. +// Then, submit the import token from the same GetParametersForImport response. +// +// When calling this operation, you must specify the following values: +// +// * The key ID or key ARN of a CMK with no key material. Its Origin must +// be EXTERNAL. To create a CMK with no key material, call CreateKey and +// set the value of its Origin parameter to EXTERNAL. To get the Origin of +// a CMK, call DescribeKey.) +// +// * The encrypted key material. To get the public key to encrypt the key +// material, call GetParametersForImport. +// +// * The import token that GetParametersForImport returned. You must use +// a public key and token from the same GetParametersForImport response. +// +// * Whether the key material expires and if so, when. If you set an expiration +// date, AWS KMS deletes the key material from the CMK on the specified date, +// and the CMK becomes unusable. To use the CMK again, you must reimport +// the same key material. The only way to change an expiration date is by +// reimporting the same key material and specifying a new expiration date. +// +// When this operation is successful, the key state of the CMK changes from +// PendingImport to Enabled, and you can use the CMK. +// +// If this operation fails, use the exception to help determine the problem. +// If the error is related to the key material, the import token, or wrapping +// key, use GetParametersForImport to get a new public key and import token +// for the CMK and repeat the import procedure. For help, see How To Import +// Key Material (https://docs.aws.amazon.com/kms/latest/developerguide/importing-keys.html#importing-keys-overview) +// in the AWS Key Management Service Developer Guide. +// +// The CMK that you use for this operation must be in a compatible key state. +// For details, see How Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation ImportKeyMaterial for usage and error information. +// +// Returned Error Types: +// * InvalidArnException +// The request was rejected because a specified ARN, or an ARN in a key policy, +// is not valid. +// +// * UnsupportedOperationException +// The request was rejected because a specified parameter is not supported or +// a specified resource is not valid for this operation. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// * InvalidCiphertextException +// From the Decrypt or ReEncrypt operation, the request was rejected because +// the specified ciphertext, or additional authenticated data incorporated into +// the ciphertext, such as the encryption context, is corrupted, missing, or +// otherwise invalid. +// +// From the ImportKeyMaterial operation, the request was rejected because AWS +// KMS could not decrypt the encrypted (wrapped) key material. +// +// * IncorrectKeyMaterialException +// The request was rejected because the key material in the request is, expired, +// invalid, or is not the same key material that was previously imported into +// this customer master key (CMK). +// +// * ExpiredImportTokenException +// The request was rejected because the specified import token is expired. Use +// GetParametersForImport to get a new import token and public key, use the +// new public key to encrypt the key material, and then try the request again. +// +// * InvalidImportTokenException +// The request was rejected because the provided import token is invalid or +// is associated with a different customer master key (CMK). +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/ImportKeyMaterial +func (c *KMS) ImportKeyMaterial(input *ImportKeyMaterialInput) (*ImportKeyMaterialOutput, error) { + req, out := c.ImportKeyMaterialRequest(input) + return out, req.Send() +} + +// ImportKeyMaterialWithContext is the same as ImportKeyMaterial with the addition of +// the ability to pass a context and additional request options. +// +// See ImportKeyMaterial for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) ImportKeyMaterialWithContext(ctx aws.Context, input *ImportKeyMaterialInput, opts ...request.Option) (*ImportKeyMaterialOutput, error) { + req, out := c.ImportKeyMaterialRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opListAliases = "ListAliases" + +// ListAliasesRequest generates a "aws/request.Request" representing the +// client's request for the ListAliases operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See ListAliases for more information on using the ListAliases +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the ListAliasesRequest method. +// req, resp := client.ListAliasesRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/ListAliases +func (c *KMS) ListAliasesRequest(input *ListAliasesInput) (req *request.Request, output *ListAliasesOutput) { + op := &request.Operation{ + Name: opListAliases, + HTTPMethod: "POST", + HTTPPath: "/", + Paginator: &request.Paginator{ + InputTokens: []string{"Marker"}, + OutputTokens: []string{"NextMarker"}, + LimitToken: "Limit", + TruncationToken: "Truncated", + }, + } + + if input == nil { + input = &ListAliasesInput{} + } + + output = &ListAliasesOutput{} + req = c.newRequest(op, input, output) + return +} + +// ListAliases API operation for AWS Key Management Service. +// +// Gets a list of aliases in the caller's AWS account and region. You cannot +// list aliases in other accounts. For more information about aliases, see CreateAlias. +// +// By default, the ListAliases command returns all aliases in the account and +// region. To get only the aliases that point to a particular customer master +// key (CMK), use the KeyId parameter. +// +// The ListAliases response can include aliases that you created and associated +// with your customer managed CMKs, and aliases that AWS created and associated +// with AWS managed CMKs in your account. You can recognize AWS aliases because +// their names have the format aws/, such as aws/dynamodb. +// +// The response might also include aliases that have no TargetKeyId field. These +// are predefined aliases that AWS has created but has not yet associated with +// a CMK. Aliases that AWS creates in your account, including predefined aliases, +// do not count against your AWS KMS aliases quota (https://docs.aws.amazon.com/kms/latest/developerguide/limits.html#aliases-limit). +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation ListAliases for usage and error information. +// +// Returned Error Types: +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InvalidMarkerException +// The request was rejected because the marker that specifies where pagination +// should next begin is not valid. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidArnException +// The request was rejected because a specified ARN, or an ARN in a key policy, +// is not valid. +// +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/ListAliases +func (c *KMS) ListAliases(input *ListAliasesInput) (*ListAliasesOutput, error) { + req, out := c.ListAliasesRequest(input) + return out, req.Send() +} + +// ListAliasesWithContext is the same as ListAliases with the addition of +// the ability to pass a context and additional request options. +// +// See ListAliases for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) ListAliasesWithContext(ctx aws.Context, input *ListAliasesInput, opts ...request.Option) (*ListAliasesOutput, error) { + req, out := c.ListAliasesRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +// ListAliasesPages iterates over the pages of a ListAliases operation, +// calling the "fn" function with the response data for each page. To stop +// iterating, return false from the fn function. +// +// See ListAliases method for more information on how to use this operation. +// +// Note: This operation can generate multiple requests to a service. +// +// // Example iterating over at most 3 pages of a ListAliases operation. +// pageNum := 0 +// err := client.ListAliasesPages(params, +// func(page *kms.ListAliasesOutput, lastPage bool) bool { +// pageNum++ +// fmt.Println(page) +// return pageNum <= 3 +// }) +// +func (c *KMS) ListAliasesPages(input *ListAliasesInput, fn func(*ListAliasesOutput, bool) bool) error { + return c.ListAliasesPagesWithContext(aws.BackgroundContext(), input, fn) +} + +// ListAliasesPagesWithContext same as ListAliasesPages except +// it takes a Context and allows setting request options on the pages. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) ListAliasesPagesWithContext(ctx aws.Context, input *ListAliasesInput, fn func(*ListAliasesOutput, bool) bool, opts ...request.Option) error { + p := request.Pagination{ + NewRequest: func() (*request.Request, error) { + var inCpy *ListAliasesInput + if input != nil { + tmp := *input + inCpy = &tmp + } + req, _ := c.ListAliasesRequest(inCpy) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return req, nil + }, + } + + for p.Next() { + if !fn(p.Page().(*ListAliasesOutput), !p.HasNextPage()) { + break + } + } + + return p.Err() +} + +const opListGrants = "ListGrants" + +// ListGrantsRequest generates a "aws/request.Request" representing the +// client's request for the ListGrants operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See ListGrants for more information on using the ListGrants +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the ListGrantsRequest method. +// req, resp := client.ListGrantsRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/ListGrants +func (c *KMS) ListGrantsRequest(input *ListGrantsInput) (req *request.Request, output *ListGrantsResponse) { + op := &request.Operation{ + Name: opListGrants, + HTTPMethod: "POST", + HTTPPath: "/", + Paginator: &request.Paginator{ + InputTokens: []string{"Marker"}, + OutputTokens: []string{"NextMarker"}, + LimitToken: "Limit", + TruncationToken: "Truncated", + }, + } + + if input == nil { + input = &ListGrantsInput{} + } + + output = &ListGrantsResponse{} + req = c.newRequest(op, input, output) + return +} + +// ListGrants API operation for AWS Key Management Service. +// +// Gets a list of all grants for the specified customer master key (CMK). +// +// To perform this operation on a CMK in a different AWS account, specify the +// key ARN in the value of the KeyId parameter. +// +// The GranteePrincipal field in the ListGrants response usually contains the +// user or role designated as the grantee principal in the grant. However, when +// the grantee principal in the grant is an AWS service, the GranteePrincipal +// field contains the service principal (https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_principal.html#principal-services), +// which might represent several different grantee principals. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation ListGrants for usage and error information. +// +// Returned Error Types: +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InvalidMarkerException +// The request was rejected because the marker that specifies where pagination +// should next begin is not valid. +// +// * InvalidArnException +// The request was rejected because a specified ARN, or an ARN in a key policy, +// is not valid. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/ListGrants +func (c *KMS) ListGrants(input *ListGrantsInput) (*ListGrantsResponse, error) { + req, out := c.ListGrantsRequest(input) + return out, req.Send() +} + +// ListGrantsWithContext is the same as ListGrants with the addition of +// the ability to pass a context and additional request options. +// +// See ListGrants for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) ListGrantsWithContext(ctx aws.Context, input *ListGrantsInput, opts ...request.Option) (*ListGrantsResponse, error) { + req, out := c.ListGrantsRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +// ListGrantsPages iterates over the pages of a ListGrants operation, +// calling the "fn" function with the response data for each page. To stop +// iterating, return false from the fn function. +// +// See ListGrants method for more information on how to use this operation. +// +// Note: This operation can generate multiple requests to a service. +// +// // Example iterating over at most 3 pages of a ListGrants operation. +// pageNum := 0 +// err := client.ListGrantsPages(params, +// func(page *kms.ListGrantsResponse, lastPage bool) bool { +// pageNum++ +// fmt.Println(page) +// return pageNum <= 3 +// }) +// +func (c *KMS) ListGrantsPages(input *ListGrantsInput, fn func(*ListGrantsResponse, bool) bool) error { + return c.ListGrantsPagesWithContext(aws.BackgroundContext(), input, fn) +} + +// ListGrantsPagesWithContext same as ListGrantsPages except +// it takes a Context and allows setting request options on the pages. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) ListGrantsPagesWithContext(ctx aws.Context, input *ListGrantsInput, fn func(*ListGrantsResponse, bool) bool, opts ...request.Option) error { + p := request.Pagination{ + NewRequest: func() (*request.Request, error) { + var inCpy *ListGrantsInput + if input != nil { + tmp := *input + inCpy = &tmp + } + req, _ := c.ListGrantsRequest(inCpy) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return req, nil + }, + } + + for p.Next() { + if !fn(p.Page().(*ListGrantsResponse), !p.HasNextPage()) { + break + } + } + + return p.Err() +} + +const opListKeyPolicies = "ListKeyPolicies" + +// ListKeyPoliciesRequest generates a "aws/request.Request" representing the +// client's request for the ListKeyPolicies operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See ListKeyPolicies for more information on using the ListKeyPolicies +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the ListKeyPoliciesRequest method. +// req, resp := client.ListKeyPoliciesRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/ListKeyPolicies +func (c *KMS) ListKeyPoliciesRequest(input *ListKeyPoliciesInput) (req *request.Request, output *ListKeyPoliciesOutput) { + op := &request.Operation{ + Name: opListKeyPolicies, + HTTPMethod: "POST", + HTTPPath: "/", + Paginator: &request.Paginator{ + InputTokens: []string{"Marker"}, + OutputTokens: []string{"NextMarker"}, + LimitToken: "Limit", + TruncationToken: "Truncated", + }, + } + + if input == nil { + input = &ListKeyPoliciesInput{} + } + + output = &ListKeyPoliciesOutput{} + req = c.newRequest(op, input, output) + return +} + +// ListKeyPolicies API operation for AWS Key Management Service. +// +// Gets the names of the key policies that are attached to a customer master +// key (CMK). This operation is designed to get policy names that you can use +// in a GetKeyPolicy operation. However, the only valid policy name is default. +// You cannot perform this operation on a CMK in a different AWS account. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation ListKeyPolicies for usage and error information. +// +// Returned Error Types: +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * InvalidArnException +// The request was rejected because a specified ARN, or an ARN in a key policy, +// is not valid. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/ListKeyPolicies +func (c *KMS) ListKeyPolicies(input *ListKeyPoliciesInput) (*ListKeyPoliciesOutput, error) { + req, out := c.ListKeyPoliciesRequest(input) + return out, req.Send() +} + +// ListKeyPoliciesWithContext is the same as ListKeyPolicies with the addition of +// the ability to pass a context and additional request options. +// +// See ListKeyPolicies for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) ListKeyPoliciesWithContext(ctx aws.Context, input *ListKeyPoliciesInput, opts ...request.Option) (*ListKeyPoliciesOutput, error) { + req, out := c.ListKeyPoliciesRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +// ListKeyPoliciesPages iterates over the pages of a ListKeyPolicies operation, +// calling the "fn" function with the response data for each page. To stop +// iterating, return false from the fn function. +// +// See ListKeyPolicies method for more information on how to use this operation. +// +// Note: This operation can generate multiple requests to a service. +// +// // Example iterating over at most 3 pages of a ListKeyPolicies operation. +// pageNum := 0 +// err := client.ListKeyPoliciesPages(params, +// func(page *kms.ListKeyPoliciesOutput, lastPage bool) bool { +// pageNum++ +// fmt.Println(page) +// return pageNum <= 3 +// }) +// +func (c *KMS) ListKeyPoliciesPages(input *ListKeyPoliciesInput, fn func(*ListKeyPoliciesOutput, bool) bool) error { + return c.ListKeyPoliciesPagesWithContext(aws.BackgroundContext(), input, fn) +} + +// ListKeyPoliciesPagesWithContext same as ListKeyPoliciesPages except +// it takes a Context and allows setting request options on the pages. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) ListKeyPoliciesPagesWithContext(ctx aws.Context, input *ListKeyPoliciesInput, fn func(*ListKeyPoliciesOutput, bool) bool, opts ...request.Option) error { + p := request.Pagination{ + NewRequest: func() (*request.Request, error) { + var inCpy *ListKeyPoliciesInput + if input != nil { + tmp := *input + inCpy = &tmp + } + req, _ := c.ListKeyPoliciesRequest(inCpy) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return req, nil + }, + } + + for p.Next() { + if !fn(p.Page().(*ListKeyPoliciesOutput), !p.HasNextPage()) { + break + } + } + + return p.Err() +} + +const opListKeys = "ListKeys" + +// ListKeysRequest generates a "aws/request.Request" representing the +// client's request for the ListKeys operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See ListKeys for more information on using the ListKeys +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the ListKeysRequest method. +// req, resp := client.ListKeysRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/ListKeys +func (c *KMS) ListKeysRequest(input *ListKeysInput) (req *request.Request, output *ListKeysOutput) { + op := &request.Operation{ + Name: opListKeys, + HTTPMethod: "POST", + HTTPPath: "/", + Paginator: &request.Paginator{ + InputTokens: []string{"Marker"}, + OutputTokens: []string{"NextMarker"}, + LimitToken: "Limit", + TruncationToken: "Truncated", + }, + } + + if input == nil { + input = &ListKeysInput{} + } + + output = &ListKeysOutput{} + req = c.newRequest(op, input, output) + return +} + +// ListKeys API operation for AWS Key Management Service. +// +// Gets a list of all customer master keys (CMKs) in the caller's AWS account +// and Region. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation ListKeys for usage and error information. +// +// Returned Error Types: +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidMarkerException +// The request was rejected because the marker that specifies where pagination +// should next begin is not valid. +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/ListKeys +func (c *KMS) ListKeys(input *ListKeysInput) (*ListKeysOutput, error) { + req, out := c.ListKeysRequest(input) + return out, req.Send() +} + +// ListKeysWithContext is the same as ListKeys with the addition of +// the ability to pass a context and additional request options. +// +// See ListKeys for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) ListKeysWithContext(ctx aws.Context, input *ListKeysInput, opts ...request.Option) (*ListKeysOutput, error) { + req, out := c.ListKeysRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +// ListKeysPages iterates over the pages of a ListKeys operation, +// calling the "fn" function with the response data for each page. To stop +// iterating, return false from the fn function. +// +// See ListKeys method for more information on how to use this operation. +// +// Note: This operation can generate multiple requests to a service. +// +// // Example iterating over at most 3 pages of a ListKeys operation. +// pageNum := 0 +// err := client.ListKeysPages(params, +// func(page *kms.ListKeysOutput, lastPage bool) bool { +// pageNum++ +// fmt.Println(page) +// return pageNum <= 3 +// }) +// +func (c *KMS) ListKeysPages(input *ListKeysInput, fn func(*ListKeysOutput, bool) bool) error { + return c.ListKeysPagesWithContext(aws.BackgroundContext(), input, fn) +} + +// ListKeysPagesWithContext same as ListKeysPages except +// it takes a Context and allows setting request options on the pages. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) ListKeysPagesWithContext(ctx aws.Context, input *ListKeysInput, fn func(*ListKeysOutput, bool) bool, opts ...request.Option) error { + p := request.Pagination{ + NewRequest: func() (*request.Request, error) { + var inCpy *ListKeysInput + if input != nil { + tmp := *input + inCpy = &tmp + } + req, _ := c.ListKeysRequest(inCpy) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return req, nil + }, + } + + for p.Next() { + if !fn(p.Page().(*ListKeysOutput), !p.HasNextPage()) { + break + } + } + + return p.Err() +} + +const opListResourceTags = "ListResourceTags" + +// ListResourceTagsRequest generates a "aws/request.Request" representing the +// client's request for the ListResourceTags operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See ListResourceTags for more information on using the ListResourceTags +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the ListResourceTagsRequest method. +// req, resp := client.ListResourceTagsRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/ListResourceTags +func (c *KMS) ListResourceTagsRequest(input *ListResourceTagsInput) (req *request.Request, output *ListResourceTagsOutput) { + op := &request.Operation{ + Name: opListResourceTags, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &ListResourceTagsInput{} + } + + output = &ListResourceTagsOutput{} + req = c.newRequest(op, input, output) + return +} + +// ListResourceTags API operation for AWS Key Management Service. +// +// Returns a list of all tags for the specified customer master key (CMK). +// +// You cannot perform this operation on a CMK in a different AWS account. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation ListResourceTags for usage and error information. +// +// Returned Error Types: +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * InvalidArnException +// The request was rejected because a specified ARN, or an ARN in a key policy, +// is not valid. +// +// * InvalidMarkerException +// The request was rejected because the marker that specifies where pagination +// should next begin is not valid. +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/ListResourceTags +func (c *KMS) ListResourceTags(input *ListResourceTagsInput) (*ListResourceTagsOutput, error) { + req, out := c.ListResourceTagsRequest(input) + return out, req.Send() +} + +// ListResourceTagsWithContext is the same as ListResourceTags with the addition of +// the ability to pass a context and additional request options. +// +// See ListResourceTags for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) ListResourceTagsWithContext(ctx aws.Context, input *ListResourceTagsInput, opts ...request.Option) (*ListResourceTagsOutput, error) { + req, out := c.ListResourceTagsRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opListRetirableGrants = "ListRetirableGrants" + +// ListRetirableGrantsRequest generates a "aws/request.Request" representing the +// client's request for the ListRetirableGrants operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See ListRetirableGrants for more information on using the ListRetirableGrants +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the ListRetirableGrantsRequest method. +// req, resp := client.ListRetirableGrantsRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/ListRetirableGrants +func (c *KMS) ListRetirableGrantsRequest(input *ListRetirableGrantsInput) (req *request.Request, output *ListGrantsResponse) { + op := &request.Operation{ + Name: opListRetirableGrants, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &ListRetirableGrantsInput{} + } + + output = &ListGrantsResponse{} + req = c.newRequest(op, input, output) + return +} + +// ListRetirableGrants API operation for AWS Key Management Service. +// +// Returns a list of all grants for which the grant's RetiringPrincipal matches +// the one specified. +// +// A typical use is to list all grants that you are able to retire. To retire +// a grant, use RetireGrant. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation ListRetirableGrants for usage and error information. +// +// Returned Error Types: +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InvalidMarkerException +// The request was rejected because the marker that specifies where pagination +// should next begin is not valid. +// +// * InvalidArnException +// The request was rejected because a specified ARN, or an ARN in a key policy, +// is not valid. +// +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/ListRetirableGrants +func (c *KMS) ListRetirableGrants(input *ListRetirableGrantsInput) (*ListGrantsResponse, error) { + req, out := c.ListRetirableGrantsRequest(input) + return out, req.Send() +} + +// ListRetirableGrantsWithContext is the same as ListRetirableGrants with the addition of +// the ability to pass a context and additional request options. +// +// See ListRetirableGrants for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) ListRetirableGrantsWithContext(ctx aws.Context, input *ListRetirableGrantsInput, opts ...request.Option) (*ListGrantsResponse, error) { + req, out := c.ListRetirableGrantsRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opPutKeyPolicy = "PutKeyPolicy" + +// PutKeyPolicyRequest generates a "aws/request.Request" representing the +// client's request for the PutKeyPolicy operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See PutKeyPolicy for more information on using the PutKeyPolicy +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the PutKeyPolicyRequest method. +// req, resp := client.PutKeyPolicyRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/PutKeyPolicy +func (c *KMS) PutKeyPolicyRequest(input *PutKeyPolicyInput) (req *request.Request, output *PutKeyPolicyOutput) { + op := &request.Operation{ + Name: opPutKeyPolicy, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &PutKeyPolicyInput{} + } + + output = &PutKeyPolicyOutput{} + req = c.newRequest(op, input, output) + req.Handlers.Unmarshal.Swap(jsonrpc.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + return +} + +// PutKeyPolicy API operation for AWS Key Management Service. +// +// Attaches a key policy to the specified customer master key (CMK). You cannot +// perform this operation on a CMK in a different AWS account. +// +// For more information about key policies, see Key Policies (https://docs.aws.amazon.com/kms/latest/developerguide/key-policies.html) +// in the AWS Key Management Service Developer Guide. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation PutKeyPolicy for usage and error information. +// +// Returned Error Types: +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * InvalidArnException +// The request was rejected because a specified ARN, or an ARN in a key policy, +// is not valid. +// +// * MalformedPolicyDocumentException +// The request was rejected because the specified policy is not syntactically +// or semantically correct. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * UnsupportedOperationException +// The request was rejected because a specified parameter is not supported or +// a specified resource is not valid for this operation. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * LimitExceededException +// The request was rejected because a quota was exceeded. For more information, +// see Quotas (https://docs.aws.amazon.com/kms/latest/developerguide/limits.html) +// in the AWS Key Management Service Developer Guide. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/PutKeyPolicy +func (c *KMS) PutKeyPolicy(input *PutKeyPolicyInput) (*PutKeyPolicyOutput, error) { + req, out := c.PutKeyPolicyRequest(input) + return out, req.Send() +} + +// PutKeyPolicyWithContext is the same as PutKeyPolicy with the addition of +// the ability to pass a context and additional request options. +// +// See PutKeyPolicy for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) PutKeyPolicyWithContext(ctx aws.Context, input *PutKeyPolicyInput, opts ...request.Option) (*PutKeyPolicyOutput, error) { + req, out := c.PutKeyPolicyRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opReEncrypt = "ReEncrypt" + +// ReEncryptRequest generates a "aws/request.Request" representing the +// client's request for the ReEncrypt operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See ReEncrypt for more information on using the ReEncrypt +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the ReEncryptRequest method. +// req, resp := client.ReEncryptRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/ReEncrypt +func (c *KMS) ReEncryptRequest(input *ReEncryptInput) (req *request.Request, output *ReEncryptOutput) { + op := &request.Operation{ + Name: opReEncrypt, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &ReEncryptInput{} + } + + output = &ReEncryptOutput{} + req = c.newRequest(op, input, output) + return +} + +// ReEncrypt API operation for AWS Key Management Service. +// +// Decrypts ciphertext and then reencrypts it entirely within AWS KMS. You can +// use this operation to change the customer master key (CMK) under which data +// is encrypted, such as when you manually rotate (https://docs.aws.amazon.com/kms/latest/developerguide/rotate-keys.html#rotate-keys-manually) +// a CMK or change the CMK that protects a ciphertext. You can also use it to +// reencrypt ciphertext under the same CMK, such as to change the encryption +// context (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#encrypt_context) +// of a ciphertext. +// +// The ReEncrypt operation can decrypt ciphertext that was encrypted by using +// an AWS KMS CMK in an AWS KMS operation, such as Encrypt or GenerateDataKey. +// It can also decrypt ciphertext that was encrypted by using the public key +// of an asymmetric CMK (https://docs.aws.amazon.com/kms/latest/developerguide/symm-asymm-concepts.html#asymmetric-cmks) +// outside of AWS KMS. However, it cannot decrypt ciphertext produced by other +// libraries, such as the AWS Encryption SDK (https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/) +// or Amazon S3 client-side encryption (https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingClientSideEncryption.html). +// These libraries return a ciphertext format that is incompatible with AWS +// KMS. +// +// When you use the ReEncrypt operation, you need to provide information for +// the decrypt operation and the subsequent encrypt operation. +// +// * If your ciphertext was encrypted under an asymmetric CMK, you must identify +// the source CMK, that is, the CMK that encrypted the ciphertext. You must +// also supply the encryption algorithm that was used. This information is +// required to decrypt the data. +// +// * It is optional, but you can specify a source CMK even when the ciphertext +// was encrypted under a symmetric CMK. This ensures that the ciphertext +// is decrypted only by using a particular CMK. If the CMK that you specify +// cannot decrypt the ciphertext, the ReEncrypt operation fails. +// +// * To reencrypt the data, you must specify the destination CMK, that is, +// the CMK that re-encrypts the data after it is decrypted. You can select +// a symmetric or asymmetric CMK. If the destination CMK is an asymmetric +// CMK, you must also provide the encryption algorithm. The algorithm that +// you choose must be compatible with the CMK. When you use an asymmetric +// CMK to encrypt or reencrypt data, be sure to record the CMK and encryption +// algorithm that you choose. You will be required to provide the same CMK +// and encryption algorithm when you decrypt the data. If the CMK and algorithm +// do not match the values used to encrypt the data, the decrypt operation +// fails. You are not required to supply the CMK ID and encryption algorithm +// when you decrypt with symmetric CMKs because AWS KMS stores this information +// in the ciphertext blob. AWS KMS cannot store metadata in ciphertext generated +// with asymmetric keys. The standard format for asymmetric key ciphertext +// does not include configurable fields. +// +// Unlike other AWS KMS API operations, ReEncrypt callers must have two permissions: +// +// * kms:ReEncryptFrom permission on the source CMK +// +// * kms:ReEncryptTo permission on the destination CMK +// +// To permit reencryption from or to a CMK, include the "kms:ReEncrypt*" permission +// in your key policy (https://docs.aws.amazon.com/kms/latest/developerguide/key-policies.html). +// This permission is automatically included in the key policy when you use +// the console to create a CMK. But you must include it manually when you create +// a CMK programmatically or when you use the PutKeyPolicy operation to set +// a key policy. +// +// The CMK that you use for this operation must be in a compatible key state. +// For details, see How Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation ReEncrypt for usage and error information. +// +// Returned Error Types: +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * DisabledException +// The request was rejected because the specified CMK is not enabled. +// +// * InvalidCiphertextException +// From the Decrypt or ReEncrypt operation, the request was rejected because +// the specified ciphertext, or additional authenticated data incorporated into +// the ciphertext, such as the encryption context, is corrupted, missing, or +// otherwise invalid. +// +// From the ImportKeyMaterial operation, the request was rejected because AWS +// KMS could not decrypt the encrypted (wrapped) key material. +// +// * KeyUnavailableException +// The request was rejected because the specified CMK was not available. You +// can retry the request. +// +// * IncorrectKeyException +// The request was rejected because the specified CMK cannot decrypt the data. +// The KeyId in a Decrypt request and the SourceKeyId in a ReEncrypt request +// must identify the same CMK that was used to encrypt the ciphertext. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InvalidKeyUsageException +// The request was rejected for one of the following reasons: +// +// * The KeyUsage value of the CMK is incompatible with the API operation. +// +// * The encryption algorithm or signing algorithm specified for the operation +// is incompatible with the type of key material in the CMK (CustomerMasterKeySpec). +// +// For encrypting, decrypting, re-encrypting, and generating data keys, the +// KeyUsage must be ENCRYPT_DECRYPT. For signing and verifying, the KeyUsage +// must be SIGN_VERIFY. To find the KeyUsage of a CMK, use the DescribeKey operation. +// +// To find the encryption or signing algorithms supported for a particular CMK, +// use the DescribeKey operation. +// +// * InvalidGrantTokenException +// The request was rejected because the specified grant token is not valid. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/ReEncrypt +func (c *KMS) ReEncrypt(input *ReEncryptInput) (*ReEncryptOutput, error) { + req, out := c.ReEncryptRequest(input) + return out, req.Send() +} + +// ReEncryptWithContext is the same as ReEncrypt with the addition of +// the ability to pass a context and additional request options. +// +// See ReEncrypt for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) ReEncryptWithContext(ctx aws.Context, input *ReEncryptInput, opts ...request.Option) (*ReEncryptOutput, error) { + req, out := c.ReEncryptRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opRetireGrant = "RetireGrant" + +// RetireGrantRequest generates a "aws/request.Request" representing the +// client's request for the RetireGrant operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See RetireGrant for more information on using the RetireGrant +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the RetireGrantRequest method. +// req, resp := client.RetireGrantRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/RetireGrant +func (c *KMS) RetireGrantRequest(input *RetireGrantInput) (req *request.Request, output *RetireGrantOutput) { + op := &request.Operation{ + Name: opRetireGrant, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &RetireGrantInput{} + } + + output = &RetireGrantOutput{} + req = c.newRequest(op, input, output) + req.Handlers.Unmarshal.Swap(jsonrpc.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + return +} + +// RetireGrant API operation for AWS Key Management Service. +// +// Retires a grant. To clean up, you can retire a grant when you're done using +// it. You should revoke a grant when you intend to actively deny operations +// that depend on it. The following are permitted to call this API: +// +// * The AWS account (root user) under which the grant was created +// +// * The RetiringPrincipal, if present in the grant +// +// * The GranteePrincipal, if RetireGrant is an operation specified in the +// grant +// +// You must identify the grant to retire by its grant token or by a combination +// of the grant ID and the Amazon Resource Name (ARN) of the customer master +// key (CMK). A grant token is a unique variable-length base64-encoded string. +// A grant ID is a 64 character unique identifier of a grant. The CreateGrant +// operation returns both. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation RetireGrant for usage and error information. +// +// Returned Error Types: +// * InvalidArnException +// The request was rejected because a specified ARN, or an ARN in a key policy, +// is not valid. +// +// * InvalidGrantTokenException +// The request was rejected because the specified grant token is not valid. +// +// * InvalidGrantIdException +// The request was rejected because the specified GrantId is not valid. +// +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/RetireGrant +func (c *KMS) RetireGrant(input *RetireGrantInput) (*RetireGrantOutput, error) { + req, out := c.RetireGrantRequest(input) + return out, req.Send() +} + +// RetireGrantWithContext is the same as RetireGrant with the addition of +// the ability to pass a context and additional request options. +// +// See RetireGrant for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) RetireGrantWithContext(ctx aws.Context, input *RetireGrantInput, opts ...request.Option) (*RetireGrantOutput, error) { + req, out := c.RetireGrantRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opRevokeGrant = "RevokeGrant" + +// RevokeGrantRequest generates a "aws/request.Request" representing the +// client's request for the RevokeGrant operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See RevokeGrant for more information on using the RevokeGrant +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the RevokeGrantRequest method. +// req, resp := client.RevokeGrantRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/RevokeGrant +func (c *KMS) RevokeGrantRequest(input *RevokeGrantInput) (req *request.Request, output *RevokeGrantOutput) { + op := &request.Operation{ + Name: opRevokeGrant, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &RevokeGrantInput{} + } + + output = &RevokeGrantOutput{} + req = c.newRequest(op, input, output) + req.Handlers.Unmarshal.Swap(jsonrpc.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + return +} + +// RevokeGrant API operation for AWS Key Management Service. +// +// Revokes the specified grant for the specified customer master key (CMK). +// You can revoke a grant to actively deny operations that depend on it. +// +// To perform this operation on a CMK in a different AWS account, specify the +// key ARN in the value of the KeyId parameter. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation RevokeGrant for usage and error information. +// +// Returned Error Types: +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InvalidArnException +// The request was rejected because a specified ARN, or an ARN in a key policy, +// is not valid. +// +// * InvalidGrantIdException +// The request was rejected because the specified GrantId is not valid. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/RevokeGrant +func (c *KMS) RevokeGrant(input *RevokeGrantInput) (*RevokeGrantOutput, error) { + req, out := c.RevokeGrantRequest(input) + return out, req.Send() +} + +// RevokeGrantWithContext is the same as RevokeGrant with the addition of +// the ability to pass a context and additional request options. +// +// See RevokeGrant for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) RevokeGrantWithContext(ctx aws.Context, input *RevokeGrantInput, opts ...request.Option) (*RevokeGrantOutput, error) { + req, out := c.RevokeGrantRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opScheduleKeyDeletion = "ScheduleKeyDeletion" + +// ScheduleKeyDeletionRequest generates a "aws/request.Request" representing the +// client's request for the ScheduleKeyDeletion operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See ScheduleKeyDeletion for more information on using the ScheduleKeyDeletion +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the ScheduleKeyDeletionRequest method. +// req, resp := client.ScheduleKeyDeletionRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/ScheduleKeyDeletion +func (c *KMS) ScheduleKeyDeletionRequest(input *ScheduleKeyDeletionInput) (req *request.Request, output *ScheduleKeyDeletionOutput) { + op := &request.Operation{ + Name: opScheduleKeyDeletion, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &ScheduleKeyDeletionInput{} + } + + output = &ScheduleKeyDeletionOutput{} + req = c.newRequest(op, input, output) + return +} + +// ScheduleKeyDeletion API operation for AWS Key Management Service. +// +// Schedules the deletion of a customer master key (CMK). You may provide a +// waiting period, specified in days, before deletion occurs. If you do not +// provide a waiting period, the default period of 30 days is used. When this +// operation is successful, the key state of the CMK changes to PendingDeletion. +// Before the waiting period ends, you can use CancelKeyDeletion to cancel the +// deletion of the CMK. After the waiting period ends, AWS KMS deletes the CMK +// and all AWS KMS data associated with it, including all aliases that refer +// to it. +// +// Deleting a CMK is a destructive and potentially dangerous operation. When +// a CMK is deleted, all data that was encrypted under the CMK is unrecoverable. +// To prevent the use of a CMK without deleting it, use DisableKey. +// +// If you schedule deletion of a CMK from a custom key store (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html), +// when the waiting period expires, ScheduleKeyDeletion deletes the CMK from +// AWS KMS. Then AWS KMS makes a best effort to delete the key material from +// the associated AWS CloudHSM cluster. However, you might need to manually +// delete the orphaned key material (https://docs.aws.amazon.com/kms/latest/developerguide/fix-keystore.html#fix-keystore-orphaned-key) +// from the cluster and its backups. +// +// You cannot perform this operation on a CMK in a different AWS account. +// +// For more information about scheduling a CMK for deletion, see Deleting Customer +// Master Keys (https://docs.aws.amazon.com/kms/latest/developerguide/deleting-keys.html) +// in the AWS Key Management Service Developer Guide. +// +// The CMK that you use for this operation must be in a compatible key state. +// For details, see How Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation ScheduleKeyDeletion for usage and error information. +// +// Returned Error Types: +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * InvalidArnException +// The request was rejected because a specified ARN, or an ARN in a key policy, +// is not valid. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/ScheduleKeyDeletion +func (c *KMS) ScheduleKeyDeletion(input *ScheduleKeyDeletionInput) (*ScheduleKeyDeletionOutput, error) { + req, out := c.ScheduleKeyDeletionRequest(input) + return out, req.Send() +} + +// ScheduleKeyDeletionWithContext is the same as ScheduleKeyDeletion with the addition of +// the ability to pass a context and additional request options. +// +// See ScheduleKeyDeletion for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) ScheduleKeyDeletionWithContext(ctx aws.Context, input *ScheduleKeyDeletionInput, opts ...request.Option) (*ScheduleKeyDeletionOutput, error) { + req, out := c.ScheduleKeyDeletionRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opSign = "Sign" + +// SignRequest generates a "aws/request.Request" representing the +// client's request for the Sign operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See Sign for more information on using the Sign +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the SignRequest method. +// req, resp := client.SignRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/Sign +func (c *KMS) SignRequest(input *SignInput) (req *request.Request, output *SignOutput) { + op := &request.Operation{ + Name: opSign, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &SignInput{} + } + + output = &SignOutput{} + req = c.newRequest(op, input, output) + return +} + +// Sign API operation for AWS Key Management Service. +// +// Creates a digital signature (https://en.wikipedia.org/wiki/Digital_signature) +// for a message or message digest by using the private key in an asymmetric +// CMK. To verify the signature, use the Verify operation, or use the public +// key in the same asymmetric CMK outside of AWS KMS. For information about +// symmetric and asymmetric CMKs, see Using Symmetric and Asymmetric CMKs (https://docs.aws.amazon.com/kms/latest/developerguide/symmetric-asymmetric.html) +// in the AWS Key Management Service Developer Guide. +// +// Digital signatures are generated and verified by using asymmetric key pair, +// such as an RSA or ECC pair that is represented by an asymmetric customer +// master key (CMK). The key owner (or an authorized user) uses their private +// key to sign a message. Anyone with the public key can verify that the message +// was signed with that particular private key and that the message hasn't changed +// since it was signed. +// +// To use the Sign operation, provide the following information: +// +// * Use the KeyId parameter to identify an asymmetric CMK with a KeyUsage +// value of SIGN_VERIFY. To get the KeyUsage value of a CMK, use the DescribeKey +// operation. The caller must have kms:Sign permission on the CMK. +// +// * Use the Message parameter to specify the message or message digest to +// sign. You can submit messages of up to 4096 bytes. To sign a larger message, +// generate a hash digest of the message, and then provide the hash digest +// in the Message parameter. To indicate whether the message is a full message +// or a digest, use the MessageType parameter. +// +// * Choose a signing algorithm that is compatible with the CMK. +// +// When signing a message, be sure to record the CMK and the signing algorithm. +// This information is required to verify the signature. +// +// To verify the signature that this operation generates, use the Verify operation. +// Or use the GetPublicKey operation to download the public key and then use +// the public key to verify the signature outside of AWS KMS. +// +// The CMK that you use for this operation must be in a compatible key state. +// For details, see How Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation Sign for usage and error information. +// +// Returned Error Types: +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * DisabledException +// The request was rejected because the specified CMK is not enabled. +// +// * KeyUnavailableException +// The request was rejected because the specified CMK was not available. You +// can retry the request. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InvalidKeyUsageException +// The request was rejected for one of the following reasons: +// +// * The KeyUsage value of the CMK is incompatible with the API operation. +// +// * The encryption algorithm or signing algorithm specified for the operation +// is incompatible with the type of key material in the CMK (CustomerMasterKeySpec). +// +// For encrypting, decrypting, re-encrypting, and generating data keys, the +// KeyUsage must be ENCRYPT_DECRYPT. For signing and verifying, the KeyUsage +// must be SIGN_VERIFY. To find the KeyUsage of a CMK, use the DescribeKey operation. +// +// To find the encryption or signing algorithms supported for a particular CMK, +// use the DescribeKey operation. +// +// * InvalidGrantTokenException +// The request was rejected because the specified grant token is not valid. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/Sign +func (c *KMS) Sign(input *SignInput) (*SignOutput, error) { + req, out := c.SignRequest(input) + return out, req.Send() +} + +// SignWithContext is the same as Sign with the addition of +// the ability to pass a context and additional request options. +// +// See Sign for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) SignWithContext(ctx aws.Context, input *SignInput, opts ...request.Option) (*SignOutput, error) { + req, out := c.SignRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opTagResource = "TagResource" + +// TagResourceRequest generates a "aws/request.Request" representing the +// client's request for the TagResource operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See TagResource for more information on using the TagResource +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the TagResourceRequest method. +// req, resp := client.TagResourceRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/TagResource +func (c *KMS) TagResourceRequest(input *TagResourceInput) (req *request.Request, output *TagResourceOutput) { + op := &request.Operation{ + Name: opTagResource, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &TagResourceInput{} + } + + output = &TagResourceOutput{} + req = c.newRequest(op, input, output) + req.Handlers.Unmarshal.Swap(jsonrpc.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + return +} + +// TagResource API operation for AWS Key Management Service. +// +// Adds or edits tags for a customer master key (CMK). You cannot perform this +// operation on a CMK in a different AWS account. +// +// Each tag consists of a tag key and a tag value. Tag keys and tag values are +// both required, but tag values can be empty (null) strings. +// +// You can only use a tag key once for each CMK. If you use the tag key again, +// AWS KMS replaces the current tag value with the specified value. +// +// For information about the rules that apply to tag keys and tag values, see +// User-Defined Tag Restrictions (https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/allocation-tag-restrictions.html) +// in the AWS Billing and Cost Management User Guide. +// +// The CMK that you use for this operation must be in a compatible key state. +// For details, see How Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation TagResource for usage and error information. +// +// Returned Error Types: +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * InvalidArnException +// The request was rejected because a specified ARN, or an ARN in a key policy, +// is not valid. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// * LimitExceededException +// The request was rejected because a quota was exceeded. For more information, +// see Quotas (https://docs.aws.amazon.com/kms/latest/developerguide/limits.html) +// in the AWS Key Management Service Developer Guide. +// +// * TagException +// The request was rejected because one or more tags are not valid. +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/TagResource +func (c *KMS) TagResource(input *TagResourceInput) (*TagResourceOutput, error) { + req, out := c.TagResourceRequest(input) + return out, req.Send() +} + +// TagResourceWithContext is the same as TagResource with the addition of +// the ability to pass a context and additional request options. +// +// See TagResource for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) TagResourceWithContext(ctx aws.Context, input *TagResourceInput, opts ...request.Option) (*TagResourceOutput, error) { + req, out := c.TagResourceRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opUntagResource = "UntagResource" + +// UntagResourceRequest generates a "aws/request.Request" representing the +// client's request for the UntagResource operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See UntagResource for more information on using the UntagResource +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the UntagResourceRequest method. +// req, resp := client.UntagResourceRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/UntagResource +func (c *KMS) UntagResourceRequest(input *UntagResourceInput) (req *request.Request, output *UntagResourceOutput) { + op := &request.Operation{ + Name: opUntagResource, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &UntagResourceInput{} + } + + output = &UntagResourceOutput{} + req = c.newRequest(op, input, output) + req.Handlers.Unmarshal.Swap(jsonrpc.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + return +} + +// UntagResource API operation for AWS Key Management Service. +// +// Removes the specified tags from the specified customer master key (CMK). +// You cannot perform this operation on a CMK in a different AWS account. +// +// To remove a tag, specify the tag key. To change the tag value of an existing +// tag key, use TagResource. +// +// The CMK that you use for this operation must be in a compatible key state. +// For details, see How Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation UntagResource for usage and error information. +// +// Returned Error Types: +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * InvalidArnException +// The request was rejected because a specified ARN, or an ARN in a key policy, +// is not valid. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// * TagException +// The request was rejected because one or more tags are not valid. +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/UntagResource +func (c *KMS) UntagResource(input *UntagResourceInput) (*UntagResourceOutput, error) { + req, out := c.UntagResourceRequest(input) + return out, req.Send() +} + +// UntagResourceWithContext is the same as UntagResource with the addition of +// the ability to pass a context and additional request options. +// +// See UntagResource for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) UntagResourceWithContext(ctx aws.Context, input *UntagResourceInput, opts ...request.Option) (*UntagResourceOutput, error) { + req, out := c.UntagResourceRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opUpdateAlias = "UpdateAlias" + +// UpdateAliasRequest generates a "aws/request.Request" representing the +// client's request for the UpdateAlias operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See UpdateAlias for more information on using the UpdateAlias +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the UpdateAliasRequest method. +// req, resp := client.UpdateAliasRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/UpdateAlias +func (c *KMS) UpdateAliasRequest(input *UpdateAliasInput) (req *request.Request, output *UpdateAliasOutput) { + op := &request.Operation{ + Name: opUpdateAlias, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &UpdateAliasInput{} + } + + output = &UpdateAliasOutput{} + req = c.newRequest(op, input, output) + req.Handlers.Unmarshal.Swap(jsonrpc.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + return +} + +// UpdateAlias API operation for AWS Key Management Service. +// +// Associates an existing AWS KMS alias with a different customer master key +// (CMK). Each alias is associated with only one CMK at a time, although a CMK +// can have multiple aliases. The alias and the CMK must be in the same AWS +// account and region. You cannot perform this operation on an alias in a different +// AWS account. +// +// The current and new CMK must be the same type (both symmetric or both asymmetric), +// and they must have the same key usage (ENCRYPT_DECRYPT or SIGN_VERIFY). This +// restriction prevents errors in code that uses aliases. If you must assign +// an alias to a different type of CMK, use DeleteAlias to delete the old alias +// and CreateAlias to create a new alias. +// +// You cannot use UpdateAlias to change an alias name. To change an alias name, +// use DeleteAlias to delete the old alias and CreateAlias to create a new alias. +// +// Because an alias is not a property of a CMK, you can create, update, and +// delete the aliases of a CMK without affecting the CMK. Also, aliases do not +// appear in the response from the DescribeKey operation. To get the aliases +// of all CMKs in the account, use the ListAliases operation. +// +// The CMK that you use for this operation must be in a compatible key state. +// For details, see How Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation UpdateAlias for usage and error information. +// +// Returned Error Types: +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * LimitExceededException +// The request was rejected because a quota was exceeded. For more information, +// see Quotas (https://docs.aws.amazon.com/kms/latest/developerguide/limits.html) +// in the AWS Key Management Service Developer Guide. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/UpdateAlias +func (c *KMS) UpdateAlias(input *UpdateAliasInput) (*UpdateAliasOutput, error) { + req, out := c.UpdateAliasRequest(input) + return out, req.Send() +} + +// UpdateAliasWithContext is the same as UpdateAlias with the addition of +// the ability to pass a context and additional request options. +// +// See UpdateAlias for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) UpdateAliasWithContext(ctx aws.Context, input *UpdateAliasInput, opts ...request.Option) (*UpdateAliasOutput, error) { + req, out := c.UpdateAliasRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opUpdateCustomKeyStore = "UpdateCustomKeyStore" + +// UpdateCustomKeyStoreRequest generates a "aws/request.Request" representing the +// client's request for the UpdateCustomKeyStore operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See UpdateCustomKeyStore for more information on using the UpdateCustomKeyStore +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the UpdateCustomKeyStoreRequest method. +// req, resp := client.UpdateCustomKeyStoreRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/UpdateCustomKeyStore +func (c *KMS) UpdateCustomKeyStoreRequest(input *UpdateCustomKeyStoreInput) (req *request.Request, output *UpdateCustomKeyStoreOutput) { + op := &request.Operation{ + Name: opUpdateCustomKeyStore, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &UpdateCustomKeyStoreInput{} + } + + output = &UpdateCustomKeyStoreOutput{} + req = c.newRequest(op, input, output) + req.Handlers.Unmarshal.Swap(jsonrpc.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + return +} + +// UpdateCustomKeyStore API operation for AWS Key Management Service. +// +// Changes the properties of a custom key store. Use the CustomKeyStoreId parameter +// to identify the custom key store you want to edit. Use the remaining parameters +// to change the properties of the custom key store. +// +// You can only update a custom key store that is disconnected. To disconnect +// the custom key store, use DisconnectCustomKeyStore. To reconnect the custom +// key store after the update completes, use ConnectCustomKeyStore. To find +// the connection state of a custom key store, use the DescribeCustomKeyStores +// operation. +// +// Use the parameters of UpdateCustomKeyStore to edit your keystore settings. +// +// * Use the NewCustomKeyStoreName parameter to change the friendly name +// of the custom key store to the value that you specify. +// +// * Use the KeyStorePassword parameter tell AWS KMS the current password +// of the kmsuser crypto user (CU) (https://docs.aws.amazon.com/kms/latest/developerguide/key-store-concepts.html#concept-kmsuser) +// in the associated AWS CloudHSM cluster. You can use this parameter to +// fix connection failures (https://docs.aws.amazon.com/kms/latest/developerguide/fix-keystore.html#fix-keystore-password) +// that occur when AWS KMS cannot log into the associated cluster because +// the kmsuser password has changed. This value does not change the password +// in the AWS CloudHSM cluster. +// +// * Use the CloudHsmClusterId parameter to associate the custom key store +// with a different, but related, AWS CloudHSM cluster. You can use this +// parameter to repair a custom key store if its AWS CloudHSM cluster becomes +// corrupted or is deleted, or when you need to create or restore a cluster +// from a backup. +// +// If the operation succeeds, it returns a JSON object with no properties. +// +// This operation is part of the Custom Key Store feature (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html) +// feature in AWS KMS, which combines the convenience and extensive integration +// of AWS KMS with the isolation and control of a single-tenant key store. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation UpdateCustomKeyStore for usage and error information. +// +// Returned Error Types: +// * CustomKeyStoreNotFoundException +// The request was rejected because AWS KMS cannot find a custom key store with +// the specified key store name or ID. +// +// * CustomKeyStoreNameInUseException +// The request was rejected because the specified custom key store name is already +// assigned to another custom key store in the account. Try again with a custom +// key store name that is unique in the account. +// +// * CloudHsmClusterNotFoundException +// The request was rejected because AWS KMS cannot find the AWS CloudHSM cluster +// with the specified cluster ID. Retry the request with a different cluster +// ID. +// +// * CloudHsmClusterNotRelatedException +// The request was rejected because the specified AWS CloudHSM cluster has a +// different cluster certificate than the original cluster. You cannot use the +// operation to specify an unrelated cluster. +// +// Specify a cluster that shares a backup history with the original cluster. +// This includes clusters that were created from a backup of the current cluster, +// and clusters that were created from the same backup that produced the current +// cluster. +// +// Clusters that share a backup history have the same cluster certificate. To +// view the cluster certificate of a cluster, use the DescribeClusters (https://docs.aws.amazon.com/cloudhsm/latest/APIReference/API_DescribeClusters.html) +// operation. +// +// * CustomKeyStoreInvalidStateException +// The request was rejected because of the ConnectionState of the custom key +// store. To get the ConnectionState of a custom key store, use the DescribeCustomKeyStores +// operation. +// +// This exception is thrown under the following conditions: +// +// * You requested the CreateKey or GenerateRandom operation in a custom +// key store that is not connected. These operations are valid only when +// the custom key store ConnectionState is CONNECTED. +// +// * You requested the UpdateCustomKeyStore or DeleteCustomKeyStore operation +// on a custom key store that is not disconnected. This operation is valid +// only when the custom key store ConnectionState is DISCONNECTED. +// +// * You requested the ConnectCustomKeyStore operation on a custom key store +// with a ConnectionState of DISCONNECTING or FAILED. This operation is valid +// for all other ConnectionState values. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * CloudHsmClusterNotActiveException +// The request was rejected because the AWS CloudHSM cluster that is associated +// with the custom key store is not active. Initialize and activate the cluster +// and try the command again. For detailed instructions, see Getting Started +// (https://docs.aws.amazon.com/cloudhsm/latest/userguide/getting-started.html) +// in the AWS CloudHSM User Guide. +// +// * CloudHsmClusterInvalidConfigurationException +// The request was rejected because the associated AWS CloudHSM cluster did +// not meet the configuration requirements for a custom key store. +// +// * The cluster must be configured with private subnets in at least two +// different Availability Zones in the Region. +// +// * The security group for the cluster (https://docs.aws.amazon.com/cloudhsm/latest/userguide/configure-sg.html) +// (cloudhsm-cluster--sg) must include inbound rules and outbound +// rules that allow TCP traffic on ports 2223-2225. The Source in the inbound +// rules and the Destination in the outbound rules must match the security +// group ID. These rules are set by default when you create the cluster. +// Do not delete or change them. To get information about a particular security +// group, use the DescribeSecurityGroups (https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeSecurityGroups.html) +// operation. +// +// * The cluster must contain at least as many HSMs as the operation requires. +// To add HSMs, use the AWS CloudHSM CreateHsm (https://docs.aws.amazon.com/cloudhsm/latest/APIReference/API_CreateHsm.html) +// operation. For the CreateCustomKeyStore, UpdateCustomKeyStore, and CreateKey +// operations, the AWS CloudHSM cluster must have at least two active HSMs, +// each in a different Availability Zone. For the ConnectCustomKeyStore operation, +// the AWS CloudHSM must contain at least one active HSM. +// +// For information about the requirements for an AWS CloudHSM cluster that is +// associated with a custom key store, see Assemble the Prerequisites (https://docs.aws.amazon.com/kms/latest/developerguide/create-keystore.html#before-keystore) +// in the AWS Key Management Service Developer Guide. For information about +// creating a private subnet for an AWS CloudHSM cluster, see Create a Private +// Subnet (https://docs.aws.amazon.com/cloudhsm/latest/userguide/create-subnets.html) +// in the AWS CloudHSM User Guide. For information about cluster security groups, +// see Configure a Default Security Group (https://docs.aws.amazon.com/cloudhsm/latest/userguide/configure-sg.html) +// in the AWS CloudHSM User Guide . +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/UpdateCustomKeyStore +func (c *KMS) UpdateCustomKeyStore(input *UpdateCustomKeyStoreInput) (*UpdateCustomKeyStoreOutput, error) { + req, out := c.UpdateCustomKeyStoreRequest(input) + return out, req.Send() +} + +// UpdateCustomKeyStoreWithContext is the same as UpdateCustomKeyStore with the addition of +// the ability to pass a context and additional request options. +// +// See UpdateCustomKeyStore for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) UpdateCustomKeyStoreWithContext(ctx aws.Context, input *UpdateCustomKeyStoreInput, opts ...request.Option) (*UpdateCustomKeyStoreOutput, error) { + req, out := c.UpdateCustomKeyStoreRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opUpdateKeyDescription = "UpdateKeyDescription" + +// UpdateKeyDescriptionRequest generates a "aws/request.Request" representing the +// client's request for the UpdateKeyDescription operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See UpdateKeyDescription for more information on using the UpdateKeyDescription +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the UpdateKeyDescriptionRequest method. +// req, resp := client.UpdateKeyDescriptionRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/UpdateKeyDescription +func (c *KMS) UpdateKeyDescriptionRequest(input *UpdateKeyDescriptionInput) (req *request.Request, output *UpdateKeyDescriptionOutput) { + op := &request.Operation{ + Name: opUpdateKeyDescription, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &UpdateKeyDescriptionInput{} + } + + output = &UpdateKeyDescriptionOutput{} + req = c.newRequest(op, input, output) + req.Handlers.Unmarshal.Swap(jsonrpc.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + return +} + +// UpdateKeyDescription API operation for AWS Key Management Service. +// +// Updates the description of a customer master key (CMK). To see the description +// of a CMK, use DescribeKey. +// +// You cannot perform this operation on a CMK in a different AWS account. +// +// The CMK that you use for this operation must be in a compatible key state. +// For details, see How Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation UpdateKeyDescription for usage and error information. +// +// Returned Error Types: +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * InvalidArnException +// The request was rejected because a specified ARN, or an ARN in a key policy, +// is not valid. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/UpdateKeyDescription +func (c *KMS) UpdateKeyDescription(input *UpdateKeyDescriptionInput) (*UpdateKeyDescriptionOutput, error) { + req, out := c.UpdateKeyDescriptionRequest(input) + return out, req.Send() +} + +// UpdateKeyDescriptionWithContext is the same as UpdateKeyDescription with the addition of +// the ability to pass a context and additional request options. +// +// See UpdateKeyDescription for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) UpdateKeyDescriptionWithContext(ctx aws.Context, input *UpdateKeyDescriptionInput, opts ...request.Option) (*UpdateKeyDescriptionOutput, error) { + req, out := c.UpdateKeyDescriptionRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +const opVerify = "Verify" + +// VerifyRequest generates a "aws/request.Request" representing the +// client's request for the Verify operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See Verify for more information on using the Verify +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the VerifyRequest method. +// req, resp := client.VerifyRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/Verify +func (c *KMS) VerifyRequest(input *VerifyInput) (req *request.Request, output *VerifyOutput) { + op := &request.Operation{ + Name: opVerify, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &VerifyInput{} + } + + output = &VerifyOutput{} + req = c.newRequest(op, input, output) + return +} + +// Verify API operation for AWS Key Management Service. +// +// Verifies a digital signature that was generated by the Sign operation. +// +// Verification confirms that an authorized user signed the message with the +// specified CMK and signing algorithm, and the message hasn't changed since +// it was signed. If the signature is verified, the value of the SignatureValid +// field in the response is True. If the signature verification fails, the Verify +// operation fails with an KMSInvalidSignatureException exception. +// +// A digital signature is generated by using the private key in an asymmetric +// CMK. The signature is verified by using the public key in the same asymmetric +// CMK. For information about symmetric and asymmetric CMKs, see Using Symmetric +// and Asymmetric CMKs (https://docs.aws.amazon.com/kms/latest/developerguide/symmetric-asymmetric.html) +// in the AWS Key Management Service Developer Guide. +// +// To verify a digital signature, you can use the Verify operation. Specify +// the same asymmetric CMK, message, and signing algorithm that were used to +// produce the signature. +// +// You can also verify the digital signature by using the public key of the +// CMK outside of AWS KMS. Use the GetPublicKey operation to download the public +// key in the asymmetric CMK and then use the public key to verify the signature +// outside of AWS KMS. The advantage of using the Verify operation is that it +// is performed within AWS KMS. As a result, it's easy to call, the operation +// is performed within the FIPS boundary, it is logged in AWS CloudTrail, and +// you can use key policy and IAM policy to determine who is authorized to use +// the CMK to verify signatures. +// +// The CMK that you use for this operation must be in a compatible key state. +// For details, see How Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Key Management Service's +// API operation Verify for usage and error information. +// +// Returned Error Types: +// * NotFoundException +// The request was rejected because the specified entity or resource could not +// be found. +// +// * DisabledException +// The request was rejected because the specified CMK is not enabled. +// +// * KeyUnavailableException +// The request was rejected because the specified CMK was not available. You +// can retry the request. +// +// * DependencyTimeoutException +// The system timed out while trying to fulfill the request. The request can +// be retried. +// +// * InvalidKeyUsageException +// The request was rejected for one of the following reasons: +// +// * The KeyUsage value of the CMK is incompatible with the API operation. +// +// * The encryption algorithm or signing algorithm specified for the operation +// is incompatible with the type of key material in the CMK (CustomerMasterKeySpec). +// +// For encrypting, decrypting, re-encrypting, and generating data keys, the +// KeyUsage must be ENCRYPT_DECRYPT. For signing and verifying, the KeyUsage +// must be SIGN_VERIFY. To find the KeyUsage of a CMK, use the DescribeKey operation. +// +// To find the encryption or signing algorithms supported for a particular CMK, +// use the DescribeKey operation. +// +// * InvalidGrantTokenException +// The request was rejected because the specified grant token is not valid. +// +// * InternalException +// The request was rejected because an internal exception occurred. The request +// can be retried. +// +// * InvalidStateException +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +// +// * KMSInvalidSignatureException +// The request was rejected because the signature verification failed. Signature +// verification fails when it cannot confirm that signature was produced by +// signing the specified message with the specified CMK and signing algorithm. +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01/Verify +func (c *KMS) Verify(input *VerifyInput) (*VerifyOutput, error) { + req, out := c.VerifyRequest(input) + return out, req.Send() +} + +// VerifyWithContext is the same as Verify with the addition of +// the ability to pass a context and additional request options. +// +// See Verify for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *KMS) VerifyWithContext(ctx aws.Context, input *VerifyInput, opts ...request.Option) (*VerifyOutput, error) { + req, out := c.VerifyRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + +// Contains information about an alias. +type AliasListEntry struct { + _ struct{} `type:"structure"` + + // String that contains the key ARN. + AliasArn *string `min:"20" type:"string"` + + // String that contains the alias. This value begins with alias/. + AliasName *string `min:"1" type:"string"` + + // String that contains the key identifier referred to by the alias. + TargetKeyId *string `min:"1" type:"string"` +} + +// String returns the string representation +func (s AliasListEntry) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s AliasListEntry) GoString() string { + return s.String() +} + +// SetAliasArn sets the AliasArn field's value. +func (s *AliasListEntry) SetAliasArn(v string) *AliasListEntry { + s.AliasArn = &v + return s +} + +// SetAliasName sets the AliasName field's value. +func (s *AliasListEntry) SetAliasName(v string) *AliasListEntry { + s.AliasName = &v + return s +} + +// SetTargetKeyId sets the TargetKeyId field's value. +func (s *AliasListEntry) SetTargetKeyId(v string) *AliasListEntry { + s.TargetKeyId = &v + return s +} + +// The request was rejected because it attempted to create a resource that already +// exists. +type AlreadyExistsException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s AlreadyExistsException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s AlreadyExistsException) GoString() string { + return s.String() +} + +func newErrorAlreadyExistsException(v protocol.ResponseMetadata) error { + return &AlreadyExistsException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *AlreadyExistsException) Code() string { + return "AlreadyExistsException" +} + +// Message returns the exception's message. +func (s *AlreadyExistsException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *AlreadyExistsException) OrigErr() error { + return nil +} + +func (s *AlreadyExistsException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *AlreadyExistsException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *AlreadyExistsException) RequestID() string { + return s.RespMetadata.RequestID +} + +type CancelKeyDeletionInput struct { + _ struct{} `type:"structure"` + + // The unique identifier for the customer master key (CMK) for which to cancel + // deletion. + // + // Specify the key ID or the Amazon Resource Name (ARN) of the CMK. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` +} + +// String returns the string representation +func (s CancelKeyDeletionInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s CancelKeyDeletionInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *CancelKeyDeletionInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "CancelKeyDeletionInput"} + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetKeyId sets the KeyId field's value. +func (s *CancelKeyDeletionInput) SetKeyId(v string) *CancelKeyDeletionInput { + s.KeyId = &v + return s +} + +type CancelKeyDeletionOutput struct { + _ struct{} `type:"structure"` + + // The Amazon Resource Name (key ARN (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#key-id-key-ARN)) + // of the CMK whose deletion is canceled. + KeyId *string `min:"1" type:"string"` +} + +// String returns the string representation +func (s CancelKeyDeletionOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s CancelKeyDeletionOutput) GoString() string { + return s.String() +} + +// SetKeyId sets the KeyId field's value. +func (s *CancelKeyDeletionOutput) SetKeyId(v string) *CancelKeyDeletionOutput { + s.KeyId = &v + return s +} + +// The request was rejected because the specified AWS CloudHSM cluster is already +// associated with a custom key store or it shares a backup history with a cluster +// that is associated with a custom key store. Each custom key store must be +// associated with a different AWS CloudHSM cluster. +// +// Clusters that share a backup history have the same cluster certificate. To +// view the cluster certificate of a cluster, use the DescribeClusters (https://docs.aws.amazon.com/cloudhsm/latest/APIReference/API_DescribeClusters.html) +// operation. +type CloudHsmClusterInUseException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s CloudHsmClusterInUseException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s CloudHsmClusterInUseException) GoString() string { + return s.String() +} + +func newErrorCloudHsmClusterInUseException(v protocol.ResponseMetadata) error { + return &CloudHsmClusterInUseException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *CloudHsmClusterInUseException) Code() string { + return "CloudHsmClusterInUseException" +} + +// Message returns the exception's message. +func (s *CloudHsmClusterInUseException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *CloudHsmClusterInUseException) OrigErr() error { + return nil +} + +func (s *CloudHsmClusterInUseException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *CloudHsmClusterInUseException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *CloudHsmClusterInUseException) RequestID() string { + return s.RespMetadata.RequestID +} + +// The request was rejected because the associated AWS CloudHSM cluster did +// not meet the configuration requirements for a custom key store. +// +// * The cluster must be configured with private subnets in at least two +// different Availability Zones in the Region. +// +// * The security group for the cluster (https://docs.aws.amazon.com/cloudhsm/latest/userguide/configure-sg.html) +// (cloudhsm-cluster--sg) must include inbound rules and outbound +// rules that allow TCP traffic on ports 2223-2225. The Source in the inbound +// rules and the Destination in the outbound rules must match the security +// group ID. These rules are set by default when you create the cluster. +// Do not delete or change them. To get information about a particular security +// group, use the DescribeSecurityGroups (https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeSecurityGroups.html) +// operation. +// +// * The cluster must contain at least as many HSMs as the operation requires. +// To add HSMs, use the AWS CloudHSM CreateHsm (https://docs.aws.amazon.com/cloudhsm/latest/APIReference/API_CreateHsm.html) +// operation. For the CreateCustomKeyStore, UpdateCustomKeyStore, and CreateKey +// operations, the AWS CloudHSM cluster must have at least two active HSMs, +// each in a different Availability Zone. For the ConnectCustomKeyStore operation, +// the AWS CloudHSM must contain at least one active HSM. +// +// For information about the requirements for an AWS CloudHSM cluster that is +// associated with a custom key store, see Assemble the Prerequisites (https://docs.aws.amazon.com/kms/latest/developerguide/create-keystore.html#before-keystore) +// in the AWS Key Management Service Developer Guide. For information about +// creating a private subnet for an AWS CloudHSM cluster, see Create a Private +// Subnet (https://docs.aws.amazon.com/cloudhsm/latest/userguide/create-subnets.html) +// in the AWS CloudHSM User Guide. For information about cluster security groups, +// see Configure a Default Security Group (https://docs.aws.amazon.com/cloudhsm/latest/userguide/configure-sg.html) +// in the AWS CloudHSM User Guide . +type CloudHsmClusterInvalidConfigurationException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s CloudHsmClusterInvalidConfigurationException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s CloudHsmClusterInvalidConfigurationException) GoString() string { + return s.String() +} + +func newErrorCloudHsmClusterInvalidConfigurationException(v protocol.ResponseMetadata) error { + return &CloudHsmClusterInvalidConfigurationException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *CloudHsmClusterInvalidConfigurationException) Code() string { + return "CloudHsmClusterInvalidConfigurationException" +} + +// Message returns the exception's message. +func (s *CloudHsmClusterInvalidConfigurationException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *CloudHsmClusterInvalidConfigurationException) OrigErr() error { + return nil +} + +func (s *CloudHsmClusterInvalidConfigurationException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *CloudHsmClusterInvalidConfigurationException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *CloudHsmClusterInvalidConfigurationException) RequestID() string { + return s.RespMetadata.RequestID +} + +// The request was rejected because the AWS CloudHSM cluster that is associated +// with the custom key store is not active. Initialize and activate the cluster +// and try the command again. For detailed instructions, see Getting Started +// (https://docs.aws.amazon.com/cloudhsm/latest/userguide/getting-started.html) +// in the AWS CloudHSM User Guide. +type CloudHsmClusterNotActiveException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s CloudHsmClusterNotActiveException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s CloudHsmClusterNotActiveException) GoString() string { + return s.String() +} + +func newErrorCloudHsmClusterNotActiveException(v protocol.ResponseMetadata) error { + return &CloudHsmClusterNotActiveException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *CloudHsmClusterNotActiveException) Code() string { + return "CloudHsmClusterNotActiveException" +} + +// Message returns the exception's message. +func (s *CloudHsmClusterNotActiveException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *CloudHsmClusterNotActiveException) OrigErr() error { + return nil +} + +func (s *CloudHsmClusterNotActiveException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *CloudHsmClusterNotActiveException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *CloudHsmClusterNotActiveException) RequestID() string { + return s.RespMetadata.RequestID +} + +// The request was rejected because AWS KMS cannot find the AWS CloudHSM cluster +// with the specified cluster ID. Retry the request with a different cluster +// ID. +type CloudHsmClusterNotFoundException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s CloudHsmClusterNotFoundException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s CloudHsmClusterNotFoundException) GoString() string { + return s.String() +} + +func newErrorCloudHsmClusterNotFoundException(v protocol.ResponseMetadata) error { + return &CloudHsmClusterNotFoundException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *CloudHsmClusterNotFoundException) Code() string { + return "CloudHsmClusterNotFoundException" +} + +// Message returns the exception's message. +func (s *CloudHsmClusterNotFoundException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *CloudHsmClusterNotFoundException) OrigErr() error { + return nil +} + +func (s *CloudHsmClusterNotFoundException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *CloudHsmClusterNotFoundException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *CloudHsmClusterNotFoundException) RequestID() string { + return s.RespMetadata.RequestID +} + +// The request was rejected because the specified AWS CloudHSM cluster has a +// different cluster certificate than the original cluster. You cannot use the +// operation to specify an unrelated cluster. +// +// Specify a cluster that shares a backup history with the original cluster. +// This includes clusters that were created from a backup of the current cluster, +// and clusters that were created from the same backup that produced the current +// cluster. +// +// Clusters that share a backup history have the same cluster certificate. To +// view the cluster certificate of a cluster, use the DescribeClusters (https://docs.aws.amazon.com/cloudhsm/latest/APIReference/API_DescribeClusters.html) +// operation. +type CloudHsmClusterNotRelatedException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s CloudHsmClusterNotRelatedException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s CloudHsmClusterNotRelatedException) GoString() string { + return s.String() +} + +func newErrorCloudHsmClusterNotRelatedException(v protocol.ResponseMetadata) error { + return &CloudHsmClusterNotRelatedException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *CloudHsmClusterNotRelatedException) Code() string { + return "CloudHsmClusterNotRelatedException" +} + +// Message returns the exception's message. +func (s *CloudHsmClusterNotRelatedException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *CloudHsmClusterNotRelatedException) OrigErr() error { + return nil +} + +func (s *CloudHsmClusterNotRelatedException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *CloudHsmClusterNotRelatedException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *CloudHsmClusterNotRelatedException) RequestID() string { + return s.RespMetadata.RequestID +} + +type ConnectCustomKeyStoreInput struct { + _ struct{} `type:"structure"` + + // Enter the key store ID of the custom key store that you want to connect. + // To find the ID of a custom key store, use the DescribeCustomKeyStores operation. + // + // CustomKeyStoreId is a required field + CustomKeyStoreId *string `min:"1" type:"string" required:"true"` +} + +// String returns the string representation +func (s ConnectCustomKeyStoreInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s ConnectCustomKeyStoreInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *ConnectCustomKeyStoreInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "ConnectCustomKeyStoreInput"} + if s.CustomKeyStoreId == nil { + invalidParams.Add(request.NewErrParamRequired("CustomKeyStoreId")) + } + if s.CustomKeyStoreId != nil && len(*s.CustomKeyStoreId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("CustomKeyStoreId", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetCustomKeyStoreId sets the CustomKeyStoreId field's value. +func (s *ConnectCustomKeyStoreInput) SetCustomKeyStoreId(v string) *ConnectCustomKeyStoreInput { + s.CustomKeyStoreId = &v + return s +} + +type ConnectCustomKeyStoreOutput struct { + _ struct{} `type:"structure"` +} + +// String returns the string representation +func (s ConnectCustomKeyStoreOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s ConnectCustomKeyStoreOutput) GoString() string { + return s.String() +} + +type CreateAliasInput struct { + _ struct{} `type:"structure"` + + // Specifies the alias name. This value must begin with alias/ followed by a + // name, such as alias/ExampleAlias. The alias name cannot begin with alias/aws/. + // The alias/aws/ prefix is reserved for AWS managed CMKs. + // + // AliasName is a required field + AliasName *string `min:"1" type:"string" required:"true"` + + // Identifies the CMK to which the alias refers. Specify the key ID or the Amazon + // Resource Name (ARN) of the CMK. You cannot specify another alias. For help + // finding the key ID and ARN, see Finding the Key ID and ARN (https://docs.aws.amazon.com/kms/latest/developerguide/viewing-keys.html#find-cmk-id-arn) + // in the AWS Key Management Service Developer Guide. + // + // TargetKeyId is a required field + TargetKeyId *string `min:"1" type:"string" required:"true"` +} + +// String returns the string representation +func (s CreateAliasInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s CreateAliasInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *CreateAliasInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "CreateAliasInput"} + if s.AliasName == nil { + invalidParams.Add(request.NewErrParamRequired("AliasName")) + } + if s.AliasName != nil && len(*s.AliasName) < 1 { + invalidParams.Add(request.NewErrParamMinLen("AliasName", 1)) + } + if s.TargetKeyId == nil { + invalidParams.Add(request.NewErrParamRequired("TargetKeyId")) + } + if s.TargetKeyId != nil && len(*s.TargetKeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("TargetKeyId", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetAliasName sets the AliasName field's value. +func (s *CreateAliasInput) SetAliasName(v string) *CreateAliasInput { + s.AliasName = &v + return s +} + +// SetTargetKeyId sets the TargetKeyId field's value. +func (s *CreateAliasInput) SetTargetKeyId(v string) *CreateAliasInput { + s.TargetKeyId = &v + return s +} + +type CreateAliasOutput struct { + _ struct{} `type:"structure"` +} + +// String returns the string representation +func (s CreateAliasOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s CreateAliasOutput) GoString() string { + return s.String() +} + +type CreateCustomKeyStoreInput struct { + _ struct{} `type:"structure"` + + // Identifies the AWS CloudHSM cluster for the custom key store. Enter the cluster + // ID of any active AWS CloudHSM cluster that is not already associated with + // a custom key store. To find the cluster ID, use the DescribeClusters (https://docs.aws.amazon.com/cloudhsm/latest/APIReference/API_DescribeClusters.html) + // operation. + // + // CloudHsmClusterId is a required field + CloudHsmClusterId *string `min:"19" type:"string" required:"true"` + + // Specifies a friendly name for the custom key store. The name must be unique + // in your AWS account. + // + // CustomKeyStoreName is a required field + CustomKeyStoreName *string `min:"1" type:"string" required:"true"` + + // Enter the password of the kmsuser crypto user (CU) account (https://docs.aws.amazon.com/kms/latest/developerguide/key-store-concepts.html#concept-kmsuser) + // in the specified AWS CloudHSM cluster. AWS KMS logs into the cluster as this + // user to manage key material on your behalf. + // + // The password must be a string of 7 to 32 characters. Its value is case sensitive. + // + // This parameter tells AWS KMS the kmsuser account password; it does not change + // the password in the AWS CloudHSM cluster. + // + // KeyStorePassword is a required field + KeyStorePassword *string `min:"7" type:"string" required:"true" sensitive:"true"` + + // Enter the content of the trust anchor certificate for the cluster. This is + // the content of the customerCA.crt file that you created when you initialized + // the cluster (https://docs.aws.amazon.com/cloudhsm/latest/userguide/initialize-cluster.html). + // + // TrustAnchorCertificate is a required field + TrustAnchorCertificate *string `min:"1" type:"string" required:"true"` +} + +// String returns the string representation +func (s CreateCustomKeyStoreInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s CreateCustomKeyStoreInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *CreateCustomKeyStoreInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "CreateCustomKeyStoreInput"} + if s.CloudHsmClusterId == nil { + invalidParams.Add(request.NewErrParamRequired("CloudHsmClusterId")) + } + if s.CloudHsmClusterId != nil && len(*s.CloudHsmClusterId) < 19 { + invalidParams.Add(request.NewErrParamMinLen("CloudHsmClusterId", 19)) + } + if s.CustomKeyStoreName == nil { + invalidParams.Add(request.NewErrParamRequired("CustomKeyStoreName")) + } + if s.CustomKeyStoreName != nil && len(*s.CustomKeyStoreName) < 1 { + invalidParams.Add(request.NewErrParamMinLen("CustomKeyStoreName", 1)) + } + if s.KeyStorePassword == nil { + invalidParams.Add(request.NewErrParamRequired("KeyStorePassword")) + } + if s.KeyStorePassword != nil && len(*s.KeyStorePassword) < 7 { + invalidParams.Add(request.NewErrParamMinLen("KeyStorePassword", 7)) + } + if s.TrustAnchorCertificate == nil { + invalidParams.Add(request.NewErrParamRequired("TrustAnchorCertificate")) + } + if s.TrustAnchorCertificate != nil && len(*s.TrustAnchorCertificate) < 1 { + invalidParams.Add(request.NewErrParamMinLen("TrustAnchorCertificate", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetCloudHsmClusterId sets the CloudHsmClusterId field's value. +func (s *CreateCustomKeyStoreInput) SetCloudHsmClusterId(v string) *CreateCustomKeyStoreInput { + s.CloudHsmClusterId = &v + return s +} + +// SetCustomKeyStoreName sets the CustomKeyStoreName field's value. +func (s *CreateCustomKeyStoreInput) SetCustomKeyStoreName(v string) *CreateCustomKeyStoreInput { + s.CustomKeyStoreName = &v + return s +} + +// SetKeyStorePassword sets the KeyStorePassword field's value. +func (s *CreateCustomKeyStoreInput) SetKeyStorePassword(v string) *CreateCustomKeyStoreInput { + s.KeyStorePassword = &v + return s +} + +// SetTrustAnchorCertificate sets the TrustAnchorCertificate field's value. +func (s *CreateCustomKeyStoreInput) SetTrustAnchorCertificate(v string) *CreateCustomKeyStoreInput { + s.TrustAnchorCertificate = &v + return s +} + +type CreateCustomKeyStoreOutput struct { + _ struct{} `type:"structure"` + + // A unique identifier for the new custom key store. + CustomKeyStoreId *string `min:"1" type:"string"` +} + +// String returns the string representation +func (s CreateCustomKeyStoreOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s CreateCustomKeyStoreOutput) GoString() string { + return s.String() +} + +// SetCustomKeyStoreId sets the CustomKeyStoreId field's value. +func (s *CreateCustomKeyStoreOutput) SetCustomKeyStoreId(v string) *CreateCustomKeyStoreOutput { + s.CustomKeyStoreId = &v + return s +} + +type CreateGrantInput struct { + _ struct{} `type:"structure"` + + // Allows a cryptographic operation (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#cryptographic-operations) + // only when the encryption context matches or includes the encryption context + // specified in this structure. For more information about encryption context, + // see Encryption Context (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#encrypt_context) + // in the AWS Key Management Service Developer Guide . + Constraints *GrantConstraints `type:"structure"` + + // A list of grant tokens. + // + // For more information, see Grant Tokens (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#grant_token) + // in the AWS Key Management Service Developer Guide. + GrantTokens []*string `type:"list"` + + // The principal that is given permission to perform the operations that the + // grant permits. + // + // To specify the principal, use the Amazon Resource Name (ARN) (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html) + // of an AWS principal. Valid AWS principals include AWS accounts (root), IAM + // users, IAM roles, federated users, and assumed role users. For examples of + // the ARN syntax to use for specifying a principal, see AWS Identity and Access + // Management (IAM) (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html#arn-syntax-iam) + // in the Example ARNs section of the AWS General Reference. + // + // GranteePrincipal is a required field + GranteePrincipal *string `min:"1" type:"string" required:"true"` + + // The unique identifier for the customer master key (CMK) that the grant applies + // to. + // + // Specify the key ID or the Amazon Resource Name (ARN) of the CMK. To specify + // a CMK in a different AWS account, you must use the key ARN. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` + + // A friendly name for identifying the grant. Use this value to prevent the + // unintended creation of duplicate grants when retrying this request. + // + // When this value is absent, all CreateGrant requests result in a new grant + // with a unique GrantId even if all the supplied parameters are identical. + // This can result in unintended duplicates when you retry the CreateGrant request. + // + // When this value is present, you can retry a CreateGrant request with identical + // parameters; if the grant already exists, the original GrantId is returned + // without creating a new grant. Note that the returned grant token is unique + // with every CreateGrant request, even when a duplicate GrantId is returned. + // All grant tokens obtained in this way can be used interchangeably. + Name *string `min:"1" type:"string"` + + // A list of operations that the grant permits. + // + // Operations is a required field + Operations []*string `type:"list" required:"true"` + + // The principal that is given permission to retire the grant by using RetireGrant + // operation. + // + // To specify the principal, use the Amazon Resource Name (ARN) (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html) + // of an AWS principal. Valid AWS principals include AWS accounts (root), IAM + // users, federated users, and assumed role users. For examples of the ARN syntax + // to use for specifying a principal, see AWS Identity and Access Management + // (IAM) (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html#arn-syntax-iam) + // in the Example ARNs section of the AWS General Reference. + RetiringPrincipal *string `min:"1" type:"string"` +} + +// String returns the string representation +func (s CreateGrantInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s CreateGrantInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *CreateGrantInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "CreateGrantInput"} + if s.GranteePrincipal == nil { + invalidParams.Add(request.NewErrParamRequired("GranteePrincipal")) + } + if s.GranteePrincipal != nil && len(*s.GranteePrincipal) < 1 { + invalidParams.Add(request.NewErrParamMinLen("GranteePrincipal", 1)) + } + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + if s.Name != nil && len(*s.Name) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Name", 1)) + } + if s.Operations == nil { + invalidParams.Add(request.NewErrParamRequired("Operations")) + } + if s.RetiringPrincipal != nil && len(*s.RetiringPrincipal) < 1 { + invalidParams.Add(request.NewErrParamMinLen("RetiringPrincipal", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetConstraints sets the Constraints field's value. +func (s *CreateGrantInput) SetConstraints(v *GrantConstraints) *CreateGrantInput { + s.Constraints = v + return s +} + +// SetGrantTokens sets the GrantTokens field's value. +func (s *CreateGrantInput) SetGrantTokens(v []*string) *CreateGrantInput { + s.GrantTokens = v + return s +} + +// SetGranteePrincipal sets the GranteePrincipal field's value. +func (s *CreateGrantInput) SetGranteePrincipal(v string) *CreateGrantInput { + s.GranteePrincipal = &v + return s +} + +// SetKeyId sets the KeyId field's value. +func (s *CreateGrantInput) SetKeyId(v string) *CreateGrantInput { + s.KeyId = &v + return s +} + +// SetName sets the Name field's value. +func (s *CreateGrantInput) SetName(v string) *CreateGrantInput { + s.Name = &v + return s +} + +// SetOperations sets the Operations field's value. +func (s *CreateGrantInput) SetOperations(v []*string) *CreateGrantInput { + s.Operations = v + return s +} + +// SetRetiringPrincipal sets the RetiringPrincipal field's value. +func (s *CreateGrantInput) SetRetiringPrincipal(v string) *CreateGrantInput { + s.RetiringPrincipal = &v + return s +} + +type CreateGrantOutput struct { + _ struct{} `type:"structure"` + + // The unique identifier for the grant. + // + // You can use the GrantId in a subsequent RetireGrant or RevokeGrant operation. + GrantId *string `min:"1" type:"string"` + + // The grant token. + // + // For more information, see Grant Tokens (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#grant_token) + // in the AWS Key Management Service Developer Guide. + GrantToken *string `min:"1" type:"string"` +} + +// String returns the string representation +func (s CreateGrantOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s CreateGrantOutput) GoString() string { + return s.String() +} + +// SetGrantId sets the GrantId field's value. +func (s *CreateGrantOutput) SetGrantId(v string) *CreateGrantOutput { + s.GrantId = &v + return s +} + +// SetGrantToken sets the GrantToken field's value. +func (s *CreateGrantOutput) SetGrantToken(v string) *CreateGrantOutput { + s.GrantToken = &v + return s +} + +type CreateKeyInput struct { + _ struct{} `type:"structure"` + + // A flag to indicate whether to bypass the key policy lockout safety check. + // + // Setting this value to true increases the risk that the CMK becomes unmanageable. + // Do not set this value to true indiscriminately. + // + // For more information, refer to the scenario in the Default Key Policy (https://docs.aws.amazon.com/kms/latest/developerguide/key-policies.html#key-policy-default-allow-root-enable-iam) + // section in the AWS Key Management Service Developer Guide . + // + // Use this parameter only when you include a policy in the request and you + // intend to prevent the principal that is making the request from making a + // subsequent PutKeyPolicy request on the CMK. + // + // The default value is false. + BypassPolicyLockoutSafetyCheck *bool `type:"boolean"` + + // Creates the CMK in the specified custom key store (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html) + // and the key material in its associated AWS CloudHSM cluster. To create a + // CMK in a custom key store, you must also specify the Origin parameter with + // a value of AWS_CLOUDHSM. The AWS CloudHSM cluster that is associated with + // the custom key store must have at least two active HSMs, each in a different + // Availability Zone in the Region. + // + // This parameter is valid only for symmetric CMKs. You cannot create an asymmetric + // CMK in a custom key store. + // + // To find the ID of a custom key store, use the DescribeCustomKeyStores operation. + // + // The response includes the custom key store ID and the ID of the AWS CloudHSM + // cluster. + // + // This operation is part of the Custom Key Store feature (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html) + // feature in AWS KMS, which combines the convenience and extensive integration + // of AWS KMS with the isolation and control of a single-tenant key store. + CustomKeyStoreId *string `min:"1" type:"string"` + + // Specifies the type of CMK to create. The default value, SYMMETRIC_DEFAULT, + // creates a CMK with a 256-bit symmetric key for encryption and decryption. + // For help choosing a key spec for your CMK, see How to Choose Your CMK Configuration + // (https://docs.aws.amazon.com/kms/latest/developerguide/symm-asymm-choose.html) + // in the AWS Key Management Service Developer Guide. + // + // The CustomerMasterKeySpec determines whether the CMK contains a symmetric + // key or an asymmetric key pair. It also determines the encryption algorithms + // or signing algorithms that the CMK supports. You can't change the CustomerMasterKeySpec + // after the CMK is created. To further restrict the algorithms that can be + // used with the CMK, use a condition key in its key policy or IAM policy. For + // more information, see kms:EncryptionAlgorithm (https://docs.aws.amazon.com/kms/latest/developerguide/policy-conditions.html#conditions-kms-encryption-algorithm) + // or kms:Signing Algorithm (https://docs.aws.amazon.com/kms/latest/developerguide/policy-conditions.html#conditions-kms-signing-algorithm) + // in the AWS Key Management Service Developer Guide. + // + // AWS services that are integrated with AWS KMS (http://aws.amazon.com/kms/features/#AWS_Service_Integration) + // use symmetric CMKs to protect your data. These services do not support asymmetric + // CMKs. For help determining whether a CMK is symmetric or asymmetric, see + // Identifying Symmetric and Asymmetric CMKs (https://docs.aws.amazon.com/kms/latest/developerguide/find-symm-asymm.html) + // in the AWS Key Management Service Developer Guide. + // + // AWS KMS supports the following key specs for CMKs: + // + // * Symmetric key (default) SYMMETRIC_DEFAULT (AES-256-GCM) + // + // * Asymmetric RSA key pairs RSA_2048 RSA_3072 RSA_4096 + // + // * Asymmetric NIST-recommended elliptic curve key pairs ECC_NIST_P256 (secp256r1) + // ECC_NIST_P384 (secp384r1) ECC_NIST_P521 (secp521r1) + // + // * Other asymmetric elliptic curve key pairs ECC_SECG_P256K1 (secp256k1), + // commonly used for cryptocurrencies. + CustomerMasterKeySpec *string `type:"string" enum:"CustomerMasterKeySpec"` + + // A description of the CMK. + // + // Use a description that helps you decide whether the CMK is appropriate for + // a task. + Description *string `type:"string"` + + // Determines the cryptographic operations (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#cryptographic-operations) + // for which you can use the CMK. The default value is ENCRYPT_DECRYPT. This + // parameter is required only for asymmetric CMKs. You can't change the KeyUsage + // value after the CMK is created. + // + // Select only one valid value. + // + // * For symmetric CMKs, omit the parameter or specify ENCRYPT_DECRYPT. + // + // * For asymmetric CMKs with RSA key material, specify ENCRYPT_DECRYPT or + // SIGN_VERIFY. + // + // * For asymmetric CMKs with ECC key material, specify SIGN_VERIFY. + KeyUsage *string `type:"string" enum:"KeyUsageType"` + + // The source of the key material for the CMK. You cannot change the origin + // after you create the CMK. The default is AWS_KMS, which means AWS KMS creates + // the key material. + // + // When the parameter value is EXTERNAL, AWS KMS creates a CMK without key material + // so that you can import key material from your existing key management infrastructure. + // For more information about importing key material into AWS KMS, see Importing + // Key Material (https://docs.aws.amazon.com/kms/latest/developerguide/importing-keys.html) + // in the AWS Key Management Service Developer Guide. This value is valid only + // for symmetric CMKs. + // + // When the parameter value is AWS_CLOUDHSM, AWS KMS creates the CMK in an AWS + // KMS custom key store (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html) + // and creates its key material in the associated AWS CloudHSM cluster. You + // must also use the CustomKeyStoreId parameter to identify the custom key store. + // This value is valid only for symmetric CMKs. + Origin *string `type:"string" enum:"OriginType"` + + // The key policy to attach to the CMK. + // + // If you provide a key policy, it must meet the following criteria: + // + // * If you don't set BypassPolicyLockoutSafetyCheck to true, the key policy + // must allow the principal that is making the CreateKey request to make + // a subsequent PutKeyPolicy request on the CMK. This reduces the risk that + // the CMK becomes unmanageable. For more information, refer to the scenario + // in the Default Key Policy (https://docs.aws.amazon.com/kms/latest/developerguide/key-policies.html#key-policy-default-allow-root-enable-iam) + // section of the AWS Key Management Service Developer Guide . + // + // * Each statement in the key policy must contain one or more principals. + // The principals in the key policy must exist and be visible to AWS KMS. + // When you create a new AWS principal (for example, an IAM user or role), + // you might need to enforce a delay before including the new principal in + // a key policy because the new principal might not be immediately visible + // to AWS KMS. For more information, see Changes that I make are not always + // immediately visible (https://docs.aws.amazon.com/IAM/latest/UserGuide/troubleshoot_general.html#troubleshoot_general_eventual-consistency) + // in the AWS Identity and Access Management User Guide. + // + // If you do not provide a key policy, AWS KMS attaches a default key policy + // to the CMK. For more information, see Default Key Policy (https://docs.aws.amazon.com/kms/latest/developerguide/key-policies.html#key-policy-default) + // in the AWS Key Management Service Developer Guide. + // + // The key policy size quota is 32 kilobytes (32768 bytes). + Policy *string `min:"1" type:"string"` + + // One or more tags. Each tag consists of a tag key and a tag value. Both the + // tag key and the tag value are required, but the tag value can be an empty + // (null) string. + // + // When you add tags to an AWS resource, AWS generates a cost allocation report + // with usage and costs aggregated by tags. For information about adding, changing, + // deleting and listing tags for CMKs, see Tagging Keys (https://docs.aws.amazon.com/kms/latest/developerguide/tagging-keys.html). + // + // Use this parameter to tag the CMK when it is created. To add tags to an existing + // CMK, use the TagResource operation. + Tags []*Tag `type:"list"` +} + +// String returns the string representation +func (s CreateKeyInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s CreateKeyInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *CreateKeyInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "CreateKeyInput"} + if s.CustomKeyStoreId != nil && len(*s.CustomKeyStoreId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("CustomKeyStoreId", 1)) + } + if s.Policy != nil && len(*s.Policy) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Policy", 1)) + } + if s.Tags != nil { + for i, v := range s.Tags { + if v == nil { + continue + } + if err := v.Validate(); err != nil { + invalidParams.AddNested(fmt.Sprintf("%s[%v]", "Tags", i), err.(request.ErrInvalidParams)) + } + } + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetBypassPolicyLockoutSafetyCheck sets the BypassPolicyLockoutSafetyCheck field's value. +func (s *CreateKeyInput) SetBypassPolicyLockoutSafetyCheck(v bool) *CreateKeyInput { + s.BypassPolicyLockoutSafetyCheck = &v + return s +} + +// SetCustomKeyStoreId sets the CustomKeyStoreId field's value. +func (s *CreateKeyInput) SetCustomKeyStoreId(v string) *CreateKeyInput { + s.CustomKeyStoreId = &v + return s +} + +// SetCustomerMasterKeySpec sets the CustomerMasterKeySpec field's value. +func (s *CreateKeyInput) SetCustomerMasterKeySpec(v string) *CreateKeyInput { + s.CustomerMasterKeySpec = &v + return s +} + +// SetDescription sets the Description field's value. +func (s *CreateKeyInput) SetDescription(v string) *CreateKeyInput { + s.Description = &v + return s +} + +// SetKeyUsage sets the KeyUsage field's value. +func (s *CreateKeyInput) SetKeyUsage(v string) *CreateKeyInput { + s.KeyUsage = &v + return s +} + +// SetOrigin sets the Origin field's value. +func (s *CreateKeyInput) SetOrigin(v string) *CreateKeyInput { + s.Origin = &v + return s +} + +// SetPolicy sets the Policy field's value. +func (s *CreateKeyInput) SetPolicy(v string) *CreateKeyInput { + s.Policy = &v + return s +} + +// SetTags sets the Tags field's value. +func (s *CreateKeyInput) SetTags(v []*Tag) *CreateKeyInput { + s.Tags = v + return s +} + +type CreateKeyOutput struct { + _ struct{} `type:"structure"` + + // Metadata associated with the CMK. + KeyMetadata *KeyMetadata `type:"structure"` +} + +// String returns the string representation +func (s CreateKeyOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s CreateKeyOutput) GoString() string { + return s.String() +} + +// SetKeyMetadata sets the KeyMetadata field's value. +func (s *CreateKeyOutput) SetKeyMetadata(v *KeyMetadata) *CreateKeyOutput { + s.KeyMetadata = v + return s +} + +// The request was rejected because the custom key store contains AWS KMS customer +// master keys (CMKs). After verifying that you do not need to use the CMKs, +// use the ScheduleKeyDeletion operation to delete the CMKs. After they are +// deleted, you can delete the custom key store. +type CustomKeyStoreHasCMKsException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s CustomKeyStoreHasCMKsException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s CustomKeyStoreHasCMKsException) GoString() string { + return s.String() +} + +func newErrorCustomKeyStoreHasCMKsException(v protocol.ResponseMetadata) error { + return &CustomKeyStoreHasCMKsException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *CustomKeyStoreHasCMKsException) Code() string { + return "CustomKeyStoreHasCMKsException" +} + +// Message returns the exception's message. +func (s *CustomKeyStoreHasCMKsException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *CustomKeyStoreHasCMKsException) OrigErr() error { + return nil +} + +func (s *CustomKeyStoreHasCMKsException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *CustomKeyStoreHasCMKsException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *CustomKeyStoreHasCMKsException) RequestID() string { + return s.RespMetadata.RequestID +} + +// The request was rejected because of the ConnectionState of the custom key +// store. To get the ConnectionState of a custom key store, use the DescribeCustomKeyStores +// operation. +// +// This exception is thrown under the following conditions: +// +// * You requested the CreateKey or GenerateRandom operation in a custom +// key store that is not connected. These operations are valid only when +// the custom key store ConnectionState is CONNECTED. +// +// * You requested the UpdateCustomKeyStore or DeleteCustomKeyStore operation +// on a custom key store that is not disconnected. This operation is valid +// only when the custom key store ConnectionState is DISCONNECTED. +// +// * You requested the ConnectCustomKeyStore operation on a custom key store +// with a ConnectionState of DISCONNECTING or FAILED. This operation is valid +// for all other ConnectionState values. +type CustomKeyStoreInvalidStateException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s CustomKeyStoreInvalidStateException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s CustomKeyStoreInvalidStateException) GoString() string { + return s.String() +} + +func newErrorCustomKeyStoreInvalidStateException(v protocol.ResponseMetadata) error { + return &CustomKeyStoreInvalidStateException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *CustomKeyStoreInvalidStateException) Code() string { + return "CustomKeyStoreInvalidStateException" +} + +// Message returns the exception's message. +func (s *CustomKeyStoreInvalidStateException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *CustomKeyStoreInvalidStateException) OrigErr() error { + return nil +} + +func (s *CustomKeyStoreInvalidStateException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *CustomKeyStoreInvalidStateException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *CustomKeyStoreInvalidStateException) RequestID() string { + return s.RespMetadata.RequestID +} + +// The request was rejected because the specified custom key store name is already +// assigned to another custom key store in the account. Try again with a custom +// key store name that is unique in the account. +type CustomKeyStoreNameInUseException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s CustomKeyStoreNameInUseException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s CustomKeyStoreNameInUseException) GoString() string { + return s.String() +} + +func newErrorCustomKeyStoreNameInUseException(v protocol.ResponseMetadata) error { + return &CustomKeyStoreNameInUseException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *CustomKeyStoreNameInUseException) Code() string { + return "CustomKeyStoreNameInUseException" +} + +// Message returns the exception's message. +func (s *CustomKeyStoreNameInUseException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *CustomKeyStoreNameInUseException) OrigErr() error { + return nil +} + +func (s *CustomKeyStoreNameInUseException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *CustomKeyStoreNameInUseException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *CustomKeyStoreNameInUseException) RequestID() string { + return s.RespMetadata.RequestID +} + +// The request was rejected because AWS KMS cannot find a custom key store with +// the specified key store name or ID. +type CustomKeyStoreNotFoundException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s CustomKeyStoreNotFoundException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s CustomKeyStoreNotFoundException) GoString() string { + return s.String() +} + +func newErrorCustomKeyStoreNotFoundException(v protocol.ResponseMetadata) error { + return &CustomKeyStoreNotFoundException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *CustomKeyStoreNotFoundException) Code() string { + return "CustomKeyStoreNotFoundException" +} + +// Message returns the exception's message. +func (s *CustomKeyStoreNotFoundException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *CustomKeyStoreNotFoundException) OrigErr() error { + return nil +} + +func (s *CustomKeyStoreNotFoundException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *CustomKeyStoreNotFoundException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *CustomKeyStoreNotFoundException) RequestID() string { + return s.RespMetadata.RequestID +} + +// Contains information about each custom key store in the custom key store +// list. +type CustomKeyStoresListEntry struct { + _ struct{} `type:"structure"` + + // A unique identifier for the AWS CloudHSM cluster that is associated with + // the custom key store. + CloudHsmClusterId *string `min:"19" type:"string"` + + // Describes the connection error. This field appears in the response only when + // the ConnectionState is FAILED. For help resolving these errors, see How to + // Fix a Connection Failure (https://docs.aws.amazon.com/kms/latest/developerguide/fix-keystore.html#fix-keystore-failed) + // in AWS Key Management Service Developer Guide. + // + // Valid values are: + // + // * CLUSTER_NOT_FOUND - AWS KMS cannot find the AWS CloudHSM cluster with + // the specified cluster ID. + // + // * INSUFFICIENT_CLOUDHSM_HSMS - The associated AWS CloudHSM cluster does + // not contain any active HSMs. To connect a custom key store to its AWS + // CloudHSM cluster, the cluster must contain at least one active HSM. + // + // * INTERNAL_ERROR - AWS KMS could not complete the request due to an internal + // error. Retry the request. For ConnectCustomKeyStore requests, disconnect + // the custom key store before trying to connect again. + // + // * INVALID_CREDENTIALS - AWS KMS does not have the correct password for + // the kmsuser crypto user in the AWS CloudHSM cluster. Before you can connect + // your custom key store to its AWS CloudHSM cluster, you must change the + // kmsuser account password and update the key store password value for the + // custom key store. + // + // * NETWORK_ERRORS - Network errors are preventing AWS KMS from connecting + // to the custom key store. + // + // * SUBNET_NOT_FOUND - A subnet in the AWS CloudHSM cluster configuration + // was deleted. If AWS KMS cannot find all of the subnets in the cluster + // configuration, attempts to connect the custom key store to the AWS CloudHSM + // cluster fail. To fix this error, create a cluster from a recent backup + // and associate it with your custom key store. (This process creates a new + // cluster configuration with a VPC and private subnets.) For details, see + // How to Fix a Connection Failure (https://docs.aws.amazon.com/kms/latest/developerguide/fix-keystore.html#fix-keystore-failed) + // in the AWS Key Management Service Developer Guide. + // + // * USER_LOCKED_OUT - The kmsuser CU account is locked out of the associated + // AWS CloudHSM cluster due to too many failed password attempts. Before + // you can connect your custom key store to its AWS CloudHSM cluster, you + // must change the kmsuser account password and update the key store password + // value for the custom key store. + // + // * USER_LOGGED_IN - The kmsuser CU account is logged into the the associated + // AWS CloudHSM cluster. This prevents AWS KMS from rotating the kmsuser + // account password and logging into the cluster. Before you can connect + // your custom key store to its AWS CloudHSM cluster, you must log the kmsuser + // CU out of the cluster. If you changed the kmsuser password to log into + // the cluster, you must also and update the key store password value for + // the custom key store. For help, see How to Log Out and Reconnect (https://docs.aws.amazon.com/kms/latest/developerguide/fix-keystore.html#login-kmsuser-2) + // in the AWS Key Management Service Developer Guide. + // + // * USER_NOT_FOUND - AWS KMS cannot find a kmsuser CU account in the associated + // AWS CloudHSM cluster. Before you can connect your custom key store to + // its AWS CloudHSM cluster, you must create a kmsuser CU account in the + // cluster, and then update the key store password value for the custom key + // store. + ConnectionErrorCode *string `type:"string" enum:"ConnectionErrorCodeType"` + + // Indicates whether the custom key store is connected to its AWS CloudHSM cluster. + // + // You can create and use CMKs in your custom key stores only when its connection + // state is CONNECTED. + // + // The value is DISCONNECTED if the key store has never been connected or you + // use the DisconnectCustomKeyStore operation to disconnect it. If the value + // is CONNECTED but you are having trouble using the custom key store, make + // sure that its associated AWS CloudHSM cluster is active and contains at least + // one active HSM. + // + // A value of FAILED indicates that an attempt to connect was unsuccessful. + // The ConnectionErrorCode field in the response indicates the cause of the + // failure. For help resolving a connection failure, see Troubleshooting a Custom + // Key Store (https://docs.aws.amazon.com/kms/latest/developerguide/fix-keystore.html) + // in the AWS Key Management Service Developer Guide. + ConnectionState *string `type:"string" enum:"ConnectionStateType"` + + // The date and time when the custom key store was created. + CreationDate *time.Time `type:"timestamp"` + + // A unique identifier for the custom key store. + CustomKeyStoreId *string `min:"1" type:"string"` + + // The user-specified friendly name for the custom key store. + CustomKeyStoreName *string `min:"1" type:"string"` + + // The trust anchor certificate of the associated AWS CloudHSM cluster. When + // you initialize the cluster (https://docs.aws.amazon.com/cloudhsm/latest/userguide/initialize-cluster.html#sign-csr), + // you create this certificate and save it in the customerCA.crt file. + TrustAnchorCertificate *string `min:"1" type:"string"` +} + +// String returns the string representation +func (s CustomKeyStoresListEntry) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s CustomKeyStoresListEntry) GoString() string { + return s.String() +} + +// SetCloudHsmClusterId sets the CloudHsmClusterId field's value. +func (s *CustomKeyStoresListEntry) SetCloudHsmClusterId(v string) *CustomKeyStoresListEntry { + s.CloudHsmClusterId = &v + return s +} + +// SetConnectionErrorCode sets the ConnectionErrorCode field's value. +func (s *CustomKeyStoresListEntry) SetConnectionErrorCode(v string) *CustomKeyStoresListEntry { + s.ConnectionErrorCode = &v + return s +} + +// SetConnectionState sets the ConnectionState field's value. +func (s *CustomKeyStoresListEntry) SetConnectionState(v string) *CustomKeyStoresListEntry { + s.ConnectionState = &v + return s +} + +// SetCreationDate sets the CreationDate field's value. +func (s *CustomKeyStoresListEntry) SetCreationDate(v time.Time) *CustomKeyStoresListEntry { + s.CreationDate = &v + return s +} + +// SetCustomKeyStoreId sets the CustomKeyStoreId field's value. +func (s *CustomKeyStoresListEntry) SetCustomKeyStoreId(v string) *CustomKeyStoresListEntry { + s.CustomKeyStoreId = &v + return s +} + +// SetCustomKeyStoreName sets the CustomKeyStoreName field's value. +func (s *CustomKeyStoresListEntry) SetCustomKeyStoreName(v string) *CustomKeyStoresListEntry { + s.CustomKeyStoreName = &v + return s +} + +// SetTrustAnchorCertificate sets the TrustAnchorCertificate field's value. +func (s *CustomKeyStoresListEntry) SetTrustAnchorCertificate(v string) *CustomKeyStoresListEntry { + s.TrustAnchorCertificate = &v + return s +} + +type DecryptInput struct { + _ struct{} `type:"structure"` + + // Ciphertext to be decrypted. The blob includes metadata. + // + // CiphertextBlob is automatically base64 encoded/decoded by the SDK. + // + // CiphertextBlob is a required field + CiphertextBlob []byte `min:"1" type:"blob" required:"true"` + + // Specifies the encryption algorithm that will be used to decrypt the ciphertext. + // Specify the same algorithm that was used to encrypt the data. If you specify + // a different algorithm, the Decrypt operation fails. + // + // This parameter is required only when the ciphertext was encrypted under an + // asymmetric CMK. The default value, SYMMETRIC_DEFAULT, represents the only + // supported algorithm that is valid for symmetric CMKs. + EncryptionAlgorithm *string `type:"string" enum:"EncryptionAlgorithmSpec"` + + // Specifies the encryption context to use when decrypting the data. An encryption + // context is valid only for cryptographic operations (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#cryptographic-operations) + // with a symmetric CMK. The standard asymmetric encryption algorithms that + // AWS KMS uses do not support an encryption context. + // + // An encryption context is a collection of non-secret key-value pairs that + // represents additional authenticated data. When you use an encryption context + // to encrypt data, you must specify the same (an exact case-sensitive match) + // encryption context to decrypt the data. An encryption context is optional + // when encrypting with a symmetric CMK, but it is highly recommended. + // + // For more information, see Encryption Context (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#encrypt_context) + // in the AWS Key Management Service Developer Guide. + EncryptionContext map[string]*string `type:"map"` + + // A list of grant tokens. + // + // For more information, see Grant Tokens (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#grant_token) + // in the AWS Key Management Service Developer Guide. + GrantTokens []*string `type:"list"` + + // Specifies the customer master key (CMK) that AWS KMS will use to decrypt + // the ciphertext. Enter a key ID of the CMK that was used to encrypt the ciphertext. + // + // If you specify a KeyId value, the Decrypt operation succeeds only if the + // specified CMK was used to encrypt the ciphertext. + // + // This parameter is required only when the ciphertext was encrypted under an + // asymmetric CMK. Otherwise, AWS KMS uses the metadata that it adds to the + // ciphertext blob to determine which CMK was used to encrypt the ciphertext. + // However, you can use this parameter to ensure that a particular CMK (of any + // kind) is used to decrypt the ciphertext. + // + // To specify a CMK, use its key ID, Amazon Resource Name (ARN), alias name, + // or alias ARN. When using an alias name, prefix it with "alias/". + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Alias name: alias/ExampleAlias + // + // * Alias ARN: arn:aws:kms:us-east-2:111122223333:alias/ExampleAlias + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. To + // get the alias name and alias ARN, use ListAliases. + KeyId *string `min:"1" type:"string"` +} + +// String returns the string representation +func (s DecryptInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s DecryptInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *DecryptInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "DecryptInput"} + if s.CiphertextBlob == nil { + invalidParams.Add(request.NewErrParamRequired("CiphertextBlob")) + } + if s.CiphertextBlob != nil && len(s.CiphertextBlob) < 1 { + invalidParams.Add(request.NewErrParamMinLen("CiphertextBlob", 1)) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetCiphertextBlob sets the CiphertextBlob field's value. +func (s *DecryptInput) SetCiphertextBlob(v []byte) *DecryptInput { + s.CiphertextBlob = v + return s +} + +// SetEncryptionAlgorithm sets the EncryptionAlgorithm field's value. +func (s *DecryptInput) SetEncryptionAlgorithm(v string) *DecryptInput { + s.EncryptionAlgorithm = &v + return s +} + +// SetEncryptionContext sets the EncryptionContext field's value. +func (s *DecryptInput) SetEncryptionContext(v map[string]*string) *DecryptInput { + s.EncryptionContext = v + return s +} + +// SetGrantTokens sets the GrantTokens field's value. +func (s *DecryptInput) SetGrantTokens(v []*string) *DecryptInput { + s.GrantTokens = v + return s +} + +// SetKeyId sets the KeyId field's value. +func (s *DecryptInput) SetKeyId(v string) *DecryptInput { + s.KeyId = &v + return s +} + +type DecryptOutput struct { + _ struct{} `type:"structure"` + + // The encryption algorithm that was used to decrypt the ciphertext. + EncryptionAlgorithm *string `type:"string" enum:"EncryptionAlgorithmSpec"` + + // The Amazon Resource Name (key ARN (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#key-id-key-ARN)) + // of the CMK that was used to decrypt the ciphertext. + KeyId *string `min:"1" type:"string"` + + // Decrypted plaintext data. When you use the HTTP API or the AWS CLI, the value + // is Base64-encoded. Otherwise, it is not Base64-encoded. + // + // Plaintext is automatically base64 encoded/decoded by the SDK. + Plaintext []byte `min:"1" type:"blob" sensitive:"true"` +} + +// String returns the string representation +func (s DecryptOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s DecryptOutput) GoString() string { + return s.String() +} + +// SetEncryptionAlgorithm sets the EncryptionAlgorithm field's value. +func (s *DecryptOutput) SetEncryptionAlgorithm(v string) *DecryptOutput { + s.EncryptionAlgorithm = &v + return s +} + +// SetKeyId sets the KeyId field's value. +func (s *DecryptOutput) SetKeyId(v string) *DecryptOutput { + s.KeyId = &v + return s +} + +// SetPlaintext sets the Plaintext field's value. +func (s *DecryptOutput) SetPlaintext(v []byte) *DecryptOutput { + s.Plaintext = v + return s +} + +type DeleteAliasInput struct { + _ struct{} `type:"structure"` + + // The alias to be deleted. The alias name must begin with alias/ followed by + // the alias name, such as alias/ExampleAlias. + // + // AliasName is a required field + AliasName *string `min:"1" type:"string" required:"true"` +} + +// String returns the string representation +func (s DeleteAliasInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s DeleteAliasInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *DeleteAliasInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "DeleteAliasInput"} + if s.AliasName == nil { + invalidParams.Add(request.NewErrParamRequired("AliasName")) + } + if s.AliasName != nil && len(*s.AliasName) < 1 { + invalidParams.Add(request.NewErrParamMinLen("AliasName", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetAliasName sets the AliasName field's value. +func (s *DeleteAliasInput) SetAliasName(v string) *DeleteAliasInput { + s.AliasName = &v + return s +} + +type DeleteAliasOutput struct { + _ struct{} `type:"structure"` +} + +// String returns the string representation +func (s DeleteAliasOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s DeleteAliasOutput) GoString() string { + return s.String() +} + +type DeleteCustomKeyStoreInput struct { + _ struct{} `type:"structure"` + + // Enter the ID of the custom key store you want to delete. To find the ID of + // a custom key store, use the DescribeCustomKeyStores operation. + // + // CustomKeyStoreId is a required field + CustomKeyStoreId *string `min:"1" type:"string" required:"true"` +} + +// String returns the string representation +func (s DeleteCustomKeyStoreInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s DeleteCustomKeyStoreInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *DeleteCustomKeyStoreInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "DeleteCustomKeyStoreInput"} + if s.CustomKeyStoreId == nil { + invalidParams.Add(request.NewErrParamRequired("CustomKeyStoreId")) + } + if s.CustomKeyStoreId != nil && len(*s.CustomKeyStoreId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("CustomKeyStoreId", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetCustomKeyStoreId sets the CustomKeyStoreId field's value. +func (s *DeleteCustomKeyStoreInput) SetCustomKeyStoreId(v string) *DeleteCustomKeyStoreInput { + s.CustomKeyStoreId = &v + return s +} + +type DeleteCustomKeyStoreOutput struct { + _ struct{} `type:"structure"` +} + +// String returns the string representation +func (s DeleteCustomKeyStoreOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s DeleteCustomKeyStoreOutput) GoString() string { + return s.String() +} + +type DeleteImportedKeyMaterialInput struct { + _ struct{} `type:"structure"` + + // Identifies the CMK from which you are deleting imported key material. The + // Origin of the CMK must be EXTERNAL. + // + // Specify the key ID or the Amazon Resource Name (ARN) of the CMK. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` +} + +// String returns the string representation +func (s DeleteImportedKeyMaterialInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s DeleteImportedKeyMaterialInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *DeleteImportedKeyMaterialInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "DeleteImportedKeyMaterialInput"} + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetKeyId sets the KeyId field's value. +func (s *DeleteImportedKeyMaterialInput) SetKeyId(v string) *DeleteImportedKeyMaterialInput { + s.KeyId = &v + return s +} + +type DeleteImportedKeyMaterialOutput struct { + _ struct{} `type:"structure"` +} + +// String returns the string representation +func (s DeleteImportedKeyMaterialOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s DeleteImportedKeyMaterialOutput) GoString() string { + return s.String() +} + +// The system timed out while trying to fulfill the request. The request can +// be retried. +type DependencyTimeoutException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s DependencyTimeoutException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s DependencyTimeoutException) GoString() string { + return s.String() +} + +func newErrorDependencyTimeoutException(v protocol.ResponseMetadata) error { + return &DependencyTimeoutException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *DependencyTimeoutException) Code() string { + return "DependencyTimeoutException" +} + +// Message returns the exception's message. +func (s *DependencyTimeoutException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *DependencyTimeoutException) OrigErr() error { + return nil +} + +func (s *DependencyTimeoutException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *DependencyTimeoutException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *DependencyTimeoutException) RequestID() string { + return s.RespMetadata.RequestID +} + +type DescribeCustomKeyStoresInput struct { + _ struct{} `type:"structure"` + + // Gets only information about the specified custom key store. Enter the key + // store ID. + // + // By default, this operation gets information about all custom key stores in + // the account and region. To limit the output to a particular custom key store, + // you can use either the CustomKeyStoreId or CustomKeyStoreName parameter, + // but not both. + CustomKeyStoreId *string `min:"1" type:"string"` + + // Gets only information about the specified custom key store. Enter the friendly + // name of the custom key store. + // + // By default, this operation gets information about all custom key stores in + // the account and region. To limit the output to a particular custom key store, + // you can use either the CustomKeyStoreId or CustomKeyStoreName parameter, + // but not both. + CustomKeyStoreName *string `min:"1" type:"string"` + + // Use this parameter to specify the maximum number of items to return. When + // this value is present, AWS KMS does not return more than the specified number + // of items, but it might return fewer. + Limit *int64 `min:"1" type:"integer"` + + // Use this parameter in a subsequent request after you receive a response with + // truncated results. Set it to the value of NextMarker from the truncated response + // you just received. + Marker *string `min:"1" type:"string"` +} + +// String returns the string representation +func (s DescribeCustomKeyStoresInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s DescribeCustomKeyStoresInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *DescribeCustomKeyStoresInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "DescribeCustomKeyStoresInput"} + if s.CustomKeyStoreId != nil && len(*s.CustomKeyStoreId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("CustomKeyStoreId", 1)) + } + if s.CustomKeyStoreName != nil && len(*s.CustomKeyStoreName) < 1 { + invalidParams.Add(request.NewErrParamMinLen("CustomKeyStoreName", 1)) + } + if s.Limit != nil && *s.Limit < 1 { + invalidParams.Add(request.NewErrParamMinValue("Limit", 1)) + } + if s.Marker != nil && len(*s.Marker) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Marker", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetCustomKeyStoreId sets the CustomKeyStoreId field's value. +func (s *DescribeCustomKeyStoresInput) SetCustomKeyStoreId(v string) *DescribeCustomKeyStoresInput { + s.CustomKeyStoreId = &v + return s +} + +// SetCustomKeyStoreName sets the CustomKeyStoreName field's value. +func (s *DescribeCustomKeyStoresInput) SetCustomKeyStoreName(v string) *DescribeCustomKeyStoresInput { + s.CustomKeyStoreName = &v + return s +} + +// SetLimit sets the Limit field's value. +func (s *DescribeCustomKeyStoresInput) SetLimit(v int64) *DescribeCustomKeyStoresInput { + s.Limit = &v + return s +} + +// SetMarker sets the Marker field's value. +func (s *DescribeCustomKeyStoresInput) SetMarker(v string) *DescribeCustomKeyStoresInput { + s.Marker = &v + return s +} + +type DescribeCustomKeyStoresOutput struct { + _ struct{} `type:"structure"` + + // Contains metadata about each custom key store. + CustomKeyStores []*CustomKeyStoresListEntry `type:"list"` + + // When Truncated is true, this element is present and contains the value to + // use for the Marker parameter in a subsequent request. + NextMarker *string `min:"1" type:"string"` + + // A flag that indicates whether there are more items in the list. When this + // value is true, the list in this response is truncated. To get more items, + // pass the value of the NextMarker element in thisresponse to the Marker parameter + // in a subsequent request. + Truncated *bool `type:"boolean"` +} + +// String returns the string representation +func (s DescribeCustomKeyStoresOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s DescribeCustomKeyStoresOutput) GoString() string { + return s.String() +} + +// SetCustomKeyStores sets the CustomKeyStores field's value. +func (s *DescribeCustomKeyStoresOutput) SetCustomKeyStores(v []*CustomKeyStoresListEntry) *DescribeCustomKeyStoresOutput { + s.CustomKeyStores = v + return s +} + +// SetNextMarker sets the NextMarker field's value. +func (s *DescribeCustomKeyStoresOutput) SetNextMarker(v string) *DescribeCustomKeyStoresOutput { + s.NextMarker = &v + return s +} + +// SetTruncated sets the Truncated field's value. +func (s *DescribeCustomKeyStoresOutput) SetTruncated(v bool) *DescribeCustomKeyStoresOutput { + s.Truncated = &v + return s +} + +type DescribeKeyInput struct { + _ struct{} `type:"structure"` + + // A list of grant tokens. + // + // For more information, see Grant Tokens (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#grant_token) + // in the AWS Key Management Service Developer Guide. + GrantTokens []*string `type:"list"` + + // Describes the specified customer master key (CMK). + // + // If you specify a predefined AWS alias (an AWS alias with no key ID), KMS + // associates the alias with an AWS managed CMK (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#master_keys) + // and returns its KeyId and Arn in the response. + // + // To specify a CMK, use its key ID, Amazon Resource Name (ARN), alias name, + // or alias ARN. When using an alias name, prefix it with "alias/". To specify + // a CMK in a different AWS account, you must use the key ARN or alias ARN. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Alias name: alias/ExampleAlias + // + // * Alias ARN: arn:aws:kms:us-east-2:111122223333:alias/ExampleAlias + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. To + // get the alias name and alias ARN, use ListAliases. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` +} + +// String returns the string representation +func (s DescribeKeyInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s DescribeKeyInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *DescribeKeyInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "DescribeKeyInput"} + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetGrantTokens sets the GrantTokens field's value. +func (s *DescribeKeyInput) SetGrantTokens(v []*string) *DescribeKeyInput { + s.GrantTokens = v + return s +} + +// SetKeyId sets the KeyId field's value. +func (s *DescribeKeyInput) SetKeyId(v string) *DescribeKeyInput { + s.KeyId = &v + return s +} + +type DescribeKeyOutput struct { + _ struct{} `type:"structure"` + + // Metadata associated with the key. + KeyMetadata *KeyMetadata `type:"structure"` +} + +// String returns the string representation +func (s DescribeKeyOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s DescribeKeyOutput) GoString() string { + return s.String() +} + +// SetKeyMetadata sets the KeyMetadata field's value. +func (s *DescribeKeyOutput) SetKeyMetadata(v *KeyMetadata) *DescribeKeyOutput { + s.KeyMetadata = v + return s +} + +type DisableKeyInput struct { + _ struct{} `type:"structure"` + + // A unique identifier for the customer master key (CMK). + // + // Specify the key ID or the Amazon Resource Name (ARN) of the CMK. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` +} + +// String returns the string representation +func (s DisableKeyInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s DisableKeyInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *DisableKeyInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "DisableKeyInput"} + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetKeyId sets the KeyId field's value. +func (s *DisableKeyInput) SetKeyId(v string) *DisableKeyInput { + s.KeyId = &v + return s +} + +type DisableKeyOutput struct { + _ struct{} `type:"structure"` +} + +// String returns the string representation +func (s DisableKeyOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s DisableKeyOutput) GoString() string { + return s.String() +} + +type DisableKeyRotationInput struct { + _ struct{} `type:"structure"` + + // Identifies a symmetric customer master key (CMK). You cannot enable automatic + // rotation of asymmetric CMKs (https://docs.aws.amazon.com/kms/latest/developerguide/symmetric-asymmetric.html#asymmetric-cmks), + // CMKs with imported key material (https://docs.aws.amazon.com/kms/latest/developerguide/importing-keys.html), + // or CMKs in a custom key store (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html). + // + // Specify the key ID or the Amazon Resource Name (ARN) of the CMK. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` +} + +// String returns the string representation +func (s DisableKeyRotationInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s DisableKeyRotationInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *DisableKeyRotationInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "DisableKeyRotationInput"} + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetKeyId sets the KeyId field's value. +func (s *DisableKeyRotationInput) SetKeyId(v string) *DisableKeyRotationInput { + s.KeyId = &v + return s +} + +type DisableKeyRotationOutput struct { + _ struct{} `type:"structure"` +} + +// String returns the string representation +func (s DisableKeyRotationOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s DisableKeyRotationOutput) GoString() string { + return s.String() +} + +// The request was rejected because the specified CMK is not enabled. +type DisabledException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s DisabledException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s DisabledException) GoString() string { + return s.String() +} + +func newErrorDisabledException(v protocol.ResponseMetadata) error { + return &DisabledException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *DisabledException) Code() string { + return "DisabledException" +} + +// Message returns the exception's message. +func (s *DisabledException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *DisabledException) OrigErr() error { + return nil +} + +func (s *DisabledException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *DisabledException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *DisabledException) RequestID() string { + return s.RespMetadata.RequestID +} + +type DisconnectCustomKeyStoreInput struct { + _ struct{} `type:"structure"` + + // Enter the ID of the custom key store you want to disconnect. To find the + // ID of a custom key store, use the DescribeCustomKeyStores operation. + // + // CustomKeyStoreId is a required field + CustomKeyStoreId *string `min:"1" type:"string" required:"true"` +} + +// String returns the string representation +func (s DisconnectCustomKeyStoreInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s DisconnectCustomKeyStoreInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *DisconnectCustomKeyStoreInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "DisconnectCustomKeyStoreInput"} + if s.CustomKeyStoreId == nil { + invalidParams.Add(request.NewErrParamRequired("CustomKeyStoreId")) + } + if s.CustomKeyStoreId != nil && len(*s.CustomKeyStoreId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("CustomKeyStoreId", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetCustomKeyStoreId sets the CustomKeyStoreId field's value. +func (s *DisconnectCustomKeyStoreInput) SetCustomKeyStoreId(v string) *DisconnectCustomKeyStoreInput { + s.CustomKeyStoreId = &v + return s +} + +type DisconnectCustomKeyStoreOutput struct { + _ struct{} `type:"structure"` +} + +// String returns the string representation +func (s DisconnectCustomKeyStoreOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s DisconnectCustomKeyStoreOutput) GoString() string { + return s.String() +} + +type EnableKeyInput struct { + _ struct{} `type:"structure"` + + // A unique identifier for the customer master key (CMK). + // + // Specify the key ID or the Amazon Resource Name (ARN) of the CMK. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` +} + +// String returns the string representation +func (s EnableKeyInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s EnableKeyInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *EnableKeyInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "EnableKeyInput"} + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetKeyId sets the KeyId field's value. +func (s *EnableKeyInput) SetKeyId(v string) *EnableKeyInput { + s.KeyId = &v + return s +} + +type EnableKeyOutput struct { + _ struct{} `type:"structure"` +} + +// String returns the string representation +func (s EnableKeyOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s EnableKeyOutput) GoString() string { + return s.String() +} + +type EnableKeyRotationInput struct { + _ struct{} `type:"structure"` + + // Identifies a symmetric customer master key (CMK). You cannot enable automatic + // rotation of asymmetric CMKs, CMKs with imported key material, or CMKs in + // a custom key store (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html). + // + // Specify the key ID or the Amazon Resource Name (ARN) of the CMK. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` +} + +// String returns the string representation +func (s EnableKeyRotationInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s EnableKeyRotationInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *EnableKeyRotationInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "EnableKeyRotationInput"} + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetKeyId sets the KeyId field's value. +func (s *EnableKeyRotationInput) SetKeyId(v string) *EnableKeyRotationInput { + s.KeyId = &v + return s +} + +type EnableKeyRotationOutput struct { + _ struct{} `type:"structure"` +} + +// String returns the string representation +func (s EnableKeyRotationOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s EnableKeyRotationOutput) GoString() string { + return s.String() +} + +type EncryptInput struct { + _ struct{} `type:"structure"` + + // Specifies the encryption algorithm that AWS KMS will use to encrypt the plaintext + // message. The algorithm must be compatible with the CMK that you specify. + // + // This parameter is required only for asymmetric CMKs. The default value, SYMMETRIC_DEFAULT, + // is the algorithm used for symmetric CMKs. If you are using an asymmetric + // CMK, we recommend RSAES_OAEP_SHA_256. + EncryptionAlgorithm *string `type:"string" enum:"EncryptionAlgorithmSpec"` + + // Specifies the encryption context that will be used to encrypt the data. An + // encryption context is valid only for cryptographic operations (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#cryptographic-operations) + // with a symmetric CMK. The standard asymmetric encryption algorithms that + // AWS KMS uses do not support an encryption context. + // + // An encryption context is a collection of non-secret key-value pairs that + // represents additional authenticated data. When you use an encryption context + // to encrypt data, you must specify the same (an exact case-sensitive match) + // encryption context to decrypt the data. An encryption context is optional + // when encrypting with a symmetric CMK, but it is highly recommended. + // + // For more information, see Encryption Context (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#encrypt_context) + // in the AWS Key Management Service Developer Guide. + EncryptionContext map[string]*string `type:"map"` + + // A list of grant tokens. + // + // For more information, see Grant Tokens (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#grant_token) + // in the AWS Key Management Service Developer Guide. + GrantTokens []*string `type:"list"` + + // A unique identifier for the customer master key (CMK). + // + // To specify a CMK, use its key ID, Amazon Resource Name (ARN), alias name, + // or alias ARN. When using an alias name, prefix it with "alias/". To specify + // a CMK in a different AWS account, you must use the key ARN or alias ARN. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Alias name: alias/ExampleAlias + // + // * Alias ARN: arn:aws:kms:us-east-2:111122223333:alias/ExampleAlias + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. To + // get the alias name and alias ARN, use ListAliases. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` + + // Data to be encrypted. + // + // Plaintext is automatically base64 encoded/decoded by the SDK. + // + // Plaintext is a required field + Plaintext []byte `min:"1" type:"blob" required:"true" sensitive:"true"` +} + +// String returns the string representation +func (s EncryptInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s EncryptInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *EncryptInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "EncryptInput"} + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + if s.Plaintext == nil { + invalidParams.Add(request.NewErrParamRequired("Plaintext")) + } + if s.Plaintext != nil && len(s.Plaintext) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Plaintext", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetEncryptionAlgorithm sets the EncryptionAlgorithm field's value. +func (s *EncryptInput) SetEncryptionAlgorithm(v string) *EncryptInput { + s.EncryptionAlgorithm = &v + return s +} + +// SetEncryptionContext sets the EncryptionContext field's value. +func (s *EncryptInput) SetEncryptionContext(v map[string]*string) *EncryptInput { + s.EncryptionContext = v + return s +} + +// SetGrantTokens sets the GrantTokens field's value. +func (s *EncryptInput) SetGrantTokens(v []*string) *EncryptInput { + s.GrantTokens = v + return s +} + +// SetKeyId sets the KeyId field's value. +func (s *EncryptInput) SetKeyId(v string) *EncryptInput { + s.KeyId = &v + return s +} + +// SetPlaintext sets the Plaintext field's value. +func (s *EncryptInput) SetPlaintext(v []byte) *EncryptInput { + s.Plaintext = v + return s +} + +type EncryptOutput struct { + _ struct{} `type:"structure"` + + // The encrypted plaintext. When you use the HTTP API or the AWS CLI, the value + // is Base64-encoded. Otherwise, it is not Base64-encoded. + // + // CiphertextBlob is automatically base64 encoded/decoded by the SDK. + CiphertextBlob []byte `min:"1" type:"blob"` + + // The encryption algorithm that was used to encrypt the plaintext. + EncryptionAlgorithm *string `type:"string" enum:"EncryptionAlgorithmSpec"` + + // The Amazon Resource Name (key ARN (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#key-id-key-ARN)) + // of the CMK that was used to encrypt the plaintext. + KeyId *string `min:"1" type:"string"` +} + +// String returns the string representation +func (s EncryptOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s EncryptOutput) GoString() string { + return s.String() +} + +// SetCiphertextBlob sets the CiphertextBlob field's value. +func (s *EncryptOutput) SetCiphertextBlob(v []byte) *EncryptOutput { + s.CiphertextBlob = v + return s +} + +// SetEncryptionAlgorithm sets the EncryptionAlgorithm field's value. +func (s *EncryptOutput) SetEncryptionAlgorithm(v string) *EncryptOutput { + s.EncryptionAlgorithm = &v + return s +} + +// SetKeyId sets the KeyId field's value. +func (s *EncryptOutput) SetKeyId(v string) *EncryptOutput { + s.KeyId = &v + return s +} + +// The request was rejected because the specified import token is expired. Use +// GetParametersForImport to get a new import token and public key, use the +// new public key to encrypt the key material, and then try the request again. +type ExpiredImportTokenException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s ExpiredImportTokenException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s ExpiredImportTokenException) GoString() string { + return s.String() +} + +func newErrorExpiredImportTokenException(v protocol.ResponseMetadata) error { + return &ExpiredImportTokenException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *ExpiredImportTokenException) Code() string { + return "ExpiredImportTokenException" +} + +// Message returns the exception's message. +func (s *ExpiredImportTokenException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *ExpiredImportTokenException) OrigErr() error { + return nil +} + +func (s *ExpiredImportTokenException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *ExpiredImportTokenException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *ExpiredImportTokenException) RequestID() string { + return s.RespMetadata.RequestID +} + +type GenerateDataKeyInput struct { + _ struct{} `type:"structure"` + + // Specifies the encryption context that will be used when encrypting the data + // key. + // + // An encryption context is a collection of non-secret key-value pairs that + // represents additional authenticated data. When you use an encryption context + // to encrypt data, you must specify the same (an exact case-sensitive match) + // encryption context to decrypt the data. An encryption context is optional + // when encrypting with a symmetric CMK, but it is highly recommended. + // + // For more information, see Encryption Context (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#encrypt_context) + // in the AWS Key Management Service Developer Guide. + EncryptionContext map[string]*string `type:"map"` + + // A list of grant tokens. + // + // For more information, see Grant Tokens (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#grant_token) + // in the AWS Key Management Service Developer Guide. + GrantTokens []*string `type:"list"` + + // Identifies the symmetric CMK that encrypts the data key. + // + // To specify a CMK, use its key ID, Amazon Resource Name (ARN), alias name, + // or alias ARN. When using an alias name, prefix it with "alias/". To specify + // a CMK in a different AWS account, you must use the key ARN or alias ARN. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Alias name: alias/ExampleAlias + // + // * Alias ARN: arn:aws:kms:us-east-2:111122223333:alias/ExampleAlias + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. To + // get the alias name and alias ARN, use ListAliases. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` + + // Specifies the length of the data key. Use AES_128 to generate a 128-bit symmetric + // key, or AES_256 to generate a 256-bit symmetric key. + // + // You must specify either the KeySpec or the NumberOfBytes parameter (but not + // both) in every GenerateDataKey request. + KeySpec *string `type:"string" enum:"DataKeySpec"` + + // Specifies the length of the data key in bytes. For example, use the value + // 64 to generate a 512-bit data key (64 bytes is 512 bits). For 128-bit (16-byte) + // and 256-bit (32-byte) data keys, use the KeySpec parameter. + // + // You must specify either the KeySpec or the NumberOfBytes parameter (but not + // both) in every GenerateDataKey request. + NumberOfBytes *int64 `min:"1" type:"integer"` +} + +// String returns the string representation +func (s GenerateDataKeyInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s GenerateDataKeyInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *GenerateDataKeyInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "GenerateDataKeyInput"} + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + if s.NumberOfBytes != nil && *s.NumberOfBytes < 1 { + invalidParams.Add(request.NewErrParamMinValue("NumberOfBytes", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetEncryptionContext sets the EncryptionContext field's value. +func (s *GenerateDataKeyInput) SetEncryptionContext(v map[string]*string) *GenerateDataKeyInput { + s.EncryptionContext = v + return s +} + +// SetGrantTokens sets the GrantTokens field's value. +func (s *GenerateDataKeyInput) SetGrantTokens(v []*string) *GenerateDataKeyInput { + s.GrantTokens = v + return s +} + +// SetKeyId sets the KeyId field's value. +func (s *GenerateDataKeyInput) SetKeyId(v string) *GenerateDataKeyInput { + s.KeyId = &v + return s +} + +// SetKeySpec sets the KeySpec field's value. +func (s *GenerateDataKeyInput) SetKeySpec(v string) *GenerateDataKeyInput { + s.KeySpec = &v + return s +} + +// SetNumberOfBytes sets the NumberOfBytes field's value. +func (s *GenerateDataKeyInput) SetNumberOfBytes(v int64) *GenerateDataKeyInput { + s.NumberOfBytes = &v + return s +} + +type GenerateDataKeyOutput struct { + _ struct{} `type:"structure"` + + // The encrypted copy of the data key. When you use the HTTP API or the AWS + // CLI, the value is Base64-encoded. Otherwise, it is not Base64-encoded. + // + // CiphertextBlob is automatically base64 encoded/decoded by the SDK. + CiphertextBlob []byte `min:"1" type:"blob"` + + // The Amazon Resource Name (key ARN (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#key-id-key-ARN)) + // of the CMK that encrypted the data key. + KeyId *string `min:"1" type:"string"` + + // The plaintext data key. When you use the HTTP API or the AWS CLI, the value + // is Base64-encoded. Otherwise, it is not Base64-encoded. Use this data key + // to encrypt your data outside of KMS. Then, remove it from memory as soon + // as possible. + // + // Plaintext is automatically base64 encoded/decoded by the SDK. + Plaintext []byte `min:"1" type:"blob" sensitive:"true"` +} + +// String returns the string representation +func (s GenerateDataKeyOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s GenerateDataKeyOutput) GoString() string { + return s.String() +} + +// SetCiphertextBlob sets the CiphertextBlob field's value. +func (s *GenerateDataKeyOutput) SetCiphertextBlob(v []byte) *GenerateDataKeyOutput { + s.CiphertextBlob = v + return s +} + +// SetKeyId sets the KeyId field's value. +func (s *GenerateDataKeyOutput) SetKeyId(v string) *GenerateDataKeyOutput { + s.KeyId = &v + return s +} + +// SetPlaintext sets the Plaintext field's value. +func (s *GenerateDataKeyOutput) SetPlaintext(v []byte) *GenerateDataKeyOutput { + s.Plaintext = v + return s +} + +type GenerateDataKeyPairInput struct { + _ struct{} `type:"structure"` + + // Specifies the encryption context that will be used when encrypting the private + // key in the data key pair. + // + // An encryption context is a collection of non-secret key-value pairs that + // represents additional authenticated data. When you use an encryption context + // to encrypt data, you must specify the same (an exact case-sensitive match) + // encryption context to decrypt the data. An encryption context is optional + // when encrypting with a symmetric CMK, but it is highly recommended. + // + // For more information, see Encryption Context (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#encrypt_context) + // in the AWS Key Management Service Developer Guide. + EncryptionContext map[string]*string `type:"map"` + + // A list of grant tokens. + // + // For more information, see Grant Tokens (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#grant_token) + // in the AWS Key Management Service Developer Guide. + GrantTokens []*string `type:"list"` + + // Specifies the symmetric CMK that encrypts the private key in the data key + // pair. You cannot specify an asymmetric CMK or a CMK in a custom key store. + // To get the type and origin of your CMK, use the DescribeKey operation. + // + // To specify a CMK, use its key ID, Amazon Resource Name (ARN), alias name, + // or alias ARN. When using an alias name, prefix it with "alias/". To specify + // a CMK in a different AWS account, you must use the key ARN or alias ARN. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Alias name: alias/ExampleAlias + // + // * Alias ARN: arn:aws:kms:us-east-2:111122223333:alias/ExampleAlias + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. To + // get the alias name and alias ARN, use ListAliases. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` + + // Determines the type of data key pair that is generated. + // + // The AWS KMS rule that restricts the use of asymmetric RSA CMKs to encrypt + // and decrypt or to sign and verify (but not both), and the rule that permits + // you to use ECC CMKs only to sign and verify, are not effective outside of + // AWS KMS. + // + // KeyPairSpec is a required field + KeyPairSpec *string `type:"string" required:"true" enum:"DataKeyPairSpec"` +} + +// String returns the string representation +func (s GenerateDataKeyPairInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s GenerateDataKeyPairInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *GenerateDataKeyPairInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "GenerateDataKeyPairInput"} + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + if s.KeyPairSpec == nil { + invalidParams.Add(request.NewErrParamRequired("KeyPairSpec")) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetEncryptionContext sets the EncryptionContext field's value. +func (s *GenerateDataKeyPairInput) SetEncryptionContext(v map[string]*string) *GenerateDataKeyPairInput { + s.EncryptionContext = v + return s +} + +// SetGrantTokens sets the GrantTokens field's value. +func (s *GenerateDataKeyPairInput) SetGrantTokens(v []*string) *GenerateDataKeyPairInput { + s.GrantTokens = v + return s +} + +// SetKeyId sets the KeyId field's value. +func (s *GenerateDataKeyPairInput) SetKeyId(v string) *GenerateDataKeyPairInput { + s.KeyId = &v + return s +} + +// SetKeyPairSpec sets the KeyPairSpec field's value. +func (s *GenerateDataKeyPairInput) SetKeyPairSpec(v string) *GenerateDataKeyPairInput { + s.KeyPairSpec = &v + return s +} + +type GenerateDataKeyPairOutput struct { + _ struct{} `type:"structure"` + + // The Amazon Resource Name (key ARN (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#key-id-key-ARN)) + // of the CMK that encrypted the private key. + KeyId *string `min:"1" type:"string"` + + // The type of data key pair that was generated. + KeyPairSpec *string `type:"string" enum:"DataKeyPairSpec"` + + // The encrypted copy of the private key. When you use the HTTP API or the AWS + // CLI, the value is Base64-encoded. Otherwise, it is not Base64-encoded. + // + // PrivateKeyCiphertextBlob is automatically base64 encoded/decoded by the SDK. + PrivateKeyCiphertextBlob []byte `min:"1" type:"blob"` + + // The plaintext copy of the private key. When you use the HTTP API or the AWS + // CLI, the value is Base64-encoded. Otherwise, it is not Base64-encoded. + // + // PrivateKeyPlaintext is automatically base64 encoded/decoded by the SDK. + PrivateKeyPlaintext []byte `min:"1" type:"blob" sensitive:"true"` + + // The public key (in plaintext). + // + // PublicKey is automatically base64 encoded/decoded by the SDK. + PublicKey []byte `min:"1" type:"blob"` +} + +// String returns the string representation +func (s GenerateDataKeyPairOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s GenerateDataKeyPairOutput) GoString() string { + return s.String() +} + +// SetKeyId sets the KeyId field's value. +func (s *GenerateDataKeyPairOutput) SetKeyId(v string) *GenerateDataKeyPairOutput { + s.KeyId = &v + return s +} + +// SetKeyPairSpec sets the KeyPairSpec field's value. +func (s *GenerateDataKeyPairOutput) SetKeyPairSpec(v string) *GenerateDataKeyPairOutput { + s.KeyPairSpec = &v + return s +} + +// SetPrivateKeyCiphertextBlob sets the PrivateKeyCiphertextBlob field's value. +func (s *GenerateDataKeyPairOutput) SetPrivateKeyCiphertextBlob(v []byte) *GenerateDataKeyPairOutput { + s.PrivateKeyCiphertextBlob = v + return s +} + +// SetPrivateKeyPlaintext sets the PrivateKeyPlaintext field's value. +func (s *GenerateDataKeyPairOutput) SetPrivateKeyPlaintext(v []byte) *GenerateDataKeyPairOutput { + s.PrivateKeyPlaintext = v + return s +} + +// SetPublicKey sets the PublicKey field's value. +func (s *GenerateDataKeyPairOutput) SetPublicKey(v []byte) *GenerateDataKeyPairOutput { + s.PublicKey = v + return s +} + +type GenerateDataKeyPairWithoutPlaintextInput struct { + _ struct{} `type:"structure"` + + // Specifies the encryption context that will be used when encrypting the private + // key in the data key pair. + // + // An encryption context is a collection of non-secret key-value pairs that + // represents additional authenticated data. When you use an encryption context + // to encrypt data, you must specify the same (an exact case-sensitive match) + // encryption context to decrypt the data. An encryption context is optional + // when encrypting with a symmetric CMK, but it is highly recommended. + // + // For more information, see Encryption Context (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#encrypt_context) + // in the AWS Key Management Service Developer Guide. + EncryptionContext map[string]*string `type:"map"` + + // A list of grant tokens. + // + // For more information, see Grant Tokens (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#grant_token) + // in the AWS Key Management Service Developer Guide. + GrantTokens []*string `type:"list"` + + // Specifies the CMK that encrypts the private key in the data key pair. You + // must specify a symmetric CMK. You cannot use an asymmetric CMK or a CMK in + // a custom key store. To get the type and origin of your CMK, use the DescribeKey + // operation. + // + // To specify a CMK, use its key ID, Amazon Resource Name (ARN), alias name, + // or alias ARN. When using an alias name, prefix it with "alias/". + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Alias name: alias/ExampleAlias + // + // * Alias ARN: arn:aws:kms:us-east-2:111122223333:alias/ExampleAlias + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. To + // get the alias name and alias ARN, use ListAliases. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` + + // Determines the type of data key pair that is generated. + // + // The AWS KMS rule that restricts the use of asymmetric RSA CMKs to encrypt + // and decrypt or to sign and verify (but not both), and the rule that permits + // you to use ECC CMKs only to sign and verify, are not effective outside of + // AWS KMS. + // + // KeyPairSpec is a required field + KeyPairSpec *string `type:"string" required:"true" enum:"DataKeyPairSpec"` +} + +// String returns the string representation +func (s GenerateDataKeyPairWithoutPlaintextInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s GenerateDataKeyPairWithoutPlaintextInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *GenerateDataKeyPairWithoutPlaintextInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "GenerateDataKeyPairWithoutPlaintextInput"} + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + if s.KeyPairSpec == nil { + invalidParams.Add(request.NewErrParamRequired("KeyPairSpec")) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetEncryptionContext sets the EncryptionContext field's value. +func (s *GenerateDataKeyPairWithoutPlaintextInput) SetEncryptionContext(v map[string]*string) *GenerateDataKeyPairWithoutPlaintextInput { + s.EncryptionContext = v + return s +} + +// SetGrantTokens sets the GrantTokens field's value. +func (s *GenerateDataKeyPairWithoutPlaintextInput) SetGrantTokens(v []*string) *GenerateDataKeyPairWithoutPlaintextInput { + s.GrantTokens = v + return s +} + +// SetKeyId sets the KeyId field's value. +func (s *GenerateDataKeyPairWithoutPlaintextInput) SetKeyId(v string) *GenerateDataKeyPairWithoutPlaintextInput { + s.KeyId = &v + return s +} + +// SetKeyPairSpec sets the KeyPairSpec field's value. +func (s *GenerateDataKeyPairWithoutPlaintextInput) SetKeyPairSpec(v string) *GenerateDataKeyPairWithoutPlaintextInput { + s.KeyPairSpec = &v + return s +} + +type GenerateDataKeyPairWithoutPlaintextOutput struct { + _ struct{} `type:"structure"` + + // The Amazon Resource Name (key ARN (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#key-id-key-ARN)) + // of the CMK that encrypted the private key. + KeyId *string `min:"1" type:"string"` + + // The type of data key pair that was generated. + KeyPairSpec *string `type:"string" enum:"DataKeyPairSpec"` + + // The encrypted copy of the private key. When you use the HTTP API or the AWS + // CLI, the value is Base64-encoded. Otherwise, it is not Base64-encoded. + // + // PrivateKeyCiphertextBlob is automatically base64 encoded/decoded by the SDK. + PrivateKeyCiphertextBlob []byte `min:"1" type:"blob"` + + // The public key (in plaintext). + // + // PublicKey is automatically base64 encoded/decoded by the SDK. + PublicKey []byte `min:"1" type:"blob"` +} + +// String returns the string representation +func (s GenerateDataKeyPairWithoutPlaintextOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s GenerateDataKeyPairWithoutPlaintextOutput) GoString() string { + return s.String() +} + +// SetKeyId sets the KeyId field's value. +func (s *GenerateDataKeyPairWithoutPlaintextOutput) SetKeyId(v string) *GenerateDataKeyPairWithoutPlaintextOutput { + s.KeyId = &v + return s +} + +// SetKeyPairSpec sets the KeyPairSpec field's value. +func (s *GenerateDataKeyPairWithoutPlaintextOutput) SetKeyPairSpec(v string) *GenerateDataKeyPairWithoutPlaintextOutput { + s.KeyPairSpec = &v + return s +} + +// SetPrivateKeyCiphertextBlob sets the PrivateKeyCiphertextBlob field's value. +func (s *GenerateDataKeyPairWithoutPlaintextOutput) SetPrivateKeyCiphertextBlob(v []byte) *GenerateDataKeyPairWithoutPlaintextOutput { + s.PrivateKeyCiphertextBlob = v + return s +} + +// SetPublicKey sets the PublicKey field's value. +func (s *GenerateDataKeyPairWithoutPlaintextOutput) SetPublicKey(v []byte) *GenerateDataKeyPairWithoutPlaintextOutput { + s.PublicKey = v + return s +} + +type GenerateDataKeyWithoutPlaintextInput struct { + _ struct{} `type:"structure"` + + // Specifies the encryption context that will be used when encrypting the data + // key. + // + // An encryption context is a collection of non-secret key-value pairs that + // represents additional authenticated data. When you use an encryption context + // to encrypt data, you must specify the same (an exact case-sensitive match) + // encryption context to decrypt the data. An encryption context is optional + // when encrypting with a symmetric CMK, but it is highly recommended. + // + // For more information, see Encryption Context (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#encrypt_context) + // in the AWS Key Management Service Developer Guide. + EncryptionContext map[string]*string `type:"map"` + + // A list of grant tokens. + // + // For more information, see Grant Tokens (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#grant_token) + // in the AWS Key Management Service Developer Guide. + GrantTokens []*string `type:"list"` + + // The identifier of the symmetric customer master key (CMK) that encrypts the + // data key. + // + // To specify a CMK, use its key ID, Amazon Resource Name (ARN), alias name, + // or alias ARN. When using an alias name, prefix it with "alias/". To specify + // a CMK in a different AWS account, you must use the key ARN or alias ARN. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Alias name: alias/ExampleAlias + // + // * Alias ARN: arn:aws:kms:us-east-2:111122223333:alias/ExampleAlias + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. To + // get the alias name and alias ARN, use ListAliases. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` + + // The length of the data key. Use AES_128 to generate a 128-bit symmetric key, + // or AES_256 to generate a 256-bit symmetric key. + KeySpec *string `type:"string" enum:"DataKeySpec"` + + // The length of the data key in bytes. For example, use the value 64 to generate + // a 512-bit data key (64 bytes is 512 bits). For common key lengths (128-bit + // and 256-bit symmetric keys), we recommend that you use the KeySpec field + // instead of this one. + NumberOfBytes *int64 `min:"1" type:"integer"` +} + +// String returns the string representation +func (s GenerateDataKeyWithoutPlaintextInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s GenerateDataKeyWithoutPlaintextInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *GenerateDataKeyWithoutPlaintextInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "GenerateDataKeyWithoutPlaintextInput"} + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + if s.NumberOfBytes != nil && *s.NumberOfBytes < 1 { + invalidParams.Add(request.NewErrParamMinValue("NumberOfBytes", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetEncryptionContext sets the EncryptionContext field's value. +func (s *GenerateDataKeyWithoutPlaintextInput) SetEncryptionContext(v map[string]*string) *GenerateDataKeyWithoutPlaintextInput { + s.EncryptionContext = v + return s +} + +// SetGrantTokens sets the GrantTokens field's value. +func (s *GenerateDataKeyWithoutPlaintextInput) SetGrantTokens(v []*string) *GenerateDataKeyWithoutPlaintextInput { + s.GrantTokens = v + return s +} + +// SetKeyId sets the KeyId field's value. +func (s *GenerateDataKeyWithoutPlaintextInput) SetKeyId(v string) *GenerateDataKeyWithoutPlaintextInput { + s.KeyId = &v + return s +} + +// SetKeySpec sets the KeySpec field's value. +func (s *GenerateDataKeyWithoutPlaintextInput) SetKeySpec(v string) *GenerateDataKeyWithoutPlaintextInput { + s.KeySpec = &v + return s +} + +// SetNumberOfBytes sets the NumberOfBytes field's value. +func (s *GenerateDataKeyWithoutPlaintextInput) SetNumberOfBytes(v int64) *GenerateDataKeyWithoutPlaintextInput { + s.NumberOfBytes = &v + return s +} + +type GenerateDataKeyWithoutPlaintextOutput struct { + _ struct{} `type:"structure"` + + // The encrypted data key. When you use the HTTP API or the AWS CLI, the value + // is Base64-encoded. Otherwise, it is not Base64-encoded. + // + // CiphertextBlob is automatically base64 encoded/decoded by the SDK. + CiphertextBlob []byte `min:"1" type:"blob"` + + // The Amazon Resource Name (key ARN (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#key-id-key-ARN)) + // of the CMK that encrypted the data key. + KeyId *string `min:"1" type:"string"` +} + +// String returns the string representation +func (s GenerateDataKeyWithoutPlaintextOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s GenerateDataKeyWithoutPlaintextOutput) GoString() string { + return s.String() +} + +// SetCiphertextBlob sets the CiphertextBlob field's value. +func (s *GenerateDataKeyWithoutPlaintextOutput) SetCiphertextBlob(v []byte) *GenerateDataKeyWithoutPlaintextOutput { + s.CiphertextBlob = v + return s +} + +// SetKeyId sets the KeyId field's value. +func (s *GenerateDataKeyWithoutPlaintextOutput) SetKeyId(v string) *GenerateDataKeyWithoutPlaintextOutput { + s.KeyId = &v + return s +} + +type GenerateRandomInput struct { + _ struct{} `type:"structure"` + + // Generates the random byte string in the AWS CloudHSM cluster that is associated + // with the specified custom key store (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html). + // To find the ID of a custom key store, use the DescribeCustomKeyStores operation. + CustomKeyStoreId *string `min:"1" type:"string"` + + // The length of the byte string. + NumberOfBytes *int64 `min:"1" type:"integer"` +} + +// String returns the string representation +func (s GenerateRandomInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s GenerateRandomInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *GenerateRandomInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "GenerateRandomInput"} + if s.CustomKeyStoreId != nil && len(*s.CustomKeyStoreId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("CustomKeyStoreId", 1)) + } + if s.NumberOfBytes != nil && *s.NumberOfBytes < 1 { + invalidParams.Add(request.NewErrParamMinValue("NumberOfBytes", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetCustomKeyStoreId sets the CustomKeyStoreId field's value. +func (s *GenerateRandomInput) SetCustomKeyStoreId(v string) *GenerateRandomInput { + s.CustomKeyStoreId = &v + return s +} + +// SetNumberOfBytes sets the NumberOfBytes field's value. +func (s *GenerateRandomInput) SetNumberOfBytes(v int64) *GenerateRandomInput { + s.NumberOfBytes = &v + return s +} + +type GenerateRandomOutput struct { + _ struct{} `type:"structure"` + + // The random byte string. When you use the HTTP API or the AWS CLI, the value + // is Base64-encoded. Otherwise, it is not Base64-encoded. + // + // Plaintext is automatically base64 encoded/decoded by the SDK. + Plaintext []byte `min:"1" type:"blob" sensitive:"true"` +} + +// String returns the string representation +func (s GenerateRandomOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s GenerateRandomOutput) GoString() string { + return s.String() +} + +// SetPlaintext sets the Plaintext field's value. +func (s *GenerateRandomOutput) SetPlaintext(v []byte) *GenerateRandomOutput { + s.Plaintext = v + return s +} + +type GetKeyPolicyInput struct { + _ struct{} `type:"structure"` + + // A unique identifier for the customer master key (CMK). + // + // Specify the key ID or the Amazon Resource Name (ARN) of the CMK. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` + + // Specifies the name of the key policy. The only valid name is default. To + // get the names of key policies, use ListKeyPolicies. + // + // PolicyName is a required field + PolicyName *string `min:"1" type:"string" required:"true"` +} + +// String returns the string representation +func (s GetKeyPolicyInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s GetKeyPolicyInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *GetKeyPolicyInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "GetKeyPolicyInput"} + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + if s.PolicyName == nil { + invalidParams.Add(request.NewErrParamRequired("PolicyName")) + } + if s.PolicyName != nil && len(*s.PolicyName) < 1 { + invalidParams.Add(request.NewErrParamMinLen("PolicyName", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetKeyId sets the KeyId field's value. +func (s *GetKeyPolicyInput) SetKeyId(v string) *GetKeyPolicyInput { + s.KeyId = &v + return s +} + +// SetPolicyName sets the PolicyName field's value. +func (s *GetKeyPolicyInput) SetPolicyName(v string) *GetKeyPolicyInput { + s.PolicyName = &v + return s +} + +type GetKeyPolicyOutput struct { + _ struct{} `type:"structure"` + + // A key policy document in JSON format. + Policy *string `min:"1" type:"string"` +} + +// String returns the string representation +func (s GetKeyPolicyOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s GetKeyPolicyOutput) GoString() string { + return s.String() +} + +// SetPolicy sets the Policy field's value. +func (s *GetKeyPolicyOutput) SetPolicy(v string) *GetKeyPolicyOutput { + s.Policy = &v + return s +} + +type GetKeyRotationStatusInput struct { + _ struct{} `type:"structure"` + + // A unique identifier for the customer master key (CMK). + // + // Specify the key ID or the Amazon Resource Name (ARN) of the CMK. To specify + // a CMK in a different AWS account, you must use the key ARN. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` +} + +// String returns the string representation +func (s GetKeyRotationStatusInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s GetKeyRotationStatusInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *GetKeyRotationStatusInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "GetKeyRotationStatusInput"} + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetKeyId sets the KeyId field's value. +func (s *GetKeyRotationStatusInput) SetKeyId(v string) *GetKeyRotationStatusInput { + s.KeyId = &v + return s +} + +type GetKeyRotationStatusOutput struct { + _ struct{} `type:"structure"` + + // A Boolean value that specifies whether key rotation is enabled. + KeyRotationEnabled *bool `type:"boolean"` +} + +// String returns the string representation +func (s GetKeyRotationStatusOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s GetKeyRotationStatusOutput) GoString() string { + return s.String() +} + +// SetKeyRotationEnabled sets the KeyRotationEnabled field's value. +func (s *GetKeyRotationStatusOutput) SetKeyRotationEnabled(v bool) *GetKeyRotationStatusOutput { + s.KeyRotationEnabled = &v + return s +} + +type GetParametersForImportInput struct { + _ struct{} `type:"structure"` + + // The identifier of the symmetric CMK into which you will import key material. + // The Origin of the CMK must be EXTERNAL. + // + // Specify the key ID or the Amazon Resource Name (ARN) of the CMK. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` + + // The algorithm you will use to encrypt the key material before importing it + // with ImportKeyMaterial. For more information, see Encrypt the Key Material + // (https://docs.aws.amazon.com/kms/latest/developerguide/importing-keys-encrypt-key-material.html) + // in the AWS Key Management Service Developer Guide. + // + // WrappingAlgorithm is a required field + WrappingAlgorithm *string `type:"string" required:"true" enum:"AlgorithmSpec"` + + // The type of wrapping key (public key) to return in the response. Only 2048-bit + // RSA public keys are supported. + // + // WrappingKeySpec is a required field + WrappingKeySpec *string `type:"string" required:"true" enum:"WrappingKeySpec"` +} + +// String returns the string representation +func (s GetParametersForImportInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s GetParametersForImportInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *GetParametersForImportInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "GetParametersForImportInput"} + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + if s.WrappingAlgorithm == nil { + invalidParams.Add(request.NewErrParamRequired("WrappingAlgorithm")) + } + if s.WrappingKeySpec == nil { + invalidParams.Add(request.NewErrParamRequired("WrappingKeySpec")) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetKeyId sets the KeyId field's value. +func (s *GetParametersForImportInput) SetKeyId(v string) *GetParametersForImportInput { + s.KeyId = &v + return s +} + +// SetWrappingAlgorithm sets the WrappingAlgorithm field's value. +func (s *GetParametersForImportInput) SetWrappingAlgorithm(v string) *GetParametersForImportInput { + s.WrappingAlgorithm = &v + return s +} + +// SetWrappingKeySpec sets the WrappingKeySpec field's value. +func (s *GetParametersForImportInput) SetWrappingKeySpec(v string) *GetParametersForImportInput { + s.WrappingKeySpec = &v + return s +} + +type GetParametersForImportOutput struct { + _ struct{} `type:"structure"` + + // The import token to send in a subsequent ImportKeyMaterial request. + // + // ImportToken is automatically base64 encoded/decoded by the SDK. + ImportToken []byte `min:"1" type:"blob"` + + // The Amazon Resource Name (key ARN (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#key-id-key-ARN)) + // of the CMK to use in a subsequent ImportKeyMaterial request. This is the + // same CMK specified in the GetParametersForImport request. + KeyId *string `min:"1" type:"string"` + + // The time at which the import token and public key are no longer valid. After + // this time, you cannot use them to make an ImportKeyMaterial request and you + // must send another GetParametersForImport request to get new ones. + ParametersValidTo *time.Time `type:"timestamp"` + + // The public key to use to encrypt the key material before importing it with + // ImportKeyMaterial. + // + // PublicKey is automatically base64 encoded/decoded by the SDK. + PublicKey []byte `min:"1" type:"blob" sensitive:"true"` +} + +// String returns the string representation +func (s GetParametersForImportOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s GetParametersForImportOutput) GoString() string { + return s.String() +} + +// SetImportToken sets the ImportToken field's value. +func (s *GetParametersForImportOutput) SetImportToken(v []byte) *GetParametersForImportOutput { + s.ImportToken = v + return s +} + +// SetKeyId sets the KeyId field's value. +func (s *GetParametersForImportOutput) SetKeyId(v string) *GetParametersForImportOutput { + s.KeyId = &v + return s +} + +// SetParametersValidTo sets the ParametersValidTo field's value. +func (s *GetParametersForImportOutput) SetParametersValidTo(v time.Time) *GetParametersForImportOutput { + s.ParametersValidTo = &v + return s +} + +// SetPublicKey sets the PublicKey field's value. +func (s *GetParametersForImportOutput) SetPublicKey(v []byte) *GetParametersForImportOutput { + s.PublicKey = v + return s +} + +type GetPublicKeyInput struct { + _ struct{} `type:"structure"` + + // A list of grant tokens. + // + // For more information, see Grant Tokens (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#grant_token) + // in the AWS Key Management Service Developer Guide. + GrantTokens []*string `type:"list"` + + // Identifies the asymmetric CMK that includes the public key. + // + // To specify a CMK, use its key ID, Amazon Resource Name (ARN), alias name, + // or alias ARN. When using an alias name, prefix it with "alias/". To specify + // a CMK in a different AWS account, you must use the key ARN or alias ARN. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Alias name: alias/ExampleAlias + // + // * Alias ARN: arn:aws:kms:us-east-2:111122223333:alias/ExampleAlias + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. To + // get the alias name and alias ARN, use ListAliases. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` +} + +// String returns the string representation +func (s GetPublicKeyInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s GetPublicKeyInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *GetPublicKeyInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "GetPublicKeyInput"} + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetGrantTokens sets the GrantTokens field's value. +func (s *GetPublicKeyInput) SetGrantTokens(v []*string) *GetPublicKeyInput { + s.GrantTokens = v + return s +} + +// SetKeyId sets the KeyId field's value. +func (s *GetPublicKeyInput) SetKeyId(v string) *GetPublicKeyInput { + s.KeyId = &v + return s +} + +type GetPublicKeyOutput struct { + _ struct{} `type:"structure"` + + // The type of the of the public key that was downloaded. + CustomerMasterKeySpec *string `type:"string" enum:"CustomerMasterKeySpec"` + + // The encryption algorithms that AWS KMS supports for this key. + // + // This information is critical. If a public key encrypts data outside of AWS + // KMS by using an unsupported encryption algorithm, the ciphertext cannot be + // decrypted. + // + // This field appears in the response only when the KeyUsage of the public key + // is ENCRYPT_DECRYPT. + EncryptionAlgorithms []*string `type:"list"` + + // The Amazon Resource Name (key ARN (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#key-id-key-ARN)) + // of the asymmetric CMK from which the public key was downloaded. + KeyId *string `min:"1" type:"string"` + + // The permitted use of the public key. Valid values are ENCRYPT_DECRYPT or + // SIGN_VERIFY. + // + // This information is critical. If a public key with SIGN_VERIFY key usage + // encrypts data outside of AWS KMS, the ciphertext cannot be decrypted. + KeyUsage *string `type:"string" enum:"KeyUsageType"` + + // The exported public key. + // + // The value is a DER-encoded X.509 public key, also known as SubjectPublicKeyInfo + // (SPKI), as defined in RFC 5280 (https://tools.ietf.org/html/rfc5280). When + // you use the HTTP API or the AWS CLI, the value is Base64-encoded. Otherwise, + // it is not Base64-encoded. + // + // PublicKey is automatically base64 encoded/decoded by the SDK. + PublicKey []byte `min:"1" type:"blob"` + + // The signing algorithms that AWS KMS supports for this key. + // + // This field appears in the response only when the KeyUsage of the public key + // is SIGN_VERIFY. + SigningAlgorithms []*string `type:"list"` +} + +// String returns the string representation +func (s GetPublicKeyOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s GetPublicKeyOutput) GoString() string { + return s.String() +} + +// SetCustomerMasterKeySpec sets the CustomerMasterKeySpec field's value. +func (s *GetPublicKeyOutput) SetCustomerMasterKeySpec(v string) *GetPublicKeyOutput { + s.CustomerMasterKeySpec = &v + return s +} + +// SetEncryptionAlgorithms sets the EncryptionAlgorithms field's value. +func (s *GetPublicKeyOutput) SetEncryptionAlgorithms(v []*string) *GetPublicKeyOutput { + s.EncryptionAlgorithms = v + return s +} + +// SetKeyId sets the KeyId field's value. +func (s *GetPublicKeyOutput) SetKeyId(v string) *GetPublicKeyOutput { + s.KeyId = &v + return s +} + +// SetKeyUsage sets the KeyUsage field's value. +func (s *GetPublicKeyOutput) SetKeyUsage(v string) *GetPublicKeyOutput { + s.KeyUsage = &v + return s +} + +// SetPublicKey sets the PublicKey field's value. +func (s *GetPublicKeyOutput) SetPublicKey(v []byte) *GetPublicKeyOutput { + s.PublicKey = v + return s +} + +// SetSigningAlgorithms sets the SigningAlgorithms field's value. +func (s *GetPublicKeyOutput) SetSigningAlgorithms(v []*string) *GetPublicKeyOutput { + s.SigningAlgorithms = v + return s +} + +// Use this structure to allow cryptographic operations (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#cryptographic-operations) +// in the grant only when the operation request includes the specified encryption +// context (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#encrypt_context). +// +// AWS KMS applies the grant constraints only to cryptographic operations that +// support an encryption context, that is, all cryptographic operations with +// a symmetric CMK (https://docs.aws.amazon.com/kms/latest/developerguide/symm-asymm-concepts.html#symmetric-cmks). +// Grant constraints are not applied to operations that do not support an encryption +// context, such as cryptographic operations with asymmetric CMKs and management +// operations, such as DescribeKey or ScheduleKeyDeletion. +// +// In a cryptographic operation, the encryption context in the decryption operation +// must be an exact, case-sensitive match for the keys and values in the encryption +// context of the encryption operation. Only the order of the pairs can vary. +// +// However, in a grant constraint, the key in each key-value pair is not case +// sensitive, but the value is case sensitive. +// +// To avoid confusion, do not use multiple encryption context pairs that differ +// only by case. To require a fully case-sensitive encryption context, use the +// kms:EncryptionContext: and kms:EncryptionContextKeys conditions in an IAM +// or key policy. For details, see kms:EncryptionContext: (https://docs.aws.amazon.com/kms/latest/developerguide/policy-conditions.html#conditions-kms-encryption-context) +// in the AWS Key Management Service Developer Guide . +type GrantConstraints struct { + _ struct{} `type:"structure"` + + // A list of key-value pairs that must match the encryption context in the cryptographic + // operation (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#cryptographic-operations) + // request. The grant allows the operation only when the encryption context + // in the request is the same as the encryption context specified in this constraint. + EncryptionContextEquals map[string]*string `type:"map"` + + // A list of key-value pairs that must be included in the encryption context + // of the cryptographic operation (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#cryptographic-operations) + // request. The grant allows the cryptographic operation only when the encryption + // context in the request includes the key-value pairs specified in this constraint, + // although it can include additional key-value pairs. + EncryptionContextSubset map[string]*string `type:"map"` +} + +// String returns the string representation +func (s GrantConstraints) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s GrantConstraints) GoString() string { + return s.String() +} + +// SetEncryptionContextEquals sets the EncryptionContextEquals field's value. +func (s *GrantConstraints) SetEncryptionContextEquals(v map[string]*string) *GrantConstraints { + s.EncryptionContextEquals = v + return s +} + +// SetEncryptionContextSubset sets the EncryptionContextSubset field's value. +func (s *GrantConstraints) SetEncryptionContextSubset(v map[string]*string) *GrantConstraints { + s.EncryptionContextSubset = v + return s +} + +// Contains information about a grant. +type GrantListEntry struct { + _ struct{} `type:"structure"` + + // A list of key-value pairs that must be present in the encryption context + // of certain subsequent operations that the grant allows. + Constraints *GrantConstraints `type:"structure"` + + // The date and time when the grant was created. + CreationDate *time.Time `type:"timestamp"` + + // The unique identifier for the grant. + GrantId *string `min:"1" type:"string"` + + // The identity that gets the permissions in the grant. + // + // The GranteePrincipal field in the ListGrants response usually contains the + // user or role designated as the grantee principal in the grant. However, when + // the grantee principal in the grant is an AWS service, the GranteePrincipal + // field contains the service principal (https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_principal.html#principal-services), + // which might represent several different grantee principals. + GranteePrincipal *string `min:"1" type:"string"` + + // The AWS account under which the grant was issued. + IssuingAccount *string `min:"1" type:"string"` + + // The unique identifier for the customer master key (CMK) to which the grant + // applies. + KeyId *string `min:"1" type:"string"` + + // The friendly name that identifies the grant. If a name was provided in the + // CreateGrant request, that name is returned. Otherwise this value is null. + Name *string `min:"1" type:"string"` + + // The list of operations permitted by the grant. + Operations []*string `type:"list"` + + // The principal that can retire the grant. + RetiringPrincipal *string `min:"1" type:"string"` +} + +// String returns the string representation +func (s GrantListEntry) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s GrantListEntry) GoString() string { + return s.String() +} + +// SetConstraints sets the Constraints field's value. +func (s *GrantListEntry) SetConstraints(v *GrantConstraints) *GrantListEntry { + s.Constraints = v + return s +} + +// SetCreationDate sets the CreationDate field's value. +func (s *GrantListEntry) SetCreationDate(v time.Time) *GrantListEntry { + s.CreationDate = &v + return s +} + +// SetGrantId sets the GrantId field's value. +func (s *GrantListEntry) SetGrantId(v string) *GrantListEntry { + s.GrantId = &v + return s +} + +// SetGranteePrincipal sets the GranteePrincipal field's value. +func (s *GrantListEntry) SetGranteePrincipal(v string) *GrantListEntry { + s.GranteePrincipal = &v + return s +} + +// SetIssuingAccount sets the IssuingAccount field's value. +func (s *GrantListEntry) SetIssuingAccount(v string) *GrantListEntry { + s.IssuingAccount = &v + return s +} + +// SetKeyId sets the KeyId field's value. +func (s *GrantListEntry) SetKeyId(v string) *GrantListEntry { + s.KeyId = &v + return s +} + +// SetName sets the Name field's value. +func (s *GrantListEntry) SetName(v string) *GrantListEntry { + s.Name = &v + return s +} + +// SetOperations sets the Operations field's value. +func (s *GrantListEntry) SetOperations(v []*string) *GrantListEntry { + s.Operations = v + return s +} + +// SetRetiringPrincipal sets the RetiringPrincipal field's value. +func (s *GrantListEntry) SetRetiringPrincipal(v string) *GrantListEntry { + s.RetiringPrincipal = &v + return s +} + +type ImportKeyMaterialInput struct { + _ struct{} `type:"structure"` + + // The encrypted key material to import. The key material must be encrypted + // with the public wrapping key that GetParametersForImport returned, using + // the wrapping algorithm that you specified in the same GetParametersForImport + // request. + // + // EncryptedKeyMaterial is automatically base64 encoded/decoded by the SDK. + // + // EncryptedKeyMaterial is a required field + EncryptedKeyMaterial []byte `min:"1" type:"blob" required:"true"` + + // Specifies whether the key material expires. The default is KEY_MATERIAL_EXPIRES, + // in which case you must include the ValidTo parameter. When this parameter + // is set to KEY_MATERIAL_DOES_NOT_EXPIRE, you must omit the ValidTo parameter. + ExpirationModel *string `type:"string" enum:"ExpirationModelType"` + + // The import token that you received in the response to a previous GetParametersForImport + // request. It must be from the same response that contained the public key + // that you used to encrypt the key material. + // + // ImportToken is automatically base64 encoded/decoded by the SDK. + // + // ImportToken is a required field + ImportToken []byte `min:"1" type:"blob" required:"true"` + + // The identifier of the symmetric CMK that receives the imported key material. + // The CMK's Origin must be EXTERNAL. This must be the same CMK specified in + // the KeyID parameter of the corresponding GetParametersForImport request. + // + // Specify the key ID or the Amazon Resource Name (ARN) of the CMK. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` + + // The time at which the imported key material expires. When the key material + // expires, AWS KMS deletes the key material and the CMK becomes unusable. You + // must omit this parameter when the ExpirationModel parameter is set to KEY_MATERIAL_DOES_NOT_EXPIRE. + // Otherwise it is required. + ValidTo *time.Time `type:"timestamp"` +} + +// String returns the string representation +func (s ImportKeyMaterialInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s ImportKeyMaterialInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *ImportKeyMaterialInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "ImportKeyMaterialInput"} + if s.EncryptedKeyMaterial == nil { + invalidParams.Add(request.NewErrParamRequired("EncryptedKeyMaterial")) + } + if s.EncryptedKeyMaterial != nil && len(s.EncryptedKeyMaterial) < 1 { + invalidParams.Add(request.NewErrParamMinLen("EncryptedKeyMaterial", 1)) + } + if s.ImportToken == nil { + invalidParams.Add(request.NewErrParamRequired("ImportToken")) + } + if s.ImportToken != nil && len(s.ImportToken) < 1 { + invalidParams.Add(request.NewErrParamMinLen("ImportToken", 1)) + } + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetEncryptedKeyMaterial sets the EncryptedKeyMaterial field's value. +func (s *ImportKeyMaterialInput) SetEncryptedKeyMaterial(v []byte) *ImportKeyMaterialInput { + s.EncryptedKeyMaterial = v + return s +} + +// SetExpirationModel sets the ExpirationModel field's value. +func (s *ImportKeyMaterialInput) SetExpirationModel(v string) *ImportKeyMaterialInput { + s.ExpirationModel = &v + return s +} + +// SetImportToken sets the ImportToken field's value. +func (s *ImportKeyMaterialInput) SetImportToken(v []byte) *ImportKeyMaterialInput { + s.ImportToken = v + return s +} + +// SetKeyId sets the KeyId field's value. +func (s *ImportKeyMaterialInput) SetKeyId(v string) *ImportKeyMaterialInput { + s.KeyId = &v + return s +} + +// SetValidTo sets the ValidTo field's value. +func (s *ImportKeyMaterialInput) SetValidTo(v time.Time) *ImportKeyMaterialInput { + s.ValidTo = &v + return s +} + +type ImportKeyMaterialOutput struct { + _ struct{} `type:"structure"` +} + +// String returns the string representation +func (s ImportKeyMaterialOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s ImportKeyMaterialOutput) GoString() string { + return s.String() +} + +// The request was rejected because the specified CMK cannot decrypt the data. +// The KeyId in a Decrypt request and the SourceKeyId in a ReEncrypt request +// must identify the same CMK that was used to encrypt the ciphertext. +type IncorrectKeyException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s IncorrectKeyException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s IncorrectKeyException) GoString() string { + return s.String() +} + +func newErrorIncorrectKeyException(v protocol.ResponseMetadata) error { + return &IncorrectKeyException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *IncorrectKeyException) Code() string { + return "IncorrectKeyException" +} + +// Message returns the exception's message. +func (s *IncorrectKeyException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *IncorrectKeyException) OrigErr() error { + return nil +} + +func (s *IncorrectKeyException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *IncorrectKeyException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *IncorrectKeyException) RequestID() string { + return s.RespMetadata.RequestID +} + +// The request was rejected because the key material in the request is, expired, +// invalid, or is not the same key material that was previously imported into +// this customer master key (CMK). +type IncorrectKeyMaterialException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s IncorrectKeyMaterialException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s IncorrectKeyMaterialException) GoString() string { + return s.String() +} + +func newErrorIncorrectKeyMaterialException(v protocol.ResponseMetadata) error { + return &IncorrectKeyMaterialException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *IncorrectKeyMaterialException) Code() string { + return "IncorrectKeyMaterialException" +} + +// Message returns the exception's message. +func (s *IncorrectKeyMaterialException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *IncorrectKeyMaterialException) OrigErr() error { + return nil +} + +func (s *IncorrectKeyMaterialException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *IncorrectKeyMaterialException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *IncorrectKeyMaterialException) RequestID() string { + return s.RespMetadata.RequestID +} + +// The request was rejected because the trust anchor certificate in the request +// is not the trust anchor certificate for the specified AWS CloudHSM cluster. +// +// When you initialize the cluster (https://docs.aws.amazon.com/cloudhsm/latest/userguide/initialize-cluster.html#sign-csr), +// you create the trust anchor certificate and save it in the customerCA.crt +// file. +type IncorrectTrustAnchorException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s IncorrectTrustAnchorException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s IncorrectTrustAnchorException) GoString() string { + return s.String() +} + +func newErrorIncorrectTrustAnchorException(v protocol.ResponseMetadata) error { + return &IncorrectTrustAnchorException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *IncorrectTrustAnchorException) Code() string { + return "IncorrectTrustAnchorException" +} + +// Message returns the exception's message. +func (s *IncorrectTrustAnchorException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *IncorrectTrustAnchorException) OrigErr() error { + return nil +} + +func (s *IncorrectTrustAnchorException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *IncorrectTrustAnchorException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *IncorrectTrustAnchorException) RequestID() string { + return s.RespMetadata.RequestID +} + +// The request was rejected because an internal exception occurred. The request +// can be retried. +type InternalException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s InternalException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s InternalException) GoString() string { + return s.String() +} + +func newErrorInternalException(v protocol.ResponseMetadata) error { + return &InternalException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *InternalException) Code() string { + return "KMSInternalException" +} + +// Message returns the exception's message. +func (s *InternalException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *InternalException) OrigErr() error { + return nil +} + +func (s *InternalException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *InternalException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *InternalException) RequestID() string { + return s.RespMetadata.RequestID +} + +// The request was rejected because the specified alias name is not valid. +type InvalidAliasNameException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s InvalidAliasNameException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s InvalidAliasNameException) GoString() string { + return s.String() +} + +func newErrorInvalidAliasNameException(v protocol.ResponseMetadata) error { + return &InvalidAliasNameException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *InvalidAliasNameException) Code() string { + return "InvalidAliasNameException" +} + +// Message returns the exception's message. +func (s *InvalidAliasNameException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *InvalidAliasNameException) OrigErr() error { + return nil +} + +func (s *InvalidAliasNameException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *InvalidAliasNameException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *InvalidAliasNameException) RequestID() string { + return s.RespMetadata.RequestID +} + +// The request was rejected because a specified ARN, or an ARN in a key policy, +// is not valid. +type InvalidArnException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s InvalidArnException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s InvalidArnException) GoString() string { + return s.String() +} + +func newErrorInvalidArnException(v protocol.ResponseMetadata) error { + return &InvalidArnException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *InvalidArnException) Code() string { + return "InvalidArnException" +} + +// Message returns the exception's message. +func (s *InvalidArnException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *InvalidArnException) OrigErr() error { + return nil +} + +func (s *InvalidArnException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *InvalidArnException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *InvalidArnException) RequestID() string { + return s.RespMetadata.RequestID +} + +// From the Decrypt or ReEncrypt operation, the request was rejected because +// the specified ciphertext, or additional authenticated data incorporated into +// the ciphertext, such as the encryption context, is corrupted, missing, or +// otherwise invalid. +// +// From the ImportKeyMaterial operation, the request was rejected because AWS +// KMS could not decrypt the encrypted (wrapped) key material. +type InvalidCiphertextException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s InvalidCiphertextException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s InvalidCiphertextException) GoString() string { + return s.String() +} + +func newErrorInvalidCiphertextException(v protocol.ResponseMetadata) error { + return &InvalidCiphertextException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *InvalidCiphertextException) Code() string { + return "InvalidCiphertextException" +} + +// Message returns the exception's message. +func (s *InvalidCiphertextException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *InvalidCiphertextException) OrigErr() error { + return nil +} + +func (s *InvalidCiphertextException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *InvalidCiphertextException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *InvalidCiphertextException) RequestID() string { + return s.RespMetadata.RequestID +} + +// The request was rejected because the specified GrantId is not valid. +type InvalidGrantIdException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s InvalidGrantIdException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s InvalidGrantIdException) GoString() string { + return s.String() +} + +func newErrorInvalidGrantIdException(v protocol.ResponseMetadata) error { + return &InvalidGrantIdException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *InvalidGrantIdException) Code() string { + return "InvalidGrantIdException" +} + +// Message returns the exception's message. +func (s *InvalidGrantIdException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *InvalidGrantIdException) OrigErr() error { + return nil +} + +func (s *InvalidGrantIdException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *InvalidGrantIdException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *InvalidGrantIdException) RequestID() string { + return s.RespMetadata.RequestID +} + +// The request was rejected because the specified grant token is not valid. +type InvalidGrantTokenException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s InvalidGrantTokenException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s InvalidGrantTokenException) GoString() string { + return s.String() +} + +func newErrorInvalidGrantTokenException(v protocol.ResponseMetadata) error { + return &InvalidGrantTokenException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *InvalidGrantTokenException) Code() string { + return "InvalidGrantTokenException" +} + +// Message returns the exception's message. +func (s *InvalidGrantTokenException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *InvalidGrantTokenException) OrigErr() error { + return nil +} + +func (s *InvalidGrantTokenException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *InvalidGrantTokenException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *InvalidGrantTokenException) RequestID() string { + return s.RespMetadata.RequestID +} + +// The request was rejected because the provided import token is invalid or +// is associated with a different customer master key (CMK). +type InvalidImportTokenException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s InvalidImportTokenException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s InvalidImportTokenException) GoString() string { + return s.String() +} + +func newErrorInvalidImportTokenException(v protocol.ResponseMetadata) error { + return &InvalidImportTokenException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *InvalidImportTokenException) Code() string { + return "InvalidImportTokenException" +} + +// Message returns the exception's message. +func (s *InvalidImportTokenException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *InvalidImportTokenException) OrigErr() error { + return nil +} + +func (s *InvalidImportTokenException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *InvalidImportTokenException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *InvalidImportTokenException) RequestID() string { + return s.RespMetadata.RequestID +} + +// The request was rejected for one of the following reasons: +// +// * The KeyUsage value of the CMK is incompatible with the API operation. +// +// * The encryption algorithm or signing algorithm specified for the operation +// is incompatible with the type of key material in the CMK (CustomerMasterKeySpec). +// +// For encrypting, decrypting, re-encrypting, and generating data keys, the +// KeyUsage must be ENCRYPT_DECRYPT. For signing and verifying, the KeyUsage +// must be SIGN_VERIFY. To find the KeyUsage of a CMK, use the DescribeKey operation. +// +// To find the encryption or signing algorithms supported for a particular CMK, +// use the DescribeKey operation. +type InvalidKeyUsageException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s InvalidKeyUsageException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s InvalidKeyUsageException) GoString() string { + return s.String() +} + +func newErrorInvalidKeyUsageException(v protocol.ResponseMetadata) error { + return &InvalidKeyUsageException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *InvalidKeyUsageException) Code() string { + return "InvalidKeyUsageException" +} + +// Message returns the exception's message. +func (s *InvalidKeyUsageException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *InvalidKeyUsageException) OrigErr() error { + return nil +} + +func (s *InvalidKeyUsageException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *InvalidKeyUsageException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *InvalidKeyUsageException) RequestID() string { + return s.RespMetadata.RequestID +} + +// The request was rejected because the marker that specifies where pagination +// should next begin is not valid. +type InvalidMarkerException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s InvalidMarkerException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s InvalidMarkerException) GoString() string { + return s.String() +} + +func newErrorInvalidMarkerException(v protocol.ResponseMetadata) error { + return &InvalidMarkerException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *InvalidMarkerException) Code() string { + return "InvalidMarkerException" +} + +// Message returns the exception's message. +func (s *InvalidMarkerException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *InvalidMarkerException) OrigErr() error { + return nil +} + +func (s *InvalidMarkerException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *InvalidMarkerException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *InvalidMarkerException) RequestID() string { + return s.RespMetadata.RequestID +} + +// The request was rejected because the state of the specified resource is not +// valid for this request. +// +// For more information about how key state affects the use of a CMK, see How +// Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) +// in the AWS Key Management Service Developer Guide . +type InvalidStateException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s InvalidStateException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s InvalidStateException) GoString() string { + return s.String() +} + +func newErrorInvalidStateException(v protocol.ResponseMetadata) error { + return &InvalidStateException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *InvalidStateException) Code() string { + return "KMSInvalidStateException" +} + +// Message returns the exception's message. +func (s *InvalidStateException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *InvalidStateException) OrigErr() error { + return nil +} + +func (s *InvalidStateException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *InvalidStateException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *InvalidStateException) RequestID() string { + return s.RespMetadata.RequestID +} + +// The request was rejected because the signature verification failed. Signature +// verification fails when it cannot confirm that signature was produced by +// signing the specified message with the specified CMK and signing algorithm. +type KMSInvalidSignatureException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s KMSInvalidSignatureException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s KMSInvalidSignatureException) GoString() string { + return s.String() +} + +func newErrorKMSInvalidSignatureException(v protocol.ResponseMetadata) error { + return &KMSInvalidSignatureException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *KMSInvalidSignatureException) Code() string { + return "KMSInvalidSignatureException" +} + +// Message returns the exception's message. +func (s *KMSInvalidSignatureException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *KMSInvalidSignatureException) OrigErr() error { + return nil +} + +func (s *KMSInvalidSignatureException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *KMSInvalidSignatureException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *KMSInvalidSignatureException) RequestID() string { + return s.RespMetadata.RequestID +} + +// Contains information about each entry in the key list. +type KeyListEntry struct { + _ struct{} `type:"structure"` + + // ARN of the key. + KeyArn *string `min:"20" type:"string"` + + // Unique identifier of the key. + KeyId *string `min:"1" type:"string"` +} + +// String returns the string representation +func (s KeyListEntry) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s KeyListEntry) GoString() string { + return s.String() +} + +// SetKeyArn sets the KeyArn field's value. +func (s *KeyListEntry) SetKeyArn(v string) *KeyListEntry { + s.KeyArn = &v + return s +} + +// SetKeyId sets the KeyId field's value. +func (s *KeyListEntry) SetKeyId(v string) *KeyListEntry { + s.KeyId = &v + return s +} + +// Contains metadata about a customer master key (CMK). +// +// This data type is used as a response element for the CreateKey and DescribeKey +// operations. +type KeyMetadata struct { + _ struct{} `type:"structure"` + + // The twelve-digit account ID of the AWS account that owns the CMK. + AWSAccountId *string `type:"string"` + + // The Amazon Resource Name (ARN) of the CMK. For examples, see AWS Key Management + // Service (AWS KMS) (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html#arn-syntax-kms) + // in the Example ARNs section of the AWS General Reference. + Arn *string `min:"20" type:"string"` + + // The cluster ID of the AWS CloudHSM cluster that contains the key material + // for the CMK. When you create a CMK in a custom key store (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html), + // AWS KMS creates the key material for the CMK in the associated AWS CloudHSM + // cluster. This value is present only when the CMK is created in a custom key + // store. + CloudHsmClusterId *string `min:"19" type:"string"` + + // The date and time when the CMK was created. + CreationDate *time.Time `type:"timestamp"` + + // A unique identifier for the custom key store (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html) + // that contains the CMK. This value is present only when the CMK is created + // in a custom key store. + CustomKeyStoreId *string `min:"1" type:"string"` + + // Describes the type of key material in the CMK. + CustomerMasterKeySpec *string `type:"string" enum:"CustomerMasterKeySpec"` + + // The date and time after which AWS KMS deletes the CMK. This value is present + // only when KeyState is PendingDeletion. + DeletionDate *time.Time `type:"timestamp"` + + // The description of the CMK. + Description *string `type:"string"` + + // Specifies whether the CMK is enabled. When KeyState is Enabled this value + // is true, otherwise it is false. + Enabled *bool `type:"boolean"` + + // The encryption algorithms that the CMK supports. You cannot use the CMK with + // other encryption algorithms within AWS KMS. + // + // This field appears only when the KeyUsage of the CMK is ENCRYPT_DECRYPT. + EncryptionAlgorithms []*string `type:"list"` + + // Specifies whether the CMK's key material expires. This value is present only + // when Origin is EXTERNAL, otherwise this value is omitted. + ExpirationModel *string `type:"string" enum:"ExpirationModelType"` + + // The globally unique identifier for the CMK. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` + + // The manager of the CMK. CMKs in your AWS account are either customer managed + // or AWS managed. For more information about the difference, see Customer Master + // Keys (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#master_keys) + // in the AWS Key Management Service Developer Guide. + KeyManager *string `type:"string" enum:"KeyManagerType"` + + // The current status of the CMK. + // + // For more information about how key state affects the use of a CMK, see Key + // state: Effect on your CMK (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) + // in the AWS Key Management Service Developer Guide. + KeyState *string `type:"string" enum:"KeyState"` + + // The cryptographic operations (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#cryptographic-operations) + // for which you can use the CMK. + KeyUsage *string `type:"string" enum:"KeyUsageType"` + + // The source of the CMK's key material. When this value is AWS_KMS, AWS KMS + // created the key material. When this value is EXTERNAL, the key material was + // imported from your existing key management infrastructure or the CMK lacks + // key material. When this value is AWS_CLOUDHSM, the key material was created + // in the AWS CloudHSM cluster associated with a custom key store. + Origin *string `type:"string" enum:"OriginType"` + + // The signing algorithms that the CMK supports. You cannot use the CMK with + // other signing algorithms within AWS KMS. + // + // This field appears only when the KeyUsage of the CMK is SIGN_VERIFY. + SigningAlgorithms []*string `type:"list"` + + // The time at which the imported key material expires. When the key material + // expires, AWS KMS deletes the key material and the CMK becomes unusable. This + // value is present only for CMKs whose Origin is EXTERNAL and whose ExpirationModel + // is KEY_MATERIAL_EXPIRES, otherwise this value is omitted. + ValidTo *time.Time `type:"timestamp"` +} + +// String returns the string representation +func (s KeyMetadata) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s KeyMetadata) GoString() string { + return s.String() +} + +// SetAWSAccountId sets the AWSAccountId field's value. +func (s *KeyMetadata) SetAWSAccountId(v string) *KeyMetadata { + s.AWSAccountId = &v + return s +} + +// SetArn sets the Arn field's value. +func (s *KeyMetadata) SetArn(v string) *KeyMetadata { + s.Arn = &v + return s +} + +// SetCloudHsmClusterId sets the CloudHsmClusterId field's value. +func (s *KeyMetadata) SetCloudHsmClusterId(v string) *KeyMetadata { + s.CloudHsmClusterId = &v + return s +} + +// SetCreationDate sets the CreationDate field's value. +func (s *KeyMetadata) SetCreationDate(v time.Time) *KeyMetadata { + s.CreationDate = &v + return s +} + +// SetCustomKeyStoreId sets the CustomKeyStoreId field's value. +func (s *KeyMetadata) SetCustomKeyStoreId(v string) *KeyMetadata { + s.CustomKeyStoreId = &v + return s +} + +// SetCustomerMasterKeySpec sets the CustomerMasterKeySpec field's value. +func (s *KeyMetadata) SetCustomerMasterKeySpec(v string) *KeyMetadata { + s.CustomerMasterKeySpec = &v + return s +} + +// SetDeletionDate sets the DeletionDate field's value. +func (s *KeyMetadata) SetDeletionDate(v time.Time) *KeyMetadata { + s.DeletionDate = &v + return s +} + +// SetDescription sets the Description field's value. +func (s *KeyMetadata) SetDescription(v string) *KeyMetadata { + s.Description = &v + return s +} + +// SetEnabled sets the Enabled field's value. +func (s *KeyMetadata) SetEnabled(v bool) *KeyMetadata { + s.Enabled = &v + return s +} + +// SetEncryptionAlgorithms sets the EncryptionAlgorithms field's value. +func (s *KeyMetadata) SetEncryptionAlgorithms(v []*string) *KeyMetadata { + s.EncryptionAlgorithms = v + return s +} + +// SetExpirationModel sets the ExpirationModel field's value. +func (s *KeyMetadata) SetExpirationModel(v string) *KeyMetadata { + s.ExpirationModel = &v + return s +} + +// SetKeyId sets the KeyId field's value. +func (s *KeyMetadata) SetKeyId(v string) *KeyMetadata { + s.KeyId = &v + return s +} + +// SetKeyManager sets the KeyManager field's value. +func (s *KeyMetadata) SetKeyManager(v string) *KeyMetadata { + s.KeyManager = &v + return s +} + +// SetKeyState sets the KeyState field's value. +func (s *KeyMetadata) SetKeyState(v string) *KeyMetadata { + s.KeyState = &v + return s +} + +// SetKeyUsage sets the KeyUsage field's value. +func (s *KeyMetadata) SetKeyUsage(v string) *KeyMetadata { + s.KeyUsage = &v + return s +} + +// SetOrigin sets the Origin field's value. +func (s *KeyMetadata) SetOrigin(v string) *KeyMetadata { + s.Origin = &v + return s +} + +// SetSigningAlgorithms sets the SigningAlgorithms field's value. +func (s *KeyMetadata) SetSigningAlgorithms(v []*string) *KeyMetadata { + s.SigningAlgorithms = v + return s +} + +// SetValidTo sets the ValidTo field's value. +func (s *KeyMetadata) SetValidTo(v time.Time) *KeyMetadata { + s.ValidTo = &v + return s +} + +// The request was rejected because the specified CMK was not available. You +// can retry the request. +type KeyUnavailableException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s KeyUnavailableException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s KeyUnavailableException) GoString() string { + return s.String() +} + +func newErrorKeyUnavailableException(v protocol.ResponseMetadata) error { + return &KeyUnavailableException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *KeyUnavailableException) Code() string { + return "KeyUnavailableException" +} + +// Message returns the exception's message. +func (s *KeyUnavailableException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *KeyUnavailableException) OrigErr() error { + return nil +} + +func (s *KeyUnavailableException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *KeyUnavailableException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *KeyUnavailableException) RequestID() string { + return s.RespMetadata.RequestID +} + +// The request was rejected because a quota was exceeded. For more information, +// see Quotas (https://docs.aws.amazon.com/kms/latest/developerguide/limits.html) +// in the AWS Key Management Service Developer Guide. +type LimitExceededException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s LimitExceededException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s LimitExceededException) GoString() string { + return s.String() +} + +func newErrorLimitExceededException(v protocol.ResponseMetadata) error { + return &LimitExceededException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *LimitExceededException) Code() string { + return "LimitExceededException" +} + +// Message returns the exception's message. +func (s *LimitExceededException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *LimitExceededException) OrigErr() error { + return nil +} + +func (s *LimitExceededException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *LimitExceededException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *LimitExceededException) RequestID() string { + return s.RespMetadata.RequestID +} + +type ListAliasesInput struct { + _ struct{} `type:"structure"` + + // Lists only aliases that refer to the specified CMK. The value of this parameter + // can be the ID or Amazon Resource Name (ARN) of a CMK in the caller's account + // and region. You cannot use an alias name or alias ARN in this value. + // + // This parameter is optional. If you omit it, ListAliases returns all aliases + // in the account and region. + KeyId *string `min:"1" type:"string"` + + // Use this parameter to specify the maximum number of items to return. When + // this value is present, AWS KMS does not return more than the specified number + // of items, but it might return fewer. + // + // This value is optional. If you include a value, it must be between 1 and + // 100, inclusive. If you do not include a value, it defaults to 50. + Limit *int64 `min:"1" type:"integer"` + + // Use this parameter in a subsequent request after you receive a response with + // truncated results. Set it to the value of NextMarker from the truncated response + // you just received. + Marker *string `min:"1" type:"string"` +} + +// String returns the string representation +func (s ListAliasesInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s ListAliasesInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *ListAliasesInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "ListAliasesInput"} + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + if s.Limit != nil && *s.Limit < 1 { + invalidParams.Add(request.NewErrParamMinValue("Limit", 1)) + } + if s.Marker != nil && len(*s.Marker) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Marker", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetKeyId sets the KeyId field's value. +func (s *ListAliasesInput) SetKeyId(v string) *ListAliasesInput { + s.KeyId = &v + return s +} + +// SetLimit sets the Limit field's value. +func (s *ListAliasesInput) SetLimit(v int64) *ListAliasesInput { + s.Limit = &v + return s +} + +// SetMarker sets the Marker field's value. +func (s *ListAliasesInput) SetMarker(v string) *ListAliasesInput { + s.Marker = &v + return s +} + +type ListAliasesOutput struct { + _ struct{} `type:"structure"` + + // A list of aliases. + Aliases []*AliasListEntry `type:"list"` + + // When Truncated is true, this element is present and contains the value to + // use for the Marker parameter in a subsequent request. + NextMarker *string `min:"1" type:"string"` + + // A flag that indicates whether there are more items in the list. When this + // value is true, the list in this response is truncated. To get more items, + // pass the value of the NextMarker element in thisresponse to the Marker parameter + // in a subsequent request. + Truncated *bool `type:"boolean"` +} + +// String returns the string representation +func (s ListAliasesOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s ListAliasesOutput) GoString() string { + return s.String() +} + +// SetAliases sets the Aliases field's value. +func (s *ListAliasesOutput) SetAliases(v []*AliasListEntry) *ListAliasesOutput { + s.Aliases = v + return s +} + +// SetNextMarker sets the NextMarker field's value. +func (s *ListAliasesOutput) SetNextMarker(v string) *ListAliasesOutput { + s.NextMarker = &v + return s +} + +// SetTruncated sets the Truncated field's value. +func (s *ListAliasesOutput) SetTruncated(v bool) *ListAliasesOutput { + s.Truncated = &v + return s +} + +type ListGrantsInput struct { + _ struct{} `type:"structure"` + + // A unique identifier for the customer master key (CMK). + // + // Specify the key ID or the Amazon Resource Name (ARN) of the CMK. To specify + // a CMK in a different AWS account, you must use the key ARN. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` + + // Use this parameter to specify the maximum number of items to return. When + // this value is present, AWS KMS does not return more than the specified number + // of items, but it might return fewer. + // + // This value is optional. If you include a value, it must be between 1 and + // 100, inclusive. If you do not include a value, it defaults to 50. + Limit *int64 `min:"1" type:"integer"` + + // Use this parameter in a subsequent request after you receive a response with + // truncated results. Set it to the value of NextMarker from the truncated response + // you just received. + Marker *string `min:"1" type:"string"` +} + +// String returns the string representation +func (s ListGrantsInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s ListGrantsInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *ListGrantsInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "ListGrantsInput"} + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + if s.Limit != nil && *s.Limit < 1 { + invalidParams.Add(request.NewErrParamMinValue("Limit", 1)) + } + if s.Marker != nil && len(*s.Marker) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Marker", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetKeyId sets the KeyId field's value. +func (s *ListGrantsInput) SetKeyId(v string) *ListGrantsInput { + s.KeyId = &v + return s +} + +// SetLimit sets the Limit field's value. +func (s *ListGrantsInput) SetLimit(v int64) *ListGrantsInput { + s.Limit = &v + return s +} + +// SetMarker sets the Marker field's value. +func (s *ListGrantsInput) SetMarker(v string) *ListGrantsInput { + s.Marker = &v + return s +} + +type ListGrantsResponse struct { + _ struct{} `type:"structure"` + + // A list of grants. + Grants []*GrantListEntry `type:"list"` + + // When Truncated is true, this element is present and contains the value to + // use for the Marker parameter in a subsequent request. + NextMarker *string `min:"1" type:"string"` + + // A flag that indicates whether there are more items in the list. When this + // value is true, the list in this response is truncated. To get more items, + // pass the value of the NextMarker element in thisresponse to the Marker parameter + // in a subsequent request. + Truncated *bool `type:"boolean"` +} + +// String returns the string representation +func (s ListGrantsResponse) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s ListGrantsResponse) GoString() string { + return s.String() +} + +// SetGrants sets the Grants field's value. +func (s *ListGrantsResponse) SetGrants(v []*GrantListEntry) *ListGrantsResponse { + s.Grants = v + return s +} + +// SetNextMarker sets the NextMarker field's value. +func (s *ListGrantsResponse) SetNextMarker(v string) *ListGrantsResponse { + s.NextMarker = &v + return s +} + +// SetTruncated sets the Truncated field's value. +func (s *ListGrantsResponse) SetTruncated(v bool) *ListGrantsResponse { + s.Truncated = &v + return s +} + +type ListKeyPoliciesInput struct { + _ struct{} `type:"structure"` + + // A unique identifier for the customer master key (CMK). + // + // Specify the key ID or the Amazon Resource Name (ARN) of the CMK. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` + + // Use this parameter to specify the maximum number of items to return. When + // this value is present, AWS KMS does not return more than the specified number + // of items, but it might return fewer. + // + // This value is optional. If you include a value, it must be between 1 and + // 1000, inclusive. If you do not include a value, it defaults to 100. + // + // Only one policy can be attached to a key. + Limit *int64 `min:"1" type:"integer"` + + // Use this parameter in a subsequent request after you receive a response with + // truncated results. Set it to the value of NextMarker from the truncated response + // you just received. + Marker *string `min:"1" type:"string"` +} + +// String returns the string representation +func (s ListKeyPoliciesInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s ListKeyPoliciesInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *ListKeyPoliciesInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "ListKeyPoliciesInput"} + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + if s.Limit != nil && *s.Limit < 1 { + invalidParams.Add(request.NewErrParamMinValue("Limit", 1)) + } + if s.Marker != nil && len(*s.Marker) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Marker", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetKeyId sets the KeyId field's value. +func (s *ListKeyPoliciesInput) SetKeyId(v string) *ListKeyPoliciesInput { + s.KeyId = &v + return s +} + +// SetLimit sets the Limit field's value. +func (s *ListKeyPoliciesInput) SetLimit(v int64) *ListKeyPoliciesInput { + s.Limit = &v + return s +} + +// SetMarker sets the Marker field's value. +func (s *ListKeyPoliciesInput) SetMarker(v string) *ListKeyPoliciesInput { + s.Marker = &v + return s +} + +type ListKeyPoliciesOutput struct { + _ struct{} `type:"structure"` + + // When Truncated is true, this element is present and contains the value to + // use for the Marker parameter in a subsequent request. + NextMarker *string `min:"1" type:"string"` + + // A list of key policy names. The only valid value is default. + PolicyNames []*string `type:"list"` + + // A flag that indicates whether there are more items in the list. When this + // value is true, the list in this response is truncated. To get more items, + // pass the value of the NextMarker element in thisresponse to the Marker parameter + // in a subsequent request. + Truncated *bool `type:"boolean"` +} + +// String returns the string representation +func (s ListKeyPoliciesOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s ListKeyPoliciesOutput) GoString() string { + return s.String() +} + +// SetNextMarker sets the NextMarker field's value. +func (s *ListKeyPoliciesOutput) SetNextMarker(v string) *ListKeyPoliciesOutput { + s.NextMarker = &v + return s +} + +// SetPolicyNames sets the PolicyNames field's value. +func (s *ListKeyPoliciesOutput) SetPolicyNames(v []*string) *ListKeyPoliciesOutput { + s.PolicyNames = v + return s +} + +// SetTruncated sets the Truncated field's value. +func (s *ListKeyPoliciesOutput) SetTruncated(v bool) *ListKeyPoliciesOutput { + s.Truncated = &v + return s +} + +type ListKeysInput struct { + _ struct{} `type:"structure"` + + // Use this parameter to specify the maximum number of items to return. When + // this value is present, AWS KMS does not return more than the specified number + // of items, but it might return fewer. + // + // This value is optional. If you include a value, it must be between 1 and + // 1000, inclusive. If you do not include a value, it defaults to 100. + Limit *int64 `min:"1" type:"integer"` + + // Use this parameter in a subsequent request after you receive a response with + // truncated results. Set it to the value of NextMarker from the truncated response + // you just received. + Marker *string `min:"1" type:"string"` +} + +// String returns the string representation +func (s ListKeysInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s ListKeysInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *ListKeysInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "ListKeysInput"} + if s.Limit != nil && *s.Limit < 1 { + invalidParams.Add(request.NewErrParamMinValue("Limit", 1)) + } + if s.Marker != nil && len(*s.Marker) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Marker", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetLimit sets the Limit field's value. +func (s *ListKeysInput) SetLimit(v int64) *ListKeysInput { + s.Limit = &v + return s +} + +// SetMarker sets the Marker field's value. +func (s *ListKeysInput) SetMarker(v string) *ListKeysInput { + s.Marker = &v + return s +} + +type ListKeysOutput struct { + _ struct{} `type:"structure"` + + // A list of customer master keys (CMKs). + Keys []*KeyListEntry `type:"list"` + + // When Truncated is true, this element is present and contains the value to + // use for the Marker parameter in a subsequent request. + NextMarker *string `min:"1" type:"string"` + + // A flag that indicates whether there are more items in the list. When this + // value is true, the list in this response is truncated. To get more items, + // pass the value of the NextMarker element in thisresponse to the Marker parameter + // in a subsequent request. + Truncated *bool `type:"boolean"` +} + +// String returns the string representation +func (s ListKeysOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s ListKeysOutput) GoString() string { + return s.String() +} + +// SetKeys sets the Keys field's value. +func (s *ListKeysOutput) SetKeys(v []*KeyListEntry) *ListKeysOutput { + s.Keys = v + return s +} + +// SetNextMarker sets the NextMarker field's value. +func (s *ListKeysOutput) SetNextMarker(v string) *ListKeysOutput { + s.NextMarker = &v + return s +} + +// SetTruncated sets the Truncated field's value. +func (s *ListKeysOutput) SetTruncated(v bool) *ListKeysOutput { + s.Truncated = &v + return s +} + +type ListResourceTagsInput struct { + _ struct{} `type:"structure"` + + // A unique identifier for the customer master key (CMK). + // + // Specify the key ID or the Amazon Resource Name (ARN) of the CMK. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` + + // Use this parameter to specify the maximum number of items to return. When + // this value is present, AWS KMS does not return more than the specified number + // of items, but it might return fewer. + // + // This value is optional. If you include a value, it must be between 1 and + // 50, inclusive. If you do not include a value, it defaults to 50. + Limit *int64 `min:"1" type:"integer"` + + // Use this parameter in a subsequent request after you receive a response with + // truncated results. Set it to the value of NextMarker from the truncated response + // you just received. + // + // Do not attempt to construct this value. Use only the value of NextMarker + // from the truncated response you just received. + Marker *string `min:"1" type:"string"` +} + +// String returns the string representation +func (s ListResourceTagsInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s ListResourceTagsInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *ListResourceTagsInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "ListResourceTagsInput"} + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + if s.Limit != nil && *s.Limit < 1 { + invalidParams.Add(request.NewErrParamMinValue("Limit", 1)) + } + if s.Marker != nil && len(*s.Marker) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Marker", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetKeyId sets the KeyId field's value. +func (s *ListResourceTagsInput) SetKeyId(v string) *ListResourceTagsInput { + s.KeyId = &v + return s +} + +// SetLimit sets the Limit field's value. +func (s *ListResourceTagsInput) SetLimit(v int64) *ListResourceTagsInput { + s.Limit = &v + return s +} + +// SetMarker sets the Marker field's value. +func (s *ListResourceTagsInput) SetMarker(v string) *ListResourceTagsInput { + s.Marker = &v + return s +} + +type ListResourceTagsOutput struct { + _ struct{} `type:"structure"` + + // When Truncated is true, this element is present and contains the value to + // use for the Marker parameter in a subsequent request. + // + // Do not assume or infer any information from this value. + NextMarker *string `min:"1" type:"string"` + + // A list of tags. Each tag consists of a tag key and a tag value. + Tags []*Tag `type:"list"` + + // A flag that indicates whether there are more items in the list. When this + // value is true, the list in this response is truncated. To get more items, + // pass the value of the NextMarker element in thisresponse to the Marker parameter + // in a subsequent request. + Truncated *bool `type:"boolean"` +} + +// String returns the string representation +func (s ListResourceTagsOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s ListResourceTagsOutput) GoString() string { + return s.String() +} + +// SetNextMarker sets the NextMarker field's value. +func (s *ListResourceTagsOutput) SetNextMarker(v string) *ListResourceTagsOutput { + s.NextMarker = &v + return s +} + +// SetTags sets the Tags field's value. +func (s *ListResourceTagsOutput) SetTags(v []*Tag) *ListResourceTagsOutput { + s.Tags = v + return s +} + +// SetTruncated sets the Truncated field's value. +func (s *ListResourceTagsOutput) SetTruncated(v bool) *ListResourceTagsOutput { + s.Truncated = &v + return s +} + +type ListRetirableGrantsInput struct { + _ struct{} `type:"structure"` + + // Use this parameter to specify the maximum number of items to return. When + // this value is present, AWS KMS does not return more than the specified number + // of items, but it might return fewer. + // + // This value is optional. If you include a value, it must be between 1 and + // 100, inclusive. If you do not include a value, it defaults to 50. + Limit *int64 `min:"1" type:"integer"` + + // Use this parameter in a subsequent request after you receive a response with + // truncated results. Set it to the value of NextMarker from the truncated response + // you just received. + Marker *string `min:"1" type:"string"` + + // The retiring principal for which to list grants. + // + // To specify the retiring principal, use the Amazon Resource Name (ARN) (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html) + // of an AWS principal. Valid AWS principals include AWS accounts (root), IAM + // users, federated users, and assumed role users. For examples of the ARN syntax + // for specifying a principal, see AWS Identity and Access Management (IAM) + // (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html#arn-syntax-iam) + // in the Example ARNs section of the Amazon Web Services General Reference. + // + // RetiringPrincipal is a required field + RetiringPrincipal *string `min:"1" type:"string" required:"true"` +} + +// String returns the string representation +func (s ListRetirableGrantsInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s ListRetirableGrantsInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *ListRetirableGrantsInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "ListRetirableGrantsInput"} + if s.Limit != nil && *s.Limit < 1 { + invalidParams.Add(request.NewErrParamMinValue("Limit", 1)) + } + if s.Marker != nil && len(*s.Marker) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Marker", 1)) + } + if s.RetiringPrincipal == nil { + invalidParams.Add(request.NewErrParamRequired("RetiringPrincipal")) + } + if s.RetiringPrincipal != nil && len(*s.RetiringPrincipal) < 1 { + invalidParams.Add(request.NewErrParamMinLen("RetiringPrincipal", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetLimit sets the Limit field's value. +func (s *ListRetirableGrantsInput) SetLimit(v int64) *ListRetirableGrantsInput { + s.Limit = &v + return s +} + +// SetMarker sets the Marker field's value. +func (s *ListRetirableGrantsInput) SetMarker(v string) *ListRetirableGrantsInput { + s.Marker = &v + return s +} + +// SetRetiringPrincipal sets the RetiringPrincipal field's value. +func (s *ListRetirableGrantsInput) SetRetiringPrincipal(v string) *ListRetirableGrantsInput { + s.RetiringPrincipal = &v + return s +} + +// The request was rejected because the specified policy is not syntactically +// or semantically correct. +type MalformedPolicyDocumentException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s MalformedPolicyDocumentException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s MalformedPolicyDocumentException) GoString() string { + return s.String() +} + +func newErrorMalformedPolicyDocumentException(v protocol.ResponseMetadata) error { + return &MalformedPolicyDocumentException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *MalformedPolicyDocumentException) Code() string { + return "MalformedPolicyDocumentException" +} + +// Message returns the exception's message. +func (s *MalformedPolicyDocumentException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *MalformedPolicyDocumentException) OrigErr() error { + return nil +} + +func (s *MalformedPolicyDocumentException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *MalformedPolicyDocumentException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *MalformedPolicyDocumentException) RequestID() string { + return s.RespMetadata.RequestID +} + +// The request was rejected because the specified entity or resource could not +// be found. +type NotFoundException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s NotFoundException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s NotFoundException) GoString() string { + return s.String() +} + +func newErrorNotFoundException(v protocol.ResponseMetadata) error { + return &NotFoundException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *NotFoundException) Code() string { + return "NotFoundException" +} + +// Message returns the exception's message. +func (s *NotFoundException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *NotFoundException) OrigErr() error { + return nil +} + +func (s *NotFoundException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *NotFoundException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *NotFoundException) RequestID() string { + return s.RespMetadata.RequestID +} + +type PutKeyPolicyInput struct { + _ struct{} `type:"structure"` + + // A flag to indicate whether to bypass the key policy lockout safety check. + // + // Setting this value to true increases the risk that the CMK becomes unmanageable. + // Do not set this value to true indiscriminately. + // + // For more information, refer to the scenario in the Default Key Policy (https://docs.aws.amazon.com/kms/latest/developerguide/key-policies.html#key-policy-default-allow-root-enable-iam) + // section in the AWS Key Management Service Developer Guide. + // + // Use this parameter only when you intend to prevent the principal that is + // making the request from making a subsequent PutKeyPolicy request on the CMK. + // + // The default value is false. + BypassPolicyLockoutSafetyCheck *bool `type:"boolean"` + + // A unique identifier for the customer master key (CMK). + // + // Specify the key ID or the Amazon Resource Name (ARN) of the CMK. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` + + // The key policy to attach to the CMK. + // + // The key policy must meet the following criteria: + // + // * If you don't set BypassPolicyLockoutSafetyCheck to true, the key policy + // must allow the principal that is making the PutKeyPolicy request to make + // a subsequent PutKeyPolicy request on the CMK. This reduces the risk that + // the CMK becomes unmanageable. For more information, refer to the scenario + // in the Default Key Policy (https://docs.aws.amazon.com/kms/latest/developerguide/key-policies.html#key-policy-default-allow-root-enable-iam) + // section of the AWS Key Management Service Developer Guide. + // + // * Each statement in the key policy must contain one or more principals. + // The principals in the key policy must exist and be visible to AWS KMS. + // When you create a new AWS principal (for example, an IAM user or role), + // you might need to enforce a delay before including the new principal in + // a key policy because the new principal might not be immediately visible + // to AWS KMS. For more information, see Changes that I make are not always + // immediately visible (https://docs.aws.amazon.com/IAM/latest/UserGuide/troubleshoot_general.html#troubleshoot_general_eventual-consistency) + // in the AWS Identity and Access Management User Guide. + // + // The key policy cannot exceed 32 kilobytes (32768 bytes). For more information, + // see Resource Quotas (https://docs.aws.amazon.com/kms/latest/developerguide/resource-limits.html) + // in the AWS Key Management Service Developer Guide. + // + // Policy is a required field + Policy *string `min:"1" type:"string" required:"true"` + + // The name of the key policy. The only valid value is default. + // + // PolicyName is a required field + PolicyName *string `min:"1" type:"string" required:"true"` +} + +// String returns the string representation +func (s PutKeyPolicyInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s PutKeyPolicyInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *PutKeyPolicyInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "PutKeyPolicyInput"} + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + if s.Policy == nil { + invalidParams.Add(request.NewErrParamRequired("Policy")) + } + if s.Policy != nil && len(*s.Policy) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Policy", 1)) + } + if s.PolicyName == nil { + invalidParams.Add(request.NewErrParamRequired("PolicyName")) + } + if s.PolicyName != nil && len(*s.PolicyName) < 1 { + invalidParams.Add(request.NewErrParamMinLen("PolicyName", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetBypassPolicyLockoutSafetyCheck sets the BypassPolicyLockoutSafetyCheck field's value. +func (s *PutKeyPolicyInput) SetBypassPolicyLockoutSafetyCheck(v bool) *PutKeyPolicyInput { + s.BypassPolicyLockoutSafetyCheck = &v + return s +} + +// SetKeyId sets the KeyId field's value. +func (s *PutKeyPolicyInput) SetKeyId(v string) *PutKeyPolicyInput { + s.KeyId = &v + return s +} + +// SetPolicy sets the Policy field's value. +func (s *PutKeyPolicyInput) SetPolicy(v string) *PutKeyPolicyInput { + s.Policy = &v + return s +} + +// SetPolicyName sets the PolicyName field's value. +func (s *PutKeyPolicyInput) SetPolicyName(v string) *PutKeyPolicyInput { + s.PolicyName = &v + return s +} + +type PutKeyPolicyOutput struct { + _ struct{} `type:"structure"` +} + +// String returns the string representation +func (s PutKeyPolicyOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s PutKeyPolicyOutput) GoString() string { + return s.String() +} + +type ReEncryptInput struct { + _ struct{} `type:"structure"` + + // Ciphertext of the data to reencrypt. + // + // CiphertextBlob is automatically base64 encoded/decoded by the SDK. + // + // CiphertextBlob is a required field + CiphertextBlob []byte `min:"1" type:"blob" required:"true"` + + // Specifies the encryption algorithm that AWS KMS will use to reecrypt the + // data after it has decrypted it. The default value, SYMMETRIC_DEFAULT, represents + // the encryption algorithm used for symmetric CMKs. + // + // This parameter is required only when the destination CMK is an asymmetric + // CMK. + DestinationEncryptionAlgorithm *string `type:"string" enum:"EncryptionAlgorithmSpec"` + + // Specifies that encryption context to use when the reencrypting the data. + // + // A destination encryption context is valid only when the destination CMK is + // a symmetric CMK. The standard ciphertext format for asymmetric CMKs does + // not include fields for metadata. + // + // An encryption context is a collection of non-secret key-value pairs that + // represents additional authenticated data. When you use an encryption context + // to encrypt data, you must specify the same (an exact case-sensitive match) + // encryption context to decrypt the data. An encryption context is optional + // when encrypting with a symmetric CMK, but it is highly recommended. + // + // For more information, see Encryption Context (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#encrypt_context) + // in the AWS Key Management Service Developer Guide. + DestinationEncryptionContext map[string]*string `type:"map"` + + // A unique identifier for the CMK that is used to reencrypt the data. Specify + // a symmetric or asymmetric CMK with a KeyUsage value of ENCRYPT_DECRYPT. To + // find the KeyUsage value of a CMK, use the DescribeKey operation. + // + // To specify a CMK, use its key ID, Amazon Resource Name (ARN), alias name, + // or alias ARN. When using an alias name, prefix it with "alias/". To specify + // a CMK in a different AWS account, you must use the key ARN or alias ARN. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Alias name: alias/ExampleAlias + // + // * Alias ARN: arn:aws:kms:us-east-2:111122223333:alias/ExampleAlias + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. To + // get the alias name and alias ARN, use ListAliases. + // + // DestinationKeyId is a required field + DestinationKeyId *string `min:"1" type:"string" required:"true"` + + // A list of grant tokens. + // + // For more information, see Grant Tokens (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#grant_token) + // in the AWS Key Management Service Developer Guide. + GrantTokens []*string `type:"list"` + + // Specifies the encryption algorithm that AWS KMS will use to decrypt the ciphertext + // before it is reencrypted. The default value, SYMMETRIC_DEFAULT, represents + // the algorithm used for symmetric CMKs. + // + // Specify the same algorithm that was used to encrypt the ciphertext. If you + // specify a different algorithm, the decrypt attempt fails. + // + // This parameter is required only when the ciphertext was encrypted under an + // asymmetric CMK. + SourceEncryptionAlgorithm *string `type:"string" enum:"EncryptionAlgorithmSpec"` + + // Specifies the encryption context to use to decrypt the ciphertext. Enter + // the same encryption context that was used to encrypt the ciphertext. + // + // An encryption context is a collection of non-secret key-value pairs that + // represents additional authenticated data. When you use an encryption context + // to encrypt data, you must specify the same (an exact case-sensitive match) + // encryption context to decrypt the data. An encryption context is optional + // when encrypting with a symmetric CMK, but it is highly recommended. + // + // For more information, see Encryption Context (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#encrypt_context) + // in the AWS Key Management Service Developer Guide. + SourceEncryptionContext map[string]*string `type:"map"` + + // A unique identifier for the CMK that is used to decrypt the ciphertext before + // it reencrypts it using the destination CMK. + // + // This parameter is required only when the ciphertext was encrypted under an + // asymmetric CMK. Otherwise, AWS KMS uses the metadata that it adds to the + // ciphertext blob to determine which CMK was used to encrypt the ciphertext. + // However, you can use this parameter to ensure that a particular CMK (of any + // kind) is used to decrypt the ciphertext before it is reencrypted. + // + // If you specify a KeyId value, the decrypt part of the ReEncrypt operation + // succeeds only if the specified CMK was used to encrypt the ciphertext. + // + // To specify a CMK, use its key ID, Amazon Resource Name (ARN), alias name, + // or alias ARN. When using an alias name, prefix it with "alias/". + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Alias name: alias/ExampleAlias + // + // * Alias ARN: arn:aws:kms:us-east-2:111122223333:alias/ExampleAlias + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. To + // get the alias name and alias ARN, use ListAliases. + SourceKeyId *string `min:"1" type:"string"` +} + +// String returns the string representation +func (s ReEncryptInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s ReEncryptInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *ReEncryptInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "ReEncryptInput"} + if s.CiphertextBlob == nil { + invalidParams.Add(request.NewErrParamRequired("CiphertextBlob")) + } + if s.CiphertextBlob != nil && len(s.CiphertextBlob) < 1 { + invalidParams.Add(request.NewErrParamMinLen("CiphertextBlob", 1)) + } + if s.DestinationKeyId == nil { + invalidParams.Add(request.NewErrParamRequired("DestinationKeyId")) + } + if s.DestinationKeyId != nil && len(*s.DestinationKeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("DestinationKeyId", 1)) + } + if s.SourceKeyId != nil && len(*s.SourceKeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("SourceKeyId", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetCiphertextBlob sets the CiphertextBlob field's value. +func (s *ReEncryptInput) SetCiphertextBlob(v []byte) *ReEncryptInput { + s.CiphertextBlob = v + return s +} + +// SetDestinationEncryptionAlgorithm sets the DestinationEncryptionAlgorithm field's value. +func (s *ReEncryptInput) SetDestinationEncryptionAlgorithm(v string) *ReEncryptInput { + s.DestinationEncryptionAlgorithm = &v + return s +} + +// SetDestinationEncryptionContext sets the DestinationEncryptionContext field's value. +func (s *ReEncryptInput) SetDestinationEncryptionContext(v map[string]*string) *ReEncryptInput { + s.DestinationEncryptionContext = v + return s +} + +// SetDestinationKeyId sets the DestinationKeyId field's value. +func (s *ReEncryptInput) SetDestinationKeyId(v string) *ReEncryptInput { + s.DestinationKeyId = &v + return s +} + +// SetGrantTokens sets the GrantTokens field's value. +func (s *ReEncryptInput) SetGrantTokens(v []*string) *ReEncryptInput { + s.GrantTokens = v + return s +} + +// SetSourceEncryptionAlgorithm sets the SourceEncryptionAlgorithm field's value. +func (s *ReEncryptInput) SetSourceEncryptionAlgorithm(v string) *ReEncryptInput { + s.SourceEncryptionAlgorithm = &v + return s +} + +// SetSourceEncryptionContext sets the SourceEncryptionContext field's value. +func (s *ReEncryptInput) SetSourceEncryptionContext(v map[string]*string) *ReEncryptInput { + s.SourceEncryptionContext = v + return s +} + +// SetSourceKeyId sets the SourceKeyId field's value. +func (s *ReEncryptInput) SetSourceKeyId(v string) *ReEncryptInput { + s.SourceKeyId = &v + return s +} + +type ReEncryptOutput struct { + _ struct{} `type:"structure"` + + // The reencrypted data. When you use the HTTP API or the AWS CLI, the value + // is Base64-encoded. Otherwise, it is not Base64-encoded. + // + // CiphertextBlob is automatically base64 encoded/decoded by the SDK. + CiphertextBlob []byte `min:"1" type:"blob"` + + // The encryption algorithm that was used to reencrypt the data. + DestinationEncryptionAlgorithm *string `type:"string" enum:"EncryptionAlgorithmSpec"` + + // The Amazon Resource Name (key ARN (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#key-id-key-ARN)) + // of the CMK that was used to reencrypt the data. + KeyId *string `min:"1" type:"string"` + + // The encryption algorithm that was used to decrypt the ciphertext before it + // was reencrypted. + SourceEncryptionAlgorithm *string `type:"string" enum:"EncryptionAlgorithmSpec"` + + // Unique identifier of the CMK used to originally encrypt the data. + SourceKeyId *string `min:"1" type:"string"` +} + +// String returns the string representation +func (s ReEncryptOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s ReEncryptOutput) GoString() string { + return s.String() +} + +// SetCiphertextBlob sets the CiphertextBlob field's value. +func (s *ReEncryptOutput) SetCiphertextBlob(v []byte) *ReEncryptOutput { + s.CiphertextBlob = v + return s +} + +// SetDestinationEncryptionAlgorithm sets the DestinationEncryptionAlgorithm field's value. +func (s *ReEncryptOutput) SetDestinationEncryptionAlgorithm(v string) *ReEncryptOutput { + s.DestinationEncryptionAlgorithm = &v + return s +} + +// SetKeyId sets the KeyId field's value. +func (s *ReEncryptOutput) SetKeyId(v string) *ReEncryptOutput { + s.KeyId = &v + return s +} + +// SetSourceEncryptionAlgorithm sets the SourceEncryptionAlgorithm field's value. +func (s *ReEncryptOutput) SetSourceEncryptionAlgorithm(v string) *ReEncryptOutput { + s.SourceEncryptionAlgorithm = &v + return s +} + +// SetSourceKeyId sets the SourceKeyId field's value. +func (s *ReEncryptOutput) SetSourceKeyId(v string) *ReEncryptOutput { + s.SourceKeyId = &v + return s +} + +type RetireGrantInput struct { + _ struct{} `type:"structure"` + + // Unique identifier of the grant to retire. The grant ID is returned in the + // response to a CreateGrant operation. + // + // * Grant ID Example - 0123456789012345678901234567890123456789012345678901234567890123 + GrantId *string `min:"1" type:"string"` + + // Token that identifies the grant to be retired. + GrantToken *string `min:"1" type:"string"` + + // The Amazon Resource Name (ARN) of the CMK associated with the grant. + // + // For example: arn:aws:kms:us-east-2:444455556666:key/1234abcd-12ab-34cd-56ef-1234567890ab + KeyId *string `min:"1" type:"string"` +} + +// String returns the string representation +func (s RetireGrantInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s RetireGrantInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *RetireGrantInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "RetireGrantInput"} + if s.GrantId != nil && len(*s.GrantId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("GrantId", 1)) + } + if s.GrantToken != nil && len(*s.GrantToken) < 1 { + invalidParams.Add(request.NewErrParamMinLen("GrantToken", 1)) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetGrantId sets the GrantId field's value. +func (s *RetireGrantInput) SetGrantId(v string) *RetireGrantInput { + s.GrantId = &v + return s +} + +// SetGrantToken sets the GrantToken field's value. +func (s *RetireGrantInput) SetGrantToken(v string) *RetireGrantInput { + s.GrantToken = &v + return s +} + +// SetKeyId sets the KeyId field's value. +func (s *RetireGrantInput) SetKeyId(v string) *RetireGrantInput { + s.KeyId = &v + return s +} + +type RetireGrantOutput struct { + _ struct{} `type:"structure"` +} + +// String returns the string representation +func (s RetireGrantOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s RetireGrantOutput) GoString() string { + return s.String() +} + +type RevokeGrantInput struct { + _ struct{} `type:"structure"` + + // Identifier of the grant to be revoked. + // + // GrantId is a required field + GrantId *string `min:"1" type:"string" required:"true"` + + // A unique identifier for the customer master key associated with the grant. + // + // Specify the key ID or the Amazon Resource Name (ARN) of the CMK. To specify + // a CMK in a different AWS account, you must use the key ARN. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` +} + +// String returns the string representation +func (s RevokeGrantInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s RevokeGrantInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *RevokeGrantInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "RevokeGrantInput"} + if s.GrantId == nil { + invalidParams.Add(request.NewErrParamRequired("GrantId")) + } + if s.GrantId != nil && len(*s.GrantId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("GrantId", 1)) + } + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetGrantId sets the GrantId field's value. +func (s *RevokeGrantInput) SetGrantId(v string) *RevokeGrantInput { + s.GrantId = &v + return s +} + +// SetKeyId sets the KeyId field's value. +func (s *RevokeGrantInput) SetKeyId(v string) *RevokeGrantInput { + s.KeyId = &v + return s +} + +type RevokeGrantOutput struct { + _ struct{} `type:"structure"` +} + +// String returns the string representation +func (s RevokeGrantOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s RevokeGrantOutput) GoString() string { + return s.String() +} + +type ScheduleKeyDeletionInput struct { + _ struct{} `type:"structure"` + + // The unique identifier of the customer master key (CMK) to delete. + // + // Specify the key ID or the Amazon Resource Name (ARN) of the CMK. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` + + // The waiting period, specified in number of days. After the waiting period + // ends, AWS KMS deletes the customer master key (CMK). + // + // This value is optional. If you include a value, it must be between 7 and + // 30, inclusive. If you do not include a value, it defaults to 30. + PendingWindowInDays *int64 `min:"1" type:"integer"` +} + +// String returns the string representation +func (s ScheduleKeyDeletionInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s ScheduleKeyDeletionInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *ScheduleKeyDeletionInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "ScheduleKeyDeletionInput"} + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + if s.PendingWindowInDays != nil && *s.PendingWindowInDays < 1 { + invalidParams.Add(request.NewErrParamMinValue("PendingWindowInDays", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetKeyId sets the KeyId field's value. +func (s *ScheduleKeyDeletionInput) SetKeyId(v string) *ScheduleKeyDeletionInput { + s.KeyId = &v + return s +} + +// SetPendingWindowInDays sets the PendingWindowInDays field's value. +func (s *ScheduleKeyDeletionInput) SetPendingWindowInDays(v int64) *ScheduleKeyDeletionInput { + s.PendingWindowInDays = &v + return s +} + +type ScheduleKeyDeletionOutput struct { + _ struct{} `type:"structure"` + + // The date and time after which AWS KMS deletes the customer master key (CMK). + DeletionDate *time.Time `type:"timestamp"` + + // The Amazon Resource Name (key ARN (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#key-id-key-ARN)) + // of the CMK whose deletion is scheduled. + KeyId *string `min:"1" type:"string"` +} + +// String returns the string representation +func (s ScheduleKeyDeletionOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s ScheduleKeyDeletionOutput) GoString() string { + return s.String() +} + +// SetDeletionDate sets the DeletionDate field's value. +func (s *ScheduleKeyDeletionOutput) SetDeletionDate(v time.Time) *ScheduleKeyDeletionOutput { + s.DeletionDate = &v + return s +} + +// SetKeyId sets the KeyId field's value. +func (s *ScheduleKeyDeletionOutput) SetKeyId(v string) *ScheduleKeyDeletionOutput { + s.KeyId = &v + return s +} + +type SignInput struct { + _ struct{} `type:"structure"` + + // A list of grant tokens. + // + // For more information, see Grant Tokens (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#grant_token) + // in the AWS Key Management Service Developer Guide. + GrantTokens []*string `type:"list"` + + // Identifies an asymmetric CMK. AWS KMS uses the private key in the asymmetric + // CMK to sign the message. The KeyUsage type of the CMK must be SIGN_VERIFY. + // To find the KeyUsage of a CMK, use the DescribeKey operation. + // + // To specify a CMK, use its key ID, Amazon Resource Name (ARN), alias name, + // or alias ARN. When using an alias name, prefix it with "alias/". To specify + // a CMK in a different AWS account, you must use the key ARN or alias ARN. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Alias name: alias/ExampleAlias + // + // * Alias ARN: arn:aws:kms:us-east-2:111122223333:alias/ExampleAlias + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. To + // get the alias name and alias ARN, use ListAliases. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` + + // Specifies the message or message digest to sign. Messages can be 0-4096 bytes. + // To sign a larger message, provide the message digest. + // + // If you provide a message, AWS KMS generates a hash digest of the message + // and then signs it. + // + // Message is automatically base64 encoded/decoded by the SDK. + // + // Message is a required field + Message []byte `min:"1" type:"blob" required:"true" sensitive:"true"` + + // Tells AWS KMS whether the value of the Message parameter is a message or + // message digest. The default value, RAW, indicates a message. To indicate + // a message digest, enter DIGEST. + MessageType *string `type:"string" enum:"MessageType"` + + // Specifies the signing algorithm to use when signing the message. + // + // Choose an algorithm that is compatible with the type and size of the specified + // asymmetric CMK. + // + // SigningAlgorithm is a required field + SigningAlgorithm *string `type:"string" required:"true" enum:"SigningAlgorithmSpec"` +} + +// String returns the string representation +func (s SignInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s SignInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *SignInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "SignInput"} + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + if s.Message == nil { + invalidParams.Add(request.NewErrParamRequired("Message")) + } + if s.Message != nil && len(s.Message) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Message", 1)) + } + if s.SigningAlgorithm == nil { + invalidParams.Add(request.NewErrParamRequired("SigningAlgorithm")) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetGrantTokens sets the GrantTokens field's value. +func (s *SignInput) SetGrantTokens(v []*string) *SignInput { + s.GrantTokens = v + return s +} + +// SetKeyId sets the KeyId field's value. +func (s *SignInput) SetKeyId(v string) *SignInput { + s.KeyId = &v + return s +} + +// SetMessage sets the Message field's value. +func (s *SignInput) SetMessage(v []byte) *SignInput { + s.Message = v + return s +} + +// SetMessageType sets the MessageType field's value. +func (s *SignInput) SetMessageType(v string) *SignInput { + s.MessageType = &v + return s +} + +// SetSigningAlgorithm sets the SigningAlgorithm field's value. +func (s *SignInput) SetSigningAlgorithm(v string) *SignInput { + s.SigningAlgorithm = &v + return s +} + +type SignOutput struct { + _ struct{} `type:"structure"` + + // The Amazon Resource Name (key ARN (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#key-id-key-ARN)) + // of the asymmetric CMK that was used to sign the message. + KeyId *string `min:"1" type:"string"` + + // The cryptographic signature that was generated for the message. + // + // * When used with the supported RSA signing algorithms, the encoding of + // this value is defined by PKCS #1 in RFC 8017 (https://tools.ietf.org/html/rfc8017). + // + // * When used with the ECDSA_SHA_256, ECDSA_SHA_384, or ECDSA_SHA_512 signing + // algorithms, this value is a DER-encoded object as defined by ANS X9.62–2005 + // and RFC 3279 Section 2.2.3 (https://tools.ietf.org/html/rfc3279#section-2.2.3). + // This is the most commonly used signature format and is appropriate for + // most uses. + // + // When you use the HTTP API or the AWS CLI, the value is Base64-encoded. Otherwise, + // it is not Base64-encoded. + // + // Signature is automatically base64 encoded/decoded by the SDK. + Signature []byte `min:"1" type:"blob"` + + // The signing algorithm that was used to sign the message. + SigningAlgorithm *string `type:"string" enum:"SigningAlgorithmSpec"` +} + +// String returns the string representation +func (s SignOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s SignOutput) GoString() string { + return s.String() +} + +// SetKeyId sets the KeyId field's value. +func (s *SignOutput) SetKeyId(v string) *SignOutput { + s.KeyId = &v + return s +} + +// SetSignature sets the Signature field's value. +func (s *SignOutput) SetSignature(v []byte) *SignOutput { + s.Signature = v + return s +} + +// SetSigningAlgorithm sets the SigningAlgorithm field's value. +func (s *SignOutput) SetSigningAlgorithm(v string) *SignOutput { + s.SigningAlgorithm = &v + return s +} + +// A key-value pair. A tag consists of a tag key and a tag value. Tag keys and +// tag values are both required, but tag values can be empty (null) strings. +// +// For information about the rules that apply to tag keys and tag values, see +// User-Defined Tag Restrictions (https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/allocation-tag-restrictions.html) +// in the AWS Billing and Cost Management User Guide. +type Tag struct { + _ struct{} `type:"structure"` + + // The key of the tag. + // + // TagKey is a required field + TagKey *string `min:"1" type:"string" required:"true"` + + // The value of the tag. + // + // TagValue is a required field + TagValue *string `type:"string" required:"true"` +} + +// String returns the string representation +func (s Tag) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s Tag) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *Tag) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "Tag"} + if s.TagKey == nil { + invalidParams.Add(request.NewErrParamRequired("TagKey")) + } + if s.TagKey != nil && len(*s.TagKey) < 1 { + invalidParams.Add(request.NewErrParamMinLen("TagKey", 1)) + } + if s.TagValue == nil { + invalidParams.Add(request.NewErrParamRequired("TagValue")) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetTagKey sets the TagKey field's value. +func (s *Tag) SetTagKey(v string) *Tag { + s.TagKey = &v + return s +} + +// SetTagValue sets the TagValue field's value. +func (s *Tag) SetTagValue(v string) *Tag { + s.TagValue = &v + return s +} + +// The request was rejected because one or more tags are not valid. +type TagException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s TagException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s TagException) GoString() string { + return s.String() +} + +func newErrorTagException(v protocol.ResponseMetadata) error { + return &TagException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *TagException) Code() string { + return "TagException" +} + +// Message returns the exception's message. +func (s *TagException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *TagException) OrigErr() error { + return nil +} + +func (s *TagException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *TagException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *TagException) RequestID() string { + return s.RespMetadata.RequestID +} + +type TagResourceInput struct { + _ struct{} `type:"structure"` + + // A unique identifier for the CMK you are tagging. + // + // Specify the key ID or the Amazon Resource Name (ARN) of the CMK. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` + + // One or more tags. Each tag consists of a tag key and a tag value. + // + // Tags is a required field + Tags []*Tag `type:"list" required:"true"` +} + +// String returns the string representation +func (s TagResourceInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s TagResourceInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *TagResourceInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "TagResourceInput"} + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + if s.Tags == nil { + invalidParams.Add(request.NewErrParamRequired("Tags")) + } + if s.Tags != nil { + for i, v := range s.Tags { + if v == nil { + continue + } + if err := v.Validate(); err != nil { + invalidParams.AddNested(fmt.Sprintf("%s[%v]", "Tags", i), err.(request.ErrInvalidParams)) + } + } + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetKeyId sets the KeyId field's value. +func (s *TagResourceInput) SetKeyId(v string) *TagResourceInput { + s.KeyId = &v + return s +} + +// SetTags sets the Tags field's value. +func (s *TagResourceInput) SetTags(v []*Tag) *TagResourceInput { + s.Tags = v + return s +} + +type TagResourceOutput struct { + _ struct{} `type:"structure"` +} + +// String returns the string representation +func (s TagResourceOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s TagResourceOutput) GoString() string { + return s.String() +} + +// The request was rejected because a specified parameter is not supported or +// a specified resource is not valid for this operation. +type UnsupportedOperationException struct { + _ struct{} `type:"structure"` + RespMetadata protocol.ResponseMetadata `json:"-" xml:"-"` + + Message_ *string `locationName:"message" type:"string"` +} + +// String returns the string representation +func (s UnsupportedOperationException) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s UnsupportedOperationException) GoString() string { + return s.String() +} + +func newErrorUnsupportedOperationException(v protocol.ResponseMetadata) error { + return &UnsupportedOperationException{ + RespMetadata: v, + } +} + +// Code returns the exception type name. +func (s *UnsupportedOperationException) Code() string { + return "UnsupportedOperationException" +} + +// Message returns the exception's message. +func (s *UnsupportedOperationException) Message() string { + if s.Message_ != nil { + return *s.Message_ + } + return "" +} + +// OrigErr always returns nil, satisfies awserr.Error interface. +func (s *UnsupportedOperationException) OrigErr() error { + return nil +} + +func (s *UnsupportedOperationException) Error() string { + return fmt.Sprintf("%s: %s", s.Code(), s.Message()) +} + +// Status code returns the HTTP status code for the request's response error. +func (s *UnsupportedOperationException) StatusCode() int { + return s.RespMetadata.StatusCode +} + +// RequestID returns the service's response RequestID for request. +func (s *UnsupportedOperationException) RequestID() string { + return s.RespMetadata.RequestID +} + +type UntagResourceInput struct { + _ struct{} `type:"structure"` + + // A unique identifier for the CMK from which you are removing tags. + // + // Specify the key ID or the Amazon Resource Name (ARN) of the CMK. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` + + // One or more tag keys. Specify only the tag keys, not the tag values. + // + // TagKeys is a required field + TagKeys []*string `type:"list" required:"true"` +} + +// String returns the string representation +func (s UntagResourceInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s UntagResourceInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *UntagResourceInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "UntagResourceInput"} + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + if s.TagKeys == nil { + invalidParams.Add(request.NewErrParamRequired("TagKeys")) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetKeyId sets the KeyId field's value. +func (s *UntagResourceInput) SetKeyId(v string) *UntagResourceInput { + s.KeyId = &v + return s +} + +// SetTagKeys sets the TagKeys field's value. +func (s *UntagResourceInput) SetTagKeys(v []*string) *UntagResourceInput { + s.TagKeys = v + return s +} + +type UntagResourceOutput struct { + _ struct{} `type:"structure"` +} + +// String returns the string representation +func (s UntagResourceOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s UntagResourceOutput) GoString() string { + return s.String() +} + +type UpdateAliasInput struct { + _ struct{} `type:"structure"` + + // Identifies the alias that is changing its CMK. This value must begin with + // alias/ followed by the alias name, such as alias/ExampleAlias. You cannot + // use UpdateAlias to change the alias name. + // + // AliasName is a required field + AliasName *string `min:"1" type:"string" required:"true"` + + // Identifies the CMK to associate with the alias. When the update operation + // completes, the alias will point to this CMK. + // + // The CMK must be in the same AWS account and Region as the alias. Also, the + // new target CMK must be the same type as the current target CMK (both symmetric + // or both asymmetric) and they must have the same key usage. + // + // Specify the key ID or the Amazon Resource Name (ARN) of the CMK. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. + // + // To verify that the alias is mapped to the correct CMK, use ListAliases. + // + // TargetKeyId is a required field + TargetKeyId *string `min:"1" type:"string" required:"true"` +} + +// String returns the string representation +func (s UpdateAliasInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s UpdateAliasInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *UpdateAliasInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "UpdateAliasInput"} + if s.AliasName == nil { + invalidParams.Add(request.NewErrParamRequired("AliasName")) + } + if s.AliasName != nil && len(*s.AliasName) < 1 { + invalidParams.Add(request.NewErrParamMinLen("AliasName", 1)) + } + if s.TargetKeyId == nil { + invalidParams.Add(request.NewErrParamRequired("TargetKeyId")) + } + if s.TargetKeyId != nil && len(*s.TargetKeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("TargetKeyId", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetAliasName sets the AliasName field's value. +func (s *UpdateAliasInput) SetAliasName(v string) *UpdateAliasInput { + s.AliasName = &v + return s +} + +// SetTargetKeyId sets the TargetKeyId field's value. +func (s *UpdateAliasInput) SetTargetKeyId(v string) *UpdateAliasInput { + s.TargetKeyId = &v + return s +} + +type UpdateAliasOutput struct { + _ struct{} `type:"structure"` +} + +// String returns the string representation +func (s UpdateAliasOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s UpdateAliasOutput) GoString() string { + return s.String() +} + +type UpdateCustomKeyStoreInput struct { + _ struct{} `type:"structure"` + + // Associates the custom key store with a related AWS CloudHSM cluster. + // + // Enter the cluster ID of the cluster that you used to create the custom key + // store or a cluster that shares a backup history and has the same cluster + // certificate as the original cluster. You cannot use this parameter to associate + // a custom key store with an unrelated cluster. In addition, the replacement + // cluster must fulfill the requirements (https://docs.aws.amazon.com/kms/latest/developerguide/create-keystore.html#before-keystore) + // for a cluster associated with a custom key store. To view the cluster certificate + // of a cluster, use the DescribeClusters (https://docs.aws.amazon.com/cloudhsm/latest/APIReference/API_DescribeClusters.html) + // operation. + CloudHsmClusterId *string `min:"19" type:"string"` + + // Identifies the custom key store that you want to update. Enter the ID of + // the custom key store. To find the ID of a custom key store, use the DescribeCustomKeyStores + // operation. + // + // CustomKeyStoreId is a required field + CustomKeyStoreId *string `min:"1" type:"string" required:"true"` + + // Enter the current password of the kmsuser crypto user (CU) in the AWS CloudHSM + // cluster that is associated with the custom key store. + // + // This parameter tells AWS KMS the current password of the kmsuser crypto user + // (CU). It does not set or change the password of any users in the AWS CloudHSM + // cluster. + KeyStorePassword *string `min:"7" type:"string" sensitive:"true"` + + // Changes the friendly name of the custom key store to the value that you specify. + // The custom key store name must be unique in the AWS account. + NewCustomKeyStoreName *string `min:"1" type:"string"` +} + +// String returns the string representation +func (s UpdateCustomKeyStoreInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s UpdateCustomKeyStoreInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *UpdateCustomKeyStoreInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "UpdateCustomKeyStoreInput"} + if s.CloudHsmClusterId != nil && len(*s.CloudHsmClusterId) < 19 { + invalidParams.Add(request.NewErrParamMinLen("CloudHsmClusterId", 19)) + } + if s.CustomKeyStoreId == nil { + invalidParams.Add(request.NewErrParamRequired("CustomKeyStoreId")) + } + if s.CustomKeyStoreId != nil && len(*s.CustomKeyStoreId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("CustomKeyStoreId", 1)) + } + if s.KeyStorePassword != nil && len(*s.KeyStorePassword) < 7 { + invalidParams.Add(request.NewErrParamMinLen("KeyStorePassword", 7)) + } + if s.NewCustomKeyStoreName != nil && len(*s.NewCustomKeyStoreName) < 1 { + invalidParams.Add(request.NewErrParamMinLen("NewCustomKeyStoreName", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetCloudHsmClusterId sets the CloudHsmClusterId field's value. +func (s *UpdateCustomKeyStoreInput) SetCloudHsmClusterId(v string) *UpdateCustomKeyStoreInput { + s.CloudHsmClusterId = &v + return s +} + +// SetCustomKeyStoreId sets the CustomKeyStoreId field's value. +func (s *UpdateCustomKeyStoreInput) SetCustomKeyStoreId(v string) *UpdateCustomKeyStoreInput { + s.CustomKeyStoreId = &v + return s +} + +// SetKeyStorePassword sets the KeyStorePassword field's value. +func (s *UpdateCustomKeyStoreInput) SetKeyStorePassword(v string) *UpdateCustomKeyStoreInput { + s.KeyStorePassword = &v + return s +} + +// SetNewCustomKeyStoreName sets the NewCustomKeyStoreName field's value. +func (s *UpdateCustomKeyStoreInput) SetNewCustomKeyStoreName(v string) *UpdateCustomKeyStoreInput { + s.NewCustomKeyStoreName = &v + return s +} + +type UpdateCustomKeyStoreOutput struct { + _ struct{} `type:"structure"` +} + +// String returns the string representation +func (s UpdateCustomKeyStoreOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s UpdateCustomKeyStoreOutput) GoString() string { + return s.String() +} + +type UpdateKeyDescriptionInput struct { + _ struct{} `type:"structure"` + + // New description for the CMK. + // + // Description is a required field + Description *string `type:"string" required:"true"` + + // A unique identifier for the customer master key (CMK). + // + // Specify the key ID or the Amazon Resource Name (ARN) of the CMK. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` +} + +// String returns the string representation +func (s UpdateKeyDescriptionInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s UpdateKeyDescriptionInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *UpdateKeyDescriptionInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "UpdateKeyDescriptionInput"} + if s.Description == nil { + invalidParams.Add(request.NewErrParamRequired("Description")) + } + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetDescription sets the Description field's value. +func (s *UpdateKeyDescriptionInput) SetDescription(v string) *UpdateKeyDescriptionInput { + s.Description = &v + return s +} + +// SetKeyId sets the KeyId field's value. +func (s *UpdateKeyDescriptionInput) SetKeyId(v string) *UpdateKeyDescriptionInput { + s.KeyId = &v + return s +} + +type UpdateKeyDescriptionOutput struct { + _ struct{} `type:"structure"` +} + +// String returns the string representation +func (s UpdateKeyDescriptionOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s UpdateKeyDescriptionOutput) GoString() string { + return s.String() +} + +type VerifyInput struct { + _ struct{} `type:"structure"` + + // A list of grant tokens. + // + // For more information, see Grant Tokens (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#grant_token) + // in the AWS Key Management Service Developer Guide. + GrantTokens []*string `type:"list"` + + // Identifies the asymmetric CMK that will be used to verify the signature. + // This must be the same CMK that was used to generate the signature. If you + // specify a different CMK, the signature verification fails. + // + // To specify a CMK, use its key ID, Amazon Resource Name (ARN), alias name, + // or alias ARN. When using an alias name, prefix it with "alias/". To specify + // a CMK in a different AWS account, you must use the key ARN or alias ARN. + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Alias name: alias/ExampleAlias + // + // * Alias ARN: arn:aws:kms:us-east-2:111122223333:alias/ExampleAlias + // + // To get the key ID and key ARN for a CMK, use ListKeys or DescribeKey. To + // get the alias name and alias ARN, use ListAliases. + // + // KeyId is a required field + KeyId *string `min:"1" type:"string" required:"true"` + + // Specifies the message that was signed. You can submit a raw message of up + // to 4096 bytes, or a hash digest of the message. If you submit a digest, use + // the MessageType parameter with a value of DIGEST. + // + // If the message specified here is different from the message that was signed, + // the signature verification fails. A message and its hash digest are considered + // to be the same message. + // + // Message is automatically base64 encoded/decoded by the SDK. + // + // Message is a required field + Message []byte `min:"1" type:"blob" required:"true" sensitive:"true"` + + // Tells AWS KMS whether the value of the Message parameter is a message or + // message digest. The default value, RAW, indicates a message. To indicate + // a message digest, enter DIGEST. + // + // Use the DIGEST value only when the value of the Message parameter is a message + // digest. If you use the DIGEST value with a raw message, the security of the + // verification operation can be compromised. + MessageType *string `type:"string" enum:"MessageType"` + + // The signature that the Sign operation generated. + // + // Signature is automatically base64 encoded/decoded by the SDK. + // + // Signature is a required field + Signature []byte `min:"1" type:"blob" required:"true"` + + // The signing algorithm that was used to sign the message. If you submit a + // different algorithm, the signature verification fails. + // + // SigningAlgorithm is a required field + SigningAlgorithm *string `type:"string" required:"true" enum:"SigningAlgorithmSpec"` +} + +// String returns the string representation +func (s VerifyInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s VerifyInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *VerifyInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "VerifyInput"} + if s.KeyId == nil { + invalidParams.Add(request.NewErrParamRequired("KeyId")) + } + if s.KeyId != nil && len(*s.KeyId) < 1 { + invalidParams.Add(request.NewErrParamMinLen("KeyId", 1)) + } + if s.Message == nil { + invalidParams.Add(request.NewErrParamRequired("Message")) + } + if s.Message != nil && len(s.Message) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Message", 1)) + } + if s.Signature == nil { + invalidParams.Add(request.NewErrParamRequired("Signature")) + } + if s.Signature != nil && len(s.Signature) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Signature", 1)) + } + if s.SigningAlgorithm == nil { + invalidParams.Add(request.NewErrParamRequired("SigningAlgorithm")) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetGrantTokens sets the GrantTokens field's value. +func (s *VerifyInput) SetGrantTokens(v []*string) *VerifyInput { + s.GrantTokens = v + return s +} + +// SetKeyId sets the KeyId field's value. +func (s *VerifyInput) SetKeyId(v string) *VerifyInput { + s.KeyId = &v + return s +} + +// SetMessage sets the Message field's value. +func (s *VerifyInput) SetMessage(v []byte) *VerifyInput { + s.Message = v + return s +} + +// SetMessageType sets the MessageType field's value. +func (s *VerifyInput) SetMessageType(v string) *VerifyInput { + s.MessageType = &v + return s +} + +// SetSignature sets the Signature field's value. +func (s *VerifyInput) SetSignature(v []byte) *VerifyInput { + s.Signature = v + return s +} + +// SetSigningAlgorithm sets the SigningAlgorithm field's value. +func (s *VerifyInput) SetSigningAlgorithm(v string) *VerifyInput { + s.SigningAlgorithm = &v + return s +} + +type VerifyOutput struct { + _ struct{} `type:"structure"` + + // The Amazon Resource Name (key ARN (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#key-id-key-ARN)) + // of the asymmetric CMK that was used to verify the signature. + KeyId *string `min:"1" type:"string"` + + // A Boolean value that indicates whether the signature was verified. A value + // of True indicates that the Signature was produced by signing the Message + // with the specified KeyID and SigningAlgorithm. If the signature is not verified, + // the Verify operation fails with a KMSInvalidSignatureException exception. + SignatureValid *bool `type:"boolean"` + + // The signing algorithm that was used to verify the signature. + SigningAlgorithm *string `type:"string" enum:"SigningAlgorithmSpec"` +} + +// String returns the string representation +func (s VerifyOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s VerifyOutput) GoString() string { + return s.String() +} + +// SetKeyId sets the KeyId field's value. +func (s *VerifyOutput) SetKeyId(v string) *VerifyOutput { + s.KeyId = &v + return s +} + +// SetSignatureValid sets the SignatureValid field's value. +func (s *VerifyOutput) SetSignatureValid(v bool) *VerifyOutput { + s.SignatureValid = &v + return s +} + +// SetSigningAlgorithm sets the SigningAlgorithm field's value. +func (s *VerifyOutput) SetSigningAlgorithm(v string) *VerifyOutput { + s.SigningAlgorithm = &v + return s +} + +const ( + // AlgorithmSpecRsaesPkcs1V15 is a AlgorithmSpec enum value + AlgorithmSpecRsaesPkcs1V15 = "RSAES_PKCS1_V1_5" + + // AlgorithmSpecRsaesOaepSha1 is a AlgorithmSpec enum value + AlgorithmSpecRsaesOaepSha1 = "RSAES_OAEP_SHA_1" + + // AlgorithmSpecRsaesOaepSha256 is a AlgorithmSpec enum value + AlgorithmSpecRsaesOaepSha256 = "RSAES_OAEP_SHA_256" +) + +const ( + // ConnectionErrorCodeTypeInvalidCredentials is a ConnectionErrorCodeType enum value + ConnectionErrorCodeTypeInvalidCredentials = "INVALID_CREDENTIALS" + + // ConnectionErrorCodeTypeClusterNotFound is a ConnectionErrorCodeType enum value + ConnectionErrorCodeTypeClusterNotFound = "CLUSTER_NOT_FOUND" + + // ConnectionErrorCodeTypeNetworkErrors is a ConnectionErrorCodeType enum value + ConnectionErrorCodeTypeNetworkErrors = "NETWORK_ERRORS" + + // ConnectionErrorCodeTypeInternalError is a ConnectionErrorCodeType enum value + ConnectionErrorCodeTypeInternalError = "INTERNAL_ERROR" + + // ConnectionErrorCodeTypeInsufficientCloudhsmHsms is a ConnectionErrorCodeType enum value + ConnectionErrorCodeTypeInsufficientCloudhsmHsms = "INSUFFICIENT_CLOUDHSM_HSMS" + + // ConnectionErrorCodeTypeUserLockedOut is a ConnectionErrorCodeType enum value + ConnectionErrorCodeTypeUserLockedOut = "USER_LOCKED_OUT" + + // ConnectionErrorCodeTypeUserNotFound is a ConnectionErrorCodeType enum value + ConnectionErrorCodeTypeUserNotFound = "USER_NOT_FOUND" + + // ConnectionErrorCodeTypeUserLoggedIn is a ConnectionErrorCodeType enum value + ConnectionErrorCodeTypeUserLoggedIn = "USER_LOGGED_IN" + + // ConnectionErrorCodeTypeSubnetNotFound is a ConnectionErrorCodeType enum value + ConnectionErrorCodeTypeSubnetNotFound = "SUBNET_NOT_FOUND" +) + +const ( + // ConnectionStateTypeConnected is a ConnectionStateType enum value + ConnectionStateTypeConnected = "CONNECTED" + + // ConnectionStateTypeConnecting is a ConnectionStateType enum value + ConnectionStateTypeConnecting = "CONNECTING" + + // ConnectionStateTypeFailed is a ConnectionStateType enum value + ConnectionStateTypeFailed = "FAILED" + + // ConnectionStateTypeDisconnected is a ConnectionStateType enum value + ConnectionStateTypeDisconnected = "DISCONNECTED" + + // ConnectionStateTypeDisconnecting is a ConnectionStateType enum value + ConnectionStateTypeDisconnecting = "DISCONNECTING" +) + +const ( + // CustomerMasterKeySpecRsa2048 is a CustomerMasterKeySpec enum value + CustomerMasterKeySpecRsa2048 = "RSA_2048" + + // CustomerMasterKeySpecRsa3072 is a CustomerMasterKeySpec enum value + CustomerMasterKeySpecRsa3072 = "RSA_3072" + + // CustomerMasterKeySpecRsa4096 is a CustomerMasterKeySpec enum value + CustomerMasterKeySpecRsa4096 = "RSA_4096" + + // CustomerMasterKeySpecEccNistP256 is a CustomerMasterKeySpec enum value + CustomerMasterKeySpecEccNistP256 = "ECC_NIST_P256" + + // CustomerMasterKeySpecEccNistP384 is a CustomerMasterKeySpec enum value + CustomerMasterKeySpecEccNistP384 = "ECC_NIST_P384" + + // CustomerMasterKeySpecEccNistP521 is a CustomerMasterKeySpec enum value + CustomerMasterKeySpecEccNistP521 = "ECC_NIST_P521" + + // CustomerMasterKeySpecEccSecgP256k1 is a CustomerMasterKeySpec enum value + CustomerMasterKeySpecEccSecgP256k1 = "ECC_SECG_P256K1" + + // CustomerMasterKeySpecSymmetricDefault is a CustomerMasterKeySpec enum value + CustomerMasterKeySpecSymmetricDefault = "SYMMETRIC_DEFAULT" +) + +const ( + // DataKeyPairSpecRsa2048 is a DataKeyPairSpec enum value + DataKeyPairSpecRsa2048 = "RSA_2048" + + // DataKeyPairSpecRsa3072 is a DataKeyPairSpec enum value + DataKeyPairSpecRsa3072 = "RSA_3072" + + // DataKeyPairSpecRsa4096 is a DataKeyPairSpec enum value + DataKeyPairSpecRsa4096 = "RSA_4096" + + // DataKeyPairSpecEccNistP256 is a DataKeyPairSpec enum value + DataKeyPairSpecEccNistP256 = "ECC_NIST_P256" + + // DataKeyPairSpecEccNistP384 is a DataKeyPairSpec enum value + DataKeyPairSpecEccNistP384 = "ECC_NIST_P384" + + // DataKeyPairSpecEccNistP521 is a DataKeyPairSpec enum value + DataKeyPairSpecEccNistP521 = "ECC_NIST_P521" + + // DataKeyPairSpecEccSecgP256k1 is a DataKeyPairSpec enum value + DataKeyPairSpecEccSecgP256k1 = "ECC_SECG_P256K1" +) + +const ( + // DataKeySpecAes256 is a DataKeySpec enum value + DataKeySpecAes256 = "AES_256" + + // DataKeySpecAes128 is a DataKeySpec enum value + DataKeySpecAes128 = "AES_128" +) + +const ( + // EncryptionAlgorithmSpecSymmetricDefault is a EncryptionAlgorithmSpec enum value + EncryptionAlgorithmSpecSymmetricDefault = "SYMMETRIC_DEFAULT" + + // EncryptionAlgorithmSpecRsaesOaepSha1 is a EncryptionAlgorithmSpec enum value + EncryptionAlgorithmSpecRsaesOaepSha1 = "RSAES_OAEP_SHA_1" + + // EncryptionAlgorithmSpecRsaesOaepSha256 is a EncryptionAlgorithmSpec enum value + EncryptionAlgorithmSpecRsaesOaepSha256 = "RSAES_OAEP_SHA_256" +) + +const ( + // ExpirationModelTypeKeyMaterialExpires is a ExpirationModelType enum value + ExpirationModelTypeKeyMaterialExpires = "KEY_MATERIAL_EXPIRES" + + // ExpirationModelTypeKeyMaterialDoesNotExpire is a ExpirationModelType enum value + ExpirationModelTypeKeyMaterialDoesNotExpire = "KEY_MATERIAL_DOES_NOT_EXPIRE" +) + +const ( + // GrantOperationDecrypt is a GrantOperation enum value + GrantOperationDecrypt = "Decrypt" + + // GrantOperationEncrypt is a GrantOperation enum value + GrantOperationEncrypt = "Encrypt" + + // GrantOperationGenerateDataKey is a GrantOperation enum value + GrantOperationGenerateDataKey = "GenerateDataKey" + + // GrantOperationGenerateDataKeyWithoutPlaintext is a GrantOperation enum value + GrantOperationGenerateDataKeyWithoutPlaintext = "GenerateDataKeyWithoutPlaintext" + + // GrantOperationReEncryptFrom is a GrantOperation enum value + GrantOperationReEncryptFrom = "ReEncryptFrom" + + // GrantOperationReEncryptTo is a GrantOperation enum value + GrantOperationReEncryptTo = "ReEncryptTo" + + // GrantOperationSign is a GrantOperation enum value + GrantOperationSign = "Sign" + + // GrantOperationVerify is a GrantOperation enum value + GrantOperationVerify = "Verify" + + // GrantOperationGetPublicKey is a GrantOperation enum value + GrantOperationGetPublicKey = "GetPublicKey" + + // GrantOperationCreateGrant is a GrantOperation enum value + GrantOperationCreateGrant = "CreateGrant" + + // GrantOperationRetireGrant is a GrantOperation enum value + GrantOperationRetireGrant = "RetireGrant" + + // GrantOperationDescribeKey is a GrantOperation enum value + GrantOperationDescribeKey = "DescribeKey" + + // GrantOperationGenerateDataKeyPair is a GrantOperation enum value + GrantOperationGenerateDataKeyPair = "GenerateDataKeyPair" + + // GrantOperationGenerateDataKeyPairWithoutPlaintext is a GrantOperation enum value + GrantOperationGenerateDataKeyPairWithoutPlaintext = "GenerateDataKeyPairWithoutPlaintext" +) + +const ( + // KeyManagerTypeAws is a KeyManagerType enum value + KeyManagerTypeAws = "AWS" + + // KeyManagerTypeCustomer is a KeyManagerType enum value + KeyManagerTypeCustomer = "CUSTOMER" +) + +const ( + // KeyStateEnabled is a KeyState enum value + KeyStateEnabled = "Enabled" + + // KeyStateDisabled is a KeyState enum value + KeyStateDisabled = "Disabled" + + // KeyStatePendingDeletion is a KeyState enum value + KeyStatePendingDeletion = "PendingDeletion" + + // KeyStatePendingImport is a KeyState enum value + KeyStatePendingImport = "PendingImport" + + // KeyStateUnavailable is a KeyState enum value + KeyStateUnavailable = "Unavailable" +) + +const ( + // KeyUsageTypeSignVerify is a KeyUsageType enum value + KeyUsageTypeSignVerify = "SIGN_VERIFY" + + // KeyUsageTypeEncryptDecrypt is a KeyUsageType enum value + KeyUsageTypeEncryptDecrypt = "ENCRYPT_DECRYPT" +) + +const ( + // MessageTypeRaw is a MessageType enum value + MessageTypeRaw = "RAW" + + // MessageTypeDigest is a MessageType enum value + MessageTypeDigest = "DIGEST" +) + +const ( + // OriginTypeAwsKms is a OriginType enum value + OriginTypeAwsKms = "AWS_KMS" + + // OriginTypeExternal is a OriginType enum value + OriginTypeExternal = "EXTERNAL" + + // OriginTypeAwsCloudhsm is a OriginType enum value + OriginTypeAwsCloudhsm = "AWS_CLOUDHSM" +) + +const ( + // SigningAlgorithmSpecRsassaPssSha256 is a SigningAlgorithmSpec enum value + SigningAlgorithmSpecRsassaPssSha256 = "RSASSA_PSS_SHA_256" + + // SigningAlgorithmSpecRsassaPssSha384 is a SigningAlgorithmSpec enum value + SigningAlgorithmSpecRsassaPssSha384 = "RSASSA_PSS_SHA_384" + + // SigningAlgorithmSpecRsassaPssSha512 is a SigningAlgorithmSpec enum value + SigningAlgorithmSpecRsassaPssSha512 = "RSASSA_PSS_SHA_512" + + // SigningAlgorithmSpecRsassaPkcs1V15Sha256 is a SigningAlgorithmSpec enum value + SigningAlgorithmSpecRsassaPkcs1V15Sha256 = "RSASSA_PKCS1_V1_5_SHA_256" + + // SigningAlgorithmSpecRsassaPkcs1V15Sha384 is a SigningAlgorithmSpec enum value + SigningAlgorithmSpecRsassaPkcs1V15Sha384 = "RSASSA_PKCS1_V1_5_SHA_384" + + // SigningAlgorithmSpecRsassaPkcs1V15Sha512 is a SigningAlgorithmSpec enum value + SigningAlgorithmSpecRsassaPkcs1V15Sha512 = "RSASSA_PKCS1_V1_5_SHA_512" + + // SigningAlgorithmSpecEcdsaSha256 is a SigningAlgorithmSpec enum value + SigningAlgorithmSpecEcdsaSha256 = "ECDSA_SHA_256" + + // SigningAlgorithmSpecEcdsaSha384 is a SigningAlgorithmSpec enum value + SigningAlgorithmSpecEcdsaSha384 = "ECDSA_SHA_384" + + // SigningAlgorithmSpecEcdsaSha512 is a SigningAlgorithmSpec enum value + SigningAlgorithmSpecEcdsaSha512 = "ECDSA_SHA_512" +) + +const ( + // WrappingKeySpecRsa2048 is a WrappingKeySpec enum value + WrappingKeySpecRsa2048 = "RSA_2048" +) diff --git a/github.com/aws/aws-sdk-go/service/kms/doc.go b/github.com/aws/aws-sdk-go/service/kms/doc.go new file mode 100644 index 0000000000..c4c2125020 --- /dev/null +++ b/github.com/aws/aws-sdk-go/service/kms/doc.go @@ -0,0 +1,98 @@ +// Code generated by private/model/cli/gen-api/main.go. DO NOT EDIT. + +// Package kms provides the client and types for making API +// requests to AWS Key Management Service. +// +// AWS Key Management Service (AWS KMS) is an encryption and key management +// web service. This guide describes the AWS KMS operations that you can call +// programmatically. For general information about AWS KMS, see the AWS Key +// Management Service Developer Guide (https://docs.aws.amazon.com/kms/latest/developerguide/). +// +// AWS provides SDKs that consist of libraries and sample code for various programming +// languages and platforms (Java, Ruby, .Net, macOS, Android, etc.). The SDKs +// provide a convenient way to create programmatic access to AWS KMS and other +// AWS services. For example, the SDKs take care of tasks such as signing requests +// (see below), managing errors, and retrying requests automatically. For more +// information about the AWS SDKs, including how to download and install them, +// see Tools for Amazon Web Services (http://aws.amazon.com/tools/). +// +// We recommend that you use the AWS SDKs to make programmatic API calls to +// AWS KMS. +// +// Clients must support TLS (Transport Layer Security) 1.0. We recommend TLS +// 1.2. Clients must also support cipher suites with Perfect Forward Secrecy +// (PFS) such as Ephemeral Diffie-Hellman (DHE) or Elliptic Curve Ephemeral +// Diffie-Hellman (ECDHE). Most modern systems such as Java 7 and later support +// these modes. +// +// Signing Requests +// +// Requests must be signed by using an access key ID and a secret access key. +// We strongly recommend that you do not use your AWS account (root) access +// key ID and secret key for everyday work with AWS KMS. Instead, use the access +// key ID and secret access key for an IAM user. You can also use the AWS Security +// Token Service to generate temporary security credentials that you can use +// to sign requests. +// +// All AWS KMS operations require Signature Version 4 (https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html). +// +// Logging API Requests +// +// AWS KMS supports AWS CloudTrail, a service that logs AWS API calls and related +// events for your AWS account and delivers them to an Amazon S3 bucket that +// you specify. By using the information collected by CloudTrail, you can determine +// what requests were made to AWS KMS, who made the request, when it was made, +// and so on. To learn more about CloudTrail, including how to turn it on and +// find your log files, see the AWS CloudTrail User Guide (https://docs.aws.amazon.com/awscloudtrail/latest/userguide/). +// +// Additional Resources +// +// For more information about credentials and request signing, see the following: +// +// * AWS Security Credentials (https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html) +// - This topic provides general information about the types of credentials +// used for accessing AWS. +// +// * Temporary Security Credentials (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html) +// - This section of the IAM User Guide describes how to create and use temporary +// security credentials. +// +// * Signature Version 4 Signing Process (https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html) +// - This set of topics walks you through the process of signing a request +// using an access key ID and a secret access key. +// +// Commonly Used API Operations +// +// Of the API operations discussed in this guide, the following will prove the +// most useful for most applications. You will likely perform operations other +// than these, such as creating keys and assigning policies, by using the console. +// +// * Encrypt +// +// * Decrypt +// +// * GenerateDataKey +// +// * GenerateDataKeyWithoutPlaintext +// +// See https://docs.aws.amazon.com/goto/WebAPI/kms-2014-11-01 for more information on this service. +// +// See kms package documentation for more information. +// https://docs.aws.amazon.com/sdk-for-go/api/service/kms/ +// +// Using the Client +// +// To contact AWS Key Management Service with the SDK use the New function to create +// a new service client. With that client you can make API requests to the service. +// These clients are safe to use concurrently. +// +// See the SDK's documentation for more information on how to use the SDK. +// https://docs.aws.amazon.com/sdk-for-go/api/ +// +// See aws.Config documentation for more information on configuring SDK clients. +// https://docs.aws.amazon.com/sdk-for-go/api/aws/#Config +// +// See the AWS Key Management Service client KMS for more +// information on creating client for this service. +// https://docs.aws.amazon.com/sdk-for-go/api/service/kms/#New +package kms diff --git a/github.com/aws/aws-sdk-go/service/kms/errors.go b/github.com/aws/aws-sdk-go/service/kms/errors.go new file mode 100644 index 0000000000..911bf576eb --- /dev/null +++ b/github.com/aws/aws-sdk-go/service/kms/errors.go @@ -0,0 +1,367 @@ +// Code generated by private/model/cli/gen-api/main.go. DO NOT EDIT. + +package kms + +import ( + "github.com/aws/aws-sdk-go/private/protocol" +) + +const ( + + // ErrCodeAlreadyExistsException for service response error code + // "AlreadyExistsException". + // + // The request was rejected because it attempted to create a resource that already + // exists. + ErrCodeAlreadyExistsException = "AlreadyExistsException" + + // ErrCodeCloudHsmClusterInUseException for service response error code + // "CloudHsmClusterInUseException". + // + // The request was rejected because the specified AWS CloudHSM cluster is already + // associated with a custom key store or it shares a backup history with a cluster + // that is associated with a custom key store. Each custom key store must be + // associated with a different AWS CloudHSM cluster. + // + // Clusters that share a backup history have the same cluster certificate. To + // view the cluster certificate of a cluster, use the DescribeClusters (https://docs.aws.amazon.com/cloudhsm/latest/APIReference/API_DescribeClusters.html) + // operation. + ErrCodeCloudHsmClusterInUseException = "CloudHsmClusterInUseException" + + // ErrCodeCloudHsmClusterInvalidConfigurationException for service response error code + // "CloudHsmClusterInvalidConfigurationException". + // + // The request was rejected because the associated AWS CloudHSM cluster did + // not meet the configuration requirements for a custom key store. + // + // * The cluster must be configured with private subnets in at least two + // different Availability Zones in the Region. + // + // * The security group for the cluster (https://docs.aws.amazon.com/cloudhsm/latest/userguide/configure-sg.html) + // (cloudhsm-cluster--sg) must include inbound rules and outbound + // rules that allow TCP traffic on ports 2223-2225. The Source in the inbound + // rules and the Destination in the outbound rules must match the security + // group ID. These rules are set by default when you create the cluster. + // Do not delete or change them. To get information about a particular security + // group, use the DescribeSecurityGroups (https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeSecurityGroups.html) + // operation. + // + // * The cluster must contain at least as many HSMs as the operation requires. + // To add HSMs, use the AWS CloudHSM CreateHsm (https://docs.aws.amazon.com/cloudhsm/latest/APIReference/API_CreateHsm.html) + // operation. For the CreateCustomKeyStore, UpdateCustomKeyStore, and CreateKey + // operations, the AWS CloudHSM cluster must have at least two active HSMs, + // each in a different Availability Zone. For the ConnectCustomKeyStore operation, + // the AWS CloudHSM must contain at least one active HSM. + // + // For information about the requirements for an AWS CloudHSM cluster that is + // associated with a custom key store, see Assemble the Prerequisites (https://docs.aws.amazon.com/kms/latest/developerguide/create-keystore.html#before-keystore) + // in the AWS Key Management Service Developer Guide. For information about + // creating a private subnet for an AWS CloudHSM cluster, see Create a Private + // Subnet (https://docs.aws.amazon.com/cloudhsm/latest/userguide/create-subnets.html) + // in the AWS CloudHSM User Guide. For information about cluster security groups, + // see Configure a Default Security Group (https://docs.aws.amazon.com/cloudhsm/latest/userguide/configure-sg.html) + // in the AWS CloudHSM User Guide . + ErrCodeCloudHsmClusterInvalidConfigurationException = "CloudHsmClusterInvalidConfigurationException" + + // ErrCodeCloudHsmClusterNotActiveException for service response error code + // "CloudHsmClusterNotActiveException". + // + // The request was rejected because the AWS CloudHSM cluster that is associated + // with the custom key store is not active. Initialize and activate the cluster + // and try the command again. For detailed instructions, see Getting Started + // (https://docs.aws.amazon.com/cloudhsm/latest/userguide/getting-started.html) + // in the AWS CloudHSM User Guide. + ErrCodeCloudHsmClusterNotActiveException = "CloudHsmClusterNotActiveException" + + // ErrCodeCloudHsmClusterNotFoundException for service response error code + // "CloudHsmClusterNotFoundException". + // + // The request was rejected because AWS KMS cannot find the AWS CloudHSM cluster + // with the specified cluster ID. Retry the request with a different cluster + // ID. + ErrCodeCloudHsmClusterNotFoundException = "CloudHsmClusterNotFoundException" + + // ErrCodeCloudHsmClusterNotRelatedException for service response error code + // "CloudHsmClusterNotRelatedException". + // + // The request was rejected because the specified AWS CloudHSM cluster has a + // different cluster certificate than the original cluster. You cannot use the + // operation to specify an unrelated cluster. + // + // Specify a cluster that shares a backup history with the original cluster. + // This includes clusters that were created from a backup of the current cluster, + // and clusters that were created from the same backup that produced the current + // cluster. + // + // Clusters that share a backup history have the same cluster certificate. To + // view the cluster certificate of a cluster, use the DescribeClusters (https://docs.aws.amazon.com/cloudhsm/latest/APIReference/API_DescribeClusters.html) + // operation. + ErrCodeCloudHsmClusterNotRelatedException = "CloudHsmClusterNotRelatedException" + + // ErrCodeCustomKeyStoreHasCMKsException for service response error code + // "CustomKeyStoreHasCMKsException". + // + // The request was rejected because the custom key store contains AWS KMS customer + // master keys (CMKs). After verifying that you do not need to use the CMKs, + // use the ScheduleKeyDeletion operation to delete the CMKs. After they are + // deleted, you can delete the custom key store. + ErrCodeCustomKeyStoreHasCMKsException = "CustomKeyStoreHasCMKsException" + + // ErrCodeCustomKeyStoreInvalidStateException for service response error code + // "CustomKeyStoreInvalidStateException". + // + // The request was rejected because of the ConnectionState of the custom key + // store. To get the ConnectionState of a custom key store, use the DescribeCustomKeyStores + // operation. + // + // This exception is thrown under the following conditions: + // + // * You requested the CreateKey or GenerateRandom operation in a custom + // key store that is not connected. These operations are valid only when + // the custom key store ConnectionState is CONNECTED. + // + // * You requested the UpdateCustomKeyStore or DeleteCustomKeyStore operation + // on a custom key store that is not disconnected. This operation is valid + // only when the custom key store ConnectionState is DISCONNECTED. + // + // * You requested the ConnectCustomKeyStore operation on a custom key store + // with a ConnectionState of DISCONNECTING or FAILED. This operation is valid + // for all other ConnectionState values. + ErrCodeCustomKeyStoreInvalidStateException = "CustomKeyStoreInvalidStateException" + + // ErrCodeCustomKeyStoreNameInUseException for service response error code + // "CustomKeyStoreNameInUseException". + // + // The request was rejected because the specified custom key store name is already + // assigned to another custom key store in the account. Try again with a custom + // key store name that is unique in the account. + ErrCodeCustomKeyStoreNameInUseException = "CustomKeyStoreNameInUseException" + + // ErrCodeCustomKeyStoreNotFoundException for service response error code + // "CustomKeyStoreNotFoundException". + // + // The request was rejected because AWS KMS cannot find a custom key store with + // the specified key store name or ID. + ErrCodeCustomKeyStoreNotFoundException = "CustomKeyStoreNotFoundException" + + // ErrCodeDependencyTimeoutException for service response error code + // "DependencyTimeoutException". + // + // The system timed out while trying to fulfill the request. The request can + // be retried. + ErrCodeDependencyTimeoutException = "DependencyTimeoutException" + + // ErrCodeDisabledException for service response error code + // "DisabledException". + // + // The request was rejected because the specified CMK is not enabled. + ErrCodeDisabledException = "DisabledException" + + // ErrCodeExpiredImportTokenException for service response error code + // "ExpiredImportTokenException". + // + // The request was rejected because the specified import token is expired. Use + // GetParametersForImport to get a new import token and public key, use the + // new public key to encrypt the key material, and then try the request again. + ErrCodeExpiredImportTokenException = "ExpiredImportTokenException" + + // ErrCodeIncorrectKeyException for service response error code + // "IncorrectKeyException". + // + // The request was rejected because the specified CMK cannot decrypt the data. + // The KeyId in a Decrypt request and the SourceKeyId in a ReEncrypt request + // must identify the same CMK that was used to encrypt the ciphertext. + ErrCodeIncorrectKeyException = "IncorrectKeyException" + + // ErrCodeIncorrectKeyMaterialException for service response error code + // "IncorrectKeyMaterialException". + // + // The request was rejected because the key material in the request is, expired, + // invalid, or is not the same key material that was previously imported into + // this customer master key (CMK). + ErrCodeIncorrectKeyMaterialException = "IncorrectKeyMaterialException" + + // ErrCodeIncorrectTrustAnchorException for service response error code + // "IncorrectTrustAnchorException". + // + // The request was rejected because the trust anchor certificate in the request + // is not the trust anchor certificate for the specified AWS CloudHSM cluster. + // + // When you initialize the cluster (https://docs.aws.amazon.com/cloudhsm/latest/userguide/initialize-cluster.html#sign-csr), + // you create the trust anchor certificate and save it in the customerCA.crt + // file. + ErrCodeIncorrectTrustAnchorException = "IncorrectTrustAnchorException" + + // ErrCodeInternalException for service response error code + // "KMSInternalException". + // + // The request was rejected because an internal exception occurred. The request + // can be retried. + ErrCodeInternalException = "KMSInternalException" + + // ErrCodeInvalidAliasNameException for service response error code + // "InvalidAliasNameException". + // + // The request was rejected because the specified alias name is not valid. + ErrCodeInvalidAliasNameException = "InvalidAliasNameException" + + // ErrCodeInvalidArnException for service response error code + // "InvalidArnException". + // + // The request was rejected because a specified ARN, or an ARN in a key policy, + // is not valid. + ErrCodeInvalidArnException = "InvalidArnException" + + // ErrCodeInvalidCiphertextException for service response error code + // "InvalidCiphertextException". + // + // From the Decrypt or ReEncrypt operation, the request was rejected because + // the specified ciphertext, or additional authenticated data incorporated into + // the ciphertext, such as the encryption context, is corrupted, missing, or + // otherwise invalid. + // + // From the ImportKeyMaterial operation, the request was rejected because AWS + // KMS could not decrypt the encrypted (wrapped) key material. + ErrCodeInvalidCiphertextException = "InvalidCiphertextException" + + // ErrCodeInvalidGrantIdException for service response error code + // "InvalidGrantIdException". + // + // The request was rejected because the specified GrantId is not valid. + ErrCodeInvalidGrantIdException = "InvalidGrantIdException" + + // ErrCodeInvalidGrantTokenException for service response error code + // "InvalidGrantTokenException". + // + // The request was rejected because the specified grant token is not valid. + ErrCodeInvalidGrantTokenException = "InvalidGrantTokenException" + + // ErrCodeInvalidImportTokenException for service response error code + // "InvalidImportTokenException". + // + // The request was rejected because the provided import token is invalid or + // is associated with a different customer master key (CMK). + ErrCodeInvalidImportTokenException = "InvalidImportTokenException" + + // ErrCodeInvalidKeyUsageException for service response error code + // "InvalidKeyUsageException". + // + // The request was rejected for one of the following reasons: + // + // * The KeyUsage value of the CMK is incompatible with the API operation. + // + // * The encryption algorithm or signing algorithm specified for the operation + // is incompatible with the type of key material in the CMK (CustomerMasterKeySpec). + // + // For encrypting, decrypting, re-encrypting, and generating data keys, the + // KeyUsage must be ENCRYPT_DECRYPT. For signing and verifying, the KeyUsage + // must be SIGN_VERIFY. To find the KeyUsage of a CMK, use the DescribeKey operation. + // + // To find the encryption or signing algorithms supported for a particular CMK, + // use the DescribeKey operation. + ErrCodeInvalidKeyUsageException = "InvalidKeyUsageException" + + // ErrCodeInvalidMarkerException for service response error code + // "InvalidMarkerException". + // + // The request was rejected because the marker that specifies where pagination + // should next begin is not valid. + ErrCodeInvalidMarkerException = "InvalidMarkerException" + + // ErrCodeInvalidStateException for service response error code + // "KMSInvalidStateException". + // + // The request was rejected because the state of the specified resource is not + // valid for this request. + // + // For more information about how key state affects the use of a CMK, see How + // Key State Affects Use of a Customer Master Key (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html) + // in the AWS Key Management Service Developer Guide . + ErrCodeInvalidStateException = "KMSInvalidStateException" + + // ErrCodeKMSInvalidSignatureException for service response error code + // "KMSInvalidSignatureException". + // + // The request was rejected because the signature verification failed. Signature + // verification fails when it cannot confirm that signature was produced by + // signing the specified message with the specified CMK and signing algorithm. + ErrCodeKMSInvalidSignatureException = "KMSInvalidSignatureException" + + // ErrCodeKeyUnavailableException for service response error code + // "KeyUnavailableException". + // + // The request was rejected because the specified CMK was not available. You + // can retry the request. + ErrCodeKeyUnavailableException = "KeyUnavailableException" + + // ErrCodeLimitExceededException for service response error code + // "LimitExceededException". + // + // The request was rejected because a quota was exceeded. For more information, + // see Quotas (https://docs.aws.amazon.com/kms/latest/developerguide/limits.html) + // in the AWS Key Management Service Developer Guide. + ErrCodeLimitExceededException = "LimitExceededException" + + // ErrCodeMalformedPolicyDocumentException for service response error code + // "MalformedPolicyDocumentException". + // + // The request was rejected because the specified policy is not syntactically + // or semantically correct. + ErrCodeMalformedPolicyDocumentException = "MalformedPolicyDocumentException" + + // ErrCodeNotFoundException for service response error code + // "NotFoundException". + // + // The request was rejected because the specified entity or resource could not + // be found. + ErrCodeNotFoundException = "NotFoundException" + + // ErrCodeTagException for service response error code + // "TagException". + // + // The request was rejected because one or more tags are not valid. + ErrCodeTagException = "TagException" + + // ErrCodeUnsupportedOperationException for service response error code + // "UnsupportedOperationException". + // + // The request was rejected because a specified parameter is not supported or + // a specified resource is not valid for this operation. + ErrCodeUnsupportedOperationException = "UnsupportedOperationException" +) + +var exceptionFromCode = map[string]func(protocol.ResponseMetadata) error{ + "AlreadyExistsException": newErrorAlreadyExistsException, + "CloudHsmClusterInUseException": newErrorCloudHsmClusterInUseException, + "CloudHsmClusterInvalidConfigurationException": newErrorCloudHsmClusterInvalidConfigurationException, + "CloudHsmClusterNotActiveException": newErrorCloudHsmClusterNotActiveException, + "CloudHsmClusterNotFoundException": newErrorCloudHsmClusterNotFoundException, + "CloudHsmClusterNotRelatedException": newErrorCloudHsmClusterNotRelatedException, + "CustomKeyStoreHasCMKsException": newErrorCustomKeyStoreHasCMKsException, + "CustomKeyStoreInvalidStateException": newErrorCustomKeyStoreInvalidStateException, + "CustomKeyStoreNameInUseException": newErrorCustomKeyStoreNameInUseException, + "CustomKeyStoreNotFoundException": newErrorCustomKeyStoreNotFoundException, + "DependencyTimeoutException": newErrorDependencyTimeoutException, + "DisabledException": newErrorDisabledException, + "ExpiredImportTokenException": newErrorExpiredImportTokenException, + "IncorrectKeyException": newErrorIncorrectKeyException, + "IncorrectKeyMaterialException": newErrorIncorrectKeyMaterialException, + "IncorrectTrustAnchorException": newErrorIncorrectTrustAnchorException, + "KMSInternalException": newErrorInternalException, + "InvalidAliasNameException": newErrorInvalidAliasNameException, + "InvalidArnException": newErrorInvalidArnException, + "InvalidCiphertextException": newErrorInvalidCiphertextException, + "InvalidGrantIdException": newErrorInvalidGrantIdException, + "InvalidGrantTokenException": newErrorInvalidGrantTokenException, + "InvalidImportTokenException": newErrorInvalidImportTokenException, + "InvalidKeyUsageException": newErrorInvalidKeyUsageException, + "InvalidMarkerException": newErrorInvalidMarkerException, + "KMSInvalidStateException": newErrorInvalidStateException, + "KMSInvalidSignatureException": newErrorKMSInvalidSignatureException, + "KeyUnavailableException": newErrorKeyUnavailableException, + "LimitExceededException": newErrorLimitExceededException, + "MalformedPolicyDocumentException": newErrorMalformedPolicyDocumentException, + "NotFoundException": newErrorNotFoundException, + "TagException": newErrorTagException, + "UnsupportedOperationException": newErrorUnsupportedOperationException, +} diff --git a/github.com/aws/aws-sdk-go/service/kms/service.go b/github.com/aws/aws-sdk-go/service/kms/service.go new file mode 100644 index 0000000000..50ca0c092e --- /dev/null +++ b/github.com/aws/aws-sdk-go/service/kms/service.go @@ -0,0 +1,103 @@ +// Code generated by private/model/cli/gen-api/main.go. DO NOT EDIT. + +package kms + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/client" + "github.com/aws/aws-sdk-go/aws/client/metadata" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/aws/signer/v4" + "github.com/aws/aws-sdk-go/private/protocol" + "github.com/aws/aws-sdk-go/private/protocol/jsonrpc" +) + +// KMS provides the API operation methods for making requests to +// AWS Key Management Service. See this package's package overview docs +// for details on the service. +// +// KMS methods are safe to use concurrently. It is not safe to +// modify mutate any of the struct's properties though. +type KMS struct { + *client.Client +} + +// Used for custom client initialization logic +var initClient func(*client.Client) + +// Used for custom request initialization logic +var initRequest func(*request.Request) + +// Service information constants +const ( + ServiceName = "kms" // Name of service. + EndpointsID = ServiceName // ID to lookup a service endpoint with. + ServiceID = "KMS" // ServiceID is a unique identifier of a specific service. +) + +// New creates a new instance of the KMS client with a session. +// If additional configuration is needed for the client instance use the optional +// aws.Config parameter to add your extra config. +// +// Example: +// mySession := session.Must(session.NewSession()) +// +// // Create a KMS client from just a session. +// svc := kms.New(mySession) +// +// // Create a KMS client with additional configuration +// svc := kms.New(mySession, aws.NewConfig().WithRegion("us-west-2")) +func New(p client.ConfigProvider, cfgs ...*aws.Config) *KMS { + c := p.ClientConfig(EndpointsID, cfgs...) + return newClient(*c.Config, c.Handlers, c.PartitionID, c.Endpoint, c.SigningRegion, c.SigningName) +} + +// newClient creates, initializes and returns a new service client instance. +func newClient(cfg aws.Config, handlers request.Handlers, partitionID, endpoint, signingRegion, signingName string) *KMS { + svc := &KMS{ + Client: client.New( + cfg, + metadata.ClientInfo{ + ServiceName: ServiceName, + ServiceID: ServiceID, + SigningName: signingName, + SigningRegion: signingRegion, + PartitionID: partitionID, + Endpoint: endpoint, + APIVersion: "2014-11-01", + JSONVersion: "1.1", + TargetPrefix: "TrentService", + }, + handlers, + ), + } + + // Handlers + svc.Handlers.Sign.PushBackNamed(v4.SignRequestHandler) + svc.Handlers.Build.PushBackNamed(jsonrpc.BuildHandler) + svc.Handlers.Unmarshal.PushBackNamed(jsonrpc.UnmarshalHandler) + svc.Handlers.UnmarshalMeta.PushBackNamed(jsonrpc.UnmarshalMetaHandler) + svc.Handlers.UnmarshalError.PushBackNamed( + protocol.NewUnmarshalErrorHandler(jsonrpc.NewUnmarshalTypedError(exceptionFromCode)).NamedHandler(), + ) + + // Run custom client initialization if present + if initClient != nil { + initClient(svc.Client) + } + + return svc +} + +// newRequest creates a new request for a KMS operation and runs any +// custom request initialization. +func (c *KMS) newRequest(op *request.Operation, params, data interface{}) *request.Request { + req := c.NewRequest(op, params, data) + + // Run custom request initialization if present + if initRequest != nil { + initRequest(req) + } + + return req +} diff --git a/github.com/aws/aws-sdk-go/service/s3/api.go b/github.com/aws/aws-sdk-go/service/s3/api.go index 3b3c2e380c..82defe7f2c 100644 --- a/github.com/aws/aws-sdk-go/service/s3/api.go +++ b/github.com/aws/aws-sdk-go/service/s3/api.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "sync" - "sync/atomic" "time" "github.com/aws/aws-sdk-go/aws" @@ -15,11 +14,13 @@ import ( "github.com/aws/aws-sdk-go/aws/awsutil" "github.com/aws/aws-sdk-go/aws/client" "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/private/checksum" "github.com/aws/aws-sdk-go/private/protocol" "github.com/aws/aws-sdk-go/private/protocol/eventstream" "github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi" "github.com/aws/aws-sdk-go/private/protocol/rest" "github.com/aws/aws-sdk-go/private/protocol/restxml" + "github.com/aws/aws-sdk-go/service/s3/internal/arn" ) const opAbortMultipartUpload = "AbortMultipartUpload" @@ -66,11 +67,31 @@ func (c *S3) AbortMultipartUploadRequest(input *AbortMultipartUploadInput) (req // AbortMultipartUpload API operation for Amazon Simple Storage Service. // -// Aborts a multipart upload. +// This operation aborts a multipart upload. After a multipart upload is aborted, +// no additional parts can be uploaded using that upload ID. The storage consumed +// by any previously uploaded parts will be freed. However, if any part uploads +// are currently in progress, those part uploads might or might not succeed. +// As a result, it might be necessary to abort a given multipart upload multiple +// times in order to completely free all storage consumed by all parts. // // To verify that all parts have been removed, so you don't get charged for -// the part storage, you should call the List Parts operation and ensure the -// parts list is empty. +// the part storage, you should call the ListParts operation and ensure that +// the parts list is empty. +// +// For information about permissions required to use the multipart upload API, +// see Multipart Upload API and Permissions (https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html). +// +// The following operations are related to AbortMultipartUpload: +// +// * CreateMultipartUpload +// +// * UploadPart +// +// * CompleteMultipartUpload +// +// * ListParts +// +// * ListMultipartUploads // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -151,6 +172,64 @@ func (c *S3) CompleteMultipartUploadRequest(input *CompleteMultipartUploadInput) // // Completes a multipart upload by assembling previously uploaded parts. // +// You first initiate the multipart upload and then upload all parts using the +// UploadPart operation. After successfully uploading all relevant parts of +// an upload, you call this operation to complete the upload. Upon receiving +// this request, Amazon S3 concatenates all the parts in ascending order by +// part number to create a new object. In the Complete Multipart Upload request, +// you must provide the parts list. You must ensure that the parts list is complete. +// This operation concatenates the parts that you provide in the list. For each +// part in the list, you must provide the part number and the ETag value, returned +// after that part was uploaded. +// +// Processing of a Complete Multipart Upload request could take several minutes +// to complete. After Amazon S3 begins processing the request, it sends an HTTP +// response header that specifies a 200 OK response. While processing is in +// progress, Amazon S3 periodically sends white space characters to keep the +// connection from timing out. Because a request could fail after the initial +// 200 OK response has been sent, it is important that you check the response +// body to determine whether the request succeeded. +// +// Note that if CompleteMultipartUpload fails, applications should be prepared +// to retry the failed requests. For more information, see Amazon S3 Error Best +// Practices (https://docs.aws.amazon.com/AmazonS3/latest/dev/ErrorBestPractices.html). +// +// For more information about multipart uploads, see Uploading Objects Using +// Multipart Upload (https://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html). +// +// For information about permissions required to use the multipart upload API, +// see Multipart Upload API and Permissions (https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html). +// +// GetBucketLifecycle has the following special errors: +// +// * Error code: EntityTooSmall Description: Your proposed upload is smaller +// than the minimum allowed object size. Each part must be at least 5 MB +// in size, except the last part. 400 Bad Request +// +// * Error code: InvalidPart Description: One or more of the specified parts +// could not be found. The part might not have been uploaded, or the specified +// entity tag might not have matched the part's entity tag. 400 Bad Request +// +// * Error code: InvalidPartOrder Description: The list of parts was not +// in ascending order. The parts list must be specified in order by part +// number. 400 Bad Request +// +// * Error code: NoSuchUpload Description: The specified multipart upload +// does not exist. The upload ID might be invalid, or the multipart upload +// might have been aborted or completed. 404 Not Found +// +// The following operations are related to CompleteMultipartUpload: +// +// * CreateMultipartUpload +// +// * UploadPart +// +// * AbortMultipartUpload +// +// * ListParts +// +// * ListMultipartUploads +// // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about // the error. @@ -225,6 +304,153 @@ func (c *S3) CopyObjectRequest(input *CopyObjectInput) (req *request.Request, ou // // Creates a copy of an object that is already stored in Amazon S3. // +// You can store individual objects of up to 5 TB in Amazon S3. You create a +// copy of your object up to 5 GB in size in a single atomic operation using +// this API. However, to copy an object greater than 5 GB, you must use the +// multipart upload Upload Part - Copy API. For more information, see Copy Object +// Using the REST Multipart Upload API (https://docs.aws.amazon.com/AmazonS3/latest/dev/CopyingObjctsUsingRESTMPUapi.html). +// +// All copy requests must be authenticated. Additionally, you must have read +// access to the source object and write access to the destination bucket. For +// more information, see REST Authentication (https://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html). +// Both the Region that you want to copy the object from and the Region that +// you want to copy the object to must be enabled for your account. +// +// A copy request might return an error when Amazon S3 receives the copy request +// or while Amazon S3 is copying the files. If the error occurs before the copy +// operation starts, you receive a standard Amazon S3 error. If the error occurs +// during the copy operation, the error response is embedded in the 200 OK response. +// This means that a 200 OK response can contain either a success or an error. +// Design your application to parse the contents of the response and handle +// it appropriately. +// +// If the copy is successful, you receive a response with information about +// the copied object. +// +// If the request is an HTTP 1.1 request, the response is chunk encoded. If +// it were not, it would not contain the content-length, and you would need +// to read the entire body. +// +// The copy request charge is based on the storage class and Region that you +// specify for the destination object. For pricing information, see Amazon S3 +// pricing (https://aws.amazon.com/s3/pricing/). +// +// Amazon S3 transfer acceleration does not support cross-Region copies. If +// you request a cross-Region copy using a transfer acceleration endpoint, you +// get a 400 Bad Request error. For more information, see Transfer Acceleration +// (https://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html). +// +// Metadata +// +// When copying an object, you can preserve all metadata (default) or specify +// new metadata. However, the ACL is not preserved and is set to private for +// the user making the request. To override the default ACL setting, specify +// a new ACL when generating a copy request. For more information, see Using +// ACLs (https://docs.aws.amazon.com/AmazonS3/latest/dev/S3_ACLs_UsingACLs.html). +// +// To specify whether you want the object metadata copied from the source object +// or replaced with metadata provided in the request, you can optionally add +// the x-amz-metadata-directive header. When you grant permissions, you can +// use the s3:x-amz-metadata-directive condition key to enforce certain metadata +// behavior when objects are uploaded. For more information, see Specifying +// Conditions in a Policy (https://docs.aws.amazon.com/AmazonS3/latest/dev/amazon-s3-policy-keys.html) +// in the Amazon S3 Developer Guide. For a complete list of Amazon S3-specific +// condition keys, see Actions, Resources, and Condition Keys for Amazon S3 +// (https://docs.aws.amazon.com/AmazonS3/latest/dev/list_amazons3.html). +// +// x-amz-copy-source-if Headers +// +// To only copy an object under certain conditions, such as whether the Etag +// matches or whether the object was modified before or after a specified date, +// use the following request parameters: +// +// * x-amz-copy-source-if-match +// +// * x-amz-copy-source-if-none-match +// +// * x-amz-copy-source-if-unmodified-since +// +// * x-amz-copy-source-if-modified-since +// +// If both the x-amz-copy-source-if-match and x-amz-copy-source-if-unmodified-since +// headers are present in the request and evaluate as follows, Amazon S3 returns +// 200 OK and copies the data: +// +// * x-amz-copy-source-if-match condition evaluates to true +// +// * x-amz-copy-source-if-unmodified-since condition evaluates to false +// +// If both the x-amz-copy-source-if-none-match and x-amz-copy-source-if-modified-since +// headers are present in the request and evaluate as follows, Amazon S3 returns +// the 412 Precondition Failed response code: +// +// * x-amz-copy-source-if-none-match condition evaluates to false +// +// * x-amz-copy-source-if-modified-since condition evaluates to true +// +// All headers with the x-amz- prefix, including x-amz-copy-source, must be +// signed. +// +// Encryption +// +// The source object that you are copying can be encrypted or unencrypted. The +// source object can be encrypted with server-side encryption using AWS managed +// encryption keys (SSE-S3 or SSE-KMS) or by using a customer-provided encryption +// key. With server-side encryption, Amazon S3 encrypts your data as it writes +// it to disks in its data centers and decrypts the data when you access it. +// +// You can optionally use the appropriate encryption-related headers to request +// server-side encryption for the target object. You have the option to provide +// your own encryption key or use SSE-S3 or SSE-KMS, regardless of the form +// of server-side encryption that was used to encrypt the source object. You +// can even request encryption if the source object was not encrypted. For more +// information about server-side encryption, see Using Server-Side Encryption +// (https://docs.aws.amazon.com/AmazonS3/latest/dev/serv-side-encryption.html). +// +// Access Control List (ACL)-Specific Request Headers +// +// When copying an object, you can optionally use headers to grant ACL-based +// permissions. By default, all objects are private. Only the owner has full +// access control. When adding a new object, you can grant permissions to individual +// AWS accounts or to predefined groups defined by Amazon S3. These permissions +// are then added to the ACL on the object. For more information, see Access +// Control List (ACL) Overview (https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html) +// and Managing ACLs Using the REST API (https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-using-rest-api.html). +// +// Storage Class Options +// +// You can use the CopyObject operation to change the storage class of an object +// that is already stored in Amazon S3 using the StorageClass parameter. For +// more information, see Storage Classes (https://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html) +// in the Amazon S3 Service Developer Guide. +// +// Versioning +// +// By default, x-amz-copy-source identifies the current version of an object +// to copy. If the current version is a delete marker, Amazon S3 behaves as +// if the object was deleted. To copy a different version, use the versionId +// subresource. +// +// If you enable versioning on the target bucket, Amazon S3 generates a unique +// version ID for the object being copied. This version ID is different from +// the version ID of the source object. Amazon S3 returns the version ID of +// the copied object in the x-amz-version-id response header in the response. +// +// If you do not enable versioning or suspend it on the target bucket, the version +// ID that Amazon S3 generates is always null. +// +// If the source object's storage class is GLACIER, you must restore a copy +// of this object before you can use it as a source object for the copy operation. +// For more information, see . +// +// The following operations are related to CopyObject: +// +// * PutObject +// +// * GetObject +// +// For more information, see Copying Objects (https://docs.aws.amazon.com/AmazonS3/latest/dev/CopyingObjectsExamples.html). +// // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about // the error. @@ -235,7 +461,7 @@ func (c *S3) CopyObjectRequest(input *CopyObjectInput) (req *request.Request, ou // Returned Error Codes: // * ErrCodeObjectNotInActiveTierError "ObjectNotInActiveTierError" // The source object of the COPY operation is not in the active tier and is -// only stored in Amazon Glacier. +// only stored in Amazon S3 Glacier. // // See also, https://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/CopyObject func (c *S3) CopyObject(input *CopyObjectInput) (*CopyObjectOutput, error) { @@ -303,7 +529,67 @@ func (c *S3) CreateBucketRequest(input *CreateBucketInput) (req *request.Request // CreateBucket API operation for Amazon Simple Storage Service. // -// Creates a new bucket. +// Creates a new bucket. To create a bucket, you must register with Amazon S3 +// and have a valid AWS Access Key ID to authenticate requests. Anonymous requests +// are never allowed to create buckets. By creating the bucket, you become the +// bucket owner. +// +// Not every string is an acceptable bucket name. For information on bucket +// naming restrictions, see Working with Amazon S3 Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html). +// +// By default, the bucket is created in the US East (N. Virginia) Region. You +// can optionally specify a Region in the request body. You might choose a Region +// to optimize latency, minimize costs, or address regulatory requirements. +// For example, if you reside in Europe, you will probably find it advantageous +// to create buckets in the Europe (Ireland) Region. For more information, see +// How to Select a Region for Your Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro). +// +// If you send your create bucket request to the s3.amazonaws.com endpoint, +// the request goes to the us-east-1 Region. Accordingly, the signature calculations +// in Signature Version 4 must use us-east-1 as the Region, even if the location +// constraint in the request specifies another Region where the bucket is to +// be created. If you create a bucket in a Region other than US East (N. Virginia), +// your application must be able to handle 307 redirect. For more information, +// see Virtual Hosting of Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html). +// +// When creating a bucket using this operation, you can optionally specify the +// accounts or groups that should be granted specific permissions on the bucket. +// There are two ways to grant the appropriate permissions using the request +// headers. +// +// * Specify a canned ACL using the x-amz-acl request header. Amazon S3 supports +// a set of predefined ACLs, known as canned ACLs. Each canned ACL has a +// predefined set of grantees and permissions. For more information, see +// Canned ACL (https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#CannedACL). +// +// * Specify access permissions explicitly using the x-amz-grant-read, x-amz-grant-write, +// x-amz-grant-read-acp, x-amz-grant-write-acp, and x-amz-grant-full-control +// headers. These headers map to the set of permissions Amazon S3 supports +// in an ACL. For more information, see Access Control List (ACL) Overview +// (https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html). You +// specify each grantee as a type=value pair, where the type is one of the +// following: id – if the value specified is the canonical user ID of an +// AWS account uri – if you are granting permissions to a predefined group +// emailAddress – if the value specified is the email address of an AWS +// account Using email addresses to specify a grantee is only supported in +// the following AWS Regions: US East (N. Virginia) US West (N. California) +// US West (Oregon) Asia Pacific (Singapore) Asia Pacific (Sydney) Asia Pacific +// (Tokyo) Europe (Ireland) South America (São Paulo) For a list of all +// the Amazon S3 supported Regions and endpoints, see Regions and Endpoints +// (https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region) in +// the AWS General Reference. For example, the following x-amz-grant-read +// header grants the AWS accounts identified by account IDs permissions to +// read object data and its metadata: x-amz-grant-read: id="11112222333", +// id="444455556666" +// +// You can use either a canned ACL or specify access permissions explicitly. +// You cannot do both. +// +// The following operations are related to CreateBucket: +// +// * PutObject +// +// * DeleteBucket // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -318,6 +604,11 @@ func (c *S3) CreateBucketRequest(input *CreateBucketInput) (req *request.Request // by all users of the system. Please select a different name and try again. // // * ErrCodeBucketAlreadyOwnedByYou "BucketAlreadyOwnedByYou" +// The bucket you tried to create already exists, and you own it. Amazon S3 +// returns this error in all AWS Regions except in the North Virginia Region. +// For legacy compatibility, if you re-create an existing bucket that you already +// own in the North Virginia Region, Amazon S3 returns 200 OK and resets the +// bucket access control lists (ACLs). // // See also, https://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/CreateBucket func (c *S3) CreateBucket(input *CreateBucketInput) (*CreateBucketOutput, error) { @@ -385,13 +676,154 @@ func (c *S3) CreateMultipartUploadRequest(input *CreateMultipartUploadInput) (re // CreateMultipartUpload API operation for Amazon Simple Storage Service. // -// Initiates a multipart upload and returns an upload ID. -// -// Note: After you initiate multipart upload and upload one or more parts, you -// must either complete or abort multipart upload in order to stop getting charged -// for storage of the uploaded parts. Only after you either complete or abort -// multipart upload, Amazon S3 frees up the parts storage and stops charging -// you for the parts storage. +// This operation initiates a multipart upload and returns an upload ID. This +// upload ID is used to associate all of the parts in the specific multipart +// upload. You specify this upload ID in each of your subsequent upload part +// requests (see UploadPart). You also include this upload ID in the final request +// to either complete or abort the multipart upload request. +// +// For more information about multipart uploads, see Multipart Upload Overview +// (https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuoverview.html). +// +// If you have configured a lifecycle rule to abort incomplete multipart uploads, +// the upload must complete within the number of days specified in the bucket +// lifecycle configuration. Otherwise, the incomplete multipart upload becomes +// eligible for an abort operation and Amazon S3 aborts the multipart upload. +// For more information, see Aborting Incomplete Multipart Uploads Using a Bucket +// Lifecycle Policy (https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuoverview.html#mpu-abort-incomplete-mpu-lifecycle-config). +// +// For information about the permissions required to use the multipart upload +// API, see Multipart Upload API and Permissions (https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html). +// +// For request signing, multipart upload is just a series of regular requests. +// You initiate a multipart upload, send one or more requests to upload parts, +// and then complete the multipart upload process. You sign each request individually. +// There is nothing special about signing multipart upload requests. For more +// information about signing, see Authenticating Requests (AWS Signature Version +// 4) (https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html). +// +// After you initiate a multipart upload and upload one or more parts, to stop +// being charged for storing the uploaded parts, you must either complete or +// abort the multipart upload. Amazon S3 frees up the space used to store the +// parts and stop charging you for storing them only after you either complete +// or abort a multipart upload. +// +// You can optionally request server-side encryption. For server-side encryption, +// Amazon S3 encrypts your data as it writes it to disks in its data centers +// and decrypts it when you access it. You can provide your own encryption key, +// or use AWS Key Management Service (AWS KMS) customer master keys (CMKs) or +// Amazon S3-managed encryption keys. If you choose to provide your own encryption +// key, the request headers you provide in UploadPart) and UploadPartCopy) requests +// must match the headers you used in the request to initiate the upload by +// using CreateMultipartUpload. +// +// To perform a multipart upload with encryption using an AWS KMS CMK, the requester +// must have permission to the kms:Encrypt, kms:Decrypt, kms:ReEncrypt*, kms:GenerateDataKey*, +// and kms:DescribeKey actions on the key. These permissions are required because +// Amazon S3 must decrypt and read data from the encrypted file parts before +// it completes the multipart upload. +// +// If your AWS Identity and Access Management (IAM) user or role is in the same +// AWS account as the AWS KMS CMK, then you must have these permissions on the +// key policy. If your IAM user or role belongs to a different account than +// the key, then you must have the permissions on both the key policy and your +// IAM user or role. +// +// For more information, see Protecting Data Using Server-Side Encryption (https://docs.aws.amazon.com/AmazonS3/latest/dev/serv-side-encryption.html). +// +// Access Permissions +// +// When copying an object, you can optionally specify the accounts or groups +// that should be granted specific permissions on the new object. There are +// two ways to grant the permissions using the request headers: +// +// * Specify a canned ACL with the x-amz-acl request header. For more information, +// see Canned ACL (https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#CannedACL). +// +// * Specify access permissions explicitly with the x-amz-grant-read, x-amz-grant-read-acp, +// x-amz-grant-write-acp, and x-amz-grant-full-control headers. These parameters +// map to the set of permissions that Amazon S3 supports in an ACL. For more +// information, see Access Control List (ACL) Overview (https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html). +// +// You can use either a canned ACL or specify access permissions explicitly. +// You cannot do both. +// +// Server-Side- Encryption-Specific Request Headers +// +// You can optionally tell Amazon S3 to encrypt data at rest using server-side +// encryption. Server-side encryption is for data encryption at rest. Amazon +// S3 encrypts your data as it writes it to disks in its data centers and decrypts +// it when you access it. The option you use depends on whether you want to +// use AWS managed encryption keys or provide your own encryption key. +// +// * Use encryption keys managed by Amazon S3 or customer master keys (CMKs) +// stored in AWS Key Management Service (AWS KMS) – If you want AWS to +// manage the keys used to encrypt data, specify the following headers in +// the request. x-amz-server-side​-encryption x-amz-server-side-encryption-aws-kms-key-id +// x-amz-server-side-encryption-context If you specify x-amz-server-side-encryption:aws:kms, +// but don't provide x-amz-server-side-encryption-aws-kms-key-id, Amazon +// S3 uses the AWS managed CMK in AWS KMS to protect the data. All GET and +// PUT requests for an object protected by AWS KMS fail if you don't make +// them with SSL or by using SigV4. For more information about server-side +// encryption with CMKs stored in AWS KMS (SSE-KMS), see Protecting Data +// Using Server-Side Encryption with CMKs stored in AWS KMS (https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html). +// +// * Use customer-provided encryption keys – If you want to manage your +// own encryption keys, provide all the following headers in the request. +// x-amz-server-side​-encryption​-customer-algorithm x-amz-server-side​-encryption​-customer-key +// x-amz-server-side​-encryption​-customer-key-MD5 For more information +// about server-side encryption with CMKs stored in AWS KMS (SSE-KMS), see +// Protecting Data Using Server-Side Encryption with CMKs stored in AWS KMS +// (https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html). +// +// Access-Control-List (ACL)-Specific Request Headers +// +// You also can use the following access control–related headers with this +// operation. By default, all objects are private. Only the owner has full access +// control. When adding a new object, you can grant permissions to individual +// AWS accounts or to predefined groups defined by Amazon S3. These permissions +// are then added to the access control list (ACL) on the object. For more information, +// see Using ACLs (https://docs.aws.amazon.com/AmazonS3/latest/dev/S3_ACLs_UsingACLs.html). +// With this operation, you can grant access permissions using one of the following +// two methods: +// +// * Specify a canned ACL (x-amz-acl) — Amazon S3 supports a set of predefined +// ACLs, known as canned ACLs. Each canned ACL has a predefined set of grantees +// and permissions. For more information, see Canned ACL (https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#CannedACL). +// +// * Specify access permissions explicitly — To explicitly grant access +// permissions to specific AWS accounts or groups, use the following headers. +// Each header maps to specific permissions that Amazon S3 supports in an +// ACL. For more information, see Access Control List (ACL) Overview (https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html). +// In the header, you specify a list of grantees who get the specific permission. +// To grant permissions explicitly, use: x-amz-grant-read x-amz-grant-write +// x-amz-grant-read-acp x-amz-grant-write-acp x-amz-grant-full-control You +// specify each grantee as a type=value pair, where the type is one of the +// following: id – if the value specified is the canonical user ID of an +// AWS account uri – if you are granting permissions to a predefined group +// emailAddress – if the value specified is the email address of an AWS +// account Using email addresses to specify a grantee is only supported in +// the following AWS Regions: US East (N. Virginia) US West (N. California) +// US West (Oregon) Asia Pacific (Singapore) Asia Pacific (Sydney) Asia Pacific +// (Tokyo) Europe (Ireland) South America (São Paulo) For a list of all +// the Amazon S3 supported Regions and endpoints, see Regions and Endpoints +// (https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region) in +// the AWS General Reference. For example, the following x-amz-grant-read +// header grants the AWS accounts identified by account IDs permissions to +// read object data and its metadata: x-amz-grant-read: id="11112222333", +// id="444455556666" +// +// The following operations are related to CreateMultipartUpload: +// +// * UploadPart +// +// * CompleteMultipartUpload +// +// * AbortMultipartUpload +// +// * ListParts +// +// * ListMultipartUploads // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -466,8 +898,14 @@ func (c *S3) DeleteBucketRequest(input *DeleteBucketInput) (req *request.Request // DeleteBucket API operation for Amazon Simple Storage Service. // -// Deletes the bucket. All objects (including all object versions and Delete -// Markers) in the bucket must be deleted before the bucket itself can be deleted. +// Deletes the bucket. All objects (including all object versions and delete +// markers) in the bucket must be deleted before the bucket itself can be deleted. +// +// Related Resources +// +// * +// +// * // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -545,6 +983,23 @@ func (c *S3) DeleteBucketAnalyticsConfigurationRequest(input *DeleteBucketAnalyt // Deletes an analytics configuration for the bucket (specified by the analytics // configuration ID). // +// To use this operation, you must have permissions to perform the s3:PutAnalyticsConfiguration +// action. The bucket owner has this permission by default. The bucket owner +// can grant this permission to others. For more information about permissions, +// see Permissions Related to Bucket Subresource Operations (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html#using-with-s3-actions-related-to-bucket-subresources) +// and Managing Access Permissions to Your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html). +// +// For information about the Amazon S3 analytics feature, see Amazon S3 Analytics +// – Storage Class Analysis (https://docs.aws.amazon.com/AmazonS3/latest/dev/analytics-storage-class.html). +// +// The following operations are related to DeleteBucketAnalyticsConfiguration: +// +// * +// +// * +// +// * +// // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about // the error. @@ -618,7 +1073,20 @@ func (c *S3) DeleteBucketCorsRequest(input *DeleteBucketCorsInput) (req *request // DeleteBucketCors API operation for Amazon Simple Storage Service. // -// Deletes the CORS configuration information set for the bucket. +// Deletes the cors configuration information set for the bucket. +// +// To use this operation, you must have permission to perform the s3:PutBucketCORS +// action. The bucket owner has this permission by default and can grant this +// permission to others. +// +// For information about cors, see Enabling Cross-Origin Resource Sharing (https://docs.aws.amazon.com/AmazonS3/latest/dev/cors.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// Related Resources: +// +// * +// +// * RESTOPTIONSobject // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -693,7 +1161,23 @@ func (c *S3) DeleteBucketEncryptionRequest(input *DeleteBucketEncryptionInput) ( // DeleteBucketEncryption API operation for Amazon Simple Storage Service. // -// Deletes the server-side encryption configuration from the bucket. +// This implementation of the DELETE operation removes default encryption from +// the bucket. For information about the Amazon S3 default encryption feature, +// see Amazon S3 Default Bucket Encryption (https://docs.aws.amazon.com/AmazonS3/latest/dev/bucket-encryption.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// To use this operation, you must have permissions to perform the s3:PutEncryptionConfiguration +// action. The bucket owner has this permission by default. The bucket owner +// can grant this permission to others. For more information about permissions, +// see Permissions Related to Bucket Subresource Operations (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html#using-with-s3-actions-related-to-bucket-subresources) +// and Managing Access Permissions to your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// Related Resources +// +// * PutBucketEncryption +// +// * GetBucketEncryption // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -771,6 +1255,23 @@ func (c *S3) DeleteBucketInventoryConfigurationRequest(input *DeleteBucketInvent // Deletes an inventory configuration (identified by the inventory ID) from // the bucket. // +// To use this operation, you must have permissions to perform the s3:PutInventoryConfiguration +// action. The bucket owner has this permission by default. The bucket owner +// can grant this permission to others. For more information about permissions, +// see Permissions Related to Bucket Subresource Operations (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html#using-with-s3-actions-related-to-bucket-subresources) +// and Managing Access Permissions to Your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html). +// +// For information about the Amazon S3 inventory feature, see Amazon S3 Inventory +// (https://docs.aws.amazon.com/AmazonS3/latest/dev/storage-inventory.html). +// +// Operations related to DeleteBucketInventoryConfiguration include: +// +// * GetBucketInventoryConfiguration +// +// * PutBucketInventoryConfiguration +// +// * ListBucketInventoryConfigurations +// // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about // the error. @@ -844,7 +1345,27 @@ func (c *S3) DeleteBucketLifecycleRequest(input *DeleteBucketLifecycleInput) (re // DeleteBucketLifecycle API operation for Amazon Simple Storage Service. // -// Deletes the lifecycle configuration from the bucket. +// Deletes the lifecycle configuration from the specified bucket. Amazon S3 +// removes all the lifecycle configuration rules in the lifecycle subresource +// associated with the bucket. Your objects never expire, and Amazon S3 no longer +// automatically deletes any objects on the basis of rules contained in the +// deleted lifecycle configuration. +// +// To use this operation, you must have permission to perform the s3:PutLifecycleConfiguration +// action. By default, the bucket owner has this permission and the bucket owner +// can grant this permission to others. +// +// There is usually some time lag before lifecycle configuration deletion is +// fully propagated to all the Amazon S3 systems. +// +// For more information about the object expiration, see Elements to Describe +// Lifecycle Actions (https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html#intro-lifecycle-rules-actions). +// +// Related actions include: +// +// * PutBucketLifecycleConfiguration +// +// * GetBucketLifecycleConfiguration // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -919,8 +1440,28 @@ func (c *S3) DeleteBucketMetricsConfigurationRequest(input *DeleteBucketMetricsC // DeleteBucketMetricsConfiguration API operation for Amazon Simple Storage Service. // -// Deletes a metrics configuration (specified by the metrics configuration ID) -// from the bucket. +// Deletes a metrics configuration for the Amazon CloudWatch request metrics +// (specified by the metrics configuration ID) from the bucket. Note that this +// doesn't include the daily storage metrics. +// +// To use this operation, you must have permissions to perform the s3:PutMetricsConfiguration +// action. The bucket owner has this permission by default. The bucket owner +// can grant this permission to others. For more information about permissions, +// see Permissions Related to Bucket Subresource Operations (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html#using-with-s3-actions-related-to-bucket-subresources) +// and Managing Access Permissions to Your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html). +// +// For information about CloudWatch request metrics for Amazon S3, see Monitoring +// Metrics with Amazon CloudWatch (https://docs.aws.amazon.com/AmazonS3/latest/dev/cloudwatch-monitoring.html). +// +// The following operations are related to DeleteBucketMetricsConfiguration: +// +// * GetBucketMetricsConfiguration +// +// * PutBucketMetricsConfiguration +// +// * ListBucketMetricsConfigurations +// +// * Monitoring Metrics with Amazon CloudWatch (https://docs.aws.amazon.com/AmazonS3/latest/dev/cloudwatch-monitoring.html) // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -995,7 +1536,29 @@ func (c *S3) DeleteBucketPolicyRequest(input *DeleteBucketPolicyInput) (req *req // DeleteBucketPolicy API operation for Amazon Simple Storage Service. // -// Deletes the policy from the bucket. +// This implementation of the DELETE operation uses the policy subresource to +// delete the policy of a specified bucket. If you are using an identity other +// than the root user of the AWS account that owns the bucket, the calling identity +// must have the DeleteBucketPolicy permissions on the specified bucket and +// belong to the bucket owner's account to use this operation. +// +// If you don't have DeleteBucketPolicy permissions, Amazon S3 returns a 403 +// Access Denied error. If you have the correct permissions, but you're not +// using an identity that belongs to the bucket owner's account, Amazon S3 returns +// a 405 Method Not Allowed error. +// +// As a security precaution, the root user of the AWS account that owns a bucket +// can always use this operation, even if the policy explicitly denies the root +// user the ability to perform this action. +// +// For more information about bucket policies, see Using Bucket Policies and +// UserPolicies (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-iam-policies.html). +// +// The following operations are related to DeleteBucketPolicy +// +// * CreateBucket +// +// * DeleteObject // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -1070,10 +1633,26 @@ func (c *S3) DeleteBucketReplicationRequest(input *DeleteBucketReplicationInput) // DeleteBucketReplication API operation for Amazon Simple Storage Service. // -// Deletes the replication configuration from the bucket. For information about -// replication configuration, see Cross-Region Replication (CRR) ( https://docs.aws.amazon.com/AmazonS3/latest/dev/crr.html) +// Deletes the replication configuration from the bucket. +// +// To use this operation, you must have permissions to perform the s3:PutReplicationConfiguration +// action. The bucket owner has these permissions by default and can grant it +// to others. For more information about permissions, see Permissions Related +// to Bucket Subresource Operations (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html#using-with-s3-actions-related-to-bucket-subresources) +// and Managing Access Permissions to Your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html). +// +// It can take a while for the deletion of a replication configuration to fully +// propagate. +// +// For information about replication configuration, see Replication (https://docs.aws.amazon.com/AmazonS3/latest/dev/replication.html) // in the Amazon S3 Developer Guide. // +// The following operations are related to DeleteBucketReplication: +// +// * PutBucketReplication +// +// * GetBucketReplication +// // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about // the error. @@ -1149,6 +1728,16 @@ func (c *S3) DeleteBucketTaggingRequest(input *DeleteBucketTaggingInput) (req *r // // Deletes the tags from the bucket. // +// To use this operation, you must have permission to perform the s3:PutBucketTagging +// action. By default, the bucket owner has this permission and can grant this +// permission to others. +// +// The following operations are related to DeleteBucketTagging: +// +// * GetBucketTagging +// +// * PutBucketTagging +// // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about // the error. @@ -1222,7 +1811,26 @@ func (c *S3) DeleteBucketWebsiteRequest(input *DeleteBucketWebsiteInput) (req *r // DeleteBucketWebsite API operation for Amazon Simple Storage Service. // -// This operation removes the website configuration from the bucket. +// This operation removes the website configuration for a bucket. Amazon S3 +// returns a 200 OK response upon successfully deleting a website configuration +// on the specified bucket. You will get a 200 OK response if the website configuration +// you are trying to delete does not exist on the bucket. Amazon S3 returns +// a 404 response if the bucket specified in the request does not exist. +// +// This DELETE operation requires the S3:DeleteBucketWebsite permission. By +// default, only the bucket owner can delete the website configuration attached +// to a bucket. However, bucket owners can grant other users permission to delete +// the website configuration by writing a bucket policy granting them the S3:DeleteBucketWebsite +// permission. +// +// For more information about hosting websites, see Hosting Websites on Amazon +// S3 (https://docs.aws.amazon.com/AmazonS3/latest/dev/WebsiteHosting.html). +// +// The following operations are related to DeleteBucketWebsite: +// +// * GetBucketWebsite +// +// * PutBucketWebsite // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -1300,6 +1908,29 @@ func (c *S3) DeleteObjectRequest(input *DeleteObjectInput) (req *request.Request // marker, which becomes the latest version of the object. If there isn't a // null version, Amazon S3 does not remove any objects. // +// To remove a specific version, you must be the bucket owner and you must use +// the version Id subresource. Using this subresource permanently deletes the +// version. If the object deleted is a delete marker, Amazon S3 sets the response +// header, x-amz-delete-marker, to true. +// +// If the object you want to delete is in a bucket where the bucket versioning +// configuration is MFA Delete enabled, you must include the x-amz-mfa request +// header in the DELETE versionId request. Requests that include x-amz-mfa must +// use HTTPS. +// +// For more information about MFA Delete, see Using MFA Delete (https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMFADelete.html). +// To see sample requests that use versioning, see Sample Request (https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETE.html#ExampleVersionObjectDelete). +// +// You can delete objects by explicitly calling the DELETE Object API or configure +// its lifecycle (PutBucketLifecycle) to enable Amazon S3 to remove them for +// you. If you want to block users or accounts from removing or deleting objects +// from your bucket, you must deny them the s3:DeleteObject, s3:DeleteObjectVersion, +// and s3:PutLifeCycleConfiguration actions. +// +// The following operation is related to DeleteObject: +// +// * PutObject +// // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about // the error. @@ -1372,7 +2003,21 @@ func (c *S3) DeleteObjectTaggingRequest(input *DeleteObjectTaggingInput) (req *r // DeleteObjectTagging API operation for Amazon Simple Storage Service. // -// Removes the tag-set from an existing object. +// Removes the entire tag set from the specified object. For more information +// about managing object tags, see Object Tagging (https://docs.aws.amazon.com/AmazonS3/latest/dev/object-tagging.html). +// +// To use this operation, you must have permission to perform the s3:DeleteObjectTagging +// action. +// +// To delete tags of a specific object version, add the versionId query parameter +// in the request. You will need permission for the s3:DeleteObjectVersionTagging +// action. +// +// The following operations are related to DeleteBucketMetricsConfiguration: +// +// * PutObjectTagging +// +// * GetObjectTagging // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -1441,13 +2086,57 @@ func (c *S3) DeleteObjectsRequest(input *DeleteObjectsInput) (req *request.Reque output = &DeleteObjectsOutput{} req = c.newRequest(op, input, output) + req.Handlers.Build.PushBackNamed(request.NamedHandler{ + Name: "contentMd5Handler", + Fn: checksum.AddBodyContentMD5Handler, + }) return } // DeleteObjects API operation for Amazon Simple Storage Service. // // This operation enables you to delete multiple objects from a bucket using -// a single HTTP request. You may specify up to 1000 keys. +// a single HTTP request. If you know the object keys that you want to delete, +// then this operation provides a suitable alternative to sending individual +// delete requests, reducing per-request overhead. +// +// The request contains a list of up to 1000 keys that you want to delete. In +// the XML, you provide the object key names, and optionally, version IDs if +// you want to delete a specific version of the object from a versioning-enabled +// bucket. For each key, Amazon S3 performs a delete operation and returns the +// result of that delete, success, or failure, in the response. Note that if +// the object specified in the request is not found, Amazon S3 returns the result +// as deleted. +// +// The operation supports two modes for the response: verbose and quiet. By +// default, the operation uses verbose mode in which the response includes the +// result of deletion of each key in your request. In quiet mode the response +// includes only keys where the delete operation encountered an error. For a +// successful deletion, the operation does not return any information about +// the delete in the response body. +// +// When performing this operation on an MFA Delete enabled bucket, that attempts +// to delete any versioned objects, you must include an MFA token. If you do +// not provide one, the entire request will fail, even if there are non-versioned +// objects you are trying to delete. If you provide an invalid token, whether +// there are versioned keys in the request or not, the entire Multi-Object Delete +// request will fail. For information about MFA Delete, see MFA Delete (https://docs.aws.amazon.com/AmazonS3/latest/dev/Versioning.html#MultiFactorAuthenticationDelete). +// +// Finally, the Content-MD5 header is required for all Multi-Object Delete requests. +// Amazon S3 uses the header value to ensure that your request body has not +// been altered in transit. +// +// The following operations are related to DeleteObjects: +// +// * CreateMultipartUpload +// +// * UploadPart +// +// * CompleteMultipartUpload +// +// * ListParts +// +// * AbortMultipartUpload // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -1522,7 +2211,21 @@ func (c *S3) DeletePublicAccessBlockRequest(input *DeletePublicAccessBlockInput) // DeletePublicAccessBlock API operation for Amazon Simple Storage Service. // -// Removes the PublicAccessBlock configuration from an Amazon S3 bucket. +// Removes the PublicAccessBlock configuration for an Amazon S3 bucket. To use +// this operation, you must have the s3:PutBucketPublicAccessBlock permission. +// For more information about permissions, see Permissions Related to Bucket +// Subresource Operations (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html#using-with-s3-actions-related-to-bucket-subresources) +// and Managing Access Permissions to Your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html). +// +// The following operations are related to DeletePublicAccessBlock: +// +// * Using Amazon S3 Block Public Access (https://docs.aws.amazon.com/AmazonS3/latest/dev/access-control-block-public-access.html) +// +// * GetPublicAccessBlock +// +// * PutPublicAccessBlock +// +// * GetBucketPolicyStatus // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -1596,7 +2299,32 @@ func (c *S3) GetBucketAccelerateConfigurationRequest(input *GetBucketAccelerateC // GetBucketAccelerateConfiguration API operation for Amazon Simple Storage Service. // -// Returns the accelerate configuration of a bucket. +// This implementation of the GET operation uses the accelerate subresource +// to return the Transfer Acceleration state of a bucket, which is either Enabled +// or Suspended. Amazon S3 Transfer Acceleration is a bucket-level feature that +// enables you to perform faster data transfers to and from Amazon S3. +// +// To use this operation, you must have permission to perform the s3:GetAccelerateConfiguration +// action. The bucket owner has this permission by default. The bucket owner +// can grant this permission to others. For more information about permissions, +// see Permissions Related to Bucket Subresource Operations (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html#using-with-s3-actions-related-to-bucket-subresources) +// and Managing Access Permissions to your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// You set the Transfer Acceleration state of an existing bucket to Enabled +// or Suspended by using the PutBucketAccelerateConfiguration operation. +// +// A GET accelerate request does not return a state value for a bucket that +// has no transfer acceleration state. A bucket has no Transfer Acceleration +// state if a state has never been set on the bucket. +// +// For more information about transfer acceleration, see Transfer Acceleration +// (https://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// Related Resources +// +// * PutBucketAccelerateConfiguration // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -1670,7 +2398,15 @@ func (c *S3) GetBucketAclRequest(input *GetBucketAclInput) (req *request.Request // GetBucketAcl API operation for Amazon Simple Storage Service. // -// Gets the access control policy for the bucket. +// This implementation of the GET operation uses the acl subresource to return +// the access control list (ACL) of a bucket. To use GET to return the ACL of +// the bucket, you must have READ_ACP access to the bucket. If READ_ACP permission +// is granted to the anonymous user, you can return the ACL of the bucket without +// using an authorization header. +// +// Related Resources +// +// * // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -1744,8 +2480,27 @@ func (c *S3) GetBucketAnalyticsConfigurationRequest(input *GetBucketAnalyticsCon // GetBucketAnalyticsConfiguration API operation for Amazon Simple Storage Service. // -// Gets an analytics configuration for the bucket (specified by the analytics -// configuration ID). +// This implementation of the GET operation returns an analytics configuration +// (identified by the analytics configuration ID) from the bucket. +// +// To use this operation, you must have permissions to perform the s3:GetAnalyticsConfiguration +// action. The bucket owner has this permission by default. The bucket owner +// can grant this permission to others. For more information about permissions, +// see Permissions Related to Bucket Subresource Operations (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html#using-with-s3-actions-related-to-bucket-subresources) +// and Managing Access Permissions to Your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// For information about Amazon S3 analytics feature, see Amazon S3 Analytics +// – Storage Class Analysis (https://docs.aws.amazon.com/AmazonS3/latest/dev/analytics-storage-class.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// Related Resources +// +// * +// +// * +// +// * // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -1819,7 +2574,20 @@ func (c *S3) GetBucketCorsRequest(input *GetBucketCorsInput) (req *request.Reque // GetBucketCors API operation for Amazon Simple Storage Service. // -// Returns the CORS configuration for the bucket. +// Returns the cors configuration information set for the bucket. +// +// To use this operation, you must have permission to perform the s3:GetBucketCORS +// action. By default, the bucket owner has this permission and can grant it +// to others. +// +// For more information about cors, see Enabling Cross-Origin Resource Sharing +// (https://docs.aws.amazon.com/AmazonS3/latest/dev/cors.html). +// +// The following operations are related to GetBucketCors: +// +// * PutBucketCors +// +// * DeleteBucketCors // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -1893,7 +2661,21 @@ func (c *S3) GetBucketEncryptionRequest(input *GetBucketEncryptionInput) (req *r // GetBucketEncryption API operation for Amazon Simple Storage Service. // -// Returns the server-side encryption configuration of a bucket. +// Returns the default encryption configuration for an Amazon S3 bucket. For +// information about the Amazon S3 default encryption feature, see Amazon S3 +// Default Bucket Encryption (https://docs.aws.amazon.com/AmazonS3/latest/dev/bucket-encryption.html). +// +// To use this operation, you must have permission to perform the s3:GetEncryptionConfiguration +// action. The bucket owner has this permission by default. The bucket owner +// can grant this permission to others. For more information about permissions, +// see Permissions Related to Bucket Subresource Operations (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html#using-with-s3-actions-related-to-bucket-subresources) +// and Managing Access Permissions to Your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html). +// +// The following operations are related to GetBucketEncryption: +// +// * PutBucketEncryption +// +// * DeleteBucketEncryption // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -1967,8 +2749,25 @@ func (c *S3) GetBucketInventoryConfigurationRequest(input *GetBucketInventoryCon // GetBucketInventoryConfiguration API operation for Amazon Simple Storage Service. // -// Returns an inventory configuration (identified by the inventory ID) from -// the bucket. +// Returns an inventory configuration (identified by the inventory configuration +// ID) from the bucket. +// +// To use this operation, you must have permissions to perform the s3:GetInventoryConfiguration +// action. The bucket owner has this permission by default and can grant this +// permission to others. For more information about permissions, see Permissions +// Related to Bucket Subresource Operations (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html#using-with-s3-actions-related-to-bucket-subresources) +// and Managing Access Permissions to Your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html). +// +// For information about the Amazon S3 inventory feature, see Amazon S3 Inventory +// (https://docs.aws.amazon.com/AmazonS3/latest/dev/storage-inventory.html). +// +// The following operations are related to GetBucketInventoryConfiguration: +// +// * DeleteBucketInventoryConfiguration +// +// * ListBucketInventoryConfigurations +// +// * PutBucketInventoryConfiguration // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -2047,7 +2846,34 @@ func (c *S3) GetBucketLifecycleRequest(input *GetBucketLifecycleInput) (req *req // GetBucketLifecycle API operation for Amazon Simple Storage Service. // -// Deprecated, see the GetBucketLifecycleConfiguration operation. +// +// For an updated version of this API, see GetBucketLifecycleConfiguration. +// If you configured a bucket lifecycle using the filter element, you should +// see the updated version of this topic. This topic is provided for backward +// compatibility. +// +// Returns the lifecycle configuration information set on the bucket. For information +// about lifecycle configuration, see Object Lifecycle Management (https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html). +// +// To use this operation, you must have permission to perform the s3:GetLifecycleConfiguration +// action. The bucket owner has this permission by default. The bucket owner +// can grant this permission to others. For more information about permissions, +// see Permissions Related to Bucket Subresource Operations (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html#using-with-s3-actions-related-to-bucket-subresources) +// and Managing Access Permissions to Your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html). +// +// GetBucketLifecycle has the following special error: +// +// * Error code: NoSuchLifecycleConfiguration Description: The lifecycle +// configuration does not exist. HTTP Status Code: 404 Not Found SOAP Fault +// Code Prefix: Client +// +// The following operations are related to GetBucketLifecycle: +// +// * GetBucketLifecycleConfiguration +// +// * PutBucketLifecycle +// +// * DeleteBucketLifecycle // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -2125,7 +2951,37 @@ func (c *S3) GetBucketLifecycleConfigurationRequest(input *GetBucketLifecycleCon // GetBucketLifecycleConfiguration API operation for Amazon Simple Storage Service. // -// Returns the lifecycle configuration information set on the bucket. +// +// Bucket lifecycle configuration now supports specifying a lifecycle rule using +// an object key name prefix, one or more object tags, or a combination of both. +// Accordingly, this section describes the latest API. The response describes +// the new filter element that you can use to specify a filter to select a subset +// of objects to which the rule applies. If you are still using previous version +// of the lifecycle configuration, it works. For the earlier API description, +// see GetBucketLifecycle. +// +// Returns the lifecycle configuration information set on the bucket. For information +// about lifecycle configuration, see Object Lifecycle Management (https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html). +// +// To use this operation, you must have permission to perform the s3:GetLifecycleConfiguration +// action. The bucket owner has this permission, by default. The bucket owner +// can grant this permission to others. For more information about permissions, +// see Permissions Related to Bucket Subresource Operations (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html#using-with-s3-actions-related-to-bucket-subresources) +// and Managing Access Permissions to Your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html). +// +// GetBucketLifecycleConfiguration has the following special error: +// +// * Error code: NoSuchLifecycleConfiguration Description: The lifecycle +// configuration does not exist. HTTP Status Code: 404 Not Found SOAP Fault +// Code Prefix: Client +// +// The following operations are related to GetBucketLifecycleConfiguration: +// +// * GetBucketLifecycle +// +// * PutBucketLifecycle +// +// * DeleteBucketLifecycle // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -2199,7 +3055,17 @@ func (c *S3) GetBucketLocationRequest(input *GetBucketLocationInput) (req *reque // GetBucketLocation API operation for Amazon Simple Storage Service. // -// Returns the region the bucket resides in. +// Returns the Region the bucket resides in. You set the bucket's Region using +// the LocationConstraint request parameter in a CreateBucket request. For more +// information, see CreateBucket. +// +// To use this implementation of the operation, you must be the bucket owner. +// +// The following operations are related to GetBucketLocation: +// +// * GetObject +// +// * CreateBucket // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -2276,6 +3142,12 @@ func (c *S3) GetBucketLoggingRequest(input *GetBucketLoggingInput) (req *request // Returns the logging status of a bucket and the permissions users have to // view and modify that status. To use GET, you must be the bucket owner. // +// The following operations are related to GetBucketLogging: +// +// * CreateBucket +// +// * PutBucketLogging +// // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about // the error. @@ -2349,7 +3221,26 @@ func (c *S3) GetBucketMetricsConfigurationRequest(input *GetBucketMetricsConfigu // GetBucketMetricsConfiguration API operation for Amazon Simple Storage Service. // // Gets a metrics configuration (specified by the metrics configuration ID) -// from the bucket. +// from the bucket. Note that this doesn't include the daily storage metrics. +// +// To use this operation, you must have permissions to perform the s3:GetMetricsConfiguration +// action. The bucket owner has this permission by default. The bucket owner +// can grant this permission to others. For more information about permissions, +// see Permissions Related to Bucket Subresource Operations (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html#using-with-s3-actions-related-to-bucket-subresources) +// and Managing Access Permissions to Your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html). +// +// For information about CloudWatch request metrics for Amazon S3, see Monitoring +// Metrics with Amazon CloudWatch (https://docs.aws.amazon.com/AmazonS3/latest/dev/cloudwatch-monitoring.html). +// +// The following operations are related to GetBucketMetricsConfiguration: +// +// * PutBucketMetricsConfiguration +// +// * DeleteBucketMetricsConfiguration +// +// * ListBucketMetricsConfigurations +// +// * Monitoring Metrics with Amazon CloudWatch (https://docs.aws.amazon.com/AmazonS3/latest/dev/cloudwatch-monitoring.html) // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -2428,7 +3319,7 @@ func (c *S3) GetBucketNotificationRequest(input *GetBucketNotificationConfigurat // GetBucketNotification API operation for Amazon Simple Storage Service. // -// Deprecated, see the GetBucketNotificationConfiguration operation. +// No longer used, see GetBucketNotificationConfiguration. // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -2508,6 +3399,22 @@ func (c *S3) GetBucketNotificationConfigurationRequest(input *GetBucketNotificat // // Returns the notification configuration of a bucket. // +// If notifications are not enabled on the bucket, the operation returns an +// empty NotificationConfiguration element. +// +// By default, you must be the bucket owner to read the notification configuration +// of a bucket. However, the bucket owner can use a bucket policy to grant permission +// to other users to read this configuration with the s3:GetBucketNotification +// permission. +// +// For more information about setting and reading the notification configuration +// on a bucket, see Setting Up Notification of Bucket Events (https://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html). +// For more information about bucket policies, see Using Bucket Policies (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-iam-policies.html). +// +// The following operation is related to GetBucketNotification: +// +// * PutBucketNotification +// // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about // the error. @@ -2580,7 +3487,26 @@ func (c *S3) GetBucketPolicyRequest(input *GetBucketPolicyInput) (req *request.R // GetBucketPolicy API operation for Amazon Simple Storage Service. // -// Returns the policy of a specified bucket. +// Returns the policy of a specified bucket. If you are using an identity other +// than the root user of the AWS account that owns the bucket, the calling identity +// must have the GetBucketPolicy permissions on the specified bucket and belong +// to the bucket owner's account in order to use this operation. +// +// If you don't have GetBucketPolicy permissions, Amazon S3 returns a 403 Access +// Denied error. If you have the correct permissions, but you're not using an +// identity that belongs to the bucket owner's account, Amazon S3 returns a +// 405 Method Not Allowed error. +// +// As a security precaution, the root user of the AWS account that owns a bucket +// can always use this operation, even if the policy explicitly denies the root +// user the ability to perform this action. +// +// For more information about bucket policies, see Using Bucket Policies and +// User Policies (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-iam-policies.html). +// +// The following operation is related to GetBucketPolicy: +// +// * GetObject // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -2655,7 +3581,22 @@ func (c *S3) GetBucketPolicyStatusRequest(input *GetBucketPolicyStatusInput) (re // GetBucketPolicyStatus API operation for Amazon Simple Storage Service. // // Retrieves the policy status for an Amazon S3 bucket, indicating whether the -// bucket is public. +// bucket is public. In order to use this operation, you must have the s3:GetBucketPolicyStatus +// permission. For more information about Amazon S3 permissions, see Specifying +// Permissions in a Policy (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html). +// +// For more information about when Amazon S3 considers a bucket public, see +// The Meaning of "Public" (https://docs.aws.amazon.com/AmazonS3/latest/dev/access-control-block-public-access.html#access-control-block-public-access-policy-status). +// +// The following operations are related to GetBucketPolicyStatus: +// +// * Using Amazon S3 Block Public Access (https://docs.aws.amazon.com/AmazonS3/latest/dev/access-control-block-public-access.html) +// +// * GetPublicAccessBlock +// +// * PutPublicAccessBlock +// +// * DeletePublicAccessBlock // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -2735,6 +3676,25 @@ func (c *S3) GetBucketReplicationRequest(input *GetBucketReplicationInput) (req // to all Amazon S3 systems. Therefore, a get request soon after put or delete // can return a wrong result. // +// For information about replication configuration, see Replication (https://docs.aws.amazon.com/AmazonS3/latest/dev/replication.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// This operation requires permissions for the s3:GetReplicationConfiguration +// action. For more information about permissions, see Using Bucket Policies +// and User Policies (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-iam-policies.html). +// +// If you include the Filter element in a replication configuration, you must +// also include the DeleteMarkerReplication and Priority elements. The response +// also returns those elements. +// +// For information about GetBucketReplication errors, see ReplicationErrorCodeList +// +// The following operations are related to GetBucketReplication: +// +// * PutBucketReplication +// +// * DeleteBucketReplication +// // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about // the error. @@ -2807,7 +3767,13 @@ func (c *S3) GetBucketRequestPaymentRequest(input *GetBucketRequestPaymentInput) // GetBucketRequestPayment API operation for Amazon Simple Storage Service. // -// Returns the request payment configuration of a bucket. +// Returns the request payment configuration of a bucket. To use this version +// of the operation, you must be the bucket owner. For more information, see +// Requester Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/RequesterPaysBuckets.html). +// +// The following operations are related to GetBucketRequestPayment: +// +// * ListObjects // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -2883,6 +3849,21 @@ func (c *S3) GetBucketTaggingRequest(input *GetBucketTaggingInput) (req *request // // Returns the tag set associated with the bucket. // +// To use this operation, you must have permission to perform the s3:GetBucketTagging +// action. By default, the bucket owner has this permission and can grant this +// permission to others. +// +// GetBucketTagging has the following special error: +// +// * Error code: NoSuchTagSetError Description: There is no tag set associated +// with the bucket. +// +// The following operations are related to GetBucketTagging: +// +// * PutBucketTagging +// +// * DeleteBucketTagging +// // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about // the error. @@ -2957,8 +3938,22 @@ func (c *S3) GetBucketVersioningRequest(input *GetBucketVersioningInput) (req *r // // Returns the versioning state of a bucket. // -// Returns awserr.Error for service API and SDK errors. Use runtime type assertions -// with awserr.Error's Code and Message methods to get detailed information about +// To retrieve the versioning state of a bucket, you must be the bucket owner. +// +// This implementation also returns the MFA Delete status of the versioning +// state. If the MFA Delete status is enabled, the bucket owner must use an +// authentication device to change the versioning state of the bucket. +// +// The following operations are related to GetBucketVersioning: +// +// * GetObject +// +// * PutObject +// +// * DeleteObject +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about // the error. // // See the AWS API reference guide for Amazon Simple Storage Service's @@ -3029,7 +4024,21 @@ func (c *S3) GetBucketWebsiteRequest(input *GetBucketWebsiteInput) (req *request // GetBucketWebsite API operation for Amazon Simple Storage Service. // -// Returns the website configuration for a bucket. +// Returns the website configuration for a bucket. To host website on Amazon +// S3, you can configure a bucket as website by adding a website configuration. +// For more information about hosting websites, see Hosting Websites on Amazon +// S3 (https://docs.aws.amazon.com/AmazonS3/latest/dev/WebsiteHosting.html). +// +// This GET operation requires the S3:GetBucketWebsite permission. By default, +// only the bucket owner can read the bucket website configuration. However, +// bucket owners can allow other users to read the website configuration by +// writing a bucket policy granting them the S3:GetBucketWebsite permission. +// +// The following operations are related to DeleteBucketWebsite: +// +// * DeleteBucketWebsite +// +// * PutBucketWebsite // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -3103,7 +4112,130 @@ func (c *S3) GetObjectRequest(input *GetObjectInput) (req *request.Request, outp // GetObject API operation for Amazon Simple Storage Service. // -// Retrieves objects from Amazon S3. +// Retrieves objects from Amazon S3. To use GET, you must have READ access to +// the object. If you grant READ access to the anonymous user, you can return +// the object without using an authorization header. +// +// An Amazon S3 bucket has no directory hierarchy such as you would find in +// a typical computer file system. You can, however, create a logical hierarchy +// by using object key names that imply a folder structure. For example, instead +// of naming an object sample.jpg, you can name it photos/2006/February/sample.jpg. +// +// To get an object from such a logical hierarchy, specify the full key name +// for the object in the GET operation. For a virtual hosted-style request example, +// if you have the object photos/2006/February/sample.jpg, specify the resource +// as /photos/2006/February/sample.jpg. For a path-style request example, if +// you have the object photos/2006/February/sample.jpg in the bucket named examplebucket, +// specify the resource as /examplebucket/photos/2006/February/sample.jpg. For +// more information about request types, see HTTP Host Header Bucket Specification +// (https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html#VirtualHostingSpecifyBucket). +// +// To distribute large files to many people, you can save bandwidth costs by +// using BitTorrent. For more information, see Amazon S3 Torrent (https://docs.aws.amazon.com/AmazonS3/latest/dev/S3Torrent.html). +// For more information about returning the ACL of an object, see GetObjectAcl. +// +// If the object you are retrieving is stored in the GLACIER or DEEP_ARCHIVE +// storage classes, before you can retrieve the object you must first restore +// a copy using . Otherwise, this operation returns an InvalidObjectStateError +// error. For information about restoring archived objects, see Restoring Archived +// Objects (https://docs.aws.amazon.com/AmazonS3/latest/dev/restoring-objects.html). +// +// Encryption request headers, like x-amz-server-side-encryption, should not +// be sent for GET requests if your object uses server-side encryption with +// CMKs stored in AWS KMS (SSE-KMS) or server-side encryption with Amazon S3–managed +// encryption keys (SSE-S3). If your object does use these types of keys, you’ll +// get an HTTP 400 BadRequest error. +// +// If you encrypt an object by using server-side encryption with customer-provided +// encryption keys (SSE-C) when you store the object in Amazon S3, then when +// you GET the object, you must use the following headers: +// +// * x-amz-server-side​-encryption​-customer-algorithm +// +// * x-amz-server-side​-encryption​-customer-key +// +// * x-amz-server-side​-encryption​-customer-key-MD5 +// +// For more information about SSE-C, see Server-Side Encryption (Using Customer-Provided +// Encryption Keys) (https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html). +// +// Assuming you have permission to read object tags (permission for the s3:GetObjectVersionTagging +// action), the response also returns the x-amz-tagging-count header that provides +// the count of number of tags associated with the object. You can use GetObjectTagging +// to retrieve the tag set associated with an object. +// +// Permissions +// +// You need the s3:GetObject permission for this operation. For more information, +// see Specifying Permissions in a Policy (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html). +// If the object you request does not exist, the error Amazon S3 returns depends +// on whether you also have the s3:ListBucket permission. +// +// * If you have the s3:ListBucket permission on the bucket, Amazon S3 will +// return an HTTP status code 404 ("no such key") error. +// +// * If you don’t have the s3:ListBucket permission, Amazon S3 will return +// an HTTP status code 403 ("access denied") error. +// +// Versioning +// +// By default, the GET operation returns the current version of an object. To +// return a different version, use the versionId subresource. +// +// If the current version of the object is a delete marker, Amazon S3 behaves +// as if the object was deleted and includes x-amz-delete-marker: true in the +// response. +// +// For more information about versioning, see PutBucketVersioning. +// +// Overriding Response Header Values +// +// There are times when you want to override certain response header values +// in a GET response. For example, you might override the Content-Disposition +// response header value in your GET request. +// +// You can override values for a set of response headers using the following +// query parameters. These response header values are sent only on a successful +// request, that is, when status code 200 OK is returned. The set of headers +// you can override using these parameters is a subset of the headers that Amazon +// S3 accepts when you create an object. The response headers that you can override +// for the GET response are Content-Type, Content-Language, Expires, Cache-Control, +// Content-Disposition, and Content-Encoding. To override these header values +// in the GET response, you use the following request parameters. +// +// You must sign the request, either using an Authorization header or a presigned +// URL, when using these parameters. They cannot be used with an unsigned (anonymous) +// request. +// +// * response-content-type +// +// * response-content-language +// +// * response-expires +// +// * response-cache-control +// +// * response-content-disposition +// +// * response-content-encoding +// +// Additional Considerations about Request Headers +// +// If both of the If-Match and If-Unmodified-Since headers are present in the +// request as follows: If-Match condition evaluates to true, and; If-Unmodified-Since +// condition evaluates to false; then, S3 returns 200 OK and the data requested. +// +// If both of the If-None-Match and If-Modified-Since headers are present in +// the request as follows:If-None-Match condition evaluates to false, and; If-Modified-Since +// condition evaluates to true; then, S3 returns 304 Not Modified response code. +// +// For more information about conditional requests, see RFC 7232 (https://tools.ietf.org/html/rfc7232). +// +// The following operations are related to GetObject: +// +// * ListBuckets +// +// * GetObjectAcl // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -3182,7 +4314,21 @@ func (c *S3) GetObjectAclRequest(input *GetObjectAclInput) (req *request.Request // GetObjectAcl API operation for Amazon Simple Storage Service. // -// Returns the access control list (ACL) of an object. +// Returns the access control list (ACL) of an object. To use this operation, +// you must have READ_ACP access to the object. +// +// Versioning +// +// By default, GET returns ACL information about the current version of an object. +// To return ACL information about a different version, use the versionId subresource. +// +// The following operations are related to GetObjectAcl: +// +// * GetObject +// +// * DeleteObject +// +// * PutObject // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -3261,7 +4407,8 @@ func (c *S3) GetObjectLegalHoldRequest(input *GetObjectLegalHoldInput) (req *req // GetObjectLegalHold API operation for Amazon Simple Storage Service. // -// Gets an object's current Legal Hold status. +// Gets an object's current Legal Hold status. For more information, see Locking +// Objects (https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock.html). // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -3337,7 +4484,8 @@ func (c *S3) GetObjectLockConfigurationRequest(input *GetObjectLockConfiguration // // Gets the Object Lock configuration for a bucket. The rule specified in the // Object Lock configuration will be applied by default to every new object -// placed in the specified bucket. +// placed in the specified bucket. For more information, see Locking Objects +// (https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock.html). // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -3411,7 +4559,8 @@ func (c *S3) GetObjectRetentionRequest(input *GetObjectRetentionInput) (req *req // GetObjectRetention API operation for Amazon Simple Storage Service. // -// Retrieves an object's retention settings. +// Retrieves an object's retention settings. For more information, see Locking +// Objects (https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock.html). // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -3485,7 +4634,25 @@ func (c *S3) GetObjectTaggingRequest(input *GetObjectTaggingInput) (req *request // GetObjectTagging API operation for Amazon Simple Storage Service. // -// Returns the tag-set of an object. +// Returns the tag-set of an object. You send the GET request against the tagging +// subresource associated with the object. +// +// To use this operation, you must have permission to perform the s3:GetObjectTagging +// action. By default, the GET operation returns information about current version +// of an object. For a versioned bucket, you can have multiple versions of an +// object in your bucket. To retrieve tags of any other version, use the versionId +// query parameter. You also need permission for the s3:GetObjectVersionTagging +// action. +// +// By default, the bucket owner has this permission and can grant this permission +// to others. +// +// For information about the Amazon S3 object tagging feature, see Object Tagging +// (https://docs.aws.amazon.com/AmazonS3/latest/dev/object-tagging.html). +// +// The following operation is related to GetObjectTagging: +// +// * PutObjectTagging // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -3559,7 +4726,19 @@ func (c *S3) GetObjectTorrentRequest(input *GetObjectTorrentInput) (req *request // GetObjectTorrent API operation for Amazon Simple Storage Service. // -// Return torrent files from a bucket. +// Return torrent files from a bucket. BitTorrent can save you bandwidth when +// you're distributing large files. For more information about BitTorrent, see +// Amazon S3 Torrent (https://docs.aws.amazon.com/AmazonS3/latest/dev/S3Torrent.html). +// +// You can get torrent only for objects that are less than 5 GB in size and +// that are not encrypted using server-side encryption with customer-provided +// encryption key. +// +// To use GET, you must have READ access to the object. +// +// The following operation is related to GetObjectTorrent: +// +// * GetObject // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -3633,7 +4812,30 @@ func (c *S3) GetPublicAccessBlockRequest(input *GetPublicAccessBlockInput) (req // GetPublicAccessBlock API operation for Amazon Simple Storage Service. // -// Retrieves the PublicAccessBlock configuration for an Amazon S3 bucket. +// Retrieves the PublicAccessBlock configuration for an Amazon S3 bucket. To +// use this operation, you must have the s3:GetBucketPublicAccessBlock permission. +// For more information about Amazon S3 permissions, see Specifying Permissions +// in a Policy (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html). +// +// When Amazon S3 evaluates the PublicAccessBlock configuration for a bucket +// or an object, it checks the PublicAccessBlock configuration for both the +// bucket (or the bucket that contains the object) and the bucket owner's account. +// If the PublicAccessBlock settings are different between the bucket and the +// account, Amazon S3 uses the most restrictive combination of the bucket-level +// and account-level settings. +// +// For more information about when Amazon S3 considers a bucket or an object +// public, see The Meaning of "Public" (https://docs.aws.amazon.com/AmazonS3/latest/dev/access-control-block-public-access.html#access-control-block-public-access-policy-status). +// +// The following operations are related to GetPublicAccessBlock: +// +// * Using Amazon S3 Block Public Access (https://docs.aws.amazon.com/AmazonS3/latest/dev/access-control-block-public-access.html) +// +// * PutPublicAccessBlock +// +// * GetPublicAccessBlock +// +// * DeletePublicAccessBlock // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -3709,7 +4911,15 @@ func (c *S3) HeadBucketRequest(input *HeadBucketInput) (req *request.Request, ou // HeadBucket API operation for Amazon Simple Storage Service. // // This operation is useful to determine if a bucket exists and you have permission -// to access it. +// to access it. The operation returns a 200 OK if the bucket exists and you +// have permission to access it. Otherwise, the operation might return responses +// such as 404 Not Found and 403 Forbidden. +// +// To use this operation, you must have permissions to perform the s3:ListBucket +// action. The bucket owner has this permission by default and can grant this +// permission to others. For more information about permissions, see Permissions +// Related to Bucket Subresource Operations (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html#using-with-s3-actions-related-to-bucket-subresources) +// and Managing Access Permissions to Your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html). // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -3792,6 +5002,63 @@ func (c *S3) HeadObjectRequest(input *HeadObjectInput) (req *request.Request, ou // object itself. This operation is useful if you're only interested in an object's // metadata. To use HEAD, you must have READ access to the object. // +// A HEAD request has the same options as a GET operation on an object. The +// response is identical to the GET response except that there is no response +// body. +// +// If you encrypt an object by using server-side encryption with customer-provided +// encryption keys (SSE-C) when you store the object in Amazon S3, then when +// you retrieve the metadata from the object, you must use the following headers: +// +// * x-amz-server-side​-encryption​-customer-algorithm +// +// * x-amz-server-side​-encryption​-customer-key +// +// * x-amz-server-side​-encryption​-customer-key-MD5 +// +// For more information about SSE-C, see Server-Side Encryption (Using Customer-Provided +// Encryption Keys) (https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html). +// +// Encryption request headers, like x-amz-server-side-encryption, should not +// be sent for GET requests if your object uses server-side encryption with +// CMKs stored in AWS KMS (SSE-KMS) or server-side encryption with Amazon S3–managed +// encryption keys (SSE-S3). If your object does use these types of keys, you’ll +// get an HTTP 400 BadRequest error. +// +// Request headers are limited to 8 KB in size. For more information, see Common +// Request Headers (https://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonRequestHeaders.html). +// +// Consider the following when using request headers: +// +// * Consideration 1 – If both of the If-Match and If-Unmodified-Since +// headers are present in the request as follows: If-Match condition evaluates +// to true, and; If-Unmodified-Since condition evaluates to false; Then Amazon +// S3 returns 200 OK and the data requested. +// +// * Consideration 2 – If both of the If-None-Match and If-Modified-Since +// headers are present in the request as follows: If-None-Match condition +// evaluates to false, and; If-Modified-Since condition evaluates to true; +// Then Amazon S3 returns the 304 Not Modified response code. +// +// For more information about conditional requests, see RFC 7232 (https://tools.ietf.org/html/rfc7232). +// +// Permissions +// +// You need the s3:GetObject permission for this operation. For more information, +// see Specifying Permissions in a Policy (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html). +// If the object you request does not exist, the error Amazon S3 returns depends +// on whether you also have the s3:ListBucket permission. +// +// * If you have the s3:ListBucket permission on the bucket, Amazon S3 returns +// an HTTP status code 404 ("no such key") error. +// +// * If you don’t have the s3:ListBucket permission, Amazon S3 returns +// an HTTP status code 403 ("access denied") error. +// +// The following operation is related to HeadObject: +// +// * GetObject +// // See http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html#RESTErrorResponses // for more information on returned errors. // @@ -3867,7 +5134,33 @@ func (c *S3) ListBucketAnalyticsConfigurationsRequest(input *ListBucketAnalytics // ListBucketAnalyticsConfigurations API operation for Amazon Simple Storage Service. // -// Lists the analytics configurations for the bucket. +// Lists the analytics configurations for the bucket. You can have up to 1,000 +// analytics configurations per bucket. +// +// This operation supports list pagination and does not return more than 100 +// configurations at a time. You should always check the IsTruncated element +// in the response. If there are no more configurations to list, IsTruncated +// is set to false. If there are more configurations to list, IsTruncated is +// set to true, and there will be a value in NextContinuationToken. You use +// the NextContinuationToken value to continue the pagination of the list by +// passing the value in continuation-token in the request to GET the next page. +// +// To use this operation, you must have permissions to perform the s3:GetAnalyticsConfiguration +// action. The bucket owner has this permission by default. The bucket owner +// can grant this permission to others. For more information about permissions, +// see Permissions Related to Bucket Subresource Operations (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html#using-with-s3-actions-related-to-bucket-subresources) +// and Managing Access Permissions to Your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html). +// +// For information about Amazon S3 analytics feature, see Amazon S3 Analytics +// – Storage Class Analysis (https://docs.aws.amazon.com/AmazonS3/latest/dev/analytics-storage-class.html). +// +// The following operations are related to ListBucketAnalyticsConfigurations: +// +// * GetBucketAnalyticsConfiguration +// +// * DeleteBucketAnalyticsConfiguration +// +// * PutBucketAnalyticsConfiguration // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -3941,7 +5234,33 @@ func (c *S3) ListBucketInventoryConfigurationsRequest(input *ListBucketInventory // ListBucketInventoryConfigurations API operation for Amazon Simple Storage Service. // -// Returns a list of inventory configurations for the bucket. +// Returns a list of inventory configurations for the bucket. You can have up +// to 1,000 analytics configurations per bucket. +// +// This operation supports list pagination and does not return more than 100 +// configurations at a time. Always check the IsTruncated element in the response. +// If there are no more configurations to list, IsTruncated is set to false. +// If there are more configurations to list, IsTruncated is set to true, and +// there is a value in NextContinuationToken. You use the NextContinuationToken +// value to continue the pagination of the list by passing the value in continuation-token +// in the request to GET the next page. +// +// To use this operation, you must have permissions to perform the s3:GetInventoryConfiguration +// action. The bucket owner has this permission by default. The bucket owner +// can grant this permission to others. For more information about permissions, +// see Permissions Related to Bucket Subresource Operations (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html#using-with-s3-actions-related-to-bucket-subresources) +// and Managing Access Permissions to Your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html). +// +// For information about the Amazon S3 inventory feature, see Amazon S3 Inventory +// (https://docs.aws.amazon.com/AmazonS3/latest/dev/storage-inventory.html) +// +// The following operations are related to ListBucketInventoryConfigurations: +// +// * GetBucketInventoryConfiguration +// +// * DeleteBucketInventoryConfiguration +// +// * PutBucketInventoryConfiguration // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -4015,7 +5334,34 @@ func (c *S3) ListBucketMetricsConfigurationsRequest(input *ListBucketMetricsConf // ListBucketMetricsConfigurations API operation for Amazon Simple Storage Service. // -// Lists the metrics configurations for the bucket. +// Lists the metrics configurations for the bucket. The metrics configurations +// are only for the request metrics of the bucket and do not provide information +// on daily storage metrics. You can have up to 1,000 configurations per bucket. +// +// This operation supports list pagination and does not return more than 100 +// configurations at a time. Always check the IsTruncated element in the response. +// If there are no more configurations to list, IsTruncated is set to false. +// If there are more configurations to list, IsTruncated is set to true, and +// there is a value in NextContinuationToken. You use the NextContinuationToken +// value to continue the pagination of the list by passing the value in continuation-token +// in the request to GET the next page. +// +// To use this operation, you must have permissions to perform the s3:GetMetricsConfiguration +// action. The bucket owner has this permission by default. The bucket owner +// can grant this permission to others. For more information about permissions, +// see Permissions Related to Bucket Subresource Operations (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html#using-with-s3-actions-related-to-bucket-subresources) +// and Managing Access Permissions to Your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html). +// +// For more information about metrics configurations and CloudWatch request +// metrics, see Monitoring Metrics with Amazon CloudWatch (https://docs.aws.amazon.com/AmazonS3/latest/dev/cloudwatch-monitoring.html). +// +// The following operations are related to ListBucketMetricsConfigurations: +// +// * PutBucketMetricsConfiguration +// +// * GetBucketMetricsConfiguration +// +// * DeleteBucketMetricsConfiguration // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -4169,7 +5515,40 @@ func (c *S3) ListMultipartUploadsRequest(input *ListMultipartUploadsInput) (req // ListMultipartUploads API operation for Amazon Simple Storage Service. // -// This operation lists in-progress multipart uploads. +// This operation lists in-progress multipart uploads. An in-progress multipart +// upload is a multipart upload that has been initiated using the Initiate Multipart +// Upload request, but has not yet been completed or aborted. +// +// This operation returns at most 1,000 multipart uploads in the response. 1,000 +// multipart uploads is the maximum number of uploads a response can include, +// which is also the default value. You can further limit the number of uploads +// in a response by specifying the max-uploads parameter in the response. If +// additional multipart uploads satisfy the list criteria, the response will +// contain an IsTruncated element with the value true. To list the additional +// multipart uploads, use the key-marker and upload-id-marker request parameters. +// +// In the response, the uploads are sorted by key. If your application has initiated +// more than one multipart upload using the same object key, then uploads in +// the response are first sorted by key. Additionally, uploads are sorted in +// ascending order within each key by the upload initiation time. +// +// For more information on multipart uploads, see Uploading Objects Using Multipart +// Upload (https://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html). +// +// For information on permissions required to use the multipart upload API, +// see Multipart Upload API and Permissions (https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html). +// +// The following operations are related to ListMultipartUploads: +// +// * CreateMultipartUpload +// +// * UploadPart +// +// * CompleteMultipartUpload +// +// * ListParts +// +// * AbortMultipartUpload // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -4210,7 +5589,7 @@ func (c *S3) ListMultipartUploadsWithContext(ctx aws.Context, input *ListMultipa // // Example iterating over at most 3 pages of a ListMultipartUploads operation. // pageNum := 0 // err := client.ListMultipartUploadsPages(params, -// func(page *ListMultipartUploadsOutput, lastPage bool) bool { +// func(page *s3.ListMultipartUploadsOutput, lastPage bool) bool { // pageNum++ // fmt.Println(page) // return pageNum <= 3 @@ -4242,10 +5621,12 @@ func (c *S3) ListMultipartUploadsPagesWithContext(ctx aws.Context, input *ListMu }, } - cont := true - for p.Next() && cont { - cont = fn(p.Page().(*ListMultipartUploadsOutput), !p.HasNextPage()) + for p.Next() { + if !fn(p.Page().(*ListMultipartUploadsOutput), !p.HasNextPage()) { + break + } } + return p.Err() } @@ -4299,7 +5680,24 @@ func (c *S3) ListObjectVersionsRequest(input *ListObjectVersionsInput) (req *req // ListObjectVersions API operation for Amazon Simple Storage Service. // -// Returns metadata about all of the versions of objects in a bucket. +// Returns metadata about all of the versions of objects in a bucket. You can +// also use request parameters as selection criteria to return metadata about +// a subset of all the object versions. +// +// A 200 OK response can contain valid or invalid XML. Make sure to design your +// application to parse the contents of the response and handle it appropriately. +// +// To use this operation, you must have READ access to the bucket. +// +// The following operations are related to ListObjectVersions: +// +// * ListObjectsV2 +// +// * GetObject +// +// * PutObject +// +// * DeleteObject // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -4340,7 +5738,7 @@ func (c *S3) ListObjectVersionsWithContext(ctx aws.Context, input *ListObjectVer // // Example iterating over at most 3 pages of a ListObjectVersions operation. // pageNum := 0 // err := client.ListObjectVersionsPages(params, -// func(page *ListObjectVersionsOutput, lastPage bool) bool { +// func(page *s3.ListObjectVersionsOutput, lastPage bool) bool { // pageNum++ // fmt.Println(page) // return pageNum <= 3 @@ -4372,10 +5770,12 @@ func (c *S3) ListObjectVersionsPagesWithContext(ctx aws.Context, input *ListObje }, } - cont := true - for p.Next() && cont { - cont = fn(p.Page().(*ListObjectVersionsOutput), !p.HasNextPage()) + for p.Next() { + if !fn(p.Page().(*ListObjectVersionsOutput), !p.HasNextPage()) { + break + } } + return p.Err() } @@ -4429,9 +5829,27 @@ func (c *S3) ListObjectsRequest(input *ListObjectsInput) (req *request.Request, // ListObjects API operation for Amazon Simple Storage Service. // -// Returns some or all (up to 1000) of the objects in a bucket. You can use +// Returns some or all (up to 1,000) of the objects in a bucket. You can use // the request parameters as selection criteria to return a subset of the objects -// in a bucket. +// in a bucket. A 200 OK response can contain valid or invalid XML. Be sure +// to design your application to parse the contents of the response and handle +// it appropriately. +// +// This API has been revised. We recommend that you use the newer version, ListObjectsV2, +// when developing applications. For backward compatibility, Amazon S3 continues +// to support ListObjects. +// +// The following operations are related to ListObjects: +// +// * ListObjectsV2 +// +// * GetObject +// +// * PutObject +// +// * CreateBucket +// +// * ListBuckets // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -4477,7 +5895,7 @@ func (c *S3) ListObjectsWithContext(ctx aws.Context, input *ListObjectsInput, op // // Example iterating over at most 3 pages of a ListObjects operation. // pageNum := 0 // err := client.ListObjectsPages(params, -// func(page *ListObjectsOutput, lastPage bool) bool { +// func(page *s3.ListObjectsOutput, lastPage bool) bool { // pageNum++ // fmt.Println(page) // return pageNum <= 3 @@ -4509,10 +5927,12 @@ func (c *S3) ListObjectsPagesWithContext(ctx aws.Context, input *ListObjectsInpu }, } - cont := true - for p.Next() && cont { - cont = fn(p.Page().(*ListObjectsOutput), !p.HasNextPage()) + for p.Next() { + if !fn(p.Page().(*ListObjectsOutput), !p.HasNextPage()) { + break + } } + return p.Err() } @@ -4566,10 +5986,34 @@ func (c *S3) ListObjectsV2Request(input *ListObjectsV2Input) (req *request.Reque // ListObjectsV2 API operation for Amazon Simple Storage Service. // -// Returns some or all (up to 1000) of the objects in a bucket. You can use +// Returns some or all (up to 1,000) of the objects in a bucket. You can use // the request parameters as selection criteria to return a subset of the objects -// in a bucket. Note: ListObjectsV2 is the revised List Objects API and we recommend -// you use this revised API for new application development. +// in a bucket. A 200 OK response can contain valid or invalid XML. Make sure +// to design your application to parse the contents of the response and handle +// it appropriately. +// +// To use this operation, you must have READ access to the bucket. +// +// To use this operation in an AWS Identity and Access Management (IAM) policy, +// you must have permissions to perform the s3:ListBucket action. The bucket +// owner has this permission by default and can grant this permission to others. +// For more information about permissions, see Permissions Related to Bucket +// Subresource Operations (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html#using-with-s3-actions-related-to-bucket-subresources) +// and Managing Access Permissions to Your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html). +// +// This section describes the latest revision of the API. We recommend that +// you use this revised API for application development. For backward compatibility, +// Amazon S3 continues to support the prior version of this API, ListObjects. +// +// To get a list of your buckets, see ListBuckets. +// +// The following operations are related to ListObjectsV2: +// +// * GetObject +// +// * PutObject +// +// * CreateBucket // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -4615,7 +6059,7 @@ func (c *S3) ListObjectsV2WithContext(ctx aws.Context, input *ListObjectsV2Input // // Example iterating over at most 3 pages of a ListObjectsV2 operation. // pageNum := 0 // err := client.ListObjectsV2Pages(params, -// func(page *ListObjectsV2Output, lastPage bool) bool { +// func(page *s3.ListObjectsV2Output, lastPage bool) bool { // pageNum++ // fmt.Println(page) // return pageNum <= 3 @@ -4647,10 +6091,12 @@ func (c *S3) ListObjectsV2PagesWithContext(ctx aws.Context, input *ListObjectsV2 }, } - cont := true - for p.Next() && cont { - cont = fn(p.Page().(*ListObjectsV2Output), !p.HasNextPage()) + for p.Next() { + if !fn(p.Page().(*ListObjectsV2Output), !p.HasNextPage()) { + break + } } + return p.Err() } @@ -4705,6 +6151,33 @@ func (c *S3) ListPartsRequest(input *ListPartsInput) (req *request.Request, outp // ListParts API operation for Amazon Simple Storage Service. // // Lists the parts that have been uploaded for a specific multipart upload. +// This operation must include the upload ID, which you obtain by sending the +// initiate multipart upload request (see CreateMultipartUpload). This request +// returns a maximum of 1,000 uploaded parts. The default number of parts returned +// is 1,000 parts. You can restrict the number of parts returned by specifying +// the max-parts request parameter. If your multipart upload consists of more +// than 1,000 parts, the response returns an IsTruncated field with the value +// of true, and a NextPartNumberMarker element. In subsequent ListParts requests +// you can include the part-number-marker query string parameter and set its +// value to the NextPartNumberMarker field value from the previous response. +// +// For more information on multipart uploads, see Uploading Objects Using Multipart +// Upload (https://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html). +// +// For information on permissions required to use the multipart upload API, +// see Multipart Upload API and Permissions (https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html). +// +// The following operations are related to ListParts: +// +// * CreateMultipartUpload +// +// * UploadPart +// +// * CompleteMultipartUpload +// +// * AbortMultipartUpload +// +// * ListMultipartUploads // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -4745,7 +6218,7 @@ func (c *S3) ListPartsWithContext(ctx aws.Context, input *ListPartsInput, opts . // // Example iterating over at most 3 pages of a ListParts operation. // pageNum := 0 // err := client.ListPartsPages(params, -// func(page *ListPartsOutput, lastPage bool) bool { +// func(page *s3.ListPartsOutput, lastPage bool) bool { // pageNum++ // fmt.Println(page) // return pageNum <= 3 @@ -4777,10 +6250,12 @@ func (c *S3) ListPartsPagesWithContext(ctx aws.Context, input *ListPartsInput, f }, } - cont := true - for p.Next() && cont { - cont = fn(p.Page().(*ListPartsOutput), !p.HasNextPage()) + for p.Next() { + if !fn(p.Page().(*ListPartsOutput), !p.HasNextPage()) { + break + } } + return p.Err() } @@ -4829,7 +6304,41 @@ func (c *S3) PutBucketAccelerateConfigurationRequest(input *PutBucketAccelerateC // PutBucketAccelerateConfiguration API operation for Amazon Simple Storage Service. // -// Sets the accelerate configuration of an existing bucket. +// Sets the accelerate configuration of an existing bucket. Amazon S3 Transfer +// Acceleration is a bucket-level feature that enables you to perform faster +// data transfers to Amazon S3. +// +// To use this operation, you must have permission to perform the s3:PutAccelerateConfiguration +// action. The bucket owner has this permission by default. The bucket owner +// can grant this permission to others. For more information about permissions, +// see Permissions Related to Bucket Subresource Operations (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html#using-with-s3-actions-related-to-bucket-subresources) +// and Managing Access Permissions to Your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html). +// +// The Transfer Acceleration state of a bucket can be set to one of the following +// two values: +// +// * Enabled – Enables accelerated data transfers to the bucket. +// +// * Suspended – Disables accelerated data transfers to the bucket. +// +// The GetBucketAccelerateConfiguration operation returns the transfer acceleration +// state of a bucket. +// +// After setting the Transfer Acceleration state of a bucket to Enabled, it +// might take up to thirty minutes before the data transfer rates to the bucket +// increase. +// +// The name of the bucket used for Transfer Acceleration must be DNS-compliant +// and must not contain periods ("."). +// +// For more information about transfer acceleration, see Transfer Acceleration +// (https://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html). +// +// The following operations are related to PutBucketAccelerateConfiguration: +// +// * GetBucketAccelerateConfiguration +// +// * CreateBucket // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -4899,12 +6408,101 @@ func (c *S3) PutBucketAclRequest(input *PutBucketAclInput) (req *request.Request output = &PutBucketAclOutput{} req = c.newRequest(op, input, output) req.Handlers.Unmarshal.Swap(restxml.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + req.Handlers.Build.PushBackNamed(request.NamedHandler{ + Name: "contentMd5Handler", + Fn: checksum.AddBodyContentMD5Handler, + }) return } // PutBucketAcl API operation for Amazon Simple Storage Service. // -// Sets the permissions on a bucket using access control lists (ACL). +// Sets the permissions on an existing bucket using access control lists (ACL). +// For more information, see Using ACLs (https://docs.aws.amazon.com/AmazonS3/latest/dev/S3_ACLs_UsingACLs.html). +// To set the ACL of a bucket, you must have WRITE_ACP permission. +// +// You can use one of the following two ways to set a bucket's permissions: +// +// * Specify the ACL in the request body +// +// * Specify permissions using request headers +// +// You cannot specify access permission using both the body and the request +// headers. +// +// Depending on your application needs, you may choose to set the ACL on a bucket +// using either the request body or the headers. For example, if you have an +// existing application that updates a bucket ACL using the request body, then +// you can continue to use that approach. +// +// Access Permissions +// +// You can set access permissions using one of the following methods: +// +// * Specify a canned ACL with the x-amz-acl request header. Amazon S3 supports +// a set of predefined ACLs, known as canned ACLs. Each canned ACL has a +// predefined set of grantees and permissions. Specify the canned ACL name +// as the value of x-amz-acl. If you use this header, you cannot use other +// access control-specific headers in your request. For more information, +// see Canned ACL (https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#CannedACL). +// +// * Specify access permissions explicitly with the x-amz-grant-read, x-amz-grant-read-acp, +// x-amz-grant-write-acp, and x-amz-grant-full-control headers. When using +// these headers, you specify explicit access permissions and grantees (AWS +// accounts or Amazon S3 groups) who will receive the permission. If you +// use these ACL-specific headers, you cannot use the x-amz-acl header to +// set a canned ACL. These parameters map to the set of permissions that +// Amazon S3 supports in an ACL. For more information, see Access Control +// List (ACL) Overview (https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html). +// You specify each grantee as a type=value pair, where the type is one of +// the following: id – if the value specified is the canonical user ID +// of an AWS account uri – if you are granting permissions to a predefined +// group emailAddress – if the value specified is the email address of +// an AWS account Using email addresses to specify a grantee is only supported +// in the following AWS Regions: US East (N. Virginia) US West (N. California) +// US West (Oregon) Asia Pacific (Singapore) Asia Pacific (Sydney) Asia Pacific +// (Tokyo) Europe (Ireland) South America (São Paulo) For a list of all +// the Amazon S3 supported Regions and endpoints, see Regions and Endpoints +// (https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region) in +// the AWS General Reference. For example, the following x-amz-grant-write +// header grants create, overwrite, and delete objects permission to LogDelivery +// group predefined by Amazon S3 and two AWS accounts identified by their +// email addresses. x-amz-grant-write: uri="http://acs.amazonaws.com/groups/s3/LogDelivery", +// id="111122223333", id="555566667777" +// +// You can use either a canned ACL or specify access permissions explicitly. +// You cannot do both. +// +// Grantee Values +// +// You can specify the person (grantee) to whom you're assigning access rights +// (using request elements) in the following ways: +// +// * By the person's ID: <>ID<><>GranteesEmail<> +// DisplayName is optional and ignored in the request +// +// * By URI: <>http://acs.amazonaws.com/groups/global/AuthenticatedUsers<> +// +// * By Email address: <>Grantees@email.com<>lt;/Grantee> +// The grantee is resolved to the CanonicalUser and, in a response to a GET +// Object acl request, appears as the CanonicalUser. Using email addresses +// to specify a grantee is only supported in the following AWS Regions: US +// East (N. Virginia) US West (N. California) US West (Oregon) Asia Pacific +// (Singapore) Asia Pacific (Sydney) Asia Pacific (Tokyo) Europe (Ireland) +// South America (São Paulo) For a list of all the Amazon S3 supported Regions +// and endpoints, see Regions and Endpoints (https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region) +// in the AWS General Reference. +// +// Related Resources +// +// * CreateBucket +// +// * DeleteBucket +// +// * GetObjectAcl // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -4980,7 +6578,50 @@ func (c *S3) PutBucketAnalyticsConfigurationRequest(input *PutBucketAnalyticsCon // PutBucketAnalyticsConfiguration API operation for Amazon Simple Storage Service. // // Sets an analytics configuration for the bucket (specified by the analytics -// configuration ID). +// configuration ID). You can have up to 1,000 analytics configurations per +// bucket. +// +// You can choose to have storage class analysis export analysis reports sent +// to a comma-separated values (CSV) flat file. See the DataExport request element. +// Reports are updated daily and are based on the object filters that you configure. +// When selecting data export, you specify a destination bucket and an optional +// destination prefix where the file is written. You can export the data to +// a destination bucket in a different account. However, the destination bucket +// must be in the same Region as the bucket that you are making the PUT analytics +// configuration to. For more information, see Amazon S3 Analytics – Storage +// Class Analysis (https://docs.aws.amazon.com/AmazonS3/latest/dev/analytics-storage-class.html). +// +// You must create a bucket policy on the destination bucket where the exported +// file is written to grant permissions to Amazon S3 to write objects to the +// bucket. For an example policy, see Granting Permissions for Amazon S3 Inventory +// and Storage Class Analysis (https://docs.aws.amazon.com/AmazonS3/latest/dev/example-bucket-policies.html#example-bucket-policies-use-case-9). +// +// To use this operation, you must have permissions to perform the s3:PutAnalyticsConfiguration +// action. The bucket owner has this permission by default. The bucket owner +// can grant this permission to others. For more information about permissions, +// see Permissions Related to Bucket Subresource Operations (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html#using-with-s3-actions-related-to-bucket-subresources) +// and Managing Access Permissions to Your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html). +// +// Special Errors +// +// * HTTP Error: HTTP 400 Bad Request Code: InvalidArgument Cause: Invalid +// argument. +// +// * HTTP Error: HTTP 400 Bad Request Code: TooManyConfigurations Cause: +// You are attempting to create a new configuration but have already reached +// the 1,000-configuration limit. +// +// * HTTP Error: HTTP 403 Forbidden Code: AccessDenied Cause: You are not +// the owner of the specified bucket, or you do not have the s3:PutAnalyticsConfiguration +// bucket permission to set the configuration on the bucket. +// +// Related Resources +// +// * +// +// * +// +// * // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -5050,12 +6691,58 @@ func (c *S3) PutBucketCorsRequest(input *PutBucketCorsInput) (req *request.Reque output = &PutBucketCorsOutput{} req = c.newRequest(op, input, output) req.Handlers.Unmarshal.Swap(restxml.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + req.Handlers.Build.PushBackNamed(request.NamedHandler{ + Name: "contentMd5Handler", + Fn: checksum.AddBodyContentMD5Handler, + }) return } // PutBucketCors API operation for Amazon Simple Storage Service. // -// Sets the CORS configuration for a bucket. +// Sets the cors configuration for your bucket. If the configuration exists, +// Amazon S3 replaces it. +// +// To use this operation, you must be allowed to perform the s3:PutBucketCORS +// action. By default, the bucket owner has this permission and can grant it +// to others. +// +// You set this configuration on a bucket so that the bucket can service cross-origin +// requests. For example, you might want to enable a request whose origin is +// http://www.example.com to access your Amazon S3 bucket at my.example.bucket.com +// by using the browser's XMLHttpRequest capability. +// +// To enable cross-origin resource sharing (CORS) on a bucket, you add the cors +// subresource to the bucket. The cors subresource is an XML document in which +// you configure rules that identify origins and the HTTP methods that can be +// executed on your bucket. The document is limited to 64 KB in size. +// +// When Amazon S3 receives a cross-origin request (or a pre-flight OPTIONS request) +// against a bucket, it evaluates the cors configuration on the bucket and uses +// the first CORSRule rule that matches the incoming browser request to enable +// a cross-origin request. For a rule to match, the following conditions must +// be met: +// +// * The request's Origin header must match AllowedOrigin elements. +// +// * The request method (for example, GET, PUT, HEAD, and so on) or the Access-Control-Request-Method +// header in case of a pre-flight OPTIONS request must be one of the AllowedMethod +// elements. +// +// * Every header specified in the Access-Control-Request-Headers request +// header of a pre-flight request must match an AllowedHeader element. +// +// For more information about CORS, go to Enabling Cross-Origin Resource Sharing +// (https://docs.aws.amazon.com/AmazonS3/latest/dev/cors.html) in the Amazon +// Simple Storage Service Developer Guide. +// +// Related Resources +// +// * GetBucketCors +// +// * DeleteBucketCors +// +// * RESTOPTIONSobject // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -5125,13 +6812,38 @@ func (c *S3) PutBucketEncryptionRequest(input *PutBucketEncryptionInput) (req *r output = &PutBucketEncryptionOutput{} req = c.newRequest(op, input, output) req.Handlers.Unmarshal.Swap(restxml.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + req.Handlers.Build.PushBackNamed(request.NamedHandler{ + Name: "contentMd5Handler", + Fn: checksum.AddBodyContentMD5Handler, + }) return } // PutBucketEncryption API operation for Amazon Simple Storage Service. // -// Creates a new server-side encryption configuration (or replaces an existing -// one, if present). +// This implementation of the PUT operation uses the encryption subresource +// to set the default encryption state of an existing bucket. +// +// This implementation of the PUT operation sets default encryption for a bucket +// using server-side encryption with Amazon S3-managed keys SSE-S3 or AWS KMS +// customer master keys (CMKs) (SSE-KMS). For information about the Amazon S3 +// default encryption feature, see Amazon S3 Default Bucket Encryption (https://docs.aws.amazon.com/AmazonS3/latest/dev/bucket-encryption.html). +// +// This operation requires AWS Signature Version 4. For more information, see +// Authenticating Requests (AWS Signature Version 4) (sig-v4-authenticating-requests.html). +// +// To use this operation, you must have permissions to perform the s3:PutEncryptionConfiguration +// action. The bucket owner has this permission by default. The bucket owner +// can grant this permission to others. For more information about permissions, +// see Permissions Related to Bucket Subresource Operations (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html#using-with-s3-actions-related-to-bucket-subresources) +// and Managing Access Permissions to Your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// Related Resources +// +// * GetBucketEncryption +// +// * DeleteBucketEncryption // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -5206,8 +6918,54 @@ func (c *S3) PutBucketInventoryConfigurationRequest(input *PutBucketInventoryCon // PutBucketInventoryConfiguration API operation for Amazon Simple Storage Service. // -// Adds an inventory configuration (identified by the inventory ID) from the -// bucket. +// This implementation of the PUT operation adds an inventory configuration +// (identified by the inventory ID) to the bucket. You can have up to 1,000 +// inventory configurations per bucket. +// +// Amazon S3 inventory generates inventories of the objects in the bucket on +// a daily or weekly basis, and the results are published to a flat file. The +// bucket that is inventoried is called the source bucket, and the bucket where +// the inventory flat file is stored is called the destination bucket. The destination +// bucket must be in the same AWS Region as the source bucket. +// +// When you configure an inventory for a source bucket, you specify the destination +// bucket where you want the inventory to be stored, and whether to generate +// the inventory daily or weekly. You can also configure what object metadata +// to include and whether to inventory all object versions or only current versions. +// For more information, see Amazon S3 Inventory (https://docs.aws.amazon.com/AmazonS3/latest/dev/storage-inventory.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// You must create a bucket policy on the destination bucket to grant permissions +// to Amazon S3 to write objects to the bucket in the defined location. For +// an example policy, see Granting Permissions for Amazon S3 Inventory and Storage +// Class Analysis (https://docs.aws.amazon.com/AmazonS3/latest/dev/example-bucket-policies.html#example-bucket-policies-use-case-9). +// +// To use this operation, you must have permissions to perform the s3:PutInventoryConfiguration +// action. The bucket owner has this permission by default and can grant this +// permission to others. For more information about permissions, see Permissions +// Related to Bucket Subresource Operations (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html#using-with-s3-actions-related-to-bucket-subresources) +// and Managing Access Permissions to Your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// Special Errors +// +// * HTTP 400 Bad Request Error Code: InvalidArgument Cause: Invalid Argument +// +// * HTTP 400 Bad Request Error Code: TooManyConfigurations Cause: You are +// attempting to create a new configuration but have already reached the +// 1,000-configuration limit. +// +// * HTTP 403 Forbidden Error Code: AccessDenied Cause: You are not the owner +// of the specified bucket, or you do not have the s3:PutInventoryConfiguration +// bucket permission to set the configuration on the bucket. +// +// Related Resources +// +// * GetBucketInventoryConfiguration +// +// * DeleteBucketInventoryConfiguration +// +// * ListBucketInventoryConfigurations // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -5282,12 +7040,64 @@ func (c *S3) PutBucketLifecycleRequest(input *PutBucketLifecycleInput) (req *req output = &PutBucketLifecycleOutput{} req = c.newRequest(op, input, output) req.Handlers.Unmarshal.Swap(restxml.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + req.Handlers.Build.PushBackNamed(request.NamedHandler{ + Name: "contentMd5Handler", + Fn: checksum.AddBodyContentMD5Handler, + }) return } // PutBucketLifecycle API operation for Amazon Simple Storage Service. // -// Deprecated, see the PutBucketLifecycleConfiguration operation. +// +// For an updated version of this API, see PutBucketLifecycleConfiguration. +// This version has been deprecated. Existing lifecycle configurations will +// work. For new lifecycle configurations, use the updated API. +// +// Creates a new lifecycle configuration for the bucket or replaces an existing +// lifecycle configuration. For information about lifecycle configuration, see +// Object Lifecycle Management (https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// By default, all Amazon S3 resources, including buckets, objects, and related +// subresources (for example, lifecycle configuration and website configuration) +// are private. Only the resource owner, the AWS account that created the resource, +// can access it. The resource owner can optionally grant access permissions +// to others by writing an access policy. For this operation, users must get +// the s3:PutLifecycleConfiguration permission. +// +// You can also explicitly deny permissions. Explicit denial also supersedes +// any other permissions. If you want to prevent users or accounts from removing +// or deleting objects from your bucket, you must deny them permissions for +// the following actions: +// +// * s3:DeleteObject +// +// * s3:DeleteObjectVersion +// +// * s3:PutLifecycleConfiguration +// +// For more information about permissions, see Managing Access Permissions to +// your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// For more examples of transitioning objects to storage classes such as STANDARD_IA +// or ONEZONE_IA, see Examples of Lifecycle Configuration (https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html#lifecycle-configuration-examples). +// +// Related Resources +// +// * GetBucketLifecycle(Deprecated) +// +// * GetBucketLifecycleConfiguration +// +// * +// +// * By default, a resource owner—in this case, a bucket owner, which is +// the AWS account that created the bucket—can perform any of the operations. +// A resource owner can also grant others permission to perform the operation. +// For more information, see the following topics in the Amazon Simple Storage +// Service Developer Guide: Specifying Permissions in a Policy (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html) +// Managing Access Permissions to your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html) // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -5361,13 +7171,78 @@ func (c *S3) PutBucketLifecycleConfigurationRequest(input *PutBucketLifecycleCon output = &PutBucketLifecycleConfigurationOutput{} req = c.newRequest(op, input, output) req.Handlers.Unmarshal.Swap(restxml.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + req.Handlers.Build.PushBackNamed(request.NamedHandler{ + Name: "contentMd5Handler", + Fn: checksum.AddBodyContentMD5Handler, + }) return } // PutBucketLifecycleConfiguration API operation for Amazon Simple Storage Service. // -// Sets lifecycle configuration for your bucket. If a lifecycle configuration -// exists, it replaces it. +// Creates a new lifecycle configuration for the bucket or replaces an existing +// lifecycle configuration. For information about lifecycle configuration, see +// Managing Access Permissions to Your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html). +// +// Bucket lifecycle configuration now supports specifying a lifecycle rule using +// an object key name prefix, one or more object tags, or a combination of both. +// Accordingly, this section describes the latest API. The previous version +// of the API supported filtering based only on an object key name prefix, which +// is supported for backward compatibility. For the related API description, +// see PutBucketLifecycle. +// +// Rules +// +// You specify the lifecycle configuration in your request body. The lifecycle +// configuration is specified as XML consisting of one or more rules. Each rule +// consists of the following: +// +// * Filter identifying a subset of objects to which the rule applies. The +// filter can be based on a key name prefix, object tags, or a combination +// of both. +// +// * Status whether the rule is in effect. +// +// * One or more lifecycle transition and expiration actions that you want +// Amazon S3 to perform on the objects identified by the filter. If the state +// of your bucket is versioning-enabled or versioning-suspended, you can +// have many versions of the same object (one current version and zero or +// more noncurrent versions). Amazon S3 provides predefined actions that +// you can specify for current and noncurrent object versions. +// +// For more information, see Object Lifecycle Management (https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html) +// and Lifecycle Configuration Elements (https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html). +// +// Permissions +// +// By default, all Amazon S3 resources are private, including buckets, objects, +// and related subresources (for example, lifecycle configuration and website +// configuration). Only the resource owner (that is, the AWS account that created +// it) can access the resource. The resource owner can optionally grant access +// permissions to others by writing an access policy. For this operation, a +// user must get the s3:PutLifecycleConfiguration permission. +// +// You can also explicitly deny permissions. Explicit deny also supersedes any +// other permissions. If you want to block users or accounts from removing or +// deleting objects from your bucket, you must deny them permissions for the +// following actions: +// +// * s3:DeleteObject +// +// * s3:DeleteObjectVersion +// +// * s3:PutLifecycleConfiguration +// +// For more information about permissions, see Managing Access Permissions to +// Your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html). +// +// The following are related to PutBucketLifecycleConfiguration: +// +// * Examples of Lifecycle Configuration (https://docs.aws.amazon.com/AmazonS3/latest/dev/lifecycle-configuration-examples.html) +// +// * GetBucketLifecycleConfiguration +// +// * DeleteBucketLifecycle // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -5437,21 +7312,68 @@ func (c *S3) PutBucketLoggingRequest(input *PutBucketLoggingInput) (req *request output = &PutBucketLoggingOutput{} req = c.newRequest(op, input, output) req.Handlers.Unmarshal.Swap(restxml.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + req.Handlers.Build.PushBackNamed(request.NamedHandler{ + Name: "contentMd5Handler", + Fn: checksum.AddBodyContentMD5Handler, + }) return } // PutBucketLogging API operation for Amazon Simple Storage Service. // // Set the logging parameters for a bucket and to specify permissions for who -// can view and modify the logging parameters. To set the logging status of +// can view and modify the logging parameters. All logs are saved to buckets +// in the same AWS Region as the source bucket. To set the logging status of // a bucket, you must be the bucket owner. // -// Returns awserr.Error for service API and SDK errors. Use runtime type assertions -// with awserr.Error's Code and Message methods to get detailed information about -// the error. +// The bucket owner is automatically granted FULL_CONTROL to all logs. You use +// the Grantee request element to grant access to other people. The Permissions +// request element specifies the kind of access the grantee has to the logs. // -// See the AWS API reference guide for Amazon Simple Storage Service's -// API operation PutBucketLogging for usage and error information. +// Grantee Values +// +// You can specify the person (grantee) to whom you're assigning access rights +// (using request elements) in the following ways: +// +// * By the person's ID: <>ID<><>GranteesEmail<> +// DisplayName is optional and ignored in the request. +// +// * By Email address: <>Grantees@email.com<> +// The grantee is resolved to the CanonicalUser and, in a response to a GET +// Object acl request, appears as the CanonicalUser. +// +// * By URI: <>http://acs.amazonaws.com/groups/global/AuthenticatedUsers<> +// +// To enable logging, you use LoggingEnabled and its children request elements. +// To disable logging, you use an empty BucketLoggingStatus request element: +// +// +// +// For more information about server access logging, see Server Access Logging +// (https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerLogs.html). +// +// For more information about creating a bucket, see CreateBucket. For more +// information about returning the logging status of a bucket, see GetBucketLogging. +// +// The following operations are related to PutBucketLogging: +// +// * PutObject +// +// * DeleteBucket +// +// * CreateBucket +// +// * GetBucketLogging +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for Amazon Simple Storage Service's +// API operation PutBucketLogging for usage and error information. // See also, https://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/PutBucketLogging func (c *S3) PutBucketLogging(input *PutBucketLoggingInput) (*PutBucketLoggingOutput, error) { req, out := c.PutBucketLoggingRequest(input) @@ -5520,7 +7442,33 @@ func (c *S3) PutBucketMetricsConfigurationRequest(input *PutBucketMetricsConfigu // PutBucketMetricsConfiguration API operation for Amazon Simple Storage Service. // // Sets a metrics configuration (specified by the metrics configuration ID) -// for the bucket. +// for the bucket. You can have up to 1,000 metrics configurations per bucket. +// If you're updating an existing metrics configuration, note that this is a +// full replacement of the existing metrics configuration. If you don't include +// the elements you want to keep, they are erased. +// +// To use this operation, you must have permissions to perform the s3:PutMetricsConfiguration +// action. The bucket owner has this permission by default. The bucket owner +// can grant this permission to others. For more information about permissions, +// see Permissions Related to Bucket Subresource Operations (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html#using-with-s3-actions-related-to-bucket-subresources) +// and Managing Access Permissions to Your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html). +// +// For information about CloudWatch request metrics for Amazon S3, see Monitoring +// Metrics with Amazon CloudWatch (https://docs.aws.amazon.com/AmazonS3/latest/dev/cloudwatch-monitoring.html). +// +// The following operations are related to PutBucketMetricsConfiguration: +// +// * DeleteBucketMetricsConfiguration +// +// * PutBucketMetricsConfiguration +// +// * ListBucketMetricsConfigurations +// +// GetBucketLifecycle has the following special error: +// +// * Error code: TooManyConfigurations Description: You are attempting to +// create a new configuration but have already reached the 1,000-configuration +// limit. HTTP Status Code: HTTP 400 Bad Request // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -5595,12 +7543,16 @@ func (c *S3) PutBucketNotificationRequest(input *PutBucketNotificationInput) (re output = &PutBucketNotificationOutput{} req = c.newRequest(op, input, output) req.Handlers.Unmarshal.Swap(restxml.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + req.Handlers.Build.PushBackNamed(request.NamedHandler{ + Name: "contentMd5Handler", + Fn: checksum.AddBodyContentMD5Handler, + }) return } // PutBucketNotification API operation for Amazon Simple Storage Service. // -// Deprecated, see the PutBucketNotificationConfiguraiton operation. +// No longer used, see the PutBucketNotificationConfiguration operation. // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -5679,7 +7631,55 @@ func (c *S3) PutBucketNotificationConfigurationRequest(input *PutBucketNotificat // PutBucketNotificationConfiguration API operation for Amazon Simple Storage Service. // -// Enables notifications of specified events for a bucket. +// Enables notifications of specified events for a bucket. For more information +// about event notifications, see Configuring Event Notifications (https://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html). +// +// Using this API, you can replace an existing notification configuration. The +// configuration is an XML file that defines the event types that you want Amazon +// S3 to publish and the destination where you want Amazon S3 to publish an +// event notification when it detects an event of the specified type. +// +// By default, your bucket has no event notifications configured. That is, the +// notification configuration will be an empty NotificationConfiguration. +// +// +// +// +// +// This operation replaces the existing notification configuration with the +// configuration you include in the request body. +// +// After Amazon S3 receives this request, it first verifies that any Amazon +// Simple Notification Service (Amazon SNS) or Amazon Simple Queue Service (Amazon +// SQS) destination exists, and that the bucket owner has permission to publish +// to it by sending a test notification. In the case of AWS Lambda destinations, +// Amazon S3 verifies that the Lambda function permissions grant Amazon S3 permission +// to invoke the function from the Amazon S3 bucket. For more information, see +// Configuring Notifications for Amazon S3 Events (https://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html). +// +// You can disable notifications by adding the empty NotificationConfiguration +// element. +// +// By default, only the bucket owner can configure notifications on a bucket. +// However, bucket owners can use a bucket policy to grant permission to other +// users to set this configuration with s3:PutBucketNotification permission. +// +// The PUT notification is an atomic operation. For example, suppose your notification +// configuration includes SNS topic, SQS queue, and Lambda function configurations. +// When you send a PUT request with this configuration, Amazon S3 sends test +// messages to your SNS topic. If the message fails, the entire PUT operation +// will fail, and Amazon S3 will not add the configuration to your bucket. +// +// Responses +// +// If the configuration in the request body includes only one TopicConfiguration +// specifying only the s3:ReducedRedundancyLostObject event type, the response +// will also include the x-amz-sns-test-message-id header containing the message +// ID of the test notification sent to the topic. +// +// The following operation is related to PutBucketNotificationConfiguration: +// +// * GetBucketNotificationConfiguration // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -5749,13 +7749,37 @@ func (c *S3) PutBucketPolicyRequest(input *PutBucketPolicyInput) (req *request.R output = &PutBucketPolicyOutput{} req = c.newRequest(op, input, output) req.Handlers.Unmarshal.Swap(restxml.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + req.Handlers.Build.PushBackNamed(request.NamedHandler{ + Name: "contentMd5Handler", + Fn: checksum.AddBodyContentMD5Handler, + }) return } // PutBucketPolicy API operation for Amazon Simple Storage Service. // -// Replaces a policy on a bucket. If the bucket already has a policy, the one -// in this request completely replaces it. +// Applies an Amazon S3 bucket policy to an Amazon S3 bucket. If you are using +// an identity other than the root user of the AWS account that owns the bucket, +// the calling identity must have the PutBucketPolicy permissions on the specified +// bucket and belong to the bucket owner's account in order to use this operation. +// +// If you don't have PutBucketPolicy permissions, Amazon S3 returns a 403 Access +// Denied error. If you have the correct permissions, but you're not using an +// identity that belongs to the bucket owner's account, Amazon S3 returns a +// 405 Method Not Allowed error. +// +// As a security precaution, the root user of the AWS account that owns a bucket +// can always use this operation, even if the policy explicitly denies the root +// user the ability to perform this action. +// +// For more information about bucket policies, see Using Bucket Policies and +// User Policies (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-iam-policies.html). +// +// The following operations are related to PutBucketPolicy: +// +// * CreateBucket +// +// * DeleteBucket // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -5825,15 +7849,66 @@ func (c *S3) PutBucketReplicationRequest(input *PutBucketReplicationInput) (req output = &PutBucketReplicationOutput{} req = c.newRequest(op, input, output) req.Handlers.Unmarshal.Swap(restxml.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + req.Handlers.Build.PushBackNamed(request.NamedHandler{ + Name: "contentMd5Handler", + Fn: checksum.AddBodyContentMD5Handler, + }) return } // PutBucketReplication API operation for Amazon Simple Storage Service. // // Creates a replication configuration or replaces an existing one. For more -// information, see Cross-Region Replication (CRR) ( https://docs.aws.amazon.com/AmazonS3/latest/dev/crr.html) +// information, see Replication (https://docs.aws.amazon.com/AmazonS3/latest/dev/replication.html) // in the Amazon S3 Developer Guide. // +// To perform this operation, the user or role performing the operation must +// have the iam:PassRole (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_passrole.html) +// permission. +// +// Specify the replication configuration in the request body. In the replication +// configuration, you provide the name of the destination bucket where you want +// Amazon S3 to replicate objects, the IAM role that Amazon S3 can assume to +// replicate objects on your behalf, and other relevant information. +// +// A replication configuration must include at least one rule, and can contain +// a maximum of 1,000. Each rule identifies a subset of objects to replicate +// by filtering the objects in the source bucket. To choose additional subsets +// of objects to replicate, add a rule for each subset. All rules must specify +// the same destination bucket. +// +// To specify a subset of the objects in the source bucket to apply a replication +// rule to, add the Filter element as a child of the Rule element. You can filter +// objects based on an object key prefix, one or more object tags, or both. +// When you add the Filter element in the configuration, you must also add the +// following elements: DeleteMarkerReplication, Status, and Priority. +// +// For information about enabling versioning on a bucket, see Using Versioning +// (https://docs.aws.amazon.com/AmazonS3/latest/dev/Versioning.html). +// +// By default, a resource owner, in this case the AWS account that created the +// bucket, can perform this operation. The resource owner can also grant others +// permissions to perform the operation. For more information about permissions, +// see Specifying Permissions in a Policy (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html) +// and Managing Access Permissions to Your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html). +// +// Handling Replication of Encrypted Objects +// +// By default, Amazon S3 doesn't replicate objects that are stored at rest using +// server-side encryption with CMKs stored in AWS KMS. To replicate AWS KMS-encrypted +// objects, add the following: SourceSelectionCriteria, SseKmsEncryptedObjects, +// Status, EncryptionConfiguration, and ReplicaKmsKeyID. For information about +// replication configuration, see Replicating Objects Created with SSE Using +// CMKs stored in AWS KMS (https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-config-for-kms-objects.html). +// +// For information on PutBucketReplication errors, see ReplicationErrorCodeList +// +// The following operations are related to PutBucketReplication: +// +// * GetBucketReplication +// +// * DeleteBucketReplication +// // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about // the error. @@ -5902,6 +7977,10 @@ func (c *S3) PutBucketRequestPaymentRequest(input *PutBucketRequestPaymentInput) output = &PutBucketRequestPaymentOutput{} req = c.newRequest(op, input, output) req.Handlers.Unmarshal.Swap(restxml.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + req.Handlers.Build.PushBackNamed(request.NamedHandler{ + Name: "contentMd5Handler", + Fn: checksum.AddBodyContentMD5Handler, + }) return } @@ -5910,8 +7989,14 @@ func (c *S3) PutBucketRequestPaymentRequest(input *PutBucketRequestPaymentInput) // Sets the request payment configuration for a bucket. By default, the bucket // owner pays for downloads from the bucket. This configuration parameter enables // the bucket owner (only) to specify that the person requesting the download -// will be charged for the download. Documentation on requester pays buckets -// can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/RequesterPaysBuckets.html +// will be charged for the download. For more information, see Requester Pays +// Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/RequesterPaysBuckets.html). +// +// The following operations are related to PutBucketRequestPayment: +// +// * CreateBucket +// +// * GetBucketRequestPayment // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -5981,6 +8066,10 @@ func (c *S3) PutBucketTaggingRequest(input *PutBucketTaggingInput) (req *request output = &PutBucketTaggingOutput{} req = c.newRequest(op, input, output) req.Handlers.Unmarshal.Swap(restxml.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + req.Handlers.Build.PushBackNamed(request.NamedHandler{ + Name: "contentMd5Handler", + Fn: checksum.AddBodyContentMD5Handler, + }) return } @@ -5988,6 +8077,47 @@ func (c *S3) PutBucketTaggingRequest(input *PutBucketTaggingInput) (req *request // // Sets the tags for a bucket. // +// Use tags to organize your AWS bill to reflect your own cost structure. To +// do this, sign up to get your AWS account bill with tag key values included. +// Then, to see the cost of combined resources, organize your billing information +// according to resources with the same tag key values. For example, you can +// tag several resources with a specific application name, and then organize +// your billing information to see the total cost of that application across +// several services. For more information, see Cost Allocation and Tagging (https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/cost-alloc-tags.html). +// +// Within a bucket, if you add a tag that has the same key as an existing tag, +// the new value overwrites the old value. For more information, see Using Cost +// Allocation in Amazon S3 Bucket Tags (https://docs.aws.amazon.com/AmazonS3/latest/dev/CostAllocTagging.html). +// +// To use this operation, you must have permissions to perform the s3:PutBucketTagging +// action. The bucket owner has this permission by default and can grant this +// permission to others. For more information about permissions, see Permissions +// Related to Bucket Subresource Operations (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html#using-with-s3-actions-related-to-bucket-subresources) +// and Managing Access Permissions to Your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html). +// +// PutBucketTagging has the following special errors: +// +// * Error code: InvalidTagError Description: The tag provided was not a +// valid tag. This error can occur if the tag did not pass input validation. +// For information about tag restrictions, see User-Defined Tag Restrictions +// (https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/allocation-tag-restrictions.html) +// and AWS-Generated Cost Allocation Tag Restrictions (https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/aws-tag-restrictions.html). +// +// * Error code: MalformedXMLError Description: The XML provided does not +// match the schema. +// +// * Error code: OperationAbortedError Description: A conflicting conditional +// operation is currently in progress against this resource. Please try again. +// +// * Error code: InternalError Description: The service was unable to apply +// the provided tag to the bucket. +// +// The following operations are related to PutBucketTagging: +// +// * GetBucketTagging +// +// * DeleteBucketTagging +// // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about // the error. @@ -6056,6 +8186,10 @@ func (c *S3) PutBucketVersioningRequest(input *PutBucketVersioningInput) (req *r output = &PutBucketVersioningOutput{} req = c.newRequest(op, input, output) req.Handlers.Unmarshal.Swap(restxml.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + req.Handlers.Build.PushBackNamed(request.NamedHandler{ + Name: "contentMd5Handler", + Fn: checksum.AddBodyContentMD5Handler, + }) return } @@ -6064,6 +8198,38 @@ func (c *S3) PutBucketVersioningRequest(input *PutBucketVersioningInput) (req *r // Sets the versioning state of an existing bucket. To set the versioning state, // you must be the bucket owner. // +// You can set the versioning state with one of the following values: +// +// Enabled—Enables versioning for the objects in the bucket. All objects added +// to the bucket receive a unique version ID. +// +// Suspended—Disables versioning for the objects in the bucket. All objects +// added to the bucket receive the version ID null. +// +// If the versioning state has never been set on a bucket, it has no versioning +// state; a GetBucketVersioning request does not return a versioning state value. +// +// If the bucket owner enables MFA Delete in the bucket versioning configuration, +// the bucket owner must include the x-amz-mfa request header and the Status +// and the MfaDelete request elements in a request to set the versioning state +// of the bucket. +// +// If you have an object expiration lifecycle policy in your non-versioned bucket +// and you want to maintain the same permanent delete behavior when you enable +// versioning, you must add a noncurrent expiration policy. The noncurrent expiration +// lifecycle policy will manage the deletes of the noncurrent object versions +// in the version-enabled bucket. (A version-enabled bucket maintains one current +// and zero or more noncurrent object versions.) For more information, see Lifecycle +// and Versioning (https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html#lifecycle-and-other-bucket-config). +// +// Related Resources +// +// * CreateBucket +// +// * DeleteBucket +// +// * GetBucketVersioning +// // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about // the error. @@ -6132,12 +8298,81 @@ func (c *S3) PutBucketWebsiteRequest(input *PutBucketWebsiteInput) (req *request output = &PutBucketWebsiteOutput{} req = c.newRequest(op, input, output) req.Handlers.Unmarshal.Swap(restxml.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + req.Handlers.Build.PushBackNamed(request.NamedHandler{ + Name: "contentMd5Handler", + Fn: checksum.AddBodyContentMD5Handler, + }) return } // PutBucketWebsite API operation for Amazon Simple Storage Service. // -// Set the website configuration for a bucket. +// Sets the configuration of the website that is specified in the website subresource. +// To configure a bucket as a website, you can add this subresource on the bucket +// with website configuration information such as the file name of the index +// document and any redirect rules. For more information, see Hosting Websites +// on Amazon S3 (https://docs.aws.amazon.com/AmazonS3/latest/dev/WebsiteHosting.html). +// +// This PUT operation requires the S3:PutBucketWebsite permission. By default, +// only the bucket owner can configure the website attached to a bucket; however, +// bucket owners can allow other users to set the website configuration by writing +// a bucket policy that grants them the S3:PutBucketWebsite permission. +// +// To redirect all website requests sent to the bucket's website endpoint, you +// add a website configuration with the following elements. Because all requests +// are sent to another website, you don't need to provide index document name +// for the bucket. +// +// * WebsiteConfiguration +// +// * RedirectAllRequestsTo +// +// * HostName +// +// * Protocol +// +// If you want granular control over redirects, you can use the following elements +// to add routing rules that describe conditions for redirecting requests and +// information about the redirect destination. In this case, the website configuration +// must provide an index document for the bucket, because some requests might +// not be redirected. +// +// * WebsiteConfiguration +// +// * IndexDocument +// +// * Suffix +// +// * ErrorDocument +// +// * Key +// +// * RoutingRules +// +// * RoutingRule +// +// * Condition +// +// * HttpErrorCodeReturnedEquals +// +// * KeyPrefixEquals +// +// * Redirect +// +// * Protocol +// +// * HostName +// +// * ReplaceKeyPrefixWith +// +// * ReplaceKeyWith +// +// * HttpRedirectCode +// +// Amazon S3 has a limitation of 50 routing rules per website configuration. +// If you require more than 50 routing rules, you can use object redirect. For +// more information, see Configuring an Object Redirect (https://docs.aws.amazon.com/AmazonS3/latest/dev/how-to-page-redirect.html) +// in the Amazon Simple Storage Service Developer Guide. // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -6211,7 +8446,70 @@ func (c *S3) PutObjectRequest(input *PutObjectInput) (req *request.Request, outp // PutObject API operation for Amazon Simple Storage Service. // -// Adds an object to a bucket. +// Adds an object to a bucket. You must have WRITE permissions on a bucket to +// add an object to it. +// +// Amazon S3 never adds partial objects; if you receive a success response, +// Amazon S3 added the entire object to the bucket. +// +// Amazon S3 is a distributed system. If it receives multiple write requests +// for the same object simultaneously, it overwrites all but the last object +// written. Amazon S3 does not provide object locking; if you need this, make +// sure to build it into your application layer or use versioning instead. +// +// To ensure that data is not corrupted traversing the network, use the Content-MD5 +// header. When you use this header, Amazon S3 checks the object against the +// provided MD5 value and, if they do not match, returns an error. Additionally, +// you can calculate the MD5 while putting an object to Amazon S3 and compare +// the returned ETag to the calculated MD5 value. +// +// The Content-MD5 header is required for any request to upload an object with +// a retention period configured using Amazon S3 Object Lock. For more information +// about Amazon S3 Object Lock, see Amazon S3 Object Lock Overview (https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// Server-side Encryption +// +// You can optionally request server-side encryption. With server-side encryption, +// Amazon S3 encrypts your data as it writes it to disks in its data centers +// and decrypts the data when you access it. You have the option to provide +// your own encryption key or use AWS managed encryption keys. For more information, +// see Using Server-Side Encryption (https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html). +// +// Access Control List (ACL)-Specific Request Headers +// +// You can use headers to grant ACL- based permissions. By default, all objects +// are private. Only the owner has full access control. When adding a new object, +// you can grant permissions to individual AWS accounts or to predefined groups +// defined by Amazon S3. These permissions are then added to the ACL on the +// object. For more information, see Access Control List (ACL) Overview (https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html) +// and Managing ACLs Using the REST API (https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-using-rest-api.html). +// +// Storage Class Options +// +// By default, Amazon S3 uses the STANDARD storage class to store newly created +// objects. The STANDARD storage class provides high durability and high availability. +// Depending on performance needs, you can specify a different storage class. +// For more information, see Storage Classes (https://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html) +// in the Amazon S3 Service Developer Guide. +// +// Versioning +// +// If you enable versioning for a bucket, Amazon S3 automatically generates +// a unique version ID for the object being stored. Amazon S3 returns this ID +// in the response. When you enable versioning for a bucket, if Amazon S3 receives +// multiple write requests for the same object simultaneously, it stores all +// of the objects. +// +// For more information about versioning, see Adding Objects to Versioning Enabled +// Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/AddingObjectstoVersioningEnabledBuckets.html). +// For information about returning the versioning state of a bucket, see GetBucketVersioning. +// +// Related Resources +// +// * CopyObject +// +// * DeleteObject // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -6280,13 +8578,97 @@ func (c *S3) PutObjectAclRequest(input *PutObjectAclInput) (req *request.Request output = &PutObjectAclOutput{} req = c.newRequest(op, input, output) + req.Handlers.Build.PushBackNamed(request.NamedHandler{ + Name: "contentMd5Handler", + Fn: checksum.AddBodyContentMD5Handler, + }) return } // PutObjectAcl API operation for Amazon Simple Storage Service. // -// uses the acl subresource to set the access control list (ACL) permissions -// for an object that already exists in a bucket +// Uses the acl subresource to set the access control list (ACL) permissions +// for an object that already exists in a bucket. You must have WRITE_ACP permission +// to set the ACL of an object. +// +// Depending on your application needs, you can choose to set the ACL on an +// object using either the request body or the headers. For example, if you +// have an existing application that updates a bucket ACL using the request +// body, you can continue to use that approach. For more information, see Access +// Control List (ACL) Overview (https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html) +// in the Amazon S3 Developer Guide. +// +// Access Permissions +// +// You can set access permissions using one of the following methods: +// +// * Specify a canned ACL with the x-amz-acl request header. Amazon S3 supports +// a set of predefined ACLs, known as canned ACLs. Each canned ACL has a +// predefined set of grantees and permissions. Specify the canned ACL name +// as the value of x-amz-acl. If you use this header, you cannot use other +// access control-specific headers in your request. For more information, +// see Canned ACL (https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#CannedACL). +// +// * Specify access permissions explicitly with the x-amz-grant-read, x-amz-grant-read-acp, +// x-amz-grant-write-acp, and x-amz-grant-full-control headers. When using +// these headers, you specify explicit access permissions and grantees (AWS +// accounts or Amazon S3 groups) who will receive the permission. If you +// use these ACL-specific headers, you cannot use x-amz-acl header to set +// a canned ACL. These parameters map to the set of permissions that Amazon +// S3 supports in an ACL. For more information, see Access Control List (ACL) +// Overview (https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html). +// You specify each grantee as a type=value pair, where the type is one of +// the following: id – if the value specified is the canonical user ID +// of an AWS account uri – if you are granting permissions to a predefined +// group emailAddress – if the value specified is the email address of +// an AWS account Using email addresses to specify a grantee is only supported +// in the following AWS Regions: US East (N. Virginia) US West (N. California) +// US West (Oregon) Asia Pacific (Singapore) Asia Pacific (Sydney) Asia Pacific +// (Tokyo) Europe (Ireland) South America (São Paulo) For a list of all +// the Amazon S3 supported Regions and endpoints, see Regions and Endpoints +// (https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region) in +// the AWS General Reference. For example, the following x-amz-grant-read +// header grants list objects permission to the two AWS accounts identified +// by their email addresses. x-amz-grant-read: emailAddress="xyz@amazon.com", +// emailAddress="abc@amazon.com" +// +// You can use either a canned ACL or specify access permissions explicitly. +// You cannot do both. +// +// Grantee Values +// +// You can specify the person (grantee) to whom you're assigning access rights +// (using request elements) in the following ways: +// +// * By the person's ID: <>ID<><>GranteesEmail<> +// DisplayName is optional and ignored in the request. +// +// * By URI: <>http://acs.amazonaws.com/groups/global/AuthenticatedUsers<> +// +// * By Email address: <>Grantees@email.com<>lt;/Grantee> +// The grantee is resolved to the CanonicalUser and, in a response to a GET +// Object acl request, appears as the CanonicalUser. Using email addresses +// to specify a grantee is only supported in the following AWS Regions: US +// East (N. Virginia) US West (N. California) US West (Oregon) Asia Pacific +// (Singapore) Asia Pacific (Sydney) Asia Pacific (Tokyo) Europe (Ireland) +// South America (São Paulo) For a list of all the Amazon S3 supported Regions +// and endpoints, see Regions and Endpoints (https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region) +// in the AWS General Reference. +// +// Versioning +// +// The ACL of an object is set at the object version level. By default, PUT +// sets the ACL of the current version of an object. To set the ACL of a different +// version, use the versionId subresource. +// +// Related Resources +// +// * CopyObject +// +// * GetObject // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -6360,6 +8742,10 @@ func (c *S3) PutObjectLegalHoldRequest(input *PutObjectLegalHoldInput) (req *req output = &PutObjectLegalHoldOutput{} req = c.newRequest(op, input, output) + req.Handlers.Build.PushBackNamed(request.NamedHandler{ + Name: "contentMd5Handler", + Fn: checksum.AddBodyContentMD5Handler, + }) return } @@ -6367,6 +8753,10 @@ func (c *S3) PutObjectLegalHoldRequest(input *PutObjectLegalHoldInput) (req *req // // Applies a Legal Hold configuration to the specified object. // +// Related Resources +// +// * Locking Objects (https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock.html) +// // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about // the error. @@ -6434,6 +8824,10 @@ func (c *S3) PutObjectLockConfigurationRequest(input *PutObjectLockConfiguration output = &PutObjectLockConfigurationOutput{} req = c.newRequest(op, input, output) + req.Handlers.Build.PushBackNamed(request.NamedHandler{ + Name: "contentMd5Handler", + Fn: checksum.AddBodyContentMD5Handler, + }) return } @@ -6443,6 +8837,13 @@ func (c *S3) PutObjectLockConfigurationRequest(input *PutObjectLockConfiguration // in the Object Lock configuration will be applied by default to every new // object placed in the specified bucket. // +// DefaultRetention requires either Days or Years. You can't specify both at +// the same time. +// +// Related Resources +// +// * Locking Objects (https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock.html) +// // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about // the error. @@ -6510,6 +8911,10 @@ func (c *S3) PutObjectRetentionRequest(input *PutObjectRetentionInput) (req *req output = &PutObjectRetentionOutput{} req = c.newRequest(op, input, output) + req.Handlers.Build.PushBackNamed(request.NamedHandler{ + Name: "contentMd5Handler", + Fn: checksum.AddBodyContentMD5Handler, + }) return } @@ -6517,6 +8922,10 @@ func (c *S3) PutObjectRetentionRequest(input *PutObjectRetentionInput) (req *req // // Places an Object Retention configuration on an object. // +// Related Resources +// +// * Locking Objects (https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock.html) +// // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about // the error. @@ -6584,12 +8993,53 @@ func (c *S3) PutObjectTaggingRequest(input *PutObjectTaggingInput) (req *request output = &PutObjectTaggingOutput{} req = c.newRequest(op, input, output) + req.Handlers.Build.PushBackNamed(request.NamedHandler{ + Name: "contentMd5Handler", + Fn: checksum.AddBodyContentMD5Handler, + }) return } // PutObjectTagging API operation for Amazon Simple Storage Service. // -// Sets the supplied tag-set to an object that already exists in a bucket +// Sets the supplied tag-set to an object that already exists in a bucket. +// +// A tag is a key-value pair. You can associate tags with an object by sending +// a PUT request against the tagging subresource that is associated with the +// object. You can retrieve tags by sending a GET request. For more information, +// see GetObjectTagging. +// +// For tagging-related restrictions related to characters and encodings, see +// Tag Restrictions (https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/allocation-tag-restrictions.html). +// Note that Amazon S3 limits the maximum number of tags to 10 tags per object. +// +// To use this operation, you must have permission to perform the s3:PutObjectTagging +// action. By default, the bucket owner has this permission and can grant this +// permission to others. +// +// To put tags of any other version, use the versionId query parameter. You +// also need permission for the s3:PutObjectVersionTagging action. +// +// For information about the Amazon S3 object tagging feature, see Object Tagging +// (https://docs.aws.amazon.com/AmazonS3/latest/dev/object-tagging.html). +// +// Special Errors +// +// * Code: InvalidTagError Cause: The tag provided was not a valid tag. This +// error can occur if the tag did not pass input validation. For more information, +// see Object Tagging (https://docs.aws.amazon.com/AmazonS3/latest/dev/object-tagging.html). +// +// * Code: MalformedXMLError Cause: The XML provided does not match the schema. +// +// * Code: OperationAbortedError Cause: A conflicting conditional operation +// is currently in progress against this resource. Please try again. +// +// * Code: InternalError Cause: The service was unable to apply the provided +// tag to the object. +// +// Related Resources +// +// * GetObjectTagging // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -6659,13 +9109,39 @@ func (c *S3) PutPublicAccessBlockRequest(input *PutPublicAccessBlockInput) (req output = &PutPublicAccessBlockOutput{} req = c.newRequest(op, input, output) req.Handlers.Unmarshal.Swap(restxml.UnmarshalHandler.Name, protocol.UnmarshalDiscardBodyHandler) + req.Handlers.Build.PushBackNamed(request.NamedHandler{ + Name: "contentMd5Handler", + Fn: checksum.AddBodyContentMD5Handler, + }) return } // PutPublicAccessBlock API operation for Amazon Simple Storage Service. // // Creates or modifies the PublicAccessBlock configuration for an Amazon S3 -// bucket. +// bucket. To use this operation, you must have the s3:PutBucketPublicAccessBlock +// permission. For more information about Amazon S3 permissions, see Specifying +// Permissions in a Policy (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html). +// +// When Amazon S3 evaluates the PublicAccessBlock configuration for a bucket +// or an object, it checks the PublicAccessBlock configuration for both the +// bucket (or the bucket that contains the object) and the bucket owner's account. +// If the PublicAccessBlock configurations are different between the bucket +// and the account, Amazon S3 uses the most restrictive combination of the bucket-level +// and account-level settings. +// +// For more information about when Amazon S3 considers a bucket or an object +// public, see The Meaning of "Public" (https://docs.aws.amazon.com/AmazonS3/latest/dev/access-control-block-public-access.html#access-control-block-public-access-policy-status). +// +// Related Resources +// +// * GetPublicAccessBlock +// +// * DeletePublicAccessBlock +// +// * GetBucketPolicyStatus +// +// * Using Amazon S3 Block Public Access (https://docs.aws.amazon.com/AmazonS3/latest/dev/access-control-block-public-access.html) // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -6741,6 +9217,190 @@ func (c *S3) RestoreObjectRequest(input *RestoreObjectInput) (req *request.Reque // // Restores an archived copy of an object back into Amazon S3 // +// This operation performs the following types of requests: +// +// * select - Perform a select query on an archived object +// +// * restore an archive - Restore an archived object +// +// To use this operation, you must have permissions to perform the s3:RestoreObject +// action. The bucket owner has this permission by default and can grant this +// permission to others. For more information about permissions, see Permissions +// Related to Bucket Subresource Operations (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html#using-with-s3-actions-related-to-bucket-subresources) +// and Managing Access Permissions to Your Amazon S3 Resources (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// Querying Archives with Select Requests +// +// You use a select type of request to perform SQL queries on archived objects. +// The archived objects that are being queried by the select request must be +// formatted as uncompressed comma-separated values (CSV) files. You can run +// queries and custom analytics on your archived data without having to restore +// your data to a hotter Amazon S3 tier. For an overview about select requests, +// see Querying Archived Objects (https://docs.aws.amazon.com/AmazonS3/latest/dev/querying-glacier-archives.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// When making a select request, do the following: +// +// * Define an output location for the select query's output. This must be +// an Amazon S3 bucket in the same AWS Region as the bucket that contains +// the archive object that is being queried. The AWS account that initiates +// the job must have permissions to write to the S3 bucket. You can specify +// the storage class and encryption for the output objects stored in the +// bucket. For more information about output, see Querying Archived Objects +// (https://docs.aws.amazon.com/AmazonS3/latest/dev/querying-glacier-archives.html) +// in the Amazon Simple Storage Service Developer Guide. For more information +// about the S3 structure in the request body, see the following: PutObject +// Managing Access with ACLs (https://docs.aws.amazon.com/AmazonS3/latest/dev/S3_ACLs_UsingACLs.html) +// in the Amazon Simple Storage Service Developer Guide Protecting Data Using +// Server-Side Encryption (https://docs.aws.amazon.com/AmazonS3/latest/dev/serv-side-encryption.html) +// in the Amazon Simple Storage Service Developer Guide +// +// * Define the SQL expression for the SELECT type of restoration for your +// query in the request body's SelectParameters structure. You can use expressions +// like the following examples. The following expression returns all records +// from the specified object. SELECT * FROM Object Assuming that you are +// not using any headers for data stored in the object, you can specify columns +// with positional headers. SELECT s._1, s._2 FROM Object s WHERE s._3 > +// 100 If you have headers and you set the fileHeaderInfo in the CSV structure +// in the request body to USE, you can specify headers in the query. (If +// you set the fileHeaderInfo field to IGNORE, the first row is skipped for +// the query.) You cannot mix ordinal positions with header column names. +// SELECT s.Id, s.FirstName, s.SSN FROM S3Object s +// +// For more information about using SQL with S3 Glacier Select restore, see +// SQL Reference for Amazon S3 Select and S3 Glacier Select (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-glacier-select-sql-reference.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// When making a select request, you can also do the following: +// +// * To expedite your queries, specify the Expedited tier. For more information +// about tiers, see "Restoring Archives," later in this topic. +// +// * Specify details about the data serialization format of both the input +// object that is being queried and the serialization of the CSV-encoded +// query results. +// +// The following are additional important facts about the select feature: +// +// * The output results are new Amazon S3 objects. Unlike archive retrievals, +// they are stored until explicitly deleted-manually or through a lifecycle +// policy. +// +// * You can issue more than one select request on the same Amazon S3 object. +// Amazon S3 doesn't deduplicate requests, so avoid issuing duplicate requests. +// +// * Amazon S3 accepts a select request even if the object has already been +// restored. A select request doesn’t return error response 409. +// +// Restoring Archives +// +// Objects in the GLACIER and DEEP_ARCHIVE storage classes are archived. To +// access an archived object, you must first initiate a restore request. This +// restores a temporary copy of the archived object. In a restore request, you +// specify the number of days that you want the restored copy to exist. After +// the specified period, Amazon S3 deletes the temporary copy but the object +// remains archived in the GLACIER or DEEP_ARCHIVE storage class that object +// was restored from. +// +// To restore a specific object version, you can provide a version ID. If you +// don't provide a version ID, Amazon S3 restores the current version. +// +// The time it takes restore jobs to finish depends on which storage class the +// object is being restored from and which data access tier you specify. +// +// When restoring an archived object (or using a select request), you can specify +// one of the following data access tier options in the Tier element of the +// request body: +// +// * Expedited - Expedited retrievals allow you to quickly access your data +// stored in the GLACIER storage class when occasional urgent requests for +// a subset of archives are required. For all but the largest archived objects +// (250 MB+), data accessed using Expedited retrievals are typically made +// available within 1–5 minutes. Provisioned capacity ensures that retrieval +// capacity for Expedited retrievals is available when you need it. Expedited +// retrievals and provisioned capacity are not available for the DEEP_ARCHIVE +// storage class. +// +// * Standard - S3 Standard retrievals allow you to access any of your archived +// objects within several hours. This is the default option for the GLACIER +// and DEEP_ARCHIVE retrieval requests that do not specify the retrieval +// option. S3 Standard retrievals typically complete within 3-5 hours from +// the GLACIER storage class and typically complete within 12 hours from +// the DEEP_ARCHIVE storage class. +// +// * Bulk - Bulk retrievals are Amazon S3 Glacier’s lowest-cost retrieval +// option, enabling you to retrieve large amounts, even petabytes, of data +// inexpensively in a day. Bulk retrievals typically complete within 5-12 +// hours from the GLACIER storage class and typically complete within 48 +// hours from the DEEP_ARCHIVE storage class. +// +// For more information about archive retrieval options and provisioned capacity +// for Expedited data access, see Restoring Archived Objects (https://docs.aws.amazon.com/AmazonS3/latest/dev/restoring-objects.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// You can use Amazon S3 restore speed upgrade to change the restore speed to +// a faster speed while it is in progress. You upgrade the speed of an in-progress +// restoration by issuing another restore request to the same object, setting +// a new Tier request element. When issuing a request to upgrade the restore +// tier, you must choose a tier that is faster than the tier that the in-progress +// restore is using. You must not change any other parameters, such as the Days +// request element. For more information, see Upgrading the Speed of an In-Progress +// Restore (https://docs.aws.amazon.com/AmazonS3/latest/dev/restoring-objects.html#restoring-objects-upgrade-tier.title.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// To get the status of object restoration, you can send a HEAD request. Operations +// return the x-amz-restore header, which provides information about the restoration +// status, in the response. You can use Amazon S3 event notifications to notify +// you when a restore is initiated or completed. For more information, see Configuring +// Amazon S3 Event Notifications (https://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// After restoring an archived object, you can update the restoration period +// by reissuing the request with a new period. Amazon S3 updates the restoration +// period relative to the current time and charges only for the request-there +// are no data transfer charges. You cannot update the restoration period when +// Amazon S3 is actively processing your current restore request for the object. +// +// If your bucket has a lifecycle configuration with a rule that includes an +// expiration action, the object expiration overrides the life span that you +// specify in a restore request. For example, if you restore an object copy +// for 10 days, but the object is scheduled to expire in 3 days, Amazon S3 deletes +// the object in 3 days. For more information about lifecycle configuration, +// see PutBucketLifecycleConfiguration and Object Lifecycle Management (https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html) +// in Amazon Simple Storage Service Developer Guide. +// +// Responses +// +// A successful operation returns either the 200 OK or 202 Accepted status code. +// +// * If the object copy is not previously restored, then Amazon S3 returns +// 202 Accepted in the response. +// +// * If the object copy is previously restored, Amazon S3 returns 200 OK +// in the response. +// +// Special Errors +// +// * Code: RestoreAlreadyInProgress Cause: Object restore is already in progress. +// (This error does not apply to SELECT type requests.) HTTP Status Code: +// 409 Conflict SOAP Fault Code Prefix: Client +// +// * Code: GlacierExpeditedRetrievalNotAvailable Cause: S3 Glacier expedited +// retrievals are currently not available. Try again later. (Returned if +// there is insufficient capacity to process the Expedited request. This +// error applies only to Expedited retrievals and not to S3 Standard or Bulk +// retrievals.) HTTP Status Code: 503 SOAP Fault Code Prefix: N/A +// +// Related Resources +// +// * PutBucketLifecycleConfiguration +// +// * GetBucketNotificationConfiguration +// +// * SQL Reference for Amazon S3 Select and S3 Glacier Select (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-glacier-select-sql-reference.html) +// in the Amazon Simple Storage Service Developer Guide +// // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about // the error. @@ -6750,7 +9410,7 @@ func (c *S3) RestoreObjectRequest(input *RestoreObjectInput) (req *request.Reque // // Returned Error Codes: // * ErrCodeObjectAlreadyInActiveTierError "ObjectAlreadyInActiveTierError" -// This operation is not allowed against this storage tier +// This operation is not allowed against this storage tier. // // See also, https://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/RestoreObject func (c *S3) RestoreObject(input *RestoreObjectInput) (*RestoreObjectOutput, error) { @@ -6813,20 +9473,103 @@ func (c *S3) SelectObjectContentRequest(input *SelectObjectContentInput) (req *r output = &SelectObjectContentOutput{} req = c.newRequest(op, input, output) + + es := newSelectObjectContentEventStream() + req.Handlers.Unmarshal.PushBack(es.setStreamCloser) + output.EventStream = es + req.Handlers.Send.Swap(client.LogHTTPResponseHandler.Name, client.LogHTTPResponseHeaderHandler) req.Handlers.Unmarshal.Swap(restxml.UnmarshalHandler.Name, rest.UnmarshalHandler) - req.Handlers.Unmarshal.PushBack(output.runEventStreamLoop) + req.Handlers.Unmarshal.PushBack(es.runOutputStream) + req.Handlers.Unmarshal.PushBack(es.runOnStreamPartClose) return } // SelectObjectContent API operation for Amazon Simple Storage Service. // // This operation filters the contents of an Amazon S3 object based on a simple -// Structured Query Language (SQL) statement. In the request, along with the -// SQL expression, you must also specify a data serialization format (JSON or -// CSV) of the object. Amazon S3 uses this to parse object data into records, -// and returns only records that match the specified SQL expression. You must -// also specify the data serialization format for the response. +// structured query language (SQL) statement. In the request, along with the +// SQL expression, you must also specify a data serialization format (JSON, +// CSV, or Apache Parquet) of the object. Amazon S3 uses this format to parse +// object data into records, and returns only records that match the specified +// SQL expression. You must also specify the data serialization format for the +// response. +// +// For more information about Amazon S3 Select, see Selecting Content from Objects +// (https://docs.aws.amazon.com/AmazonS3/latest/dev/selecting-content-from-objects.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// For more information about using SQL with Amazon S3 Select, see SQL Reference +// for Amazon S3 Select and S3 Glacier Select (https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-glacier-select-sql-reference.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// Permissions +// +// You must have s3:GetObject permission for this operation. Amazon S3 Select +// does not support anonymous access. For more information about permissions, +// see Specifying Permissions in a Policy (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// Object Data Formats +// +// You can use Amazon S3 Select to query objects that have the following format +// properties: +// +// * CSV, JSON, and Parquet - Objects must be in CSV, JSON, or Parquet format. +// +// * UTF-8 - UTF-8 is the only encoding type Amazon S3 Select supports. +// +// * GZIP or BZIP2 - CSV and JSON files can be compressed using GZIP or BZIP2. +// GZIP and BZIP2 are the only compression formats that Amazon S3 Select +// supports for CSV and JSON files. Amazon S3 Select supports columnar compression +// for Parquet using GZIP or Snappy. Amazon S3 Select does not support whole-object +// compression for Parquet objects. +// +// * Server-side encryption - Amazon S3 Select supports querying objects +// that are protected with server-side encryption. For objects that are encrypted +// with customer-provided encryption keys (SSE-C), you must use HTTPS, and +// you must use the headers that are documented in the GetObject. For more +// information about SSE-C, see Server-Side Encryption (Using Customer-Provided +// Encryption Keys) (https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html) +// in the Amazon Simple Storage Service Developer Guide. For objects that +// are encrypted with Amazon S3 managed encryption keys (SSE-S3) and customer +// master keys (CMKs) stored in AWS Key Management Service (SSE-KMS), server-side +// encryption is handled transparently, so you don't need to specify anything. +// For more information about server-side encryption, including SSE-S3 and +// SSE-KMS, see Protecting Data Using Server-Side Encryption (https://docs.aws.amazon.com/AmazonS3/latest/dev/serv-side-encryption.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// Working with the Response Body +// +// Given the response size is unknown, Amazon S3 Select streams the response +// as a series of messages and includes a Transfer-Encoding header with chunked +// as its value in the response. For more information, see RESTSelectObjectAppendix . +// +// GetObject Support +// +// The SelectObjectContent operation does not support the following GetObject +// functionality. For more information, see GetObject. +// +// * Range: Although you can specify a scan range for an Amazon S3 Select +// request (see SelectObjectContentRequest$ScanRange in the request parameters), +// you cannot specify the range of bytes of an object to return. +// +// * GLACIER, DEEP_ARCHIVE and REDUCED_REDUNDANCY storage classes: You cannot +// specify the GLACIER, DEEP_ARCHIVE, or REDUCED_REDUNDANCY storage classes. +// For more information, about storage classes see Storage Classes (https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html#storage-class-intro) +// in the Amazon Simple Storage Service Developer Guide. +// +// Special Errors +// +// For a list of special errors for this operation, see SelectObjectContentErrorCodeList +// +// Related Resources +// +// * GetObject +// +// * GetBucketLifecycleConfiguration +// +// * PutBucketLifecycleConfiguration // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -6856,36 +9599,180 @@ func (c *S3) SelectObjectContentWithContext(ctx aws.Context, input *SelectObject return out, req.Send() } -const opUploadPart = "UploadPart" +var _ awserr.Error -// UploadPartRequest generates a "aws/request.Request" representing the -// client's request for the UploadPart operation. The "output" return -// value will be populated with the request's response once the request completes -// successfully. -// -// Use "Send" method on the returned Request to send the API call to the service. -// the "output" return value is not valid until after Send returns without error. -// -// See UploadPart for more information on using the UploadPart -// API call, and error handling. -// -// This method is useful when you want to inject custom logic or configuration -// into the SDK's request lifecycle. Such as custom headers, or retry logic. -// -// -// // Example sending a request using the UploadPartRequest method. -// req, resp := client.UploadPartRequest(params) -// -// err := req.Send() -// if err == nil { // resp is now filled -// fmt.Println(resp) -// } -// -// See also, https://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/UploadPart -func (c *S3) UploadPartRequest(input *UploadPartInput) (req *request.Request, output *UploadPartOutput) { - op := &request.Operation{ - Name: opUploadPart, - HTTPMethod: "PUT", +// SelectObjectContentEventStream provides the event stream handling for the SelectObjectContent. +type SelectObjectContentEventStream struct { + + // Reader is the EventStream reader for the SelectObjectContentEventStream + // events. This value is automatically set by the SDK when the API call is made + // Use this member when unit testing your code with the SDK to mock out the + // EventStream Reader. + // + // Must not be nil. + Reader SelectObjectContentEventStreamReader + + outputReader io.ReadCloser + + // StreamCloser is the io.Closer for the EventStream connection. For HTTP + // EventStream this is the response Body. The stream will be closed when + // the Close method of the EventStream is called. + StreamCloser io.Closer + + done chan struct{} + closeOnce sync.Once + err *eventstreamapi.OnceError +} + +func newSelectObjectContentEventStream() *SelectObjectContentEventStream { + return &SelectObjectContentEventStream{ + done: make(chan struct{}), + err: eventstreamapi.NewOnceError(), + } +} + +func (es *SelectObjectContentEventStream) setStreamCloser(r *request.Request) { + es.StreamCloser = r.HTTPResponse.Body +} + +func (es *SelectObjectContentEventStream) runOnStreamPartClose(r *request.Request) { + if es.done == nil { + return + } + go es.waitStreamPartClose() + +} + +func (es *SelectObjectContentEventStream) waitStreamPartClose() { + var outputErrCh <-chan struct{} + if v, ok := es.Reader.(interface{ ErrorSet() <-chan struct{} }); ok { + outputErrCh = v.ErrorSet() + } + var outputClosedCh <-chan struct{} + if v, ok := es.Reader.(interface{ Closed() <-chan struct{} }); ok { + outputClosedCh = v.Closed() + } + + select { + case <-es.done: + case <-outputErrCh: + es.err.SetError(es.Reader.Err()) + es.Close() + case <-outputClosedCh: + if err := es.Reader.Err(); err != nil { + es.err.SetError(es.Reader.Err()) + } + es.Close() + } +} + +// Events returns a channel to read events from. +// +// These events are: +// +// * ContinuationEvent +// * EndEvent +// * ProgressEvent +// * RecordsEvent +// * StatsEvent +// * SelectObjectContentEventStreamUnknownEvent +func (es *SelectObjectContentEventStream) Events() <-chan SelectObjectContentEventStreamEvent { + return es.Reader.Events() +} + +func (es *SelectObjectContentEventStream) runOutputStream(r *request.Request) { + var opts []func(*eventstream.Decoder) + if r.Config.Logger != nil && r.Config.LogLevel.Matches(aws.LogDebugWithEventStreamBody) { + opts = append(opts, eventstream.DecodeWithLogger(r.Config.Logger)) + } + + unmarshalerForEvent := unmarshalerForSelectObjectContentEventStreamEvent{ + metadata: protocol.ResponseMetadata{ + StatusCode: r.HTTPResponse.StatusCode, + RequestID: r.RequestID, + }, + }.UnmarshalerForEventName + + decoder := eventstream.NewDecoder(r.HTTPResponse.Body, opts...) + eventReader := eventstreamapi.NewEventReader(decoder, + protocol.HandlerPayloadUnmarshal{ + Unmarshalers: r.Handlers.UnmarshalStream, + }, + unmarshalerForEvent, + ) + + es.outputReader = r.HTTPResponse.Body + es.Reader = newReadSelectObjectContentEventStream(eventReader) +} + +// Close closes the stream. This will also cause the stream to be closed. +// Close must be called when done using the stream API. Not calling Close +// may result in resource leaks. +// +// You can use the closing of the Reader's Events channel to terminate your +// application's read from the API's stream. +// +func (es *SelectObjectContentEventStream) Close() (err error) { + es.closeOnce.Do(es.safeClose) + return es.Err() +} + +func (es *SelectObjectContentEventStream) safeClose() { + if es.done != nil { + close(es.done) + } + + es.Reader.Close() + if es.outputReader != nil { + es.outputReader.Close() + } + + es.StreamCloser.Close() +} + +// Err returns any error that occurred while reading or writing EventStream +// Events from the service API's response. Returns nil if there were no errors. +func (es *SelectObjectContentEventStream) Err() error { + if err := es.err.Err(); err != nil { + return err + } + if err := es.Reader.Err(); err != nil { + return err + } + + return nil +} + +const opUploadPart = "UploadPart" + +// UploadPartRequest generates a "aws/request.Request" representing the +// client's request for the UploadPart operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See UploadPart for more information on using the UploadPart +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the UploadPartRequest method. +// req, resp := client.UploadPartRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/UploadPart +func (c *S3) UploadPartRequest(input *UploadPartInput) (req *request.Request, output *UploadPartOutput) { + op := &request.Operation{ + Name: opUploadPart, + HTTPMethod: "PUT", HTTPPath: "/{Bucket}/{Key+}", } @@ -6902,12 +9789,87 @@ func (c *S3) UploadPartRequest(input *UploadPartInput) (req *request.Request, ou // // Uploads a part in a multipart upload. // +// In this operation, you provide part data in your request. However, you have +// an option to specify your existing Amazon S3 object as a data source for +// the part you are uploading. To upload a part from an existing object, you +// use the UploadPartCopy operation. +// +// You must initiate a multipart upload (see CreateMultipartUpload) before you +// can upload any part. In response to your initiate request, Amazon S3 returns +// an upload ID, a unique identifier, that you must include in your upload part +// request. +// +// Part numbers can be any number from 1 to 10,000, inclusive. A part number +// uniquely identifies a part and also defines its position within the object +// being created. If you upload a new part using the same part number that was +// used with a previous part, the previously uploaded part is overwritten. Each +// part must be at least 5 MB in size, except the last part. There is no size +// limit on the last part of your multipart upload. +// +// To ensure that data is not corrupted when traversing the network, specify +// the Content-MD5 header in the upload part request. Amazon S3 checks the part +// data against the provided MD5 value. If they do not match, Amazon S3 returns +// an error. +// // Note: After you initiate multipart upload and upload one or more parts, you // must either complete or abort multipart upload in order to stop getting charged // for storage of the uploaded parts. Only after you either complete or abort // multipart upload, Amazon S3 frees up the parts storage and stops charging // you for the parts storage. // +// For more information on multipart uploads, go to Multipart Upload Overview +// (https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuoverview.html) in the +// Amazon Simple Storage Service Developer Guide . +// +// For information on the permissions required to use the multipart upload API, +// go to Multipart Upload API and Permissions (https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// You can optionally request server-side encryption where Amazon S3 encrypts +// your data as it writes it to disks in its data centers and decrypts it for +// you when you access it. You have the option of providing your own encryption +// key, or you can use the AWS managed encryption keys. If you choose to provide +// your own encryption key, the request headers you provide in the request must +// match the headers you used in the request to initiate the upload by using +// CreateMultipartUpload. For more information, go to Using Server-Side Encryption +// (https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// Server-side encryption is supported by the S3 Multipart Upload actions. Unless +// you are using a customer-provided encryption key, you don't need to specify +// the encryption parameters in each UploadPart request. Instead, you only need +// to specify the server-side encryption parameters in the initial Initiate +// Multipart request. For more information, see CreateMultipartUpload. +// +// If you requested server-side encryption using a customer-provided encryption +// key in your initiate multipart upload request, you must provide identical +// encryption information in each part upload using the following headers. +// +// * x-amz-server-side​-encryption​-customer-algorithm +// +// * x-amz-server-side​-encryption​-customer-key +// +// * x-amz-server-side​-encryption​-customer-key-MD5 +// +// Special Errors +// +// * Code: NoSuchUpload Cause: The specified multipart upload does not exist. +// The upload ID might be invalid, or the multipart upload might have been +// aborted or completed. HTTP Status Code: 404 Not Found SOAP Fault Code +// Prefix: Client +// +// Related Resources +// +// * CreateMultipartUpload +// +// * CompleteMultipartUpload +// +// * AbortMultipartUpload +// +// * ListParts +// +// * ListMultipartUploads +// // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about // the error. @@ -6980,7 +9942,94 @@ func (c *S3) UploadPartCopyRequest(input *UploadPartCopyInput) (req *request.Req // UploadPartCopy API operation for Amazon Simple Storage Service. // -// Uploads a part by copying data from an existing object as data source. +// Uploads a part by copying data from an existing object as data source. You +// specify the data source by adding the request header x-amz-copy-source in +// your request and a byte range by adding the request header x-amz-copy-source-range +// in your request. +// +// The minimum allowable part size for a multipart upload is 5 MB. For more +// information about multipart upload limits, go to Quick Facts (https://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// Instead of using an existing object as part data, you might use the UploadPart +// operation and provide data in your request. +// +// You must initiate a multipart upload before you can upload any part. In response +// to your initiate request. Amazon S3 returns a unique identifier, the upload +// ID, that you must include in your upload part request. +// +// For more information about using the UploadPartCopy operation, see the following: +// +// * For conceptual information about multipart uploads, see Uploading Objects +// Using Multipart Upload (https://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// * For information about permissions required to use the multipart upload +// API, see Multipart Upload API and Permissions (https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// * For information about copying objects using a single atomic operation +// vs. the multipart upload, see Operations on Objects (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectOperations.html) +// in the Amazon Simple Storage Service Developer Guide. +// +// * For information about using server-side encryption with customer-provided +// encryption keys with the UploadPartCopy operation, see CopyObject and +// UploadPart. +// +// Note the following additional considerations about the request headers x-amz-copy-source-if-match, +// x-amz-copy-source-if-none-match, x-amz-copy-source-if-unmodified-since, and +// x-amz-copy-source-if-modified-since: +// +// * Consideration 1 - If both of the x-amz-copy-source-if-match and x-amz-copy-source-if-unmodified-since +// headers are present in the request as follows: x-amz-copy-source-if-match +// condition evaluates to true, and; x-amz-copy-source-if-unmodified-since +// condition evaluates to false; Amazon S3 returns 200 OK and copies the +// data. +// +// * Consideration 2 - If both of the x-amz-copy-source-if-none-match and +// x-amz-copy-source-if-modified-since headers are present in the request +// as follows: x-amz-copy-source-if-none-match condition evaluates to false, +// and; x-amz-copy-source-if-modified-since condition evaluates to true; +// Amazon S3 returns 412 Precondition Failed response code. +// +// Versioning +// +// If your bucket has versioning enabled, you could have multiple versions of +// the same object. By default, x-amz-copy-source identifies the current version +// of the object to copy. If the current version is a delete marker and you +// don't specify a versionId in the x-amz-copy-source, Amazon S3 returns a 404 +// error, because the object does not exist. If you specify versionId in the +// x-amz-copy-source and the versionId is a delete marker, Amazon S3 returns +// an HTTP 400 error, because you are not allowed to specify a delete marker +// as a version for the x-amz-copy-source. +// +// You can optionally specify a specific version of the source object to copy +// by adding the versionId subresource as shown in the following example: +// +// x-amz-copy-source: /bucket/object?versionId=version id +// +// Special Errors +// +// * Code: NoSuchUpload Cause: The specified multipart upload does not exist. +// The upload ID might be invalid, or the multipart upload might have been +// aborted or completed. HTTP Status Code: 404 Not Found +// +// * Code: InvalidRequest Cause: The specified copy source is not supported +// as a byte-range copy source. HTTP Status Code: 400 Bad Request +// +// Related Resources +// +// * CreateMultipartUpload +// +// * UploadPart +// +// * CompleteMultipartUpload +// +// * AbortMultipartUpload +// +// * ListParts +// +// * ListMultipartUploads // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -7010,13 +10059,16 @@ func (c *S3) UploadPartCopyWithContext(ctx aws.Context, input *UploadPartCopyInp return out, req.Send() } -// Specifies the days since the initiation of an Incomplete Multipart Upload -// that Lifecycle will wait before permanently removing all parts of the upload. +// Specifies the days since the initiation of an incomplete multipart upload +// that Amazon S3 will wait before permanently removing all parts of the upload. +// For more information, see Aborting Incomplete Multipart Uploads Using a Bucket +// Lifecycle Policy (https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuoverview.html#mpu-abort-incomplete-mpu-lifecycle-config) +// in the Amazon Simple Storage Service Developer Guide. type AbortIncompleteMultipartUpload struct { _ struct{} `type:"structure"` - // Indicates the number of days that must pass since initiation for Lifecycle - // to abort an Incomplete Multipart Upload. + // Specifies the number of days after which Amazon S3 aborts an incomplete multipart + // upload. DaysAfterInitiation *int64 `type:"integer"` } @@ -7037,20 +10089,34 @@ func (s *AbortIncompleteMultipartUpload) SetDaysAfterInitiation(v int64) *AbortI } type AbortMultipartUploadInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"AbortMultipartUploadRequest" type:"structure"` + // The bucket name to which the upload was taking place. + // + // When using this API with an access point, you must direct requests to the + // access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com. + // When using this operation using an access point through the AWS SDKs, you + // provide the access point ARN in place of the bucket name. For more information + // about access point ARNs, see Using Access Points (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-access-points.html) + // in the Amazon Simple Storage Service Developer Guide. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + // Key of the object for which the multipart upload was initiated. + // // Key is a required field Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // Documentation on downloading objects from requester pays buckets can be found - // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` + // Upload ID that identifies the multipart upload. + // // UploadId is a required field UploadId *string `location:"querystring" locationName:"uploadId" type:"string" required:"true"` } @@ -7121,6 +10187,20 @@ func (s *AbortMultipartUploadInput) SetUploadId(v string) *AbortMultipartUploadI return s } +func (s *AbortMultipartUploadInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *AbortMultipartUploadInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type AbortMultipartUploadOutput struct { _ struct{} `type:"structure"` @@ -7145,10 +10225,13 @@ func (s *AbortMultipartUploadOutput) SetRequestCharged(v string) *AbortMultipart return s } +// Configures the transfer acceleration state for an Amazon S3 bucket. For more +// information, see Amazon S3 Transfer Acceleration (https://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html) +// in the Amazon Simple Storage Service Developer Guide. type AccelerateConfiguration struct { _ struct{} `type:"structure"` - // The accelerate configuration of the bucket. + // Specifies the transfer acceleration status of the bucket. Status *string `type:"string" enum:"BucketAccelerateStatus"` } @@ -7168,12 +10251,14 @@ func (s *AccelerateConfiguration) SetStatus(v string) *AccelerateConfiguration { return s } +// Contains the elements that set the ACL permissions for an object per grantee. type AccessControlPolicy struct { _ struct{} `type:"structure"` // A list of grants. Grants []*Grant `locationName:"AccessControlList" locationNameList:"Grant" type:"list"` + // Container for the bucket owner's display name and ID. Owner *Owner `type:"structure"` } @@ -7223,7 +10308,9 @@ func (s *AccessControlPolicy) SetOwner(v *Owner) *AccessControlPolicy { type AccessControlTranslation struct { _ struct{} `type:"structure"` - // The override value for the owner of the replica object. + // Specifies the replica ownership. For default and valid values, see PUT bucket + // replication (https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUTreplication.html) + // in the Amazon Simple Storage Service API Reference. // // Owner is a required field Owner *string `type:"string" required:"true" enum:"OwnerOverride"` @@ -7258,10 +10345,14 @@ func (s *AccessControlTranslation) SetOwner(v string) *AccessControlTranslation return s } +// A conjunction (logical AND) of predicates, which is used in evaluating a +// metrics filter. The operator must have at least two predicates in any combination, +// and an object must match all of the predicates for the filter to apply. type AnalyticsAndOperator struct { _ struct{} `type:"structure"` - // The prefix to use when evaluating an AND predicate. + // The prefix to use when evaluating an AND predicate: The prefix that an object + // must have to be included in the metrics results. Prefix *string `type:"string"` // The list of tags to use when evaluating an AND predicate. @@ -7310,6 +10401,8 @@ func (s *AnalyticsAndOperator) SetTags(v []*Tag) *AnalyticsAndOperator { return s } +// Specifies the configuration and any analyses for the analytics filter of +// an Amazon S3 bucket. type AnalyticsConfiguration struct { _ struct{} `type:"structure"` @@ -7318,13 +10411,13 @@ type AnalyticsConfiguration struct { // If no filter is provided, all objects will be considered in any analysis. Filter *AnalyticsFilter `type:"structure"` - // The identifier used to represent an analytics configuration. + // The ID that identifies the analytics configuration. // // Id is a required field Id *string `type:"string" required:"true"` - // If present, it indicates that data related to access patterns will be collected - // and made available to analyze the tradeoffs between different storage classes. + // Contains data related to access patterns to be collected and made available + // to analyze the tradeoffs between different storage classes. // // StorageClassAnalysis is a required field StorageClassAnalysis *StorageClassAnalysis `type:"structure" required:"true"` @@ -7384,6 +10477,7 @@ func (s *AnalyticsConfiguration) SetStorageClassAnalysis(v *StorageClassAnalysis return s } +// Where to publish the analytics results. type AnalyticsExportDestination struct { _ struct{} `type:"structure"` @@ -7427,6 +10521,9 @@ func (s *AnalyticsExportDestination) SetS3BucketDestination(v *AnalyticsS3Bucket return s } +// The filter used to describe a set of objects for analyses. A filter must +// have exactly one prefix, one tag, or one conjunction (AnalyticsAndOperator). +// If no filter is provided, all objects will be considered in any analysis. type AnalyticsFilter struct { _ struct{} `type:"structure"` @@ -7489,25 +10586,28 @@ func (s *AnalyticsFilter) SetTag(v *Tag) *AnalyticsFilter { return s } +// Contains information about where to publish the analytics results. type AnalyticsS3BucketDestination struct { _ struct{} `type:"structure"` - // The Amazon resource name (ARN) of the bucket to which data is exported. + // The Amazon Resource Name (ARN) of the bucket to which data is exported. // // Bucket is a required field Bucket *string `type:"string" required:"true"` - // The account ID that owns the destination bucket. If no account ID is provided, - // the owner will not be validated prior to exporting data. + // The account ID that owns the destination S3 bucket. If no account ID is provided, + // the owner is not validated before exporting data. + // + // Although this value is optional, we strongly recommend that you set it to + // help prevent problems if the destination bucket ownership changes. BucketAccountId *string `type:"string"` - // The file format used when exporting data to Amazon S3. + // Specifies the file format used when exporting data to Amazon S3. // // Format is a required field Format *string `type:"string" required:"true" enum:"AnalyticsS3ExportFileFormat"` - // The prefix to use when exporting data. The exported data begins with this - // prefix. + // The prefix to use when exporting data. The prefix is prepended to all results. Prefix *string `type:"string"` } @@ -7568,6 +10668,8 @@ func (s *AnalyticsS3BucketDestination) SetPrefix(v string) *AnalyticsS3BucketDes return s } +// In terms of implementation, a Bucket is a resource. An Amazon S3 bucket name +// is globally unique, and the namespace is shared by all AWS accounts. type Bucket struct { _ struct{} `type:"structure"` @@ -7600,9 +10702,14 @@ func (s *Bucket) SetName(v string) *Bucket { return s } +// Specifies the lifecycle configuration for objects in an Amazon S3 bucket. +// For more information, see Object Lifecycle Management (https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html) +// in the Amazon Simple Storage Service Developer Guide. type BucketLifecycleConfiguration struct { _ struct{} `type:"structure"` + // A lifecycle rule for individual objects in an Amazon S3 bucket. + // // Rules is a required field Rules []*LifecycleRule `locationName:"Rule" type:"list" flattened:"true" required:"true"` } @@ -7646,12 +10753,14 @@ func (s *BucketLifecycleConfiguration) SetRules(v []*LifecycleRule) *BucketLifec return s } +// Container for logging status information. type BucketLoggingStatus struct { _ struct{} `type:"structure"` - // Container for logging information. Presence of this element indicates that - // logging is enabled. Parameters TargetBucket and TargetPrefix are required - // in this case. + // Describes where logs are stored and the prefix that Amazon S3 assigns to + // all log object keys for a bucket. For more information, see PUT Bucket logging + // (https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUTlogging.html) + // in the Amazon Simple Storage Service API Reference. LoggingEnabled *LoggingEnabled `type:"structure"` } @@ -7686,9 +10795,16 @@ func (s *BucketLoggingStatus) SetLoggingEnabled(v *LoggingEnabled) *BucketLoggin return s } +// Describes the cross-origin access configuration for objects in an Amazon +// S3 bucket. For more information, see Enabling Cross-Origin Resource Sharing +// (https://docs.aws.amazon.com/AmazonS3/latest/dev/cors.html) in the Amazon +// Simple Storage Service Developer Guide. type CORSConfiguration struct { _ struct{} `type:"structure"` + // A set of origins and methods (cross-origin access that you want to allow). + // You can add up to 100 rules to the configuration. + // // CORSRules is a required field CORSRules []*CORSRule `locationName:"CORSRule" type:"list" flattened:"true" required:"true"` } @@ -7732,14 +10848,18 @@ func (s *CORSConfiguration) SetCORSRules(v []*CORSRule) *CORSConfiguration { return s } +// Specifies a cross-origin access rule for an Amazon S3 bucket. type CORSRule struct { _ struct{} `type:"structure"` - // Specifies which headers are allowed in a pre-flight OPTIONS request. + // Headers that are specified in the Access-Control-Request-Headers header. + // These headers are allowed in a preflight OPTIONS request. In response to + // any preflight OPTIONS request, Amazon S3 returns any requested headers that + // are allowed. AllowedHeaders []*string `locationName:"AllowedHeader" type:"list" flattened:"true"` - // Identifies HTTP methods that the domain/origin specified in the rule is allowed - // to execute. + // An HTTP method that you allow the origin to execute. Valid values are GET, + // PUT, HEAD, POST, and DELETE. // // AllowedMethods is a required field AllowedMethods []*string `locationName:"AllowedMethod" type:"list" flattened:"true" required:"true"` @@ -7815,7 +10935,8 @@ func (s *CORSRule) SetMaxAgeSeconds(v int64) *CORSRule { return s } -// Describes how a CSV-formatted input object is formatted. +// Describes how an uncompressed comma-separated values (CSV)-formatted input +// object is formatted. type CSVInput struct { _ struct{} `type:"structure"` @@ -7824,24 +10945,45 @@ type CSVInput struct { // to TRUE may lower performance. AllowQuotedRecordDelimiter *bool `type:"boolean"` - // The single character used to indicate a row should be ignored when present - // at the start of a row. + // A single character used to indicate that a row should be ignored when the + // character is present at the start of that row. You can specify any character + // to indicate a comment line. Comments *string `type:"string"` - // The value used to separate individual fields in a record. + // A single character used to separate individual fields in a record. You can + // specify an arbitrary delimiter. FieldDelimiter *string `type:"string"` - // Describes the first line of input. Valid values: None, Ignore, Use. + // Describes the first line of input. Valid values are: + // + // * NONE: First line is not a header. + // + // * IGNORE: First line is a header, but you can't use the header values + // to indicate the column in an expression. You can use column position (such + // as _1, _2, …) to indicate the column (SELECT s._1 FROM OBJECT s). + // + // * Use: First line is a header, and you can use the header value to identify + // a column in an expression (SELECT "name" FROM OBJECT). FileHeaderInfo *string `type:"string" enum:"FileHeaderInfo"` - // Value used for escaping where the field delimiter is part of the value. + // A single character used for escaping when the field delimiter is part of + // the value. For example, if the value is a, b, Amazon S3 wraps this field + // value in quotation marks, as follows: " a , b ". + // + // Type: String + // + // Default: " + // + // Ancestors: CSV QuoteCharacter *string `type:"string"` - // The single character used for escaping the quote character inside an already - // escaped value. + // A single character used for escaping the quotation mark character inside + // an already escaped value. For example, the value """ a , b """ is parsed + // as " a , b ". QuoteEscapeCharacter *string `type:"string"` - // The value used to separate individual records. + // A single character used to separate individual records in the input. Instead + // of the default value, you can specify an arbitrary delimiter. RecordDelimiter *string `type:"string"` } @@ -7897,24 +11039,33 @@ func (s *CSVInput) SetRecordDelimiter(v string) *CSVInput { return s } -// Describes how CSV-formatted results are formatted. +// Describes how uncompressed comma-separated values (CSV)-formatted results +// are formatted. type CSVOutput struct { _ struct{} `type:"structure"` - // The value used to separate individual fields in a record. + // The value used to separate individual fields in a record. You can specify + // an arbitrary delimiter. FieldDelimiter *string `type:"string"` - // The value used for escaping where the field delimiter is part of the value. + // A single character used for escaping when the field delimiter is part of + // the value. For example, if the value is a, b, Amazon S3 wraps this field + // value in quotation marks, as follows: " a , b ". QuoteCharacter *string `type:"string"` - // Th single character used for escaping the quote character inside an already + // The single character used for escaping the quote character inside an already // escaped value. QuoteEscapeCharacter *string `type:"string"` - // Indicates whether or not all output fields should be quoted. + // Indicates whether to use quotation marks around output fields. + // + // * ALWAYS: Always use quotation marks for output fields. + // + // * ASNEEDED: Use quotation marks for output fields when needed. QuoteFields *string `type:"string" enum:"QuoteFields"` - // The value used to separate individual records. + // A single character used to separate individual records in the output. Instead + // of the default value, you can specify an arbitrary delimiter. RecordDelimiter *string `type:"string"` } @@ -7958,9 +11109,12 @@ func (s *CSVOutput) SetRecordDelimiter(v string) *CSVOutput { return s } +// Container for specifying the AWS Lambda notification configuration. type CloudFunctionConfiguration struct { _ struct{} `type:"structure"` + // Lambda cloud function ARN that Amazon S3 can invoke when it detects events + // of the specified type. CloudFunction *string `type:"string"` // The bucket event for which to send notifications. @@ -7968,12 +11122,14 @@ type CloudFunctionConfiguration struct { // Deprecated: Event has been deprecated Event *string `deprecated:"true" type:"string" enum:"Event"` + // Bucket events for which to send notifications. Events []*string `locationName:"Event" type:"list" flattened:"true"` // An optional unique identifier for configurations in a notification configuration. // If you don't provide one, Amazon S3 will assign an ID. Id *string `type:"string"` + // The role supporting the invocation of the Lambda function InvocationRole *string `type:"string"` } @@ -8017,9 +11173,15 @@ func (s *CloudFunctionConfiguration) SetInvocationRole(v string) *CloudFunctionC return s } +// Container for all (if there are any) keys between Prefix and the next occurrence +// of the string specified by a delimiter. CommonPrefixes lists keys that act +// like subdirectories in the directory specified by Prefix. For example, if +// the prefix is notes/ and the delimiter is a slash (/) as in notes/summer/july, +// the common prefix is notes/summer/. type CommonPrefix struct { _ struct{} `type:"structure"` + // Container for the specified common prefix. Prefix *string `type:"string"` } @@ -8040,22 +11202,30 @@ func (s *CommonPrefix) SetPrefix(v string) *CommonPrefix { } type CompleteMultipartUploadInput struct { - _ struct{} `type:"structure" payload:"MultipartUpload"` + _ struct{} `locationName:"CompleteMultipartUploadRequest" type:"structure" payload:"MultipartUpload"` + // Name of the bucket to which the multipart upload was initiated. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + // Object key for which the multipart upload was initiated. + // // Key is a required field Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"` + // The container for the multipart upload request information. MultipartUpload *CompletedMultipartUpload `locationName:"CompleteMultipartUpload" type:"structure" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // Documentation on downloading objects from requester pays buckets can be found - // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` + // ID for the initiated multipart upload. + // // UploadId is a required field UploadId *string `location:"querystring" locationName:"uploadId" type:"string" required:"true"` } @@ -8132,35 +11302,61 @@ func (s *CompleteMultipartUploadInput) SetUploadId(v string) *CompleteMultipartU return s } +func (s *CompleteMultipartUploadInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *CompleteMultipartUploadInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type CompleteMultipartUploadOutput struct { _ struct{} `type:"structure"` + // The name of the bucket that contains the newly created object. Bucket *string `type:"string"` - // Entity tag of the object. + // Entity tag that identifies the newly created object's data. Objects with + // different object data will have different entity tags. The entity tag is + // an opaque string. The entity tag may or may not be an MD5 digest of the object + // data. If the entity tag is not an MD5 digest of the object data, it will + // contain one or more nonhexadecimal characters and/or will consist of less + // than 32 or more than 32 hexadecimal digits. ETag *string `type:"string"` // If the object expiration is configured, this will contain the expiration // date (expiry-date) and rule ID (rule-id). The value of rule-id is URL encoded. Expiration *string `location:"header" locationName:"x-amz-expiration" type:"string"` + // The object key of the newly created object. Key *string `min:"1" type:"string"` + // The URI that identifies the newly created object. Location *string `type:"string"` // If present, indicates that the requester was successfully charged for the // request. RequestCharged *string `location:"header" locationName:"x-amz-request-charged" type:"string" enum:"RequestCharged"` - // If present, specifies the ID of the AWS Key Management Service (KMS) master - // encryption key that was used for the object. + // If present, specifies the ID of the AWS Key Management Service (AWS KMS) + // symmetric customer managed customer master key (CMK) that was used for the + // object. SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` - // The Server-side encryption algorithm used when storing this object in S3 - // (e.g., AES256, aws:kms). + // If you specified server-side encryption either with an Amazon S3-managed + // encryption key or an AWS KMS customer master key (CMK) in your initiate multipart + // upload request, the response includes this header. It confirms the encryption + // algorithm that Amazon S3 used to encrypt the object. ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string" enum:"ServerSideEncryption"` - // Version of the object. + // Version ID of the newly created object, in case the bucket has versioning + // turned on. VersionId *string `location:"header" locationName:"x-amz-version-id" type:"string"` } @@ -8235,9 +11431,11 @@ func (s *CompleteMultipartUploadOutput) SetVersionId(v string) *CompleteMultipar return s } +// The container for the completed multipart upload details. type CompletedMultipartUpload struct { _ struct{} `type:"structure"` + // Array of CompletedPart data types. Parts []*CompletedPart `locationName:"Part" type:"list" flattened:"true"` } @@ -8257,6 +11455,7 @@ func (s *CompletedMultipartUpload) SetParts(v []*CompletedPart) *CompletedMultip return s } +// Details of the parts that were uploaded. type CompletedPart struct { _ struct{} `type:"structure"` @@ -8290,6 +11489,10 @@ func (s *CompletedPart) SetPartNumber(v int64) *CompletedPart { return s } +// A container for describing a condition that must be met for the specified +// redirect to apply. For example, 1. If request is for pages in the /docs folder, +// redirect to the /documents folder. 2. If request results in HTTP error 4xx, +// redirect request to another host where you might process the error. type Condition struct { _ struct{} `type:"structure"` @@ -8358,12 +11561,21 @@ func (s *ContinuationEvent) UnmarshalEvent( return nil } +// MarshalEvent marshals the type into an stream event value. This method +// should only used internally within the SDK's EventStream handling. +func (s *ContinuationEvent) MarshalEvent(pm protocol.PayloadMarshaler) (msg eventstream.Message, err error) { + msg.Headers.Set(eventstreamapi.MessageTypeHeader, eventstream.StringValue(eventstreamapi.EventMessageType)) + return msg, err +} + type CopyObjectInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"CopyObjectRequest" type:"structure"` // The canned ACL to apply to the object. ACL *string `location:"header" locationName:"x-amz-acl" type:"string" enum:"ObjectCannedACL"` + // The name of the destination bucket. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` @@ -8403,17 +11615,18 @@ type CopyObjectInput struct { // Copies the object if it hasn't been modified since the specified time. CopySourceIfUnmodifiedSince *time.Time `location:"header" locationName:"x-amz-copy-source-if-unmodified-since" type:"timestamp"` - // Specifies the algorithm to use when decrypting the source object (e.g., AES256). + // Specifies the algorithm to use when decrypting the source object (for example, + // AES256). CopySourceSSECustomerAlgorithm *string `location:"header" locationName:"x-amz-copy-source-server-side-encryption-customer-algorithm" type:"string"` // Specifies the customer-provided encryption key for Amazon S3 to use to decrypt // the source object. The encryption key provided in this header must be one // that was used when the source object was created. - CopySourceSSECustomerKey *string `location:"header" locationName:"x-amz-copy-source-server-side-encryption-customer-key" type:"string" sensitive:"true"` + CopySourceSSECustomerKey *string `marshal-as:"blob" location:"header" locationName:"x-amz-copy-source-server-side-encryption-customer-key" type:"string" sensitive:"true"` // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. - // Amazon S3 uses this header for a message integrity check to ensure the encryption - // key was transmitted without error. + // Amazon S3 uses this header for a message integrity check to ensure that the + // encryption key was transmitted without error. CopySourceSSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-copy-source-server-side-encryption-customer-key-MD5" type:"string"` // The date and time at which the object is no longer cacheable. @@ -8431,6 +11644,8 @@ type CopyObjectInput struct { // Allows grantee to write the ACL for the applicable object. GrantWriteACP *string `location:"header" locationName:"x-amz-grant-write-acp" type:"string"` + // The key of the destination object. + // // Key is a required field Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"` @@ -8450,35 +11665,44 @@ type CopyObjectInput struct { // The date and time when you want the copied object's Object Lock to expire. ObjectLockRetainUntilDate *time.Time `location:"header" locationName:"x-amz-object-lock-retain-until-date" type:"timestamp" timestampFormat:"iso8601"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // Documentation on downloading objects from requester pays buckets can be found - // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` - // Specifies the algorithm to use to when encrypting the object (e.g., AES256). + // Specifies the algorithm to use to when encrypting the object (for example, + // AES256). SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` // Specifies the customer-provided encryption key for Amazon S3 to use in encrypting // data. This value is used to store the object and then it is discarded; Amazon - // does not store the encryption key. The key must be appropriate for use with - // the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm + // S3 does not store the encryption key. The key must be appropriate for use + // with the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm // header. - SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"` + SSECustomerKey *string `marshal-as:"blob" location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"` // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. - // Amazon S3 uses this header for a message integrity check to ensure the encryption - // key was transmitted without error. + // Amazon S3 uses this header for a message integrity check to ensure that the + // encryption key was transmitted without error. SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` + // Specifies the AWS KMS Encryption Context to use for object encryption. The + // value of this header is a base64-encoded UTF-8 string holding JSON with the + // encryption context key-value pairs. + SSEKMSEncryptionContext *string `location:"header" locationName:"x-amz-server-side-encryption-context" type:"string" sensitive:"true"` + // Specifies the AWS KMS key ID to use for object encryption. All GET and PUT // requests for an object protected by AWS KMS will fail if not made via SSL - // or using SigV4. Documentation on configuring any of the officially supported - // AWS SDKs and CLI can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingAWSSDK.html#specify-signature-version + // or using SigV4. For information about configuring using any of the officially + // supported AWS SDKs and AWS CLI, see Specifying the Signature Version in Request + // Authentication (https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingAWSSDK.html#specify-signature-version) + // in the Amazon S3 Developer Guide. SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` - // The Server-side encryption algorithm used when storing this object in S3 - // (e.g., AES256, aws:kms). + // The server-side encryption algorithm used when storing this object in Amazon + // S3 (for example, AES256, aws:kms). ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string" enum:"ServerSideEncryption"` // The type of storage to use for the object. Defaults to 'STANDARD'. @@ -8486,7 +11710,7 @@ type CopyObjectInput struct { // The tag-set for the object destination object this value must be used in // conjunction with the TaggingDirective. The tag-set must be encoded as URL - // Query parameters + // Query parameters. Tagging *string `location:"header" locationName:"x-amz-tagging" type:"string"` // Specifies whether the object tag-set are copied from the source object or @@ -8735,6 +11959,12 @@ func (s *CopyObjectInput) SetSSECustomerKeyMD5(v string) *CopyObjectInput { return s } +// SetSSEKMSEncryptionContext sets the SSEKMSEncryptionContext field's value. +func (s *CopyObjectInput) SetSSEKMSEncryptionContext(v string) *CopyObjectInput { + s.SSEKMSEncryptionContext = &v + return s +} + // SetSSEKMSKeyId sets the SSEKMSKeyId field's value. func (s *CopyObjectInput) SetSSEKMSKeyId(v string) *CopyObjectInput { s.SSEKMSKeyId = &v @@ -8771,11 +12001,27 @@ func (s *CopyObjectInput) SetWebsiteRedirectLocation(v string) *CopyObjectInput return s } +func (s *CopyObjectInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *CopyObjectInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type CopyObjectOutput struct { _ struct{} `type:"structure" payload:"CopyObjectResult"` + // Container for all response elements. CopyObjectResult *CopyObjectResult `type:"structure"` + // Version of the copied object in the destination bucket. CopySourceVersionId *string `location:"header" locationName:"x-amz-copy-source-version-id" type:"string"` // If the object expiration is configured, the response includes this header. @@ -8791,16 +12037,22 @@ type CopyObjectOutput struct { SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` // If server-side encryption with a customer-provided encryption key was requested, - // the response will include this header to provide round trip message integrity + // the response will include this header to provide round-trip message integrity // verification of the customer-provided encryption key. SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` - // If present, specifies the ID of the AWS Key Management Service (KMS) master - // encryption key that was used for the object. + // If present, specifies the AWS KMS Encryption Context to use for object encryption. + // The value of this header is a base64-encoded UTF-8 string holding JSON with + // the encryption context key-value pairs. + SSEKMSEncryptionContext *string `location:"header" locationName:"x-amz-server-side-encryption-context" type:"string" sensitive:"true"` + + // If present, specifies the ID of the AWS Key Management Service (AWS KMS) + // symmetric customer managed customer master key (CMK) that was used for the + // object. SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` - // The Server-side encryption algorithm used when storing this object in S3 - // (e.g., AES256, aws:kms). + // The server-side encryption algorithm used when storing this object in Amazon + // S3 (for example, AES256, aws:kms). ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string" enum:"ServerSideEncryption"` // Version ID of the newly created copy. @@ -8853,6 +12105,12 @@ func (s *CopyObjectOutput) SetSSECustomerKeyMD5(v string) *CopyObjectOutput { return s } +// SetSSEKMSEncryptionContext sets the SSEKMSEncryptionContext field's value. +func (s *CopyObjectOutput) SetSSEKMSEncryptionContext(v string) *CopyObjectOutput { + s.SSEKMSEncryptionContext = &v + return s +} + // SetSSEKMSKeyId sets the SSEKMSKeyId field's value. func (s *CopyObjectOutput) SetSSEKMSKeyId(v string) *CopyObjectOutput { s.SSEKMSKeyId = &v @@ -8871,11 +12129,16 @@ func (s *CopyObjectOutput) SetVersionId(v string) *CopyObjectOutput { return s } +// Container for all response elements. type CopyObjectResult struct { _ struct{} `type:"structure"` + // Returns the ETag of the new object. The ETag reflects only changes to the + // contents of an object, not its metadata. The source and destination ETag + // is identical for a successfully copied object. ETag *string `type:"string"` + // Returns the date that the object was last modified. LastModified *time.Time `type:"timestamp"` } @@ -8901,6 +12164,7 @@ func (s *CopyObjectResult) SetLastModified(v time.Time) *CopyObjectResult { return s } +// Container for all response elements. type CopyPartResult struct { _ struct{} `type:"structure"` @@ -8933,11 +12197,12 @@ func (s *CopyPartResult) SetLastModified(v time.Time) *CopyPartResult { return s } +// The configuration information for the bucket. type CreateBucketConfiguration struct { _ struct{} `type:"structure"` - // Specifies the region where the bucket will be created. If you don't specify - // a region, the bucket will be created in US Standard. + // Specifies the Region where the bucket will be created. If you don't specify + // a Region, the bucket is created in the US East (N. Virginia) Region (us-east-1). LocationConstraint *string `type:"string" enum:"BucketLocationConstraint"` } @@ -8958,14 +12223,17 @@ func (s *CreateBucketConfiguration) SetLocationConstraint(v string) *CreateBucke } type CreateBucketInput struct { - _ struct{} `type:"structure" payload:"CreateBucketConfiguration"` + _ struct{} `locationName:"CreateBucketRequest" type:"structure" payload:"CreateBucketConfiguration"` // The canned ACL to apply to the bucket. ACL *string `location:"header" locationName:"x-amz-acl" type:"string" enum:"BucketCannedACL"` + // The name of the bucket to create. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + // The configuration information for the bucket. CreateBucketConfiguration *CreateBucketConfiguration `locationName:"CreateBucketConfiguration" type:"structure" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` // Allows grantee the read, write, read ACP, and write ACP permissions on the @@ -9078,6 +12346,9 @@ func (s *CreateBucketInput) SetObjectLockEnabledForBucket(v bool) *CreateBucketI type CreateBucketOutput struct { _ struct{} `type:"structure"` + // Specifies the Region where the bucket will be created. If you are creating + // a bucket on the US East (N. Virginia) Region (us-east-1), you do not need + // to specify the location. Location *string `location:"header" locationName:"Location" type:"string"` } @@ -9098,11 +12369,13 @@ func (s *CreateBucketOutput) SetLocation(v string) *CreateBucketOutput { } type CreateMultipartUploadInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"CreateMultipartUploadRequest" type:"structure"` // The canned ACL to apply to the object. ACL *string `location:"header" locationName:"x-amz-acl" type:"string" enum:"ObjectCannedACL"` + // The name of the bucket to which to initiate the upload + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` @@ -9138,6 +12411,8 @@ type CreateMultipartUploadInput struct { // Allows grantee to write the ACL for the applicable object. GrantWriteACP *string `location:"header" locationName:"x-amz-grant-write-acp" type:"string"` + // Object key for which the multipart upload is to be initiated. + // // Key is a required field Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"` @@ -9153,41 +12428,50 @@ type CreateMultipartUploadInput struct { // Specifies the date and time when you want the Object Lock to expire. ObjectLockRetainUntilDate *time.Time `location:"header" locationName:"x-amz-object-lock-retain-until-date" type:"timestamp" timestampFormat:"iso8601"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // Documentation on downloading objects from requester pays buckets can be found - // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` - // Specifies the algorithm to use to when encrypting the object (e.g., AES256). + // Specifies the algorithm to use to when encrypting the object (for example, + // AES256). SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` // Specifies the customer-provided encryption key for Amazon S3 to use in encrypting // data. This value is used to store the object and then it is discarded; Amazon - // does not store the encryption key. The key must be appropriate for use with - // the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm + // S3 does not store the encryption key. The key must be appropriate for use + // with the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm // header. - SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"` + SSECustomerKey *string `marshal-as:"blob" location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"` // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. - // Amazon S3 uses this header for a message integrity check to ensure the encryption - // key was transmitted without error. + // Amazon S3 uses this header for a message integrity check to ensure that the + // encryption key was transmitted without error. SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` - // Specifies the AWS KMS key ID to use for object encryption. All GET and PUT - // requests for an object protected by AWS KMS will fail if not made via SSL - // or using SigV4. Documentation on configuring any of the officially supported - // AWS SDKs and CLI can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingAWSSDK.html#specify-signature-version + // Specifies the AWS KMS Encryption Context to use for object encryption. The + // value of this header is a base64-encoded UTF-8 string holding JSON with the + // encryption context key-value pairs. + SSEKMSEncryptionContext *string `location:"header" locationName:"x-amz-server-side-encryption-context" type:"string" sensitive:"true"` + + // Specifies the ID of the symmetric customer managed AWS KMS CMK to use for + // object encryption. All GET and PUT requests for an object protected by AWS + // KMS will fail if not made via SSL or using SigV4. For information about configuring + // using any of the officially supported AWS SDKs and AWS CLI, see Specifying + // the Signature Version in Request Authentication (https://docs.aws.amazon.com/http:/docs.aws.amazon.com/AmazonS3/latest/dev/UsingAWSSDK.html#specify-signature-version) + // in the Amazon S3 Developer Guide. SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` - // The Server-side encryption algorithm used when storing this object in S3 - // (e.g., AES256, aws:kms). + // The server-side encryption algorithm used when storing this object in Amazon + // S3 (for example, AES256, aws:kms). ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string" enum:"ServerSideEncryption"` // The type of storage to use for the object. Defaults to 'STANDARD'. StorageClass *string `location:"header" locationName:"x-amz-storage-class" type:"string" enum:"StorageClass"` - // The tag-set for the object. The tag-set must be encoded as URL Query parameters + // The tag-set for the object. The tag-set must be encoded as URL Query parameters. Tagging *string `location:"header" locationName:"x-amz-tagging" type:"string"` // If the bucket is configured as a website, redirects requests for this object @@ -9368,8 +12652,14 @@ func (s *CreateMultipartUploadInput) SetSSECustomerKeyMD5(v string) *CreateMulti return s } -// SetSSEKMSKeyId sets the SSEKMSKeyId field's value. -func (s *CreateMultipartUploadInput) SetSSEKMSKeyId(v string) *CreateMultipartUploadInput { +// SetSSEKMSEncryptionContext sets the SSEKMSEncryptionContext field's value. +func (s *CreateMultipartUploadInput) SetSSEKMSEncryptionContext(v string) *CreateMultipartUploadInput { + s.SSEKMSEncryptionContext = &v + return s +} + +// SetSSEKMSKeyId sets the SSEKMSKeyId field's value. +func (s *CreateMultipartUploadInput) SetSSEKMSKeyId(v string) *CreateMultipartUploadInput { s.SSEKMSKeyId = &v return s } @@ -9398,17 +12688,47 @@ func (s *CreateMultipartUploadInput) SetWebsiteRedirectLocation(v string) *Creat return s } +func (s *CreateMultipartUploadInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *CreateMultipartUploadInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type CreateMultipartUploadOutput struct { _ struct{} `type:"structure"` - // Date when multipart upload will become eligible for abort operation by lifecycle. + // If the bucket has a lifecycle rule configured with an action to abort incomplete + // multipart uploads and the prefix in the lifecycle rule matches the object + // name in the request, the response includes this header. The header indicates + // when the initiated multipart upload becomes eligible for an abort operation. + // For more information, see Aborting Incomplete Multipart Uploads Using a Bucket + // Lifecycle Policy (https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuoverview.html#mpu-abort-incomplete-mpu-lifecycle-config). + // + // The response also includes the x-amz-abort-rule-id header that provides the + // ID of the lifecycle configuration rule that defines this action. AbortDate *time.Time `location:"header" locationName:"x-amz-abort-date" type:"timestamp"` - // Id of the lifecycle rule that makes a multipart upload eligible for abort - // operation. + // This header is returned along with the x-amz-abort-date header. It identifies + // the applicable lifecycle configuration rule that defines the action to abort + // incomplete multipart uploads. AbortRuleId *string `location:"header" locationName:"x-amz-abort-rule-id" type:"string"` // Name of the bucket to which the multipart upload was initiated. + // + // When using this API with an access point, you must direct requests to the + // access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com. + // When using this operation using an access point through the AWS SDKs, you + // provide the access point ARN in place of the bucket name. For more information + // about access point ARNs, see Using Access Points (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-access-points.html) + // in the Amazon Simple Storage Service Developer Guide. Bucket *string `locationName:"Bucket" type:"string"` // Object key for which the multipart upload was initiated. @@ -9424,16 +12744,22 @@ type CreateMultipartUploadOutput struct { SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` // If server-side encryption with a customer-provided encryption key was requested, - // the response will include this header to provide round trip message integrity + // the response will include this header to provide round-trip message integrity // verification of the customer-provided encryption key. SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` - // If present, specifies the ID of the AWS Key Management Service (KMS) master - // encryption key that was used for the object. + // If present, specifies the AWS KMS Encryption Context to use for object encryption. + // The value of this header is a base64-encoded UTF-8 string holding JSON with + // the encryption context key-value pairs. + SSEKMSEncryptionContext *string `location:"header" locationName:"x-amz-server-side-encryption-context" type:"string" sensitive:"true"` + + // If present, specifies the ID of the AWS Key Management Service (AWS KMS) + // symmetric customer managed customer master key (CMK) that was used for the + // object. SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` - // The Server-side encryption algorithm used when storing this object in S3 - // (e.g., AES256, aws:kms). + // The server-side encryption algorithm used when storing this object in Amazon + // S3 (for example, AES256, aws:kms). ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string" enum:"ServerSideEncryption"` // ID for the initiated multipart upload. @@ -9499,6 +12825,12 @@ func (s *CreateMultipartUploadOutput) SetSSECustomerKeyMD5(v string) *CreateMult return s } +// SetSSEKMSEncryptionContext sets the SSEKMSEncryptionContext field's value. +func (s *CreateMultipartUploadOutput) SetSSEKMSEncryptionContext(v string) *CreateMultipartUploadOutput { + s.SSEKMSEncryptionContext = &v + return s +} + // SetSSEKMSKeyId sets the SSEKMSKeyId field's value. func (s *CreateMultipartUploadOutput) SetSSEKMSKeyId(v string) *CreateMultipartUploadOutput { s.SSEKMSKeyId = &v @@ -9561,9 +12893,12 @@ func (s *DefaultRetention) SetYears(v int64) *DefaultRetention { return s } +// Container for the objects to delete. type Delete struct { _ struct{} `type:"structure"` + // The objects to delete. + // // Objects is a required field Objects []*ObjectIdentifier `locationName:"Object" type:"list" flattened:"true" required:"true"` @@ -9618,14 +12953,14 @@ func (s *Delete) SetQuiet(v bool) *Delete { } type DeleteBucketAnalyticsConfigurationInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"DeleteBucketAnalyticsConfigurationRequest" type:"structure"` // The name of the bucket from which an analytics configuration is deleted. // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` - // The identifier used to represent an analytics configuration. + // The ID that identifies the analytics configuration. // // Id is a required field Id *string `location:"querystring" locationName:"id" type:"string" required:"true"` @@ -9679,6 +13014,20 @@ func (s *DeleteBucketAnalyticsConfigurationInput) SetId(v string) *DeleteBucketA return s } +func (s *DeleteBucketAnalyticsConfigurationInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *DeleteBucketAnalyticsConfigurationInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type DeleteBucketAnalyticsConfigurationOutput struct { _ struct{} `type:"structure"` } @@ -9694,8 +13043,10 @@ func (s DeleteBucketAnalyticsConfigurationOutput) GoString() string { } type DeleteBucketCorsInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"DeleteBucketCorsRequest" type:"structure"` + // Specifies the bucket whose cors configuration is being deleted. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` } @@ -9739,6 +13090,20 @@ func (s *DeleteBucketCorsInput) getBucket() (v string) { return *s.Bucket } +func (s *DeleteBucketCorsInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *DeleteBucketCorsInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type DeleteBucketCorsOutput struct { _ struct{} `type:"structure"` } @@ -9754,7 +13119,7 @@ func (s DeleteBucketCorsOutput) GoString() string { } type DeleteBucketEncryptionInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"DeleteBucketEncryptionRequest" type:"structure"` // The name of the bucket containing the server-side encryption configuration // to delete. @@ -9802,6 +13167,20 @@ func (s *DeleteBucketEncryptionInput) getBucket() (v string) { return *s.Bucket } +func (s *DeleteBucketEncryptionInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *DeleteBucketEncryptionInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type DeleteBucketEncryptionOutput struct { _ struct{} `type:"structure"` } @@ -9817,8 +13196,10 @@ func (s DeleteBucketEncryptionOutput) GoString() string { } type DeleteBucketInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"DeleteBucketRequest" type:"structure"` + // Specifies the bucket being deleted. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` } @@ -9862,8 +13243,22 @@ func (s *DeleteBucketInput) getBucket() (v string) { return *s.Bucket } +func (s *DeleteBucketInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *DeleteBucketInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type DeleteBucketInventoryConfigurationInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"DeleteBucketInventoryConfigurationRequest" type:"structure"` // The name of the bucket containing the inventory configuration to delete. // @@ -9924,6 +13319,20 @@ func (s *DeleteBucketInventoryConfigurationInput) SetId(v string) *DeleteBucketI return s } +func (s *DeleteBucketInventoryConfigurationInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *DeleteBucketInventoryConfigurationInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type DeleteBucketInventoryConfigurationOutput struct { _ struct{} `type:"structure"` } @@ -9939,8 +13348,10 @@ func (s DeleteBucketInventoryConfigurationOutput) GoString() string { } type DeleteBucketLifecycleInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"DeleteBucketLifecycleRequest" type:"structure"` + // The bucket name of the lifecycle to delete. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` } @@ -9984,6 +13395,20 @@ func (s *DeleteBucketLifecycleInput) getBucket() (v string) { return *s.Bucket } +func (s *DeleteBucketLifecycleInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *DeleteBucketLifecycleInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type DeleteBucketLifecycleOutput struct { _ struct{} `type:"structure"` } @@ -9999,7 +13424,7 @@ func (s DeleteBucketLifecycleOutput) GoString() string { } type DeleteBucketMetricsConfigurationInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"DeleteBucketMetricsConfigurationRequest" type:"structure"` // The name of the bucket containing the metrics configuration to delete. // @@ -10060,6 +13485,20 @@ func (s *DeleteBucketMetricsConfigurationInput) SetId(v string) *DeleteBucketMet return s } +func (s *DeleteBucketMetricsConfigurationInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *DeleteBucketMetricsConfigurationInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type DeleteBucketMetricsConfigurationOutput struct { _ struct{} `type:"structure"` } @@ -10089,8 +13528,10 @@ func (s DeleteBucketOutput) GoString() string { } type DeleteBucketPolicyInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"DeleteBucketPolicyRequest" type:"structure"` + // The bucket name. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` } @@ -10134,6 +13575,20 @@ func (s *DeleteBucketPolicyInput) getBucket() (v string) { return *s.Bucket } +func (s *DeleteBucketPolicyInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *DeleteBucketPolicyInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type DeleteBucketPolicyOutput struct { _ struct{} `type:"structure"` } @@ -10149,13 +13604,10 @@ func (s DeleteBucketPolicyOutput) GoString() string { } type DeleteBucketReplicationInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"DeleteBucketReplicationRequest" type:"structure"` // The bucket name. // - // It can take a while to propagate the deletion of a replication configuration - // to all Amazon S3 systems. - // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` } @@ -10199,6 +13651,20 @@ func (s *DeleteBucketReplicationInput) getBucket() (v string) { return *s.Bucket } +func (s *DeleteBucketReplicationInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *DeleteBucketReplicationInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type DeleteBucketReplicationOutput struct { _ struct{} `type:"structure"` } @@ -10214,8 +13680,10 @@ func (s DeleteBucketReplicationOutput) GoString() string { } type DeleteBucketTaggingInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"DeleteBucketTaggingRequest" type:"structure"` + // The bucket that has the tag set to be removed. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` } @@ -10259,6 +13727,20 @@ func (s *DeleteBucketTaggingInput) getBucket() (v string) { return *s.Bucket } +func (s *DeleteBucketTaggingInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *DeleteBucketTaggingInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type DeleteBucketTaggingOutput struct { _ struct{} `type:"structure"` } @@ -10274,8 +13756,10 @@ func (s DeleteBucketTaggingOutput) GoString() string { } type DeleteBucketWebsiteInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"DeleteBucketWebsiteRequest" type:"structure"` + // The bucket name for which you want to remove the website configuration. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` } @@ -10319,6 +13803,20 @@ func (s *DeleteBucketWebsiteInput) getBucket() (v string) { return *s.Bucket } +func (s *DeleteBucketWebsiteInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *DeleteBucketWebsiteInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type DeleteBucketWebsiteOutput struct { _ struct{} `type:"structure"` } @@ -10333,6 +13831,7 @@ func (s DeleteBucketWebsiteOutput) GoString() string { return s.String() } +// Information about the delete marker. type DeleteMarkerEntry struct { _ struct{} `type:"structure"` @@ -10346,6 +13845,7 @@ type DeleteMarkerEntry struct { // Date and time the object was last modified. LastModified *time.Time `type:"timestamp"` + // The account that created the delete marker.> Owner *Owner `type:"structure"` // Version ID of an object. @@ -10392,11 +13892,21 @@ func (s *DeleteMarkerEntry) SetVersionId(v string) *DeleteMarkerEntry { return s } -// Specifies whether Amazon S3 should replicate delete makers. +// Specifies whether Amazon S3 replicates the delete markers. If you specify +// a Filter, you must specify this element. However, in the latest version of +// replication configuration (when Filter is specified), Amazon S3 doesn't replicate +// delete markers. Therefore, the DeleteMarkerReplication element can contain +// only Disabled. For an example configuration, see Basic Rule +// Configuration (https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html#replication-config-min-rule-config). +// +// If you don't specify the Filter element, Amazon S3 assumes that the replication +// configuration is the earlier version, V1. In the earlier version, Amazon +// S3 handled replication of delete markers differently. For more information, +// see Backward Compatibility (https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html#replication-backward-compat-considerations). type DeleteMarkerReplication struct { _ struct{} `type:"structure"` - // The status of the delete marker replication. + // Indicates whether to replicate delete markers. // // In the current implementation, Amazon S3 doesn't replicate the delete markers. // The status must be Disabled. @@ -10420,8 +13930,17 @@ func (s *DeleteMarkerReplication) SetStatus(v string) *DeleteMarkerReplication { } type DeleteObjectInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"DeleteObjectRequest" type:"structure"` + // The bucket name of the bucket containing the object. + // + // When using this API with an access point, you must direct requests to the + // access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com. + // When using this operation using an access point through the AWS SDKs, you + // provide the access point ARN in place of the bucket name. For more information + // about access point ARNs, see Using Access Points (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-access-points.html) + // in the Amazon Simple Storage Service Developer Guide. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` @@ -10429,17 +13948,22 @@ type DeleteObjectInput struct { // to process this operation. BypassGovernanceRetention *bool `location:"header" locationName:"x-amz-bypass-governance-retention" type:"boolean"` + // Key name of the object to delete. + // // Key is a required field Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"` // The concatenation of the authentication device's serial number, a space, - // and the value that is displayed on your authentication device. + // and the value that is displayed on your authentication device. Required to + // permanently delete a versioned object if versioning is configured with MFA + // delete enabled. MFA *string `location:"header" locationName:"x-amz-mfa" type:"string"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // Documentation on downloading objects from requester pays buckets can be found - // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` // VersionId used to reference a specific version of the object. @@ -10521,6 +14045,20 @@ func (s *DeleteObjectInput) SetVersionId(v string) *DeleteObjectInput { return s } +func (s *DeleteObjectInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *DeleteObjectInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type DeleteObjectOutput struct { _ struct{} `type:"structure"` @@ -10566,11 +14104,22 @@ func (s *DeleteObjectOutput) SetVersionId(v string) *DeleteObjectOutput { } type DeleteObjectTaggingInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"DeleteObjectTaggingRequest" type:"structure"` + // The bucket name containing the objects from which to remove the tags. + // + // When using this API with an access point, you must direct requests to the + // access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com. + // When using this operation using an access point through the AWS SDKs, you + // provide the access point ARN in place of the bucket name. For more information + // about access point ARNs, see Using Access Points (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-access-points.html) + // in the Amazon Simple Storage Service Developer Guide. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + // Name of the tag. + // // Key is a required field Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"` @@ -10635,6 +14184,20 @@ func (s *DeleteObjectTaggingInput) SetVersionId(v string) *DeleteObjectTaggingIn return s } +func (s *DeleteObjectTaggingInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *DeleteObjectTaggingInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type DeleteObjectTaggingOutput struct { _ struct{} `type:"structure"` @@ -10659,8 +14222,17 @@ func (s *DeleteObjectTaggingOutput) SetVersionId(v string) *DeleteObjectTaggingO } type DeleteObjectsInput struct { - _ struct{} `type:"structure" payload:"Delete"` + _ struct{} `locationName:"DeleteObjectsRequest" type:"structure" payload:"Delete"` + // The bucket name containing the objects to delete. + // + // When using this API with an access point, you must direct requests to the + // access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com. + // When using this operation using an access point through the AWS SDKs, you + // provide the access point ARN in place of the bucket name. For more information + // about access point ARNs, see Using Access Points (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-access-points.html) + // in the Amazon Simple Storage Service Developer Guide. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` @@ -10669,17 +14241,22 @@ type DeleteObjectsInput struct { // operation. BypassGovernanceRetention *bool `location:"header" locationName:"x-amz-bypass-governance-retention" type:"boolean"` + // Container for the request. + // // Delete is a required field Delete *Delete `locationName:"Delete" type:"structure" required:"true" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` // The concatenation of the authentication device's serial number, a space, - // and the value that is displayed on your authentication device. + // and the value that is displayed on your authentication device. Required to + // permanently delete a versioned object if versioning is configured with MFA + // delete enabled. MFA *string `location:"header" locationName:"x-amz-mfa" type:"string"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // Documentation on downloading objects from requester pays buckets can be found - // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` } @@ -10754,11 +14331,29 @@ func (s *DeleteObjectsInput) SetRequestPayer(v string) *DeleteObjectsInput { return s } +func (s *DeleteObjectsInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *DeleteObjectsInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type DeleteObjectsOutput struct { _ struct{} `type:"structure"` + // Container element for a successful delete. It identifies the object that + // was successfully deleted. Deleted []*DeletedObject `type:"list" flattened:"true"` + // Container for a failed delete operation that describes the object that Amazon + // S3 attempted to delete and the error it encountered. Errors []*Error `locationName:"Error" type:"list" flattened:"true"` // If present, indicates that the requester was successfully charged for the @@ -10795,7 +14390,7 @@ func (s *DeleteObjectsOutput) SetRequestCharged(v string) *DeleteObjectsOutput { } type DeletePublicAccessBlockInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"DeletePublicAccessBlockRequest" type:"structure"` // The Amazon S3 bucket whose PublicAccessBlock configuration you want to delete. // @@ -10842,6 +14437,20 @@ func (s *DeletePublicAccessBlockInput) getBucket() (v string) { return *s.Bucket } +func (s *DeletePublicAccessBlockInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *DeletePublicAccessBlockInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type DeletePublicAccessBlockOutput struct { _ struct{} `type:"structure"` } @@ -10856,15 +14465,24 @@ func (s DeletePublicAccessBlockOutput) GoString() string { return s.String() } +// Information about the deleted object. type DeletedObject struct { _ struct{} `type:"structure"` + // Specifies whether the versioned object that was permanently deleted was (true) + // or was not (false) a delete marker. In a simple DELETE, this header indicates + // whether (true) or not (false) a delete marker was created. DeleteMarker *bool `type:"boolean"` + // The version ID of the delete marker created as a result of the DELETE operation. + // If you delete a specific object version, the value returned by this header + // is the version ID of the object version deleted. DeleteMarkerVersionId *string `type:"string"` + // The name of the deleted object. Key *string `min:"1" type:"string"` + // The version ID of the deleted object. VersionId *string `type:"string"` } @@ -10902,33 +14520,28 @@ func (s *DeletedObject) SetVersionId(v string) *DeletedObject { return s } -// A container for information about the replication destination. +// Specifies information about where to publish analysis or configuration results +// for an Amazon S3 bucket and S3 Replication Time Control (S3 RTC). type Destination struct { _ struct{} `type:"structure"` - // A container for information about access control for replicas. - // - // Use this element only in a cross-account scenario where source and destination - // bucket owners are not the same to change replica ownership to the AWS account - // that owns the destination bucket. If you don't add this element to the replication - // configuration, the replicas are owned by same AWS account that owns the source - // object. + // Specify this only in a cross-account scenario (where source and destination + // bucket owners are not the same), and you want to change replica ownership + // to the AWS account that owns the destination bucket. If this is not specified + // in the replication configuration, the replicas are owned by same AWS account + // that owns the source object. AccessControlTranslation *AccessControlTranslation `type:"structure"` - // The account ID of the destination bucket. Currently, Amazon S3 verifies this - // value only if Access Control Translation is enabled. - // - // In a cross-account scenario, if you change replica ownership to the AWS account - // that owns the destination bucket by adding the AccessControlTranslation element, - // this is the account ID of the owner of the destination bucket. + // Destination bucket owner account ID. In a cross-account scenario, if you + // direct Amazon S3 to change replica ownership to the AWS account that owns + // the destination bucket by specifying the AccessControlTranslation property, + // this is the account ID of the destination bucket owner. For more information, + // see Replication Additional Configuration: Changing the Replica Owner (https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-change-owner.html) + // in the Amazon Simple Storage Service Developer Guide. Account *string `type:"string"` // The Amazon Resource Name (ARN) of the bucket where you want Amazon S3 to - // store replicas of the object identified by the rule. - // - // If there are multiple rules in your replication configuration, all rules - // must specify the same bucket as the destination. A replication configuration - // can replicate objects to only one destination bucket. + // store the results. // // Bucket is a required field Bucket *string `type:"string" required:"true"` @@ -10937,8 +14550,23 @@ type Destination struct { // is specified, you must specify this element. EncryptionConfiguration *EncryptionConfiguration `type:"structure"` - // The class of storage used to store the object. By default Amazon S3 uses - // storage class of the source object when creating a replica. + // A container specifying replication metrics-related settings enabling metrics + // and Amazon S3 events for S3 Replication Time Control (S3 RTC). Must be specified + // together with a ReplicationTime block. + Metrics *Metrics `type:"structure"` + + // A container specifying S3 Replication Time Control (S3 RTC), including whether + // S3 RTC is enabled and the time when all objects and operations on objects + // must be replicated. Must be specified together with a Metrics block. + ReplicationTime *ReplicationTime `type:"structure"` + + // The storage class to use when replicating objects, such as S3 Standard or + // reduced redundancy. By default, Amazon S3 uses the storage class of the source + // object to create the object replica. + // + // For valid values, see the StorageClass element of the PUT Bucket replication + // (https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUTreplication.html) + // action in the Amazon Simple Storage Service API Reference. StorageClass *string `type:"string" enum:"StorageClass"` } @@ -10963,6 +14591,16 @@ func (s *Destination) Validate() error { invalidParams.AddNested("AccessControlTranslation", err.(request.ErrInvalidParams)) } } + if s.Metrics != nil { + if err := s.Metrics.Validate(); err != nil { + invalidParams.AddNested("Metrics", err.(request.ErrInvalidParams)) + } + } + if s.ReplicationTime != nil { + if err := s.ReplicationTime.Validate(); err != nil { + invalidParams.AddNested("ReplicationTime", err.(request.ErrInvalidParams)) + } + } if invalidParams.Len() > 0 { return invalidParams @@ -11001,19 +14639,30 @@ func (s *Destination) SetEncryptionConfiguration(v *EncryptionConfiguration) *De return s } +// SetMetrics sets the Metrics field's value. +func (s *Destination) SetMetrics(v *Metrics) *Destination { + s.Metrics = v + return s +} + +// SetReplicationTime sets the ReplicationTime field's value. +func (s *Destination) SetReplicationTime(v *ReplicationTime) *Destination { + s.ReplicationTime = v + return s +} + // SetStorageClass sets the StorageClass field's value. func (s *Destination) SetStorageClass(v string) *Destination { s.StorageClass = &v return s } -// Describes the server-side encryption that will be applied to the restore -// results. +// Contains the type of server-side encryption used. type Encryption struct { _ struct{} `type:"structure"` // The server-side encryption algorithm used when storing job results in Amazon - // S3 (e.g., AES256, aws:kms). + // S3 (for example, AES256, aws:kms). // // EncryptionType is a required field EncryptionType *string `type:"string" required:"true" enum:"ServerSideEncryption"` @@ -11022,8 +14671,11 @@ type Encryption struct { // the encryption context for the restore results. KMSContext *string `type:"string"` - // If the encryption type is aws:kms, this optional value specifies the AWS - // KMS key ID to use for encryption of job results. + // If the encryption type is aws:kms, this optional value specifies the ID of + // the symmetric customer managed AWS KMS CMK to use for encryption of job results. + // Amazon S3 only supports symmetric CMKs. For more information, see Using Symmetric + // and Asymmetric Keys (https://docs.aws.amazon.com/kms/latest/developerguide/symmetric-asymmetric.html) + // in the AWS Key Management Service Developer Guide. KMSKeyId *string `type:"string" sensitive:"true"` } @@ -11068,13 +14720,17 @@ func (s *Encryption) SetKMSKeyId(v string) *Encryption { return s } -// A container for information about the encryption-based configuration for -// replicas. +// Specifies encryption-related information for an Amazon S3 bucket that is +// a destination for replicated objects. type EncryptionConfiguration struct { _ struct{} `type:"structure"` - // The ID of the AWS KMS key for the AWS Region where the destination bucket - // resides. Amazon S3 uses this key to encrypt the replica object. + // Specifies the ID (Key ARN or Alias ARN) of the customer managed customer + // master key (CMK) stored in AWS Key Management Service (KMS) for the destination + // bucket. Amazon S3 uses this key to encrypt replica objects. Amazon S3 only + // supports symmetric customer managed CMKs. For more information, see Using + // Symmetric and Asymmetric Keys (https://docs.aws.amazon.com/kms/latest/developerguide/symmetric-asymmetric.html) + // in the AWS Key Management Service Developer Guide. ReplicaKmsKeyID *string `type:"string"` } @@ -11094,6 +14750,9 @@ func (s *EncryptionConfiguration) SetReplicaKmsKeyID(v string) *EncryptionConfig return s } +// A message that indicates the request is complete and no more messages will +// be sent. You should not assume that the request is complete until the client +// receives an EndEvent. type EndEvent struct { _ struct{} `locationName:"EndEvent" type:"structure"` } @@ -11120,15 +14779,382 @@ func (s *EndEvent) UnmarshalEvent( return nil } +// MarshalEvent marshals the type into an stream event value. This method +// should only used internally within the SDK's EventStream handling. +func (s *EndEvent) MarshalEvent(pm protocol.PayloadMarshaler) (msg eventstream.Message, err error) { + msg.Headers.Set(eventstreamapi.MessageTypeHeader, eventstream.StringValue(eventstreamapi.EventMessageType)) + return msg, err +} + +// Container for all error elements. type Error struct { _ struct{} `type:"structure"` + // The error code is a string that uniquely identifies an error condition. It + // is meant to be read and understood by programs that detect and handle errors + // by type. + // + // Amazon S3 error codes + // + // * Code: AccessDenied Description: Access Denied HTTP Status Code: 403 + // Forbidden SOAP Fault Code Prefix: Client + // + // * Code: AccountProblem Description: There is a problem with your AWS account + // that prevents the operation from completing successfully. Contact AWS + // Support for further assistance. HTTP Status Code: 403 Forbidden SOAP Fault + // Code Prefix: Client + // + // * Code: AllAccessDisabled Description: All access to this Amazon S3 resource + // has been disabled. Contact AWS Support for further assistance. HTTP Status + // Code: 403 Forbidden SOAP Fault Code Prefix: Client + // + // * Code: AmbiguousGrantByEmailAddress Description: The email address you + // provided is associated with more than one account. HTTP Status Code: 400 + // Bad Request SOAP Fault Code Prefix: Client + // + // * Code: AuthorizationHeaderMalformed Description: The authorization header + // you provided is invalid. HTTP Status Code: 400 Bad Request HTTP Status + // Code: N/A + // + // * Code: BadDigest Description: The Content-MD5 you specified did not match + // what we received. HTTP Status Code: 400 Bad Request SOAP Fault Code Prefix: + // Client + // + // * Code: BucketAlreadyExists Description: The requested bucket name is + // not available. The bucket namespace is shared by all users of the system. + // Please select a different name and try again. HTTP Status Code: 409 Conflict + // SOAP Fault Code Prefix: Client + // + // * Code: BucketAlreadyOwnedByYou Description: The bucket you tried to create + // already exists, and you own it. Amazon S3 returns this error in all AWS + // Regions except in the North Virginia Region. For legacy compatibility, + // if you re-create an existing bucket that you already own in the North + // Virginia Region, Amazon S3 returns 200 OK and resets the bucket access + // control lists (ACLs). Code: 409 Conflict (in all Regions except the North + // Virginia Region) SOAP Fault Code Prefix: Client + // + // * Code: BucketNotEmpty Description: The bucket you tried to delete is + // not empty. HTTP Status Code: 409 Conflict SOAP Fault Code Prefix: Client + // + // * Code: CredentialsNotSupported Description: This request does not support + // credentials. HTTP Status Code: 400 Bad Request SOAP Fault Code Prefix: + // Client + // + // * Code: CrossLocationLoggingProhibited Description: Cross-location logging + // not allowed. Buckets in one geographic location cannot log information + // to a bucket in another location. HTTP Status Code: 403 Forbidden SOAP + // Fault Code Prefix: Client + // + // * Code: EntityTooSmall Description: Your proposed upload is smaller than + // the minimum allowed object size. HTTP Status Code: 400 Bad Request SOAP + // Fault Code Prefix: Client + // + // * Code: EntityTooLarge Description: Your proposed upload exceeds the maximum + // allowed object size. HTTP Status Code: 400 Bad Request SOAP Fault Code + // Prefix: Client + // + // * Code: ExpiredToken Description: The provided token has expired. HTTP + // Status Code: 400 Bad Request SOAP Fault Code Prefix: Client + // + // * Code: IllegalVersioningConfigurationException Description: Indicates + // that the versioning configuration specified in the request is invalid. + // HTTP Status Code: 400 Bad Request SOAP Fault Code Prefix: Client + // + // * Code: IncompleteBody Description: You did not provide the number of + // bytes specified by the Content-Length HTTP header HTTP Status Code: 400 + // Bad Request SOAP Fault Code Prefix: Client + // + // * Code: IncorrectNumberOfFilesInPostRequest Description: POST requires + // exactly one file upload per request. HTTP Status Code: 400 Bad Request + // SOAP Fault Code Prefix: Client + // + // * Code: InlineDataTooLarge Description: Inline data exceeds the maximum + // allowed size. HTTP Status Code: 400 Bad Request SOAP Fault Code Prefix: + // Client + // + // * Code: InternalError Description: We encountered an internal error. Please + // try again. HTTP Status Code: 500 Internal Server Error SOAP Fault Code + // Prefix: Server + // + // * Code: InvalidAccessKeyId Description: The AWS access key ID you provided + // does not exist in our records. HTTP Status Code: 403 Forbidden SOAP Fault + // Code Prefix: Client + // + // * Code: InvalidAddressingHeader Description: You must specify the Anonymous + // role. HTTP Status Code: N/A SOAP Fault Code Prefix: Client + // + // * Code: InvalidArgument Description: Invalid Argument HTTP Status Code: + // 400 Bad Request SOAP Fault Code Prefix: Client + // + // * Code: InvalidBucketName Description: The specified bucket is not valid. + // HTTP Status Code: 400 Bad Request SOAP Fault Code Prefix: Client + // + // * Code: InvalidBucketState Description: The request is not valid with + // the current state of the bucket. HTTP Status Code: 409 Conflict SOAP Fault + // Code Prefix: Client + // + // * Code: InvalidDigest Description: The Content-MD5 you specified is not + // valid. HTTP Status Code: 400 Bad Request SOAP Fault Code Prefix: Client + // + // * Code: InvalidEncryptionAlgorithmError Description: The encryption request + // you specified is not valid. The valid value is AES256. HTTP Status Code: + // 400 Bad Request SOAP Fault Code Prefix: Client + // + // * Code: InvalidLocationConstraint Description: The specified location + // constraint is not valid. For more information about Regions, see How to + // Select a Region for Your Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro). + // HTTP Status Code: 400 Bad Request SOAP Fault Code Prefix: Client + // + // * Code: InvalidObjectState Description: The operation is not valid for + // the current state of the object. HTTP Status Code: 403 Forbidden SOAP + // Fault Code Prefix: Client + // + // * Code: InvalidPart Description: One or more of the specified parts could + // not be found. The part might not have been uploaded, or the specified + // entity tag might not have matched the part's entity tag. HTTP Status Code: + // 400 Bad Request SOAP Fault Code Prefix: Client + // + // * Code: InvalidPartOrder Description: The list of parts was not in ascending + // order. Parts list must be specified in order by part number. HTTP Status + // Code: 400 Bad Request SOAP Fault Code Prefix: Client + // + // * Code: InvalidPayer Description: All access to this object has been disabled. + // Please contact AWS Support for further assistance. HTTP Status Code: 403 + // Forbidden SOAP Fault Code Prefix: Client + // + // * Code: InvalidPolicyDocument Description: The content of the form does + // not meet the conditions specified in the policy document. HTTP Status + // Code: 400 Bad Request SOAP Fault Code Prefix: Client + // + // * Code: InvalidRange Description: The requested range cannot be satisfied. + // HTTP Status Code: 416 Requested Range Not Satisfiable SOAP Fault Code + // Prefix: Client + // + // * Code: InvalidRequest Description: Please use AWS4-HMAC-SHA256. HTTP + // Status Code: 400 Bad Request Code: N/A + // + // * Code: InvalidRequest Description: SOAP requests must be made over an + // HTTPS connection. HTTP Status Code: 400 Bad Request SOAP Fault Code Prefix: + // Client + // + // * Code: InvalidRequest Description: Amazon S3 Transfer Acceleration is + // not supported for buckets with non-DNS compliant names. HTTP Status Code: + // 400 Bad Request Code: N/A + // + // * Code: InvalidRequest Description: Amazon S3 Transfer Acceleration is + // not supported for buckets with periods (.) in their names. HTTP Status + // Code: 400 Bad Request Code: N/A + // + // * Code: InvalidRequest Description: Amazon S3 Transfer Accelerate endpoint + // only supports virtual style requests. HTTP Status Code: 400 Bad Request + // Code: N/A + // + // * Code: InvalidRequest Description: Amazon S3 Transfer Accelerate is not + // configured on this bucket. HTTP Status Code: 400 Bad Request Code: N/A + // + // * Code: InvalidRequest Description: Amazon S3 Transfer Accelerate is disabled + // on this bucket. HTTP Status Code: 400 Bad Request Code: N/A + // + // * Code: InvalidRequest Description: Amazon S3 Transfer Acceleration is + // not supported on this bucket. Contact AWS Support for more information. + // HTTP Status Code: 400 Bad Request Code: N/A + // + // * Code: InvalidRequest Description: Amazon S3 Transfer Acceleration cannot + // be enabled on this bucket. Contact AWS Support for more information. HTTP + // Status Code: 400 Bad Request Code: N/A + // + // * Code: InvalidSecurity Description: The provided security credentials + // are not valid. HTTP Status Code: 403 Forbidden SOAP Fault Code Prefix: + // Client + // + // * Code: InvalidSOAPRequest Description: The SOAP request body is invalid. + // HTTP Status Code: 400 Bad Request SOAP Fault Code Prefix: Client + // + // * Code: InvalidStorageClass Description: The storage class you specified + // is not valid. HTTP Status Code: 400 Bad Request SOAP Fault Code Prefix: + // Client + // + // * Code: InvalidTargetBucketForLogging Description: The target bucket for + // logging does not exist, is not owned by you, or does not have the appropriate + // grants for the log-delivery group. HTTP Status Code: 400 Bad Request SOAP + // Fault Code Prefix: Client + // + // * Code: InvalidToken Description: The provided token is malformed or otherwise + // invalid. HTTP Status Code: 400 Bad Request SOAP Fault Code Prefix: Client + // + // * Code: InvalidURI Description: Couldn't parse the specified URI. HTTP + // Status Code: 400 Bad Request SOAP Fault Code Prefix: Client + // + // * Code: KeyTooLongError Description: Your key is too long. HTTP Status + // Code: 400 Bad Request SOAP Fault Code Prefix: Client + // + // * Code: MalformedACLError Description: The XML you provided was not well-formed + // or did not validate against our published schema. HTTP Status Code: 400 + // Bad Request SOAP Fault Code Prefix: Client + // + // * Code: MalformedPOSTRequest Description: The body of your POST request + // is not well-formed multipart/form-data. HTTP Status Code: 400 Bad Request + // SOAP Fault Code Prefix: Client + // + // * Code: MalformedXML Description: This happens when the user sends malformed + // XML (XML that doesn't conform to the published XSD) for the configuration. + // The error message is, "The XML you provided was not well-formed or did + // not validate against our published schema." HTTP Status Code: 400 Bad + // Request SOAP Fault Code Prefix: Client + // + // * Code: MaxMessageLengthExceeded Description: Your request was too big. + // HTTP Status Code: 400 Bad Request SOAP Fault Code Prefix: Client + // + // * Code: MaxPostPreDataLengthExceededError Description: Your POST request + // fields preceding the upload file were too large. HTTP Status Code: 400 + // Bad Request SOAP Fault Code Prefix: Client + // + // * Code: MetadataTooLarge Description: Your metadata headers exceed the + // maximum allowed metadata size. HTTP Status Code: 400 Bad Request SOAP + // Fault Code Prefix: Client + // + // * Code: MethodNotAllowed Description: The specified method is not allowed + // against this resource. HTTP Status Code: 405 Method Not Allowed SOAP Fault + // Code Prefix: Client + // + // * Code: MissingAttachment Description: A SOAP attachment was expected, + // but none were found. HTTP Status Code: N/A SOAP Fault Code Prefix: Client + // + // * Code: MissingContentLength Description: You must provide the Content-Length + // HTTP header. HTTP Status Code: 411 Length Required SOAP Fault Code Prefix: + // Client + // + // * Code: MissingRequestBodyError Description: This happens when the user + // sends an empty XML document as a request. The error message is, "Request + // body is empty." HTTP Status Code: 400 Bad Request SOAP Fault Code Prefix: + // Client + // + // * Code: MissingSecurityElement Description: The SOAP 1.1 request is missing + // a security element. HTTP Status Code: 400 Bad Request SOAP Fault Code + // Prefix: Client + // + // * Code: MissingSecurityHeader Description: Your request is missing a required + // header. HTTP Status Code: 400 Bad Request SOAP Fault Code Prefix: Client + // + // * Code: NoLoggingStatusForKey Description: There is no such thing as a + // logging status subresource for a key. HTTP Status Code: 400 Bad Request + // SOAP Fault Code Prefix: Client + // + // * Code: NoSuchBucket Description: The specified bucket does not exist. + // HTTP Status Code: 404 Not Found SOAP Fault Code Prefix: Client + // + // * Code: NoSuchBucketPolicy Description: The specified bucket does not + // have a bucket policy. HTTP Status Code: 404 Not Found SOAP Fault Code + // Prefix: Client + // + // * Code: NoSuchKey Description: The specified key does not exist. HTTP + // Status Code: 404 Not Found SOAP Fault Code Prefix: Client + // + // * Code: NoSuchLifecycleConfiguration Description: The lifecycle configuration + // does not exist. HTTP Status Code: 404 Not Found SOAP Fault Code Prefix: + // Client + // + // * Code: NoSuchUpload Description: The specified multipart upload does + // not exist. The upload ID might be invalid, or the multipart upload might + // have been aborted or completed. HTTP Status Code: 404 Not Found SOAP Fault + // Code Prefix: Client + // + // * Code: NoSuchVersion Description: Indicates that the version ID specified + // in the request does not match an existing version. HTTP Status Code: 404 + // Not Found SOAP Fault Code Prefix: Client + // + // * Code: NotImplemented Description: A header you provided implies functionality + // that is not implemented. HTTP Status Code: 501 Not Implemented SOAP Fault + // Code Prefix: Server + // + // * Code: NotSignedUp Description: Your account is not signed up for the + // Amazon S3 service. You must sign up before you can use Amazon S3. You + // can sign up at the following URL: https://aws.amazon.com/s3 HTTP Status + // Code: 403 Forbidden SOAP Fault Code Prefix: Client + // + // * Code: OperationAborted Description: A conflicting conditional operation + // is currently in progress against this resource. Try again. HTTP Status + // Code: 409 Conflict SOAP Fault Code Prefix: Client + // + // * Code: PermanentRedirect Description: The bucket you are attempting to + // access must be addressed using the specified endpoint. Send all future + // requests to this endpoint. HTTP Status Code: 301 Moved Permanently SOAP + // Fault Code Prefix: Client + // + // * Code: PreconditionFailed Description: At least one of the preconditions + // you specified did not hold. HTTP Status Code: 412 Precondition Failed + // SOAP Fault Code Prefix: Client + // + // * Code: Redirect Description: Temporary redirect. HTTP Status Code: 307 + // Moved Temporarily SOAP Fault Code Prefix: Client + // + // * Code: RestoreAlreadyInProgress Description: Object restore is already + // in progress. HTTP Status Code: 409 Conflict SOAP Fault Code Prefix: Client + // + // * Code: RequestIsNotMultiPartContent Description: Bucket POST must be + // of the enclosure-type multipart/form-data. HTTP Status Code: 400 Bad Request + // SOAP Fault Code Prefix: Client + // + // * Code: RequestTimeout Description: Your socket connection to the server + // was not read from or written to within the timeout period. HTTP Status + // Code: 400 Bad Request SOAP Fault Code Prefix: Client + // + // * Code: RequestTimeTooSkewed Description: The difference between the request + // time and the server's time is too large. HTTP Status Code: 403 Forbidden + // SOAP Fault Code Prefix: Client + // + // * Code: RequestTorrentOfBucketError Description: Requesting the torrent + // file of a bucket is not permitted. HTTP Status Code: 400 Bad Request SOAP + // Fault Code Prefix: Client + // + // * Code: SignatureDoesNotMatch Description: The request signature we calculated + // does not match the signature you provided. Check your AWS secret access + // key and signing method. For more information, see REST Authentication + // (https://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html) + // and SOAP Authentication (https://docs.aws.amazon.com/AmazonS3/latest/dev/SOAPAuthentication.html) + // for details. HTTP Status Code: 403 Forbidden SOAP Fault Code Prefix: Client + // + // * Code: ServiceUnavailable Description: Reduce your request rate. HTTP + // Status Code: 503 Service Unavailable SOAP Fault Code Prefix: Server + // + // * Code: SlowDown Description: Reduce your request rate. HTTP Status Code: + // 503 Slow Down SOAP Fault Code Prefix: Server + // + // * Code: TemporaryRedirect Description: You are being redirected to the + // bucket while DNS updates. HTTP Status Code: 307 Moved Temporarily SOAP + // Fault Code Prefix: Client + // + // * Code: TokenRefreshRequired Description: The provided token must be refreshed. + // HTTP Status Code: 400 Bad Request SOAP Fault Code Prefix: Client + // + // * Code: TooManyBuckets Description: You have attempted to create more + // buckets than allowed. HTTP Status Code: 400 Bad Request SOAP Fault Code + // Prefix: Client + // + // * Code: UnexpectedContent Description: This request does not support content. + // HTTP Status Code: 400 Bad Request SOAP Fault Code Prefix: Client + // + // * Code: UnresolvableGrantByEmailAddress Description: The email address + // you provided does not match any account on record. HTTP Status Code: 400 + // Bad Request SOAP Fault Code Prefix: Client + // + // * Code: UserKeyMustBeSpecified Description: The bucket POST must contain + // the specified field name. If it is specified, check the order of the fields. + // HTTP Status Code: 400 Bad Request SOAP Fault Code Prefix: Client Code *string `type:"string"` + // The error key. Key *string `min:"1" type:"string"` + // The error message contains a generic description of the error condition in + // English. It is intended for a human audience. Simple programs display the + // message directly to the end user if they encounter an error condition they + // don't know how or don't care to handle. Sophisticated programs with more + // exhaustive error handling and proper internationalization are more likely + // to ignore the error message. Message *string `type:"string"` + // The version ID of the error. VersionId *string `type:"string"` } @@ -11166,6 +15192,7 @@ func (s *Error) SetVersionId(v string) *Error { return s } +// The error information. type ErrorDocument struct { _ struct{} `type:"structure"` @@ -11207,18 +15234,58 @@ func (s *ErrorDocument) SetKey(v string) *ErrorDocument { return s } -// A container for a key value pair that defines the criteria for the filter -// rule. +// Optional configuration to replicate existing source bucket objects. For more +// information, see Replicating Existing Objects (https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-what-is-isnot-replicated.html#existing-object-replication) +// in the Amazon S3 Developer Guide. +type ExistingObjectReplication struct { + _ struct{} `type:"structure"` + + // Status is a required field + Status *string `type:"string" required:"true" enum:"ExistingObjectReplicationStatus"` +} + +// String returns the string representation +func (s ExistingObjectReplication) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s ExistingObjectReplication) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *ExistingObjectReplication) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "ExistingObjectReplication"} + if s.Status == nil { + invalidParams.Add(request.NewErrParamRequired("Status")) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetStatus sets the Status field's value. +func (s *ExistingObjectReplication) SetStatus(v string) *ExistingObjectReplication { + s.Status = &v + return s +} + +// Specifies the Amazon S3 object key name to filter on and whether to filter +// on the suffix or prefix of the key name. type FilterRule struct { _ struct{} `type:"structure"` // The object key name prefix or suffix identifying one or more objects to which - // the filtering rule applies. The maximum prefix length is 1,024 characters. - // Overlapping prefixes and suffixes are not supported. For more information, - // see Configuring Event Notifications (http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html) + // the filtering rule applies. The maximum length is 1,024 characters. Overlapping + // prefixes and suffixes are not supported. For more information, see Configuring + // Event Notifications (https://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html) // in the Amazon Simple Storage Service Developer Guide. Name *string `type:"string" enum:"FilterRuleName"` + // The value that the filter searches for in object key names. Value *string `type:"string"` } @@ -11245,7 +15312,7 @@ func (s *FilterRule) SetValue(v string) *FilterRule { } type GetBucketAccelerateConfigurationInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"GetBucketAccelerateConfigurationRequest" type:"structure"` // Name of the bucket for which the accelerate configuration is retrieved. // @@ -11292,6 +15359,20 @@ func (s *GetBucketAccelerateConfigurationInput) getBucket() (v string) { return *s.Bucket } +func (s *GetBucketAccelerateConfigurationInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *GetBucketAccelerateConfigurationInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type GetBucketAccelerateConfigurationOutput struct { _ struct{} `type:"structure"` @@ -11316,8 +15397,10 @@ func (s *GetBucketAccelerateConfigurationOutput) SetStatus(v string) *GetBucketA } type GetBucketAclInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"GetBucketAclRequest" type:"structure"` + // Specifies the S3 bucket whose ACL is being requested. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` } @@ -11361,12 +15444,27 @@ func (s *GetBucketAclInput) getBucket() (v string) { return *s.Bucket } +func (s *GetBucketAclInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *GetBucketAclInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type GetBucketAclOutput struct { _ struct{} `type:"structure"` // A list of grants. Grants []*Grant `locationName:"AccessControlList" locationNameList:"Grant" type:"list"` + // Container for the bucket owner's display name and ID. Owner *Owner `type:"structure"` } @@ -11393,14 +15491,14 @@ func (s *GetBucketAclOutput) SetOwner(v *Owner) *GetBucketAclOutput { } type GetBucketAnalyticsConfigurationInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"GetBucketAnalyticsConfigurationRequest" type:"structure"` // The name of the bucket from which an analytics configuration is retrieved. // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` - // The identifier used to represent an analytics configuration. + // The ID that identifies the analytics configuration. // // Id is a required field Id *string `location:"querystring" locationName:"id" type:"string" required:"true"` @@ -11454,6 +15552,20 @@ func (s *GetBucketAnalyticsConfigurationInput) SetId(v string) *GetBucketAnalyti return s } +func (s *GetBucketAnalyticsConfigurationInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *GetBucketAnalyticsConfigurationInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type GetBucketAnalyticsConfigurationOutput struct { _ struct{} `type:"structure" payload:"AnalyticsConfiguration"` @@ -11478,8 +15590,10 @@ func (s *GetBucketAnalyticsConfigurationOutput) SetAnalyticsConfiguration(v *Ana } type GetBucketCorsInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"GetBucketCorsRequest" type:"structure"` + // The bucket name for which to get the cors configuration. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` } @@ -11523,9 +15637,25 @@ func (s *GetBucketCorsInput) getBucket() (v string) { return *s.Bucket } +func (s *GetBucketCorsInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *GetBucketCorsInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type GetBucketCorsOutput struct { _ struct{} `type:"structure"` + // A set of origins and methods (cross-origin access that you want to allow). + // You can add up to 100 rules to the configuration. CORSRules []*CORSRule `locationName:"CORSRule" type:"list" flattened:"true"` } @@ -11546,7 +15676,7 @@ func (s *GetBucketCorsOutput) SetCORSRules(v []*CORSRule) *GetBucketCorsOutput { } type GetBucketEncryptionInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"GetBucketEncryptionRequest" type:"structure"` // The name of the bucket from which the server-side encryption configuration // is retrieved. @@ -11594,11 +15724,24 @@ func (s *GetBucketEncryptionInput) getBucket() (v string) { return *s.Bucket } +func (s *GetBucketEncryptionInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *GetBucketEncryptionInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type GetBucketEncryptionOutput struct { _ struct{} `type:"structure" payload:"ServerSideEncryptionConfiguration"` - // Container for server-side encryption configuration rules. Currently S3 supports - // one rule only. + // Specifies the default server-side-encryption configuration. ServerSideEncryptionConfiguration *ServerSideEncryptionConfiguration `type:"structure"` } @@ -11619,7 +15762,7 @@ func (s *GetBucketEncryptionOutput) SetServerSideEncryptionConfiguration(v *Serv } type GetBucketInventoryConfigurationInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"GetBucketInventoryConfigurationRequest" type:"structure"` // The name of the bucket containing the inventory configuration to retrieve. // @@ -11680,6 +15823,20 @@ func (s *GetBucketInventoryConfigurationInput) SetId(v string) *GetBucketInvento return s } +func (s *GetBucketInventoryConfigurationInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *GetBucketInventoryConfigurationInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type GetBucketInventoryConfigurationOutput struct { _ struct{} `type:"structure" payload:"InventoryConfiguration"` @@ -11704,8 +15861,10 @@ func (s *GetBucketInventoryConfigurationOutput) SetInventoryConfiguration(v *Inv } type GetBucketLifecycleConfigurationInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"GetBucketLifecycleConfigurationRequest" type:"structure"` + // The name of the bucket for which to get the lifecycle information. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` } @@ -11744,14 +15903,29 @@ func (s *GetBucketLifecycleConfigurationInput) SetBucket(v string) *GetBucketLif func (s *GetBucketLifecycleConfigurationInput) getBucket() (v string) { if s.Bucket == nil { - return v + return v + } + return *s.Bucket +} + +func (s *GetBucketLifecycleConfigurationInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *GetBucketLifecycleConfigurationInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false } - return *s.Bucket + return arn.IsARN(*s.Bucket) } type GetBucketLifecycleConfigurationOutput struct { _ struct{} `type:"structure"` + // Container for a lifecycle rule. Rules []*LifecycleRule `locationName:"Rule" type:"list" flattened:"true"` } @@ -11772,8 +15946,10 @@ func (s *GetBucketLifecycleConfigurationOutput) SetRules(v []*LifecycleRule) *Ge } type GetBucketLifecycleInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"GetBucketLifecycleRequest" type:"structure"` + // The name of the bucket for which to get the lifecycle information. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` } @@ -11817,9 +15993,24 @@ func (s *GetBucketLifecycleInput) getBucket() (v string) { return *s.Bucket } +func (s *GetBucketLifecycleInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *GetBucketLifecycleInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type GetBucketLifecycleOutput struct { _ struct{} `type:"structure"` + // Container for a lifecycle rule. Rules []*Rule `locationName:"Rule" type:"list" flattened:"true"` } @@ -11840,8 +16031,10 @@ func (s *GetBucketLifecycleOutput) SetRules(v []*Rule) *GetBucketLifecycleOutput } type GetBucketLocationInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"GetBucketLocationRequest" type:"structure"` + // The name of the bucket for which to get the location. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` } @@ -11885,9 +16078,26 @@ func (s *GetBucketLocationInput) getBucket() (v string) { return *s.Bucket } +func (s *GetBucketLocationInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *GetBucketLocationInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type GetBucketLocationOutput struct { _ struct{} `type:"structure"` + // Specifies the Region where the bucket resides. For a list of all the Amazon + // S3 supported location constraints by Region, see Regions and Endpoints (https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region). + // Buckets in Region us-east-1 have a LocationConstraint of null. LocationConstraint *string `type:"string" enum:"BucketLocationConstraint"` } @@ -11908,8 +16118,10 @@ func (s *GetBucketLocationOutput) SetLocationConstraint(v string) *GetBucketLoca } type GetBucketLoggingInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"GetBucketLoggingRequest" type:"structure"` + // The bucket name for which to get the logging information. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` } @@ -11953,12 +16165,27 @@ func (s *GetBucketLoggingInput) getBucket() (v string) { return *s.Bucket } +func (s *GetBucketLoggingInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *GetBucketLoggingInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type GetBucketLoggingOutput struct { _ struct{} `type:"structure"` - // Container for logging information. Presence of this element indicates that - // logging is enabled. Parameters TargetBucket and TargetPrefix are required - // in this case. + // Describes where logs are stored and the prefix that Amazon S3 assigns to + // all log object keys for a bucket. For more information, see PUT Bucket logging + // (https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUTlogging.html) + // in the Amazon Simple Storage Service API Reference. LoggingEnabled *LoggingEnabled `type:"structure"` } @@ -11979,7 +16206,7 @@ func (s *GetBucketLoggingOutput) SetLoggingEnabled(v *LoggingEnabled) *GetBucket } type GetBucketMetricsConfigurationInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"GetBucketMetricsConfigurationRequest" type:"structure"` // The name of the bucket containing the metrics configuration to retrieve. // @@ -12040,6 +16267,20 @@ func (s *GetBucketMetricsConfigurationInput) SetId(v string) *GetBucketMetricsCo return s } +func (s *GetBucketMetricsConfigurationInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *GetBucketMetricsConfigurationInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type GetBucketMetricsConfigurationOutput struct { _ struct{} `type:"structure" payload:"MetricsConfiguration"` @@ -12064,9 +16305,9 @@ func (s *GetBucketMetricsConfigurationOutput) SetMetricsConfiguration(v *Metrics } type GetBucketNotificationConfigurationRequest struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"GetBucketNotificationConfigurationRequest" type:"structure"` - // Name of the bucket to get the notification configuration for. + // Name of the bucket for which to get the notification configuration. // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` @@ -12111,9 +16352,25 @@ func (s *GetBucketNotificationConfigurationRequest) getBucket() (v string) { return *s.Bucket } +func (s *GetBucketNotificationConfigurationRequest) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *GetBucketNotificationConfigurationRequest) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type GetBucketPolicyInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"GetBucketPolicyRequest" type:"structure"` + // The bucket name for which to get the bucket policy. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` } @@ -12157,6 +16414,20 @@ func (s *GetBucketPolicyInput) getBucket() (v string) { return *s.Bucket } +func (s *GetBucketPolicyInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *GetBucketPolicyInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type GetBucketPolicyOutput struct { _ struct{} `type:"structure" payload:"Policy"` @@ -12181,7 +16452,7 @@ func (s *GetBucketPolicyOutput) SetPolicy(v string) *GetBucketPolicyOutput { } type GetBucketPolicyStatusInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"GetBucketPolicyStatusRequest" type:"structure"` // The name of the Amazon S3 bucket whose policy status you want to retrieve. // @@ -12228,6 +16499,20 @@ func (s *GetBucketPolicyStatusInput) getBucket() (v string) { return *s.Bucket } +func (s *GetBucketPolicyStatusInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *GetBucketPolicyStatusInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type GetBucketPolicyStatusOutput struct { _ struct{} `type:"structure" payload:"PolicyStatus"` @@ -12252,8 +16537,10 @@ func (s *GetBucketPolicyStatusOutput) SetPolicyStatus(v *PolicyStatus) *GetBucke } type GetBucketReplicationInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"GetBucketReplicationRequest" type:"structure"` + // The bucket name for which to get the replication information. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` } @@ -12297,6 +16584,20 @@ func (s *GetBucketReplicationInput) getBucket() (v string) { return *s.Bucket } +func (s *GetBucketReplicationInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *GetBucketReplicationInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type GetBucketReplicationOutput struct { _ struct{} `type:"structure" payload:"ReplicationConfiguration"` @@ -12322,8 +16623,10 @@ func (s *GetBucketReplicationOutput) SetReplicationConfiguration(v *ReplicationC } type GetBucketRequestPaymentInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"GetBucketRequestPaymentRequest" type:"structure"` + // The name of the bucket for which to get the payment request configuration + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` } @@ -12367,6 +16670,20 @@ func (s *GetBucketRequestPaymentInput) getBucket() (v string) { return *s.Bucket } +func (s *GetBucketRequestPaymentInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *GetBucketRequestPaymentInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type GetBucketRequestPaymentOutput struct { _ struct{} `type:"structure"` @@ -12391,8 +16708,10 @@ func (s *GetBucketRequestPaymentOutput) SetPayer(v string) *GetBucketRequestPaym } type GetBucketTaggingInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"GetBucketTaggingRequest" type:"structure"` + // The name of the bucket for which to get the tagging information. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` } @@ -12436,9 +16755,25 @@ func (s *GetBucketTaggingInput) getBucket() (v string) { return *s.Bucket } +func (s *GetBucketTaggingInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *GetBucketTaggingInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type GetBucketTaggingOutput struct { _ struct{} `type:"structure"` + // Contains the tag set. + // // TagSet is a required field TagSet []*Tag `locationNameList:"Tag" type:"list" required:"true"` } @@ -12460,8 +16795,10 @@ func (s *GetBucketTaggingOutput) SetTagSet(v []*Tag) *GetBucketTaggingOutput { } type GetBucketVersioningInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"GetBucketVersioningRequest" type:"structure"` + // The name of the bucket for which to get the versioning information. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` } @@ -12505,6 +16842,20 @@ func (s *GetBucketVersioningInput) getBucket() (v string) { return *s.Bucket } +func (s *GetBucketVersioningInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *GetBucketVersioningInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type GetBucketVersioningOutput struct { _ struct{} `type:"structure"` @@ -12540,8 +16891,10 @@ func (s *GetBucketVersioningOutput) SetStatus(v string) *GetBucketVersioningOutp } type GetBucketWebsiteInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"GetBucketWebsiteRequest" type:"structure"` + // The bucket name for which to get the website configuration. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` } @@ -12585,15 +16938,34 @@ func (s *GetBucketWebsiteInput) getBucket() (v string) { return *s.Bucket } +func (s *GetBucketWebsiteInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *GetBucketWebsiteInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type GetBucketWebsiteOutput struct { _ struct{} `type:"structure"` + // The object key name of the website error document to use for 4XX class errors. ErrorDocument *ErrorDocument `type:"structure"` + // The name of the index document for the website (for example index.html). IndexDocument *IndexDocument `type:"structure"` + // Specifies the redirect behavior of all requests to a website endpoint of + // an Amazon S3 bucket. RedirectAllRequestsTo *RedirectAllRequestsTo `type:"structure"` + // Rules that define when a redirect is applied and the redirect behavior. RoutingRules []*RoutingRule `locationNameList:"RoutingRule" type:"list"` } @@ -12632,18 +17004,30 @@ func (s *GetBucketWebsiteOutput) SetRoutingRules(v []*RoutingRule) *GetBucketWeb } type GetObjectAclInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"GetObjectAclRequest" type:"structure"` + // The bucket name that contains the object for which to get the ACL information. + // + // When using this API with an access point, you must direct requests to the + // access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com. + // When using this operation using an access point through the AWS SDKs, you + // provide the access point ARN in place of the bucket name. For more information + // about access point ARNs, see Using Access Points (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-access-points.html) + // in the Amazon Simple Storage Service Developer Guide. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + // The key of the object for which to get the ACL information. + // // Key is a required field Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // Documentation on downloading objects from requester pays buckets can be found - // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` // VersionId used to reference a specific version of the object. @@ -12713,12 +17097,27 @@ func (s *GetObjectAclInput) SetVersionId(v string) *GetObjectAclInput { return s } +func (s *GetObjectAclInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *GetObjectAclInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type GetObjectAclOutput struct { _ struct{} `type:"structure"` // A list of grants. Grants []*Grant `locationName:"AccessControlList" locationNameList:"Grant" type:"list"` + // Container for the bucket owner's display name and ID. Owner *Owner `type:"structure"` // If present, indicates that the requester was successfully charged for the @@ -12755,8 +17154,17 @@ func (s *GetObjectAclOutput) SetRequestCharged(v string) *GetObjectAclOutput { } type GetObjectInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"GetObjectRequest" type:"structure"` + // The bucket name containing the object. + // + // When using this API with an access point, you must direct requests to the + // access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com. + // When using this operation using an access point through the AWS SDKs, you + // provide the access point ARN in place of the bucket name. For more information + // about access point ARNs, see Using Access Points (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-access-points.html) + // in the Amazon Simple Storage Service Developer Guide. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` @@ -12776,6 +17184,8 @@ type GetObjectInput struct { // otherwise return a 412 (precondition failed). IfUnmodifiedSince *time.Time `location:"header" locationName:"If-Unmodified-Since" type:"timestamp"` + // Key of the object to get. + // // Key is a required field Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"` @@ -12785,13 +17195,17 @@ type GetObjectInput struct { PartNumber *int64 `location:"querystring" locationName:"partNumber" type:"integer"` // Downloads the specified range bytes of an object. For more information about - // the HTTP Range header, go to http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35. + // the HTTP Range header, see https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35 + // (https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35). + // + // Amazon S3 doesn't support retrieving multiple ranges of data per GET request. Range *string `location:"header" locationName:"Range" type:"string"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // Documentation on downloading objects from requester pays buckets can be found - // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` // Sets the Cache-Control header of the response. @@ -12812,19 +17226,20 @@ type GetObjectInput struct { // Sets the Expires header of the response. ResponseExpires *time.Time `location:"querystring" locationName:"response-expires" type:"timestamp"` - // Specifies the algorithm to use to when encrypting the object (e.g., AES256). + // Specifies the algorithm to use to when encrypting the object (for example, + // AES256). SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` // Specifies the customer-provided encryption key for Amazon S3 to use in encrypting // data. This value is used to store the object and then it is discarded; Amazon - // does not store the encryption key. The key must be appropriate for use with - // the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm + // S3 does not store the encryption key. The key must be appropriate for use + // with the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm // header. - SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"` + SSECustomerKey *string `marshal-as:"blob" location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"` // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. - // Amazon S3 uses this header for a message integrity check to ensure the encryption - // key was transmitted without error. + // Amazon S3 uses this header for a message integrity check to ensure that the + // encryption key was transmitted without error. SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` // VersionId used to reference a specific version of the object. @@ -12991,10 +17406,32 @@ func (s *GetObjectInput) SetVersionId(v string) *GetObjectInput { return s } +func (s *GetObjectInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *GetObjectInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type GetObjectLegalHoldInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"GetObjectLegalHoldRequest" type:"structure"` - // The bucket containing the object whose Legal Hold status you want to retrieve. + // The bucket name containing the object whose Legal Hold status you want to + // retrieve. + // + // When using this API with an access point, you must direct requests to the + // access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com. + // When using this operation using an access point through the AWS SDKs, you + // provide the access point ARN in place of the bucket name. For more information + // about access point ARNs, see Using Access Points (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-access-points.html) + // in the Amazon Simple Storage Service Developer Guide. // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` @@ -13004,10 +17441,11 @@ type GetObjectLegalHoldInput struct { // Key is a required field Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // Documentation on downloading objects from requester pays buckets can be found - // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` // The version ID of the object whose Legal Hold status you want to retrieve. @@ -13077,6 +17515,20 @@ func (s *GetObjectLegalHoldInput) SetVersionId(v string) *GetObjectLegalHoldInpu return s } +func (s *GetObjectLegalHoldInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *GetObjectLegalHoldInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type GetObjectLegalHoldOutput struct { _ struct{} `type:"structure" payload:"LegalHold"` @@ -13101,7 +17553,7 @@ func (s *GetObjectLegalHoldOutput) SetLegalHold(v *ObjectLockLegalHold) *GetObje } type GetObjectLockConfigurationInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"GetObjectLockConfigurationRequest" type:"structure"` // The bucket whose Object Lock configuration you want to retrieve. // @@ -13148,6 +17600,20 @@ func (s *GetObjectLockConfigurationInput) getBucket() (v string) { return *s.Bucket } +func (s *GetObjectLockConfigurationInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *GetObjectLockConfigurationInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type GetObjectLockConfigurationOutput struct { _ struct{} `type:"structure" payload:"ObjectLockConfiguration"` @@ -13174,6 +17640,7 @@ func (s *GetObjectLockConfigurationOutput) SetObjectLockConfiguration(v *ObjectL type GetObjectOutput struct { _ struct{} `type:"structure" payload:"Body"` + // Indicates that a range of bytes was specified. AcceptRanges *string `location:"header" locationName:"accept-ranges" type:"string"` // Object data. @@ -13207,11 +17674,11 @@ type GetObjectOutput struct { DeleteMarker *bool `location:"header" locationName:"x-amz-delete-marker" type:"boolean"` // An ETag is an opaque identifier assigned by a web server to a specific version - // of a resource found at a URL + // of a resource found at a URL. ETag *string `location:"header" locationName:"ETag" type:"string"` // If the object expiration is configured (see PUT Bucket lifecycle), the response - // includes this header. It includes the expiry-date and rule-id key value pairs + // includes this header. It includes the expiry-date and rule-id key-value pairs // providing object expiration information. The value of the rule-id is URL // encoded. Expiration *string `location:"header" locationName:"x-amz-expiration" type:"string"` @@ -13223,6 +17690,10 @@ type GetObjectOutput struct { LastModified *time.Time `location:"header" locationName:"Last-Modified" type:"timestamp"` // A map of metadata to store with the object in S3. + // + // By default unmarshaled keys are written as a map keys in following canonicalized format: + // the first letter and any letter following a hyphen will be capitalized, and the rest as lowercase. + // Set `aws.Config.LowerCaseHeaderMaps` to `true` to write unmarshaled keys to the map as lowercase. Metadata map[string]*string `location:"headers" locationName:"x-amz-meta-" type:"map"` // This is set to the number of metadata entries not returned in x-amz-meta @@ -13244,6 +17715,8 @@ type GetObjectOutput struct { // The count of parts this object has. PartsCount *int64 `location:"header" locationName:"x-amz-mp-parts-count" type:"integer"` + // Amazon S3 can return this if your request involves a bucket that is either + // a source or destination in a replication rule. ReplicationStatus *string `location:"header" locationName:"x-amz-replication-status" type:"string" enum:"ReplicationStatus"` // If present, indicates that the requester was successfully charged for the @@ -13260,18 +17733,21 @@ type GetObjectOutput struct { SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` // If server-side encryption with a customer-provided encryption key was requested, - // the response will include this header to provide round trip message integrity + // the response will include this header to provide round-trip message integrity // verification of the customer-provided encryption key. SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` - // If present, specifies the ID of the AWS Key Management Service (KMS) master - // encryption key that was used for the object. + // If present, specifies the ID of the AWS Key Management Service (AWS KMS) + // symmetric customer managed customer master key (CMK) that was used for the + // object. SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` - // The Server-side encryption algorithm used when storing this object in S3 - // (e.g., AES256, aws:kms). + // The server-side encryption algorithm used when storing this object in Amazon + // S3 (for example, AES256, aws:kms). ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string" enum:"ServerSideEncryption"` + // Provides storage class information of the object. Amazon S3 returns this + // header for all objects except for S3 Standard storage class objects. StorageClass *string `location:"header" locationName:"x-amz-storage-class" type:"string" enum:"StorageClass"` // The number of tags, if any, on the object. @@ -13483,9 +17959,17 @@ func (s *GetObjectOutput) SetWebsiteRedirectLocation(v string) *GetObjectOutput } type GetObjectRetentionInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"GetObjectRetentionRequest" type:"structure"` - // The bucket containing the object whose retention settings you want to retrieve. + // The bucket name containing the object whose retention settings you want to + // retrieve. + // + // When using this API with an access point, you must direct requests to the + // access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com. + // When using this operation using an access point through the AWS SDKs, you + // provide the access point ARN in place of the bucket name. For more information + // about access point ARNs, see Using Access Points (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-access-points.html) + // in the Amazon Simple Storage Service Developer Guide. // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` @@ -13495,10 +17979,11 @@ type GetObjectRetentionInput struct { // Key is a required field Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // Documentation on downloading objects from requester pays buckets can be found - // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` // The version ID for the object whose retention settings you want to retrieve. @@ -13568,6 +18053,20 @@ func (s *GetObjectRetentionInput) SetVersionId(v string) *GetObjectRetentionInpu return s } +func (s *GetObjectRetentionInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *GetObjectRetentionInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type GetObjectRetentionOutput struct { _ struct{} `type:"structure" payload:"Retention"` @@ -13592,14 +18091,26 @@ func (s *GetObjectRetentionOutput) SetRetention(v *ObjectLockRetention) *GetObje } type GetObjectTaggingInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"GetObjectTaggingRequest" type:"structure"` + // The bucket name containing the object for which to get the tagging information. + // + // When using this API with an access point, you must direct requests to the + // access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com. + // When using this operation using an access point through the AWS SDKs, you + // provide the access point ARN in place of the bucket name. For more information + // about access point ARNs, see Using Access Points (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-access-points.html) + // in the Amazon Simple Storage Service Developer Guide. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + // Object key for which to get the tagging information. + // // Key is a required field Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"` + // The versionId of the object for which to get the tagging information. VersionId *string `location:"querystring" locationName:"versionId" type:"string"` } @@ -13660,12 +18171,29 @@ func (s *GetObjectTaggingInput) SetVersionId(v string) *GetObjectTaggingInput { return s } +func (s *GetObjectTaggingInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *GetObjectTaggingInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type GetObjectTaggingOutput struct { _ struct{} `type:"structure"` + // Contains the tag set. + // // TagSet is a required field TagSet []*Tag `locationNameList:"Tag" type:"list" required:"true"` + // The versionId of the object for which you got the tagging information. VersionId *string `location:"header" locationName:"x-amz-version-id" type:"string"` } @@ -13692,18 +18220,24 @@ func (s *GetObjectTaggingOutput) SetVersionId(v string) *GetObjectTaggingOutput } type GetObjectTorrentInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"GetObjectTorrentRequest" type:"structure"` + // The name of the bucket containing the object for which to get the torrent + // files. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + // The object key for which to get the information. + // // Key is a required field Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // Documentation on downloading objects from requester pays buckets can be found - // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` } @@ -13764,9 +18298,24 @@ func (s *GetObjectTorrentInput) SetRequestPayer(v string) *GetObjectTorrentInput return s } +func (s *GetObjectTorrentInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *GetObjectTorrentInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type GetObjectTorrentOutput struct { _ struct{} `type:"structure" payload:"Body"` + // A Bencoded dictionary as defined by the BitTorrent specification Body io.ReadCloser `type:"blob"` // If present, indicates that the requester was successfully charged for the @@ -13797,7 +18346,7 @@ func (s *GetObjectTorrentOutput) SetRequestCharged(v string) *GetObjectTorrentOu } type GetPublicAccessBlockInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"GetPublicAccessBlockRequest" type:"structure"` // The name of the Amazon S3 bucket whose PublicAccessBlock configuration you // want to retrieve. @@ -13845,6 +18394,20 @@ func (s *GetPublicAccessBlockInput) getBucket() (v string) { return *s.Bucket } +func (s *GetPublicAccessBlockInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *GetPublicAccessBlockInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type GetPublicAccessBlockOutput struct { _ struct{} `type:"structure" payload:"PublicAccessBlockConfiguration"` @@ -13869,10 +18432,11 @@ func (s *GetPublicAccessBlockOutput) SetPublicAccessBlockConfiguration(v *Public return s } +// Container for S3 Glacier job parameters. type GlacierJobParameters struct { _ struct{} `type:"structure"` - // Glacier retrieval tier at which the restore will be processed. + // S3 Glacier retrieval tier at which the restore will be processed. // // Tier is a required field Tier *string `type:"string" required:"true" enum:"Tier"` @@ -13907,9 +18471,11 @@ func (s *GlacierJobParameters) SetTier(v string) *GlacierJobParameters { return s } +// Container for grant information. type Grant struct { _ struct{} `type:"structure"` + // The person being granted permissions. Grantee *Grantee `type:"structure" xmlPrefix:"xsi" xmlURI:"http://www.w3.org/2001/XMLSchema-instance"` // Specifies the permission given to the grantee. @@ -13953,6 +18519,7 @@ func (s *Grant) SetPermission(v string) *Grant { return s } +// Container for the person being granted permissions. type Grantee struct { _ struct{} `type:"structure" xmlPrefix:"xsi" xmlURI:"http://www.w3.org/2001/XMLSchema-instance"` @@ -13960,6 +18527,29 @@ type Grantee struct { DisplayName *string `type:"string"` // Email address of the grantee. + // + // Using email addresses to specify a grantee is only supported in the following + // AWS Regions: + // + // * US East (N. Virginia) + // + // * US West (N. California) + // + // * US West (Oregon) + // + // * Asia Pacific (Singapore) + // + // * Asia Pacific (Sydney) + // + // * Asia Pacific (Tokyo) + // + // * Europe (Ireland) + // + // * South America (São Paulo) + // + // For a list of all the Amazon S3 supported Regions and endpoints, see Regions + // and Endpoints (https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region) + // in the AWS General Reference. EmailAddress *string `type:"string"` // The canonical user ID of the grantee. @@ -14028,8 +18618,10 @@ func (s *Grantee) SetURI(v string) *Grantee { } type HeadBucketInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"HeadBucketRequest" type:"structure"` + // The bucket name. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` } @@ -14073,6 +18665,20 @@ func (s *HeadBucketInput) getBucket() (v string) { return *s.Bucket } +func (s *HeadBucketInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *HeadBucketInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type HeadBucketOutput struct { _ struct{} `type:"structure"` } @@ -14088,8 +18694,10 @@ func (s HeadBucketOutput) GoString() string { } type HeadObjectInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"HeadObjectRequest" type:"structure"` + // The name of the bucket containing the object. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` @@ -14109,6 +18717,8 @@ type HeadObjectInput struct { // otherwise return a 412 (precondition failed). IfUnmodifiedSince *time.Time `location:"header" locationName:"If-Unmodified-Since" type:"timestamp"` + // The object key. + // // Key is a required field Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"` @@ -14119,28 +18729,32 @@ type HeadObjectInput struct { PartNumber *int64 `location:"querystring" locationName:"partNumber" type:"integer"` // Downloads the specified range bytes of an object. For more information about - // the HTTP Range header, go to http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35. + // the HTTP Range header, see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35. + // + // Amazon S3 doesn't support retrieving multiple ranges of data per GET request. Range *string `location:"header" locationName:"Range" type:"string"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // Documentation on downloading objects from requester pays buckets can be found - // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` - // Specifies the algorithm to use to when encrypting the object (e.g., AES256). + // Specifies the algorithm to use to when encrypting the object (for example, + // AES256). SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` // Specifies the customer-provided encryption key for Amazon S3 to use in encrypting // data. This value is used to store the object and then it is discarded; Amazon - // does not store the encryption key. The key must be appropriate for use with - // the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm + // S3 does not store the encryption key. The key must be appropriate for use + // with the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm // header. - SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"` + SSECustomerKey *string `marshal-as:"blob" location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"` // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. - // Amazon S3 uses this header for a message integrity check to ensure the encryption - // key was transmitted without error. + // Amazon S3 uses this header for a message integrity check to ensure that the + // encryption key was transmitted without error. SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` // VersionId used to reference a specific version of the object. @@ -14271,9 +18885,24 @@ func (s *HeadObjectInput) SetVersionId(v string) *HeadObjectInput { return s } +func (s *HeadObjectInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *HeadObjectInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type HeadObjectOutput struct { _ struct{} `type:"structure"` + // Indicates that a range of bytes was specified. AcceptRanges *string `location:"header" locationName:"accept-ranges" type:"string"` // Specifies caching behavior along the request/reply chain. @@ -14301,11 +18930,11 @@ type HeadObjectOutput struct { DeleteMarker *bool `location:"header" locationName:"x-amz-delete-marker" type:"boolean"` // An ETag is an opaque identifier assigned by a web server to a specific version - // of a resource found at a URL + // of a resource found at a URL. ETag *string `location:"header" locationName:"ETag" type:"string"` // If the object expiration is configured (see PUT Bucket lifecycle), the response - // includes this header. It includes the expiry-date and rule-id key value pairs + // includes this header. It includes the expiry-date and rule-id key-value pairs // providing object expiration information. The value of the rule-id is URL // encoded. Expiration *string `location:"header" locationName:"x-amz-expiration" type:"string"` @@ -14317,6 +18946,10 @@ type HeadObjectOutput struct { LastModified *time.Time `location:"header" locationName:"Last-Modified" type:"timestamp"` // A map of metadata to store with the object in S3. + // + // By default unmarshaled keys are written as a map keys in following canonicalized format: + // the first letter and any letter following a hyphen will be capitalized, and the rest as lowercase. + // Set `aws.Config.LowerCaseHeaderMaps` to `true` to write unmarshaled keys to the map as lowercase. Metadata map[string]*string `location:"headers" locationName:"x-amz-meta-" type:"map"` // This is set to the number of metadata entries not returned in x-amz-meta @@ -14325,26 +18958,69 @@ type HeadObjectOutput struct { // you can create metadata whose values are not legal HTTP headers. MissingMeta *int64 `location:"header" locationName:"x-amz-missing-meta" type:"integer"` - // The Legal Hold status for the specified object. + // Specifies whether a legal hold is in effect for this object. This header + // is only returned if the requester has the s3:GetObjectLegalHold permission. + // This header is not returned if the specified version of this object has never + // had a legal hold applied. For more information about S3 Object Lock, see + // Object Lock (https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock.html). ObjectLockLegalHoldStatus *string `location:"header" locationName:"x-amz-object-lock-legal-hold" type:"string" enum:"ObjectLockLegalHoldStatus"` - // The Object Lock mode currently in place for this object. + // The Object Lock mode, if any, that's in effect for this object. This header + // is only returned if the requester has the s3:GetObjectRetention permission. + // For more information about S3 Object Lock, see Object Lock (https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock.html). ObjectLockMode *string `location:"header" locationName:"x-amz-object-lock-mode" type:"string" enum:"ObjectLockMode"` - // The date and time when this object's Object Lock will expire. + // The date and time when the Object Lock retention period expires. This header + // is only returned if the requester has the s3:GetObjectRetention permission. ObjectLockRetainUntilDate *time.Time `location:"header" locationName:"x-amz-object-lock-retain-until-date" type:"timestamp" timestampFormat:"iso8601"` // The count of parts this object has. PartsCount *int64 `location:"header" locationName:"x-amz-mp-parts-count" type:"integer"` + // Amazon S3 can return this header if your request involves a bucket that is + // either a source or destination in a replication rule. + // + // In replication, you have a source bucket on which you configure replication + // and destination bucket where Amazon S3 stores object replicas. When you request + // an object (GetObject) or object metadata (HeadObject) from these buckets, + // Amazon S3 will return the x-amz-replication-status header in the response + // as follows: + // + // * If requesting an object from the source bucket — Amazon S3 will return + // the x-amz-replication-status header if the object in your request is eligible + // for replication. For example, suppose that in your replication configuration, + // you specify object prefix TaxDocs requesting Amazon S3 to replicate objects + // with key prefix TaxDocs. Any objects you upload with this key name prefix, + // for example TaxDocs/document1.pdf, are eligible for replication. For any + // object request with this key name prefix, Amazon S3 will return the x-amz-replication-status + // header with value PENDING, COMPLETED or FAILED indicating object replication + // status. + // + // * If requesting an object from the destination bucket — Amazon S3 will + // return the x-amz-replication-status header with value REPLICA if the object + // in your request is a replica that Amazon S3 created. + // + // For more information, see Replication (https://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html). ReplicationStatus *string `location:"header" locationName:"x-amz-replication-status" type:"string" enum:"ReplicationStatus"` // If present, indicates that the requester was successfully charged for the // request. RequestCharged *string `location:"header" locationName:"x-amz-request-charged" type:"string" enum:"RequestCharged"` - // Provides information about object restoration operation and expiration time - // of the restored object copy. + // If the object is an archived object (an object whose storage class is GLACIER), + // the response includes this header if either the archive restoration is in + // progress (see RestoreObject or an archive copy is already restored. + // + // If an archive copy is already restored, the header value indicates when Amazon + // S3 is scheduled to delete the object copy. For example: + // + // x-amz-restore: ongoing-request="false", expiry-date="Fri, 23 Dec 2012 00:00:00 + // GMT" + // + // If the object restoration is in progress, the header returns the value ongoing-request="true". + // + // For more information about archiving objects, see Transitioning Objects: + // General Considerations (https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html#lifecycle-transition-general-considerations). Restore *string `location:"header" locationName:"x-amz-restore" type:"string"` // If server-side encryption with a customer-provided encryption key was requested, @@ -14353,18 +19029,25 @@ type HeadObjectOutput struct { SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` // If server-side encryption with a customer-provided encryption key was requested, - // the response will include this header to provide round trip message integrity + // the response will include this header to provide round-trip message integrity // verification of the customer-provided encryption key. SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` - // If present, specifies the ID of the AWS Key Management Service (KMS) master - // encryption key that was used for the object. + // If present, specifies the ID of the AWS Key Management Service (AWS KMS) + // symmetric customer managed customer master key (CMK) that was used for the + // object. SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` - // The Server-side encryption algorithm used when storing this object in S3 - // (e.g., AES256, aws:kms). + // If the object is stored using server-side encryption either with an AWS KMS + // customer master key (CMK) or an Amazon S3-managed encryption key, the response + // includes this header with the value of the server-side encryption algorithm + // used when storing this object in Amazon S3 (for example, AES256, aws:kms). ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string" enum:"ServerSideEncryption"` + // Provides storage class information of the object. Amazon S3 returns this + // header for all objects except for S3 Standard storage class objects. + // + // For more information, see Storage Classes (https://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html). StorageClass *string `location:"header" locationName:"x-amz-storage-class" type:"string" enum:"StorageClass"` // Version of the object. @@ -14554,13 +19237,15 @@ func (s *HeadObjectOutput) SetWebsiteRedirectLocation(v string) *HeadObjectOutpu return s } +// Container for the Suffix element. type IndexDocument struct { _ struct{} `type:"structure"` // A suffix that is appended to a request that is for a directory on the website - // endpoint (e.g. if the suffix is index.html and you make a request to samplebucket/images/ - // the data that is returned will be for the object with the key name images/index.html) - // The suffix must not be empty and must not include a slash character. + // endpoint (for example,if the suffix is index.html and you make a request + // to samplebucket/images/ the data that is returned will be for the object + // with the key name images/index.html) The suffix must not be empty and must + // not include a slash character. // // Suffix is a required field Suffix *string `type:"string" required:"true"` @@ -14595,6 +19280,7 @@ func (s *IndexDocument) SetSuffix(v string) *IndexDocument { return s } +// Container element that identifies who initiated the multipart upload. type Initiator struct { _ struct{} `type:"structure"` @@ -14680,6 +19366,9 @@ func (s *InputSerialization) SetParquet(v *ParquetInput) *InputSerialization { return s } +// Specifies the inventory configuration for an Amazon S3 bucket. For more information, +// see GET Bucket inventory (https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGETInventoryConfig.html) +// in the Amazon Simple Storage Service API Reference. type InventoryConfiguration struct { _ struct{} `type:"structure"` @@ -14697,12 +19386,16 @@ type InventoryConfiguration struct { // Id is a required field Id *string `type:"string" required:"true"` - // Specifies which object version(s) to included in the inventory results. + // Object versions to include in the inventory list. If set to All, the list + // includes all the object versions, which adds the version-related fields VersionId, + // IsLatest, and DeleteMarker to the list. If set to Current, the list does + // not contain these version-related fields. // // IncludedObjectVersions is a required field IncludedObjectVersions *string `type:"string" required:"true" enum:"InventoryIncludedObjectVersions"` - // Specifies whether the inventory is enabled or disabled. + // Specifies whether the inventory is enabled or disabled. If set to True, an + // inventory list is generated. If set to False, no inventory list is generated. // // IsEnabled is a required field IsEnabled *bool `type:"boolean" required:"true"` @@ -14808,6 +19501,7 @@ func (s *InventoryConfiguration) SetSchedule(v *InventorySchedule) *InventoryCon return s } +// Specifies the inventory configuration for an Amazon S3 bucket. type InventoryDestination struct { _ struct{} `type:"structure"` @@ -14857,10 +19551,10 @@ func (s *InventoryDestination) SetS3BucketDestination(v *InventoryS3BucketDestin type InventoryEncryption struct { _ struct{} `type:"structure"` - // Specifies the use of SSE-KMS to encrypt delivered Inventory reports. + // Specifies the use of SSE-KMS to encrypt delivered inventory reports. SSEKMS *SSEKMS `locationName:"SSE-KMS" type:"structure"` - // Specifies the use of SSE-S3 to encrypt delivered Inventory reports. + // Specifies the use of SSE-S3 to encrypt delivered inventory reports. SSES3 *SSES3 `locationName:"SSE-S3" type:"structure"` } @@ -14901,6 +19595,8 @@ func (s *InventoryEncryption) SetSSES3(v *SSES3) *InventoryEncryption { return s } +// Specifies an inventory filter. The inventory only includes objects that meet +// the filter's criteria. type InventoryFilter struct { _ struct{} `type:"structure"` @@ -14939,13 +19635,19 @@ func (s *InventoryFilter) SetPrefix(v string) *InventoryFilter { return s } +// Contains the bucket name, file format, bucket owner (optional), and prefix +// (optional) where inventory results are published. type InventoryS3BucketDestination struct { _ struct{} `type:"structure"` - // The ID of the account that owns the destination bucket. + // The account ID that owns the destination S3 bucket. If no account ID is provided, + // the owner is not validated before exporting data. + // + // Although this value is optional, we strongly recommend that you set it to + // help prevent problems if the destination bucket ownership changes. AccountId *string `type:"string"` - // The Amazon resource name (ARN) of the bucket where inventory results will + // The Amazon Resource Name (ARN) of the bucket where inventory results will // be published. // // Bucket is a required field @@ -15032,6 +19734,7 @@ func (s *InventoryS3BucketDestination) SetPrefix(v string) *InventoryS3BucketDes return s } +// Specifies the schedule for generating inventory results. type InventorySchedule struct { _ struct{} `type:"structure"` @@ -15070,6 +19773,7 @@ func (s *InventorySchedule) SetFrequency(v string) *InventorySchedule { return s } +// Specifies JSON as object's input serialization format. type JSONInput struct { _ struct{} `type:"structure"` @@ -15093,10 +19797,12 @@ func (s *JSONInput) SetType(v string) *JSONInput { return s } +// Specifies JSON as request's output serialization format. type JSONOutput struct { _ struct{} `type:"structure"` - // The value used to separate individual records in the output. + // The value used to separate individual records in the output. If no value + // is specified, Amazon S3 uses a newline character ('\n'). RecordDelimiter *string `type:"string"` } @@ -15120,7 +19826,7 @@ func (s *JSONOutput) SetRecordDelimiter(v string) *JSONOutput { type KeyFilter struct { _ struct{} `type:"structure"` - // A list of containers for the key value pair that defines the criteria for + // A list of containers for the key-value pair that defines the criteria for // the filter rule. FilterRules []*FilterRule `locationName:"FilterRule" type:"list" flattened:"true"` } @@ -15145,11 +19851,15 @@ func (s *KeyFilter) SetFilterRules(v []*FilterRule) *KeyFilter { type LambdaFunctionConfiguration struct { _ struct{} `type:"structure"` + // The Amazon S3 bucket event for which to invoke the AWS Lambda function. For + // more information, see Supported Event Types (https://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html) + // in the Amazon Simple Storage Service Developer Guide. + // // Events is a required field Events []*string `locationName:"Event" type:"list" flattened:"true" required:"true"` - // A container for object key name filtering rules. For information about key - // name filtering, see Configuring Event Notifications (http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html) + // Specifies object key name filtering rules. For information about key name + // filtering, see Configuring Event Notifications (https://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html) // in the Amazon Simple Storage Service Developer Guide. Filter *NotificationConfigurationFilter `type:"structure"` @@ -15157,8 +19867,8 @@ type LambdaFunctionConfiguration struct { // If you don't provide one, Amazon S3 will assign an ID. Id *string `type:"string"` - // The Amazon Resource Name (ARN) of the Lambda cloud function that Amazon S3 - // can invoke when it detects events of the specified type. + // The Amazon Resource Name (ARN) of the AWS Lambda function that Amazon S3 + // invokes when the specified event type occurs. // // LambdaFunctionArn is a required field LambdaFunctionArn *string `locationName:"CloudFunction" type:"string" required:"true"` @@ -15214,9 +19924,12 @@ func (s *LambdaFunctionConfiguration) SetLambdaFunctionArn(v string) *LambdaFunc return s } +// Container for lifecycle rules. You can add as many as 1000 rules. type LifecycleConfiguration struct { _ struct{} `type:"structure"` + // Specifies lifecycle configuration rules for an Amazon S3 bucket. + // // Rules is a required field Rules []*Rule `locationName:"Rule" type:"list" flattened:"true" required:"true"` } @@ -15260,6 +19973,7 @@ func (s *LifecycleConfiguration) SetRules(v []*Rule) *LifecycleConfiguration { return s } +// Container for the expiration for the lifecycle of the object. type LifecycleExpiration struct { _ struct{} `type:"structure"` @@ -15306,13 +20020,19 @@ func (s *LifecycleExpiration) SetExpiredObjectDeleteMarker(v bool) *LifecycleExp return s } +// A lifecycle rule for individual objects in an Amazon S3 bucket. type LifecycleRule struct { _ struct{} `type:"structure"` - // Specifies the days since the initiation of an Incomplete Multipart Upload - // that Lifecycle will wait before permanently removing all parts of the upload. + // Specifies the days since the initiation of an incomplete multipart upload + // that Amazon S3 will wait before permanently removing all parts of the upload. + // For more information, see Aborting Incomplete Multipart Uploads Using a Bucket + // Lifecycle Policy (https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuoverview.html#mpu-abort-incomplete-mpu-lifecycle-config) + // in the Amazon Simple Storage Service Developer Guide. AbortIncompleteMultipartUpload *AbortIncompleteMultipartUpload `type:"structure"` + // Specifies the expiration for the lifecycle of the object in the form of date, + // days and, whether the object has a delete marker. Expiration *LifecycleExpiration `type:"structure"` // The Filter is used to identify objects that a Lifecycle Rule applies to. @@ -15329,10 +20049,15 @@ type LifecycleRule struct { // period in the object's lifetime. NoncurrentVersionExpiration *NoncurrentVersionExpiration `type:"structure"` + // Specifies the transition rule for the lifecycle rule that describes when + // noncurrent objects transition to a specific storage class. If your bucket + // is versioning-enabled (or versioning is suspended), you can set this action + // to request that Amazon S3 transition noncurrent object versions to a specific + // storage class at a set period in the object's lifetime. NoncurrentVersionTransitions []*NoncurrentVersionTransition `locationName:"NoncurrentVersionTransition" type:"list" flattened:"true"` // Prefix identifying one or more objects to which the rule applies. This is - // deprecated; use Filter instead. + // No longer used; use Filter instead. // // Deprecated: Prefix has been deprecated Prefix *string `deprecated:"true" type:"string"` @@ -15343,6 +20068,7 @@ type LifecycleRule struct { // Status is a required field Status *string `type:"string" required:"true" enum:"ExpirationStatus"` + // Specifies when an Amazon S3 object transitions to a specified storage class. Transitions []*Transition `locationName:"Transition" type:"list" flattened:"true"` } @@ -15434,6 +20160,7 @@ func (s *LifecycleRule) SetTransitions(v []*Transition) *LifecycleRule { type LifecycleRuleAndOperator struct { _ struct{} `type:"structure"` + // Prefix identifying one or more objects to which the rule applies. Prefix *string `type:"string"` // All of these tags must exist in the object's tag set in order for the rule @@ -15549,7 +20276,7 @@ func (s *LifecycleRuleFilter) SetTag(v *Tag) *LifecycleRuleFilter { } type ListBucketAnalyticsConfigurationsInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"ListBucketAnalyticsConfigurationsRequest" type:"structure"` // The name of the bucket from which analytics configurations are retrieved. // @@ -15606,13 +20333,28 @@ func (s *ListBucketAnalyticsConfigurationsInput) SetContinuationToken(v string) return s } +func (s *ListBucketAnalyticsConfigurationsInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *ListBucketAnalyticsConfigurationsInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type ListBucketAnalyticsConfigurationsOutput struct { _ struct{} `type:"structure"` // The list of analytics configurations for a bucket. AnalyticsConfigurationList []*AnalyticsConfiguration `locationName:"AnalyticsConfiguration" type:"list" flattened:"true"` - // The ContinuationToken that represents where this request began. + // The marker that is used as a starting point for this analytics configuration + // list response. This value is present if it was sent in the request. ContinuationToken *string `type:"string"` // Indicates whether the returned list of analytics configurations is complete. @@ -15661,7 +20403,7 @@ func (s *ListBucketAnalyticsConfigurationsOutput) SetNextContinuationToken(v str } type ListBucketInventoryConfigurationsInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"ListBucketInventoryConfigurationsRequest" type:"structure"` // The name of the bucket containing the inventory configurations to retrieve. // @@ -15720,6 +20462,20 @@ func (s *ListBucketInventoryConfigurationsInput) SetContinuationToken(v string) return s } +func (s *ListBucketInventoryConfigurationsInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *ListBucketInventoryConfigurationsInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type ListBucketInventoryConfigurationsOutput struct { _ struct{} `type:"structure"` @@ -15730,8 +20486,9 @@ type ListBucketInventoryConfigurationsOutput struct { // The list of inventory configurations for a bucket. InventoryConfigurationList []*InventoryConfiguration `locationName:"InventoryConfiguration" type:"list" flattened:"true"` - // Indicates whether the returned list of inventory configurations is truncated - // in this response. A value of true indicates that the list is truncated. + // Tells whether the returned list of inventory configurations is complete. + // A value of true indicates that the list is not complete and the NextContinuationToken + // is provided for a subsequent request. IsTruncated *bool `type:"boolean"` // The marker used to continue this inventory configuration listing. Use the @@ -15775,7 +20532,7 @@ func (s *ListBucketInventoryConfigurationsOutput) SetNextContinuationToken(v str } type ListBucketMetricsConfigurationsInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"ListBucketMetricsConfigurationsRequest" type:"structure"` // The name of the bucket containing the metrics configurations to retrieve. // @@ -15834,6 +20591,20 @@ func (s *ListBucketMetricsConfigurationsInput) SetContinuationToken(v string) *L return s } +func (s *ListBucketMetricsConfigurationsInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *ListBucketMetricsConfigurationsInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type ListBucketMetricsConfigurationsOutput struct { _ struct{} `type:"structure"` @@ -15907,8 +20678,10 @@ func (s ListBucketsInput) GoString() string { type ListBucketsOutput struct { _ struct{} `type:"structure"` + // The list of buckets owned by the requestor. Buckets []*Bucket `locationNameList:"Bucket" type:"list"` + // The owner of the buckets listed. Owner *Owner `type:"structure"` } @@ -15935,12 +20708,28 @@ func (s *ListBucketsOutput) SetOwner(v *Owner) *ListBucketsOutput { } type ListMultipartUploadsInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"ListMultipartUploadsRequest" type:"structure"` + // Name of the bucket to which the multipart upload was initiated. + // + // When using this API with an access point, you must direct requests to the + // access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com. + // When using this operation using an access point through the AWS SDKs, you + // provide the access point ARN in place of the bucket name. For more information + // about access point ARNs, see Using Access Points (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-access-points.html) + // in the Amazon Simple Storage Service Developer Guide. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` // Character you use to group keys. + // + // All keys that contain the same string between the prefix, if specified, and + // the first occurrence of the delimiter after the prefix are grouped under + // a single result element, CommonPrefixes. If you don't specify the prefix + // parameter, then the substring starts at the beginning of the key. The keys + // that are grouped under CommonPrefixes result element are not returned elsewhere + // in the response. Delimiter *string `location:"querystring" locationName:"delimiter" type:"string"` // Requests Amazon S3 to encode the object keys in the response and specifies @@ -15953,6 +20742,13 @@ type ListMultipartUploadsInput struct { // Together with upload-id-marker, this parameter specifies the multipart upload // after which listing should begin. + // + // If upload-id-marker is not specified, only the keys lexicographically greater + // than the specified key-marker will be included in the list. + // + // If upload-id-marker is specified, any multipart uploads for a key equal to + // the key-marker might also be included, provided those multipart uploads have + // upload IDs lexicographically greater than the specified upload-id-marker. KeyMarker *string `location:"querystring" locationName:"key-marker" type:"string"` // Sets the maximum number of multipart uploads, from 1 to 1,000, to return @@ -15961,12 +20757,16 @@ type ListMultipartUploadsInput struct { MaxUploads *int64 `location:"querystring" locationName:"max-uploads" type:"integer"` // Lists in-progress uploads only for those keys that begin with the specified - // prefix. + // prefix. You can use prefixes to separate a bucket into different grouping + // of keys. (You can think of using prefix to make groups in the same way you'd + // use a folder in a file system.) Prefix *string `location:"querystring" locationName:"prefix" type:"string"` // Together with key-marker, specifies the multipart upload after which listing // should begin. If key-marker is not specified, the upload-id-marker parameter - // is ignored. + // is ignored. Otherwise, any multipart uploads for a key equal to the key-marker + // might be included in the list only if they have an upload ID lexicographically + // greater than the specified upload-id-marker. UploadIdMarker *string `location:"querystring" locationName:"upload-id-marker" type:"string"` } @@ -16045,17 +20845,42 @@ func (s *ListMultipartUploadsInput) SetUploadIdMarker(v string) *ListMultipartUp return s } +func (s *ListMultipartUploadsInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *ListMultipartUploadsInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type ListMultipartUploadsOutput struct { _ struct{} `type:"structure"` // Name of the bucket to which the multipart upload was initiated. Bucket *string `type:"string"` + // If you specify a delimiter in the request, then the result returns each distinct + // key prefix containing the delimiter in a CommonPrefixes element. The distinct + // key prefixes are returned in the Prefix child element. CommonPrefixes []*CommonPrefix `type:"list" flattened:"true"` + // Contains the delimiter you specified in the request. If you don't specify + // a delimiter in your request, this element is absent from the response. Delimiter *string `type:"string"` // Encoding type used by Amazon S3 to encode object keys in the response. + // + // If you specify encoding-type request parameter, Amazon S3 includes this element + // in the response, and returns encoded key name values in the following response + // elements: + // + // Delimiter, KeyMarker, Prefix, NextKeyMarker, Key. EncodingType *string `type:"string" enum:"EncodingType"` // Indicates whether the returned list of multipart uploads is truncated. A @@ -16086,6 +20911,8 @@ type ListMultipartUploadsOutput struct { // Upload ID after which listing began. UploadIdMarker *string `type:"string"` + // Container for elements related to a particular multipart upload. A response + // can contain zero or more Upload elements. Uploads []*MultipartUpload `locationName:"Upload" type:"list" flattened:"true"` } @@ -16179,12 +21006,25 @@ func (s *ListMultipartUploadsOutput) SetUploads(v []*MultipartUpload) *ListMulti } type ListObjectVersionsInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"ListObjectVersionsRequest" type:"structure"` + // The bucket name that contains the objects. + // + // When using this API with an access point, you must direct requests to the + // access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com. + // When using this operation using an access point through the AWS SDKs, you + // provide the access point ARN in place of the bucket name. For more information + // about access point ARNs, see Using Access Points (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-access-points.html) + // in the Amazon Simple Storage Service Developer Guide. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` - // A delimiter is a character you use to group keys. + // A delimiter is a character that you specify to group keys. All keys that + // contain the same string between the prefix and the first occurrence of the + // delimiter are grouped under a single result element in CommonPrefixes. These + // groups are counted as one result against the max-keys limitation. These keys + // are not returned elsewhere in the response. Delimiter *string `location:"querystring" locationName:"delimiter" type:"string"` // Requests Amazon S3 to encode the object keys in the response and specifies @@ -16198,11 +21038,19 @@ type ListObjectVersionsInput struct { // Specifies the key to start with when listing objects in a bucket. KeyMarker *string `location:"querystring" locationName:"key-marker" type:"string"` - // Sets the maximum number of keys returned in the response. The response might - // contain fewer keys but will never contain more. + // Sets the maximum number of keys returned in the response. By default the + // API returns up to 1,000 key names. The response might contain fewer keys + // but will never contain more. If additional keys satisfy the search criteria, + // but were not returned because max-keys was exceeded, the response contains + // true. To return the additional keys, see key-marker + // and version-id-marker. MaxKeys *int64 `location:"querystring" locationName:"max-keys" type:"integer"` - // Limits the response to keys that begin with the specified prefix. + // Use this parameter to select only those keys that begin with the specified + // prefix. You can use prefixes to separate a bucket into different groupings + // of keys. (You can think of using prefix to make groups in the same way you'd + // use a folder in a file system.) You can use prefix with delimiter to roll + // up numerous objects into a single result under CommonPrefixes. Prefix *string `location:"querystring" locationName:"prefix" type:"string"` // Specifies the object version you want to start listing from. @@ -16284,42 +21132,81 @@ func (s *ListObjectVersionsInput) SetVersionIdMarker(v string) *ListObjectVersio return s } +func (s *ListObjectVersionsInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *ListObjectVersionsInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type ListObjectVersionsOutput struct { _ struct{} `type:"structure"` + // All of the keys rolled up into a common prefix count as a single return when + // calculating the number of returns. CommonPrefixes []*CommonPrefix `type:"list" flattened:"true"` + // Container for an object that is a delete marker. DeleteMarkers []*DeleteMarkerEntry `locationName:"DeleteMarker" type:"list" flattened:"true"` + // The delimiter grouping the included keys. A delimiter is a character that + // you specify to group keys. All keys that contain the same string between + // the prefix and the first occurrence of the delimiter are grouped under a + // single result element in CommonPrefixes. These groups are counted as one + // result against the max-keys limitation. These keys are not returned elsewhere + // in the response. Delimiter *string `type:"string"` - // Encoding type used by Amazon S3 to encode object keys in the response. + // Encoding type used by Amazon S3 to encode object key names in the XML response. + // + // If you specify encoding-type request parameter, Amazon S3 includes this element + // in the response, and returns encoded key name values in the following response + // elements: + // + // KeyMarker, NextKeyMarker, Prefix, Key, and Delimiter. EncodingType *string `type:"string" enum:"EncodingType"` - // A flag that indicates whether or not Amazon S3 returned all of the results - // that satisfied the search criteria. If your results were truncated, you can - // make a follow-up paginated request using the NextKeyMarker and NextVersionIdMarker + // A flag that indicates whether Amazon S3 returned all of the results that + // satisfied the search criteria. If your results were truncated, you can make + // a follow-up paginated request using the NextKeyMarker and NextVersionIdMarker // response parameters as a starting place in another request to return the // rest of the results. IsTruncated *bool `type:"boolean"` - // Marks the last Key returned in a truncated response. + // Marks the last key returned in a truncated response. KeyMarker *string `type:"string"` + // Specifies the maximum number of objects to return. MaxKeys *int64 `type:"integer"` + // Bucket name. Name *string `type:"string"` - // Use this value for the key marker request parameter in a subsequent request. + // When the number of responses exceeds the value of MaxKeys, NextKeyMarker + // specifies the first key not returned that satisfies the search criteria. + // Use this value for the key-marker request parameter in a subsequent request. NextKeyMarker *string `type:"string"` - // Use this value for the next version id marker parameter in a subsequent request. + // When the number of responses exceeds the value of MaxKeys, NextVersionIdMarker + // specifies the first object version not returned that satisfies the search + // criteria. Use this value for the version-id-marker request parameter in a + // subsequent request. NextVersionIdMarker *string `type:"string"` + // Selects objects that start with the value supplied by this parameter. Prefix *string `type:"string"` + // Marks the last version of the key returned in a truncated response. VersionIdMarker *string `type:"string"` + // Container for version information. Versions []*ObjectVersion `locationName:"Version" type:"list" flattened:"true"` } @@ -16412,8 +21299,10 @@ func (s *ListObjectVersionsOutput) SetVersions(v []*ObjectVersion) *ListObjectVe } type ListObjectsInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"ListObjectsRequest" type:"structure"` + // The name of the bucket containing the objects. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` @@ -16431,8 +21320,9 @@ type ListObjectsInput struct { // Specifies the key to start with when listing objects in a bucket. Marker *string `location:"querystring" locationName:"marker" type:"string"` - // Sets the maximum number of keys returned in the response. The response might - // contain fewer keys but will never contain more. + // Sets the maximum number of keys returned in the response. By default the + // API returns up to 1,000 key names. The response might contain fewer keys + // but will never contain more. MaxKeys *int64 `location:"querystring" locationName:"max-keys" type:"integer"` // Limits the response to keys that begin with the specified prefix. @@ -16519,26 +21409,65 @@ func (s *ListObjectsInput) SetRequestPayer(v string) *ListObjectsInput { return s } +func (s *ListObjectsInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *ListObjectsInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type ListObjectsOutput struct { _ struct{} `type:"structure"` + // All of the keys rolled up in a common prefix count as a single return when + // calculating the number of returns. + // + // A response can contain CommonPrefixes only if you specify a delimiter. + // + // CommonPrefixes contains all (if there are any) keys between Prefix and the + // next occurrence of the string specified by the delimiter. + // + // CommonPrefixes lists keys that act like subdirectories in the directory specified + // by Prefix. + // + // For example, if the prefix is notes/ and the delimiter is a slash (/) as + // in notes/summer/july, the common prefix is notes/summer/. All of the keys + // that roll up into a common prefix count as a single return when calculating + // the number of returns. CommonPrefixes []*CommonPrefix `type:"list" flattened:"true"` + // Metadata about each object returned. Contents []*Object `type:"list" flattened:"true"` + // Causes keys that contain the same string between the prefix and the first + // occurrence of the delimiter to be rolled up into a single result element + // in the CommonPrefixes collection. These rolled-up keys are not returned elsewhere + // in the response. Each rolled-up result counts as only one return against + // the MaxKeys value. Delimiter *string `type:"string"` // Encoding type used by Amazon S3 to encode object keys in the response. EncodingType *string `type:"string" enum:"EncodingType"` - // A flag that indicates whether or not Amazon S3 returned all of the results - // that satisfied the search criteria. + // A flag that indicates whether Amazon S3 returned all of the results that + // satisfied the search criteria. IsTruncated *bool `type:"boolean"` + // Indicates where in the bucket listing begins. Marker is included in the response + // if it was sent with the request. Marker *string `type:"string"` + // The maximum number of keys returned in the response body. MaxKeys *int64 `type:"integer"` + // Bucket name. Name *string `type:"string"` // When response is truncated (the IsTruncated element value in the response @@ -16550,6 +21479,7 @@ type ListObjectsOutput struct { // subsequent request to get the next set of object keys. NextMarker *string `type:"string"` + // Keys that begin with the indicated prefix. Prefix *string `type:"string"` } @@ -16624,16 +21554,23 @@ func (s *ListObjectsOutput) SetPrefix(v string) *ListObjectsOutput { } type ListObjectsV2Input struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"ListObjectsV2Request" type:"structure"` - // Name of the bucket to list. + // Bucket name to list. + // + // When using this API with an access point, you must direct requests to the + // access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com. + // When using this operation using an access point through the AWS SDKs, you + // provide the access point ARN in place of the bucket name. For more information + // about access point ARNs, see Using Access Points (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-access-points.html) + // in the Amazon Simple Storage Service Developer Guide. // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` // ContinuationToken indicates Amazon S3 that the list is being continued on // this bucket with a token. ContinuationToken is obfuscated and is not a real - // key + // key. ContinuationToken *string `location:"querystring" locationName:"continuation-token" type:"string"` // A delimiter is a character you use to group keys. @@ -16644,11 +21581,12 @@ type ListObjectsV2Input struct { // The owner field is not present in listV2 by default, if you want to return // owner field with each key in the result then set the fetch owner field to - // true + // true. FetchOwner *bool `location:"querystring" locationName:"fetch-owner" type:"boolean"` - // Sets the maximum number of keys returned in the response. The response might - // contain fewer keys but will never contain more. + // Sets the maximum number of keys returned in the response. By default the + // API returns up to 1,000 key names. The response might contain fewer keys + // but will never contain more. MaxKeys *int64 `location:"querystring" locationName:"max-keys" type:"integer"` // Limits the response to keys that begin with the specified prefix. @@ -16660,7 +21598,7 @@ type ListObjectsV2Input struct { RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` // StartAfter is where you want Amazon S3 to start listing from. Amazon S3 starts - // listing after this specified key. StartAfter can be any key in the bucket + // listing after this specified key. StartAfter can be any key in the bucket. StartAfter *string `location:"querystring" locationName:"start-after" type:"string"` } @@ -16751,29 +21689,65 @@ func (s *ListObjectsV2Input) SetStartAfter(v string) *ListObjectsV2Input { return s } +func (s *ListObjectsV2Input) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *ListObjectsV2Input) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type ListObjectsV2Output struct { _ struct{} `type:"structure"` + // All of the keys rolled up into a common prefix count as a single return when + // calculating the number of returns. + // + // A response can contain CommonPrefixes only if you specify a delimiter. + // // CommonPrefixes contains all (if there are any) keys between Prefix and the - // next occurrence of the string specified by delimiter + // next occurrence of the string specified by a delimiter. + // + // CommonPrefixes lists keys that act like subdirectories in the directory specified + // by Prefix. + // + // For example, if the prefix is notes/ and the delimiter is a slash (/) as + // in notes/summer/july, the common prefix is notes/summer/. All of the keys + // that roll up into a common prefix count as a single return when calculating + // the number of returns. CommonPrefixes []*CommonPrefix `type:"list" flattened:"true"` // Metadata about each object returned. Contents []*Object `type:"list" flattened:"true"` - // ContinuationToken indicates Amazon S3 that the list is being continued on - // this bucket with a token. ContinuationToken is obfuscated and is not a real - // key + // If ContinuationToken was sent with the request, it is included in the response. ContinuationToken *string `type:"string"` - // A delimiter is a character you use to group keys. + // Causes keys that contain the same string between the prefix and the first + // occurrence of the delimiter to be rolled up into a single result element + // in the CommonPrefixes collection. These rolled-up keys are not returned elsewhere + // in the response. Each rolled-up result counts as only one return against + // the MaxKeys value. Delimiter *string `type:"string"` - // Encoding type used by Amazon S3 to encode object keys in the response. + // Encoding type used by Amazon S3 to encode object key names in the XML response. + // + // If you specify the encoding-type request parameter, Amazon S3 includes this + // element in the response, and returns encoded key name values in the following + // response elements: + // + // Delimiter, Prefix, Key, and StartAfter. EncodingType *string `type:"string" enum:"EncodingType"` - // A flag that indicates whether or not Amazon S3 returned all of the results - // that satisfied the search criteria. + // Set to false if all of the results were returned. Set to true if more keys + // are available to return. If the number of results exceeds that specified + // by MaxKeys, all of the results might not be returned. IsTruncated *bool `type:"boolean"` // KeyCount is the number of keys returned with this request. KeyCount will @@ -16781,24 +21755,31 @@ type ListObjectsV2Output struct { // result will include less than equals 50 keys KeyCount *int64 `type:"integer"` - // Sets the maximum number of keys returned in the response. The response might - // contain fewer keys but will never contain more. + // Sets the maximum number of keys returned in the response. By default the + // API returns up to 1,000 key names. The response might contain fewer keys + // but will never contain more. MaxKeys *int64 `type:"integer"` - // Name of the bucket to list. + // Bucket name. + // + // When using this API with an access point, you must direct requests to the + // access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com. + // When using this operation using an access point through the AWS SDKs, you + // provide the access point ARN in place of the bucket name. For more information + // about access point ARNs, see Using Access Points (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-access-points.html) + // in the Amazon Simple Storage Service Developer Guide. Name *string `type:"string"` - // NextContinuationToken is sent when isTruncated is true which means there + // NextContinuationToken is sent when isTruncated is true, which means there // are more keys in the bucket that can be listed. The next list requests to // Amazon S3 can be continued with this NextContinuationToken. NextContinuationToken // is obfuscated and is not a real key NextContinuationToken *string `type:"string"` - // Limits the response to keys that begin with the specified prefix. + // Keys that begin with the indicated prefix. Prefix *string `type:"string"` - // StartAfter is where you want Amazon S3 to start listing from. Amazon S3 starts - // listing after this specified key. StartAfter can be any key in the bucket + // If StartAfter was sent with the request, it is included in the response. StartAfter *string `type:"string"` } @@ -16885,11 +21866,22 @@ func (s *ListObjectsV2Output) SetStartAfter(v string) *ListObjectsV2Output { } type ListPartsInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"ListPartsRequest" type:"structure"` + // Name of the bucket to which the parts are being uploaded. + // + // When using this API with an access point, you must direct requests to the + // access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com. + // When using this operation using an access point through the AWS SDKs, you + // provide the access point ARN in place of the bucket name. For more information + // about access point ARNs, see Using Access Points (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-access-points.html) + // in the Amazon Simple Storage Service Developer Guide. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + // Object key for which the multipart upload was initiated. + // // Key is a required field Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"` @@ -16900,10 +21892,11 @@ type ListPartsInput struct { // part numbers will be listed. PartNumberMarker *int64 `location:"querystring" locationName:"part-number-marker" type:"integer"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // Documentation on downloading objects from requester pays buckets can be found - // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` // Upload ID identifying the multipart upload whose parts are being listed. @@ -16990,23 +21983,51 @@ func (s *ListPartsInput) SetUploadId(v string) *ListPartsInput { return s } +func (s *ListPartsInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *ListPartsInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type ListPartsOutput struct { _ struct{} `type:"structure"` - // Date when multipart upload will become eligible for abort operation by lifecycle. + // If the bucket has a lifecycle rule configured with an action to abort incomplete + // multipart uploads and the prefix in the lifecycle rule matches the object + // name in the request, then the response includes this header indicating when + // the initiated multipart upload will become eligible for abort operation. + // For more information, see Aborting Incomplete Multipart Uploads Using a Bucket + // Lifecycle Policy (https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuoverview.html#mpu-abort-incomplete-mpu-lifecycle-config). + // + // The response will also include the x-amz-abort-rule-id header that will provide + // the ID of the lifecycle configuration rule that defines this action. AbortDate *time.Time `location:"header" locationName:"x-amz-abort-date" type:"timestamp"` - // Id of the lifecycle rule that makes a multipart upload eligible for abort - // operation. + // This header is returned along with the x-amz-abort-date header. It identifies + // applicable lifecycle configuration rule that defines the action to abort + // incomplete multipart uploads. AbortRuleId *string `location:"header" locationName:"x-amz-abort-rule-id" type:"string"` // Name of the bucket to which the multipart upload was initiated. Bucket *string `type:"string"` - // Identifies who initiated the multipart upload. + // Container element that identifies who initiated the multipart upload. If + // the initiator is an AWS account, this element provides the same information + // as the Owner element. If the initiator is an IAM User, this element provides + // the user ARN and display name. Initiator *Initiator `type:"structure"` - // Indicates whether the returned list of parts is truncated. + // Indicates whether the returned list of parts is truncated. A true value indicates + // that the list was truncated. A list can be truncated if the number of parts + // exceeds the limit returned in the MaxParts element. IsTruncated *bool `type:"boolean"` // Object key for which the multipart upload was initiated. @@ -17020,18 +22041,26 @@ type ListPartsOutput struct { // in a subsequent request. NextPartNumberMarker *int64 `type:"integer"` + // Container element that identifies the object owner, after the object is created. + // If multipart upload is initiated by an IAM user, this element provides the + // parent account ID and display name. Owner *Owner `type:"structure"` - // Part number after which listing begins. + // When a list is truncated, this element specifies the last part in the list, + // as well as the value to use for the part-number-marker request parameter + // in a subsequent request. PartNumberMarker *int64 `type:"integer"` + // Container for elements related to a particular part. A response can contain + // zero or more Part elements. Parts []*Part `locationName:"Part" type:"list" flattened:"true"` // If present, indicates that the requester was successfully charged for the // request. RequestCharged *string `location:"header" locationName:"x-amz-request-charged" type:"string" enum:"RequestCharged"` - // The class of storage used to store the object. + // Class of storage (STANDARD or REDUCED_REDUNDANCY) used to store the uploaded + // object. StorageClass *string `type:"string" enum:"StorageClass"` // Upload ID identifying the multipart upload whose parts are being listed. @@ -17139,7 +22168,8 @@ func (s *ListPartsOutput) SetUploadId(v string) *ListPartsOutput { return s } -// Describes an S3 location that will receive the results of the restore request. +// Describes an Amazon S3 location that will receive the results of the restore +// request. type Location struct { _ struct{} `type:"structure"` @@ -17154,8 +22184,7 @@ type Location struct { // The canned ACL to apply to the restore results. CannedACL *string `type:"string" enum:"ObjectCannedACL"` - // Describes the server-side encryption that will be applied to the restore - // results. + // Contains the type of server-side encryption used. Encryption *Encryption `type:"structure"` // The prefix that is prepended to the restore results for this request. @@ -17267,26 +22296,29 @@ func (s *Location) SetUserMetadata(v []*MetadataEntry) *Location { return s } -// Container for logging information. Presence of this element indicates that -// logging is enabled. Parameters TargetBucket and TargetPrefix are required -// in this case. +// Describes where logs are stored and the prefix that Amazon S3 assigns to +// all log object keys for a bucket. For more information, see PUT Bucket logging +// (https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUTlogging.html) +// in the Amazon Simple Storage Service API Reference. type LoggingEnabled struct { _ struct{} `type:"structure"` // Specifies the bucket where you want Amazon S3 to store server access logs. // You can have your logs delivered to any bucket that you own, including the // same bucket that is being logged. You can also configure multiple buckets - // to deliver their logs to the same target bucket. In this case you should + // to deliver their logs to the same target bucket. In this case, you should // choose a different TargetPrefix for each source bucket so that the delivered // log files can be distinguished by key. // // TargetBucket is a required field TargetBucket *string `type:"string" required:"true"` + // Container for granting information. TargetGrants []*TargetGrant `locationNameList:"Grant" type:"list"` - // This element lets you specify a prefix for the keys that the log files will - // be stored under. + // A prefix for all log object keys. If you store log files from multiple Amazon + // S3 buckets in a single bucket, you can use a prefix to distinguish which + // log files came from which bucket. // // TargetPrefix is a required field TargetPrefix *string `type:"string" required:"true"` @@ -17350,8 +22382,10 @@ func (s *LoggingEnabled) SetTargetPrefix(v string) *LoggingEnabled { type MetadataEntry struct { _ struct{} `type:"structure"` + // Name of the Object. Name *string `type:"string"` + // Value of the Object. Value *string `type:"string"` } @@ -17377,6 +22411,65 @@ func (s *MetadataEntry) SetValue(v string) *MetadataEntry { return s } +// A container specifying replication metrics-related settings enabling metrics +// and Amazon S3 events for S3 Replication Time Control (S3 RTC). Must be specified +// together with a ReplicationTime block. +type Metrics struct { + _ struct{} `type:"structure"` + + // A container specifying the time threshold for emitting the s3:Replication:OperationMissedThreshold + // event. + // + // EventThreshold is a required field + EventThreshold *ReplicationTimeValue `type:"structure" required:"true"` + + // Specifies whether the replication metrics are enabled. + // + // Status is a required field + Status *string `type:"string" required:"true" enum:"MetricsStatus"` +} + +// String returns the string representation +func (s Metrics) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s Metrics) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *Metrics) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "Metrics"} + if s.EventThreshold == nil { + invalidParams.Add(request.NewErrParamRequired("EventThreshold")) + } + if s.Status == nil { + invalidParams.Add(request.NewErrParamRequired("Status")) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetEventThreshold sets the EventThreshold field's value. +func (s *Metrics) SetEventThreshold(v *ReplicationTimeValue) *Metrics { + s.EventThreshold = v + return s +} + +// SetStatus sets the Status field's value. +func (s *Metrics) SetStatus(v string) *Metrics { + s.Status = &v + return s +} + +// A conjunction (logical AND) of predicates, which is used in evaluating a +// metrics filter. The operator must have at least two predicates, and an object +// must match all of the predicates in order for the filter to apply. type MetricsAndOperator struct { _ struct{} `type:"structure"` @@ -17429,6 +22522,13 @@ func (s *MetricsAndOperator) SetTags(v []*Tag) *MetricsAndOperator { return s } +// Specifies a metrics configuration for the CloudWatch request metrics (specified +// by the metrics configuration ID) from an Amazon S3 bucket. If you're updating +// an existing metrics configuration, note that this is a full replacement of +// the existing metrics configuration. If you don't include the elements you +// want to keep, they are erased. For more information, see PUT Bucket metrics +// (https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUTMetricConfiguration.html) +// in the Amazon Simple Storage Service API Reference. type MetricsConfiguration struct { _ struct{} `type:"structure"` @@ -17483,6 +22583,9 @@ func (s *MetricsConfiguration) SetId(v string) *MetricsConfiguration { return s } +// Specifies a metrics configuration filter. The metrics configuration only +// includes objects that meet the filter's criteria. A filter must be a prefix, +// a tag, or a conjunction (MetricsAndOperator). type MetricsFilter struct { _ struct{} `type:"structure"` @@ -17546,6 +22649,7 @@ func (s *MetricsFilter) SetTag(v *Tag) *MetricsFilter { return s } +// Container for the MultipartUpload for the Amazon S3 object. type MultipartUpload struct { _ struct{} `type:"structure"` @@ -17558,6 +22662,7 @@ type MultipartUpload struct { // Key of the object for which the multipart upload was initiated. Key *string `min:"1" type:"string"` + // Specifies the owner of the object that is part of the multipart upload. Owner *Owner `type:"structure"` // The class of storage used to store the object. @@ -17624,8 +22729,8 @@ type NoncurrentVersionExpiration struct { // Specifies the number of days an object is noncurrent before Amazon S3 can // perform the associated action. For information about the noncurrent days // calculations, see How Amazon S3 Calculates When an Object Became Noncurrent - // (http://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html) in - // the Amazon Simple Storage Service Developer Guide. + // (https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html#non-current-days-calculations) + // in the Amazon Simple Storage Service Developer Guide. NoncurrentDays *int64 `type:"integer"` } @@ -17646,19 +22751,20 @@ func (s *NoncurrentVersionExpiration) SetNoncurrentDays(v int64) *NoncurrentVers } // Container for the transition rule that describes when noncurrent objects -// transition to the STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING or GLACIER -// storage class. If your bucket is versioning-enabled (or versioning is suspended), -// you can set this action to request that Amazon S3 transition noncurrent object -// versions to the STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING or GLACIER storage -// class at a specific period in the object's lifetime. +// transition to the STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING, GLACIER, +// or DEEP_ARCHIVE storage class. If your bucket is versioning-enabled (or versioning +// is suspended), you can set this action to request that Amazon S3 transition +// noncurrent object versions to the STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING, +// GLACIER, or DEEP_ARCHIVE storage class at a specific period in the object's +// lifetime. type NoncurrentVersionTransition struct { _ struct{} `type:"structure"` // Specifies the number of days an object is noncurrent before Amazon S3 can // perform the associated action. For information about the noncurrent days - // calculations, see How Amazon S3 Calculates When an Object Became Noncurrent - // (http://docs.aws.amazon.com/AmazonS3/latest/dev/s3-access-control.html) in - // the Amazon Simple Storage Service Developer Guide. + // calculations, see How Amazon S3 Calculates How Long an Object Has Been Noncurrent + // (https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html#non-current-days-calculations) + // in the Amazon Simple Storage Service Developer Guide. NoncurrentDays *int64 `type:"integer"` // The class of storage used to store the object. @@ -17692,10 +22798,16 @@ func (s *NoncurrentVersionTransition) SetStorageClass(v string) *NoncurrentVersi type NotificationConfiguration struct { _ struct{} `type:"structure"` + // Describes the AWS Lambda functions to invoke and the events for which to + // invoke them. LambdaFunctionConfigurations []*LambdaFunctionConfiguration `locationName:"CloudFunctionConfiguration" type:"list" flattened:"true"` + // The Amazon Simple Queue Service queues to publish messages to and the events + // for which to publish messages. QueueConfigurations []*QueueConfiguration `locationName:"QueueConfiguration" type:"list" flattened:"true"` + // The topic to which notifications are sent and the events for which notifications + // are generated. TopicConfigurations []*TopicConfiguration `locationName:"TopicConfiguration" type:"list" flattened:"true"` } @@ -17770,10 +22882,17 @@ func (s *NotificationConfiguration) SetTopicConfigurations(v []*TopicConfigurati type NotificationConfigurationDeprecated struct { _ struct{} `type:"structure"` + // Container for specifying the AWS Lambda notification configuration. CloudFunctionConfiguration *CloudFunctionConfiguration `type:"structure"` + // This data type is deprecated. This data type specifies the configuration + // for publishing messages to an Amazon Simple Queue Service (Amazon SQS) queue + // when Amazon S3 detects specified events. QueueConfiguration *QueueConfigurationDeprecated `type:"structure"` + // This data type is deprecated. A container for specifying the configuration + // for publication of messages to an Amazon Simple Notification Service (Amazon + // SNS) topic when Amazon S3 detects specified events. TopicConfiguration *TopicConfigurationDeprecated `type:"structure"` } @@ -17805,8 +22924,8 @@ func (s *NotificationConfigurationDeprecated) SetTopicConfiguration(v *TopicConf return s } -// A container for object key name filtering rules. For information about key -// name filtering, see Configuring Event Notifications (http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html) +// Specifies object key name filtering rules. For information about key name +// filtering, see Configuring Event Notifications (https://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html) // in the Amazon Simple Storage Service Developer Guide. type NotificationConfigurationFilter struct { _ struct{} `type:"structure"` @@ -17831,17 +22950,25 @@ func (s *NotificationConfigurationFilter) SetKey(v *KeyFilter) *NotificationConf return s } +// An object consists of data and its descriptive metadata. type Object struct { _ struct{} `type:"structure"` + // The entity tag is an MD5 hash of the object. ETag reflects only changes to + // the contents of an object, not its metadata. ETag *string `type:"string"` + // The name that you assign to an object. You use the object key to retrieve + // the object. Key *string `min:"1" type:"string"` + // The date the Object was Last Modified LastModified *time.Time `type:"timestamp"` + // The owner of the object Owner *Owner `type:"structure"` + // Size in bytes of the object Size *int64 `type:"integer"` // The class of storage used to store the object. @@ -17894,6 +23021,7 @@ func (s *Object) SetStorageClass(v string) *Object { return s } +// Object Identifier is unique value to identify objects. type ObjectIdentifier struct { _ struct{} `type:"structure"` @@ -18059,9 +23187,11 @@ func (s *ObjectLockRule) SetDefaultRetention(v *DefaultRetention) *ObjectLockRul return s } +// The version of an object. type ObjectVersion struct { _ struct{} `type:"structure"` + // The entity tag is an MD5 hash of that version of the object. ETag *string `type:"string"` // Specifies whether the object is (true) or is not (false) the latest version @@ -18074,6 +23204,7 @@ type ObjectVersion struct { // Date and time the object was last modified. LastModified *time.Time `type:"timestamp"` + // Specifies the owner of the object. Owner *Owner `type:"structure"` // Size in bytes of the object. @@ -18216,11 +23347,14 @@ func (s *OutputSerialization) SetJSON(v *JSONOutput) *OutputSerialization { return s } +// Container for the owner's display name and ID. type Owner struct { _ struct{} `type:"structure"` + // Container for the display name of the owner. DisplayName *string `type:"string"` + // Container for the ID of the owner. ID *string `type:"string"` } @@ -18246,6 +23380,7 @@ func (s *Owner) SetID(v string) *Owner { return s } +// Container for Parquet. type ParquetInput struct { _ struct{} `type:"structure"` } @@ -18260,6 +23395,7 @@ func (s ParquetInput) GoString() string { return s.String() } +// Container for elements related to a part. type Part struct { _ struct{} `type:"structure"` @@ -18336,6 +23472,7 @@ func (s *PolicyStatus) SetIsPublic(v bool) *PolicyStatus { return s } +// This data type contains information about progress of an operation. type Progress struct { _ struct{} `type:"structure"` @@ -18377,6 +23514,7 @@ func (s *Progress) SetBytesScanned(v int64) *Progress { return s } +// This data type contains information about the progress event of an operation. type ProgressEvent struct { _ struct{} `locationName:"ProgressEvent" type:"structure" payload:"Details"` @@ -18417,6 +23555,23 @@ func (s *ProgressEvent) UnmarshalEvent( return nil } +// MarshalEvent marshals the type into an stream event value. This method +// should only used internally within the SDK's EventStream handling. +func (s *ProgressEvent) MarshalEvent(pm protocol.PayloadMarshaler) (msg eventstream.Message, err error) { + msg.Headers.Set(eventstreamapi.MessageTypeHeader, eventstream.StringValue(eventstreamapi.EventMessageType)) + var buf bytes.Buffer + if err = pm.MarshalPayload(&buf, s); err != nil { + return eventstream.Message{}, err + } + msg.Payload = buf.Bytes() + return msg, err +} + +// The PublicAccessBlock configuration that you want to apply to this Amazon +// S3 bucket. You can enable the configuration options in any combination. For +// more information about when Amazon S3 considers a bucket or object public, +// see The Meaning of "Public" (https://docs.aws.amazon.com/AmazonS3/latest/dev/access-control-block-public-access.html#access-control-block-public-access-policy-status) +// in the Amazon Simple Storage Service Developer Guide. type PublicAccessBlockConfiguration struct { _ struct{} `type:"structure"` @@ -18429,6 +23584,8 @@ type PublicAccessBlockConfiguration struct { // // * PUT Object calls fail if the request includes a public ACL. // + // * PUT Bucket calls fail if the request includes a public ACL. + // // Enabling this setting doesn't affect existing policies or ACLs. BlockPublicAcls *bool `locationName:"BlockPublicAcls" type:"boolean"` @@ -18493,9 +23650,9 @@ func (s *PublicAccessBlockConfiguration) SetRestrictPublicBuckets(v bool) *Publi } type PutBucketAccelerateConfigurationInput struct { - _ struct{} `type:"structure" payload:"AccelerateConfiguration"` + _ struct{} `locationName:"PutBucketAccelerateConfigurationRequest" type:"structure" payload:"AccelerateConfiguration"` - // Specifies the Accelerate Configuration you want to set for the bucket. + // Container for setting the transfer acceleration state. // // AccelerateConfiguration is a required field AccelerateConfiguration *AccelerateConfiguration `locationName:"AccelerateConfiguration" type:"structure" required:"true" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` @@ -18554,6 +23711,20 @@ func (s *PutBucketAccelerateConfigurationInput) getBucket() (v string) { return *s.Bucket } +func (s *PutBucketAccelerateConfigurationInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *PutBucketAccelerateConfigurationInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type PutBucketAccelerateConfigurationOutput struct { _ struct{} `type:"structure"` } @@ -18569,13 +23740,16 @@ func (s PutBucketAccelerateConfigurationOutput) GoString() string { } type PutBucketAclInput struct { - _ struct{} `type:"structure" payload:"AccessControlPolicy"` + _ struct{} `locationName:"PutBucketAclRequest" type:"structure" payload:"AccessControlPolicy"` // The canned ACL to apply to the bucket. ACL *string `location:"header" locationName:"x-amz-acl" type:"string" enum:"BucketCannedACL"` + // Contains the elements that set the ACL permissions for an object per grantee. AccessControlPolicy *AccessControlPolicy `locationName:"AccessControlPolicy" type:"structure" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` + // The bucket to which to apply the ACL. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` @@ -18682,6 +23856,20 @@ func (s *PutBucketAclInput) SetGrantWriteACP(v string) *PutBucketAclInput { return s } +func (s *PutBucketAclInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *PutBucketAclInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type PutBucketAclOutput struct { _ struct{} `type:"structure"` } @@ -18697,7 +23885,7 @@ func (s PutBucketAclOutput) GoString() string { } type PutBucketAnalyticsConfigurationInput struct { - _ struct{} `type:"structure" payload:"AnalyticsConfiguration"` + _ struct{} `locationName:"PutBucketAnalyticsConfigurationRequest" type:"structure" payload:"AnalyticsConfiguration"` // The configuration and any analyses for the analytics filter. // @@ -18709,7 +23897,7 @@ type PutBucketAnalyticsConfigurationInput struct { // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` - // The identifier used to represent an analytics configuration. + // The ID that identifies the analytics configuration. // // Id is a required field Id *string `location:"querystring" locationName:"id" type:"string" required:"true"` @@ -18777,6 +23965,20 @@ func (s *PutBucketAnalyticsConfigurationInput) SetId(v string) *PutBucketAnalyti return s } +func (s *PutBucketAnalyticsConfigurationInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *PutBucketAnalyticsConfigurationInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type PutBucketAnalyticsConfigurationOutput struct { _ struct{} `type:"structure"` } @@ -18792,11 +23994,18 @@ func (s PutBucketAnalyticsConfigurationOutput) GoString() string { } type PutBucketCorsInput struct { - _ struct{} `type:"structure" payload:"CORSConfiguration"` + _ struct{} `locationName:"PutBucketCorsRequest" type:"structure" payload:"CORSConfiguration"` + // Specifies the bucket impacted by the corsconfiguration. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + // Describes the cross-origin access configuration for objects in an Amazon + // S3 bucket. For more information, see Enabling Cross-Origin Resource Sharing + // (https://docs.aws.amazon.com/AmazonS3/latest/dev/cors.html) in the Amazon + // Simple Storage Service Developer Guide. + // // CORSConfiguration is a required field CORSConfiguration *CORSConfiguration `locationName:"CORSConfiguration" type:"structure" required:"true" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` } @@ -18854,6 +24063,20 @@ func (s *PutBucketCorsInput) SetCORSConfiguration(v *CORSConfiguration) *PutBuck return s } +func (s *PutBucketCorsInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *PutBucketCorsInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type PutBucketCorsOutput struct { _ struct{} `type:"structure"` } @@ -18869,16 +24092,18 @@ func (s PutBucketCorsOutput) GoString() string { } type PutBucketEncryptionInput struct { - _ struct{} `type:"structure" payload:"ServerSideEncryptionConfiguration"` + _ struct{} `locationName:"PutBucketEncryptionRequest" type:"structure" payload:"ServerSideEncryptionConfiguration"` - // The name of the bucket for which the server-side encryption configuration - // is set. + // Specifies default encryption for a bucket using server-side encryption with + // Amazon S3-managed keys (SSE-S3) or customer master keys stored in AWS KMS + // (SSE-KMS). For information about the Amazon S3 default encryption feature, + // see Amazon S3 Default Bucket Encryption (https://docs.aws.amazon.com/AmazonS3/latest/dev/bucket-encryption.html) + // in the Amazon Simple Storage Service Developer Guide. // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` - // Container for server-side encryption configuration rules. Currently S3 supports - // one rule only. + // Specifies the default server-side-encryption configuration. // // ServerSideEncryptionConfiguration is a required field ServerSideEncryptionConfiguration *ServerSideEncryptionConfiguration `locationName:"ServerSideEncryptionConfiguration" type:"structure" required:"true" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` @@ -18937,6 +24162,20 @@ func (s *PutBucketEncryptionInput) SetServerSideEncryptionConfiguration(v *Serve return s } +func (s *PutBucketEncryptionInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *PutBucketEncryptionInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type PutBucketEncryptionOutput struct { _ struct{} `type:"structure"` } @@ -18952,7 +24191,7 @@ func (s PutBucketEncryptionOutput) GoString() string { } type PutBucketInventoryConfigurationInput struct { - _ struct{} `type:"structure" payload:"InventoryConfiguration"` + _ struct{} `locationName:"PutBucketInventoryConfigurationRequest" type:"structure" payload:"InventoryConfiguration"` // The name of the bucket where the inventory configuration will be stored. // @@ -19032,6 +24271,20 @@ func (s *PutBucketInventoryConfigurationInput) SetInventoryConfiguration(v *Inve return s } +func (s *PutBucketInventoryConfigurationInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *PutBucketInventoryConfigurationInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type PutBucketInventoryConfigurationOutput struct { _ struct{} `type:"structure"` } @@ -19047,11 +24300,14 @@ func (s PutBucketInventoryConfigurationOutput) GoString() string { } type PutBucketLifecycleConfigurationInput struct { - _ struct{} `type:"structure" payload:"LifecycleConfiguration"` + _ struct{} `locationName:"PutBucketLifecycleConfigurationRequest" type:"structure" payload:"LifecycleConfiguration"` + // The name of the bucket for which to set the configuration. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + // Container for lifecycle rules. You can add as many as 1,000 rules. LifecycleConfiguration *BucketLifecycleConfiguration `locationName:"LifecycleConfiguration" type:"structure" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` } @@ -19105,6 +24361,20 @@ func (s *PutBucketLifecycleConfigurationInput) SetLifecycleConfiguration(v *Buck return s } +func (s *PutBucketLifecycleConfigurationInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *PutBucketLifecycleConfigurationInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type PutBucketLifecycleConfigurationOutput struct { _ struct{} `type:"structure"` } @@ -19120,11 +24390,12 @@ func (s PutBucketLifecycleConfigurationOutput) GoString() string { } type PutBucketLifecycleInput struct { - _ struct{} `type:"structure" payload:"LifecycleConfiguration"` + _ struct{} `locationName:"PutBucketLifecycleRequest" type:"structure" payload:"LifecycleConfiguration"` // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + // Container for lifecycle rules. You can add as many as 1000 rules. LifecycleConfiguration *LifecycleConfiguration `locationName:"LifecycleConfiguration" type:"structure" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` } @@ -19178,6 +24449,20 @@ func (s *PutBucketLifecycleInput) SetLifecycleConfiguration(v *LifecycleConfigur return s } +func (s *PutBucketLifecycleInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *PutBucketLifecycleInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type PutBucketLifecycleOutput struct { _ struct{} `type:"structure"` } @@ -19193,11 +24478,15 @@ func (s PutBucketLifecycleOutput) GoString() string { } type PutBucketLoggingInput struct { - _ struct{} `type:"structure" payload:"BucketLoggingStatus"` + _ struct{} `locationName:"PutBucketLoggingRequest" type:"structure" payload:"BucketLoggingStatus"` + // The name of the bucket for which to set the logging parameters. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + // Container for logging status information. + // // BucketLoggingStatus is a required field BucketLoggingStatus *BucketLoggingStatus `locationName:"BucketLoggingStatus" type:"structure" required:"true" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` } @@ -19255,6 +24544,20 @@ func (s *PutBucketLoggingInput) SetBucketLoggingStatus(v *BucketLoggingStatus) * return s } +func (s *PutBucketLoggingInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *PutBucketLoggingInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type PutBucketLoggingOutput struct { _ struct{} `type:"structure"` } @@ -19270,7 +24573,7 @@ func (s PutBucketLoggingOutput) GoString() string { } type PutBucketMetricsConfigurationInput struct { - _ struct{} `type:"structure" payload:"MetricsConfiguration"` + _ struct{} `locationName:"PutBucketMetricsConfigurationRequest" type:"structure" payload:"MetricsConfiguration"` // The name of the bucket for which the metrics configuration is set. // @@ -19350,6 +24653,20 @@ func (s *PutBucketMetricsConfigurationInput) SetMetricsConfiguration(v *MetricsC return s } +func (s *PutBucketMetricsConfigurationInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *PutBucketMetricsConfigurationInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type PutBucketMetricsConfigurationOutput struct { _ struct{} `type:"structure"` } @@ -19365,8 +24682,10 @@ func (s PutBucketMetricsConfigurationOutput) GoString() string { } type PutBucketNotificationConfigurationInput struct { - _ struct{} `type:"structure" payload:"NotificationConfiguration"` + _ struct{} `locationName:"PutBucketNotificationConfigurationRequest" type:"structure" payload:"NotificationConfiguration"` + // The name of the bucket. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` @@ -19430,6 +24749,20 @@ func (s *PutBucketNotificationConfigurationInput) SetNotificationConfiguration(v return s } +func (s *PutBucketNotificationConfigurationInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *PutBucketNotificationConfigurationInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type PutBucketNotificationConfigurationOutput struct { _ struct{} `type:"structure"` } @@ -19445,11 +24778,15 @@ func (s PutBucketNotificationConfigurationOutput) GoString() string { } type PutBucketNotificationInput struct { - _ struct{} `type:"structure" payload:"NotificationConfiguration"` + _ struct{} `locationName:"PutBucketNotificationRequest" type:"structure" payload:"NotificationConfiguration"` + // The name of the bucket. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + // The container for the configuration. + // // NotificationConfiguration is a required field NotificationConfiguration *NotificationConfigurationDeprecated `locationName:"NotificationConfiguration" type:"structure" required:"true" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` } @@ -19502,6 +24839,20 @@ func (s *PutBucketNotificationInput) SetNotificationConfiguration(v *Notificatio return s } +func (s *PutBucketNotificationInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *PutBucketNotificationInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type PutBucketNotificationOutput struct { _ struct{} `type:"structure"` } @@ -19517,8 +24868,10 @@ func (s PutBucketNotificationOutput) GoString() string { } type PutBucketPolicyInput struct { - _ struct{} `type:"structure" payload:"Policy"` + _ struct{} `locationName:"PutBucketPolicyRequest" type:"structure" payload:"Policy"` + // The name of the bucket. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` @@ -19586,6 +24939,20 @@ func (s *PutBucketPolicyInput) SetPolicy(v string) *PutBucketPolicyInput { return s } +func (s *PutBucketPolicyInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *PutBucketPolicyInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type PutBucketPolicyOutput struct { _ struct{} `type:"structure"` } @@ -19601,8 +24968,10 @@ func (s PutBucketPolicyOutput) GoString() string { } type PutBucketReplicationInput struct { - _ struct{} `type:"structure" payload:"ReplicationConfiguration"` + _ struct{} `locationName:"PutBucketReplicationRequest" type:"structure" payload:"ReplicationConfiguration"` + // The name of the bucket + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` @@ -19611,6 +24980,8 @@ type PutBucketReplicationInput struct { // // ReplicationConfiguration is a required field ReplicationConfiguration *ReplicationConfiguration `locationName:"ReplicationConfiguration" type:"structure" required:"true" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` + + Token *string `location:"header" locationName:"x-amz-bucket-object-lock-token" type:"string"` } // String returns the string representation @@ -19666,6 +25037,26 @@ func (s *PutBucketReplicationInput) SetReplicationConfiguration(v *ReplicationCo return s } +// SetToken sets the Token field's value. +func (s *PutBucketReplicationInput) SetToken(v string) *PutBucketReplicationInput { + s.Token = &v + return s +} + +func (s *PutBucketReplicationInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *PutBucketReplicationInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type PutBucketReplicationOutput struct { _ struct{} `type:"structure"` } @@ -19681,11 +25072,15 @@ func (s PutBucketReplicationOutput) GoString() string { } type PutBucketRequestPaymentInput struct { - _ struct{} `type:"structure" payload:"RequestPaymentConfiguration"` + _ struct{} `locationName:"PutBucketRequestPaymentRequest" type:"structure" payload:"RequestPaymentConfiguration"` + // The bucket name. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + // Container for Payer. + // // RequestPaymentConfiguration is a required field RequestPaymentConfiguration *RequestPaymentConfiguration `locationName:"RequestPaymentConfiguration" type:"structure" required:"true" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` } @@ -19743,6 +25138,20 @@ func (s *PutBucketRequestPaymentInput) SetRequestPaymentConfiguration(v *Request return s } +func (s *PutBucketRequestPaymentInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *PutBucketRequestPaymentInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type PutBucketRequestPaymentOutput struct { _ struct{} `type:"structure"` } @@ -19758,11 +25167,15 @@ func (s PutBucketRequestPaymentOutput) GoString() string { } type PutBucketTaggingInput struct { - _ struct{} `type:"structure" payload:"Tagging"` + _ struct{} `locationName:"PutBucketTaggingRequest" type:"structure" payload:"Tagging"` + // The bucket name. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + // Container for the TagSet and Tag elements. + // // Tagging is a required field Tagging *Tagging `locationName:"Tagging" type:"structure" required:"true" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` } @@ -19820,6 +25233,20 @@ func (s *PutBucketTaggingInput) SetTagging(v *Tagging) *PutBucketTaggingInput { return s } +func (s *PutBucketTaggingInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *PutBucketTaggingInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type PutBucketTaggingOutput struct { _ struct{} `type:"structure"` } @@ -19835,8 +25262,10 @@ func (s PutBucketTaggingOutput) GoString() string { } type PutBucketVersioningInput struct { - _ struct{} `type:"structure" payload:"VersioningConfiguration"` + _ struct{} `locationName:"PutBucketVersioningRequest" type:"structure" payload:"VersioningConfiguration"` + // The bucket name. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` @@ -19844,6 +25273,8 @@ type PutBucketVersioningInput struct { // and the value that is displayed on your authentication device. MFA *string `location:"header" locationName:"x-amz-mfa" type:"string"` + // Container for setting the versioning state. + // // VersioningConfiguration is a required field VersioningConfiguration *VersioningConfiguration `locationName:"VersioningConfiguration" type:"structure" required:"true" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` } @@ -19902,6 +25333,20 @@ func (s *PutBucketVersioningInput) SetVersioningConfiguration(v *VersioningConfi return s } +func (s *PutBucketVersioningInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *PutBucketVersioningInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type PutBucketVersioningOutput struct { _ struct{} `type:"structure"` } @@ -19917,11 +25362,15 @@ func (s PutBucketVersioningOutput) GoString() string { } type PutBucketWebsiteInput struct { - _ struct{} `type:"structure" payload:"WebsiteConfiguration"` + _ struct{} `locationName:"PutBucketWebsiteRequest" type:"structure" payload:"WebsiteConfiguration"` + // The bucket name. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + // Container for the request. + // // WebsiteConfiguration is a required field WebsiteConfiguration *WebsiteConfiguration `locationName:"WebsiteConfiguration" type:"structure" required:"true" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` } @@ -19979,6 +25428,20 @@ func (s *PutBucketWebsiteInput) SetWebsiteConfiguration(v *WebsiteConfiguration) return s } +func (s *PutBucketWebsiteInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *PutBucketWebsiteInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type PutBucketWebsiteOutput struct { _ struct{} `type:"structure"` } @@ -19994,13 +25457,25 @@ func (s PutBucketWebsiteOutput) GoString() string { } type PutObjectAclInput struct { - _ struct{} `type:"structure" payload:"AccessControlPolicy"` + _ struct{} `locationName:"PutObjectAclRequest" type:"structure" payload:"AccessControlPolicy"` - // The canned ACL to apply to the object. + // The canned ACL to apply to the object. For more information, see Canned ACL + // (https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#CannedACL). ACL *string `location:"header" locationName:"x-amz-acl" type:"string" enum:"ObjectCannedACL"` + // Contains the elements that set the ACL permissions for an object per grantee. AccessControlPolicy *AccessControlPolicy `locationName:"AccessControlPolicy" type:"structure" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` + // The bucket name that contains the object to which you want to attach the + // ACL. + // + // When using this API with an access point, you must direct requests to the + // access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com. + // When using this operation using an access point through the AWS SDKs, you + // provide the access point ARN in place of the bucket name. For more information + // about access point ARNs, see Using Access Points (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-access-points.html) + // in the Amazon Simple Storage Service Developer Guide. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` @@ -20020,13 +25495,16 @@ type PutObjectAclInput struct { // Allows grantee to write the ACL for the applicable bucket. GrantWriteACP *string `location:"header" locationName:"x-amz-grant-write-acp" type:"string"` + // Key for which the PUT operation was initiated. + // // Key is a required field Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // Documentation on downloading objects from requester pays buckets can be found - // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` // VersionId used to reference a specific version of the object. @@ -20143,6 +25621,20 @@ func (s *PutObjectAclInput) SetVersionId(v string) *PutObjectAclInput { return s } +func (s *PutObjectAclInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *PutObjectAclInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type PutObjectAclOutput struct { _ struct{} `type:"structure"` @@ -20168,44 +25660,64 @@ func (s *PutObjectAclOutput) SetRequestCharged(v string) *PutObjectAclOutput { } type PutObjectInput struct { - _ struct{} `type:"structure" payload:"Body"` + _ struct{} `locationName:"PutObjectRequest" type:"structure" payload:"Body"` - // The canned ACL to apply to the object. + // The canned ACL to apply to the object. For more information, see Canned ACL + // (https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#CannedACL). ACL *string `location:"header" locationName:"x-amz-acl" type:"string" enum:"ObjectCannedACL"` // Object data. Body io.ReadSeeker `type:"blob"` - // Name of the bucket to which the PUT operation was initiated. + // Bucket name to which the PUT operation was initiated. + // + // When using this API with an access point, you must direct requests to the + // access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com. + // When using this operation using an access point through the AWS SDKs, you + // provide the access point ARN in place of the bucket name. For more information + // about access point ARNs, see Using Access Points (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-access-points.html) + // in the Amazon Simple Storage Service Developer Guide. // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` - // Specifies caching behavior along the request/reply chain. + // Can be used to specify caching behavior along the request/reply chain. For + // more information, see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9 + // (http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9). CacheControl *string `location:"header" locationName:"Cache-Control" type:"string"` - // Specifies presentational information for the object. + // Specifies presentational information for the object. For more information, + // see http://www.w3.org/Protocols/rfc2616/rfc2616-sec19.html#sec19.5.1 (http://www.w3.org/Protocols/rfc2616/rfc2616-sec19.html#sec19.5.1). ContentDisposition *string `location:"header" locationName:"Content-Disposition" type:"string"` // Specifies what content encodings have been applied to the object and thus // what decoding mechanisms must be applied to obtain the media-type referenced - // by the Content-Type header field. + // by the Content-Type header field. For more information, see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.11 + // (http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.11). ContentEncoding *string `location:"header" locationName:"Content-Encoding" type:"string"` // The language the content is in. ContentLanguage *string `location:"header" locationName:"Content-Language" type:"string"` // Size of the body in bytes. This parameter is useful when the size of the - // body cannot be determined automatically. + // body cannot be determined automatically. For more information, see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.13 + // (http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.13). ContentLength *int64 `location:"header" locationName:"Content-Length" type:"long"` - // The base64-encoded 128-bit MD5 digest of the part data. + // The base64-encoded 128-bit MD5 digest of the message (without the headers) + // according to RFC 1864. This header can be used as a message integrity check + // to verify that the data is the same data that was originally sent. Although + // it is optional, we recommend using the Content-MD5 mechanism as an end-to-end + // integrity check. For more information about REST request authentication, + // see REST Authentication (https://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html). ContentMD5 *string `location:"header" locationName:"Content-MD5" type:"string"` - // A standard MIME type describing the format of the object data. + // A standard MIME type describing the format of the contents. For more information, + // see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.17 (http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.17). ContentType *string `location:"header" locationName:"Content-Type" type:"string"` - // The date and time at which the object is no longer cacheable. + // The date and time at which the object is no longer cacheable. For more information, + // see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.21 (http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.21). Expires *time.Time `location:"header" locationName:"Expires" type:"timestamp"` // Gives the grantee READ, READ_ACP, and WRITE_ACP permissions on the object. @@ -20228,7 +25740,8 @@ type PutObjectInput struct { // A map of metadata to store with the object in S3. Metadata map[string]*string `location:"headers" locationName:"x-amz-meta-" type:"map"` - // The Legal Hold status that you want to apply to the specified object. + // Specifies whether a legal hold will be applied to this object. For more information + // about S3 Object Lock, see Object Lock (https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock.html). ObjectLockLegalHoldStatus *string `location:"header" locationName:"x-amz-object-lock-legal-hold" type:"string" enum:"ObjectLockLegalHoldStatus"` // The Object Lock mode that you want to apply to this object. @@ -20237,38 +25750,52 @@ type PutObjectInput struct { // The date and time when you want this object's Object Lock to expire. ObjectLockRetainUntilDate *time.Time `location:"header" locationName:"x-amz-object-lock-retain-until-date" type:"timestamp" timestampFormat:"iso8601"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // Documentation on downloading objects from requester pays buckets can be found - // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` - // Specifies the algorithm to use to when encrypting the object (e.g., AES256). + // Specifies the algorithm to use to when encrypting the object (for example, + // AES256). SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` // Specifies the customer-provided encryption key for Amazon S3 to use in encrypting // data. This value is used to store the object and then it is discarded; Amazon - // does not store the encryption key. The key must be appropriate for use with - // the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm + // S3 does not store the encryption key. The key must be appropriate for use + // with the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm // header. - SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"` + SSECustomerKey *string `marshal-as:"blob" location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"` // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. - // Amazon S3 uses this header for a message integrity check to ensure the encryption - // key was transmitted without error. + // Amazon S3 uses this header for a message integrity check to ensure that the + // encryption key was transmitted without error. SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` - // Specifies the AWS KMS key ID to use for object encryption. All GET and PUT - // requests for an object protected by AWS KMS will fail if not made via SSL - // or using SigV4. Documentation on configuring any of the officially supported - // AWS SDKs and CLI can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingAWSSDK.html#specify-signature-version + // Specifies the AWS KMS Encryption Context to use for object encryption. The + // value of this header is a base64-encoded UTF-8 string holding JSON with the + // encryption context key-value pairs. + SSEKMSEncryptionContext *string `location:"header" locationName:"x-amz-server-side-encryption-context" type:"string" sensitive:"true"` + + // If x-amz-server-side-encryption is present and has the value of aws:kms, + // this header specifies the ID of the AWS Key Management Service (AWS KMS) + // symmetrical customer managed customer master key (CMK) that was used for + // the object. + // + // If the value of x-amz-server-side-encryption is aws:kms, this header specifies + // the ID of the symmetric customer managed AWS KMS CMK that will be used for + // the object. If you specify x-amz-server-side-encryption:aws:kms, but do not + // providex-amz-server-side-encryption-aws-kms-key-id, Amazon S3 uses the AWS + // managed CMK in AWS to protect the data. SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` - // The Server-side encryption algorithm used when storing this object in S3 - // (e.g., AES256, aws:kms). + // The server-side encryption algorithm used when storing this object in Amazon + // S3 (for example, AES256, aws:kms). ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string" enum:"ServerSideEncryption"` - // The type of storage to use for the object. Defaults to 'STANDARD'. + // If you don't specify, S3 Standard is the default storage class. Amazon S3 + // supports other storage classes. StorageClass *string `location:"header" locationName:"x-amz-storage-class" type:"string" enum:"StorageClass"` // The tag-set for the object. The tag-set must be encoded as URL Query parameters. @@ -20277,7 +25804,22 @@ type PutObjectInput struct { // If the bucket is configured as a website, redirects requests for this object // to another object in the same bucket or to an external URL. Amazon S3 stores - // the value of this header in the object metadata. + // the value of this header in the object metadata. For information about object + // metadata, see Object Key and Metadata (https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html). + // + // In the following example, the request header sets the redirect to an object + // (anotherPage.html) in the same bucket: + // + // x-amz-website-redirect-location: /anotherPage.html + // + // In the following example, the request header sets the object redirect to + // another website: + // + // x-amz-website-redirect-location: http://www.example.com/ + // + // For more information about website hosting in Amazon S3, see Hosting Websites + // on Amazon S3 (https://docs.aws.amazon.com/AmazonS3/latest/dev/WebsiteHosting.html) + // and How to Configure Website Page Redirects (https://docs.aws.amazon.com/AmazonS3/latest/dev/how-to-page-redirect.html). WebsiteRedirectLocation *string `location:"header" locationName:"x-amz-website-redirect-location" type:"string"` } @@ -20471,6 +26013,12 @@ func (s *PutObjectInput) SetSSECustomerKeyMD5(v string) *PutObjectInput { return s } +// SetSSEKMSEncryptionContext sets the SSEKMSEncryptionContext field's value. +func (s *PutObjectInput) SetSSEKMSEncryptionContext(v string) *PutObjectInput { + s.SSEKMSEncryptionContext = &v + return s +} + // SetSSEKMSKeyId sets the SSEKMSKeyId field's value. func (s *PutObjectInput) SetSSEKMSKeyId(v string) *PutObjectInput { s.SSEKMSKeyId = &v @@ -20501,10 +26049,32 @@ func (s *PutObjectInput) SetWebsiteRedirectLocation(v string) *PutObjectInput { return s } +func (s *PutObjectInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *PutObjectInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type PutObjectLegalHoldInput struct { - _ struct{} `type:"structure" payload:"LegalHold"` + _ struct{} `locationName:"PutObjectLegalHoldRequest" type:"structure" payload:"LegalHold"` - // The bucket containing the object that you want to place a Legal Hold on. + // The bucket name containing the object that you want to place a Legal Hold + // on. + // + // When using this API with an access point, you must direct requests to the + // access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com. + // When using this operation using an access point through the AWS SDKs, you + // provide the access point ARN in place of the bucket name. For more information + // about access point ARNs, see Using Access Points (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-access-points.html) + // in the Amazon Simple Storage Service Developer Guide. // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` @@ -20518,10 +26088,11 @@ type PutObjectLegalHoldInput struct { // specified object. LegalHold *ObjectLockLegalHold `locationName:"LegalHold" type:"structure" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // Documentation on downloading objects from requester pays buckets can be found - // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` // The version ID of the object that you want to place a Legal Hold on. @@ -20597,6 +26168,20 @@ func (s *PutObjectLegalHoldInput) SetVersionId(v string) *PutObjectLegalHoldInpu return s } +func (s *PutObjectLegalHoldInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *PutObjectLegalHoldInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type PutObjectLegalHoldOutput struct { _ struct{} `type:"structure"` @@ -20622,7 +26207,7 @@ func (s *PutObjectLegalHoldOutput) SetRequestCharged(v string) *PutObjectLegalHo } type PutObjectLockConfigurationInput struct { - _ struct{} `type:"structure" payload:"ObjectLockConfiguration"` + _ struct{} `locationName:"PutObjectLockConfigurationRequest" type:"structure" payload:"ObjectLockConfiguration"` // The bucket whose Object Lock configuration you want to create or replace. // @@ -20632,12 +26217,14 @@ type PutObjectLockConfigurationInput struct { // The Object Lock configuration that you want to apply to the specified bucket. ObjectLockConfiguration *ObjectLockConfiguration `locationName:"ObjectLockConfiguration" type:"structure" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // Documentation on downloading objects from requester pays buckets can be found - // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` + // A token to allow Object Lock to be enabled for an existing bucket. Token *string `location:"header" locationName:"x-amz-bucket-object-lock-token" type:"string"` } @@ -20698,6 +26285,20 @@ func (s *PutObjectLockConfigurationInput) SetToken(v string) *PutObjectLockConfi return s } +func (s *PutObjectLockConfigurationInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *PutObjectLockConfigurationInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type PutObjectLockConfigurationOutput struct { _ struct{} `type:"structure"` @@ -20728,8 +26329,10 @@ type PutObjectOutput struct { // Entity tag for the uploaded object. ETag *string `location:"header" locationName:"ETag" type:"string"` - // If the object expiration is configured, this will contain the expiration - // date (expiry-date) and rule ID (rule-id). The value of rule-id is URL encoded. + // If the expiration is configured for the object (see PutBucketLifecycleConfiguration), + // the response includes this header. It includes the expiry-date and rule-id + // key-value pairs that provide information about object expiration. The value + // of the rule-id is URL encoded. Expiration *string `location:"header" locationName:"x-amz-expiration" type:"string"` // If present, indicates that the requester was successfully charged for the @@ -20742,16 +26345,25 @@ type PutObjectOutput struct { SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` // If server-side encryption with a customer-provided encryption key was requested, - // the response will include this header to provide round trip message integrity + // the response will include this header to provide round-trip message integrity // verification of the customer-provided encryption key. SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` - // If present, specifies the ID of the AWS Key Management Service (KMS) master - // encryption key that was used for the object. + // If present, specifies the AWS KMS Encryption Context to use for object encryption. + // The value of this header is a base64-encoded UTF-8 string holding JSON with + // the encryption context key-value pairs. + SSEKMSEncryptionContext *string `location:"header" locationName:"x-amz-server-side-encryption-context" type:"string" sensitive:"true"` + + // If x-amz-server-side-encryption is present and has the value of aws:kms, + // this header specifies the ID of the AWS Key Management Service (AWS KMS) + // symmetric customer managed customer master key (CMK) that was used for the + // object. SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` - // The Server-side encryption algorithm used when storing this object in S3 - // (e.g., AES256, aws:kms). + // If you specified server-side encryption either with an AWS KMS customer master + // key (CMK) or Amazon S3-managed encryption key in your PUT request, the response + // includes this header. It confirms the encryption algorithm that Amazon S3 + // used to encrypt the object. ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string" enum:"ServerSideEncryption"` // Version of the object. @@ -20798,6 +26410,12 @@ func (s *PutObjectOutput) SetSSECustomerKeyMD5(v string) *PutObjectOutput { return s } +// SetSSEKMSEncryptionContext sets the SSEKMSEncryptionContext field's value. +func (s *PutObjectOutput) SetSSEKMSEncryptionContext(v string) *PutObjectOutput { + s.SSEKMSEncryptionContext = &v + return s +} + // SetSSEKMSKeyId sets the SSEKMSKeyId field's value. func (s *PutObjectOutput) SetSSEKMSKeyId(v string) *PutObjectOutput { s.SSEKMSKeyId = &v @@ -20817,15 +26435,22 @@ func (s *PutObjectOutput) SetVersionId(v string) *PutObjectOutput { } type PutObjectRetentionInput struct { - _ struct{} `type:"structure" payload:"Retention"` + _ struct{} `locationName:"PutObjectRetentionRequest" type:"structure" payload:"Retention"` - // The bucket that contains the object you want to apply this Object Retention + // The bucket name that contains the object you want to apply this Object Retention // configuration to. // + // When using this API with an access point, you must direct requests to the + // access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com. + // When using this operation using an access point through the AWS SDKs, you + // provide the access point ARN in place of the bucket name. For more information + // about access point ARNs, see Using Access Points (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-access-points.html) + // in the Amazon Simple Storage Service Developer Guide. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` - // Indicates whether this operation should bypass Governance-mode restrictions.j + // Indicates whether this operation should bypass Governance-mode restrictions. BypassGovernanceRetention *bool `location:"header" locationName:"x-amz-bypass-governance-retention" type:"boolean"` // The key name for the object that you want to apply this Object Retention @@ -20834,10 +26459,11 @@ type PutObjectRetentionInput struct { // Key is a required field Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // Documentation on downloading objects from requester pays buckets can be found - // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` // The container element for the Object Retention configuration. @@ -20923,6 +26549,20 @@ func (s *PutObjectRetentionInput) SetVersionId(v string) *PutObjectRetentionInpu return s } +func (s *PutObjectRetentionInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *PutObjectRetentionInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type PutObjectRetentionOutput struct { _ struct{} `type:"structure"` @@ -20948,17 +26588,31 @@ func (s *PutObjectRetentionOutput) SetRequestCharged(v string) *PutObjectRetenti } type PutObjectTaggingInput struct { - _ struct{} `type:"structure" payload:"Tagging"` + _ struct{} `locationName:"PutObjectTaggingRequest" type:"structure" payload:"Tagging"` + // The bucket name containing the object. + // + // When using this API with an access point, you must direct requests to the + // access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com. + // When using this operation using an access point through the AWS SDKs, you + // provide the access point ARN in place of the bucket name. For more information + // about access point ARNs, see Using Access Points (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-access-points.html) + // in the Amazon Simple Storage Service Developer Guide. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + // Name of the tag. + // // Key is a required field Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"` + // Container for the TagSet and Tag elements + // // Tagging is a required field Tagging *Tagging `locationName:"Tagging" type:"structure" required:"true" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` + // The versionId of the object that the tag-set will be added to. VersionId *string `location:"querystring" locationName:"versionId" type:"string"` } @@ -21033,9 +26687,24 @@ func (s *PutObjectTaggingInput) SetVersionId(v string) *PutObjectTaggingInput { return s } +func (s *PutObjectTaggingInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *PutObjectTaggingInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type PutObjectTaggingOutput struct { _ struct{} `type:"structure"` + // The versionId of the object the tag-set was added to. VersionId *string `location:"header" locationName:"x-amz-version-id" type:"string"` } @@ -21056,7 +26725,7 @@ func (s *PutObjectTaggingOutput) SetVersionId(v string) *PutObjectTaggingOutput } type PutPublicAccessBlockInput struct { - _ struct{} `type:"structure" payload:"PublicAccessBlockConfiguration"` + _ struct{} `locationName:"PutPublicAccessBlockRequest" type:"structure" payload:"PublicAccessBlockConfiguration"` // The name of the Amazon S3 bucket whose PublicAccessBlock configuration you // want to set. @@ -21122,6 +26791,20 @@ func (s *PutPublicAccessBlockInput) SetPublicAccessBlockConfiguration(v *PublicA return s } +func (s *PutPublicAccessBlockInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *PutPublicAccessBlockInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type PutPublicAccessBlockOutput struct { _ struct{} `type:"structure"` } @@ -21136,17 +26819,18 @@ func (s PutPublicAccessBlockOutput) GoString() string { return s.String() } -// A container for specifying the configuration for publication of messages -// to an Amazon Simple Queue Service (Amazon SQS) queue.when Amazon S3 detects -// specified events. +// Specifies the configuration for publishing messages to an Amazon Simple Queue +// Service (Amazon SQS) queue when Amazon S3 detects specified events. type QueueConfiguration struct { _ struct{} `type:"structure"` + // A collection of bucket events for which to send notifications + // // Events is a required field Events []*string `locationName:"Event" type:"list" flattened:"true" required:"true"` - // A container for object key name filtering rules. For information about key - // name filtering, see Configuring Event Notifications (http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html) + // Specifies object key name filtering rules. For information about key name + // filtering, see Configuring Event Notifications (https://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html) // in the Amazon Simple Storage Service Developer Guide. Filter *NotificationConfigurationFilter `type:"structure"` @@ -21155,7 +26839,7 @@ type QueueConfiguration struct { Id *string `type:"string"` // The Amazon Resource Name (ARN) of the Amazon SQS queue to which Amazon S3 - // will publish a message when it detects events of the specified type. + // publishes a message when it detects events of the specified type. // // QueueArn is a required field QueueArn *string `locationName:"Queue" type:"string" required:"true"` @@ -21211,6 +26895,10 @@ func (s *QueueConfiguration) SetQueueArn(v string) *QueueConfiguration { return s } +// This data type is deprecated. Use QueueConfiguration for the same purposes. +// This data type specifies the configuration for publishing messages to an +// Amazon Simple Queue Service (Amazon SQS) queue when Amazon S3 detects specified +// events. type QueueConfigurationDeprecated struct { _ struct{} `type:"structure"` @@ -21219,12 +26907,15 @@ type QueueConfigurationDeprecated struct { // Deprecated: Event has been deprecated Event *string `deprecated:"true" type:"string" enum:"Event"` + // A collection of bucket events for which to send notifications Events []*string `locationName:"Event" type:"list" flattened:"true"` // An optional unique identifier for configurations in a notification configuration. // If you don't provide one, Amazon S3 will assign an ID. Id *string `type:"string"` + // The Amazon Resource Name (ARN) of the Amazon SQS queue to which Amazon S3 + // publishes a message when it detects events of the specified type. Queue *string `type:"string"` } @@ -21262,6 +26953,7 @@ func (s *QueueConfigurationDeprecated) SetQueue(v string) *QueueConfigurationDep return s } +// The container for the records event. type RecordsEvent struct { _ struct{} `locationName:"RecordsEvent" type:"structure" payload:"Payload"` @@ -21301,6 +26993,17 @@ func (s *RecordsEvent) UnmarshalEvent( return nil } +// MarshalEvent marshals the type into an stream event value. This method +// should only used internally within the SDK's EventStream handling. +func (s *RecordsEvent) MarshalEvent(pm protocol.PayloadMarshaler) (msg eventstream.Message, err error) { + msg.Headers.Set(eventstreamapi.MessageTypeHeader, eventstream.StringValue(eventstreamapi.EventMessageType)) + msg.Headers.Set(":content-type", eventstream.StringValue("application/octet-stream")) + msg.Payload = s.Payload + return msg, err +} + +// Specifies how requests are redirected. In the event of an error, you can +// specify a different error code to return. type Redirect struct { _ struct{} `type:"structure"` @@ -21311,8 +27014,8 @@ type Redirect struct { // siblings is present. HttpRedirectCode *string `type:"string"` - // Protocol to use (http, https) when redirecting requests. The default is the - // protocol that is used in the original request. + // Protocol to use when redirecting requests. The default is the protocol that + // is used in the original request. Protocol *string `type:"string" enum:"Protocol"` // The object key prefix to use in the redirect request. For example, to redirect @@ -21324,7 +27027,7 @@ type Redirect struct { ReplaceKeyPrefixWith *string `type:"string"` // The specific object key to use in the redirect request. For example, redirect - // request to error.html. Not required if one of the sibling is present. Can + // request to error.html. Not required if one of the siblings is present. Can // be present only if ReplaceKeyPrefixWith is not provided. ReplaceKeyWith *string `type:"string"` } @@ -21369,16 +27072,18 @@ func (s *Redirect) SetReplaceKeyWith(v string) *Redirect { return s } +// Specifies the redirect behavior of all requests to a website endpoint of +// an Amazon S3 bucket. type RedirectAllRequestsTo struct { _ struct{} `type:"structure"` - // Name of the host where requests will be redirected. + // Name of the host where requests are redirected. // // HostName is a required field HostName *string `type:"string" required:"true"` - // Protocol to use (http, https) when redirecting requests. The default is the - // protocol that is used in the original request. + // Protocol to use when redirecting requests. The default is the protocol that + // is used in the original request. Protocol *string `type:"string" enum:"Protocol"` } @@ -21423,7 +27128,9 @@ type ReplicationConfiguration struct { _ struct{} `type:"structure"` // The Amazon Resource Name (ARN) of the AWS Identity and Access Management - // (IAM) role that Amazon S3 can assume when replicating the objects. + // (IAM) role that Amazon S3 assumes when replicating objects. For more information, + // see How to Set Up Replication (https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-how-setup.html) + // in the Amazon Simple Storage Service Developer Guide. // // Role is a required field Role *string `type:"string" required:"true"` @@ -21483,18 +27190,34 @@ func (s *ReplicationConfiguration) SetRules(v []*ReplicationRule) *ReplicationCo return s } -// A container for information about a specific replication rule. +// Specifies which Amazon S3 objects to replicate and where to store the replicas. type ReplicationRule struct { _ struct{} `type:"structure"` - // Specifies whether Amazon S3 should replicate delete makers. + // Specifies whether Amazon S3 replicates the delete markers. If you specify + // a Filter, you must specify this element. However, in the latest version of + // replication configuration (when Filter is specified), Amazon S3 doesn't replicate + // delete markers. Therefore, the DeleteMarkerReplication element can contain + // only Disabled. For an example configuration, see Basic Rule + // Configuration (https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html#replication-config-min-rule-config). + // + // If you don't specify the Filter element, Amazon S3 assumes that the replication + // configuration is the earlier version, V1. In the earlier version, Amazon + // S3 handled replication of delete markers differently. For more information, + // see Backward Compatibility (https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html#replication-backward-compat-considerations). DeleteMarkerReplication *DeleteMarkerReplication `type:"structure"` - // A container for information about the replication destination. + // A container for information about the replication destination and its configurations + // including enabling the S3 Replication Time Control (S3 RTC). // // Destination is a required field Destination *Destination `type:"structure" required:"true"` + // Optional configuration to replicate existing source bucket objects. For more + // information, see Replicating Existing Objects (https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-what-is-isnot-replicated.html#existing-object-replication) + // in the Amazon S3 Developer Guide. + ExistingObjectReplication *ExistingObjectReplication `type:"structure"` + // A filter that identifies the subset of objects to which the replication rule // applies. A Filter must specify exactly one Prefix, Tag, or an And child element. Filter *ReplicationRuleFilter `type:"structure"` @@ -21502,8 +27225,9 @@ type ReplicationRule struct { // A unique identifier for the rule. The maximum value is 255 characters. ID *string `type:"string"` - // An object keyname prefix that identifies the object or objects to which the - // rule applies. The maximum prefix length is 1,024 characters. + // An object key name prefix that identifies the object or objects to which + // the rule applies. The maximum prefix length is 1,024 characters. To include + // all objects in a bucket, specify an empty string. // // Deprecated: Prefix has been deprecated Prefix *string `deprecated:"true" type:"string"` @@ -21513,27 +27237,24 @@ type ReplicationRule struct { // when filtering. If two or more rules identify the same object based on a // specified filter, the rule with higher priority takes precedence. For example: // - // * Same object quality prefix based filter criteria If prefixes you specified + // * Same object quality prefix-based filter criteria if prefixes you specified // in multiple rules overlap // - // * Same object qualify tag based filter criteria specified in multiple + // * Same object qualify tag-based filter criteria specified in multiple // rules // - // For more information, see Cross-Region Replication (CRR) ( https://docs.aws.amazon.com/AmazonS3/latest/dev/crr.html) - // in the Amazon S3 Developer Guide. + // For more information, see Replication (https://docs.aws.amazon.com/AmazonS3/latest/dev/replication.html) + // in the Amazon Simple Storage Service Developer Guide. Priority *int64 `type:"integer"` // A container that describes additional filters for identifying the source // objects that you want to replicate. You can choose to enable or disable the // replication of these objects. Currently, Amazon S3 supports only the filter // that you can specify for objects created with server-side encryption using - // an AWS KMS-Managed Key (SSE-KMS). - // - // If you want Amazon S3 to replicate objects created with server-side encryption - // using AWS KMS-Managed Keys. + // a customer master key (CMK) stored in AWS Key Management Service (SSE-KMS). SourceSelectionCriteria *SourceSelectionCriteria `type:"structure"` - // If status isn't enabled, the rule is ignored. + // Specifies whether the rule is enabled. // // Status is a required field Status *string `type:"string" required:"true" enum:"ReplicationRuleStatus"` @@ -21563,6 +27284,11 @@ func (s *ReplicationRule) Validate() error { invalidParams.AddNested("Destination", err.(request.ErrInvalidParams)) } } + if s.ExistingObjectReplication != nil { + if err := s.ExistingObjectReplication.Validate(); err != nil { + invalidParams.AddNested("ExistingObjectReplication", err.(request.ErrInvalidParams)) + } + } if s.Filter != nil { if err := s.Filter.Validate(); err != nil { invalidParams.AddNested("Filter", err.(request.ErrInvalidParams)) @@ -21592,6 +27318,12 @@ func (s *ReplicationRule) SetDestination(v *Destination) *ReplicationRule { return s } +// SetExistingObjectReplication sets the ExistingObjectReplication field's value. +func (s *ReplicationRule) SetExistingObjectReplication(v *ExistingObjectReplication) *ReplicationRule { + s.ExistingObjectReplication = v + return s +} + // SetFilter sets the Filter field's value. func (s *ReplicationRule) SetFilter(v *ReplicationRuleFilter) *ReplicationRule { s.Filter = v @@ -21628,11 +27360,25 @@ func (s *ReplicationRule) SetStatus(v string) *ReplicationRule { return s } +// A container for specifying rule filters. The filters determine the subset +// of objects to which the rule applies. This element is required only if you +// specify more than one filter. +// +// For example: +// +// * If you specify both a Prefix and a Tag filter, wrap these filters in +// an And tag. +// +// * If you specify a filter based on multiple tags, wrap the Tag elements +// in an And tag type ReplicationRuleAndOperator struct { _ struct{} `type:"structure"` + // An object key name prefix that identifies the subset of objects to which + // the rule applies. Prefix *string `type:"string"` + // An array of tags containing key and value pairs. Tags []*Tag `locationName:"Tag" locationNameList:"Tag" type:"list" flattened:"true"` } @@ -21694,8 +27440,8 @@ type ReplicationRuleFilter struct { // in an And tag. And *ReplicationRuleAndOperator `type:"structure"` - // An object keyname prefix that identifies the subset of objects to which the - // rule applies. + // An object key name prefix that identifies the subset of objects to which + // the rule applies. Prefix *string `type:"string"` // A container for specifying a tag key and value. @@ -21752,6 +27498,91 @@ func (s *ReplicationRuleFilter) SetTag(v *Tag) *ReplicationRuleFilter { return s } +// A container specifying S3 Replication Time Control (S3 RTC) related information, +// including whether S3 RTC is enabled and the time when all objects and operations +// on objects must be replicated. Must be specified together with a Metrics +// block. +type ReplicationTime struct { + _ struct{} `type:"structure"` + + // Specifies whether the replication time is enabled. + // + // Status is a required field + Status *string `type:"string" required:"true" enum:"ReplicationTimeStatus"` + + // A container specifying the time by which replication should be complete for + // all objects and operations on objects. + // + // Time is a required field + Time *ReplicationTimeValue `type:"structure" required:"true"` +} + +// String returns the string representation +func (s ReplicationTime) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s ReplicationTime) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *ReplicationTime) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "ReplicationTime"} + if s.Status == nil { + invalidParams.Add(request.NewErrParamRequired("Status")) + } + if s.Time == nil { + invalidParams.Add(request.NewErrParamRequired("Time")) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetStatus sets the Status field's value. +func (s *ReplicationTime) SetStatus(v string) *ReplicationTime { + s.Status = &v + return s +} + +// SetTime sets the Time field's value. +func (s *ReplicationTime) SetTime(v *ReplicationTimeValue) *ReplicationTime { + s.Time = v + return s +} + +// A container specifying the time value for S3 Replication Time Control (S3 +// RTC) and replication metrics EventThreshold. +type ReplicationTimeValue struct { + _ struct{} `type:"structure"` + + // Contains an integer specifying time in minutes. + // + // Valid values: 15 minutes. + Minutes *int64 `type:"integer"` +} + +// String returns the string representation +func (s ReplicationTimeValue) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s ReplicationTimeValue) GoString() string { + return s.String() +} + +// SetMinutes sets the Minutes field's value. +func (s *ReplicationTimeValue) SetMinutes(v int64) *ReplicationTimeValue { + s.Minutes = &v + return s +} + +// Container for Payer. type RequestPaymentConfiguration struct { _ struct{} `type:"structure"` @@ -21790,6 +27621,7 @@ func (s *RequestPaymentConfiguration) SetPayer(v string) *RequestPaymentConfigur return s } +// Container for specifying if periodic QueryProgress messages should be sent. type RequestProgress struct { _ struct{} `type:"structure"` @@ -21815,23 +27647,36 @@ func (s *RequestProgress) SetEnabled(v bool) *RequestProgress { } type RestoreObjectInput struct { - _ struct{} `type:"structure" payload:"RestoreRequest"` + _ struct{} `locationName:"RestoreObjectRequest" type:"structure" payload:"RestoreRequest"` + // The bucket name or containing the object to restore. + // + // When using this API with an access point, you must direct requests to the + // access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com. + // When using this operation using an access point through the AWS SDKs, you + // provide the access point ARN in place of the bucket name. For more information + // about access point ARNs, see Using Access Points (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-access-points.html) + // in the Amazon Simple Storage Service Developer Guide. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + // Object key for which the operation was initiated. + // // Key is a required field Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // Documentation on downloading objects from requester pays buckets can be found - // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` // Container for restore job parameters. RestoreRequest *RestoreRequest `locationName:"RestoreRequest" type:"structure" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` + // VersionId used to reference a specific version of the object. VersionId *string `location:"querystring" locationName:"versionId" type:"string"` } @@ -21909,6 +27754,20 @@ func (s *RestoreObjectInput) SetVersionId(v string) *RestoreObjectInput { return s } +func (s *RestoreObjectInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *RestoreObjectInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type RestoreObjectOutput struct { _ struct{} `type:"structure"` @@ -21954,7 +27813,7 @@ type RestoreRequest struct { // The optional description for the job. Description *string `type:"string"` - // Glacier related parameters pertaining to this job. Do not use with restores + // S3 Glacier related parameters pertaining to this job. Do not use with restores // that specify OutputLocation. GlacierJobParameters *GlacierJobParameters `type:"structure"` @@ -21964,7 +27823,7 @@ type RestoreRequest struct { // Describes the parameters for Select job types. SelectParameters *SelectParameters `type:"structure"` - // Glacier retrieval tier at which the restore will be processed. + // S3 Glacier retrieval tier at which the restore will be processed. Tier *string `type:"string" enum:"Tier"` // Type of restore request. @@ -22048,6 +27907,7 @@ func (s *RestoreRequest) SetType(v string) *RestoreRequest { return s } +// Specifies the redirect behavior and when a redirect is applied. type RoutingRule struct { _ struct{} `type:"structure"` @@ -22100,16 +27960,24 @@ func (s *RoutingRule) SetRedirect(v *Redirect) *RoutingRule { return s } +// Specifies lifecycle rules for an Amazon S3 bucket. For more information, +// see Put Bucket Lifecycle Configuration (https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUTlifecycle.html) +// in the Amazon Simple Storage Service API Reference. For examples, see Put +// Bucket Lifecycle Configuration Examples (https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html#API_PutBucketLifecycleConfiguration_Examples) type Rule struct { _ struct{} `type:"structure"` - // Specifies the days since the initiation of an Incomplete Multipart Upload - // that Lifecycle will wait before permanently removing all parts of the upload. + // Specifies the days since the initiation of an incomplete multipart upload + // that Amazon S3 will wait before permanently removing all parts of the upload. + // For more information, see Aborting Incomplete Multipart Uploads Using a Bucket + // Lifecycle Policy (https://docs.aws.amazon.com/AmazonS3/latest/dev/mpuoverview.html#mpu-abort-incomplete-mpu-lifecycle-config) + // in the Amazon Simple Storage Service Developer Guide. AbortIncompleteMultipartUpload *AbortIncompleteMultipartUpload `type:"structure"` + // Specifies the expiration for the lifecycle of the object. Expiration *LifecycleExpiration `type:"structure"` - // Unique identifier for the rule. The value cannot be longer than 255 characters. + // Unique identifier for the rule. The value can't be longer than 255 characters. ID *string `type:"string"` // Specifies when noncurrent object versions expire. Upon expiration, Amazon @@ -22120,24 +27988,30 @@ type Rule struct { NoncurrentVersionExpiration *NoncurrentVersionExpiration `type:"structure"` // Container for the transition rule that describes when noncurrent objects - // transition to the STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING or GLACIER - // storage class. If your bucket is versioning-enabled (or versioning is suspended), - // you can set this action to request that Amazon S3 transition noncurrent object - // versions to the STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING or GLACIER storage - // class at a specific period in the object's lifetime. + // transition to the STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING, GLACIER, + // or DEEP_ARCHIVE storage class. If your bucket is versioning-enabled (or versioning + // is suspended), you can set this action to request that Amazon S3 transition + // noncurrent object versions to the STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING, + // GLACIER, or DEEP_ARCHIVE storage class at a specific period in the object's + // lifetime. NoncurrentVersionTransition *NoncurrentVersionTransition `type:"structure"` - // Prefix identifying one or more objects to which the rule applies. + // Object key prefix that identifies one or more objects to which this rule + // applies. // // Prefix is a required field Prefix *string `type:"string" required:"true"` - // If 'Enabled', the rule is currently being applied. If 'Disabled', the rule - // is not currently being applied. + // If Enabled, the rule is currently being applied. If Disabled, the rule is + // not currently being applied. // // Status is a required field Status *string `type:"string" required:"true" enum:"ExpirationStatus"` + // Specifies when an object transitions to a specified storage class. For more + // information about Amazon S3 lifecycle configuration rules, see Transitioning + // Objects Using Amazon S3 Lifecycle (https://docs.aws.amazon.com/AmazonS3/latest/dev/lifecycle-transition-general-considerations.html) + // in the Amazon Simple Storage Service Developer Guide. Transition *Transition `type:"structure"` } @@ -22215,12 +28089,12 @@ func (s *Rule) SetTransition(v *Transition) *Rule { return s } -// Specifies the use of SSE-KMS to encrypt delivered Inventory reports. +// Specifies the use of SSE-KMS to encrypt delivered inventory reports. type SSEKMS struct { _ struct{} `locationName:"SSE-KMS" type:"structure"` - // Specifies the ID of the AWS Key Management Service (KMS) master encryption - // key to use for encrypting Inventory reports. + // Specifies the ID of the AWS Key Management Service (AWS KMS) symmetric customer + // managed customer master key (CMK) to use for encrypting inventory reports. // // KeyId is a required field KeyId *string `type:"string" required:"true" sensitive:"true"` @@ -22255,7 +28129,7 @@ func (s *SSEKMS) SetKeyId(v string) *SSEKMS { return s } -// Specifies the use of SSE-S3 to encrypt delivered Inventory reports. +// Specifies the use of SSE-S3 to encrypt delivered inventory reports. type SSES3 struct { _ struct{} `locationName:"SSE-S3" type:"structure"` } @@ -22270,75 +28144,51 @@ func (s SSES3) GoString() string { return s.String() } -// SelectObjectContentEventStream provides handling of EventStreams for -// the SelectObjectContent API. -// -// Use this type to receive SelectObjectContentEventStream events. The events -// can be read from the Events channel member. -// -// The events that can be received are: -// -// * ContinuationEvent -// * EndEvent -// * ProgressEvent -// * RecordsEvent -// * StatsEvent -type SelectObjectContentEventStream struct { - // Reader is the EventStream reader for the SelectObjectContentEventStream - // events. This value is automatically set by the SDK when the API call is made - // Use this member when unit testing your code with the SDK to mock out the - // EventStream Reader. - // - // Must not be nil. - Reader SelectObjectContentEventStreamReader +// Specifies the byte range of the object to get the records from. A record +// is processed when its first byte is contained by the range. This parameter +// is optional, but when specified, it must not be empty. See RFC 2616, Section +// 14.35.1 about how to specify the start and end of the range. +type ScanRange struct { + _ struct{} `type:"structure"` - // StreamCloser is the io.Closer for the EventStream connection. For HTTP - // EventStream this is the response Body. The stream will be closed when - // the Close method of the EventStream is called. - StreamCloser io.Closer + // Specifies the end of the byte range. This parameter is optional. Valid values: + // non-negative integers. The default value is one less than the size of the + // object being queried. If only the End parameter is supplied, it is interpreted + // to mean scan the last N bytes of the file. For example, 50 + // means scan the last 50 bytes. + End *int64 `type:"long"` + + // Specifies the start of the byte range. This parameter is optional. Valid + // values: non-negative integers. The default value is 0. If only start is supplied, + // it means scan from that point to the end of the file.For example; 50 + // means scan from byte 50 until the end of the file. + Start *int64 `type:"long"` } -// Close closes the EventStream. This will also cause the Events channel to be -// closed. You can use the closing of the Events channel to terminate your -// application's read from the API's EventStream. -// -// Will close the underlying EventStream reader. For EventStream over HTTP -// connection this will also close the HTTP connection. -// -// Close must be called when done using the EventStream API. Not calling Close -// may result in resource leaks. -func (es *SelectObjectContentEventStream) Close() (err error) { - es.Reader.Close() - return es.Err() +// String returns the string representation +func (s ScanRange) String() string { + return awsutil.Prettify(s) } -// Err returns any error that occurred while reading EventStream Events from -// the service API's response. Returns nil if there were no errors. -func (es *SelectObjectContentEventStream) Err() error { - if err := es.Reader.Err(); err != nil { - return err - } - es.StreamCloser.Close() +// GoString returns the string representation +func (s ScanRange) GoString() string { + return s.String() +} - return nil +// SetEnd sets the End field's value. +func (s *ScanRange) SetEnd(v int64) *ScanRange { + s.End = &v + return s } -// Events returns a channel to read EventStream Events from the -// SelectObjectContent API. -// -// These events are: -// -// * ContinuationEvent -// * EndEvent -// * ProgressEvent -// * RecordsEvent -// * StatsEvent -func (es *SelectObjectContentEventStream) Events() <-chan SelectObjectContentEventStreamEvent { - return es.Reader.Events() +// SetStart sets the Start field's value. +func (s *ScanRange) SetStart(v int64) *ScanRange { + s.Start = &v + return s } // SelectObjectContentEventStreamEvent groups together all EventStream -// events read from the SelectObjectContent API. +// events writes for SelectObjectContentEventStream. // // These events are: // @@ -22349,11 +28199,12 @@ func (es *SelectObjectContentEventStream) Events() <-chan SelectObjectContentEve // * StatsEvent type SelectObjectContentEventStreamEvent interface { eventSelectObjectContentEventStream() + eventstreamapi.Marshaler + eventstreamapi.Unmarshaler } -// SelectObjectContentEventStreamReader provides the interface for reading EventStream -// Events from the SelectObjectContent API. The -// default implementation for this interface will be SelectObjectContentEventStream. +// SelectObjectContentEventStreamReader provides the interface for reading to the stream. The +// default implementation for this interface will be SelectObjectContentEventStreamData. // // The reader's Close method must allow multiple concurrent calls. // @@ -22364,12 +28215,12 @@ type SelectObjectContentEventStreamEvent interface { // * ProgressEvent // * RecordsEvent // * StatsEvent +// * SelectObjectContentEventStreamUnknownEvent type SelectObjectContentEventStreamReader interface { // Returns a channel of events as they are read from the event stream. Events() <-chan SelectObjectContentEventStreamEvent - // Close will close the underlying event stream reader. For event stream over - // HTTP this will also close the HTTP connection. + // Close will stop the reader reading events from the stream. Close() error // Returns any error that has occurred while reading from the event stream. @@ -22379,57 +28230,44 @@ type SelectObjectContentEventStreamReader interface { type readSelectObjectContentEventStream struct { eventReader *eventstreamapi.EventReader stream chan SelectObjectContentEventStreamEvent - errVal atomic.Value + err *eventstreamapi.OnceError done chan struct{} closeOnce sync.Once } -func newReadSelectObjectContentEventStream( - reader io.ReadCloser, - unmarshalers request.HandlerList, - logger aws.Logger, - logLevel aws.LogLevelType, -) *readSelectObjectContentEventStream { +func newReadSelectObjectContentEventStream(eventReader *eventstreamapi.EventReader) *readSelectObjectContentEventStream { r := &readSelectObjectContentEventStream{ - stream: make(chan SelectObjectContentEventStreamEvent), - done: make(chan struct{}), + eventReader: eventReader, + stream: make(chan SelectObjectContentEventStreamEvent), + done: make(chan struct{}), + err: eventstreamapi.NewOnceError(), } - - r.eventReader = eventstreamapi.NewEventReader( - reader, - protocol.HandlerPayloadUnmarshal{ - Unmarshalers: unmarshalers, - }, - r.unmarshalerForEventType, - ) - r.eventReader.UseLogger(logger, logLevel) + go r.readEventStream() return r } -// Close will close the underlying event stream reader. For EventStream over -// HTTP this will also close the HTTP connection. +// Close will close the underlying event stream reader. func (r *readSelectObjectContentEventStream) Close() error { r.closeOnce.Do(r.safeClose) - return r.Err() } +func (r *readSelectObjectContentEventStream) ErrorSet() <-chan struct{} { + return r.err.ErrorSet() +} + +func (r *readSelectObjectContentEventStream) Closed() <-chan struct{} { + return r.done +} + func (r *readSelectObjectContentEventStream) safeClose() { close(r.done) - err := r.eventReader.Close() - if err != nil { - r.errVal.Store(err) - } } func (r *readSelectObjectContentEventStream) Err() error { - if v := r.errVal.Load(); v != nil { - return v.(error) - } - - return nil + return r.err.Err() } func (r *readSelectObjectContentEventStream) Events() <-chan SelectObjectContentEventStreamEvent { @@ -22437,6 +28275,7 @@ func (r *readSelectObjectContentEventStream) Events() <-chan SelectObjectContent } func (r *readSelectObjectContentEventStream) readEventStream() { + defer r.Close() defer close(r.stream) for { @@ -22451,7 +28290,10 @@ func (r *readSelectObjectContentEventStream) readEventStream() { return default: } - r.errVal.Store(err) + if _, ok := err.(*eventstreamapi.UnknownMessageTypeError); ok { + continue + } + r.err.SetError(err) return } @@ -22463,40 +28305,63 @@ func (r *readSelectObjectContentEventStream) readEventStream() { } } -func (r *readSelectObjectContentEventStream) unmarshalerForEventType( - eventType string, -) (eventstreamapi.Unmarshaler, error) { +type unmarshalerForSelectObjectContentEventStreamEvent struct { + metadata protocol.ResponseMetadata +} + +func (u unmarshalerForSelectObjectContentEventStreamEvent) UnmarshalerForEventName(eventType string) (eventstreamapi.Unmarshaler, error) { switch eventType { case "Cont": return &ContinuationEvent{}, nil - case "End": return &EndEvent{}, nil - case "Progress": return &ProgressEvent{}, nil - case "Records": return &RecordsEvent{}, nil - case "Stats": return &StatsEvent{}, nil default: - return nil, awserr.New( - request.ErrCodeSerialization, - fmt.Sprintf("unknown event type name, %s, for SelectObjectContentEventStream", eventType), - nil, - ) + return &SelectObjectContentEventStreamUnknownEvent{Type: eventType}, nil } } +// SelectObjectContentEventStreamUnknownEvent provides a failsafe event for the +// SelectObjectContentEventStream group of events when an unknown event is received. +type SelectObjectContentEventStreamUnknownEvent struct { + Type string + Message eventstream.Message +} + +// The SelectObjectContentEventStreamUnknownEvent is and event in the SelectObjectContentEventStream +// group of events. +func (s *SelectObjectContentEventStreamUnknownEvent) eventSelectObjectContentEventStream() {} + +// MarshalEvent marshals the type into an stream event value. This method +// should only used internally within the SDK's EventStream handling. +func (e *SelectObjectContentEventStreamUnknownEvent) MarshalEvent(pm protocol.PayloadMarshaler) ( + msg eventstream.Message, err error, +) { + return e.Message.Clone(), nil +} + +// UnmarshalEvent unmarshals the EventStream Message into the SelectObjectContentEventStreamData value. +// This method is only used internally within the SDK's EventStream handling. +func (e *SelectObjectContentEventStreamUnknownEvent) UnmarshalEvent( + payloadUnmarshaler protocol.PayloadUnmarshaler, + msg eventstream.Message, +) error { + e.Message = msg.Clone() + return nil +} + // Request to filter the contents of an Amazon S3 object based on a simple Structured // Query Language (SQL) statement. In the request, along with the SQL expression, // you must specify a data serialization format (JSON or CSV) of the object. // Amazon S3 uses this to parse object data into records. It returns only records // that match the specified SQL expression. You must also specify the data serialization // format for the response. For more information, see S3Select API Documentation -// (http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html). +// (https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html). type SelectObjectContentInput struct { _ struct{} `locationName:"SelectObjectContentRequest" type:"structure" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` @@ -22510,7 +28375,7 @@ type SelectObjectContentInput struct { // Expression is a required field Expression *string `type:"string" required:"true"` - // The type of the provided expression (for example., SQL). + // The type of the provided expression (for example, SQL). // // ExpressionType is a required field ExpressionType *string `type:"string" required:"true" enum:"ExpressionType"` @@ -22533,17 +28398,35 @@ type SelectObjectContentInput struct { // Specifies if periodic request progress information should be enabled. RequestProgress *RequestProgress `type:"structure"` - // The SSE Algorithm used to encrypt the object. For more information, see - // Server-Side Encryption (Using Customer-Provided Encryption Keys (http://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html). + // The SSE Algorithm used to encrypt the object. For more information, see Server-Side + // Encryption (Using Customer-Provided Encryption Keys (https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html). SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` - // The SSE Customer Key. For more information, see Server-Side Encryption (Using - // Customer-Provided Encryption Keys (http://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html). - SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"` + // The SSE Customer Key. For more information, see Server-Side Encryption (Using + // Customer-Provided Encryption Keys (https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html). + SSECustomerKey *string `marshal-as:"blob" location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"` - // The SSE Customer Key MD5. For more information, see Server-Side Encryption - // (Using Customer-Provided Encryption Keys (http://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html). + // The SSE Customer Key MD5. For more information, see Server-Side Encryption + // (Using Customer-Provided Encryption Keys (https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html). SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` + + // Specifies the byte range of the object to get the records from. A record + // is processed when its first byte is contained by the range. This parameter + // is optional, but when specified, it must not be empty. See RFC 2616, Section + // 14.35.1 about how to specify the start and end of the range. + // + // ScanRangemay be used in the following ways: + // + // * 50100 - process only + // the records starting between the bytes 50 and 100 (inclusive, counting + // from zero) + // + // * 50 - process only the records + // starting after the byte 50 + // + // * 50 - process only the records within + // the last 50 bytes of the file. + ScanRange *ScanRange `type:"structure"` } // String returns the string representation @@ -22664,11 +28547,30 @@ func (s *SelectObjectContentInput) SetSSECustomerKeyMD5(v string) *SelectObjectC return s } +// SetScanRange sets the ScanRange field's value. +func (s *SelectObjectContentInput) SetScanRange(v *ScanRange) *SelectObjectContentInput { + s.ScanRange = v + return s +} + +func (s *SelectObjectContentInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *SelectObjectContentInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type SelectObjectContentOutput struct { _ struct{} `type:"structure" payload:"Payload"` - // Use EventStream to use the API's stream. - EventStream *SelectObjectContentEventStream `type:"structure"` + EventStream *SelectObjectContentEventStream } // String returns the string representation @@ -22681,29 +28583,17 @@ func (s SelectObjectContentOutput) GoString() string { return s.String() } -// SetEventStream sets the EventStream field's value. func (s *SelectObjectContentOutput) SetEventStream(v *SelectObjectContentEventStream) *SelectObjectContentOutput { s.EventStream = v return s } +func (s *SelectObjectContentOutput) GetEventStream() *SelectObjectContentEventStream { + return s.EventStream +} -func (s *SelectObjectContentOutput) runEventStreamLoop(r *request.Request) { - if r.Error != nil { - return - } - reader := newReadSelectObjectContentEventStream( - r.HTTPResponse.Body, - r.Handlers.UnmarshalStream, - r.Config.Logger, - r.Config.LogLevel.Value(), - ) - go reader.readEventStream() - - eventStream := &SelectObjectContentEventStream{ - StreamCloser: r.HTTPResponse.Body, - Reader: reader, - } - s.EventStream = eventStream +// GetStream returns the type to interact with the event stream. +func (s *SelectObjectContentOutput) GetStream() *SelectObjectContentEventStream { + return s.EventStream } // Describes the parameters for Select job types. @@ -22715,7 +28605,7 @@ type SelectParameters struct { // Expression is a required field Expression *string `type:"string" required:"true"` - // The type of the provided expression (e.g., SQL). + // The type of the provided expression (for example, SQL). // // ExpressionType is a required field ExpressionType *string `type:"string" required:"true" enum:"ExpressionType"` @@ -22788,13 +28678,31 @@ func (s *SelectParameters) SetOutputSerialization(v *OutputSerialization) *Selec } // Describes the default server-side encryption to apply to new objects in the -// bucket. If Put Object request does not specify any server-side encryption, -// this default encryption will be applied. +// bucket. If a PUT Object request doesn't specify any server-side encryption, +// this default encryption will be applied. For more information, see PUT Bucket +// encryption (https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUTencryption.html) +// in the Amazon Simple Storage Service API Reference. type ServerSideEncryptionByDefault struct { _ struct{} `type:"structure"` - // KMS master key ID to use for the default encryption. This parameter is allowed - // if SSEAlgorithm is aws:kms. + // AWS Key Management Service (KMS) customer master key ID to use for the default + // encryption. This parameter is allowed if and only if SSEAlgorithm is set + // to aws:kms. + // + // You can specify the key ID or the Amazon Resource Name (ARN) of the CMK. + // However, if you are using encryption with cross-account operations, you must + // use a fully qualified CMK ARN. For more information, see Using encryption + // for cross-account operations (https://docs.aws.amazon.com/AmazonS3/latest/dev/bucket-encryption.html#bucket-encryption-update-bucket-policy). + // + // For example: + // + // * Key ID: 1234abcd-12ab-34cd-56ef-1234567890ab + // + // * Key ARN: arn:aws:kms:us-east-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab + // + // Amazon S3 only supports symmetric CMKs and not asymmetric CMKs. For more + // information, see Using Symmetric and Asymmetric Keys (https://docs.aws.amazon.com/kms/latest/developerguide/symmetric-asymmetric.html) + // in the AWS Key Management Service Developer Guide. KMSMasterKeyID *string `type:"string" sensitive:"true"` // Server-side encryption algorithm to use for the default encryption. @@ -22838,8 +28746,7 @@ func (s *ServerSideEncryptionByDefault) SetSSEAlgorithm(v string) *ServerSideEnc return s } -// Container for server-side encryption configuration rules. Currently S3 supports -// one rule only. +// Specifies the default server-side-encryption configuration. type ServerSideEncryptionConfiguration struct { _ struct{} `type:"structure"` @@ -22889,13 +28796,12 @@ func (s *ServerSideEncryptionConfiguration) SetRules(v []*ServerSideEncryptionRu return s } -// Container for information about a particular server-side encryption configuration -// rule. +// Specifies the default server-side encryption configuration. type ServerSideEncryptionRule struct { _ struct{} `type:"structure"` - // Describes the default server-side encryption to apply to new objects in the - // bucket. If Put Object request does not specify any server-side encryption, + // Specifies the default server-side encryption to apply to new objects in the + // bucket. If a PUT Object request doesn't specify any server-side encryption, // this default encryption will be applied. ApplyServerSideEncryptionByDefault *ServerSideEncryptionByDefault `type:"structure"` } @@ -22931,13 +28837,17 @@ func (s *ServerSideEncryptionRule) SetApplyServerSideEncryptionByDefault(v *Serv return s } -// A container for filters that define which source objects should be replicated. +// A container that describes additional filters for identifying the source +// objects that you want to replicate. You can choose to enable or disable the +// replication of these objects. Currently, Amazon S3 supports only the filter +// that you can specify for objects created with server-side encryption using +// a customer master key (CMK) stored in AWS Key Management Service (SSE-KMS). type SourceSelectionCriteria struct { _ struct{} `type:"structure"` - // A container for filter information for the selection of S3 objects encrypted - // with AWS KMS. If you include SourceSelectionCriteria in the replication configuration, - // this element is required. + // A container for filter information for the selection of Amazon S3 objects + // encrypted with AWS KMS. If you include SourceSelectionCriteria in the replication + // configuration, this element is required. SseKmsEncryptedObjects *SseKmsEncryptedObjects `type:"structure"` } @@ -22977,8 +28887,8 @@ func (s *SourceSelectionCriteria) SetSseKmsEncryptedObjects(v *SseKmsEncryptedOb type SseKmsEncryptedObjects struct { _ struct{} `type:"structure"` - // If the status is not Enabled, replication for S3 objects encrypted with AWS - // KMS is disabled. + // Specifies whether Amazon S3 replicates objects created with server-side encryption + // using a customer master key (CMK) stored in AWS Key Management Service. // // Status is a required field Status *string `type:"string" required:"true" enum:"SseKmsEncryptedObjectsStatus"` @@ -23013,6 +28923,7 @@ func (s *SseKmsEncryptedObjects) SetStatus(v string) *SseKmsEncryptedObjects { return s } +// Container for the stats details. type Stats struct { _ struct{} `type:"structure"` @@ -23054,6 +28965,7 @@ func (s *Stats) SetBytesScanned(v int64) *Stats { return s } +// Container for the Stats Event. type StatsEvent struct { _ struct{} `locationName:"StatsEvent" type:"structure" payload:"Details"` @@ -23094,11 +29006,26 @@ func (s *StatsEvent) UnmarshalEvent( return nil } +// MarshalEvent marshals the type into an stream event value. This method +// should only used internally within the SDK's EventStream handling. +func (s *StatsEvent) MarshalEvent(pm protocol.PayloadMarshaler) (msg eventstream.Message, err error) { + msg.Headers.Set(eventstreamapi.MessageTypeHeader, eventstream.StringValue(eventstreamapi.EventMessageType)) + var buf bytes.Buffer + if err = pm.MarshalPayload(&buf, s); err != nil { + return eventstream.Message{}, err + } + msg.Payload = buf.Bytes() + return msg, err +} + +// Specifies data related to access patterns to be collected and made available +// to analyze the tradeoffs between different storage classes for an Amazon +// S3 bucket. type StorageClassAnalysis struct { _ struct{} `type:"structure"` - // A container used to describe how data related to the storage class analysis - // should be exported. + // Specifies how data related to the storage class analysis for an Amazon S3 + // bucket should be exported. DataExport *StorageClassAnalysisDataExport `type:"structure"` } @@ -23133,6 +29060,8 @@ func (s *StorageClassAnalysis) SetDataExport(v *StorageClassAnalysisDataExport) return s } +// Container for data related to the storage class analysis for an Amazon S3 +// bucket for export. type StorageClassAnalysisDataExport struct { _ struct{} `type:"structure"` @@ -23190,6 +29119,7 @@ func (s *StorageClassAnalysisDataExport) SetOutputSchemaVersion(v string) *Stora return s } +// A container of a key value name pair. type Tag struct { _ struct{} `type:"structure"` @@ -23245,9 +29175,12 @@ func (s *Tag) SetValue(v string) *Tag { return s } +// Container for TagSet elements. type Tagging struct { _ struct{} `type:"structure"` + // A collection for a set of tags + // // TagSet is a required field TagSet []*Tag `locationNameList:"Tag" type:"list" required:"true"` } @@ -23291,9 +29224,11 @@ func (s *Tagging) SetTagSet(v []*Tag) *Tagging { return s } +// Container for granting information. type TargetGrant struct { _ struct{} `type:"structure"` + // Container for the person being granted permissions. Grantee *Grantee `type:"structure" xmlPrefix:"xsi" xmlURI:"http://www.w3.org/2001/XMLSchema-instance"` // Logging permissions assigned to the Grantee for the bucket. @@ -23338,16 +29273,20 @@ func (s *TargetGrant) SetPermission(v string) *TargetGrant { } // A container for specifying the configuration for publication of messages -// to an Amazon Simple Notification Service (Amazon SNS) topic.when Amazon S3 +// to an Amazon Simple Notification Service (Amazon SNS) topic when Amazon S3 // detects specified events. type TopicConfiguration struct { _ struct{} `type:"structure"` + // The Amazon S3 bucket event about which to send notifications. For more information, + // see Supported Event Types (https://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html) + // in the Amazon Simple Storage Service Developer Guide. + // // Events is a required field Events []*string `locationName:"Event" type:"list" flattened:"true" required:"true"` - // A container for object key name filtering rules. For information about key - // name filtering, see Configuring Event Notifications (http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html) + // Specifies object key name filtering rules. For information about key name + // filtering, see Configuring Event Notifications (https://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html) // in the Amazon Simple Storage Service Developer Guide. Filter *NotificationConfigurationFilter `type:"structure"` @@ -23356,7 +29295,7 @@ type TopicConfiguration struct { Id *string `type:"string"` // The Amazon Resource Name (ARN) of the Amazon SNS topic to which Amazon S3 - // will publish a message when it detects events of the specified type. + // publishes a message when it detects events of the specified type. // // TopicArn is a required field TopicArn *string `locationName:"Topic" type:"string" required:"true"` @@ -23412,6 +29351,10 @@ func (s *TopicConfiguration) SetTopicArn(v string) *TopicConfiguration { return s } +// A container for specifying the configuration for publication of messages +// to an Amazon Simple Notification Service (Amazon SNS) topic when Amazon S3 +// detects specified events. This data type is deprecated. Use TopicConfiguration +// instead. type TopicConfigurationDeprecated struct { _ struct{} `type:"structure"` @@ -23420,6 +29363,7 @@ type TopicConfigurationDeprecated struct { // Deprecated: Event has been deprecated Event *string `deprecated:"true" type:"string" enum:"Event"` + // A collection of events related to objects Events []*string `locationName:"Event" type:"list" flattened:"true"` // An optional unique identifier for configurations in a notification configuration. @@ -23465,18 +29409,22 @@ func (s *TopicConfigurationDeprecated) SetTopic(v string) *TopicConfigurationDep return s } +// Specifies when an object transitions to a specified storage class. For more +// information about Amazon S3 lifecycle configuration rules, see Transitioning +// Objects Using Amazon S3 Lifecycle (https://docs.aws.amazon.com/AmazonS3/latest/dev/lifecycle-transition-general-considerations.html) +// in the Amazon Simple Storage Service Developer Guide. type Transition struct { _ struct{} `type:"structure"` - // Indicates at what date the object is to be moved or deleted. Should be in - // GMT ISO 8601 Format. + // Indicates when objects are transitioned to the specified storage class. The + // date value must be in ISO 8601 format. The time is always midnight UTC. Date *time.Time `type:"timestamp" timestampFormat:"iso8601"` - // Indicates the lifetime, in days, of the objects that are subject to the rule. - // The value must be a non-zero positive integer. + // Indicates the number of days after creation when objects are transitioned + // to the specified storage class. The value must be a positive integer. Days *int64 `type:"integer"` - // The class of storage used to store the object. + // The storage class to which you want the object to transition. StorageClass *string `type:"string" enum:"TransitionStorageClass"` } @@ -23509,8 +29457,10 @@ func (s *Transition) SetStorageClass(v string) *Transition { } type UploadPartCopyInput struct { - _ struct{} `type:"structure"` + _ struct{} `locationName:"UploadPartCopyRequest" type:"structure"` + // The bucket name. + // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` @@ -23536,23 +29486,26 @@ type UploadPartCopyInput struct { // The range of bytes to copy from the source object. The range value must use // the form bytes=first-last, where the first and last are the zero-based byte // offsets to copy. For example, bytes=0-9 indicates that you want to copy the - // first ten bytes of the source. You can copy a range only if the source object - // is greater than 5 GB. + // first 10 bytes of the source. You can copy a range only if the source object + // is greater than 5 MB. CopySourceRange *string `location:"header" locationName:"x-amz-copy-source-range" type:"string"` - // Specifies the algorithm to use when decrypting the source object (e.g., AES256). + // Specifies the algorithm to use when decrypting the source object (for example, + // AES256). CopySourceSSECustomerAlgorithm *string `location:"header" locationName:"x-amz-copy-source-server-side-encryption-customer-algorithm" type:"string"` // Specifies the customer-provided encryption key for Amazon S3 to use to decrypt // the source object. The encryption key provided in this header must be one // that was used when the source object was created. - CopySourceSSECustomerKey *string `location:"header" locationName:"x-amz-copy-source-server-side-encryption-customer-key" type:"string" sensitive:"true"` + CopySourceSSECustomerKey *string `marshal-as:"blob" location:"header" locationName:"x-amz-copy-source-server-side-encryption-customer-key" type:"string" sensitive:"true"` // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. - // Amazon S3 uses this header for a message integrity check to ensure the encryption - // key was transmitted without error. + // Amazon S3 uses this header for a message integrity check to ensure that the + // encryption key was transmitted without error. CopySourceSSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-copy-source-server-side-encryption-customer-key-MD5" type:"string"` + // Object key for which the multipart upload was initiated. + // // Key is a required field Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"` @@ -23562,26 +29515,28 @@ type UploadPartCopyInput struct { // PartNumber is a required field PartNumber *int64 `location:"querystring" locationName:"partNumber" type:"integer" required:"true"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // Documentation on downloading objects from requester pays buckets can be found - // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` - // Specifies the algorithm to use to when encrypting the object (e.g., AES256). + // Specifies the algorithm to use to when encrypting the object (for example, + // AES256). SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` // Specifies the customer-provided encryption key for Amazon S3 to use in encrypting // data. This value is used to store the object and then it is discarded; Amazon - // does not store the encryption key. The key must be appropriate for use with - // the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm + // S3 does not store the encryption key. The key must be appropriate for use + // with the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm // header. This must be the same encryption key specified in the initiate multipart // upload request. - SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"` + SSECustomerKey *string `marshal-as:"blob" location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"` // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. - // Amazon S3 uses this header for a message integrity check to ensure the encryption - // key was transmitted without error. + // Amazon S3 uses this header for a message integrity check to ensure that the + // encryption key was transmitted without error. SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` // Upload ID identifying the multipart upload whose part is being copied. @@ -23754,9 +29709,24 @@ func (s *UploadPartCopyInput) SetUploadId(v string) *UploadPartCopyInput { return s } +func (s *UploadPartCopyInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *UploadPartCopyInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type UploadPartCopyOutput struct { _ struct{} `type:"structure" payload:"CopyPartResult"` + // Container for all response elements. CopyPartResult *CopyPartResult `type:"structure"` // The version of the source object that was copied, if you have enabled versioning @@ -23773,16 +29743,17 @@ type UploadPartCopyOutput struct { SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` // If server-side encryption with a customer-provided encryption key was requested, - // the response will include this header to provide round trip message integrity + // the response will include this header to provide round-trip message integrity // verification of the customer-provided encryption key. SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` - // If present, specifies the ID of the AWS Key Management Service (KMS) master - // encryption key that was used for the object. + // If present, specifies the ID of the AWS Key Management Service (AWS KMS) + // symmetric customer managed customer master key (CMK) that was used for the + // object. SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` - // The Server-side encryption algorithm used when storing this object in S3 - // (e.g., AES256, aws:kms). + // The server-side encryption algorithm used when storing this object in Amazon + // S3 (for example, AES256, aws:kms). ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string" enum:"ServerSideEncryption"` } @@ -23839,7 +29810,7 @@ func (s *UploadPartCopyOutput) SetServerSideEncryption(v string) *UploadPartCopy } type UploadPartInput struct { - _ struct{} `type:"structure" payload:"Body"` + _ struct{} `locationName:"UploadPartRequest" type:"structure" payload:"Body"` // Object data. Body io.ReadSeeker `type:"blob"` @@ -23853,7 +29824,9 @@ type UploadPartInput struct { // body cannot be determined automatically. ContentLength *int64 `location:"header" locationName:"Content-Length" type:"long"` - // The base64-encoded 128-bit MD5 digest of the part data. + // The base64-encoded 128-bit MD5 digest of the part data. This parameter is + // auto-populated when using the command from the CLI. This parameter is required + // if object lock parameters are specified. ContentMD5 *string `location:"header" locationName:"Content-MD5" type:"string"` // Object key for which the multipart upload was initiated. @@ -23867,26 +29840,28 @@ type UploadPartInput struct { // PartNumber is a required field PartNumber *int64 `location:"querystring" locationName:"partNumber" type:"integer" required:"true"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // Documentation on downloading objects from requester pays buckets can be found - // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` - // Specifies the algorithm to use to when encrypting the object (e.g., AES256). + // Specifies the algorithm to use to when encrypting the object (for example, + // AES256). SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` // Specifies the customer-provided encryption key for Amazon S3 to use in encrypting // data. This value is used to store the object and then it is discarded; Amazon - // does not store the encryption key. The key must be appropriate for use with - // the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm + // S3 does not store the encryption key. The key must be appropriate for use + // with the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm // header. This must be the same encryption key specified in the initiate multipart // upload request. - SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"` + SSECustomerKey *string `marshal-as:"blob" location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"` // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. - // Amazon S3 uses this header for a message integrity check to ensure the encryption - // key was transmitted without error. + // Amazon S3 uses this header for a message integrity check to ensure that the + // encryption key was transmitted without error. SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` // Upload ID identifying the multipart upload whose part is being uploaded. @@ -24013,6 +29988,20 @@ func (s *UploadPartInput) SetUploadId(v string) *UploadPartInput { return s } +func (s *UploadPartInput) getEndpointARN() (arn.Resource, error) { + if s.Bucket == nil { + return nil, fmt.Errorf("member Bucket is nil") + } + return parseEndpointARN(*s.Bucket) +} + +func (s *UploadPartInput) hasEndpointARN() bool { + if s.Bucket == nil { + return false + } + return arn.IsARN(*s.Bucket) +} + type UploadPartOutput struct { _ struct{} `type:"structure"` @@ -24029,16 +30018,16 @@ type UploadPartOutput struct { SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` // If server-side encryption with a customer-provided encryption key was requested, - // the response will include this header to provide round trip message integrity + // the response will include this header to provide round-trip message integrity // verification of the customer-provided encryption key. SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` - // If present, specifies the ID of the AWS Key Management Service (KMS) master - // encryption key that was used for the object. + // If present, specifies the ID of the AWS Key Management Service (AWS KMS) + // symmetric customer managed customer master key (CMK) was used for the object. SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` - // The Server-side encryption algorithm used when storing this object in S3 - // (e.g., AES256, aws:kms). + // The server-side encryption algorithm used when storing this object in Amazon + // S3 (for example, AES256, aws:kms). ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string" enum:"ServerSideEncryption"` } @@ -24088,6 +30077,9 @@ func (s *UploadPartOutput) SetServerSideEncryption(v string) *UploadPartOutput { return s } +// Describes the versioning state of an Amazon S3 bucket. For more information, +// see PUT Bucket versioning (https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUTVersioningStatus.html) +// in the Amazon Simple Storage Service API Reference. type VersioningConfiguration struct { _ struct{} `type:"structure"` @@ -24122,15 +30114,22 @@ func (s *VersioningConfiguration) SetStatus(v string) *VersioningConfiguration { return s } +// Specifies website configuration parameters for an Amazon S3 bucket. type WebsiteConfiguration struct { _ struct{} `type:"structure"` + // The name of the error document for the website. ErrorDocument *ErrorDocument `type:"structure"` + // The name of the index document for the website. IndexDocument *IndexDocument `type:"structure"` + // The redirect behavior for every request to this bucket's website endpoint. + // + // If you specify this property, you can't specify any other property. RedirectAllRequestsTo *RedirectAllRequestsTo `type:"structure"` + // Rules that define when a redirect is applied and the redirect behavior. RoutingRules []*RoutingRule `locationNameList:"RoutingRule" type:"list"` } @@ -24343,11 +30342,37 @@ const ( // EventS3ObjectRemovedDeleteMarkerCreated is a Event enum value EventS3ObjectRemovedDeleteMarkerCreated = "s3:ObjectRemoved:DeleteMarkerCreated" + // EventS3ObjectRestore is a Event enum value + EventS3ObjectRestore = "s3:ObjectRestore:*" + // EventS3ObjectRestorePost is a Event enum value EventS3ObjectRestorePost = "s3:ObjectRestore:Post" // EventS3ObjectRestoreCompleted is a Event enum value EventS3ObjectRestoreCompleted = "s3:ObjectRestore:Completed" + + // EventS3Replication is a Event enum value + EventS3Replication = "s3:Replication:*" + + // EventS3ReplicationOperationFailedReplication is a Event enum value + EventS3ReplicationOperationFailedReplication = "s3:Replication:OperationFailedReplication" + + // EventS3ReplicationOperationNotTracked is a Event enum value + EventS3ReplicationOperationNotTracked = "s3:Replication:OperationNotTracked" + + // EventS3ReplicationOperationMissedThreshold is a Event enum value + EventS3ReplicationOperationMissedThreshold = "s3:Replication:OperationMissedThreshold" + + // EventS3ReplicationOperationReplicatedAfterThreshold is a Event enum value + EventS3ReplicationOperationReplicatedAfterThreshold = "s3:Replication:OperationReplicatedAfterThreshold" +) + +const ( + // ExistingObjectReplicationStatusEnabled is a ExistingObjectReplicationStatus enum value + ExistingObjectReplicationStatusEnabled = "Enabled" + + // ExistingObjectReplicationStatusDisabled is a ExistingObjectReplicationStatus enum value + ExistingObjectReplicationStatusDisabled = "Disabled" ) const ( @@ -24439,6 +30464,9 @@ const ( // InventoryOptionalFieldObjectLockLegalHoldStatus is a InventoryOptionalField enum value InventoryOptionalFieldObjectLockLegalHoldStatus = "ObjectLockLegalHoldStatus" + + // InventoryOptionalFieldIntelligentTieringAccessTier is a InventoryOptionalField enum value + InventoryOptionalFieldIntelligentTieringAccessTier = "IntelligentTieringAccessTier" ) const ( @@ -24473,6 +30501,14 @@ const ( MetadataDirectiveReplace = "REPLACE" ) +const ( + // MetricsStatusEnabled is a MetricsStatus enum value + MetricsStatusEnabled = "Enabled" + + // MetricsStatusDisabled is a MetricsStatus enum value + MetricsStatusDisabled = "Disabled" +) + const ( // ObjectCannedACLPrivate is a ObjectCannedACL enum value ObjectCannedACLPrivate = "private" @@ -24543,6 +30579,9 @@ const ( // ObjectStorageClassIntelligentTiering is a ObjectStorageClass enum value ObjectStorageClassIntelligentTiering = "INTELLIGENT_TIERING" + + // ObjectStorageClassDeepArchive is a ObjectStorageClass enum value + ObjectStorageClassDeepArchive = "DEEP_ARCHIVE" ) const ( @@ -24618,6 +30657,14 @@ const ( ReplicationStatusReplica = "REPLICA" ) +const ( + // ReplicationTimeStatusEnabled is a ReplicationTimeStatus enum value + ReplicationTimeStatusEnabled = "Enabled" + + // ReplicationTimeStatusDisabled is a ReplicationTimeStatus enum value + ReplicationTimeStatusDisabled = "Disabled" +) + // If present, indicates that the requester was successfully charged for the // request. const ( @@ -24625,10 +30672,11 @@ const ( RequestChargedRequester = "requester" ) -// Confirms that the requester knows that she or he will be charged for the -// request. Bucket owners need not specify this parameter in their requests. -// Documentation on downloading objects from requester pays buckets can be found -// at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html +// Confirms that the requester knows that they will be charged for the request. +// Bucket owners need not specify this parameter in their requests. For information +// about downloading objects from requester pays buckets, see Downloading Objects +// in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) +// in the Amazon S3 Developer Guide. const ( // RequestPayerRequester is a RequestPayer enum value RequestPayerRequester = "requester" @@ -24673,6 +30721,9 @@ const ( // StorageClassGlacier is a StorageClass enum value StorageClassGlacier = "GLACIER" + + // StorageClassDeepArchive is a StorageClass enum value + StorageClassDeepArchive = "DEEP_ARCHIVE" ) const ( @@ -24711,6 +30762,9 @@ const ( // TransitionStorageClassIntelligentTiering is a TransitionStorageClass enum value TransitionStorageClassIntelligentTiering = "INTELLIGENT_TIERING" + + // TransitionStorageClassDeepArchive is a TransitionStorageClass enum value + TransitionStorageClassDeepArchive = "DEEP_ARCHIVE" ) const ( diff --git a/github.com/aws/aws-sdk-go/service/s3/body_hash.go b/github.com/aws/aws-sdk-go/service/s3/body_hash.go index 5c8ce5cc8a..407f06b6ed 100644 --- a/github.com/aws/aws-sdk-go/service/s3/body_hash.go +++ b/github.com/aws/aws-sdk-go/service/s3/body_hash.go @@ -13,7 +13,6 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/internal/sdkio" ) const ( @@ -25,30 +24,6 @@ const ( appendMD5TxEncoding = "append-md5" ) -// contentMD5 computes and sets the HTTP Content-MD5 header for requests that -// require it. -func contentMD5(r *request.Request) { - h := md5.New() - - if !aws.IsReaderSeekable(r.Body) { - if r.Config.Logger != nil { - r.Config.Logger.Log(fmt.Sprintf( - "Unable to compute Content-MD5 for unseekable body, S3.%s", - r.Operation.Name)) - } - return - } - - if _, err := copySeekableBody(h, r.Body); err != nil { - r.Error = awserr.New("ContentMD5", "failed to compute body MD5", err) - return - } - - // encode the md5 checksum in base64 and set the request header. - v := base64.StdEncoding.EncodeToString(h.Sum(nil)) - r.HTTPRequest.Header.Set(contentMD5Header, v) -} - // computeBodyHashes will add Content MD5 and Content Sha256 hashes to the // request. If the body is not seekable or S3DisableContentMD5Validation set // this handler will be ignored. @@ -90,7 +65,7 @@ func computeBodyHashes(r *request.Request) { dst = io.MultiWriter(hashers...) } - if _, err := copySeekableBody(dst, r.Body); err != nil { + if _, err := aws.CopySeekableBody(dst, r.Body); err != nil { r.Error = awserr.New("BodyHashError", "failed to compute body hashes", err) return } @@ -119,28 +94,6 @@ const ( sha256HexEncLen = sha256.Size * 2 // hex.EncodedLen ) -func copySeekableBody(dst io.Writer, src io.ReadSeeker) (int64, error) { - curPos, err := src.Seek(0, sdkio.SeekCurrent) - if err != nil { - return 0, err - } - - // hash the body. seek back to the first position after reading to reset - // the body for transmission. copy errors may be assumed to be from the - // body. - n, err := io.Copy(dst, src) - if err != nil { - return n, err - } - - _, err = src.Seek(curPos, sdkio.SeekStart) - if err != nil { - return n, err - } - - return n, nil -} - // Adds the x-amz-te: append_md5 header to the request. This requests the service // responds with a trailing MD5 checksum. // diff --git a/github.com/aws/aws-sdk-go/service/s3/bucket_location.go b/github.com/aws/aws-sdk-go/service/s3/bucket_location.go index bc68a46acf..9ba8a78872 100644 --- a/github.com/aws/aws-sdk-go/service/s3/bucket_location.go +++ b/github.com/aws/aws-sdk-go/service/s3/bucket_location.go @@ -80,7 +80,8 @@ func buildGetBucketLocation(r *request.Request) { out := r.Data.(*GetBucketLocationOutput) b, err := ioutil.ReadAll(r.HTTPResponse.Body) if err != nil { - r.Error = awserr.New("SerializationError", "failed reading response body", err) + r.Error = awserr.New(request.ErrCodeSerialization, + "failed reading response body", err) return } diff --git a/github.com/aws/aws-sdk-go/service/s3/customizations.go b/github.com/aws/aws-sdk-go/service/s3/customizations.go index 95f2456363..a7698d5eb9 100644 --- a/github.com/aws/aws-sdk-go/service/s3/customizations.go +++ b/github.com/aws/aws-sdk-go/service/s3/customizations.go @@ -4,6 +4,7 @@ import ( "github.com/aws/aws-sdk-go/aws/client" "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/internal/s3err" + "github.com/aws/aws-sdk-go/service/s3/internal/arn" ) func init() { @@ -13,11 +14,12 @@ func init() { func defaultInitClientFn(c *client.Client) { // Support building custom endpoints based on config - c.Handlers.Build.PushFront(updateEndpointForS3Config) + c.Handlers.Build.PushFront(endpointHandler) // Require SSL when using SSE keys c.Handlers.Validate.PushBack(validateSSERequiresSSL) - c.Handlers.Build.PushBack(computeSSEKeys) + c.Handlers.Build.PushBack(computeSSEKeyMD5) + c.Handlers.Build.PushBack(computeCopySourceSSEKeyMD5) // S3 uses custom error unmarshaling logic c.Handlers.UnmarshalError.Clear() @@ -26,17 +28,11 @@ func defaultInitClientFn(c *client.Client) { } func defaultInitRequestFn(r *request.Request) { - // Add reuest handlers for specific platforms. + // Add request handlers for specific platforms. // e.g. 100-continue support for PUT requests using Go 1.6 platformRequestHandlers(r) switch r.Operation.Name { - case opPutBucketCors, opPutBucketLifecycle, opPutBucketPolicy, - opPutBucketTagging, opDeleteObjects, opPutBucketLifecycleConfiguration, - opPutObjectLegalHold, opPutObjectRetention, opPutObjectLockConfiguration, - opPutBucketReplication: - // These S3 operations require Content-MD5 to be set - r.Handlers.Build.PushBack(contentMD5) case opGetBucketLocation: // GetBucketLocation has custom parsing logic r.Handlers.Unmarshal.PushFront(buildGetBucketLocation) @@ -72,3 +68,8 @@ type sseCustomerKeyGetter interface { type copySourceSSECustomerKeyGetter interface { getCopySourceSSECustomerKey() string } + +type endpointARNGetter interface { + getEndpointARN() (arn.Resource, error) + hasEndpointARN() bool +} diff --git a/github.com/aws/aws-sdk-go/service/s3/doc_custom.go b/github.com/aws/aws-sdk-go/service/s3/doc_custom.go index 39b912c260..4b65f71531 100644 --- a/github.com/aws/aws-sdk-go/service/s3/doc_custom.go +++ b/github.com/aws/aws-sdk-go/service/s3/doc_custom.go @@ -63,6 +63,20 @@ // See the s3manager package's Downloader type documentation for more information. // https://docs.aws.amazon.com/sdk-for-go/api/service/s3/s3manager/#Downloader // +// Automatic URI cleaning +// +// Interacting with objects whose keys contain adjacent slashes (e.g. bucketname/foo//bar/objectname) +// requires setting DisableRestProtocolURICleaning to true in the aws.Config struct +// used by the service client. +// +// svc := s3.New(sess, &aws.Config{ +// DisableRestProtocolURICleaning: aws.Bool(true), +// }) +// out, err := svc.GetObject(&s3.GetObjectInput { +// Bucket: aws.String("bucketname"), +// Key: aws.String("//foo//bar//moo"), +// }) +// // Get Bucket Region // // GetBucketRegion will attempt to get the region for a bucket using a region diff --git a/github.com/aws/aws-sdk-go/service/s3/endpoint.go b/github.com/aws/aws-sdk-go/service/s3/endpoint.go new file mode 100644 index 0000000000..c4048fbfb6 --- /dev/null +++ b/github.com/aws/aws-sdk-go/service/s3/endpoint.go @@ -0,0 +1,233 @@ +package s3 + +import ( + "net/url" + "strings" + + "github.com/aws/aws-sdk-go/aws" + awsarn "github.com/aws/aws-sdk-go/aws/arn" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/endpoints" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/private/protocol" + "github.com/aws/aws-sdk-go/service/s3/internal/arn" +) + +// Used by shapes with members decorated as endpoint ARN. +func parseEndpointARN(v string) (arn.Resource, error) { + return arn.ParseResource(v, accessPointResourceParser) +} + +func accessPointResourceParser(a awsarn.ARN) (arn.Resource, error) { + resParts := arn.SplitResource(a.Resource) + switch resParts[0] { + case "accesspoint": + return arn.ParseAccessPointResource(a, resParts[1:]) + default: + return nil, arn.InvalidARNError{ARN: a, Reason: "unknown resource type"} + } +} + +func endpointHandler(req *request.Request) { + endpoint, ok := req.Params.(endpointARNGetter) + if !ok || !endpoint.hasEndpointARN() { + updateBucketEndpointFromParams(req) + return + } + + resource, err := endpoint.getEndpointARN() + if err != nil { + req.Error = newInvalidARNError(nil, err) + return + } + + resReq := resourceRequest{ + Resource: resource, + Request: req, + } + + if resReq.IsCrossPartition() { + req.Error = newClientPartitionMismatchError(resource, + req.ClientInfo.PartitionID, aws.StringValue(req.Config.Region), nil) + return + } + + if !resReq.AllowCrossRegion() && resReq.IsCrossRegion() { + req.Error = newClientRegionMismatchError(resource, + req.ClientInfo.PartitionID, aws.StringValue(req.Config.Region), nil) + return + } + + if resReq.HasCustomEndpoint() { + req.Error = newInvalidARNWithCustomEndpointError(resource, nil) + return + } + + switch tv := resource.(type) { + case arn.AccessPointARN: + err = updateRequestAccessPointEndpoint(req, tv) + if err != nil { + req.Error = err + } + default: + req.Error = newInvalidARNError(resource, nil) + } +} + +type resourceRequest struct { + Resource arn.Resource + Request *request.Request +} + +func (r resourceRequest) ARN() awsarn.ARN { + return r.Resource.GetARN() +} + +func (r resourceRequest) AllowCrossRegion() bool { + return aws.BoolValue(r.Request.Config.S3UseARNRegion) +} + +func (r resourceRequest) UseFIPS() bool { + return isFIPS(aws.StringValue(r.Request.Config.Region)) +} + +func (r resourceRequest) IsCrossPartition() bool { + return r.Request.ClientInfo.PartitionID != r.Resource.GetARN().Partition +} + +func (r resourceRequest) IsCrossRegion() bool { + return isCrossRegion(r.Request, r.Resource.GetARN().Region) +} + +func (r resourceRequest) HasCustomEndpoint() bool { + return len(aws.StringValue(r.Request.Config.Endpoint)) > 0 +} + +func isFIPS(clientRegion string) bool { + return strings.HasPrefix(clientRegion, "fips-") || strings.HasSuffix(clientRegion, "-fips") +} +func isCrossRegion(req *request.Request, otherRegion string) bool { + return req.ClientInfo.SigningRegion != otherRegion +} + +func updateBucketEndpointFromParams(r *request.Request) { + bucket, ok := bucketNameFromReqParams(r.Params) + if !ok { + // Ignore operation requests if the bucket name was not provided + // if this is an input validation error the validation handler + // will report it. + return + } + updateEndpointForS3Config(r, bucket) +} + +func updateRequestAccessPointEndpoint(req *request.Request, accessPoint arn.AccessPointARN) error { + // Accelerate not supported + if aws.BoolValue(req.Config.S3UseAccelerate) { + return newClientConfiguredForAccelerateError(accessPoint, + req.ClientInfo.PartitionID, aws.StringValue(req.Config.Region), nil) + } + + // Ignore the disable host prefix for access points since custom endpoints + // are not supported. + req.Config.DisableEndpointHostPrefix = aws.Bool(false) + + if err := accessPointEndpointBuilder(accessPoint).Build(req); err != nil { + return err + } + + removeBucketFromPath(req.HTTPRequest.URL) + + return nil +} + +func removeBucketFromPath(u *url.URL) { + u.Path = strings.Replace(u.Path, "/{Bucket}", "", -1) + if u.Path == "" { + u.Path = "/" + } +} + +type accessPointEndpointBuilder arn.AccessPointARN + +const ( + accessPointPrefixLabel = "accesspoint" + accountIDPrefixLabel = "accountID" + accesPointPrefixTemplate = "{" + accessPointPrefixLabel + "}-{" + accountIDPrefixLabel + "}." +) + +func (a accessPointEndpointBuilder) Build(req *request.Request) error { + resolveRegion := arn.AccessPointARN(a).Region + cfgRegion := aws.StringValue(req.Config.Region) + + if isFIPS(cfgRegion) { + if aws.BoolValue(req.Config.S3UseARNRegion) && isCrossRegion(req, resolveRegion) { + // FIPS with cross region is not supported, the SDK must fail + // because there is no well defined method for SDK to construct a + // correct FIPS endpoint. + return newClientConfiguredForCrossRegionFIPSError(arn.AccessPointARN(a), + req.ClientInfo.PartitionID, cfgRegion, nil) + } + resolveRegion = cfgRegion + } + + endpoint, err := resolveRegionalEndpoint(req, resolveRegion) + if err != nil { + return newFailedToResolveEndpointError(arn.AccessPointARN(a), + req.ClientInfo.PartitionID, cfgRegion, err) + } + + if err = updateRequestEndpoint(req, endpoint.URL); err != nil { + return err + } + + const serviceEndpointLabel = "s3-accesspoint" + + // dualstack provided by endpoint resolver + cfgHost := req.HTTPRequest.URL.Host + if strings.HasPrefix(cfgHost, "s3") { + req.HTTPRequest.URL.Host = serviceEndpointLabel + cfgHost[2:] + } + + protocol.HostPrefixBuilder{ + Prefix: accesPointPrefixTemplate, + LabelsFn: a.hostPrefixLabelValues, + }.Build(req) + + req.ClientInfo.SigningName = endpoint.SigningName + req.ClientInfo.SigningRegion = endpoint.SigningRegion + + err = protocol.ValidateEndpointHost(req.Operation.Name, req.HTTPRequest.URL.Host) + if err != nil { + return newInvalidARNError(arn.AccessPointARN(a), err) + } + + return nil +} + +func (a accessPointEndpointBuilder) hostPrefixLabelValues() map[string]string { + return map[string]string{ + accessPointPrefixLabel: arn.AccessPointARN(a).AccessPointName, + accountIDPrefixLabel: arn.AccessPointARN(a).AccountID, + } +} + +func resolveRegionalEndpoint(r *request.Request, region string) (endpoints.ResolvedEndpoint, error) { + return r.Config.EndpointResolver.EndpointFor(EndpointsID, region, func(opts *endpoints.Options) { + opts.DisableSSL = aws.BoolValue(r.Config.DisableSSL) + opts.UseDualStack = aws.BoolValue(r.Config.UseDualStack) + opts.S3UsEast1RegionalEndpoint = endpoints.RegionalS3UsEast1Endpoint + }) +} + +func updateRequestEndpoint(r *request.Request, endpoint string) (err error) { + endpoint = endpoints.AddScheme(endpoint, aws.BoolValue(r.Config.DisableSSL)) + + r.HTTPRequest.URL, err = url.Parse(endpoint + r.Operation.HTTPPath) + if err != nil { + return awserr.New(request.ErrCodeSerialization, + "failed to parse endpoint URL", err) + } + + return nil +} diff --git a/github.com/aws/aws-sdk-go/service/s3/endpoint_errors.go b/github.com/aws/aws-sdk-go/service/s3/endpoint_errors.go new file mode 100644 index 0000000000..9df03e78d3 --- /dev/null +++ b/github.com/aws/aws-sdk-go/service/s3/endpoint_errors.go @@ -0,0 +1,151 @@ +package s3 + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/s3/internal/arn" +) + +const ( + invalidARNErrorErrCode = "InvalidARNError" + configurationErrorErrCode = "ConfigurationError" +) + +type invalidARNError struct { + message string + resource arn.Resource + origErr error +} + +func (e invalidARNError) Error() string { + var extra string + if e.resource != nil { + extra = "ARN: " + e.resource.String() + } + return awserr.SprintError(e.Code(), e.Message(), extra, e.origErr) +} + +func (e invalidARNError) Code() string { + return invalidARNErrorErrCode +} + +func (e invalidARNError) Message() string { + return e.message +} + +func (e invalidARNError) OrigErr() error { + return e.origErr +} + +func newInvalidARNError(resource arn.Resource, err error) invalidARNError { + return invalidARNError{ + message: "invalid ARN", + origErr: err, + resource: resource, + } +} + +func newInvalidARNWithCustomEndpointError(resource arn.Resource, err error) invalidARNError { + return invalidARNError{ + message: "resource ARN not supported with custom client endpoints", + origErr: err, + resource: resource, + } +} + +// ARN not supported for the target partition +func newInvalidARNWithUnsupportedPartitionError(resource arn.Resource, err error) invalidARNError { + return invalidARNError{ + message: "resource ARN not supported for the target ARN partition", + origErr: err, + resource: resource, + } +} + +type configurationError struct { + message string + resource arn.Resource + clientPartitionID string + clientRegion string + origErr error +} + +func (e configurationError) Error() string { + extra := fmt.Sprintf("ARN: %s, client partition: %s, client region: %s", + e.resource, e.clientPartitionID, e.clientRegion) + + return awserr.SprintError(e.Code(), e.Message(), extra, e.origErr) +} + +func (e configurationError) Code() string { + return configurationErrorErrCode +} + +func (e configurationError) Message() string { + return e.message +} + +func (e configurationError) OrigErr() error { + return e.origErr +} + +func newClientPartitionMismatchError(resource arn.Resource, clientPartitionID, clientRegion string, err error) configurationError { + return configurationError{ + message: "client partition does not match provided ARN partition", + origErr: err, + resource: resource, + clientPartitionID: clientPartitionID, + clientRegion: clientRegion, + } +} + +func newClientRegionMismatchError(resource arn.Resource, clientPartitionID, clientRegion string, err error) configurationError { + return configurationError{ + message: "client region does not match provided ARN region", + origErr: err, + resource: resource, + clientPartitionID: clientPartitionID, + clientRegion: clientRegion, + } +} + +func newFailedToResolveEndpointError(resource arn.Resource, clientPartitionID, clientRegion string, err error) configurationError { + return configurationError{ + message: "endpoint resolver failed to find an endpoint for the provided ARN region", + origErr: err, + resource: resource, + clientPartitionID: clientPartitionID, + clientRegion: clientRegion, + } +} + +func newClientConfiguredForFIPSError(resource arn.Resource, clientPartitionID, clientRegion string, err error) configurationError { + return configurationError{ + message: "client configured for fips but cross-region resource ARN provided", + origErr: err, + resource: resource, + clientPartitionID: clientPartitionID, + clientRegion: clientRegion, + } +} + +func newClientConfiguredForAccelerateError(resource arn.Resource, clientPartitionID, clientRegion string, err error) configurationError { + return configurationError{ + message: "client configured for S3 Accelerate but is supported with resource ARN", + origErr: err, + resource: resource, + clientPartitionID: clientPartitionID, + clientRegion: clientRegion, + } +} + +func newClientConfiguredForCrossRegionFIPSError(resource arn.Resource, clientPartitionID, clientRegion string, err error) configurationError { + return configurationError{ + message: "client configured for FIPS with cross-region enabled but is supported with cross-region resource ARN", + origErr: err, + resource: resource, + clientPartitionID: clientPartitionID, + clientRegion: clientRegion, + } +} diff --git a/github.com/aws/aws-sdk-go/service/s3/errors.go b/github.com/aws/aws-sdk-go/service/s3/errors.go index 931cb17bb0..49aeff16f2 100644 --- a/github.com/aws/aws-sdk-go/service/s3/errors.go +++ b/github.com/aws/aws-sdk-go/service/s3/errors.go @@ -13,6 +13,12 @@ const ( // ErrCodeBucketAlreadyOwnedByYou for service response error code // "BucketAlreadyOwnedByYou". + // + // The bucket you tried to create already exists, and you own it. Amazon S3 + // returns this error in all AWS Regions except in the North Virginia Region. + // For legacy compatibility, if you re-create an existing bucket that you already + // own in the North Virginia Region, Amazon S3 returns 200 OK and resets the + // bucket access control lists (ACLs). ErrCodeBucketAlreadyOwnedByYou = "BucketAlreadyOwnedByYou" // ErrCodeNoSuchBucket for service response error code @@ -36,13 +42,13 @@ const ( // ErrCodeObjectAlreadyInActiveTierError for service response error code // "ObjectAlreadyInActiveTierError". // - // This operation is not allowed against this storage tier + // This operation is not allowed against this storage tier. ErrCodeObjectAlreadyInActiveTierError = "ObjectAlreadyInActiveTierError" // ErrCodeObjectNotInActiveTierError for service response error code // "ObjectNotInActiveTierError". // // The source object of the COPY operation is not in the active tier and is - // only stored in Amazon Glacier. + // only stored in Amazon S3 Glacier. ErrCodeObjectNotInActiveTierError = "ObjectNotInActiveTierError" ) diff --git a/github.com/aws/aws-sdk-go/service/s3/host_style_bucket.go b/github.com/aws/aws-sdk-go/service/s3/host_style_bucket.go index a7fbc2de2f..81cdec1ae7 100644 --- a/github.com/aws/aws-sdk-go/service/s3/host_style_bucket.go +++ b/github.com/aws/aws-sdk-go/service/s3/host_style_bucket.go @@ -30,10 +30,10 @@ var accelerateOpBlacklist = operationBlacklist{ opListBuckets, opCreateBucket, opDeleteBucket, } -// Request handler to automatically add the bucket name to the endpoint domain +// Automatically add the bucket name to the endpoint domain // if possible. This style of bucket is valid for all bucket names which are // DNS compatible and do not contain "." -func updateEndpointForS3Config(r *request.Request) { +func updateEndpointForS3Config(r *request.Request, bucketName string) { forceHostStyle := aws.BoolValue(r.Config.S3ForcePathStyle) accelerate := aws.BoolValue(r.Config.S3UseAccelerate) @@ -43,45 +43,29 @@ func updateEndpointForS3Config(r *request.Request) { r.Config.Logger.Log("ERROR: aws.Config.S3UseAccelerate is not compatible with aws.Config.S3ForcePathStyle, ignoring S3ForcePathStyle.") } } - updateEndpointForAccelerate(r) + updateEndpointForAccelerate(r, bucketName) } else if !forceHostStyle && r.Operation.Name != opGetBucketLocation { - updateEndpointForHostStyle(r) + updateEndpointForHostStyle(r, bucketName) } } -func updateEndpointForHostStyle(r *request.Request) { - bucket, ok := bucketNameFromReqParams(r.Params) - if !ok { - // Ignore operation requests if the bucketname was not provided - // if this is an input validation error the validation handler - // will report it. - return - } - - if !hostCompatibleBucketName(r.HTTPRequest.URL, bucket) { +func updateEndpointForHostStyle(r *request.Request, bucketName string) { + if !hostCompatibleBucketName(r.HTTPRequest.URL, bucketName) { // bucket name must be valid to put into the host return } - moveBucketToHost(r.HTTPRequest.URL, bucket) + moveBucketToHost(r.HTTPRequest.URL, bucketName) } var ( accelElem = []byte("s3-accelerate.dualstack.") ) -func updateEndpointForAccelerate(r *request.Request) { - bucket, ok := bucketNameFromReqParams(r.Params) - if !ok { - // Ignore operation requests if the bucketname was not provided - // if this is an input validation error the validation handler - // will report it. - return - } - - if !hostCompatibleBucketName(r.HTTPRequest.URL, bucket) { +func updateEndpointForAccelerate(r *request.Request, bucketName string) { + if !hostCompatibleBucketName(r.HTTPRequest.URL, bucketName) { r.Error = awserr.New("InvalidParameterException", - fmt.Sprintf("bucket name %s is not compatible with S3 Accelerate", bucket), + fmt.Sprintf("bucket name %s is not compatible with S3 Accelerate", bucketName), nil) return } @@ -106,7 +90,7 @@ func updateEndpointForAccelerate(r *request.Request) { r.HTTPRequest.URL.Host = strings.Join(parts, ".") - moveBucketToHost(r.HTTPRequest.URL, bucket) + moveBucketToHost(r.HTTPRequest.URL, bucketName) } // Attempts to retrieve the bucket name from the request input parameters. @@ -148,8 +132,5 @@ func dnsCompatibleBucketName(bucket string) bool { // moveBucketToHost moves the bucket name from the URI path to URL host. func moveBucketToHost(u *url.URL, bucket string) { u.Host = bucket + "." + u.Host - u.Path = strings.Replace(u.Path, "/{Bucket}", "", -1) - if u.Path == "" { - u.Path = "/" - } + removeBucketFromPath(u) } diff --git a/github.com/aws/aws-sdk-go/service/s3/internal/arn/accesspoint_arn.go b/github.com/aws/aws-sdk-go/service/s3/internal/arn/accesspoint_arn.go new file mode 100644 index 0000000000..2f93f96fd5 --- /dev/null +++ b/github.com/aws/aws-sdk-go/service/s3/internal/arn/accesspoint_arn.go @@ -0,0 +1,45 @@ +package arn + +import ( + "strings" + + "github.com/aws/aws-sdk-go/aws/arn" +) + +// AccessPointARN provides representation +type AccessPointARN struct { + arn.ARN + AccessPointName string +} + +// GetARN returns the base ARN for the Access Point resource +func (a AccessPointARN) GetARN() arn.ARN { + return a.ARN +} + +// ParseAccessPointResource attempts to parse the ARN's resource as an +// AccessPoint resource. +func ParseAccessPointResource(a arn.ARN, resParts []string) (AccessPointARN, error) { + if len(a.Region) == 0 { + return AccessPointARN{}, InvalidARNError{a, "region not set"} + } + if len(a.AccountID) == 0 { + return AccessPointARN{}, InvalidARNError{a, "account-id not set"} + } + if len(resParts) == 0 { + return AccessPointARN{}, InvalidARNError{a, "resource-id not set"} + } + if len(resParts) > 1 { + return AccessPointARN{}, InvalidARNError{a, "sub resource not supported"} + } + + resID := resParts[0] + if len(strings.TrimSpace(resID)) == 0 { + return AccessPointARN{}, InvalidARNError{a, "resource-id not set"} + } + + return AccessPointARN{ + ARN: a, + AccessPointName: resID, + }, nil +} diff --git a/github.com/aws/aws-sdk-go/service/s3/internal/arn/arn.go b/github.com/aws/aws-sdk-go/service/s3/internal/arn/arn.go new file mode 100644 index 0000000000..a942d887f7 --- /dev/null +++ b/github.com/aws/aws-sdk-go/service/s3/internal/arn/arn.go @@ -0,0 +1,71 @@ +package arn + +import ( + "strings" + + "github.com/aws/aws-sdk-go/aws/arn" +) + +// Resource provides the interfaces abstracting ARNs of specific resource +// types. +type Resource interface { + GetARN() arn.ARN + String() string +} + +// ResourceParser provides the function for parsing an ARN's resource +// component into a typed resource. +type ResourceParser func(arn.ARN) (Resource, error) + +// ParseResource parses an AWS ARN into a typed resource for the S3 API. +func ParseResource(s string, resParser ResourceParser) (resARN Resource, err error) { + a, err := arn.Parse(s) + if err != nil { + return nil, err + } + + if len(a.Partition) == 0 { + return nil, InvalidARNError{a, "partition not set"} + } + if a.Service != "s3" { + return nil, InvalidARNError{a, "service is not S3"} + } + if len(a.Resource) == 0 { + return nil, InvalidARNError{a, "resource not set"} + } + + return resParser(a) +} + +// SplitResource splits the resource components by the ARN resource delimiters. +func SplitResource(v string) []string { + var parts []string + var offset int + + for offset <= len(v) { + idx := strings.IndexAny(v[offset:], "/:") + if idx < 0 { + parts = append(parts, v[offset:]) + break + } + parts = append(parts, v[offset:idx+offset]) + offset += idx + 1 + } + + return parts +} + +// IsARN returns whether the given string is an ARN +func IsARN(s string) bool { + return arn.IsARN(s) +} + +// InvalidARNError provides the error for an invalid ARN error. +type InvalidARNError struct { + ARN arn.ARN + Reason string +} + +func (e InvalidARNError) Error() string { + return "invalid Amazon S3 ARN, " + e.Reason + ", " + e.ARN.String() +} diff --git a/github.com/aws/aws-sdk-go/service/s3/s3manager/batch.go b/github.com/aws/aws-sdk-go/service/s3/s3manager/batch.go index 18215574bf..22bd0b7ce5 100644 --- a/github.com/aws/aws-sdk-go/service/s3/s3manager/batch.go +++ b/github.com/aws/aws-sdk-go/service/s3/s3manager/batch.go @@ -273,7 +273,7 @@ type DeleteObjectsIterator struct { inc bool } -// Next will increment the default iterator's index and and ensure that there +// Next will increment the default iterator's index and ensure that there // is another object to iterator to. func (iter *DeleteObjectsIterator) Next() bool { if iter.inc { @@ -458,7 +458,7 @@ type DownloadObjectsIterator struct { inc bool } -// Next will increment the default iterator's index and and ensure that there +// Next will increment the default iterator's index and ensure that there // is another object to iterator to. func (batcher *DownloadObjectsIterator) Next() bool { if batcher.inc { @@ -497,7 +497,7 @@ type UploadObjectsIterator struct { inc bool } -// Next will increment the default iterator's index and and ensure that there +// Next will increment the default iterator's index and ensure that there // is another object to iterator to. func (batcher *UploadObjectsIterator) Next() bool { if batcher.inc { diff --git a/github.com/aws/aws-sdk-go/service/s3/s3manager/bucket_region.go b/github.com/aws/aws-sdk-go/service/s3/s3manager/bucket_region.go index f61665a58a..9cc1e5970c 100644 --- a/github.com/aws/aws-sdk-go/service/s3/s3manager/bucket_region.go +++ b/github.com/aws/aws-sdk-go/service/s3/s3manager/bucket_region.go @@ -3,6 +3,7 @@ package s3manager import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/client" + "github.com/aws/aws-sdk-go/aws/corehandlers" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/service/s3" @@ -35,6 +36,30 @@ import ( // } // fmt.Printf("Bucket %s is in %s region\n", bucket, region) // +// By default the request will be made to the Amazon S3 endpoint using the Path +// style addressing. +// +// s3.us-west-2.amazonaws.com/bucketname +// +// This is not compatible with Amazon S3's FIPS endpoints. To override this +// behavior to use Virtual Host style addressing, provide a functional option +// that will set the Request's Config.S3ForcePathStyle to aws.Bool(false). +// +// region, err := s3manager.GetBucketRegion(ctx, sess, "bucketname", "us-west-2", func(r *request.Request) { +// r.S3ForcePathStyle = aws.Bool(false) +// }) +// +// To configure the GetBucketRegion to make a request via the Amazon +// S3 FIPS endpoints directly when a FIPS region name is not available, (e.g. +// fips-us-gov-west-1) set the Config.Endpoint on the Session, or client the +// utility is called with. The hint region will be ignored if an endpoint URL +// is configured on the session or client. +// +// sess, err := session.NewSession(&aws.Config{ +// Endpoint: aws.String("https://s3-fips.us-west-2.amazonaws.com"), +// }) +// +// region, err := s3manager.GetBucketRegion(context.Background(), sess, "bucketname", "") func GetBucketRegion(ctx aws.Context, c client.ConfigProvider, bucket, regionHint string, opts ...request.Option) (string, error) { var cfg aws.Config if len(regionHint) != 0 { @@ -50,12 +75,38 @@ const bucketRegionHeader = "X-Amz-Bucket-Region" // that it takes a S3 service client instead of a Session. The regionHint is // derived from the region the S3 service client was created in. // +// By default the request will be made to the Amazon S3 endpoint using the Path +// style addressing. +// +// s3.us-west-2.amazonaws.com/bucketname +// +// This is not compatible with Amazon S3's FIPS endpoints. To override this +// behavior to use Virtual Host style addressing, provide a functional option +// that will set the Request's Config.S3ForcePathStyle to aws.Bool(false). +// +// region, err := s3manager.GetBucketRegionWithClient(ctx, client, "bucketname", func(r *request.Request) { +// r.S3ForcePathStyle = aws.Bool(false) +// }) +// +// To configure the GetBucketRegion to make a request via the Amazon +// S3 FIPS endpoints directly when a FIPS region name is not available, (e.g. +// fips-us-gov-west-1) set the Config.Endpoint on the Session, or client the +// utility is called with. The hint region will be ignored if an endpoint URL +// is configured on the session or client. +// +// region, err := s3manager.GetBucketRegionWithClient(context.Background(), +// s3.New(sess, &aws.Config{ +// Endpoint: aws.String("https://s3-fips.us-west-2.amazonaws.com"), +// }), +// "bucketname") +// // See GetBucketRegion for more information. func GetBucketRegionWithClient(ctx aws.Context, svc s3iface.S3API, bucket string, opts ...request.Option) (string, error) { req, _ := svc.HeadBucketRequest(&s3.HeadBucketInput{ Bucket: aws.String(bucket), }) req.Config.S3ForcePathStyle = aws.Bool(true) + req.Config.Credentials = credentials.AnonymousCredentials req.SetContext(ctx) @@ -75,6 +126,16 @@ func GetBucketRegionWithClient(ctx aws.Context, svc s3iface.S3API, bucket string r.HTTPResponse.Status = "OK" r.Error = nil }) + // Replace the endpoint validation handler to not require a region if an + // endpoint URL was specified. Since these requests are not authenticated, + // requiring a region is not needed when an endpoint URL is provided. + req.Handlers.Validate.Swap( + corehandlers.ValidateEndpointHandler.Name, + request.NamedHandler{ + Name: "validateEndpointWithoutRegion", + Fn: validateEndpointWithoutRegion, + }, + ) req.ApplyOptions(opts...) @@ -86,3 +147,13 @@ func GetBucketRegionWithClient(ctx aws.Context, svc s3iface.S3API, bucket string return bucketRegion, nil } + +func validateEndpointWithoutRegion(r *request.Request) { + // Check if the caller provided an explicit URL instead of one derived by + // the SDK's endpoint resolver. For GetBucketRegion, with an explicit + // endpoint URL, a region is not needed. If no endpoint URL is provided, + // fallback the SDK's standard endpoint validation handler. + if len(aws.StringValue(r.Config.Endpoint)) == 0 { + corehandlers.ValidateEndpointHandler.Fn(r) + } +} diff --git a/github.com/aws/aws-sdk-go/service/s3/s3manager/buffered_read_seeker.go b/github.com/aws/aws-sdk-go/service/s3/s3manager/buffered_read_seeker.go new file mode 100644 index 0000000000..f1d9e85c7b --- /dev/null +++ b/github.com/aws/aws-sdk-go/service/s3/s3manager/buffered_read_seeker.go @@ -0,0 +1,81 @@ +package s3manager + +import ( + "io" + + "github.com/aws/aws-sdk-go/internal/sdkio" +) + +// BufferedReadSeeker is buffered io.ReadSeeker +type BufferedReadSeeker struct { + r io.ReadSeeker + buffer []byte + readIdx, writeIdx int +} + +// NewBufferedReadSeeker returns a new BufferedReadSeeker +// if len(b) == 0 then the buffer will be initialized to 64 KiB. +func NewBufferedReadSeeker(r io.ReadSeeker, b []byte) *BufferedReadSeeker { + if len(b) == 0 { + b = make([]byte, 64*1024) + } + return &BufferedReadSeeker{r: r, buffer: b} +} + +func (b *BufferedReadSeeker) reset(r io.ReadSeeker) { + b.r = r + b.readIdx, b.writeIdx = 0, 0 +} + +// Read will read up len(p) bytes into p and will return +// the number of bytes read and any error that occurred. +// If the len(p) > the buffer size then a single read request +// will be issued to the underlying io.ReadSeeker for len(p) bytes. +// A Read request will at most perform a single Read to the underlying +// io.ReadSeeker, and may return < len(p) if serviced from the buffer. +func (b *BufferedReadSeeker) Read(p []byte) (n int, err error) { + if len(p) == 0 { + return n, err + } + + if b.readIdx == b.writeIdx { + if len(p) >= len(b.buffer) { + n, err = b.r.Read(p) + return n, err + } + b.readIdx, b.writeIdx = 0, 0 + + n, err = b.r.Read(b.buffer) + if n == 0 { + return n, err + } + + b.writeIdx += n + } + + n = copy(p, b.buffer[b.readIdx:b.writeIdx]) + b.readIdx += n + + return n, err +} + +// Seek will position then underlying io.ReadSeeker to the given offset +// and will clear the buffer. +func (b *BufferedReadSeeker) Seek(offset int64, whence int) (int64, error) { + n, err := b.r.Seek(offset, whence) + + b.reset(b.r) + + return n, err +} + +// ReadAt will read up to len(p) bytes at the given file offset. +// This will result in the buffer being cleared. +func (b *BufferedReadSeeker) ReadAt(p []byte, off int64) (int, error) { + _, err := b.Seek(off, sdkio.SeekStart) + if err != nil { + return 0, err + } + + return b.Read(p) +} diff --git a/github.com/aws/aws-sdk-go/service/s3/s3manager/default_read_seeker_write_to.go b/github.com/aws/aws-sdk-go/service/s3/s3manager/default_read_seeker_write_to.go new file mode 100644 index 0000000000..42276530a8 --- /dev/null +++ b/github.com/aws/aws-sdk-go/service/s3/s3manager/default_read_seeker_write_to.go @@ -0,0 +1,7 @@ +// +build !windows + +package s3manager + +func defaultUploadBufferProvider() ReadSeekerWriteToProvider { + return nil +} diff --git a/github.com/aws/aws-sdk-go/service/s3/s3manager/default_read_seeker_write_to_windows.go b/github.com/aws/aws-sdk-go/service/s3/s3manager/default_read_seeker_write_to_windows.go new file mode 100644 index 0000000000..687082c306 --- /dev/null +++ b/github.com/aws/aws-sdk-go/service/s3/s3manager/default_read_seeker_write_to_windows.go @@ -0,0 +1,5 @@ +package s3manager + +func defaultUploadBufferProvider() ReadSeekerWriteToProvider { + return NewBufferedReadSeekerWriteToPool(1024 * 1024) +} diff --git a/github.com/aws/aws-sdk-go/service/s3/s3manager/default_writer_read_from.go b/github.com/aws/aws-sdk-go/service/s3/s3manager/default_writer_read_from.go new file mode 100644 index 0000000000..ada50c2435 --- /dev/null +++ b/github.com/aws/aws-sdk-go/service/s3/s3manager/default_writer_read_from.go @@ -0,0 +1,7 @@ +// +build !windows + +package s3manager + +func defaultDownloadBufferProvider() WriterReadFromProvider { + return nil +} diff --git a/github.com/aws/aws-sdk-go/service/s3/s3manager/default_writer_read_from_windows.go b/github.com/aws/aws-sdk-go/service/s3/s3manager/default_writer_read_from_windows.go new file mode 100644 index 0000000000..7e9d9579f6 --- /dev/null +++ b/github.com/aws/aws-sdk-go/service/s3/s3manager/default_writer_read_from_windows.go @@ -0,0 +1,5 @@ +package s3manager + +func defaultDownloadBufferProvider() WriterReadFromProvider { + return NewPooledBufferedWriterReadFromProvider(1024 * 1024) +} diff --git a/github.com/aws/aws-sdk-go/service/s3/s3manager/download.go b/github.com/aws/aws-sdk-go/service/s3/s3manager/download.go index 4a9ad65e49..4b54b7c033 100644 --- a/github.com/aws/aws-sdk-go/service/s3/s3manager/download.go +++ b/github.com/aws/aws-sdk-go/service/s3/s3manager/download.go @@ -25,13 +25,25 @@ const DefaultDownloadPartSize = 1024 * 1024 * 5 // when using Download(). const DefaultDownloadConcurrency = 5 +type errReadingBody struct { + err error +} + +func (e *errReadingBody) Error() string { + return fmt.Sprintf("failed to read part body: %v", e.err) +} + +func (e *errReadingBody) Unwrap() error { + return e.err +} + // The Downloader structure that calls Download(). It is safe to call Download() // on this structure for multiple objects and across concurrent goroutines. // Mutating the Downloader's properties is not safe to be done concurrently. type Downloader struct { - // The buffer size (in bytes) to use when buffering data into chunks and - // sending them as parts to S3. The minimum allowed part size is 5MB, and - // if this value is set to zero, the DefaultDownloadPartSize value will be used. + // The size (in bytes) to request from S3 for each part. + // The minimum allowed part size is 5MB, and if this value is set to zero, + // the DefaultDownloadPartSize value will be used. // // PartSize is ignored if the Range input parameter is provided. PartSize int64 @@ -50,6 +62,14 @@ type Downloader struct { // List of request options that will be passed down to individual API // operation requests made by the downloader. RequestOptions []request.Option + + // Defines the buffer strategy used when downloading a part. + // + // If a WriterReadFromProvider is given the Download manager + // will pass the io.WriterAt of the Download request to the provider + // and will use the returned WriterReadFrom from the provider as the + // destination writer when copying from http response body. + BufferProvider WriterReadFromProvider } // WithDownloaderRequestOptions appends to the Downloader's API request options. @@ -77,10 +97,15 @@ func WithDownloaderRequestOptions(opts ...request.Option) func(*Downloader) { // d.PartSize = 64 * 1024 * 1024 // 64MB per part // }) func NewDownloader(c client.ConfigProvider, options ...func(*Downloader)) *Downloader { + return newDownloader(s3.New(c), options...) +} + +func newDownloader(client s3iface.S3API, options ...func(*Downloader)) *Downloader { d := &Downloader{ - S3: s3.New(c), - PartSize: DefaultDownloadPartSize, - Concurrency: DefaultDownloadConcurrency, + S3: client, + PartSize: DefaultDownloadPartSize, + Concurrency: DefaultDownloadConcurrency, + BufferProvider: defaultDownloadBufferProvider(), } for _, option := range options { option(d) @@ -99,7 +124,7 @@ func NewDownloader(c client.ConfigProvider, options ...func(*Downloader)) *Downl // sess := session.Must(session.NewSession()) // // // The S3 client the S3 Downloader will use -// s3Svc := s3.new(sess) +// s3Svc := s3.New(sess) // // // Create a downloader with the s3 client and default options // downloader := s3manager.NewDownloaderWithClient(s3Svc) @@ -109,16 +134,7 @@ func NewDownloader(c client.ConfigProvider, options ...func(*Downloader)) *Downl // d.PartSize = 64 * 1024 * 1024 // 64MB per part // }) func NewDownloaderWithClient(svc s3iface.S3API, options ...func(*Downloader)) *Downloader { - d := &Downloader{ - S3: svc, - PartSize: DefaultDownloadPartSize, - Concurrency: DefaultDownloadConcurrency, - } - for _, option := range options { - option(d) - } - - return d + return newDownloader(svc, options...) } type maxRetrier interface { @@ -126,7 +142,8 @@ type maxRetrier interface { } // Download downloads an object in S3 and writes the payload into w using -// concurrent GET requests. +// concurrent GET requests. The n int64 returned is the size of the object downloaded +// in bytes. // // Additional functional options can be provided to configure the individual // download. These options are copies of the Downloader instance Download is called from. @@ -148,7 +165,8 @@ func (d Downloader) Download(w io.WriterAt, input *s3.GetObjectInput, options .. } // DownloadWithContext downloads an object in S3 and writes the payload into w -// using concurrent GET requests. +// using concurrent GET requests. The n int64 returned is the size of the object downloaded +// in bytes. // // DownloadWithContext is the same as Download with the additional support for // Context input parameters. The Context must not be nil. A nil Context will @@ -403,18 +421,20 @@ func (d *downloader) downloadChunk(chunk dlchunk) error { var n int64 var err error for retry := 0; retry <= d.partBodyMaxRetries; retry++ { - var resp *s3.GetObjectOutput - resp, err = d.cfg.S3.GetObjectWithContext(d.ctx, in, d.cfg.RequestOptions...) - if err != nil { - return err - } - d.setTotalBytes(resp) // Set total if not yet set. - - n, err = io.Copy(&chunk, resp.Body) - resp.Body.Close() + n, err = d.tryDownloadChunk(in, &chunk) if err == nil { break } + // Check if the returned error is an errReadingBody. + // If err is errReadingBody this indicates that an error + // occurred while copying the http response body. + // If this occurs we unwrap the err to set the underlying error + // and attempt any remaining retries. + if bodyErr, ok := err.(*errReadingBody); ok { + err = bodyErr.Unwrap() + } else { + return err + } chunk.cur = 0 logMessage(d.cfg.S3, aws.LogDebugWithRequestRetries, @@ -427,6 +447,28 @@ func (d *downloader) downloadChunk(chunk dlchunk) error { return err } +func (d *downloader) tryDownloadChunk(in *s3.GetObjectInput, w io.Writer) (int64, error) { + cleanup := func() {} + if d.cfg.BufferProvider != nil { + w, cleanup = d.cfg.BufferProvider.GetReadFrom(w) + } + defer cleanup() + + resp, err := d.cfg.S3.GetObjectWithContext(d.ctx, in, d.cfg.RequestOptions...) + if err != nil { + return 0, err + } + d.setTotalBytes(resp) // Set total if not yet set. + + n, err := io.Copy(w, resp.Body) + resp.Body.Close() + if err != nil { + return n, &errReadingBody{err: err} + } + + return n, nil +} + func logMessage(svc s3iface.S3API, level aws.LogLevelType, msg string) { s, ok := svc.(*s3.S3) if !ok { diff --git a/github.com/aws/aws-sdk-go/service/s3/s3manager/pool.go b/github.com/aws/aws-sdk-go/service/s3/s3manager/pool.go new file mode 100644 index 0000000000..05113286d3 --- /dev/null +++ b/github.com/aws/aws-sdk-go/service/s3/s3manager/pool.go @@ -0,0 +1,244 @@ +package s3manager + +import ( + "fmt" + "sync" + + "github.com/aws/aws-sdk-go/aws" +) + +type byteSlicePool interface { + Get(aws.Context) (*[]byte, error) + Put(*[]byte) + ModifyCapacity(int) + SliceSize() int64 + Close() +} + +type maxSlicePool struct { + // allocator is defined as a function pointer to allow + // for test cases to instrument custom tracers when allocations + // occur. + allocator sliceAllocator + + slices chan *[]byte + allocations chan struct{} + capacityChange chan struct{} + + max int + sliceSize int64 + + mtx sync.RWMutex +} + +func newMaxSlicePool(sliceSize int64) *maxSlicePool { + p := &maxSlicePool{sliceSize: sliceSize} + p.allocator = p.newSlice + + return p +} + +var errZeroCapacity = fmt.Errorf("get called on zero capacity pool") + +func (p *maxSlicePool) Get(ctx aws.Context) (*[]byte, error) { + // check if context is canceled before attempting to get a slice + // this ensures priority is given to the cancel case first + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + p.mtx.RLock() + + for { + select { + case bs, ok := <-p.slices: + p.mtx.RUnlock() + if !ok { + // attempt to get on a zero capacity pool + return nil, errZeroCapacity + } + return bs, nil + case _, ok := <-p.allocations: + p.mtx.RUnlock() + if !ok { + // attempt to get on a zero capacity pool + return nil, errZeroCapacity + } + return p.allocator(), nil + case <-ctx.Done(): + p.mtx.RUnlock() + return nil, ctx.Err() + default: + // In the event that there are no slices or allocations available + // This prevents some deadlock situations that can occur around sync.RWMutex + // When a lock request occurs on ModifyCapacity, no new readers are allowed to acquire a read lock. + // By releasing the read lock here and waiting for a notification, we prevent a deadlock situation where + // Get could hold the read lock indefinitely waiting for capacity, ModifyCapacity is waiting for a write lock, + // and a Put is blocked trying to get a read-lock which is blocked by ModifyCapacity. + + // Short-circuit if the pool capacity is zero. + if p.max == 0 { + p.mtx.RUnlock() + return nil, errZeroCapacity + } + + // Since we will be releasing the read-lock we need to take the reference to the channel. + // Since channels are references we will still get notified if slices are added, or if + // the channel is closed due to a capacity modification. This specifically avoids a data race condition + // where ModifyCapacity both closes a channel and initializes a new one while we don't have a read-lock. + c := p.capacityChange + + p.mtx.RUnlock() + + select { + case _ = <-c: + p.mtx.RLock() + case <-ctx.Done(): + return nil, ctx.Err() + } + } + } +} + +func (p *maxSlicePool) Put(bs *[]byte) { + p.mtx.RLock() + defer p.mtx.RUnlock() + + if p.max == 0 { + return + } + + select { + case p.slices <- bs: + p.notifyCapacity() + default: + // If the new channel when attempting to add the slice then we drop the slice. + // The logic here is to prevent a deadlock situation if channel is already at max capacity. + // Allows us to reap allocations that are returned and are no longer needed. + } +} + +func (p *maxSlicePool) ModifyCapacity(delta int) { + if delta == 0 { + return + } + + p.mtx.Lock() + defer p.mtx.Unlock() + + p.max += delta + + if p.max == 0 { + p.empty() + return + } + + if p.capacityChange != nil { + close(p.capacityChange) + } + p.capacityChange = make(chan struct{}, p.max) + + origAllocations := p.allocations + p.allocations = make(chan struct{}, p.max) + + newAllocs := len(origAllocations) + delta + for i := 0; i < newAllocs; i++ { + p.allocations <- struct{}{} + } + + if origAllocations != nil { + close(origAllocations) + } + + origSlices := p.slices + p.slices = make(chan *[]byte, p.max) + if origSlices == nil { + return + } + + close(origSlices) + for bs := range origSlices { + select { + case p.slices <- bs: + default: + // If the new channel blocks while adding slices from the old channel + // then we drop the slice. The logic here is to prevent a deadlock situation + // if the new channel has a smaller capacity then the old. + } + } +} + +func (p *maxSlicePool) notifyCapacity() { + select { + case p.capacityChange <- struct{}{}: + default: + // This *shouldn't* happen as the channel is both buffered to the max pool capacity size and is resized + // on capacity modifications. This is just a safety to ensure that a blocking situation can't occur. + } +} + +func (p *maxSlicePool) SliceSize() int64 { + return p.sliceSize +} + +func (p *maxSlicePool) Close() { + p.mtx.Lock() + defer p.mtx.Unlock() + p.empty() +} + +func (p *maxSlicePool) empty() { + p.max = 0 + + if p.capacityChange != nil { + close(p.capacityChange) + p.capacityChange = nil + } + + if p.allocations != nil { + close(p.allocations) + for range p.allocations { + // drain channel + } + p.allocations = nil + } + + if p.slices != nil { + close(p.slices) + for range p.slices { + // drain channel + } + p.slices = nil + } +} + +func (p *maxSlicePool) newSlice() *[]byte { + bs := make([]byte, p.sliceSize) + return &bs +} + +type returnCapacityPoolCloser struct { + byteSlicePool + returnCapacity int +} + +func (n *returnCapacityPoolCloser) ModifyCapacity(delta int) { + if delta > 0 { + n.returnCapacity = -1 * delta + } + n.byteSlicePool.ModifyCapacity(delta) +} + +func (n *returnCapacityPoolCloser) Close() { + if n.returnCapacity < 0 { + n.byteSlicePool.ModifyCapacity(n.returnCapacity) + } +} + +type sliceAllocator func() *[]byte + +var newByteSlicePool = func(sliceSize int64) byteSlicePool { + return newMaxSlicePool(sliceSize) +} diff --git a/github.com/aws/aws-sdk-go/service/s3/s3manager/read_seeker_write_to.go b/github.com/aws/aws-sdk-go/service/s3/s3manager/read_seeker_write_to.go new file mode 100644 index 0000000000..f62e1a45ee --- /dev/null +++ b/github.com/aws/aws-sdk-go/service/s3/s3manager/read_seeker_write_to.go @@ -0,0 +1,65 @@ +package s3manager + +import ( + "io" + "sync" +) + +// ReadSeekerWriteTo defines an interface implementing io.WriteTo and io.ReadSeeker +type ReadSeekerWriteTo interface { + io.ReadSeeker + io.WriterTo +} + +// BufferedReadSeekerWriteTo wraps a BufferedReadSeeker with an io.WriteAt +// implementation. +type BufferedReadSeekerWriteTo struct { + *BufferedReadSeeker +} + +// WriteTo writes to the given io.Writer from BufferedReadSeeker until there's no more data to write or +// an error occurs. Returns the number of bytes written and any error encountered during the write. +func (b *BufferedReadSeekerWriteTo) WriteTo(writer io.Writer) (int64, error) { + return io.Copy(writer, b.BufferedReadSeeker) +} + +// ReadSeekerWriteToProvider provides an implementation of io.WriteTo for an io.ReadSeeker +type ReadSeekerWriteToProvider interface { + GetWriteTo(seeker io.ReadSeeker) (r ReadSeekerWriteTo, cleanup func()) +} + +// BufferedReadSeekerWriteToPool uses a sync.Pool to create and reuse +// []byte slices for buffering parts in memory +type BufferedReadSeekerWriteToPool struct { + pool sync.Pool +} + +// NewBufferedReadSeekerWriteToPool will return a new BufferedReadSeekerWriteToPool that will create +// a pool of reusable buffers . If size is less then < 64 KiB then the buffer +// will default to 64 KiB. Reason: io.Copy from writers or readers that don't support io.WriteTo or io.ReadFrom +// respectively will default to copying 32 KiB. +func NewBufferedReadSeekerWriteToPool(size int) *BufferedReadSeekerWriteToPool { + if size < 65536 { + size = 65536 + } + + return &BufferedReadSeekerWriteToPool{ + pool: sync.Pool{New: func() interface{} { + return make([]byte, size) + }}, + } +} + +// GetWriteTo will wrap the provided io.ReadSeeker with a BufferedReadSeekerWriteTo. +// The provided cleanup must be called after operations have been completed on the +// returned io.ReadSeekerWriteTo in order to signal the return of resources to the pool. +func (p *BufferedReadSeekerWriteToPool) GetWriteTo(seeker io.ReadSeeker) (r ReadSeekerWriteTo, cleanup func()) { + buffer := p.pool.Get().([]byte) + + r = &BufferedReadSeekerWriteTo{BufferedReadSeeker: NewBufferedReadSeeker(seeker, buffer)} + cleanup = func() { + p.pool.Put(buffer) + } + + return r, cleanup +} diff --git a/github.com/aws/aws-sdk-go/service/s3/s3manager/upload.go b/github.com/aws/aws-sdk-go/service/s3/s3manager/upload.go index a9a005f0ac..8770d40411 100644 --- a/github.com/aws/aws-sdk-go/service/s3/s3manager/upload.go +++ b/github.com/aws/aws-sdk-go/service/s3/s3manager/upload.go @@ -11,6 +11,7 @@ import ( "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/awsutil" "github.com/aws/aws-sdk-go/aws/client" + "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3iface" @@ -144,8 +145,13 @@ type Uploader struct { // MaxUploadParts is the max number of parts which will be uploaded to S3. // Will be used to calculate the partsize of the object to be uploaded. // E.g: 5GB file, with MaxUploadParts set to 100, will upload the file - // as 100, 50MB parts. - // With a limited of s3.MaxUploadParts (10,000 parts). + // as 100, 50MB parts. With a limited of s3.MaxUploadParts (10,000 parts). + // + // MaxUploadParts must not be used to limit the total number of bytes uploaded. + // Use a type like to io.LimitReader (https://golang.org/pkg/io/#LimitedReader) + // instead. An io.LimitReader is helpful when uploading an unbounded reader + // to S3, and you know its maximum size. Otherwise the reader's io.EOF returned + // error must be used to signal end of stream. // // Defaults to package const's MaxUploadParts value. MaxUploadParts int @@ -156,6 +162,12 @@ type Uploader struct { // List of request options that will be passed down to individual API // operation requests made by the uploader. RequestOptions []request.Option + + // Defines the buffer strategy used when uploading a part + BufferProvider ReadSeekerWriteToProvider + + // partPool allows for the re-usage of streaming payload part buffers between upload calls + partPool byteSlicePool } // NewUploader creates a new Uploader instance to upload objects to S3. Pass In @@ -175,18 +187,25 @@ type Uploader struct { // u.PartSize = 64 * 1024 * 1024 // 64MB per part // }) func NewUploader(c client.ConfigProvider, options ...func(*Uploader)) *Uploader { + return newUploader(s3.New(c), options...) +} + +func newUploader(client s3iface.S3API, options ...func(*Uploader)) *Uploader { u := &Uploader{ - S3: s3.New(c), + S3: client, PartSize: DefaultUploadPartSize, Concurrency: DefaultUploadConcurrency, LeavePartsOnError: false, MaxUploadParts: MaxUploadParts, + BufferProvider: defaultUploadBufferProvider(), } for _, option := range options { option(u) } + u.partPool = newByteSlicePool(u.PartSize) + return u } @@ -209,19 +228,7 @@ func NewUploader(c client.ConfigProvider, options ...func(*Uploader)) *Uploader // u.PartSize = 64 * 1024 * 1024 // 64MB per part // }) func NewUploaderWithClient(svc s3iface.S3API, options ...func(*Uploader)) *Uploader { - u := &Uploader{ - S3: svc, - PartSize: DefaultUploadPartSize, - Concurrency: DefaultUploadConcurrency, - LeavePartsOnError: false, - MaxUploadParts: MaxUploadParts, - } - - for _, option := range options { - option(u) - } - - return u + return newUploader(svc, options...) } // Upload uploads an object to S3, intelligently buffering large files into @@ -281,6 +288,7 @@ func (u Uploader) UploadWithContext(ctx aws.Context, input *UploadInput, opts .. for _, opt := range opts { opt(&i.cfg) } + i.cfg.RequestOptions = append(i.cfg.RequestOptions, request.WithAppendUserAgent("S3Manager")) return i.upload() @@ -350,14 +358,15 @@ type uploader struct { readerPos int64 // current reader position totalSize int64 // set to -1 if the size is not known - - bufferPool sync.Pool } // internal logic for deciding whether to upload a single part or use a // multipart upload. func (u *uploader) upload() (*UploadOutput, error) { - u.init() + if err := u.init(); err != nil { + return nil, awserr.New("ReadRequestBody", "unable to initialize upload", err) + } + defer u.cfg.partPool.Close() if u.cfg.PartSize < MinUploadPartSize { msg := fmt.Sprintf("part size must be at least %d bytes", MinUploadPartSize) @@ -365,19 +374,20 @@ func (u *uploader) upload() (*UploadOutput, error) { } // Do one read to determine if we have more than one part - reader, _, part, err := u.nextReader() + reader, _, cleanup, err := u.nextReader() if err == io.EOF { // single part - return u.singlePart(reader) + return u.singlePart(reader, cleanup) } else if err != nil { + cleanup() return nil, awserr.New("ReadRequestBody", "read upload data failed", err) } mu := multiuploader{uploader: u} - return mu.upload(reader, part) + return mu.upload(reader, cleanup) } // init will initialize all default options. -func (u *uploader) init() { +func (u *uploader) init() error { if u.cfg.Concurrency == 0 { u.cfg.Concurrency = DefaultUploadConcurrency } @@ -388,24 +398,35 @@ func (u *uploader) init() { u.cfg.MaxUploadParts = MaxUploadParts } - u.bufferPool = sync.Pool{ - New: func() interface{} { return make([]byte, u.cfg.PartSize) }, + // Try to get the total size for some optimizations + if err := u.initSize(); err != nil { + return err } - // Try to get the total size for some optimizations - u.initSize() + // If PartSize was changed or partPool was never setup then we need to allocated a new pool + // so that we return []byte slices of the correct size + poolCap := u.cfg.Concurrency + 1 + if u.cfg.partPool == nil || u.cfg.partPool.SliceSize() != u.cfg.PartSize { + u.cfg.partPool = newByteSlicePool(u.cfg.PartSize) + u.cfg.partPool.ModifyCapacity(poolCap) + } else { + u.cfg.partPool = &returnCapacityPoolCloser{byteSlicePool: u.cfg.partPool} + u.cfg.partPool.ModifyCapacity(poolCap) + } + + return nil } // initSize tries to detect the total stream size, setting u.totalSize. If // the size is not known, totalSize is set to -1. -func (u *uploader) initSize() { +func (u *uploader) initSize() error { u.totalSize = -1 switch r := u.in.Body.(type) { case io.Seeker: n, err := aws.SeekerLen(r) if err != nil { - return + return err } u.totalSize = n @@ -417,17 +438,15 @@ func (u *uploader) initSize() { u.cfg.PartSize = (u.totalSize / int64(u.cfg.MaxUploadParts)) + 1 } } + + return nil } // nextReader returns a seekable reader representing the next packet of data. // This operation increases the shared u.readerPos counter, but note that it // does not need to be wrapped in a mutex because nextReader is only called // from the main thread. -func (u *uploader) nextReader() (io.ReadSeeker, int, []byte, error) { - type readerAtSeeker interface { - io.ReaderAt - io.ReadSeeker - } +func (u *uploader) nextReader() (io.ReadSeeker, int, func(), error) { switch r := u.in.Body.(type) { case readerAtSeeker: var err error @@ -442,17 +461,36 @@ func (u *uploader) nextReader() (io.ReadSeeker, int, []byte, error) { } } - reader := io.NewSectionReader(r, u.readerPos, n) + var ( + reader io.ReadSeeker + cleanup func() + ) + + reader = io.NewSectionReader(r, u.readerPos, n) + if u.cfg.BufferProvider != nil { + reader, cleanup = u.cfg.BufferProvider.GetWriteTo(reader) + } else { + cleanup = func() {} + } + u.readerPos += n - return reader, int(n), nil, err + return reader, int(n), cleanup, err default: - part := u.bufferPool.Get().([]byte) - n, err := readFillBuf(r, part) + part, err := u.cfg.partPool.Get(u.ctx) + if err != nil { + return nil, 0, func() {}, err + } + + n, err := readFillBuf(r, *part) u.readerPos += int64(n) - return bytes.NewReader(part[0:n]), n, part, err + cleanup := func() { + u.cfg.partPool.Put(part) + } + + return bytes.NewReader((*part)[0:n]), n, cleanup, err } } @@ -469,10 +507,12 @@ func readFillBuf(r io.Reader, b []byte) (offset int, err error) { // singlePart contains upload logic for uploading a single chunk via // a regular PutObject request. Multipart requests require at least two // parts, or at least 5MB of data. -func (u *uploader) singlePart(buf io.ReadSeeker) (*UploadOutput, error) { +func (u *uploader) singlePart(r io.ReadSeeker, cleanup func()) (*UploadOutput, error) { + defer cleanup() + params := &s3.PutObjectInput{} awsutil.Copy(params, u.in) - params.Body = buf + params.Body = r // Need to use request form because URL generated in request is // used in return. @@ -502,9 +542,9 @@ type multiuploader struct { // keeps track of a single chunk of data being sent to S3. type chunk struct { - buf io.ReadSeeker - part []byte - num int64 + buf io.ReadSeeker + num int64 + cleanup func() } // completedParts is a wrapper to make parts sortable by their part number, @@ -517,13 +557,14 @@ func (a completedParts) Less(i, j int) bool { return *a[i].PartNumber < *a[j].Pa // upload will perform a multipart upload using the firstBuf buffer containing // the first chunk of data. -func (u *multiuploader) upload(firstBuf io.ReadSeeker, firstPart []byte) (*UploadOutput, error) { +func (u *multiuploader) upload(firstBuf io.ReadSeeker, cleanup func()) (*UploadOutput, error) { params := &s3.CreateMultipartUploadInput{} awsutil.Copy(params, u.in) // Create the multipart resp, err := u.cfg.S3.CreateMultipartUploadWithContext(u.ctx, params, u.cfg.RequestOptions...) if err != nil { + cleanup() return nil, err } u.uploadID = *resp.UploadId @@ -537,46 +578,29 @@ func (u *multiuploader) upload(firstBuf io.ReadSeeker, firstPart []byte) (*Uploa // Send part 1 to the workers var num int64 = 1 - ch <- chunk{buf: firstBuf, part: firstPart, num: num} + ch <- chunk{buf: firstBuf, num: num, cleanup: cleanup} // Read and queue the rest of the parts for u.geterr() == nil && err == nil { - num++ - // This upload exceeded maximum number of supported parts, error now. - if num > int64(u.cfg.MaxUploadParts) || num > int64(MaxUploadParts) { - var msg string - if num > int64(u.cfg.MaxUploadParts) { - msg = fmt.Sprintf("exceeded total allowed configured MaxUploadParts (%d). Adjust PartSize to fit in this limit", - u.cfg.MaxUploadParts) - } else { - msg = fmt.Sprintf("exceeded total allowed S3 limit MaxUploadParts (%d). Adjust PartSize to fit in this limit", - MaxUploadParts) + var ( + reader io.ReadSeeker + nextChunkLen int + ok bool + ) + + reader, nextChunkLen, cleanup, err = u.nextReader() + ok, err = u.shouldContinue(num, nextChunkLen, err) + if !ok { + cleanup() + if err != nil { + u.seterr(err) } - u.seterr(awserr.New("TotalPartsExceeded", msg, nil)) - break - } - - var reader io.ReadSeeker - var nextChunkLen int - var part []byte - reader, nextChunkLen, part, err = u.nextReader() - - if err != nil && err != io.EOF { - u.seterr(awserr.New( - "ReadRequestBody", - "read multipart upload data failed", - err)) break } - if nextChunkLen == 0 { - // No need to upload empty part, if file was empty to start - // with empty single part would of been created and never - // started multipart upload. - break - } + num++ - ch <- chunk{buf: reader, part: part, num: num} + ch <- chunk{buf: reader, num: num, cleanup: cleanup} } // Close the channel, wait for workers, and complete upload @@ -593,13 +617,53 @@ func (u *multiuploader) upload(firstBuf io.ReadSeeker, firstPart []byte) (*Uploa uploadID: u.uploadID, } } + + // Create a presigned URL of the S3 Get Object in order to have parity with + // single part upload. + getReq, _ := u.cfg.S3.GetObjectRequest(&s3.GetObjectInput{ + Bucket: u.in.Bucket, + Key: u.in.Key, + }) + getReq.Config.Credentials = credentials.AnonymousCredentials + getReq.SetContext(u.ctx) + uploadLocation, _, _ := getReq.PresignRequest(1) + return &UploadOutput{ - Location: aws.StringValue(complete.Location), + Location: uploadLocation, VersionID: complete.VersionId, UploadID: u.uploadID, }, nil } +func (u *multiuploader) shouldContinue(part int64, nextChunkLen int, err error) (bool, error) { + if err != nil && err != io.EOF { + return false, awserr.New("ReadRequestBody", "read multipart upload data failed", err) + } + + if nextChunkLen == 0 { + // No need to upload empty part, if file was empty to start + // with empty single part would of been created and never + // started multipart upload. + return false, nil + } + + part++ + // This upload exceeded maximum number of supported parts, error now. + if part > int64(u.cfg.MaxUploadParts) || part > int64(MaxUploadParts) { + var msg string + if part > int64(u.cfg.MaxUploadParts) { + msg = fmt.Sprintf("exceeded total allowed configured MaxUploadParts (%d). Adjust PartSize to fit in this limit", + u.cfg.MaxUploadParts) + } else { + msg = fmt.Sprintf("exceeded total allowed S3 limit MaxUploadParts (%d). Adjust PartSize to fit in this limit", + MaxUploadParts) + } + return false, awserr.New("TotalPartsExceeded", msg, nil) + } + + return true, err +} + // readChunk runs in worker goroutines to pull chunks off of the ch channel // and send() them as UploadPart requests. func (u *multiuploader) readChunk(ch chan chunk) { @@ -616,6 +680,8 @@ func (u *multiuploader) readChunk(ch chan chunk) { u.seterr(err) } } + + data.cleanup() } } @@ -631,9 +697,8 @@ func (u *multiuploader) send(c chunk) error { SSECustomerKey: u.in.SSECustomerKey, PartNumber: &c.num, } + resp, err := u.cfg.S3.UploadPartWithContext(u.ctx, params, u.cfg.RequestOptions...) - // put the byte array back into the pool to conserve memory - u.bufferPool.Put(c.part) if err != nil { return err } @@ -705,3 +770,8 @@ func (u *multiuploader) complete() *s3.CompleteMultipartUploadOutput { return resp } + +type readerAtSeeker interface { + io.ReaderAt + io.ReadSeeker +} diff --git a/github.com/aws/aws-sdk-go/service/s3/s3manager/upload_input.go b/github.com/aws/aws-sdk-go/service/s3/s3manager/upload_input.go index ebf48204c1..c8810c11bb 100644 --- a/github.com/aws/aws-sdk-go/service/s3/s3manager/upload_input.go +++ b/github.com/aws/aws-sdk-go/service/s3/s3manager/upload_input.go @@ -12,40 +12,59 @@ import ( // package's PutObjectInput with the exception that the Body member is an // io.Reader instead of an io.ReadSeeker. type UploadInput struct { - _ struct{} `type:"structure" payload:"Body"` + _ struct{} `locationName:"PutObjectRequest" type:"structure" payload:"Body"` - // The canned ACL to apply to the object. + // The canned ACL to apply to the object. For more information, see Canned ACL + // (https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#CannedACL). ACL *string `location:"header" locationName:"x-amz-acl" type:"string" enum:"ObjectCannedACL"` // The readable body payload to send to S3. Body io.Reader - // Name of the bucket to which the PUT operation was initiated. + // Bucket name to which the PUT operation was initiated. + // + // When using this API with an access point, you must direct requests to the + // access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com. + // When using this operation using an access point through the AWS SDKs, you + // provide the access point ARN in place of the bucket name. For more information + // about access point ARNs, see Using Access Points (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-access-points.html) + // in the Amazon Simple Storage Service Developer Guide. // // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` - // Specifies caching behavior along the request/reply chain. + // Can be used to specify caching behavior along the request/reply chain. For + // more information, see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9 + // (http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9). CacheControl *string `location:"header" locationName:"Cache-Control" type:"string"` - // Specifies presentational information for the object. + // Specifies presentational information for the object. For more information, + // see http://www.w3.org/Protocols/rfc2616/rfc2616-sec19.html#sec19.5.1 (http://www.w3.org/Protocols/rfc2616/rfc2616-sec19.html#sec19.5.1). ContentDisposition *string `location:"header" locationName:"Content-Disposition" type:"string"` // Specifies what content encodings have been applied to the object and thus // what decoding mechanisms must be applied to obtain the media-type referenced - // by the Content-Type header field. + // by the Content-Type header field. For more information, see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.11 + // (http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.11). ContentEncoding *string `location:"header" locationName:"Content-Encoding" type:"string"` // The language the content is in. ContentLanguage *string `location:"header" locationName:"Content-Language" type:"string"` - // The base64-encoded 128-bit MD5 digest of the part data. + // The base64-encoded 128-bit MD5 digest of the message (without the headers) + // according to RFC 1864. This header can be used as a message integrity check + // to verify that the data is the same data that was originally sent. Although + // it is optional, we recommend using the Content-MD5 mechanism as an end-to-end + // integrity check. For more information about REST request authentication, + // see REST Authentication (https://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html). ContentMD5 *string `location:"header" locationName:"Content-MD5" type:"string"` - // A standard MIME type describing the format of the object data. + // A standard MIME type describing the format of the contents. For more information, + // see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.17 (http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.17). ContentType *string `location:"header" locationName:"Content-Type" type:"string"` - // The date and time at which the object is no longer cacheable. + // The date and time at which the object is no longer cacheable. For more information, + // see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.21 (http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.21). Expires *time.Time `location:"header" locationName:"Expires" type:"timestamp"` // Gives the grantee READ, READ_ACP, and WRITE_ACP permissions on the object. @@ -68,7 +87,8 @@ type UploadInput struct { // A map of metadata to store with the object in S3. Metadata map[string]*string `location:"headers" locationName:"x-amz-meta-" type:"map"` - // The Legal Hold status that you want to apply to the specified object. + // Specifies whether a legal hold will be applied to this object. For more information + // about S3 Object Lock, see Object Lock (https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock.html). ObjectLockLegalHoldStatus *string `location:"header" locationName:"x-amz-object-lock-legal-hold" type:"string" enum:"ObjectLockLegalHoldStatus"` // The Object Lock mode that you want to apply to this object. @@ -77,38 +97,52 @@ type UploadInput struct { // The date and time when you want this object's Object Lock to expire. ObjectLockRetainUntilDate *time.Time `location:"header" locationName:"x-amz-object-lock-retain-until-date" type:"timestamp" timestampFormat:"iso8601"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // Documentation on downloading objects from requester pays buckets can be found - // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` - // Specifies the algorithm to use to when encrypting the object (e.g., AES256). + // Specifies the algorithm to use to when encrypting the object (for example, + // AES256). SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` // Specifies the customer-provided encryption key for Amazon S3 to use in encrypting // data. This value is used to store the object and then it is discarded; Amazon - // does not store the encryption key. The key must be appropriate for use with - // the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm + // S3 does not store the encryption key. The key must be appropriate for use + // with the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm // header. - SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"` + SSECustomerKey *string `marshal-as:"blob" location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"` // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. - // Amazon S3 uses this header for a message integrity check to ensure the encryption - // key was transmitted without error. + // Amazon S3 uses this header for a message integrity check to ensure that the + // encryption key was transmitted without error. SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` - // Specifies the AWS KMS key ID to use for object encryption. All GET and PUT - // requests for an object protected by AWS KMS will fail if not made via SSL - // or using SigV4. Documentation on configuring any of the officially supported - // AWS SDKs and CLI can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingAWSSDK.html#specify-signature-version + // Specifies the AWS KMS Encryption Context to use for object encryption. The + // value of this header is a base64-encoded UTF-8 string holding JSON with the + // encryption context key-value pairs. + SSEKMSEncryptionContext *string `location:"header" locationName:"x-amz-server-side-encryption-context" type:"string" sensitive:"true"` + + // If x-amz-server-side-encryption is present and has the value of aws:kms, + // this header specifies the ID of the AWS Key Management Service (AWS KMS) + // symmetrical customer managed customer master key (CMK) that was used for + // the object. + // + // If the value of x-amz-server-side-encryption is aws:kms, this header specifies + // the ID of the symmetric customer managed AWS KMS CMK that will be used for + // the object. If you specify x-amz-server-side-encryption:aws:kms, but do not + // providex-amz-server-side-encryption-aws-kms-key-id, Amazon S3 uses the AWS + // managed CMK in AWS to protect the data. SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` - // The Server-side encryption algorithm used when storing this object in S3 - // (e.g., AES256, aws:kms). + // The server-side encryption algorithm used when storing this object in Amazon + // S3 (for example, AES256, aws:kms). ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string" enum:"ServerSideEncryption"` - // The type of storage to use for the object. Defaults to 'STANDARD'. + // If you don't specify, S3 Standard is the default storage class. Amazon S3 + // supports other storage classes. StorageClass *string `location:"header" locationName:"x-amz-storage-class" type:"string" enum:"StorageClass"` // The tag-set for the object. The tag-set must be encoded as URL Query parameters. @@ -117,6 +151,21 @@ type UploadInput struct { // If the bucket is configured as a website, redirects requests for this object // to another object in the same bucket or to an external URL. Amazon S3 stores - // the value of this header in the object metadata. + // the value of this header in the object metadata. For information about object + // metadata, see Object Key and Metadata (https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html). + // + // In the following example, the request header sets the redirect to an object + // (anotherPage.html) in the same bucket: + // + // x-amz-website-redirect-location: /anotherPage.html + // + // In the following example, the request header sets the object redirect to + // another website: + // + // x-amz-website-redirect-location: http://www.example.com/ + // + // For more information about website hosting in Amazon S3, see Hosting Websites + // on Amazon S3 (https://docs.aws.amazon.com/AmazonS3/latest/dev/WebsiteHosting.html) + // and How to Configure Website Page Redirects (https://docs.aws.amazon.com/AmazonS3/latest/dev/how-to-page-redirect.html). WebsiteRedirectLocation *string `location:"header" locationName:"x-amz-website-redirect-location" type:"string"` } diff --git a/github.com/aws/aws-sdk-go/service/s3/s3manager/writer_read_from.go b/github.com/aws/aws-sdk-go/service/s3/s3manager/writer_read_from.go new file mode 100644 index 0000000000..765dc07ca3 --- /dev/null +++ b/github.com/aws/aws-sdk-go/service/s3/s3manager/writer_read_from.go @@ -0,0 +1,75 @@ +package s3manager + +import ( + "bufio" + "io" + "sync" + + "github.com/aws/aws-sdk-go/internal/sdkio" +) + +// WriterReadFrom defines an interface implementing io.Writer and io.ReaderFrom +type WriterReadFrom interface { + io.Writer + io.ReaderFrom +} + +// WriterReadFromProvider provides an implementation of io.ReadFrom for the given io.Writer +type WriterReadFromProvider interface { + GetReadFrom(writer io.Writer) (w WriterReadFrom, cleanup func()) +} + +type bufferedWriter interface { + WriterReadFrom + Flush() error + Reset(io.Writer) +} + +type bufferedReadFrom struct { + bufferedWriter +} + +func (b *bufferedReadFrom) ReadFrom(r io.Reader) (int64, error) { + n, err := b.bufferedWriter.ReadFrom(r) + if flushErr := b.Flush(); flushErr != nil && err == nil { + err = flushErr + } + return n, err +} + +// PooledBufferedReadFromProvider is a WriterReadFromProvider that uses a sync.Pool +// to manage allocation and reuse of *bufio.Writer structures. +type PooledBufferedReadFromProvider struct { + pool sync.Pool +} + +// NewPooledBufferedWriterReadFromProvider returns a new PooledBufferedReadFromProvider +// Size is used to control the size of the underlying *bufio.Writer created for +// calls to GetReadFrom. +func NewPooledBufferedWriterReadFromProvider(size int) *PooledBufferedReadFromProvider { + if size < int(32*sdkio.KibiByte) { + size = int(64 * sdkio.KibiByte) + } + + return &PooledBufferedReadFromProvider{ + pool: sync.Pool{ + New: func() interface{} { + return &bufferedReadFrom{bufferedWriter: bufio.NewWriterSize(nil, size)} + }, + }, + } +} + +// GetReadFrom takes an io.Writer and wraps it with a type which satisfies the WriterReadFrom +// interface/ Additionally a cleanup function is provided which must be called after usage of the WriterReadFrom +// has been completed in order to allow the reuse of the *bufio.Writer +func (p *PooledBufferedReadFromProvider) GetReadFrom(writer io.Writer) (r WriterReadFrom, cleanup func()) { + buffer := p.pool.Get().(*bufferedReadFrom) + buffer.Reset(writer) + r = buffer + cleanup = func() { + buffer.Reset(nil) // Reset to nil writer to release reference + p.pool.Put(buffer) + } + return r, cleanup +} diff --git a/github.com/aws/aws-sdk-go/service/s3/service.go b/github.com/aws/aws-sdk-go/service/s3/service.go index d17dcc9dad..b4c07b4d47 100644 --- a/github.com/aws/aws-sdk-go/service/s3/service.go +++ b/github.com/aws/aws-sdk-go/service/s3/service.go @@ -31,7 +31,7 @@ var initRequest func(*request.Request) const ( ServiceName = "s3" // Name of service. EndpointsID = ServiceName // ID to lookup a service endpoint with. - ServiceID = "S3" // ServiceID is a unique identifer of a specific service. + ServiceID = "S3" // ServiceID is a unique identifier of a specific service. ) // New creates a new instance of the S3 client with a session. @@ -39,6 +39,8 @@ const ( // aws.Config parameter to add your extra config. // // Example: +// mySession := session.Must(session.NewSession()) +// // // Create a S3 client from just a session. // svc := s3.New(mySession) // @@ -46,11 +48,11 @@ const ( // svc := s3.New(mySession, aws.NewConfig().WithRegion("us-west-2")) func New(p client.ConfigProvider, cfgs ...*aws.Config) *S3 { c := p.ClientConfig(EndpointsID, cfgs...) - return newClient(*c.Config, c.Handlers, c.Endpoint, c.SigningRegion, c.SigningName) + return newClient(*c.Config, c.Handlers, c.PartitionID, c.Endpoint, c.SigningRegion, c.SigningName) } // newClient creates, initializes and returns a new service client instance. -func newClient(cfg aws.Config, handlers request.Handlers, endpoint, signingRegion, signingName string) *S3 { +func newClient(cfg aws.Config, handlers request.Handlers, partitionID, endpoint, signingRegion, signingName string) *S3 { svc := &S3{ Client: client.New( cfg, @@ -59,6 +61,7 @@ func newClient(cfg aws.Config, handlers request.Handlers, endpoint, signingRegio ServiceID: ServiceID, SigningName: signingName, SigningRegion: signingRegion, + PartitionID: partitionID, Endpoint: endpoint, APIVersion: "2006-03-01", }, @@ -75,6 +78,7 @@ func newClient(cfg aws.Config, handlers request.Handlers, endpoint, signingRegio svc.Handlers.UnmarshalMeta.PushBackNamed(restxml.UnmarshalMetaHandler) svc.Handlers.UnmarshalError.PushBackNamed(restxml.UnmarshalErrorHandler) + svc.Handlers.BuildStream.PushBackNamed(restxml.BuildHandler) svc.Handlers.UnmarshalStream.PushBackNamed(restxml.UnmarshalHandler) // Run custom client initialization if present diff --git a/github.com/aws/aws-sdk-go/service/s3/sse.go b/github.com/aws/aws-sdk-go/service/s3/sse.go index 8010c4fa19..b71c835dee 100644 --- a/github.com/aws/aws-sdk-go/service/s3/sse.go +++ b/github.com/aws/aws-sdk-go/service/s3/sse.go @@ -3,6 +3,7 @@ package s3 import ( "crypto/md5" "encoding/base64" + "net/http" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/request" @@ -30,25 +31,54 @@ func validateSSERequiresSSL(r *request.Request) { } } -func computeSSEKeys(r *request.Request) { - headers := []string{ - "x-amz-server-side-encryption-customer-key", - "x-amz-copy-source-server-side-encryption-customer-key", +const ( + sseKeyHeader = "x-amz-server-side-encryption-customer-key" + sseKeyMD5Header = sseKeyHeader + "-md5" +) + +func computeSSEKeyMD5(r *request.Request) { + var key string + if g, ok := r.Params.(sseCustomerKeyGetter); ok { + key = g.getSSECustomerKey() + } + + computeKeyMD5(sseKeyHeader, sseKeyMD5Header, key, r.HTTPRequest) +} + +const ( + copySrcSSEKeyHeader = "x-amz-copy-source-server-side-encryption-customer-key" + copySrcSSEKeyMD5Header = copySrcSSEKeyHeader + "-md5" +) + +func computeCopySourceSSEKeyMD5(r *request.Request) { + var key string + if g, ok := r.Params.(copySourceSSECustomerKeyGetter); ok { + key = g.getCopySourceSSECustomerKey() } - for _, h := range headers { - md5h := h + "-md5" - if key := r.HTTPRequest.Header.Get(h); key != "" { - // Base64-encode the value - b64v := base64.StdEncoding.EncodeToString([]byte(key)) - r.HTTPRequest.Header.Set(h, b64v) - - // Add MD5 if it wasn't computed - if r.HTTPRequest.Header.Get(md5h) == "" { - sum := md5.Sum([]byte(key)) - b64sum := base64.StdEncoding.EncodeToString(sum[:]) - r.HTTPRequest.Header.Set(md5h, b64sum) - } + computeKeyMD5(copySrcSSEKeyHeader, copySrcSSEKeyMD5Header, key, r.HTTPRequest) +} + +func computeKeyMD5(keyHeader, keyMD5Header, key string, r *http.Request) { + if len(key) == 0 { + // Backwards compatiablity where user just set the header value instead + // of using the API parameter, or setting the header value for an + // operation without the parameters modeled. + key = r.Header.Get(keyHeader) + if len(key) == 0 { + return } + + // In backwards compatiable, the header's value is not base64 encoded, + // and needs to be encoded and updated by the SDK's customizations. + b64Key := base64.StdEncoding.EncodeToString([]byte(key)) + r.Header.Set(keyHeader, b64Key) + } + + // Only update Key's MD5 if not already set. + if len(r.Header.Get(keyMD5Header)) == 0 { + sum := md5.Sum([]byte(key)) + keyMD5 := base64.StdEncoding.EncodeToString(sum[:]) + r.Header.Set(keyMD5Header, keyMD5) } } diff --git a/github.com/aws/aws-sdk-go/service/s3/statusok_error.go b/github.com/aws/aws-sdk-go/service/s3/statusok_error.go index fde3050f95..247770e4c8 100644 --- a/github.com/aws/aws-sdk-go/service/s3/statusok_error.go +++ b/github.com/aws/aws-sdk-go/service/s3/statusok_error.go @@ -2,6 +2,7 @@ package s3 import ( "bytes" + "io" "io/ioutil" "net/http" @@ -14,7 +15,7 @@ func copyMultipartStatusOKUnmarhsalError(r *request.Request) { b, err := ioutil.ReadAll(r.HTTPResponse.Body) if err != nil { r.Error = awserr.NewRequestFailure( - awserr.New("SerializationError", "unable to read response body", err), + awserr.New(request.ErrCodeSerialization, "unable to read response body", err), r.HTTPResponse.StatusCode, r.RequestID, ) @@ -24,17 +25,18 @@ func copyMultipartStatusOKUnmarhsalError(r *request.Request) { r.HTTPResponse.Body = ioutil.NopCloser(body) defer body.Seek(0, sdkio.SeekStart) - if body.Len() == 0 { - // If there is no body don't attempt to parse the body. - return - } - unmarshalError(r) if err, ok := r.Error.(awserr.Error); ok && err != nil { - if err.Code() == "SerializationError" { + if err.Code() == request.ErrCodeSerialization && + err.OrigErr() != io.EOF { r.Error = nil return } - r.HTTPResponse.StatusCode = http.StatusServiceUnavailable + // if empty payload + if err.OrigErr() == io.EOF { + r.HTTPResponse.StatusCode = http.StatusInternalServerError + } else { + r.HTTPResponse.StatusCode = http.StatusServiceUnavailable + } } } diff --git a/github.com/aws/aws-sdk-go/service/s3/unmarshal_error.go b/github.com/aws/aws-sdk-go/service/s3/unmarshal_error.go index 12c0612c8d..6eecf66910 100644 --- a/github.com/aws/aws-sdk-go/service/s3/unmarshal_error.go +++ b/github.com/aws/aws-sdk-go/service/s3/unmarshal_error.go @@ -1,6 +1,7 @@ package s3 import ( + "bytes" "encoding/xml" "fmt" "io" @@ -11,6 +12,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil" ) type xmlErrorResponse struct { @@ -26,40 +28,57 @@ func unmarshalError(r *request.Request) { // Bucket exists in a different region, and request needs // to be made to the correct region. if r.HTTPResponse.StatusCode == http.StatusMovedPermanently { + msg := fmt.Sprintf( + "incorrect region, the bucket is not in '%s' region at endpoint '%s'", + aws.StringValue(r.Config.Region), + aws.StringValue(r.Config.Endpoint), + ) + if v := r.HTTPResponse.Header.Get("x-amz-bucket-region"); len(v) != 0 { + msg += fmt.Sprintf(", bucket is in '%s' region", v) + } r.Error = awserr.NewRequestFailure( - awserr.New("BucketRegionError", - fmt.Sprintf("incorrect region, the bucket is not in '%s' region", - aws.StringValue(r.Config.Region)), - nil), + awserr.New("BucketRegionError", msg, nil), r.HTTPResponse.StatusCode, r.RequestID, ) return } - var errCode, errMsg string - // Attempt to parse error from body if it is known - resp := &xmlErrorResponse{} - err := xml.NewDecoder(r.HTTPResponse.Body).Decode(resp) - if err != nil && err != io.EOF { - errCode = "SerializationError" - errMsg = "failed to decode S3 XML error response" + var errResp xmlErrorResponse + var err error + if r.HTTPResponse.StatusCode >= 200 && r.HTTPResponse.StatusCode < 300 { + err = s3unmarshalXMLError(&errResp, r.HTTPResponse.Body) } else { - errCode = resp.Code - errMsg = resp.Message - err = nil + err = xmlutil.UnmarshalXMLError(&errResp, r.HTTPResponse.Body) + } + + if err != nil { + var errorMsg string + if err == io.EOF { + errorMsg = "empty response payload" + } else { + errorMsg = "failed to unmarshal error message" + } + + r.Error = awserr.NewRequestFailure( + awserr.New(request.ErrCodeSerialization, + errorMsg, err), + r.HTTPResponse.StatusCode, + r.RequestID, + ) + return } // Fallback to status code converted to message if still no error code - if len(errCode) == 0 { + if len(errResp.Code) == 0 { statusText := http.StatusText(r.HTTPResponse.StatusCode) - errCode = strings.Replace(statusText, " ", "", -1) - errMsg = statusText + errResp.Code = strings.Replace(statusText, " ", "", -1) + errResp.Message = statusText } r.Error = awserr.NewRequestFailure( - awserr.New(errCode, errMsg, err), + awserr.New(errResp.Code, errResp.Message, err), r.HTTPResponse.StatusCode, r.RequestID, ) @@ -75,3 +94,21 @@ type RequestFailure interface { // Host ID is the S3 Host ID needed for debug, and contacting support HostID() string } + +// s3unmarshalXMLError is s3 specific xml error unmarshaler +// for 200 OK errors and response payloads. +// This function differs from the xmlUtil.UnmarshalXMLError +// func. It does not ignore the EOF error and passes it up. +// Related to bug fix for `s3 200 OK response with empty payload` +func s3unmarshalXMLError(v interface{}, stream io.Reader) error { + var errBuf bytes.Buffer + body := io.TeeReader(stream, &errBuf) + + err := xml.NewDecoder(body).Decode(v) + if err != nil && err != io.EOF { + return awserr.NewUnmarshalError(err, + "failed to unmarshal error message", errBuf.Bytes()) + } + + return err +} diff --git a/github.com/aws/aws-sdk-go/service/sts/api.go b/github.com/aws/aws-sdk-go/service/sts/api.go index ee908f9167..550b5f687f 100644 --- a/github.com/aws/aws-sdk-go/service/sts/api.go +++ b/github.com/aws/aws-sdk-go/service/sts/api.go @@ -3,10 +3,12 @@ package sts import ( + "fmt" "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awsutil" + "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/request" ) @@ -54,39 +56,29 @@ func (c *STS) AssumeRoleRequest(input *AssumeRoleInput) (req *request.Request, o // AssumeRole API operation for AWS Security Token Service. // -// Returns a set of temporary security credentials (consisting of an access -// key ID, a secret access key, and a security token) that you can use to access -// AWS resources that you might not normally have access to. Typically, you -// use AssumeRole for cross-account access or federation. For a comparison of -// AssumeRole with the other APIs that produce temporary credentials, see Requesting -// Temporary Security Credentials (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html) -// and Comparing the AWS STS APIs (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison) +// Returns a set of temporary security credentials that you can use to access +// AWS resources that you might not normally have access to. These temporary +// credentials consist of an access key ID, a secret access key, and a security +// token. Typically, you use AssumeRole within your account or for cross-account +// access. For a comparison of AssumeRole with other API operations that produce +// temporary credentials, see Requesting Temporary Security Credentials (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html) +// and Comparing the AWS STS API operations (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison) // in the IAM User Guide. // -// Important: You cannot call AssumeRole by using AWS root account credentials; -// access is denied. You must use credentials for an IAM user or an IAM role -// to call AssumeRole. +// You cannot use AWS account root user credentials to call AssumeRole. You +// must use credentials for an IAM user or an IAM role to call AssumeRole. // // For cross-account access, imagine that you own multiple accounts and need // to access resources in each account. You could create long-term credentials // in each account to access those resources. However, managing all those credentials // and remembering which one can access which account can be time consuming. -// Instead, you can create one set of long-term credentials in one account and -// then use temporary security credentials to access all the other accounts +// Instead, you can create one set of long-term credentials in one account. +// Then use temporary security credentials to access all the other accounts // by assuming roles in those accounts. For more information about roles, see -// IAM Roles (Delegation and Federation) (http://docs.aws.amazon.com/IAM/latest/UserGuide/roles-toplevel.html) +// IAM Roles (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html) // in the IAM User Guide. // -// For federation, you can, for example, grant single sign-on access to the -// AWS Management Console. If you already have an identity and authentication -// system in your corporate network, you don't have to recreate user identities -// in AWS in order to grant those user identities access to AWS. Instead, after -// a user has been authenticated, you call AssumeRole (and specify the role -// with the appropriate permissions) to get temporary security credentials for -// that user. With those temporary security credentials, you construct a sign-in -// URL that users can use to access the console. For more information, see Common -// Scenarios for Temporary Credentials (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html#sts-introduction) -// in the IAM User Guide. +// Session Duration // // By default, the temporary security credentials created by AssumeRole last // for one hour. However, you can use the optional DurationSeconds parameter @@ -94,69 +86,93 @@ func (c *STS) AssumeRoleRequest(input *AssumeRoleInput) (req *request.Request, o // seconds (15 minutes) up to the maximum session duration setting for the role. // This setting can have a value from 1 hour to 12 hours. To learn how to view // the maximum value for your role, see View the Maximum Session Duration Setting -// for a Role (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html#id_roles_use_view-role-max-session) +// for a Role (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html#id_roles_use_view-role-max-session) // in the IAM User Guide. The maximum session duration limit applies when you -// use the AssumeRole* API operations or the assume-role* CLI operations but -// does not apply when you use those operations to create a console URL. For -// more information, see Using IAM Roles (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html) +// use the AssumeRole* API operations or the assume-role* CLI commands. However +// the limit does not apply when you use those operations to create a console +// URL. For more information, see Using IAM Roles (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html) // in the IAM User Guide. // +// Permissions +// // The temporary security credentials created by AssumeRole can be used to make -// API calls to any AWS service with the following exception: you cannot call -// the STS service's GetFederationToken or GetSessionToken APIs. -// -// Optionally, you can pass an IAM access policy to this operation. If you choose -// not to pass a policy, the temporary security credentials that are returned -// by the operation have the permissions that are defined in the access policy -// of the role that is being assumed. If you pass a policy to this operation, -// the temporary security credentials that are returned by the operation have -// the permissions that are allowed by both the access policy of the role that -// is being assumed, and the policy that you pass. This gives you a way to further -// restrict the permissions for the resulting temporary security credentials. -// You cannot use the passed policy to grant permissions that are in excess -// of those allowed by the access policy of the role that is being assumed. -// For more information, see Permissions for AssumeRole, AssumeRoleWithSAML, -// and AssumeRoleWithWebIdentity (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_control-access_assumerole.html) +// API calls to any AWS service with the following exception: You cannot call +// the AWS STS GetFederationToken or GetSessionToken API operations. +// +// (Optional) You can pass inline or managed session policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session) +// to this operation. You can pass a single JSON policy document to use as an +// inline session policy. You can also specify up to 10 managed policies to +// use as managed session policies. The plain text that you use for both inline +// and managed session policies can't exceed 2,048 characters. Passing policies +// to this operation returns new temporary credentials. The resulting session's +// permissions are the intersection of the role's identity-based policy and +// the session policies. You can use the role's temporary credentials in subsequent +// AWS API calls to access resources in the account that owns the role. You +// cannot use session policies to grant more permissions than those allowed +// by the identity-based policy of the role that is being assumed. For more +// information, see Session Policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session) // in the IAM User Guide. // -// To assume a role, your AWS account must be trusted by the role. The trust -// relationship is defined in the role's trust policy when the role is created. -// That trust policy states which accounts are allowed to delegate access to -// this account's role. -// -// The user who wants to access the role must also have permissions delegated -// from the role's administrator. If the user is in a different account than -// the role, then the user's administrator must attach a policy that allows -// the user to call AssumeRole on the ARN of the role in the other account. -// If the user is in the same account as the role, then you can either attach -// a policy to the user (identical to the previous different account user), -// or you can add the user as a principal directly in the role's trust policy. -// In this case, the trust policy acts as the only resource-based policy in -// IAM, and users in the same account as the role do not need explicit permission -// to assume the role. For more information about trust policies and resource-based -// policies, see IAM Policies (http://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html) +// To assume a role from a different account, your AWS account must be trusted +// by the role. The trust relationship is defined in the role's trust policy +// when the role is created. That trust policy states which accounts are allowed +// to delegate that access to users in the account. +// +// A user who wants to access a role in a different account must also have permissions +// that are delegated from the user account administrator. The administrator +// must attach a policy that allows the user to call AssumeRole for the ARN +// of the role in the other account. If the user is in the same account as the +// role, then you can do either of the following: +// +// * Attach a policy to the user (identical to the previous user in a different +// account). +// +// * Add the user as a principal directly in the role's trust policy. +// +// In this case, the trust policy acts as an IAM resource-based policy. Users +// in the same account as the role do not need explicit permission to assume +// the role. For more information about trust policies and resource-based policies, +// see IAM Policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html) +// in the IAM User Guide. +// +// Tags +// +// (Optional) You can pass tag key-value pairs to your session. These tags are +// called session tags. For more information about session tags, see Passing +// Session Tags in STS (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_session-tags.html) +// in the IAM User Guide. +// +// An administrator must grant you the permissions necessary to pass session +// tags. The administrator can also create granular permissions to allow you +// to pass only specific session tags. For more information, see Tutorial: Using +// Tags for Attribute-Based Access Control (https://docs.aws.amazon.com/IAM/latest/UserGuide/tutorial_attribute-based-access-control.html) +// in the IAM User Guide. +// +// You can set the session tags as transitive. Transitive tags persist during +// role chaining. For more information, see Chaining Roles with Session Tags +// (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_session-tags.html#id_session-tags_role-chaining) // in the IAM User Guide. // // Using MFA with AssumeRole // -// You can optionally include multi-factor authentication (MFA) information -// when you call AssumeRole. This is useful for cross-account scenarios in which -// you want to make sure that the user who is assuming the role has been authenticated -// using an AWS MFA device. In that scenario, the trust policy of the role being -// assumed includes a condition that tests for MFA authentication; if the caller -// does not include valid MFA information, the request to assume the role is -// denied. The condition in a trust policy that tests for MFA authentication -// might look like the following example. +// (Optional) You can include multi-factor authentication (MFA) information +// when you call AssumeRole. This is useful for cross-account scenarios to ensure +// that the user that assumes the role has been authenticated with an AWS MFA +// device. In that scenario, the trust policy of the role being assumed includes +// a condition that tests for MFA authentication. If the caller does not include +// valid MFA information, the request to assume the role is denied. The condition +// in a trust policy that tests for MFA authentication might look like the following +// example. // // "Condition": {"Bool": {"aws:MultiFactorAuthPresent": true}} // -// For more information, see Configuring MFA-Protected API Access (http://docs.aws.amazon.com/IAM/latest/UserGuide/MFAProtectedAPI.html) +// For more information, see Configuring MFA-Protected API Access (https://docs.aws.amazon.com/IAM/latest/UserGuide/MFAProtectedAPI.html) // in the IAM User Guide guide. // // To use MFA with AssumeRole, you pass values for the SerialNumber and TokenCode // parameters. The SerialNumber value identifies the user's hardware or virtual // MFA device. The TokenCode is the time-based one-time password (TOTP) that -// the MFA devices produces. +// the MFA device produces. // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -171,15 +187,24 @@ func (c *STS) AssumeRoleRequest(input *AssumeRoleInput) (req *request.Request, o // message describes the specific error. // // * ErrCodePackedPolicyTooLargeException "PackedPolicyTooLarge" -// The request was rejected because the policy document was too large. The error -// message describes how big the policy document is, in packed form, as a percentage -// of what the API allows. +// The request was rejected because the total packed size of the session policies +// and session tags combined was too large. An AWS conversion compresses the +// session policy document, session policy ARNs, and session tags into a packed +// binary format that has a separate limit. The error message indicates by percentage +// how close the policies and tags are to the upper size limit. For more information, +// see Passing Session Tags in STS (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_session-tags.html) +// in the IAM User Guide. +// +// You could receive this error even though you meet other defined session policy +// and session tag limits. For more information, see IAM and STS Entity Character +// Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html) +// in the IAM User Guide. // // * ErrCodeRegionDisabledException "RegionDisabledException" // STS is not activated in the requested region for the account that is being // asked to generate credentials. The account administrator must use the IAM // console to activate STS in that region. For more information, see Activating -// and Deactivating AWS STS in an AWS Region (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html) +// and Deactivating AWS STS in an AWS Region (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html) // in the IAM User Guide. // // See also, https://docs.aws.amazon.com/goto/WebAPI/sts-2011-06-15/AssumeRole @@ -243,6 +268,7 @@ func (c *STS) AssumeRoleWithSAMLRequest(input *AssumeRoleWithSAMLInput) (req *re output = &AssumeRoleWithSAMLOutput{} req = c.newRequest(op, input, output) + req.Config.Credentials = credentials.AnonymousCredentials return } @@ -252,15 +278,17 @@ func (c *STS) AssumeRoleWithSAMLRequest(input *AssumeRoleWithSAMLInput) (req *re // via a SAML authentication response. This operation provides a mechanism for // tying an enterprise identity store or directory to role-based AWS access // without user-specific credentials or configuration. For a comparison of AssumeRoleWithSAML -// with the other APIs that produce temporary credentials, see Requesting Temporary -// Security Credentials (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html) -// and Comparing the AWS STS APIs (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison) +// with the other API operations that produce temporary credentials, see Requesting +// Temporary Security Credentials (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html) +// and Comparing the AWS STS API operations (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison) // in the IAM User Guide. // // The temporary security credentials returned by this operation consist of // an access key ID, a secret access key, and a security token. Applications // can use these temporary security credentials to sign calls to AWS services. // +// Session Duration +// // By default, the temporary security credentials created by AssumeRoleWithSAML // last for one hour. However, you can use the optional DurationSeconds parameter // to specify the duration of your session. Your role session lasts for the @@ -269,38 +297,33 @@ func (c *STS) AssumeRoleWithSAMLRequest(input *AssumeRoleWithSAMLInput) (req *re // a DurationSeconds value from 900 seconds (15 minutes) up to the maximum session // duration setting for the role. This setting can have a value from 1 hour // to 12 hours. To learn how to view the maximum value for your role, see View -// the Maximum Session Duration Setting for a Role (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html#id_roles_use_view-role-max-session) +// the Maximum Session Duration Setting for a Role (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html#id_roles_use_view-role-max-session) // in the IAM User Guide. The maximum session duration limit applies when you -// use the AssumeRole* API operations or the assume-role* CLI operations but -// does not apply when you use those operations to create a console URL. For -// more information, see Using IAM Roles (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html) +// use the AssumeRole* API operations or the assume-role* CLI commands. However +// the limit does not apply when you use those operations to create a console +// URL. For more information, see Using IAM Roles (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html) // in the IAM User Guide. // +// Permissions +// // The temporary security credentials created by AssumeRoleWithSAML can be used // to make API calls to any AWS service with the following exception: you cannot -// call the STS service's GetFederationToken or GetSessionToken APIs. -// -// Optionally, you can pass an IAM access policy to this operation. If you choose -// not to pass a policy, the temporary security credentials that are returned -// by the operation have the permissions that are defined in the access policy -// of the role that is being assumed. If you pass a policy to this operation, -// the temporary security credentials that are returned by the operation have -// the permissions that are allowed by the intersection of both the access policy -// of the role that is being assumed, and the policy that you pass. This means -// that both policies must grant the permission for the action to be allowed. -// This gives you a way to further restrict the permissions for the resulting -// temporary security credentials. You cannot use the passed policy to grant -// permissions that are in excess of those allowed by the access policy of the -// role that is being assumed. For more information, see Permissions for AssumeRole, -// AssumeRoleWithSAML, and AssumeRoleWithWebIdentity (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_control-access_assumerole.html) +// call the STS GetFederationToken or GetSessionToken API operations. +// +// (Optional) You can pass inline or managed session policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session) +// to this operation. You can pass a single JSON policy document to use as an +// inline session policy. You can also specify up to 10 managed policies to +// use as managed session policies. The plain text that you use for both inline +// and managed session policies can't exceed 2,048 characters. Passing policies +// to this operation returns new temporary credentials. The resulting session's +// permissions are the intersection of the role's identity-based policy and +// the session policies. You can use the role's temporary credentials in subsequent +// AWS API calls to access resources in the account that owns the role. You +// cannot use session policies to grant more permissions than those allowed +// by the identity-based policy of the role that is being assumed. For more +// information, see Session Policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session) // in the IAM User Guide. // -// Before your application can call AssumeRoleWithSAML, you must configure your -// SAML identity provider (IdP) to issue the claims required by AWS. Additionally, -// you must use AWS Identity and Access Management (IAM) to create a SAML provider -// entity in your AWS account that represents your identity provider, and create -// an IAM role that specifies this SAML provider in its trust policy. -// // Calling AssumeRoleWithSAML does not require the use of AWS security credentials. // The identity of the caller is validated by using keys in the metadata document // that is uploaded for the SAML provider entity for your identity provider. @@ -308,21 +331,63 @@ func (c *STS) AssumeRoleWithSAMLRequest(input *AssumeRoleWithSAMLInput) (req *re // Calling AssumeRoleWithSAML can result in an entry in your AWS CloudTrail // logs. The entry includes the value in the NameID element of the SAML assertion. // We recommend that you use a NameIDType that is not associated with any personally -// identifiable information (PII). For example, you could instead use the Persistent -// Identifier (urn:oasis:names:tc:SAML:2.0:nameid-format:persistent). +// identifiable information (PII). For example, you could instead use the persistent +// identifier (urn:oasis:names:tc:SAML:2.0:nameid-format:persistent). +// +// Tags +// +// (Optional) You can configure your IdP to pass attributes into your SAML assertion +// as session tags. Each session tag consists of a key name and an associated +// value. For more information about session tags, see Passing Session Tags +// in STS (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_session-tags.html) +// in the IAM User Guide. +// +// You can pass up to 50 session tags. The plain text session tag keys can’t +// exceed 128 characters and the values can’t exceed 256 characters. For these +// and additional limits, see IAM and STS Character Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-limits.html#reference_iam-limits-entity-length) +// in the IAM User Guide. +// +// An AWS conversion compresses the passed session policies and session tags +// into a packed binary format that has a separate limit. Your request can fail +// for this limit even if your plain text meets the other requirements. The +// PackedPolicySize response element indicates by percentage how close the policies +// and tags for your request are to the upper size limit. +// +// You can pass a session tag with the same key as a tag that is attached to +// the role. When you do, session tags override the role's tags with the same +// key. +// +// An administrator must grant you the permissions necessary to pass session +// tags. The administrator can also create granular permissions to allow you +// to pass only specific session tags. For more information, see Tutorial: Using +// Tags for Attribute-Based Access Control (https://docs.aws.amazon.com/IAM/latest/UserGuide/tutorial_attribute-based-access-control.html) +// in the IAM User Guide. +// +// You can set the session tags as transitive. Transitive tags persist during +// role chaining. For more information, see Chaining Roles with Session Tags +// (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_session-tags.html#id_session-tags_role-chaining) +// in the IAM User Guide. +// +// SAML Configuration +// +// Before your application can call AssumeRoleWithSAML, you must configure your +// SAML identity provider (IdP) to issue the claims required by AWS. Additionally, +// you must use AWS Identity and Access Management (IAM) to create a SAML provider +// entity in your AWS account that represents your identity provider. You must +// also create an IAM role that specifies this SAML provider in its trust policy. // // For more information, see the following resources: // -// * About SAML 2.0-based Federation (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_saml.html) +// * About SAML 2.0-based Federation (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_saml.html) // in the IAM User Guide. // -// * Creating SAML Identity Providers (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_create_saml.html) +// * Creating SAML Identity Providers (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_create_saml.html) // in the IAM User Guide. // -// * Configuring a Relying Party and Claims (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_create_saml_relying-party.html) +// * Configuring a Relying Party and Claims (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_create_saml_relying-party.html) // in the IAM User Guide. // -// * Creating a Role for SAML 2.0 Federation (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-idp_saml.html) +// * Creating a Role for SAML 2.0 Federation (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-idp_saml.html) // in the IAM User Guide. // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions @@ -338,9 +403,18 @@ func (c *STS) AssumeRoleWithSAMLRequest(input *AssumeRoleWithSAMLInput) (req *re // message describes the specific error. // // * ErrCodePackedPolicyTooLargeException "PackedPolicyTooLarge" -// The request was rejected because the policy document was too large. The error -// message describes how big the policy document is, in packed form, as a percentage -// of what the API allows. +// The request was rejected because the total packed size of the session policies +// and session tags combined was too large. An AWS conversion compresses the +// session policy document, session policy ARNs, and session tags into a packed +// binary format that has a separate limit. The error message indicates by percentage +// how close the policies and tags are to the upper size limit. For more information, +// see Passing Session Tags in STS (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_session-tags.html) +// in the IAM User Guide. +// +// You could receive this error even though you meet other defined session policy +// and session tag limits. For more information, see IAM and STS Entity Character +// Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html) +// in the IAM User Guide. // // * ErrCodeIDPRejectedClaimException "IDPRejectedClaim" // The identity provider (IdP) reported that authentication failed. This might @@ -361,7 +435,7 @@ func (c *STS) AssumeRoleWithSAMLRequest(input *AssumeRoleWithSAMLInput) (req *re // STS is not activated in the requested region for the account that is being // asked to generate credentials. The account administrator must use the IAM // console to activate STS in that region. For more information, see Activating -// and Deactivating AWS STS in an AWS Region (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html) +// and Deactivating AWS STS in an AWS Region (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html) // in the IAM User Guide. // // See also, https://docs.aws.amazon.com/goto/WebAPI/sts-2011-06-15/AssumeRoleWithSAML @@ -425,41 +499,44 @@ func (c *STS) AssumeRoleWithWebIdentityRequest(input *AssumeRoleWithWebIdentityI output = &AssumeRoleWithWebIdentityOutput{} req = c.newRequest(op, input, output) + req.Config.Credentials = credentials.AnonymousCredentials return } // AssumeRoleWithWebIdentity API operation for AWS Security Token Service. // // Returns a set of temporary security credentials for users who have been authenticated -// in a mobile or web application with a web identity provider, such as Amazon -// Cognito, Login with Amazon, Facebook, Google, or any OpenID Connect-compatible -// identity provider. +// in a mobile or web application with a web identity provider. Example providers +// include Amazon Cognito, Login with Amazon, Facebook, Google, or any OpenID +// Connect-compatible identity provider. // // For mobile applications, we recommend that you use Amazon Cognito. You can -// use Amazon Cognito with the AWS SDK for iOS (http://aws.amazon.com/sdkforios/) -// and the AWS SDK for Android (http://aws.amazon.com/sdkforandroid/) to uniquely -// identify a user and supply the user with a consistent identity throughout -// the lifetime of an application. -// -// To learn more about Amazon Cognito, see Amazon Cognito Overview (http://docs.aws.amazon.com/mobile/sdkforandroid/developerguide/cognito-auth.html#d0e840) -// in the AWS SDK for Android Developer Guide guide and Amazon Cognito Overview -// (http://docs.aws.amazon.com/mobile/sdkforios/developerguide/cognito-auth.html#d0e664) +// use Amazon Cognito with the AWS SDK for iOS Developer Guide (http://aws.amazon.com/sdkforios/) +// and the AWS SDK for Android Developer Guide (http://aws.amazon.com/sdkforandroid/) +// to uniquely identify a user. You can also supply the user with a consistent +// identity throughout the lifetime of an application. +// +// To learn more about Amazon Cognito, see Amazon Cognito Overview (https://docs.aws.amazon.com/mobile/sdkforandroid/developerguide/cognito-auth.html#d0e840) +// in AWS SDK for Android Developer Guide and Amazon Cognito Overview (https://docs.aws.amazon.com/mobile/sdkforios/developerguide/cognito-auth.html#d0e664) // in the AWS SDK for iOS Developer Guide. // // Calling AssumeRoleWithWebIdentity does not require the use of AWS security // credentials. Therefore, you can distribute an application (for example, on // mobile devices) that requests temporary security credentials without including -// long-term AWS credentials in the application, and without deploying server-based -// proxy services that use long-term AWS credentials. Instead, the identity -// of the caller is validated by using a token from the web identity provider. -// For a comparison of AssumeRoleWithWebIdentity with the other APIs that produce -// temporary credentials, see Requesting Temporary Security Credentials (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html) -// and Comparing the AWS STS APIs (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison) +// long-term AWS credentials in the application. You also don't need to deploy +// server-based proxy services that use long-term AWS credentials. Instead, +// the identity of the caller is validated by using a token from the web identity +// provider. For a comparison of AssumeRoleWithWebIdentity with the other API +// operations that produce temporary credentials, see Requesting Temporary Security +// Credentials (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html) +// and Comparing the AWS STS API operations (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison) // in the IAM User Guide. // // The temporary security credentials returned by this API consist of an access // key ID, a secret access key, and a security token. Applications can use these -// temporary security credentials to sign calls to AWS service APIs. +// temporary security credentials to sign calls to AWS service API operations. +// +// Session Duration // // By default, the temporary security credentials created by AssumeRoleWithWebIdentity // last for one hour. However, you can use the optional DurationSeconds parameter @@ -467,31 +544,69 @@ func (c *STS) AssumeRoleWithWebIdentityRequest(input *AssumeRoleWithWebIdentityI // seconds (15 minutes) up to the maximum session duration setting for the role. // This setting can have a value from 1 hour to 12 hours. To learn how to view // the maximum value for your role, see View the Maximum Session Duration Setting -// for a Role (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html#id_roles_use_view-role-max-session) +// for a Role (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html#id_roles_use_view-role-max-session) // in the IAM User Guide. The maximum session duration limit applies when you -// use the AssumeRole* API operations or the assume-role* CLI operations but -// does not apply when you use those operations to create a console URL. For -// more information, see Using IAM Roles (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html) +// use the AssumeRole* API operations or the assume-role* CLI commands. However +// the limit does not apply when you use those operations to create a console +// URL. For more information, see Using IAM Roles (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html) // in the IAM User Guide. // +// Permissions +// // The temporary security credentials created by AssumeRoleWithWebIdentity can // be used to make API calls to any AWS service with the following exception: -// you cannot call the STS service's GetFederationToken or GetSessionToken APIs. -// -// Optionally, you can pass an IAM access policy to this operation. If you choose -// not to pass a policy, the temporary security credentials that are returned -// by the operation have the permissions that are defined in the access policy -// of the role that is being assumed. If you pass a policy to this operation, -// the temporary security credentials that are returned by the operation have -// the permissions that are allowed by both the access policy of the role that -// is being assumed, and the policy that you pass. This gives you a way to further -// restrict the permissions for the resulting temporary security credentials. -// You cannot use the passed policy to grant permissions that are in excess -// of those allowed by the access policy of the role that is being assumed. -// For more information, see Permissions for AssumeRole, AssumeRoleWithSAML, -// and AssumeRoleWithWebIdentity (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_control-access_assumerole.html) +// you cannot call the STS GetFederationToken or GetSessionToken API operations. +// +// (Optional) You can pass inline or managed session policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session) +// to this operation. You can pass a single JSON policy document to use as an +// inline session policy. You can also specify up to 10 managed policies to +// use as managed session policies. The plain text that you use for both inline +// and managed session policies can't exceed 2,048 characters. Passing policies +// to this operation returns new temporary credentials. The resulting session's +// permissions are the intersection of the role's identity-based policy and +// the session policies. You can use the role's temporary credentials in subsequent +// AWS API calls to access resources in the account that owns the role. You +// cannot use session policies to grant more permissions than those allowed +// by the identity-based policy of the role that is being assumed. For more +// information, see Session Policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session) // in the IAM User Guide. // +// Tags +// +// (Optional) You can configure your IdP to pass attributes into your web identity +// token as session tags. Each session tag consists of a key name and an associated +// value. For more information about session tags, see Passing Session Tags +// in STS (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_session-tags.html) +// in the IAM User Guide. +// +// You can pass up to 50 session tags. The plain text session tag keys can’t +// exceed 128 characters and the values can’t exceed 256 characters. For these +// and additional limits, see IAM and STS Character Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-limits.html#reference_iam-limits-entity-length) +// in the IAM User Guide. +// +// An AWS conversion compresses the passed session policies and session tags +// into a packed binary format that has a separate limit. Your request can fail +// for this limit even if your plain text meets the other requirements. The +// PackedPolicySize response element indicates by percentage how close the policies +// and tags for your request are to the upper size limit. +// +// You can pass a session tag with the same key as a tag that is attached to +// the role. When you do, the session tag overrides the role tag with the same +// key. +// +// An administrator must grant you the permissions necessary to pass session +// tags. The administrator can also create granular permissions to allow you +// to pass only specific session tags. For more information, see Tutorial: Using +// Tags for Attribute-Based Access Control (https://docs.aws.amazon.com/IAM/latest/UserGuide/tutorial_attribute-based-access-control.html) +// in the IAM User Guide. +// +// You can set the session tags as transitive. Transitive tags persist during +// role chaining. For more information, see Chaining Roles with Session Tags +// (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_session-tags.html#id_session-tags_role-chaining) +// in the IAM User Guide. +// +// Identities +// // Before your application can call AssumeRoleWithWebIdentity, you must have // an identity token from a supported identity provider and create a role that // the application can assume. The role that your application assumes must trust @@ -508,21 +623,19 @@ func (c *STS) AssumeRoleWithWebIdentityRequest(input *AssumeRoleWithWebIdentityI // For more information about how to use web identity federation and the AssumeRoleWithWebIdentity // API, see the following resources: // -// * Using Web Identity Federation APIs for Mobile Apps (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_oidc_manual.html) -// and Federation Through a Web-based Identity Provider (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#api_assumerolewithwebidentity). -// -// -// * Web Identity Federation Playground (https://web-identity-federation-playground.s3.amazonaws.com/index.html). -// This interactive website lets you walk through the process of authenticating -// via Login with Amazon, Facebook, or Google, getting temporary security -// credentials, and then using those credentials to make a request to AWS. +// * Using Web Identity Federation API Operations for Mobile Apps (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_oidc_manual.html) +// and Federation Through a Web-based Identity Provider (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#api_assumerolewithwebidentity). // +// * Web Identity Federation Playground (https://web-identity-federation-playground.s3.amazonaws.com/index.html). +// Walk through the process of authenticating through Login with Amazon, +// Facebook, or Google, getting temporary security credentials, and then +// using those credentials to make a request to AWS. // -// * AWS SDK for iOS (http://aws.amazon.com/sdkforios/) and AWS SDK for Android -// (http://aws.amazon.com/sdkforandroid/). These toolkits contain sample -// apps that show how to invoke the identity providers, and then how to use -// the information from these providers to get and use temporary security -// credentials. +// * AWS SDK for iOS Developer Guide (http://aws.amazon.com/sdkforios/) and +// AWS SDK for Android Developer Guide (http://aws.amazon.com/sdkforandroid/). +// These toolkits contain sample apps that show how to invoke the identity +// providers. The toolkits then show how to use the information from these +// providers to get and use temporary security credentials. // // * Web Identity Federation with Mobile Applications (http://aws.amazon.com/articles/web-identity-federation-with-mobile-applications). // This article discusses web identity federation and shows an example of @@ -542,9 +655,18 @@ func (c *STS) AssumeRoleWithWebIdentityRequest(input *AssumeRoleWithWebIdentityI // message describes the specific error. // // * ErrCodePackedPolicyTooLargeException "PackedPolicyTooLarge" -// The request was rejected because the policy document was too large. The error -// message describes how big the policy document is, in packed form, as a percentage -// of what the API allows. +// The request was rejected because the total packed size of the session policies +// and session tags combined was too large. An AWS conversion compresses the +// session policy document, session policy ARNs, and session tags into a packed +// binary format that has a separate limit. The error message indicates by percentage +// how close the policies and tags are to the upper size limit. For more information, +// see Passing Session Tags in STS (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_session-tags.html) +// in the IAM User Guide. +// +// You could receive this error even though you meet other defined session policy +// and session tag limits. For more information, see IAM and STS Entity Character +// Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html) +// in the IAM User Guide. // // * ErrCodeIDPRejectedClaimException "IDPRejectedClaim" // The identity provider (IdP) reported that authentication failed. This might @@ -554,11 +676,11 @@ func (c *STS) AssumeRoleWithWebIdentityRequest(input *AssumeRoleWithWebIdentityI // can also mean that the claim has expired or has been explicitly revoked. // // * ErrCodeIDPCommunicationErrorException "IDPCommunicationError" -// The request could not be fulfilled because the non-AWS identity provider -// (IDP) that was asked to verify the incoming identity token could not be reached. -// This is often a transient error caused by network conditions. Retry the request +// The request could not be fulfilled because the identity provider (IDP) that +// was asked to verify the incoming identity token could not be reached. This +// is often a transient error caused by network conditions. Retry the request // a limited number of times so that you don't exceed the request rate. If the -// error persists, the non-AWS identity provider might be down or not responding. +// error persists, the identity provider might be down or not responding. // // * ErrCodeInvalidIdentityTokenException "InvalidIdentityToken" // The web identity token that was passed could not be validated by AWS. Get @@ -572,7 +694,7 @@ func (c *STS) AssumeRoleWithWebIdentityRequest(input *AssumeRoleWithWebIdentityI // STS is not activated in the requested region for the account that is being // asked to generate credentials. The account administrator must use the IAM // console to activate STS in that region. For more information, see Activating -// and Deactivating AWS STS in an AWS Region (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html) +// and Deactivating AWS STS in an AWS Region (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html) // in the IAM User Guide. // // See also, https://docs.aws.amazon.com/goto/WebAPI/sts-2011-06-15/AssumeRoleWithWebIdentity @@ -644,17 +766,17 @@ func (c *STS) DecodeAuthorizationMessageRequest(input *DecodeAuthorizationMessag // Decodes additional information about the authorization status of a request // from an encoded message returned in response to an AWS request. // -// For example, if a user is not authorized to perform an action that he or -// she has requested, the request returns a Client.UnauthorizedOperation response -// (an HTTP 403 response). Some AWS actions additionally return an encoded message -// that can provide details about this authorization failure. +// For example, if a user is not authorized to perform an operation that he +// or she has requested, the request returns a Client.UnauthorizedOperation +// response (an HTTP 403 response). Some AWS operations additionally return +// an encoded message that can provide details about this authorization failure. // -// Only certain AWS actions return an encoded authorization message. The documentation -// for an individual action indicates whether that action returns an encoded -// message in addition to returning an HTTP code. +// Only certain AWS operations return an encoded authorization message. The +// documentation for an individual operation indicates whether that operation +// returns an encoded message in addition to returning an HTTP code. // // The message is encoded because the details of the authorization status can -// constitute privileged information that the user who requested the action +// constitute privileged information that the user who requested the operation // should not see. To decode an authorization status message, a user must be // granted permissions via an IAM policy to request the DecodeAuthorizationMessage // (sts:DecodeAuthorizationMessage) action. @@ -663,7 +785,7 @@ func (c *STS) DecodeAuthorizationMessageRequest(input *DecodeAuthorizationMessag // // * Whether the request was denied due to an explicit deny or due to the // absence of an explicit allow. For more information, see Determining Whether -// a Request is Allowed or Denied (http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_evaluation-logic.html#policy-eval-denyallow) +// a Request is Allowed or Denied (https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_evaluation-logic.html#policy-eval-denyallow) // in the IAM User Guide. // // * The principal who made the request. @@ -709,6 +831,103 @@ func (c *STS) DecodeAuthorizationMessageWithContext(ctx aws.Context, input *Deco return out, req.Send() } +const opGetAccessKeyInfo = "GetAccessKeyInfo" + +// GetAccessKeyInfoRequest generates a "aws/request.Request" representing the +// client's request for the GetAccessKeyInfo operation. The "output" return +// value will be populated with the request's response once the request completes +// successfully. +// +// Use "Send" method on the returned Request to send the API call to the service. +// the "output" return value is not valid until after Send returns without error. +// +// See GetAccessKeyInfo for more information on using the GetAccessKeyInfo +// API call, and error handling. +// +// This method is useful when you want to inject custom logic or configuration +// into the SDK's request lifecycle. Such as custom headers, or retry logic. +// +// +// // Example sending a request using the GetAccessKeyInfoRequest method. +// req, resp := client.GetAccessKeyInfoRequest(params) +// +// err := req.Send() +// if err == nil { // resp is now filled +// fmt.Println(resp) +// } +// +// See also, https://docs.aws.amazon.com/goto/WebAPI/sts-2011-06-15/GetAccessKeyInfo +func (c *STS) GetAccessKeyInfoRequest(input *GetAccessKeyInfoInput) (req *request.Request, output *GetAccessKeyInfoOutput) { + op := &request.Operation{ + Name: opGetAccessKeyInfo, + HTTPMethod: "POST", + HTTPPath: "/", + } + + if input == nil { + input = &GetAccessKeyInfoInput{} + } + + output = &GetAccessKeyInfoOutput{} + req = c.newRequest(op, input, output) + return +} + +// GetAccessKeyInfo API operation for AWS Security Token Service. +// +// Returns the account identifier for the specified access key ID. +// +// Access keys consist of two parts: an access key ID (for example, AKIAIOSFODNN7EXAMPLE) +// and a secret access key (for example, wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY). +// For more information about access keys, see Managing Access Keys for IAM +// Users (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html) +// in the IAM User Guide. +// +// When you pass an access key ID to this operation, it returns the ID of the +// AWS account to which the keys belong. Access key IDs beginning with AKIA +// are long-term credentials for an IAM user or the AWS account root user. Access +// key IDs beginning with ASIA are temporary credentials that are created using +// STS operations. If the account in the response belongs to you, you can sign +// in as the root user and review your root user access keys. Then, you can +// pull a credentials report (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_getting-report.html) +// to learn which IAM user owns the keys. To learn who requested the temporary +// credentials for an ASIA access key, view the STS events in your CloudTrail +// logs (https://docs.aws.amazon.com/IAM/latest/UserGuide/cloudtrail-integration.html) +// in the IAM User Guide. +// +// This operation does not indicate the state of the access key. The key might +// be active, inactive, or deleted. Active keys might not have permissions to +// perform an operation. Providing a deleted access key might return an error +// that the key doesn't exist. +// +// Returns awserr.Error for service API and SDK errors. Use runtime type assertions +// with awserr.Error's Code and Message methods to get detailed information about +// the error. +// +// See the AWS API reference guide for AWS Security Token Service's +// API operation GetAccessKeyInfo for usage and error information. +// See also, https://docs.aws.amazon.com/goto/WebAPI/sts-2011-06-15/GetAccessKeyInfo +func (c *STS) GetAccessKeyInfo(input *GetAccessKeyInfoInput) (*GetAccessKeyInfoOutput, error) { + req, out := c.GetAccessKeyInfoRequest(input) + return out, req.Send() +} + +// GetAccessKeyInfoWithContext is the same as GetAccessKeyInfo with the addition of +// the ability to pass a context and additional request options. +// +// See GetAccessKeyInfo for details on how to use this API operation. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func (c *STS) GetAccessKeyInfoWithContext(ctx aws.Context, input *GetAccessKeyInfoInput, opts ...request.Option) (*GetAccessKeyInfoOutput, error) { + req, out := c.GetAccessKeyInfoRequest(input) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return out, req.Send() +} + const opGetCallerIdentity = "GetCallerIdentity" // GetCallerIdentityRequest generates a "aws/request.Request" representing the @@ -753,8 +972,16 @@ func (c *STS) GetCallerIdentityRequest(input *GetCallerIdentityInput) (req *requ // GetCallerIdentity API operation for AWS Security Token Service. // -// Returns details about the IAM identity whose credentials are used to call -// the API. +// Returns details about the IAM user or role whose credentials are used to +// call the operation. +// +// No permissions are required to perform this operation. If an administrator +// adds a policy to your IAM user or role that explicitly denies access to the +// sts:GetCallerIdentity action, you can still perform this operation. Permissions +// are not required because the same information is returned when an IAM user +// or role is denied access. To view an example response, see I Am Not Authorized +// to Perform: iam:DeleteVirtualMFADevice (https://docs.aws.amazon.com/IAM/latest/UserGuide/troubleshoot_general.html#troubleshoot_general_access-denied-delete-mfa) +// in the IAM User Guide. // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -831,81 +1058,92 @@ func (c *STS) GetFederationTokenRequest(input *GetFederationTokenInput) (req *re // Returns a set of temporary security credentials (consisting of an access // key ID, a secret access key, and a security token) for a federated user. // A typical use is in a proxy application that gets temporary security credentials -// on behalf of distributed applications inside a corporate network. Because -// you must call the GetFederationToken action using the long-term security -// credentials of an IAM user, this call is appropriate in contexts where those +// on behalf of distributed applications inside a corporate network. You must +// call the GetFederationToken operation using the long-term security credentials +// of an IAM user. As a result, this call is appropriate in contexts where those // credentials can be safely stored, usually in a server-based application. -// For a comparison of GetFederationToken with the other APIs that produce temporary -// credentials, see Requesting Temporary Security Credentials (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html) -// and Comparing the AWS STS APIs (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison) +// For a comparison of GetFederationToken with the other API operations that +// produce temporary credentials, see Requesting Temporary Security Credentials +// (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html) +// and Comparing the AWS STS API operations (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison) // in the IAM User Guide. // -// If you are creating a mobile-based or browser-based app that can authenticate +// You can create a mobile-based or browser-based app that can authenticate // users using a web identity provider like Login with Amazon, Facebook, Google, -// or an OpenID Connect-compatible identity provider, we recommend that you -// use Amazon Cognito (http://aws.amazon.com/cognito/) or AssumeRoleWithWebIdentity. +// or an OpenID Connect-compatible identity provider. In this case, we recommend +// that you use Amazon Cognito (http://aws.amazon.com/cognito/) or AssumeRoleWithWebIdentity. // For more information, see Federation Through a Web-based Identity Provider -// (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#api_assumerolewithwebidentity). -// -// The GetFederationToken action must be called by using the long-term AWS security -// credentials of an IAM user. You can also call GetFederationToken using the -// security credentials of an AWS root account, but we do not recommended it. -// Instead, we recommend that you create an IAM user for the purpose of the -// proxy application and then attach a policy to the IAM user that limits federated -// users to only the actions and resources that they need access to. For more -// information, see IAM Best Practices (http://docs.aws.amazon.com/IAM/latest/UserGuide/best-practices.html) +// (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#api_assumerolewithwebidentity) // in the IAM User Guide. // -// The temporary security credentials that are obtained by using the long-term -// credentials of an IAM user are valid for the specified duration, from 900 -// seconds (15 minutes) up to a maximium of 129600 seconds (36 hours). The default -// is 43200 seconds (12 hours). Temporary credentials that are obtained by using -// AWS root account credentials have a maximum duration of 3600 seconds (1 hour). -// -// The temporary security credentials created by GetFederationToken can be used -// to make API calls to any AWS service with the following exceptions: +// You can also call GetFederationToken using the security credentials of an +// AWS account root user, but we do not recommend it. Instead, we recommend +// that you create an IAM user for the purpose of the proxy application. Then +// attach a policy to the IAM user that limits federated users to only the actions +// and resources that they need to access. For more information, see IAM Best +// Practices (https://docs.aws.amazon.com/IAM/latest/UserGuide/best-practices.html) +// in the IAM User Guide. // -// * You cannot use these credentials to call any IAM APIs. +// Session duration // -// * You cannot call any STS APIs except GetCallerIdentity. +// The temporary credentials are valid for the specified duration, from 900 +// seconds (15 minutes) up to a maximum of 129,600 seconds (36 hours). The default +// session duration is 43,200 seconds (12 hours). Temporary credentials that +// are obtained by using AWS account root user credentials have a maximum duration +// of 3,600 seconds (1 hour). // // Permissions // -// The permissions for the temporary security credentials returned by GetFederationToken -// are determined by a combination of the following: -// -// * The policy or policies that are attached to the IAM user whose credentials -// are used to call GetFederationToken. -// -// * The policy that is passed as a parameter in the call. -// -// The passed policy is attached to the temporary security credentials that -// result from the GetFederationToken API call--that is, to the federated user. -// When the federated user makes an AWS request, AWS evaluates the policy attached -// to the federated user in combination with the policy or policies attached -// to the IAM user whose credentials were used to call GetFederationToken. AWS -// allows the federated user's request only when both the federated user and -// the IAM user are explicitly allowed to perform the requested action. The -// passed policy cannot grant more permissions than those that are defined in -// the IAM user policy. -// -// A typical use case is that the permissions of the IAM user whose credentials -// are used to call GetFederationToken are designed to allow access to all the -// actions and resources that any federated user will need. Then, for individual -// users, you pass a policy to the operation that scopes down the permissions -// to a level that's appropriate to that individual user, using a policy that -// allows only a subset of permissions that are granted to the IAM user. -// -// If you do not pass a policy, the resulting temporary security credentials -// have no effective permissions. The only exception is when the temporary security -// credentials are used to access a resource that has a resource-based policy -// that specifically allows the federated user to access the resource. -// -// For more information about how permissions work, see Permissions for GetFederationToken -// (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_control-access_getfederationtoken.html). -// For information about using GetFederationToken to create temporary security -// credentials, see GetFederationToken—Federation Through a Custom Identity -// Broker (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#api_getfederationtoken). +// You can use the temporary credentials created by GetFederationToken in any +// AWS service except the following: +// +// * You cannot call any IAM operations using the AWS CLI or the AWS API. +// +// * You cannot call any STS operations except GetCallerIdentity. +// +// You must pass an inline or managed session policy (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session) +// to this operation. You can pass a single JSON policy document to use as an +// inline session policy. You can also specify up to 10 managed policies to +// use as managed session policies. The plain text that you use for both inline +// and managed session policies can't exceed 2,048 characters. +// +// Though the session policy parameters are optional, if you do not pass a policy, +// then the resulting federated user session has no permissions. When you pass +// session policies, the session permissions are the intersection of the IAM +// user policies and the session policies that you pass. This gives you a way +// to further restrict the permissions for a federated user. You cannot use +// session policies to grant more permissions than those that are defined in +// the permissions policy of the IAM user. For more information, see Session +// Policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session) +// in the IAM User Guide. For information about using GetFederationToken to +// create temporary security credentials, see GetFederationToken—Federation +// Through a Custom Identity Broker (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#api_getfederationtoken). +// +// You can use the credentials to access a resource that has a resource-based +// policy. If that policy specifically references the federated user session +// in the Principal element of the policy, the session has the permissions allowed +// by the policy. These permissions are granted in addition to the permissions +// granted by the session policies. +// +// Tags +// +// (Optional) You can pass tag key-value pairs to your session. These are called +// session tags. For more information about session tags, see Passing Session +// Tags in STS (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_session-tags.html) +// in the IAM User Guide. +// +// An administrator must grant you the permissions necessary to pass session +// tags. The administrator can also create granular permissions to allow you +// to pass only specific session tags. For more information, see Tutorial: Using +// Tags for Attribute-Based Access Control (https://docs.aws.amazon.com/IAM/latest/UserGuide/tutorial_attribute-based-access-control.html) +// in the IAM User Guide. +// +// Tag key–value pairs are not case sensitive, but case is preserved. This +// means that you cannot have separate Department and department tag keys. Assume +// that the user that you are federating has the Department=Marketing tag and +// you pass the department=engineering session tag. Department and department +// are not saved as separate tags, and the session tag passed in the request +// takes precedence over the user tag. // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -920,15 +1158,24 @@ func (c *STS) GetFederationTokenRequest(input *GetFederationTokenInput) (req *re // message describes the specific error. // // * ErrCodePackedPolicyTooLargeException "PackedPolicyTooLarge" -// The request was rejected because the policy document was too large. The error -// message describes how big the policy document is, in packed form, as a percentage -// of what the API allows. +// The request was rejected because the total packed size of the session policies +// and session tags combined was too large. An AWS conversion compresses the +// session policy document, session policy ARNs, and session tags into a packed +// binary format that has a separate limit. The error message indicates by percentage +// how close the policies and tags are to the upper size limit. For more information, +// see Passing Session Tags in STS (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_session-tags.html) +// in the IAM User Guide. +// +// You could receive this error even though you meet other defined session policy +// and session tag limits. For more information, see IAM and STS Entity Character +// Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html) +// in the IAM User Guide. // // * ErrCodeRegionDisabledException "RegionDisabledException" // STS is not activated in the requested region for the account that is being // asked to generate credentials. The account administrator must use the IAM // console to activate STS in that region. For more information, see Activating -// and Deactivating AWS STS in an AWS Region (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html) +// and Deactivating AWS STS in an AWS Region (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html) // in the IAM User Guide. // // See also, https://docs.aws.amazon.com/goto/WebAPI/sts-2011-06-15/GetFederationToken @@ -1000,48 +1247,51 @@ func (c *STS) GetSessionTokenRequest(input *GetSessionTokenInput) (req *request. // Returns a set of temporary credentials for an AWS account or IAM user. The // credentials consist of an access key ID, a secret access key, and a security // token. Typically, you use GetSessionToken if you want to use MFA to protect -// programmatic calls to specific AWS APIs like Amazon EC2 StopInstances. MFA-enabled -// IAM users would need to call GetSessionToken and submit an MFA code that -// is associated with their MFA device. Using the temporary security credentials -// that are returned from the call, IAM users can then make programmatic calls -// to APIs that require MFA authentication. If you do not supply a correct MFA -// code, then the API returns an access denied error. For a comparison of GetSessionToken -// with the other APIs that produce temporary credentials, see Requesting Temporary -// Security Credentials (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html) -// and Comparing the AWS STS APIs (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison) +// programmatic calls to specific AWS API operations like Amazon EC2 StopInstances. +// MFA-enabled IAM users would need to call GetSessionToken and submit an MFA +// code that is associated with their MFA device. Using the temporary security +// credentials that are returned from the call, IAM users can then make programmatic +// calls to API operations that require MFA authentication. If you do not supply +// a correct MFA code, then the API returns an access denied error. For a comparison +// of GetSessionToken with the other API operations that produce temporary credentials, +// see Requesting Temporary Security Credentials (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html) +// and Comparing the AWS STS API operations (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison) // in the IAM User Guide. // -// The GetSessionToken action must be called by using the long-term AWS security -// credentials of the AWS account or an IAM user. Credentials that are created -// by IAM users are valid for the duration that you specify, from 900 seconds -// (15 minutes) up to a maximum of 129600 seconds (36 hours), with a default -// of 43200 seconds (12 hours); credentials that are created by using account -// credentials can range from 900 seconds (15 minutes) up to a maximum of 3600 -// seconds (1 hour), with a default of 1 hour. +// Session Duration +// +// The GetSessionToken operation must be called by using the long-term AWS security +// credentials of the AWS account root user or an IAM user. Credentials that +// are created by IAM users are valid for the duration that you specify. This +// duration can range from 900 seconds (15 minutes) up to a maximum of 129,600 +// seconds (36 hours), with a default of 43,200 seconds (12 hours). Credentials +// based on account credentials can range from 900 seconds (15 minutes) up to +// 3,600 seconds (1 hour), with a default of 1 hour. +// +// Permissions // // The temporary security credentials created by GetSessionToken can be used // to make API calls to any AWS service with the following exceptions: // -// * You cannot call any IAM APIs unless MFA authentication information is -// included in the request. +// * You cannot call any IAM API operations unless MFA authentication information +// is included in the request. // -// * You cannot call any STS API exceptAssumeRole or GetCallerIdentity. +// * You cannot call any STS API except AssumeRole or GetCallerIdentity. // -// We recommend that you do not call GetSessionToken with root account credentials. -// Instead, follow our best practices (http://docs.aws.amazon.com/IAM/latest/UserGuide/best-practices.html#create-iam-users) +// We recommend that you do not call GetSessionToken with AWS account root user +// credentials. Instead, follow our best practices (https://docs.aws.amazon.com/IAM/latest/UserGuide/best-practices.html#create-iam-users) // by creating one or more IAM users, giving them the necessary permissions, // and using IAM users for everyday interaction with AWS. // -// The permissions associated with the temporary security credentials returned -// by GetSessionToken are based on the permissions associated with account or -// IAM user whose credentials are used to call the action. If GetSessionToken -// is called using root account credentials, the temporary credentials have -// root account permissions. Similarly, if GetSessionToken is called using the -// credentials of an IAM user, the temporary credentials have the same permissions -// as the IAM user. +// The credentials that are returned by GetSessionToken are based on permissions +// associated with the user whose credentials were used to call the operation. +// If GetSessionToken is called using AWS account root user credentials, the +// temporary credentials have root user permissions. Similarly, if GetSessionToken +// is called using the credentials of an IAM user, the temporary credentials +// have the same permissions as the IAM user. // // For more information about using GetSessionToken to create temporary credentials, -// go to Temporary Credentials for Users in Untrusted Environments (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#api_getsessiontoken) +// go to Temporary Credentials for Users in Untrusted Environments (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#api_getsessiontoken) // in the IAM User Guide. // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions @@ -1056,7 +1306,7 @@ func (c *STS) GetSessionTokenRequest(input *GetSessionTokenInput) (req *request. // STS is not activated in the requested region for the account that is being // asked to generate credentials. The account administrator must use the IAM // console to activate STS in that region. For more information, see Activating -// and Deactivating AWS STS in an AWS Region (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html) +// and Deactivating AWS STS in an AWS Region (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html) // in the IAM User Guide. // // See also, https://docs.aws.amazon.com/goto/WebAPI/sts-2011-06-15/GetSessionToken @@ -1091,7 +1341,7 @@ type AssumeRoleInput struct { // a session duration of 12 hours, but your administrator set the maximum session // duration to 6 hours, your operation fails. To learn how to view the maximum // value for your role, see View the Maximum Session Duration Setting for a - // Role (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html#id_roles_use_view-role-max-session) + // Role (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html#id_roles_use_view-role-max-session) // in the IAM User Guide. // // By default, the value is set to 3600 seconds. @@ -1101,51 +1351,77 @@ type AssumeRoleInput struct { // to the federation endpoint for a console sign-in token takes a SessionDuration // parameter that specifies the maximum length of the console session. For more // information, see Creating a URL that Enables Federated Users to Access the - // AWS Management Console (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_enable-console-custom-url.html) + // AWS Management Console (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_enable-console-custom-url.html) // in the IAM User Guide. DurationSeconds *int64 `min:"900" type:"integer"` - // A unique identifier that is used by third parties when assuming roles in - // their customers' accounts. For each role that the third party can assume, - // they should instruct their customers to ensure the role's trust policy checks - // for the external ID that the third party generated. Each time the third party - // assumes the role, they should pass the customer's external ID. The external - // ID is useful in order to help third parties bind a role to the customer who - // created it. For more information about the external ID, see How to Use an - // External ID When Granting Access to Your AWS Resources to a Third Party (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-user_externalid.html) + // A unique identifier that might be required when you assume a role in another + // account. If the administrator of the account to which the role belongs provided + // you with an external ID, then provide that value in the ExternalId parameter. + // This value can be any string, such as a passphrase or account number. A cross-account + // role is usually set up to trust everyone in an account. Therefore, the administrator + // of the trusting account might send an external ID to the administrator of + // the trusted account. That way, only someone with the ID can assume the role, + // rather than everyone in the account. For more information about the external + // ID, see How to Use an External ID When Granting Access to Your AWS Resources + // to a Third Party (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-user_externalid.html) // in the IAM User Guide. // - // The regex used to validated this parameter is a string of characters consisting + // The regex used to validate this parameter is a string of characters consisting // of upper- and lower-case alphanumeric characters with no spaces. You can // also include underscores or any of the following characters: =,.@:/- ExternalId *string `min:"2" type:"string"` - // An IAM policy in JSON format. - // - // This parameter is optional. If you pass a policy, the temporary security - // credentials that are returned by the operation have the permissions that - // are allowed by both (the intersection of) the access policy of the role that - // is being assumed, and the policy that you pass. This gives you a way to further - // restrict the permissions for the resulting temporary security credentials. - // You cannot use the passed policy to grant permissions that are in excess - // of those allowed by the access policy of the role that is being assumed. - // For more information, see Permissions for AssumeRole, AssumeRoleWithSAML, - // and AssumeRoleWithWebIdentity (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_control-access_assumerole.html) + // An IAM policy in JSON format that you want to use as an inline session policy. + // + // This parameter is optional. Passing policies to this operation returns new + // temporary credentials. The resulting session's permissions are the intersection + // of the role's identity-based policy and the session policies. You can use + // the role's temporary credentials in subsequent AWS API calls to access resources + // in the account that owns the role. You cannot use session policies to grant + // more permissions than those allowed by the identity-based policy of the role + // that is being assumed. For more information, see Session Policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session) // in the IAM User Guide. // - // The format for this parameter, as described by its regex pattern, is a string - // of characters up to 2048 characters in length. The characters can be any - // ASCII character from the space character to the end of the valid character - // list (\u0020-\u00FF). It can also include the tab (\u0009), linefeed (\u000A), + // The plain text that you use for both inline and managed session policies + // can't exceed 2,048 characters. The JSON policy characters can be any ASCII + // character from the space character to the end of the valid character list + // (\u0020 through \u00FF). It can also include the tab (\u0009), linefeed (\u000A), // and carriage return (\u000D) characters. // - // The policy plain text must be 2048 bytes or shorter. However, an internal - // conversion compresses it into a packed binary format with a separate limit. - // The PackedPolicySize response element indicates by percentage how close to - // the upper size limit the policy is, with 100% equaling the maximum allowed - // size. + // An AWS conversion compresses the passed session policies and session tags + // into a packed binary format that has a separate limit. Your request can fail + // for this limit even if your plain text meets the other requirements. The + // PackedPolicySize response element indicates by percentage how close the policies + // and tags for your request are to the upper size limit. Policy *string `min:"1" type:"string"` + // The Amazon Resource Names (ARNs) of the IAM managed policies that you want + // to use as managed session policies. The policies must exist in the same account + // as the role. + // + // This parameter is optional. You can provide up to 10 managed policy ARNs. + // However, the plain text that you use for both inline and managed session + // policies can't exceed 2,048 characters. For more information about ARNs, + // see Amazon Resource Names (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html) + // in the AWS General Reference. + // + // An AWS conversion compresses the passed session policies and session tags + // into a packed binary format that has a separate limit. Your request can fail + // for this limit even if your plain text meets the other requirements. The + // PackedPolicySize response element indicates by percentage how close the policies + // and tags for your request are to the upper size limit. + // + // Passing policies to this operation returns new temporary credentials. The + // resulting session's permissions are the intersection of the role's identity-based + // policy and the session policies. You can use the role's temporary credentials + // in subsequent AWS API calls to access resources in the account that owns + // the role. You cannot use session policies to grant more permissions than + // those allowed by the identity-based policy of the role that is being assumed. + // For more information, see Session Policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session) + // in the IAM User Guide. + PolicyArns []*PolicyDescriptorType `type:"list"` + // The Amazon Resource Name (ARN) of the role to assume. // // RoleArn is a required field @@ -1158,8 +1434,8 @@ type AssumeRoleInput struct { // scenarios, the role session name is visible to, and can be logged by the // account that owns the role. The role session name is also used in the ARN // of the assumed role principal. This means that subsequent cross-account API - // requests using the temporary security credentials will expose the role session - // name to the external account in their CloudTrail logs. + // requests that use the temporary security credentials will expose the role + // session name to the external account in their AWS CloudTrail logs. // // The regex used to validate this parameter is a string of characters consisting // of upper- and lower-case alphanumeric characters with no spaces. You can @@ -1179,6 +1455,41 @@ type AssumeRoleInput struct { // also include underscores or any of the following characters: =,.@- SerialNumber *string `min:"9" type:"string"` + // A list of session tags that you want to pass. Each session tag consists of + // a key name and an associated value. For more information about session tags, + // see Tagging AWS STS Sessions (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_session-tags.html) + // in the IAM User Guide. + // + // This parameter is optional. You can pass up to 50 session tags. The plain + // text session tag keys can’t exceed 128 characters, and the values can’t + // exceed 256 characters. For these and additional limits, see IAM and STS Character + // Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-limits.html#reference_iam-limits-entity-length) + // in the IAM User Guide. + // + // An AWS conversion compresses the passed session policies and session tags + // into a packed binary format that has a separate limit. Your request can fail + // for this limit even if your plain text meets the other requirements. The + // PackedPolicySize response element indicates by percentage how close the policies + // and tags for your request are to the upper size limit. + // + // You can pass a session tag with the same key as a tag that is already attached + // to the role. When you do, session tags override a role tag with the same + // key. + // + // Tag key–value pairs are not case sensitive, but case is preserved. This + // means that you cannot have separate Department and department tag keys. Assume + // that the role has the Department=Marketing tag and you pass the department=engineering + // session tag. Department and department are not saved as separate tags, and + // the session tag passed in the request takes precedence over the role tag. + // + // Additionally, if you used temporary credentials to perform this operation, + // the new session inherits any transitive session tags from the calling session. + // If you pass a session tag with the same key as an inherited tag, the operation + // fails. To view the inherited tags for a session, see the AWS CloudTrail logs. + // For more information, see Viewing Session Tags in CloudTrail (https://docs.aws.amazon.com/IAM/latest/UserGuide/session-tags.html#id_session-tags_ctlogs) + // in the IAM User Guide. + Tags []*Tag `type:"list"` + // The value provided by the MFA device, if the trust policy of the role being // assumed requires MFA (that is, if the policy includes a condition that tests // for MFA). If the role being assumed requires MFA and if the TokenCode value @@ -1187,6 +1498,19 @@ type AssumeRoleInput struct { // The format for this parameter, as described by its regex pattern, is a sequence // of six numeric digits. TokenCode *string `min:"6" type:"string"` + + // A list of keys for session tags that you want to set as transitive. If you + // set a tag key as transitive, the corresponding key and value passes to subsequent + // sessions in a role chain. For more information, see Chaining Roles with Session + // Tags (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_session-tags.html#id_session-tags_role-chaining) + // in the IAM User Guide. + // + // This parameter is optional. When you set session tags as transitive, the + // session policy and session tags packed binary limit is not affected. + // + // If you choose not to specify a transitive tag key, then no tags are passed + // from this session to any subsequent sessions. + TransitiveTagKeys []*string `type:"list"` } // String returns the string representation @@ -1229,6 +1553,26 @@ func (s *AssumeRoleInput) Validate() error { if s.TokenCode != nil && len(*s.TokenCode) < 6 { invalidParams.Add(request.NewErrParamMinLen("TokenCode", 6)) } + if s.PolicyArns != nil { + for i, v := range s.PolicyArns { + if v == nil { + continue + } + if err := v.Validate(); err != nil { + invalidParams.AddNested(fmt.Sprintf("%s[%v]", "PolicyArns", i), err.(request.ErrInvalidParams)) + } + } + } + if s.Tags != nil { + for i, v := range s.Tags { + if v == nil { + continue + } + if err := v.Validate(); err != nil { + invalidParams.AddNested(fmt.Sprintf("%s[%v]", "Tags", i), err.(request.ErrInvalidParams)) + } + } + } if invalidParams.Len() > 0 { return invalidParams @@ -1254,6 +1598,12 @@ func (s *AssumeRoleInput) SetPolicy(v string) *AssumeRoleInput { return s } +// SetPolicyArns sets the PolicyArns field's value. +func (s *AssumeRoleInput) SetPolicyArns(v []*PolicyDescriptorType) *AssumeRoleInput { + s.PolicyArns = v + return s +} + // SetRoleArn sets the RoleArn field's value. func (s *AssumeRoleInput) SetRoleArn(v string) *AssumeRoleInput { s.RoleArn = &v @@ -1272,12 +1622,24 @@ func (s *AssumeRoleInput) SetSerialNumber(v string) *AssumeRoleInput { return s } +// SetTags sets the Tags field's value. +func (s *AssumeRoleInput) SetTags(v []*Tag) *AssumeRoleInput { + s.Tags = v + return s +} + // SetTokenCode sets the TokenCode field's value. func (s *AssumeRoleInput) SetTokenCode(v string) *AssumeRoleInput { s.TokenCode = &v return s } +// SetTransitiveTagKeys sets the TransitiveTagKeys field's value. +func (s *AssumeRoleInput) SetTransitiveTagKeys(v []*string) *AssumeRoleInput { + s.TransitiveTagKeys = v + return s +} + // Contains the response to a successful AssumeRole request, including temporary // AWS credentials that can be used to make AWS requests. type AssumeRoleOutput struct { @@ -1293,15 +1655,14 @@ type AssumeRoleOutput struct { // The temporary security credentials, which include an access key ID, a secret // access key, and a security (or session) token. // - // Note: The size of the security token that STS APIs return is not fixed. We - // strongly recommend that you make no assumptions about the maximum size. As - // of this writing, the typical size is less than 4096 bytes, but that can vary. - // Also, future updates to AWS might require larger sizes. + // The size of the security token that STS API operations return is not fixed. + // We strongly recommend that you make no assumptions about the maximum size. Credentials *Credentials `type:"structure"` - // A percentage value that indicates the size of the policy in packed form. - // The service rejects any policy with a packed size greater than 100 percent, - // which means the policy exceeded the allowed space. + // A percentage value that indicates the packed size of the session policies + // and session tags combined passed in the request. The request fails if the + // packed size is greater than 100 percent, which means the policies and tags + // exceeded the allowed space. PackedPolicySize *int64 `type:"integer"` } @@ -1346,7 +1707,7 @@ type AssumeRoleWithSAMLInput struct { // specify a session duration of 12 hours, but your administrator set the maximum // session duration to 6 hours, your operation fails. To learn how to view the // maximum value for your role, see View the Maximum Session Duration Setting - // for a Role (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html#id_roles_use_view-role-max-session) + // for a Role (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html#id_roles_use_view-role-max-session) // in the IAM User Guide. // // By default, the value is set to 3600 seconds. @@ -1356,36 +1717,60 @@ type AssumeRoleWithSAMLInput struct { // to the federation endpoint for a console sign-in token takes a SessionDuration // parameter that specifies the maximum length of the console session. For more // information, see Creating a URL that Enables Federated Users to Access the - // AWS Management Console (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_enable-console-custom-url.html) + // AWS Management Console (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_enable-console-custom-url.html) // in the IAM User Guide. DurationSeconds *int64 `min:"900" type:"integer"` - // An IAM policy in JSON format. - // - // The policy parameter is optional. If you pass a policy, the temporary security - // credentials that are returned by the operation have the permissions that - // are allowed by both the access policy of the role that is being assumed, - // and the policy that you pass. This gives you a way to further restrict the - // permissions for the resulting temporary security credentials. You cannot - // use the passed policy to grant permissions that are in excess of those allowed - // by the access policy of the role that is being assumed. For more information, - // Permissions for AssumeRole, AssumeRoleWithSAML, and AssumeRoleWithWebIdentity - // (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_control-access_assumerole.html) + // An IAM policy in JSON format that you want to use as an inline session policy. + // + // This parameter is optional. Passing policies to this operation returns new + // temporary credentials. The resulting session's permissions are the intersection + // of the role's identity-based policy and the session policies. You can use + // the role's temporary credentials in subsequent AWS API calls to access resources + // in the account that owns the role. You cannot use session policies to grant + // more permissions than those allowed by the identity-based policy of the role + // that is being assumed. For more information, see Session Policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session) // in the IAM User Guide. // - // The format for this parameter, as described by its regex pattern, is a string - // of characters up to 2048 characters in length. The characters can be any - // ASCII character from the space character to the end of the valid character - // list (\u0020-\u00FF). It can also include the tab (\u0009), linefeed (\u000A), + // The plain text that you use for both inline and managed session policies + // can't exceed 2,048 characters. The JSON policy characters can be any ASCII + // character from the space character to the end of the valid character list + // (\u0020 through \u00FF). It can also include the tab (\u0009), linefeed (\u000A), // and carriage return (\u000D) characters. // - // The policy plain text must be 2048 bytes or shorter. However, an internal - // conversion compresses it into a packed binary format with a separate limit. - // The PackedPolicySize response element indicates by percentage how close to - // the upper size limit the policy is, with 100% equaling the maximum allowed - // size. + // An AWS conversion compresses the passed session policies and session tags + // into a packed binary format that has a separate limit. Your request can fail + // for this limit even if your plain text meets the other requirements. The + // PackedPolicySize response element indicates by percentage how close the policies + // and tags for your request are to the upper size limit. Policy *string `min:"1" type:"string"` + // The Amazon Resource Names (ARNs) of the IAM managed policies that you want + // to use as managed session policies. The policies must exist in the same account + // as the role. + // + // This parameter is optional. You can provide up to 10 managed policy ARNs. + // However, the plain text that you use for both inline and managed session + // policies can't exceed 2,048 characters. For more information about ARNs, + // see Amazon Resource Names (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html) + // in the AWS General Reference. + // + // An AWS conversion compresses the passed session policies and session tags + // into a packed binary format that has a separate limit. Your request can fail + // for this limit even if your plain text meets the other requirements. The + // PackedPolicySize response element indicates by percentage how close the policies + // and tags for your request are to the upper size limit. + // + // Passing policies to this operation returns new temporary credentials. The + // resulting session's permissions are the intersection of the role's identity-based + // policy and the session policies. You can use the role's temporary credentials + // in subsequent AWS API calls to access resources in the account that owns + // the role. You cannot use session policies to grant more permissions than + // those allowed by the identity-based policy of the role that is being assumed. + // For more information, see Session Policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session) + // in the IAM User Guide. + PolicyArns []*PolicyDescriptorType `type:"list"` + // The Amazon Resource Name (ARN) of the SAML provider in IAM that describes // the IdP. // @@ -1399,11 +1784,11 @@ type AssumeRoleWithSAMLInput struct { // The base-64 encoded SAML authentication response provided by the IdP. // - // For more information, see Configuring a Relying Party and Adding Claims (http://docs.aws.amazon.com/IAM/latest/UserGuide/create-role-saml-IdP-tasks.html) - // in the Using IAM guide. + // For more information, see Configuring a Relying Party and Adding Claims (https://docs.aws.amazon.com/IAM/latest/UserGuide/create-role-saml-IdP-tasks.html) + // in the IAM User Guide. // // SAMLAssertion is a required field - SAMLAssertion *string `min:"4" type:"string" required:"true"` + SAMLAssertion *string `min:"4" type:"string" required:"true" sensitive:"true"` } // String returns the string representation @@ -1443,6 +1828,16 @@ func (s *AssumeRoleWithSAMLInput) Validate() error { if s.SAMLAssertion != nil && len(*s.SAMLAssertion) < 4 { invalidParams.Add(request.NewErrParamMinLen("SAMLAssertion", 4)) } + if s.PolicyArns != nil { + for i, v := range s.PolicyArns { + if v == nil { + continue + } + if err := v.Validate(); err != nil { + invalidParams.AddNested(fmt.Sprintf("%s[%v]", "PolicyArns", i), err.(request.ErrInvalidParams)) + } + } + } if invalidParams.Len() > 0 { return invalidParams @@ -1462,6 +1857,12 @@ func (s *AssumeRoleWithSAMLInput) SetPolicy(v string) *AssumeRoleWithSAMLInput { return s } +// SetPolicyArns sets the PolicyArns field's value. +func (s *AssumeRoleWithSAMLInput) SetPolicyArns(v []*PolicyDescriptorType) *AssumeRoleWithSAMLInput { + s.PolicyArns = v + return s +} + // SetPrincipalArn sets the PrincipalArn field's value. func (s *AssumeRoleWithSAMLInput) SetPrincipalArn(v string) *AssumeRoleWithSAMLInput { s.PrincipalArn = &v @@ -1496,10 +1897,8 @@ type AssumeRoleWithSAMLOutput struct { // The temporary security credentials, which include an access key ID, a secret // access key, and a security (or session) token. // - // Note: The size of the security token that STS APIs return is not fixed. We - // strongly recommend that you make no assumptions about the maximum size. As - // of this writing, the typical size is less than 4096 bytes, but that can vary. - // Also, future updates to AWS might require larger sizes. + // The size of the security token that STS API operations return is not fixed. + // We strongly recommend that you make no assumptions about the maximum size. Credentials *Credentials `type:"structure"` // The value of the Issuer element of the SAML assertion. @@ -1516,9 +1915,10 @@ type AssumeRoleWithSAMLOutput struct { // ) ) NameQualifier *string `type:"string"` - // A percentage value that indicates the size of the policy in packed form. - // The service rejects any policy with a packed size greater than 100 percent, - // which means the policy exceeded the allowed space. + // A percentage value that indicates the packed size of the session policies + // and session tags combined passed in the request. The request fails if the + // packed size is greater than 100 percent, which means the policies and tags + // exceeded the allowed space. PackedPolicySize *int64 `type:"integer"` // The value of the NameID element in the Subject element of the SAML assertion. @@ -1603,7 +2003,7 @@ type AssumeRoleWithWebIdentityInput struct { // a session duration of 12 hours, but your administrator set the maximum session // duration to 6 hours, your operation fails. To learn how to view the maximum // value for your role, see View the Maximum Session Duration Setting for a - // Role (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html#id_roles_use_view-role-max-session) + // Role (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html#id_roles_use_view-role-max-session) // in the IAM User Guide. // // By default, the value is set to 3600 seconds. @@ -1613,35 +2013,60 @@ type AssumeRoleWithWebIdentityInput struct { // to the federation endpoint for a console sign-in token takes a SessionDuration // parameter that specifies the maximum length of the console session. For more // information, see Creating a URL that Enables Federated Users to Access the - // AWS Management Console (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_enable-console-custom-url.html) + // AWS Management Console (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_enable-console-custom-url.html) // in the IAM User Guide. DurationSeconds *int64 `min:"900" type:"integer"` - // An IAM policy in JSON format. + // An IAM policy in JSON format that you want to use as an inline session policy. // - // The policy parameter is optional. If you pass a policy, the temporary security - // credentials that are returned by the operation have the permissions that - // are allowed by both the access policy of the role that is being assumed, - // and the policy that you pass. This gives you a way to further restrict the - // permissions for the resulting temporary security credentials. You cannot - // use the passed policy to grant permissions that are in excess of those allowed - // by the access policy of the role that is being assumed. For more information, - // see Permissions for AssumeRoleWithWebIdentity (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_control-access_assumerole.html) + // This parameter is optional. Passing policies to this operation returns new + // temporary credentials. The resulting session's permissions are the intersection + // of the role's identity-based policy and the session policies. You can use + // the role's temporary credentials in subsequent AWS API calls to access resources + // in the account that owns the role. You cannot use session policies to grant + // more permissions than those allowed by the identity-based policy of the role + // that is being assumed. For more information, see Session Policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session) // in the IAM User Guide. // - // The format for this parameter, as described by its regex pattern, is a string - // of characters up to 2048 characters in length. The characters can be any - // ASCII character from the space character to the end of the valid character - // list (\u0020-\u00FF). It can also include the tab (\u0009), linefeed (\u000A), + // The plain text that you use for both inline and managed session policies + // can't exceed 2,048 characters. The JSON policy characters can be any ASCII + // character from the space character to the end of the valid character list + // (\u0020 through \u00FF). It can also include the tab (\u0009), linefeed (\u000A), // and carriage return (\u000D) characters. // - // The policy plain text must be 2048 bytes or shorter. However, an internal - // conversion compresses it into a packed binary format with a separate limit. - // The PackedPolicySize response element indicates by percentage how close to - // the upper size limit the policy is, with 100% equaling the maximum allowed - // size. + // An AWS conversion compresses the passed session policies and session tags + // into a packed binary format that has a separate limit. Your request can fail + // for this limit even if your plain text meets the other requirements. The + // PackedPolicySize response element indicates by percentage how close the policies + // and tags for your request are to the upper size limit. Policy *string `min:"1" type:"string"` + // The Amazon Resource Names (ARNs) of the IAM managed policies that you want + // to use as managed session policies. The policies must exist in the same account + // as the role. + // + // This parameter is optional. You can provide up to 10 managed policy ARNs. + // However, the plain text that you use for both inline and managed session + // policies can't exceed 2,048 characters. For more information about ARNs, + // see Amazon Resource Names (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html) + // in the AWS General Reference. + // + // An AWS conversion compresses the passed session policies and session tags + // into a packed binary format that has a separate limit. Your request can fail + // for this limit even if your plain text meets the other requirements. The + // PackedPolicySize response element indicates by percentage how close the policies + // and tags for your request are to the upper size limit. + // + // Passing policies to this operation returns new temporary credentials. The + // resulting session's permissions are the intersection of the role's identity-based + // policy and the session policies. You can use the role's temporary credentials + // in subsequent AWS API calls to access resources in the account that owns + // the role. You cannot use session policies to grant more permissions than + // those allowed by the identity-based policy of the role that is being assumed. + // For more information, see Session Policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session) + // in the IAM User Guide. + PolicyArns []*PolicyDescriptorType `type:"list"` + // The fully qualified host component of the domain name of the identity provider. // // Specify this value only for OAuth 2.0 access tokens. Currently www.amazon.com @@ -1675,7 +2100,7 @@ type AssumeRoleWithWebIdentityInput struct { // the application makes an AssumeRoleWithWebIdentity call. // // WebIdentityToken is a required field - WebIdentityToken *string `min:"4" type:"string" required:"true"` + WebIdentityToken *string `min:"4" type:"string" required:"true" sensitive:"true"` } // String returns the string representation @@ -1718,6 +2143,16 @@ func (s *AssumeRoleWithWebIdentityInput) Validate() error { if s.WebIdentityToken != nil && len(*s.WebIdentityToken) < 4 { invalidParams.Add(request.NewErrParamMinLen("WebIdentityToken", 4)) } + if s.PolicyArns != nil { + for i, v := range s.PolicyArns { + if v == nil { + continue + } + if err := v.Validate(); err != nil { + invalidParams.AddNested(fmt.Sprintf("%s[%v]", "PolicyArns", i), err.(request.ErrInvalidParams)) + } + } + } if invalidParams.Len() > 0 { return invalidParams @@ -1737,6 +2172,12 @@ func (s *AssumeRoleWithWebIdentityInput) SetPolicy(v string) *AssumeRoleWithWebI return s } +// SetPolicyArns sets the PolicyArns field's value. +func (s *AssumeRoleWithWebIdentityInput) SetPolicyArns(v []*PolicyDescriptorType) *AssumeRoleWithWebIdentityInput { + s.PolicyArns = v + return s +} + // SetProviderId sets the ProviderId field's value. func (s *AssumeRoleWithWebIdentityInput) SetProviderId(v string) *AssumeRoleWithWebIdentityInput { s.ProviderId = &v @@ -1781,19 +2222,18 @@ type AssumeRoleWithWebIdentityOutput struct { // The temporary security credentials, which include an access key ID, a secret // access key, and a security token. // - // Note: The size of the security token that STS APIs return is not fixed. We - // strongly recommend that you make no assumptions about the maximum size. As - // of this writing, the typical size is less than 4096 bytes, but that can vary. - // Also, future updates to AWS might require larger sizes. + // The size of the security token that STS API operations return is not fixed. + // We strongly recommend that you make no assumptions about the maximum size. Credentials *Credentials `type:"structure"` - // A percentage value that indicates the size of the policy in packed form. - // The service rejects any policy with a packed size greater than 100 percent, - // which means the policy exceeded the allowed space. + // A percentage value that indicates the packed size of the session policies + // and session tags combined passed in the request. The request fails if the + // packed size is greater than 100 percent, which means the policies and tags + // exceeded the allowed space. PackedPolicySize *int64 `type:"integer"` // The issuing authority of the web identity token presented. For OpenID Connect - // ID Tokens this contains the value of the iss field. For OAuth 2.0 access + // ID tokens, this contains the value of the iss field. For OAuth 2.0 access // tokens, this contains the value of the ProviderId parameter that was passed // in the AssumeRoleWithWebIdentity request. Provider *string `type:"string"` @@ -1860,8 +2300,8 @@ type AssumedRoleUser struct { // The ARN of the temporary security credentials that are returned from the // AssumeRole action. For more information about ARNs and how to use them in - // policies, see IAM Identifiers (http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html) - // in Using IAM. + // policies, see IAM Identifiers (https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html) + // in the IAM User Guide. // // Arn is a required field Arn *string `min:"20" type:"string" required:"true"` @@ -2028,8 +2468,8 @@ type FederatedUser struct { // The ARN that specifies the federated user that is associated with the credentials. // For more information about ARNs and how to use them in policies, see IAM - // Identifiers (http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html) - // in Using IAM. + // Identifiers (https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html) + // in the IAM User Guide. // // Arn is a required field Arn *string `min:"20" type:"string" required:"true"` @@ -2063,6 +2503,73 @@ func (s *FederatedUser) SetFederatedUserId(v string) *FederatedUser { return s } +type GetAccessKeyInfoInput struct { + _ struct{} `type:"structure"` + + // The identifier of an access key. + // + // This parameter allows (through its regex pattern) a string of characters + // that can consist of any upper- or lowercase letter or digit. + // + // AccessKeyId is a required field + AccessKeyId *string `min:"16" type:"string" required:"true"` +} + +// String returns the string representation +func (s GetAccessKeyInfoInput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s GetAccessKeyInfoInput) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *GetAccessKeyInfoInput) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "GetAccessKeyInfoInput"} + if s.AccessKeyId == nil { + invalidParams.Add(request.NewErrParamRequired("AccessKeyId")) + } + if s.AccessKeyId != nil && len(*s.AccessKeyId) < 16 { + invalidParams.Add(request.NewErrParamMinLen("AccessKeyId", 16)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetAccessKeyId sets the AccessKeyId field's value. +func (s *GetAccessKeyInfoInput) SetAccessKeyId(v string) *GetAccessKeyInfoInput { + s.AccessKeyId = &v + return s +} + +type GetAccessKeyInfoOutput struct { + _ struct{} `type:"structure"` + + // The number used to identify the AWS account. + Account *string `type:"string"` +} + +// String returns the string representation +func (s GetAccessKeyInfoOutput) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s GetAccessKeyInfoOutput) GoString() string { + return s.String() +} + +// SetAccount sets the Account field's value. +func (s *GetAccessKeyInfoOutput) SetAccount(v string) *GetAccessKeyInfoOutput { + s.Account = &v + return s +} + type GetCallerIdentityInput struct { _ struct{} `type:"structure"` } @@ -2090,8 +2597,8 @@ type GetCallerIdentityOutput struct { Arn *string `min:"20" type:"string"` // The unique identifier of the calling entity. The exact value depends on the - // type of entity making the call. The values returned are those listed in the - // aws:userid column in the Principal table (http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_variables.html#principaltable) + // type of entity that is making the call. The values returned are those listed + // in the aws:userid column in the Principal table (https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_variables.html#principaltable) // found on the Policy Variables reference page in the IAM User Guide. UserId *string `type:"string"` } @@ -2128,12 +2635,11 @@ type GetFederationTokenInput struct { _ struct{} `type:"structure"` // The duration, in seconds, that the session should last. Acceptable durations - // for federation sessions range from 900 seconds (15 minutes) to 129600 seconds - // (36 hours), with 43200 seconds (12 hours) as the default. Sessions obtained - // using AWS account (root) credentials are restricted to a maximum of 3600 + // for federation sessions range from 900 seconds (15 minutes) to 129,600 seconds + // (36 hours), with 43,200 seconds (12 hours) as the default. Sessions obtained + // using AWS account root user credentials are restricted to a maximum of 3,600 // seconds (one hour). If the specified duration is longer than one hour, the - // session obtained by using AWS account (root) credentials defaults to one - // hour. + // session obtained by using root user credentials defaults to one hour. DurationSeconds *int64 `min:"900" type:"integer"` // The name of the federated user. The name is used as an identifier for the @@ -2148,36 +2654,107 @@ type GetFederationTokenInput struct { // Name is a required field Name *string `min:"2" type:"string" required:"true"` - // An IAM policy in JSON format that is passed with the GetFederationToken call - // and evaluated along with the policy or policies that are attached to the - // IAM user whose credentials are used to call GetFederationToken. The passed - // policy is used to scope down the permissions that are available to the IAM - // user, by allowing only a subset of the permissions that are granted to the - // IAM user. The passed policy cannot grant more permissions than those granted - // to the IAM user. The final permissions for the federated user are the most - // restrictive set based on the intersection of the passed policy and the IAM - // user policy. - // - // If you do not pass a policy, the resulting temporary security credentials - // have no effective permissions. The only exception is when the temporary security - // credentials are used to access a resource that has a resource-based policy - // that specifically allows the federated user to access the resource. - // - // The format for this parameter, as described by its regex pattern, is a string - // of characters up to 2048 characters in length. The characters can be any - // ASCII character from the space character to the end of the valid character - // list (\u0020-\u00FF). It can also include the tab (\u0009), linefeed (\u000A), - // and carriage return (\u000D) characters. + // An IAM policy in JSON format that you want to use as an inline session policy. + // + // You must pass an inline or managed session policy (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session) + // to this operation. You can pass a single JSON policy document to use as an + // inline session policy. You can also specify up to 10 managed policies to + // use as managed session policies. + // + // This parameter is optional. However, if you do not pass any session policies, + // then the resulting federated user session has no permissions. + // + // When you pass session policies, the session permissions are the intersection + // of the IAM user policies and the session policies that you pass. This gives + // you a way to further restrict the permissions for a federated user. You cannot + // use session policies to grant more permissions than those that are defined + // in the permissions policy of the IAM user. For more information, see Session + // Policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session) + // in the IAM User Guide. + // + // The resulting credentials can be used to access a resource that has a resource-based + // policy. If that policy specifically references the federated user session + // in the Principal element of the policy, the session has the permissions allowed + // by the policy. These permissions are granted in addition to the permissions + // that are granted by the session policies. // - // The policy plain text must be 2048 bytes or shorter. However, an internal - // conversion compresses it into a packed binary format with a separate limit. - // The PackedPolicySize response element indicates by percentage how close to - // the upper size limit the policy is, with 100% equaling the maximum allowed - // size. + // The plain text that you use for both inline and managed session policies + // can't exceed 2,048 characters. The JSON policy characters can be any ASCII + // character from the space character to the end of the valid character list + // (\u0020 through \u00FF). It can also include the tab (\u0009), linefeed (\u000A), + // and carriage return (\u000D) characters. // - // For more information about how permissions work, see Permissions for GetFederationToken - // (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_control-access_getfederationtoken.html). + // An AWS conversion compresses the passed session policies and session tags + // into a packed binary format that has a separate limit. Your request can fail + // for this limit even if your plain text meets the other requirements. The + // PackedPolicySize response element indicates by percentage how close the policies + // and tags for your request are to the upper size limit. Policy *string `min:"1" type:"string"` + + // The Amazon Resource Names (ARNs) of the IAM managed policies that you want + // to use as a managed session policy. The policies must exist in the same account + // as the IAM user that is requesting federated access. + // + // You must pass an inline or managed session policy (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session) + // to this operation. You can pass a single JSON policy document to use as an + // inline session policy. You can also specify up to 10 managed policies to + // use as managed session policies. The plain text that you use for both inline + // and managed session policies can't exceed 2,048 characters. You can provide + // up to 10 managed policy ARNs. For more information about ARNs, see Amazon + // Resource Names (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html) + // in the AWS General Reference. + // + // This parameter is optional. However, if you do not pass any session policies, + // then the resulting federated user session has no permissions. + // + // When you pass session policies, the session permissions are the intersection + // of the IAM user policies and the session policies that you pass. This gives + // you a way to further restrict the permissions for a federated user. You cannot + // use session policies to grant more permissions than those that are defined + // in the permissions policy of the IAM user. For more information, see Session + // Policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session) + // in the IAM User Guide. + // + // The resulting credentials can be used to access a resource that has a resource-based + // policy. If that policy specifically references the federated user session + // in the Principal element of the policy, the session has the permissions allowed + // by the policy. These permissions are granted in addition to the permissions + // that are granted by the session policies. + // + // An AWS conversion compresses the passed session policies and session tags + // into a packed binary format that has a separate limit. Your request can fail + // for this limit even if your plain text meets the other requirements. The + // PackedPolicySize response element indicates by percentage how close the policies + // and tags for your request are to the upper size limit. + PolicyArns []*PolicyDescriptorType `type:"list"` + + // A list of session tags. Each session tag consists of a key name and an associated + // value. For more information about session tags, see Passing Session Tags + // in STS (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_session-tags.html) + // in the IAM User Guide. + // + // This parameter is optional. You can pass up to 50 session tags. The plain + // text session tag keys can’t exceed 128 characters and the values can’t + // exceed 256 characters. For these and additional limits, see IAM and STS Character + // Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-limits.html#reference_iam-limits-entity-length) + // in the IAM User Guide. + // + // An AWS conversion compresses the passed session policies and session tags + // into a packed binary format that has a separate limit. Your request can fail + // for this limit even if your plain text meets the other requirements. The + // PackedPolicySize response element indicates by percentage how close the policies + // and tags for your request are to the upper size limit. + // + // You can pass a session tag with the same key as a tag that is already attached + // to the user you are federating. When you do, session tags override a user + // tag with the same key. + // + // Tag key–value pairs are not case sensitive, but case is preserved. This + // means that you cannot have separate Department and department tag keys. Assume + // that the role has the Department=Marketing tag and you pass the department=engineering + // session tag. Department and department are not saved as separate tags, and + // the session tag passed in the request takes precedence over the role tag. + Tags []*Tag `type:"list"` } // String returns the string representation @@ -2205,6 +2782,26 @@ func (s *GetFederationTokenInput) Validate() error { if s.Policy != nil && len(*s.Policy) < 1 { invalidParams.Add(request.NewErrParamMinLen("Policy", 1)) } + if s.PolicyArns != nil { + for i, v := range s.PolicyArns { + if v == nil { + continue + } + if err := v.Validate(); err != nil { + invalidParams.AddNested(fmt.Sprintf("%s[%v]", "PolicyArns", i), err.(request.ErrInvalidParams)) + } + } + } + if s.Tags != nil { + for i, v := range s.Tags { + if v == nil { + continue + } + if err := v.Validate(); err != nil { + invalidParams.AddNested(fmt.Sprintf("%s[%v]", "Tags", i), err.(request.ErrInvalidParams)) + } + } + } if invalidParams.Len() > 0 { return invalidParams @@ -2230,6 +2827,18 @@ func (s *GetFederationTokenInput) SetPolicy(v string) *GetFederationTokenInput { return s } +// SetPolicyArns sets the PolicyArns field's value. +func (s *GetFederationTokenInput) SetPolicyArns(v []*PolicyDescriptorType) *GetFederationTokenInput { + s.PolicyArns = v + return s +} + +// SetTags sets the Tags field's value. +func (s *GetFederationTokenInput) SetTags(v []*Tag) *GetFederationTokenInput { + s.Tags = v + return s +} + // Contains the response to a successful GetFederationToken request, including // temporary AWS credentials that can be used to make AWS requests. type GetFederationTokenOutput struct { @@ -2238,10 +2847,8 @@ type GetFederationTokenOutput struct { // The temporary security credentials, which include an access key ID, a secret // access key, and a security (or session) token. // - // Note: The size of the security token that STS APIs return is not fixed. We - // strongly recommend that you make no assumptions about the maximum size. As - // of this writing, the typical size is less than 4096 bytes, but that can vary. - // Also, future updates to AWS might require larger sizes. + // The size of the security token that STS API operations return is not fixed. + // We strongly recommend that you make no assumptions about the maximum size. Credentials *Credentials `type:"structure"` // Identifiers for the federated user associated with the credentials (such @@ -2250,9 +2857,10 @@ type GetFederationTokenOutput struct { // an Amazon S3 bucket policy. FederatedUser *FederatedUser `type:"structure"` - // A percentage value indicating the size of the policy in packed form. The - // service rejects policies for which the packed size is greater than 100 percent - // of the allowed value. + // A percentage value that indicates the packed size of the session policies + // and session tags combined passed in the request. The request fails if the + // packed size is greater than 100 percent, which means the policies and tags + // exceeded the allowed space. PackedPolicySize *int64 `type:"integer"` } @@ -2288,11 +2896,11 @@ type GetSessionTokenInput struct { _ struct{} `type:"structure"` // The duration, in seconds, that the credentials should remain valid. Acceptable - // durations for IAM user sessions range from 900 seconds (15 minutes) to 129600 - // seconds (36 hours), with 43200 seconds (12 hours) as the default. Sessions - // for AWS account owners are restricted to a maximum of 3600 seconds (one hour). - // If the duration is longer than one hour, the session for AWS account owners - // defaults to one hour. + // durations for IAM user sessions range from 900 seconds (15 minutes) to 129,600 + // seconds (36 hours), with 43,200 seconds (12 hours) as the default. Sessions + // for AWS account owners are restricted to a maximum of 3,600 seconds (one + // hour). If the duration is longer than one hour, the session for AWS account + // owners defaults to one hour. DurationSeconds *int64 `min:"900" type:"integer"` // The identification number of the MFA device that is associated with the IAM @@ -2303,16 +2911,16 @@ type GetSessionTokenInput struct { // You can find the device for an IAM user by going to the AWS Management Console // and viewing the user's security credentials. // - // The regex used to validated this parameter is a string of characters consisting + // The regex used to validate this parameter is a string of characters consisting // of upper- and lower-case alphanumeric characters with no spaces. You can // also include underscores or any of the following characters: =,.@:/- SerialNumber *string `min:"9" type:"string"` // The value provided by the MFA device, if MFA is required. If any policy requires // the IAM user to submit an MFA code, specify this value. If MFA authentication - // is required, and the user does not provide a code when requesting a set of - // temporary security credentials, the user will receive an "access denied" - // response when requesting resources that require MFA authentication. + // is required, the user must provide a code when requesting a set of temporary + // security credentials. A user who fails to provide the code receives an "access + // denied" response when requesting resources that require MFA authentication. // // The format for this parameter, as described by its regex pattern, is a sequence // of six numeric digits. @@ -2374,10 +2982,8 @@ type GetSessionTokenOutput struct { // The temporary security credentials, which include an access key ID, a secret // access key, and a security (or session) token. // - // Note: The size of the security token that STS APIs return is not fixed. We - // strongly recommend that you make no assumptions about the maximum size. As - // of this writing, the typical size is less than 4096 bytes, but that can vary. - // Also, future updates to AWS might require larger sizes. + // The size of the security token that STS API operations return is not fixed. + // We strongly recommend that you make no assumptions about the maximum size. Credentials *Credentials `type:"structure"` } @@ -2396,3 +3002,114 @@ func (s *GetSessionTokenOutput) SetCredentials(v *Credentials) *GetSessionTokenO s.Credentials = v return s } + +// A reference to the IAM managed policy that is passed as a session policy +// for a role session or a federated user session. +type PolicyDescriptorType struct { + _ struct{} `type:"structure"` + + // The Amazon Resource Name (ARN) of the IAM managed policy to use as a session + // policy for the role. For more information about ARNs, see Amazon Resource + // Names (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html) + // in the AWS General Reference. + Arn *string `locationName:"arn" min:"20" type:"string"` +} + +// String returns the string representation +func (s PolicyDescriptorType) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s PolicyDescriptorType) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *PolicyDescriptorType) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "PolicyDescriptorType"} + if s.Arn != nil && len(*s.Arn) < 20 { + invalidParams.Add(request.NewErrParamMinLen("Arn", 20)) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetArn sets the Arn field's value. +func (s *PolicyDescriptorType) SetArn(v string) *PolicyDescriptorType { + s.Arn = &v + return s +} + +// You can pass custom key-value pair attributes when you assume a role or federate +// a user. These are called session tags. You can then use the session tags +// to control access to resources. For more information, see Tagging AWS STS +// Sessions (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_session-tags.html) +// in the IAM User Guide. +type Tag struct { + _ struct{} `type:"structure"` + + // The key for a session tag. + // + // You can pass up to 50 session tags. The plain text session tag keys can’t + // exceed 128 characters. For these and additional limits, see IAM and STS Character + // Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-limits.html#reference_iam-limits-entity-length) + // in the IAM User Guide. + // + // Key is a required field + Key *string `min:"1" type:"string" required:"true"` + + // The value for a session tag. + // + // You can pass up to 50 session tags. The plain text session tag values can’t + // exceed 256 characters. For these and additional limits, see IAM and STS Character + // Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_iam-limits.html#reference_iam-limits-entity-length) + // in the IAM User Guide. + // + // Value is a required field + Value *string `type:"string" required:"true"` +} + +// String returns the string representation +func (s Tag) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s Tag) GoString() string { + return s.String() +} + +// Validate inspects the fields of the type to determine if they are valid. +func (s *Tag) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "Tag"} + if s.Key == nil { + invalidParams.Add(request.NewErrParamRequired("Key")) + } + if s.Key != nil && len(*s.Key) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Key", 1)) + } + if s.Value == nil { + invalidParams.Add(request.NewErrParamRequired("Value")) + } + + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} + +// SetKey sets the Key field's value. +func (s *Tag) SetKey(v string) *Tag { + s.Key = &v + return s +} + +// SetValue sets the Value field's value. +func (s *Tag) SetValue(v string) *Tag { + s.Value = &v + return s +} diff --git a/github.com/aws/aws-sdk-go/service/sts/customizations.go b/github.com/aws/aws-sdk-go/service/sts/customizations.go index 4010cc7fa1..d5307fcaa0 100644 --- a/github.com/aws/aws-sdk-go/service/sts/customizations.go +++ b/github.com/aws/aws-sdk-go/service/sts/customizations.go @@ -3,10 +3,9 @@ package sts import "github.com/aws/aws-sdk-go/aws/request" func init() { - initRequest = func(r *request.Request) { - switch r.Operation.Name { - case opAssumeRoleWithSAML, opAssumeRoleWithWebIdentity: - r.Handlers.Sign.Clear() // these operations are unsigned - } - } + initRequest = customizeRequest +} + +func customizeRequest(r *request.Request) { + r.RetryErrorCodes = append(r.RetryErrorCodes, ErrCodeIDPCommunicationErrorException) } diff --git a/github.com/aws/aws-sdk-go/service/sts/doc.go b/github.com/aws/aws-sdk-go/service/sts/doc.go index ef681ab0c6..fcb720dcac 100644 --- a/github.com/aws/aws-sdk-go/service/sts/doc.go +++ b/github.com/aws/aws-sdk-go/service/sts/doc.go @@ -7,22 +7,14 @@ // request temporary, limited-privilege credentials for AWS Identity and Access // Management (IAM) users or for users that you authenticate (federated users). // This guide provides descriptions of the STS API. For more detailed information -// about using this service, go to Temporary Security Credentials (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html). -// -// As an alternative to using the API, you can use one of the AWS SDKs, which -// consist of libraries and sample code for various programming languages and -// platforms (Java, Ruby, .NET, iOS, Android, etc.). The SDKs provide a convenient -// way to create programmatic access to STS. For example, the SDKs take care -// of cryptographically signing requests, managing errors, and retrying requests -// automatically. For information about the AWS SDKs, including how to download -// and install them, see the Tools for Amazon Web Services page (http://aws.amazon.com/tools/). +// about using this service, go to Temporary Security Credentials (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html). // // For information about setting up signatures and authorization through the -// API, go to Signing AWS API Requests (http://docs.aws.amazon.com/general/latest/gr/signing_aws_api_requests.html) +// API, go to Signing AWS API Requests (https://docs.aws.amazon.com/general/latest/gr/signing_aws_api_requests.html) // in the AWS General Reference. For general information about the Query API, -// go to Making Query Requests (http://docs.aws.amazon.com/IAM/latest/UserGuide/IAM_UsingQueryAPI.html) +// go to Making Query Requests (https://docs.aws.amazon.com/IAM/latest/UserGuide/IAM_UsingQueryAPI.html) // in Using IAM. For information about using security tokens with other AWS -// products, go to AWS Services That Work with IAM (http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_aws-services-that-work-with-iam.html) +// products, go to AWS Services That Work with IAM (https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_aws-services-that-work-with-iam.html) // in the IAM User Guide. // // If you're new to AWS and need additional technical information about a specific @@ -31,14 +23,38 @@ // // Endpoints // -// The AWS Security Token Service (STS) has a default endpoint of https://sts.amazonaws.com -// that maps to the US East (N. Virginia) region. Additional regions are available -// and are activated by default. For more information, see Activating and Deactivating -// AWS STS in an AWS Region (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html) +// By default, AWS Security Token Service (STS) is available as a global service, +// and all AWS STS requests go to a single endpoint at https://sts.amazonaws.com. +// Global requests map to the US East (N. Virginia) region. AWS recommends using +// Regional AWS STS endpoints instead of the global endpoint to reduce latency, +// build in redundancy, and increase session token validity. For more information, +// see Managing AWS STS in an AWS Region (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html) +// in the IAM User Guide. +// +// Most AWS Regions are enabled for operations in all AWS services by default. +// Those Regions are automatically activated for use with AWS STS. Some Regions, +// such as Asia Pacific (Hong Kong), must be manually enabled. To learn more +// about enabling and disabling AWS Regions, see Managing AWS Regions (https://docs.aws.amazon.com/general/latest/gr/rande-manage.html) +// in the AWS General Reference. When you enable these AWS Regions, they are +// automatically activated for use with AWS STS. You cannot activate the STS +// endpoint for a Region that is disabled. Tokens that are valid in all AWS +// Regions are longer than tokens that are valid in Regions that are enabled +// by default. Changing this setting might affect existing systems where you +// temporarily store tokens. For more information, see Managing Global Endpoint +// Session Tokens (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html#sts-regions-manage-tokens) // in the IAM User Guide. // -// For information about STS endpoints, see Regions and Endpoints (http://docs.aws.amazon.com/general/latest/gr/rande.html#sts_region) -// in the AWS General Reference. +// After you activate a Region for use with AWS STS, you can direct AWS STS +// API calls to that Region. AWS STS recommends that you provide both the Region +// and endpoint when you make calls to a Regional endpoint. You can provide +// the Region alone for manually enabled Regions, such as Asia Pacific (Hong +// Kong). In this case, the calls are directed to the STS Regional endpoint. +// However, if you provide the Region alone for Regions enabled by default, +// the calls are directed to the global endpoint of https://sts.amazonaws.com. +// +// To view the list of AWS STS endpoints and whether they are active by default, +// see Writing Code to Use AWS STS Regions (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html#id_credentials_temp_enable-regions_writing_code) +// in the IAM User Guide. // // Recording API requests // @@ -46,8 +62,28 @@ // your AWS account and delivers log files to an Amazon S3 bucket. By using // information collected by CloudTrail, you can determine what requests were // successfully made to STS, who made the request, when it was made, and so -// on. To learn more about CloudTrail, including how to turn it on and find -// your log files, see the AWS CloudTrail User Guide (http://docs.aws.amazon.com/awscloudtrail/latest/userguide/what_is_cloud_trail_top_level.html). +// on. +// +// If you activate AWS STS endpoints in Regions other than the default global +// endpoint, then you must also turn on CloudTrail logging in those Regions. +// This is necessary to record any AWS STS API calls that are made in those +// Regions. For more information, see Turning On CloudTrail in Additional Regions +// (https://docs.aws.amazon.com/awscloudtrail/latest/userguide/aggregating_logs_regions_turn_on_ct.html) +// in the AWS CloudTrail User Guide. +// +// AWS Security Token Service (STS) is a global service with a single endpoint +// at https://sts.amazonaws.com. Calls to this endpoint are logged as calls +// to a global service. However, because this endpoint is physically located +// in the US East (N. Virginia) Region, your logs list us-east-1 as the event +// Region. CloudTrail does not write these logs to the US East (Ohio) Region +// unless you choose to include global service logs in that Region. CloudTrail +// writes calls to all Regional endpoints to their respective Regions. For example, +// calls to sts.us-east-2.amazonaws.com are published to the US East (Ohio) +// Region and calls to sts.eu-central-1.amazonaws.com are published to the EU +// (Frankfurt) Region. +// +// To learn more about CloudTrail, including how to turn it on and find your +// log files, see the AWS CloudTrail User Guide (https://docs.aws.amazon.com/awscloudtrail/latest/userguide/what_is_cloud_trail_top_level.html). // // See https://docs.aws.amazon.com/goto/WebAPI/sts-2011-06-15 for more information on this service. // diff --git a/github.com/aws/aws-sdk-go/service/sts/errors.go b/github.com/aws/aws-sdk-go/service/sts/errors.go index e24884ef37..a233f542ef 100644 --- a/github.com/aws/aws-sdk-go/service/sts/errors.go +++ b/github.com/aws/aws-sdk-go/service/sts/errors.go @@ -14,11 +14,11 @@ const ( // ErrCodeIDPCommunicationErrorException for service response error code // "IDPCommunicationError". // - // The request could not be fulfilled because the non-AWS identity provider - // (IDP) that was asked to verify the incoming identity token could not be reached. - // This is often a transient error caused by network conditions. Retry the request + // The request could not be fulfilled because the identity provider (IDP) that + // was asked to verify the incoming identity token could not be reached. This + // is often a transient error caused by network conditions. Retry the request // a limited number of times so that you don't exceed the request rate. If the - // error persists, the non-AWS identity provider might be down or not responding. + // error persists, the identity provider might be down or not responding. ErrCodeIDPCommunicationErrorException = "IDPCommunicationError" // ErrCodeIDPRejectedClaimException for service response error code @@ -56,9 +56,18 @@ const ( // ErrCodePackedPolicyTooLargeException for service response error code // "PackedPolicyTooLarge". // - // The request was rejected because the policy document was too large. The error - // message describes how big the policy document is, in packed form, as a percentage - // of what the API allows. + // The request was rejected because the total packed size of the session policies + // and session tags combined was too large. An AWS conversion compresses the + // session policy document, session policy ARNs, and session tags into a packed + // binary format that has a separate limit. The error message indicates by percentage + // how close the policies and tags are to the upper size limit. For more information, + // see Passing Session Tags in STS (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_session-tags.html) + // in the IAM User Guide. + // + // You could receive this error even though you meet other defined session policy + // and session tag limits. For more information, see IAM and STS Entity Character + // Limits (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html) + // in the IAM User Guide. ErrCodePackedPolicyTooLargeException = "PackedPolicyTooLarge" // ErrCodeRegionDisabledException for service response error code @@ -67,7 +76,7 @@ const ( // STS is not activated in the requested region for the account that is being // asked to generate credentials. The account administrator must use the IAM // console to activate STS in that region. For more information, see Activating - // and Deactivating AWS STS in an AWS Region (http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html) + // and Deactivating AWS STS in an AWS Region (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html) // in the IAM User Guide. ErrCodeRegionDisabledException = "RegionDisabledException" ) diff --git a/github.com/aws/aws-sdk-go/service/sts/service.go b/github.com/aws/aws-sdk-go/service/sts/service.go index 185c914d1b..d34a685533 100644 --- a/github.com/aws/aws-sdk-go/service/sts/service.go +++ b/github.com/aws/aws-sdk-go/service/sts/service.go @@ -31,7 +31,7 @@ var initRequest func(*request.Request) const ( ServiceName = "sts" // Name of service. EndpointsID = ServiceName // ID to lookup a service endpoint with. - ServiceID = "STS" // ServiceID is a unique identifer of a specific service. + ServiceID = "STS" // ServiceID is a unique identifier of a specific service. ) // New creates a new instance of the STS client with a session. @@ -39,6 +39,8 @@ const ( // aws.Config parameter to add your extra config. // // Example: +// mySession := session.Must(session.NewSession()) +// // // Create a STS client from just a session. // svc := sts.New(mySession) // @@ -46,11 +48,11 @@ const ( // svc := sts.New(mySession, aws.NewConfig().WithRegion("us-west-2")) func New(p client.ConfigProvider, cfgs ...*aws.Config) *STS { c := p.ClientConfig(EndpointsID, cfgs...) - return newClient(*c.Config, c.Handlers, c.Endpoint, c.SigningRegion, c.SigningName) + return newClient(*c.Config, c.Handlers, c.PartitionID, c.Endpoint, c.SigningRegion, c.SigningName) } // newClient creates, initializes and returns a new service client instance. -func newClient(cfg aws.Config, handlers request.Handlers, endpoint, signingRegion, signingName string) *STS { +func newClient(cfg aws.Config, handlers request.Handlers, partitionID, endpoint, signingRegion, signingName string) *STS { svc := &STS{ Client: client.New( cfg, @@ -59,6 +61,7 @@ func newClient(cfg aws.Config, handlers request.Handlers, endpoint, signingRegio ServiceID: ServiceID, SigningName: signingName, SigningRegion: signingRegion, + PartitionID: partitionID, Endpoint: endpoint, APIVersion: "2011-06-15", }, diff --git a/github.com/aws/aws-sdk-go/service/sts/stsiface/interface.go b/github.com/aws/aws-sdk-go/service/sts/stsiface/interface.go new file mode 100644 index 0000000000..e2e1d6efe5 --- /dev/null +++ b/github.com/aws/aws-sdk-go/service/sts/stsiface/interface.go @@ -0,0 +1,96 @@ +// Code generated by private/model/cli/gen-api/main.go. DO NOT EDIT. + +// Package stsiface provides an interface to enable mocking the AWS Security Token Service service client +// for testing your code. +// +// It is important to note that this interface will have breaking changes +// when the service model is updated and adds new API operations, paginators, +// and waiters. +package stsiface + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/sts" +) + +// STSAPI provides an interface to enable mocking the +// sts.STS service client's API operation, +// paginators, and waiters. This make unit testing your code that calls out +// to the SDK's service client's calls easier. +// +// The best way to use this interface is so the SDK's service client's calls +// can be stubbed out for unit testing your code with the SDK without needing +// to inject custom request handlers into the SDK's request pipeline. +// +// // myFunc uses an SDK service client to make a request to +// // AWS Security Token Service. +// func myFunc(svc stsiface.STSAPI) bool { +// // Make svc.AssumeRole request +// } +// +// func main() { +// sess := session.New() +// svc := sts.New(sess) +// +// myFunc(svc) +// } +// +// In your _test.go file: +// +// // Define a mock struct to be used in your unit tests of myFunc. +// type mockSTSClient struct { +// stsiface.STSAPI +// } +// func (m *mockSTSClient) AssumeRole(input *sts.AssumeRoleInput) (*sts.AssumeRoleOutput, error) { +// // mock response/functionality +// } +// +// func TestMyFunc(t *testing.T) { +// // Setup Test +// mockSvc := &mockSTSClient{} +// +// myfunc(mockSvc) +// +// // Verify myFunc's functionality +// } +// +// It is important to note that this interface will have breaking changes +// when the service model is updated and adds new API operations, paginators, +// and waiters. Its suggested to use the pattern above for testing, or using +// tooling to generate mocks to satisfy the interfaces. +type STSAPI interface { + AssumeRole(*sts.AssumeRoleInput) (*sts.AssumeRoleOutput, error) + AssumeRoleWithContext(aws.Context, *sts.AssumeRoleInput, ...request.Option) (*sts.AssumeRoleOutput, error) + AssumeRoleRequest(*sts.AssumeRoleInput) (*request.Request, *sts.AssumeRoleOutput) + + AssumeRoleWithSAML(*sts.AssumeRoleWithSAMLInput) (*sts.AssumeRoleWithSAMLOutput, error) + AssumeRoleWithSAMLWithContext(aws.Context, *sts.AssumeRoleWithSAMLInput, ...request.Option) (*sts.AssumeRoleWithSAMLOutput, error) + AssumeRoleWithSAMLRequest(*sts.AssumeRoleWithSAMLInput) (*request.Request, *sts.AssumeRoleWithSAMLOutput) + + AssumeRoleWithWebIdentity(*sts.AssumeRoleWithWebIdentityInput) (*sts.AssumeRoleWithWebIdentityOutput, error) + AssumeRoleWithWebIdentityWithContext(aws.Context, *sts.AssumeRoleWithWebIdentityInput, ...request.Option) (*sts.AssumeRoleWithWebIdentityOutput, error) + AssumeRoleWithWebIdentityRequest(*sts.AssumeRoleWithWebIdentityInput) (*request.Request, *sts.AssumeRoleWithWebIdentityOutput) + + DecodeAuthorizationMessage(*sts.DecodeAuthorizationMessageInput) (*sts.DecodeAuthorizationMessageOutput, error) + DecodeAuthorizationMessageWithContext(aws.Context, *sts.DecodeAuthorizationMessageInput, ...request.Option) (*sts.DecodeAuthorizationMessageOutput, error) + DecodeAuthorizationMessageRequest(*sts.DecodeAuthorizationMessageInput) (*request.Request, *sts.DecodeAuthorizationMessageOutput) + + GetAccessKeyInfo(*sts.GetAccessKeyInfoInput) (*sts.GetAccessKeyInfoOutput, error) + GetAccessKeyInfoWithContext(aws.Context, *sts.GetAccessKeyInfoInput, ...request.Option) (*sts.GetAccessKeyInfoOutput, error) + GetAccessKeyInfoRequest(*sts.GetAccessKeyInfoInput) (*request.Request, *sts.GetAccessKeyInfoOutput) + + GetCallerIdentity(*sts.GetCallerIdentityInput) (*sts.GetCallerIdentityOutput, error) + GetCallerIdentityWithContext(aws.Context, *sts.GetCallerIdentityInput, ...request.Option) (*sts.GetCallerIdentityOutput, error) + GetCallerIdentityRequest(*sts.GetCallerIdentityInput) (*request.Request, *sts.GetCallerIdentityOutput) + + GetFederationToken(*sts.GetFederationTokenInput) (*sts.GetFederationTokenOutput, error) + GetFederationTokenWithContext(aws.Context, *sts.GetFederationTokenInput, ...request.Option) (*sts.GetFederationTokenOutput, error) + GetFederationTokenRequest(*sts.GetFederationTokenInput) (*request.Request, *sts.GetFederationTokenOutput) + + GetSessionToken(*sts.GetSessionTokenInput) (*sts.GetSessionTokenOutput, error) + GetSessionTokenWithContext(aws.Context, *sts.GetSessionTokenInput, ...request.Option) (*sts.GetSessionTokenOutput, error) + GetSessionTokenRequest(*sts.GetSessionTokenInput) (*request.Request, *sts.GetSessionTokenOutput) +} + +var _ STSAPI = (*sts.STS)(nil) diff --git a/github.com/go-sql-driver/mysql/.travis.yml b/github.com/go-sql-driver/mysql/.travis.yml index 75505f1440..56fcf25f20 100644 --- a/github.com/go-sql-driver/mysql/.travis.yml +++ b/github.com/go-sql-driver/mysql/.travis.yml @@ -1,10 +1,10 @@ sudo: false language: go go: - - 1.8.x - - 1.9.x - 1.10.x - 1.11.x + - 1.12.x + - 1.13.x - master before_install: @@ -99,6 +99,28 @@ matrix: - export MYSQL_TEST_ADDR=127.0.0.1:3307 - export MYSQL_TEST_CONCURRENT=1 + - os: osx + osx_image: xcode10.1 + addons: + homebrew: + packages: + - mysql + update: true + go: 1.12.x + before_install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls + before_script: + - echo -e "[server]\ninnodb_log_file_size=256MB\ninnodb_buffer_pool_size=512MB\nmax_allowed_packet=16MB\nlocal_infile=1" >> /usr/local/etc/my.cnf + - mysql.server start + - mysql -uroot -e 'CREATE USER gotest IDENTIFIED BY "secret"' + - mysql -uroot -e 'GRANT ALL ON *.* TO gotest' + - mysql -uroot -e 'create database gotest;' + - export MYSQL_TEST_USER=gotest + - export MYSQL_TEST_PASS=secret + - export MYSQL_TEST_ADDR=127.0.0.1:3306 + - export MYSQL_TEST_CONCURRENT=1 + script: - go test -v -covermode=count -coverprofile=coverage.out - go vet ./... diff --git a/github.com/go-sql-driver/mysql/AUTHORS b/github.com/go-sql-driver/mysql/AUTHORS index 5ce4f7eca1..ad59898006 100644 --- a/github.com/go-sql-driver/mysql/AUTHORS +++ b/github.com/go-sql-driver/mysql/AUTHORS @@ -27,6 +27,7 @@ Daniël van Eeden Dave Protasowski DisposaBoy Egor Smolyakov +Erwan Martin Evan Shaw Frederick Mayle Gustavo Kristic @@ -34,6 +35,7 @@ Hajime Nakagami Hanno Braun Henri Yandell Hirotaka Yamamoto +Huyiguang ICHINOSE Shogo Ilia Cimpoes INADA Naoki @@ -41,6 +43,8 @@ Jacek Szwec James Harr Jeff Hodges Jeffrey Charles +Jerome Meyer +Jiajia Zhong Jian Zhen Joshua Prunier Julien Lefevre @@ -59,6 +63,7 @@ Lucas Liu Luke Scott Maciej Zimnoch Michael Woolnough +Nathanial Murphy Nicola Peduzzi Olivier Mengué oscarzhao @@ -70,12 +75,15 @@ Richard Wilkes Robert Russell Runrioter Wung Shuode Li +Simon J Mudd Soroush Pour Stan Putrya Stanley Gunawan Steven Hartland Thomas Wodarek +Tim Ruffles Tom Jenkinson +Vladimir Kovpak Xiangyu Hu Xiaobing Jiang Xiuming Chen @@ -85,10 +93,13 @@ Zhenye Xie Barracuda Networks, Inc. Counting Ltd. +DigitalOcean Inc. +Facebook Inc. +GitHub Inc. Google Inc. InfoSum Ltd. Keybase Inc. +Multiplay Ltd. Percona LLC Pivotal Inc. Stripe Inc. -Multiplay Ltd. diff --git a/github.com/go-sql-driver/mysql/CHANGELOG.md b/github.com/go-sql-driver/mysql/CHANGELOG.md index 2d87d74c97..9cb97b38de 100644 --- a/github.com/go-sql-driver/mysql/CHANGELOG.md +++ b/github.com/go-sql-driver/mysql/CHANGELOG.md @@ -1,3 +1,42 @@ +## Version 1.5 (2020-01-07) + +Changes: + + - Dropped support Go 1.9 and lower (#823, #829, #886, #1016, #1017) + - Improve buffer handling (#890) + - Document potentially insecure TLS configs (#901) + - Use a double-buffering scheme to prevent data races (#943) + - Pass uint64 values without converting them to string (#838, #955) + - Update collations and make utf8mb4 default (#877, #1054) + - Make NullTime compatible with sql.NullTime in Go 1.13+ (#995) + - Removed CloudSQL support (#993, #1007) + - Add Go Module support (#1003) + +New Features: + + - Implement support of optional TLS (#900) + - Check connection liveness (#934, #964, #997, #1048, #1051, #1052) + - Implement Connector Interface (#941, #958, #1020, #1035) + +Bugfixes: + + - Mark connections as bad on error during ping (#875) + - Mark connections as bad on error during dial (#867) + - Fix connection leak caused by rapid context cancellation (#1024) + - Mark connections as bad on error during Conn.Prepare (#1030) + + +## Version 1.4.1 (2018-11-14) + +Bugfixes: + + - Fix TIME format for binary columns (#818) + - Fix handling of empty auth plugin names (#835) + - Fix caching_sha2_password with empty password (#826) + - Fix canceled context broke mysqlConn (#862) + - Fix OldAuthSwitchRequest support (#870) + - Fix Auth Response packet for cleartext password (#887) + ## Version 1.4 (2018-06-03) Changes: diff --git a/github.com/go-sql-driver/mysql/CONTRIBUTING.md b/github.com/go-sql-driver/mysql/CONTRIBUTING.md deleted file mode 100644 index 8fe16bcb49..0000000000 --- a/github.com/go-sql-driver/mysql/CONTRIBUTING.md +++ /dev/null @@ -1,23 +0,0 @@ -# Contributing Guidelines - -## Reporting Issues - -Before creating a new Issue, please check first if a similar Issue [already exists](https://github.com/go-sql-driver/mysql/issues?state=open) or was [recently closed](https://github.com/go-sql-driver/mysql/issues?direction=desc&page=1&sort=updated&state=closed). - -## Contributing Code - -By contributing to this project, you share your code under the Mozilla Public License 2, as specified in the LICENSE file. -Don't forget to add yourself to the AUTHORS file. - -### Code Review - -Everyone is invited to review and comment on pull requests. -If it looks fine to you, comment with "LGTM" (Looks good to me). - -If changes are required, notice the reviewers with "PTAL" (Please take another look) after committing the fixes. - -Before merging the Pull Request, at least one [team member](https://github.com/go-sql-driver?tab=members) must have commented with "LGTM". - -## Development Ideas - -If you are looking for ideas for code contributions, please check our [Development Ideas](https://github.com/go-sql-driver/mysql/wiki/Development-Ideas) Wiki page. diff --git a/github.com/go-sql-driver/mysql/README.md b/github.com/go-sql-driver/mysql/README.md index 341d9194c1..d2627a41a7 100644 --- a/github.com/go-sql-driver/mysql/README.md +++ b/github.com/go-sql-driver/mysql/README.md @@ -40,7 +40,7 @@ A MySQL-Driver for Go's [database/sql](https://golang.org/pkg/database/sql/) pac * Optional placeholder interpolation ## Requirements - * Go 1.8 or higher. We aim to support the 3 latest versions of Go. + * Go 1.10 or higher. We aim to support the 3 latest versions of Go. * MySQL (4.1+), MariaDB, Percona Server, Google CloudSQL or Sphinx (2.2.3+) --------------------------------------- @@ -166,18 +166,34 @@ Sets the charset used for client-server interaction (`"SET NAMES "`). If Usage of the `charset` parameter is discouraged because it issues additional queries to the server. Unless you need the fallback behavior, please use `collation` instead. +##### `checkConnLiveness` + +``` +Type: bool +Valid Values: true, false +Default: true +``` + +On supported platforms connections retrieved from the connection pool are checked for liveness before using them. If the check fails, the respective connection is marked as bad and the query retried with another connection. +`checkConnLiveness=false` disables this liveness check of connections. + ##### `collation` ``` Type: string Valid Values: -Default: utf8_general_ci +Default: utf8mb4_general_ci ``` Sets the collation used for client-server interaction on connection. In contrast to `charset`, `collation` does not issue additional queries. If the specified collation is unavailable on the target server, the connection will fail. A list of valid charsets for a server is retrievable with `SHOW COLLATION`. +The default collation (`utf8mb4_general_ci`) is supported from MySQL 5.5. You should use an older collation (e.g. `utf8_general_ci`) for older MySQL. + +Collations for charset "ucs2", "utf16", "utf16le", and "utf32" can not be used ([ref](https://dev.mysql.com/doc/refman/5.7/en/charset-connection.html#charset-connection-impermissible-client-charset)). + + ##### `clientFoundRows` ``` @@ -391,14 +407,9 @@ TCP on a remote host, e.g. Amazon RDS: id:password@tcp(your-amazonaws-uri.com:3306)/dbname ``` -Google Cloud SQL on App Engine (First Generation MySQL Server): -``` -user@cloudsql(project-id:instance-name)/dbname -``` - -Google Cloud SQL on App Engine (Second Generation MySQL Server): +Google Cloud SQL on App Engine: ``` -user@cloudsql(project-id:regionname:instance-name)/dbname +user:password@unix(/cloudsql/project-id:region-name:instance-name)/dbname ``` TCP using default port (3306) on localhost: @@ -444,7 +455,7 @@ See the [godoc of Go-MySQL-Driver](https://godoc.org/github.com/go-sql-driver/my ### `time.Time` support The default internal output type of MySQL `DATE` and `DATETIME` values is `[]byte` which allows you to scan the value into a `[]byte`, `string` or `sql.RawBytes` variable in your program. -However, many want to scan MySQL `DATE` and `DATETIME` values into `time.Time` variables, which is the logical opposite in Go to `DATE` and `DATETIME` in MySQL. You can do that by changing the internal output type from `[]byte` to `time.Time` with the DSN parameter `parseTime=true`. You can set the default [`time.Time` location](https://golang.org/pkg/time/#Location) with the `loc` DSN parameter. +However, many want to scan MySQL `DATE` and `DATETIME` values into `time.Time` variables, which is the logical equivalent in Go to `DATE` and `DATETIME` in MySQL. You can do that by changing the internal output type from `[]byte` to `time.Time` with the DSN parameter `parseTime=true`. You can set the default [`time.Time` location](https://golang.org/pkg/time/#Location) with the `loc` DSN parameter. **Caution:** As of Go 1.1, this makes `time.Time` the only variable type you can scan `DATE` and `DATETIME` values into. This breaks for example [`sql.RawBytes` support](https://github.com/go-sql-driver/mysql/wiki/Examples#rawbytes). @@ -452,13 +463,13 @@ Alternatively you can use the [`NullTime`](https://godoc.org/github.com/go-sql-d ### Unicode support -Since version 1.1 Go-MySQL-Driver automatically uses the collation `utf8_general_ci` by default. +Since version 1.5 Go-MySQL-Driver automatically uses the collation ` utf8mb4_general_ci` by default. Other collations / charsets can be set using the [`collation`](#collation) DSN parameter. Version 1.0 of the driver recommended adding `&charset=utf8` (alias for `SET NAMES utf8`) to the DSN to enable proper UTF-8 support. This is not necessary anymore. The [`collation`](#collation) parameter should be preferred to set another collation / charset than the default. -See http://dev.mysql.com/doc/refman/5.7/en/charset-unicode.html for more details on MySQL's Unicode support. +See http://dev.mysql.com/doc/refman/8.0/en/charset-unicode.html for more details on MySQL's Unicode support. ## Testing / Development To run the driver tests you may need to adjust the configuration. See the [Testing Wiki-Page](https://github.com/go-sql-driver/mysql/wiki/Testing "Testing") for details. diff --git a/github.com/go-sql-driver/mysql/buffer.go b/github.com/go-sql-driver/mysql/buffer.go index 19486bd6f6..0774c5c8c2 100644 --- a/github.com/go-sql-driver/mysql/buffer.go +++ b/github.com/go-sql-driver/mysql/buffer.go @@ -15,47 +15,69 @@ import ( ) const defaultBufSize = 4096 +const maxCachedBufSize = 256 * 1024 // A buffer which is used for both reading and writing. // This is possible since communication on each connection is synchronous. // In other words, we can't write and read simultaneously on the same connection. // The buffer is similar to bufio.Reader / Writer but zero-copy-ish // Also highly optimized for this particular use case. +// This buffer is backed by two byte slices in a double-buffering scheme type buffer struct { buf []byte // buf is a byte buffer who's length and capacity are equal. nc net.Conn idx int length int timeout time.Duration + dbuf [2][]byte // dbuf is an array with the two byte slices that back this buffer + flipcnt uint // flipccnt is the current buffer counter for double-buffering } // newBuffer allocates and returns a new buffer. func newBuffer(nc net.Conn) buffer { + fg := make([]byte, defaultBufSize) return buffer{ - buf: make([]byte, defaultBufSize), - nc: nc, + buf: fg, + nc: nc, + dbuf: [2][]byte{fg, nil}, } } +// flip replaces the active buffer with the background buffer +// this is a delayed flip that simply increases the buffer counter; +// the actual flip will be performed the next time we call `buffer.fill` +func (b *buffer) flip() { + b.flipcnt += 1 +} + // fill reads into the buffer until at least _need_ bytes are in it func (b *buffer) fill(need int) error { n := b.length + // fill data into its double-buffering target: if we've called + // flip on this buffer, we'll be copying to the background buffer, + // and then filling it with network data; otherwise we'll just move + // the contents of the current buffer to the front before filling it + dest := b.dbuf[b.flipcnt&1] + + // grow buffer if necessary to fit the whole packet. + if need > len(dest) { + // Round up to the next multiple of the default size + dest = make([]byte, ((need/defaultBufSize)+1)*defaultBufSize) - // move existing data to the beginning - if n > 0 && b.idx > 0 { - copy(b.buf[0:n], b.buf[b.idx:]) + // if the allocated buffer is not too large, move it to backing storage + // to prevent extra allocations on applications that perform large reads + if len(dest) <= maxCachedBufSize { + b.dbuf[b.flipcnt&1] = dest + } } - // grow buffer if necessary - // TODO: let the buffer shrink again at some point - // Maybe keep the org buf slice and swap back? - if need > len(b.buf) { - // Round up to the next multiple of the default size - newBuf := make([]byte, ((need/defaultBufSize)+1)*defaultBufSize) - copy(newBuf, b.buf) - b.buf = newBuf + // if we're filling the fg buffer, move the existing data to the start of it. + // if we're filling the bg buffer, copy over the data + if n > 0 { + copy(dest[:n], b.buf[b.idx:]) } + b.buf = dest b.idx = 0 for { diff --git a/github.com/go-sql-driver/mysql/collations.go b/github.com/go-sql-driver/mysql/collations.go index 136c9e4d1e..8d2b556767 100644 --- a/github.com/go-sql-driver/mysql/collations.go +++ b/github.com/go-sql-driver/mysql/collations.go @@ -8,183 +8,190 @@ package mysql -const defaultCollation = "utf8_general_ci" +const defaultCollation = "utf8mb4_general_ci" const binaryCollation = "binary" // A list of available collations mapped to the internal ID. // To update this map use the following MySQL query: -// SELECT COLLATION_NAME, ID FROM information_schema.COLLATIONS +// SELECT COLLATION_NAME, ID FROM information_schema.COLLATIONS WHERE ID<256 ORDER BY ID +// +// Handshake packet have only 1 byte for collation_id. So we can't use collations with ID > 255. +// +// ucs2, utf16, and utf32 can't be used for connection charset. +// https://dev.mysql.com/doc/refman/5.7/en/charset-connection.html#charset-connection-impermissible-client-charset +// They are commented out to reduce this map. var collations = map[string]byte{ - "big5_chinese_ci": 1, - "latin2_czech_cs": 2, - "dec8_swedish_ci": 3, - "cp850_general_ci": 4, - "latin1_german1_ci": 5, - "hp8_english_ci": 6, - "koi8r_general_ci": 7, - "latin1_swedish_ci": 8, - "latin2_general_ci": 9, - "swe7_swedish_ci": 10, - "ascii_general_ci": 11, - "ujis_japanese_ci": 12, - "sjis_japanese_ci": 13, - "cp1251_bulgarian_ci": 14, - "latin1_danish_ci": 15, - "hebrew_general_ci": 16, - "tis620_thai_ci": 18, - "euckr_korean_ci": 19, - "latin7_estonian_cs": 20, - "latin2_hungarian_ci": 21, - "koi8u_general_ci": 22, - "cp1251_ukrainian_ci": 23, - "gb2312_chinese_ci": 24, - "greek_general_ci": 25, - "cp1250_general_ci": 26, - "latin2_croatian_ci": 27, - "gbk_chinese_ci": 28, - "cp1257_lithuanian_ci": 29, - "latin5_turkish_ci": 30, - "latin1_german2_ci": 31, - "armscii8_general_ci": 32, - "utf8_general_ci": 33, - "cp1250_czech_cs": 34, - "ucs2_general_ci": 35, - "cp866_general_ci": 36, - "keybcs2_general_ci": 37, - "macce_general_ci": 38, - "macroman_general_ci": 39, - "cp852_general_ci": 40, - "latin7_general_ci": 41, - "latin7_general_cs": 42, - "macce_bin": 43, - "cp1250_croatian_ci": 44, - "utf8mb4_general_ci": 45, - "utf8mb4_bin": 46, - "latin1_bin": 47, - "latin1_general_ci": 48, - "latin1_general_cs": 49, - "cp1251_bin": 50, - "cp1251_general_ci": 51, - "cp1251_general_cs": 52, - "macroman_bin": 53, - "utf16_general_ci": 54, - "utf16_bin": 55, - "utf16le_general_ci": 56, - "cp1256_general_ci": 57, - "cp1257_bin": 58, - "cp1257_general_ci": 59, - "utf32_general_ci": 60, - "utf32_bin": 61, - "utf16le_bin": 62, - "binary": 63, - "armscii8_bin": 64, - "ascii_bin": 65, - "cp1250_bin": 66, - "cp1256_bin": 67, - "cp866_bin": 68, - "dec8_bin": 69, - "greek_bin": 70, - "hebrew_bin": 71, - "hp8_bin": 72, - "keybcs2_bin": 73, - "koi8r_bin": 74, - "koi8u_bin": 75, - "latin2_bin": 77, - "latin5_bin": 78, - "latin7_bin": 79, - "cp850_bin": 80, - "cp852_bin": 81, - "swe7_bin": 82, - "utf8_bin": 83, - "big5_bin": 84, - "euckr_bin": 85, - "gb2312_bin": 86, - "gbk_bin": 87, - "sjis_bin": 88, - "tis620_bin": 89, - "ucs2_bin": 90, - "ujis_bin": 91, - "geostd8_general_ci": 92, - "geostd8_bin": 93, - "latin1_spanish_ci": 94, - "cp932_japanese_ci": 95, - "cp932_bin": 96, - "eucjpms_japanese_ci": 97, - "eucjpms_bin": 98, - "cp1250_polish_ci": 99, - "utf16_unicode_ci": 101, - "utf16_icelandic_ci": 102, - "utf16_latvian_ci": 103, - "utf16_romanian_ci": 104, - "utf16_slovenian_ci": 105, - "utf16_polish_ci": 106, - "utf16_estonian_ci": 107, - "utf16_spanish_ci": 108, - "utf16_swedish_ci": 109, - "utf16_turkish_ci": 110, - "utf16_czech_ci": 111, - "utf16_danish_ci": 112, - "utf16_lithuanian_ci": 113, - "utf16_slovak_ci": 114, - "utf16_spanish2_ci": 115, - "utf16_roman_ci": 116, - "utf16_persian_ci": 117, - "utf16_esperanto_ci": 118, - "utf16_hungarian_ci": 119, - "utf16_sinhala_ci": 120, - "utf16_german2_ci": 121, - "utf16_croatian_ci": 122, - "utf16_unicode_520_ci": 123, - "utf16_vietnamese_ci": 124, - "ucs2_unicode_ci": 128, - "ucs2_icelandic_ci": 129, - "ucs2_latvian_ci": 130, - "ucs2_romanian_ci": 131, - "ucs2_slovenian_ci": 132, - "ucs2_polish_ci": 133, - "ucs2_estonian_ci": 134, - "ucs2_spanish_ci": 135, - "ucs2_swedish_ci": 136, - "ucs2_turkish_ci": 137, - "ucs2_czech_ci": 138, - "ucs2_danish_ci": 139, - "ucs2_lithuanian_ci": 140, - "ucs2_slovak_ci": 141, - "ucs2_spanish2_ci": 142, - "ucs2_roman_ci": 143, - "ucs2_persian_ci": 144, - "ucs2_esperanto_ci": 145, - "ucs2_hungarian_ci": 146, - "ucs2_sinhala_ci": 147, - "ucs2_german2_ci": 148, - "ucs2_croatian_ci": 149, - "ucs2_unicode_520_ci": 150, - "ucs2_vietnamese_ci": 151, - "ucs2_general_mysql500_ci": 159, - "utf32_unicode_ci": 160, - "utf32_icelandic_ci": 161, - "utf32_latvian_ci": 162, - "utf32_romanian_ci": 163, - "utf32_slovenian_ci": 164, - "utf32_polish_ci": 165, - "utf32_estonian_ci": 166, - "utf32_spanish_ci": 167, - "utf32_swedish_ci": 168, - "utf32_turkish_ci": 169, - "utf32_czech_ci": 170, - "utf32_danish_ci": 171, - "utf32_lithuanian_ci": 172, - "utf32_slovak_ci": 173, - "utf32_spanish2_ci": 174, - "utf32_roman_ci": 175, - "utf32_persian_ci": 176, - "utf32_esperanto_ci": 177, - "utf32_hungarian_ci": 178, - "utf32_sinhala_ci": 179, - "utf32_german2_ci": 180, - "utf32_croatian_ci": 181, - "utf32_unicode_520_ci": 182, - "utf32_vietnamese_ci": 183, + "big5_chinese_ci": 1, + "latin2_czech_cs": 2, + "dec8_swedish_ci": 3, + "cp850_general_ci": 4, + "latin1_german1_ci": 5, + "hp8_english_ci": 6, + "koi8r_general_ci": 7, + "latin1_swedish_ci": 8, + "latin2_general_ci": 9, + "swe7_swedish_ci": 10, + "ascii_general_ci": 11, + "ujis_japanese_ci": 12, + "sjis_japanese_ci": 13, + "cp1251_bulgarian_ci": 14, + "latin1_danish_ci": 15, + "hebrew_general_ci": 16, + "tis620_thai_ci": 18, + "euckr_korean_ci": 19, + "latin7_estonian_cs": 20, + "latin2_hungarian_ci": 21, + "koi8u_general_ci": 22, + "cp1251_ukrainian_ci": 23, + "gb2312_chinese_ci": 24, + "greek_general_ci": 25, + "cp1250_general_ci": 26, + "latin2_croatian_ci": 27, + "gbk_chinese_ci": 28, + "cp1257_lithuanian_ci": 29, + "latin5_turkish_ci": 30, + "latin1_german2_ci": 31, + "armscii8_general_ci": 32, + "utf8_general_ci": 33, + "cp1250_czech_cs": 34, + //"ucs2_general_ci": 35, + "cp866_general_ci": 36, + "keybcs2_general_ci": 37, + "macce_general_ci": 38, + "macroman_general_ci": 39, + "cp852_general_ci": 40, + "latin7_general_ci": 41, + "latin7_general_cs": 42, + "macce_bin": 43, + "cp1250_croatian_ci": 44, + "utf8mb4_general_ci": 45, + "utf8mb4_bin": 46, + "latin1_bin": 47, + "latin1_general_ci": 48, + "latin1_general_cs": 49, + "cp1251_bin": 50, + "cp1251_general_ci": 51, + "cp1251_general_cs": 52, + "macroman_bin": 53, + //"utf16_general_ci": 54, + //"utf16_bin": 55, + //"utf16le_general_ci": 56, + "cp1256_general_ci": 57, + "cp1257_bin": 58, + "cp1257_general_ci": 59, + //"utf32_general_ci": 60, + //"utf32_bin": 61, + //"utf16le_bin": 62, + "binary": 63, + "armscii8_bin": 64, + "ascii_bin": 65, + "cp1250_bin": 66, + "cp1256_bin": 67, + "cp866_bin": 68, + "dec8_bin": 69, + "greek_bin": 70, + "hebrew_bin": 71, + "hp8_bin": 72, + "keybcs2_bin": 73, + "koi8r_bin": 74, + "koi8u_bin": 75, + "utf8_tolower_ci": 76, + "latin2_bin": 77, + "latin5_bin": 78, + "latin7_bin": 79, + "cp850_bin": 80, + "cp852_bin": 81, + "swe7_bin": 82, + "utf8_bin": 83, + "big5_bin": 84, + "euckr_bin": 85, + "gb2312_bin": 86, + "gbk_bin": 87, + "sjis_bin": 88, + "tis620_bin": 89, + //"ucs2_bin": 90, + "ujis_bin": 91, + "geostd8_general_ci": 92, + "geostd8_bin": 93, + "latin1_spanish_ci": 94, + "cp932_japanese_ci": 95, + "cp932_bin": 96, + "eucjpms_japanese_ci": 97, + "eucjpms_bin": 98, + "cp1250_polish_ci": 99, + //"utf16_unicode_ci": 101, + //"utf16_icelandic_ci": 102, + //"utf16_latvian_ci": 103, + //"utf16_romanian_ci": 104, + //"utf16_slovenian_ci": 105, + //"utf16_polish_ci": 106, + //"utf16_estonian_ci": 107, + //"utf16_spanish_ci": 108, + //"utf16_swedish_ci": 109, + //"utf16_turkish_ci": 110, + //"utf16_czech_ci": 111, + //"utf16_danish_ci": 112, + //"utf16_lithuanian_ci": 113, + //"utf16_slovak_ci": 114, + //"utf16_spanish2_ci": 115, + //"utf16_roman_ci": 116, + //"utf16_persian_ci": 117, + //"utf16_esperanto_ci": 118, + //"utf16_hungarian_ci": 119, + //"utf16_sinhala_ci": 120, + //"utf16_german2_ci": 121, + //"utf16_croatian_ci": 122, + //"utf16_unicode_520_ci": 123, + //"utf16_vietnamese_ci": 124, + //"ucs2_unicode_ci": 128, + //"ucs2_icelandic_ci": 129, + //"ucs2_latvian_ci": 130, + //"ucs2_romanian_ci": 131, + //"ucs2_slovenian_ci": 132, + //"ucs2_polish_ci": 133, + //"ucs2_estonian_ci": 134, + //"ucs2_spanish_ci": 135, + //"ucs2_swedish_ci": 136, + //"ucs2_turkish_ci": 137, + //"ucs2_czech_ci": 138, + //"ucs2_danish_ci": 139, + //"ucs2_lithuanian_ci": 140, + //"ucs2_slovak_ci": 141, + //"ucs2_spanish2_ci": 142, + //"ucs2_roman_ci": 143, + //"ucs2_persian_ci": 144, + //"ucs2_esperanto_ci": 145, + //"ucs2_hungarian_ci": 146, + //"ucs2_sinhala_ci": 147, + //"ucs2_german2_ci": 148, + //"ucs2_croatian_ci": 149, + //"ucs2_unicode_520_ci": 150, + //"ucs2_vietnamese_ci": 151, + //"ucs2_general_mysql500_ci": 159, + //"utf32_unicode_ci": 160, + //"utf32_icelandic_ci": 161, + //"utf32_latvian_ci": 162, + //"utf32_romanian_ci": 163, + //"utf32_slovenian_ci": 164, + //"utf32_polish_ci": 165, + //"utf32_estonian_ci": 166, + //"utf32_spanish_ci": 167, + //"utf32_swedish_ci": 168, + //"utf32_turkish_ci": 169, + //"utf32_czech_ci": 170, + //"utf32_danish_ci": 171, + //"utf32_lithuanian_ci": 172, + //"utf32_slovak_ci": 173, + //"utf32_spanish2_ci": 174, + //"utf32_roman_ci": 175, + //"utf32_persian_ci": 176, + //"utf32_esperanto_ci": 177, + //"utf32_hungarian_ci": 178, + //"utf32_sinhala_ci": 179, + //"utf32_german2_ci": 180, + //"utf32_croatian_ci": 181, + //"utf32_unicode_520_ci": 182, + //"utf32_vietnamese_ci": 183, "utf8_unicode_ci": 192, "utf8_icelandic_ci": 193, "utf8_latvian_ci": 194, @@ -234,18 +241,25 @@ var collations = map[string]byte{ "utf8mb4_croatian_ci": 245, "utf8mb4_unicode_520_ci": 246, "utf8mb4_vietnamese_ci": 247, + "gb18030_chinese_ci": 248, + "gb18030_bin": 249, + "gb18030_unicode_520_ci": 250, + "utf8mb4_0900_ai_ci": 255, } // A blacklist of collations which is unsafe to interpolate parameters. // These multibyte encodings may contains 0x5c (`\`) in their trailing bytes. var unsafeCollations = map[string]bool{ - "big5_chinese_ci": true, - "sjis_japanese_ci": true, - "gbk_chinese_ci": true, - "big5_bin": true, - "gb2312_bin": true, - "gbk_bin": true, - "sjis_bin": true, - "cp932_japanese_ci": true, - "cp932_bin": true, + "big5_chinese_ci": true, + "sjis_japanese_ci": true, + "gbk_chinese_ci": true, + "big5_bin": true, + "gb2312_bin": true, + "gbk_bin": true, + "sjis_bin": true, + "cp932_japanese_ci": true, + "cp932_bin": true, + "gb18030_chinese_ci": true, + "gb18030_bin": true, + "gb18030_unicode_520_ci": true, } diff --git a/github.com/go-sql-driver/mysql/conncheck.go b/github.com/go-sql-driver/mysql/conncheck.go new file mode 100644 index 0000000000..024eb28589 --- /dev/null +++ b/github.com/go-sql-driver/mysql/conncheck.go @@ -0,0 +1,54 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2019 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build linux darwin dragonfly freebsd netbsd openbsd solaris illumos + +package mysql + +import ( + "errors" + "io" + "net" + "syscall" +) + +var errUnexpectedRead = errors.New("unexpected read from socket") + +func connCheck(conn net.Conn) error { + var sysErr error + + sysConn, ok := conn.(syscall.Conn) + if !ok { + return nil + } + rawConn, err := sysConn.SyscallConn() + if err != nil { + return err + } + + err = rawConn.Read(func(fd uintptr) bool { + var buf [1]byte + n, err := syscall.Read(int(fd), buf[:]) + switch { + case n == 0 && err == nil: + sysErr = io.EOF + case n > 0: + sysErr = errUnexpectedRead + case err == syscall.EAGAIN || err == syscall.EWOULDBLOCK: + sysErr = nil + default: + sysErr = err + } + return true + }) + if err != nil { + return err + } + + return sysErr +} diff --git a/github.com/go-sql-driver/mysql/appengine.go b/github.com/go-sql-driver/mysql/conncheck_dummy.go similarity index 59% rename from github.com/go-sql-driver/mysql/appengine.go rename to github.com/go-sql-driver/mysql/conncheck_dummy.go index be41f2ee6d..ea7fb607ac 100644 --- a/github.com/go-sql-driver/mysql/appengine.go +++ b/github.com/go-sql-driver/mysql/conncheck_dummy.go @@ -1,19 +1,17 @@ // Go MySQL Driver - A MySQL-Driver for Go's database/sql package // -// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// Copyright 2019 The Go-MySQL-Driver Authors. All rights reserved. // // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at http://mozilla.org/MPL/2.0/. -// +build appengine +// +build !linux,!darwin,!dragonfly,!freebsd,!netbsd,!openbsd,!solaris,!illumos package mysql -import ( - "google.golang.org/appengine/cloudsql" -) +import "net" -func init() { - RegisterDial("cloudsql", cloudsql.Dial) +func connCheck(conn net.Conn) error { + return nil } diff --git a/github.com/go-sql-driver/mysql/connection.go b/github.com/go-sql-driver/mysql/connection.go index fc4ec7597d..e4bb59e67c 100644 --- a/github.com/go-sql-driver/mysql/connection.go +++ b/github.com/go-sql-driver/mysql/connection.go @@ -22,6 +22,7 @@ import ( type mysqlConn struct { buf buffer netConn net.Conn + rawConn net.Conn // underlying connection when netConn is TLS connection. affectedRows uint64 insertId uint64 cfg *Config @@ -32,6 +33,7 @@ type mysqlConn struct { status statusFlag sequence uint8 parseTime bool + reset bool // set when the Go SQL package calls ResetSession // for context support (Go 1.8+) watching bool @@ -152,7 +154,9 @@ func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) { // Send command err := mc.writeCommandPacketStr(comStmtPrepare, query) if err != nil { - return nil, mc.markBadConn(err) + // STMT_PREPARE is safe to retry. So we can return ErrBadConn here. + errLog.Print(err) + return nil, driver.ErrBadConn } stmt := &mysqlStmt{ @@ -211,6 +215,9 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin switch v := arg.(type) { case int64: buf = strconv.AppendInt(buf, v, 10) + case uint64: + // Handle uint64 explicitly because our custom ConvertValue emits unsigned values + buf = strconv.AppendUint(buf, v, 10) case float64: buf = strconv.AppendFloat(buf, v, 'g', -1, 64) case bool: @@ -639,5 +646,6 @@ func (mc *mysqlConn) ResetSession(ctx context.Context) error { if mc.closed.IsSet() { return driver.ErrBadConn } + mc.reset = true return nil } diff --git a/github.com/go-sql-driver/mysql/connector.go b/github.com/go-sql-driver/mysql/connector.go new file mode 100644 index 0000000000..d567b4e4fc --- /dev/null +++ b/github.com/go-sql-driver/mysql/connector.go @@ -0,0 +1,146 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "context" + "database/sql/driver" + "net" +) + +type connector struct { + cfg *Config // immutable private copy. +} + +// Connect implements driver.Connector interface. +// Connect returns a connection to the database. +func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { + var err error + + // New mysqlConn + mc := &mysqlConn{ + maxAllowedPacket: maxPacketSize, + maxWriteSize: maxPacketSize - 1, + closech: make(chan struct{}), + cfg: c.cfg, + } + mc.parseTime = mc.cfg.ParseTime + + // Connect to Server + dialsLock.RLock() + dial, ok := dials[mc.cfg.Net] + dialsLock.RUnlock() + if ok { + dctx := ctx + if mc.cfg.Timeout > 0 { + var cancel context.CancelFunc + dctx, cancel = context.WithTimeout(ctx, c.cfg.Timeout) + defer cancel() + } + mc.netConn, err = dial(dctx, mc.cfg.Addr) + } else { + nd := net.Dialer{Timeout: mc.cfg.Timeout} + mc.netConn, err = nd.DialContext(ctx, mc.cfg.Net, mc.cfg.Addr) + } + + if err != nil { + return nil, err + } + + // Enable TCP Keepalives on TCP connections + if tc, ok := mc.netConn.(*net.TCPConn); ok { + if err := tc.SetKeepAlive(true); err != nil { + // Don't send COM_QUIT before handshake. + mc.netConn.Close() + mc.netConn = nil + return nil, err + } + } + + // Call startWatcher for context support (From Go 1.8) + mc.startWatcher() + if err := mc.watchCancel(ctx); err != nil { + mc.cleanup() + return nil, err + } + defer mc.finish() + + mc.buf = newBuffer(mc.netConn) + + // Set I/O timeouts + mc.buf.timeout = mc.cfg.ReadTimeout + mc.writeTimeout = mc.cfg.WriteTimeout + + // Reading Handshake Initialization Packet + authData, plugin, err := mc.readHandshakePacket() + if err != nil { + mc.cleanup() + return nil, err + } + + if plugin == "" { + plugin = defaultAuthPlugin + } + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + // try the default auth plugin, if using the requested plugin failed + errLog.Print("could not use requested auth plugin '"+plugin+"': ", err.Error()) + plugin = defaultAuthPlugin + authResp, err = mc.auth(authData, plugin) + if err != nil { + mc.cleanup() + return nil, err + } + } + if err = mc.writeHandshakeResponsePacket(authResp, plugin); err != nil { + mc.cleanup() + return nil, err + } + + // Handle response to auth packet, switch methods if possible + if err = mc.handleAuthResult(authData, plugin); err != nil { + // Authentication failed and MySQL has already closed the connection + // (https://dev.mysql.com/doc/internals/en/authentication-fails.html). + // Do not send COM_QUIT, just cleanup and return the error. + mc.cleanup() + return nil, err + } + + if mc.cfg.MaxAllowedPacket > 0 { + mc.maxAllowedPacket = mc.cfg.MaxAllowedPacket + } else { + // Get max allowed packet size + maxap, err := mc.getSystemVar("max_allowed_packet") + if err != nil { + mc.Close() + return nil, err + } + mc.maxAllowedPacket = stringToInt(maxap) - 1 + } + if mc.maxAllowedPacket < maxPacketSize { + mc.maxWriteSize = mc.maxAllowedPacket + } + + // Handle DSN Params + err = mc.handleParams() + if err != nil { + mc.Close() + return nil, err + } + + return mc, nil +} + +// Driver implements driver.Connector interface. +// Driver returns &MySQLDriver{}. +func (c *connector) Driver() driver.Driver { + return &MySQLDriver{} +} diff --git a/github.com/go-sql-driver/mysql/driver.go b/github.com/go-sql-driver/mysql/driver.go index 9f4967087f..c1bdf1199b 100644 --- a/github.com/go-sql-driver/mysql/driver.go +++ b/github.com/go-sql-driver/mysql/driver.go @@ -17,6 +17,7 @@ package mysql import ( + "context" "database/sql" "database/sql/driver" "net" @@ -29,141 +30,78 @@ type MySQLDriver struct{} // DialFunc is a function which can be used to establish the network connection. // Custom dial functions must be registered with RegisterDial +// +// Deprecated: users should register a DialContextFunc instead type DialFunc func(addr string) (net.Conn, error) +// DialContextFunc is a function which can be used to establish the network connection. +// Custom dial functions must be registered with RegisterDialContext +type DialContextFunc func(ctx context.Context, addr string) (net.Conn, error) + var ( dialsLock sync.RWMutex - dials map[string]DialFunc + dials map[string]DialContextFunc ) -// RegisterDial registers a custom dial function. It can then be used by the +// RegisterDialContext registers a custom dial function. It can then be used by the // network address mynet(addr), where mynet is the registered new network. -// addr is passed as a parameter to the dial function. -func RegisterDial(net string, dial DialFunc) { +// The current context for the connection and its address is passed to the dial function. +func RegisterDialContext(net string, dial DialContextFunc) { dialsLock.Lock() defer dialsLock.Unlock() if dials == nil { - dials = make(map[string]DialFunc) + dials = make(map[string]DialContextFunc) } dials[net] = dial } +// RegisterDial registers a custom dial function. It can then be used by the +// network address mynet(addr), where mynet is the registered new network. +// addr is passed as a parameter to the dial function. +// +// Deprecated: users should call RegisterDialContext instead +func RegisterDial(network string, dial DialFunc) { + RegisterDialContext(network, func(_ context.Context, addr string) (net.Conn, error) { + return dial(addr) + }) +} + // Open new Connection. // See https://github.com/go-sql-driver/mysql#dsn-data-source-name for how // the DSN string is formatted func (d MySQLDriver) Open(dsn string) (driver.Conn, error) { - var err error - - // New mysqlConn - mc := &mysqlConn{ - maxAllowedPacket: maxPacketSize, - maxWriteSize: maxPacketSize - 1, - closech: make(chan struct{}), - } - mc.cfg, err = ParseDSN(dsn) + cfg, err := ParseDSN(dsn) if err != nil { return nil, err } - mc.parseTime = mc.cfg.ParseTime - - // Connect to Server - dialsLock.RLock() - dial, ok := dials[mc.cfg.Net] - dialsLock.RUnlock() - if ok { - mc.netConn, err = dial(mc.cfg.Addr) - } else { - nd := net.Dialer{Timeout: mc.cfg.Timeout} - mc.netConn, err = nd.Dial(mc.cfg.Net, mc.cfg.Addr) - } - if err != nil { - if nerr, ok := err.(net.Error); ok && nerr.Temporary() { - errLog.Print("net.Error from Dial()': ", nerr.Error()) - return nil, driver.ErrBadConn - } - return nil, err - } - - // Enable TCP Keepalives on TCP connections - if tc, ok := mc.netConn.(*net.TCPConn); ok { - if err := tc.SetKeepAlive(true); err != nil { - // Don't send COM_QUIT before handshake. - mc.netConn.Close() - mc.netConn = nil - return nil, err - } - } - - // Call startWatcher for context support (From Go 1.8) - mc.startWatcher() - - mc.buf = newBuffer(mc.netConn) - - // Set I/O timeouts - mc.buf.timeout = mc.cfg.ReadTimeout - mc.writeTimeout = mc.cfg.WriteTimeout - - // Reading Handshake Initialization Packet - authData, plugin, err := mc.readHandshakePacket() - if err != nil { - mc.cleanup() - return nil, err - } - if plugin == "" { - plugin = defaultAuthPlugin + c := &connector{ + cfg: cfg, } + return c.Connect(context.Background()) +} - // Send Client Authentication Packet - authResp, err := mc.auth(authData, plugin) - if err != nil { - // try the default auth plugin, if using the requested plugin failed - errLog.Print("could not use requested auth plugin '"+plugin+"': ", err.Error()) - plugin = defaultAuthPlugin - authResp, err = mc.auth(authData, plugin) - if err != nil { - mc.cleanup() - return nil, err - } - } - if err = mc.writeHandshakeResponsePacket(authResp, plugin); err != nil { - mc.cleanup() - return nil, err - } +func init() { + sql.Register("mysql", &MySQLDriver{}) +} - // Handle response to auth packet, switch methods if possible - if err = mc.handleAuthResult(authData, plugin); err != nil { - // Authentication failed and MySQL has already closed the connection - // (https://dev.mysql.com/doc/internals/en/authentication-fails.html). - // Do not send COM_QUIT, just cleanup and return the error. - mc.cleanup() +// NewConnector returns new driver.Connector. +func NewConnector(cfg *Config) (driver.Connector, error) { + cfg = cfg.Clone() + // normalize the contents of cfg so calls to NewConnector have the same + // behavior as MySQLDriver.OpenConnector + if err := cfg.normalize(); err != nil { return nil, err } + return &connector{cfg: cfg}, nil +} - if mc.cfg.MaxAllowedPacket > 0 { - mc.maxAllowedPacket = mc.cfg.MaxAllowedPacket - } else { - // Get max allowed packet size - maxap, err := mc.getSystemVar("max_allowed_packet") - if err != nil { - mc.Close() - return nil, err - } - mc.maxAllowedPacket = stringToInt(maxap) - 1 - } - if mc.maxAllowedPacket < maxPacketSize { - mc.maxWriteSize = mc.maxAllowedPacket - } - - // Handle DSN Params - err = mc.handleParams() +// OpenConnector implements driver.DriverContext. +func (d MySQLDriver) OpenConnector(dsn string) (driver.Connector, error) { + cfg, err := ParseDSN(dsn) if err != nil { - mc.Close() return nil, err } - - return mc, nil -} - -func init() { - sql.Register("mysql", &MySQLDriver{}) + return &connector{ + cfg: cfg, + }, nil } diff --git a/github.com/go-sql-driver/mysql/dsn.go b/github.com/go-sql-driver/mysql/dsn.go index b9134722eb..75c8c24896 100644 --- a/github.com/go-sql-driver/mysql/dsn.go +++ b/github.com/go-sql-driver/mysql/dsn.go @@ -14,6 +14,7 @@ import ( "crypto/tls" "errors" "fmt" + "math/big" "net" "net/url" "sort" @@ -54,6 +55,7 @@ type Config struct { AllowCleartextPasswords bool // Allows the cleartext client side plugin AllowNativePasswords bool // Allows the native password authentication method AllowOldPasswords bool // Allows the old insecure password method + CheckConnLiveness bool // Check connections for liveness before using them ClientFoundRows bool // Return number of matching rows instead of rows changed ColumnsWithAlias bool // Prepend table alias to column names InterpolateParams bool // Interpolate placeholders into query string @@ -69,9 +71,30 @@ func NewConfig() *Config { Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, + CheckConnLiveness: true, } } +func (cfg *Config) Clone() *Config { + cp := *cfg + if cp.tls != nil { + cp.tls = cfg.tls.Clone() + } + if len(cp.Params) > 0 { + cp.Params = make(map[string]string, len(cfg.Params)) + for k, v := range cfg.Params { + cp.Params[k] = v + } + } + if cfg.pubKey != nil { + cp.pubKey = &rsa.PublicKey{ + N: new(big.Int).Set(cfg.pubKey.N), + E: cfg.pubKey.E, + } + } + return &cp +} + func (cfg *Config) normalize() error { if cfg.InterpolateParams && unsafeCollations[cfg.Collation] { return errInvalidDSNUnsafeCollation @@ -92,23 +115,54 @@ func (cfg *Config) normalize() error { default: return errors.New("default addr for network '" + cfg.Net + "' unknown") } - } else if cfg.Net == "tcp" { cfg.Addr = ensureHavePort(cfg.Addr) } - if cfg.tls != nil { - if cfg.tls.ServerName == "" && !cfg.tls.InsecureSkipVerify { - host, _, err := net.SplitHostPort(cfg.Addr) - if err == nil { - cfg.tls.ServerName = host - } + switch cfg.TLSConfig { + case "false", "": + // don't set anything + case "true": + cfg.tls = &tls.Config{} + case "skip-verify", "preferred": + cfg.tls = &tls.Config{InsecureSkipVerify: true} + default: + cfg.tls = getTLSConfigClone(cfg.TLSConfig) + if cfg.tls == nil { + return errors.New("invalid value / unknown config name: " + cfg.TLSConfig) + } + } + + if cfg.tls != nil && cfg.tls.ServerName == "" && !cfg.tls.InsecureSkipVerify { + host, _, err := net.SplitHostPort(cfg.Addr) + if err == nil { + cfg.tls.ServerName = host + } + } + + if cfg.ServerPubKey != "" { + cfg.pubKey = getServerPubKey(cfg.ServerPubKey) + if cfg.pubKey == nil { + return errors.New("invalid value / unknown server pub key name: " + cfg.ServerPubKey) } } return nil } +func writeDSNParam(buf *bytes.Buffer, hasParam *bool, name, value string) { + buf.Grow(1 + len(name) + 1 + len(value)) + if !*hasParam { + *hasParam = true + buf.WriteByte('?') + } else { + buf.WriteByte('&') + } + buf.WriteString(name) + buf.WriteByte('=') + buf.WriteString(value) +} + // FormatDSN formats the given Config into a DSN string which can be passed to // the driver. func (cfg *Config) FormatDSN() string { @@ -147,165 +201,75 @@ func (cfg *Config) FormatDSN() string { } if cfg.AllowCleartextPasswords { - if hasParam { - buf.WriteString("&allowCleartextPasswords=true") - } else { - hasParam = true - buf.WriteString("?allowCleartextPasswords=true") - } + writeDSNParam(&buf, &hasParam, "allowCleartextPasswords", "true") } if !cfg.AllowNativePasswords { - if hasParam { - buf.WriteString("&allowNativePasswords=false") - } else { - hasParam = true - buf.WriteString("?allowNativePasswords=false") - } + writeDSNParam(&buf, &hasParam, "allowNativePasswords", "false") } if cfg.AllowOldPasswords { - if hasParam { - buf.WriteString("&allowOldPasswords=true") - } else { - hasParam = true - buf.WriteString("?allowOldPasswords=true") - } + writeDSNParam(&buf, &hasParam, "allowOldPasswords", "true") + } + + if !cfg.CheckConnLiveness { + writeDSNParam(&buf, &hasParam, "checkConnLiveness", "false") } if cfg.ClientFoundRows { - if hasParam { - buf.WriteString("&clientFoundRows=true") - } else { - hasParam = true - buf.WriteString("?clientFoundRows=true") - } + writeDSNParam(&buf, &hasParam, "clientFoundRows", "true") } if col := cfg.Collation; col != defaultCollation && len(col) > 0 { - if hasParam { - buf.WriteString("&collation=") - } else { - hasParam = true - buf.WriteString("?collation=") - } - buf.WriteString(col) + writeDSNParam(&buf, &hasParam, "collation", col) } if cfg.ColumnsWithAlias { - if hasParam { - buf.WriteString("&columnsWithAlias=true") - } else { - hasParam = true - buf.WriteString("?columnsWithAlias=true") - } + writeDSNParam(&buf, &hasParam, "columnsWithAlias", "true") } if cfg.InterpolateParams { - if hasParam { - buf.WriteString("&interpolateParams=true") - } else { - hasParam = true - buf.WriteString("?interpolateParams=true") - } + writeDSNParam(&buf, &hasParam, "interpolateParams", "true") } if cfg.Loc != time.UTC && cfg.Loc != nil { - if hasParam { - buf.WriteString("&loc=") - } else { - hasParam = true - buf.WriteString("?loc=") - } - buf.WriteString(url.QueryEscape(cfg.Loc.String())) + writeDSNParam(&buf, &hasParam, "loc", url.QueryEscape(cfg.Loc.String())) } if cfg.MultiStatements { - if hasParam { - buf.WriteString("&multiStatements=true") - } else { - hasParam = true - buf.WriteString("?multiStatements=true") - } + writeDSNParam(&buf, &hasParam, "multiStatements", "true") } if cfg.ParseTime { - if hasParam { - buf.WriteString("&parseTime=true") - } else { - hasParam = true - buf.WriteString("?parseTime=true") - } + writeDSNParam(&buf, &hasParam, "parseTime", "true") } if cfg.ReadTimeout > 0 { - if hasParam { - buf.WriteString("&readTimeout=") - } else { - hasParam = true - buf.WriteString("?readTimeout=") - } - buf.WriteString(cfg.ReadTimeout.String()) + writeDSNParam(&buf, &hasParam, "readTimeout", cfg.ReadTimeout.String()) } if cfg.RejectReadOnly { - if hasParam { - buf.WriteString("&rejectReadOnly=true") - } else { - hasParam = true - buf.WriteString("?rejectReadOnly=true") - } + writeDSNParam(&buf, &hasParam, "rejectReadOnly", "true") } if len(cfg.ServerPubKey) > 0 { - if hasParam { - buf.WriteString("&serverPubKey=") - } else { - hasParam = true - buf.WriteString("?serverPubKey=") - } - buf.WriteString(url.QueryEscape(cfg.ServerPubKey)) + writeDSNParam(&buf, &hasParam, "serverPubKey", url.QueryEscape(cfg.ServerPubKey)) } if cfg.Timeout > 0 { - if hasParam { - buf.WriteString("&timeout=") - } else { - hasParam = true - buf.WriteString("?timeout=") - } - buf.WriteString(cfg.Timeout.String()) + writeDSNParam(&buf, &hasParam, "timeout", cfg.Timeout.String()) } if len(cfg.TLSConfig) > 0 { - if hasParam { - buf.WriteString("&tls=") - } else { - hasParam = true - buf.WriteString("?tls=") - } - buf.WriteString(url.QueryEscape(cfg.TLSConfig)) + writeDSNParam(&buf, &hasParam, "tls", url.QueryEscape(cfg.TLSConfig)) } if cfg.WriteTimeout > 0 { - if hasParam { - buf.WriteString("&writeTimeout=") - } else { - hasParam = true - buf.WriteString("?writeTimeout=") - } - buf.WriteString(cfg.WriteTimeout.String()) + writeDSNParam(&buf, &hasParam, "writeTimeout", cfg.WriteTimeout.String()) } if cfg.MaxAllowedPacket != defaultMaxAllowedPacket { - if hasParam { - buf.WriteString("&maxAllowedPacket=") - } else { - hasParam = true - buf.WriteString("?maxAllowedPacket=") - } - buf.WriteString(strconv.Itoa(cfg.MaxAllowedPacket)) - + writeDSNParam(&buf, &hasParam, "maxAllowedPacket", strconv.Itoa(cfg.MaxAllowedPacket)) } // other params @@ -316,16 +280,7 @@ func (cfg *Config) FormatDSN() string { } sort.Strings(params) for _, param := range params { - if hasParam { - buf.WriteByte('&') - } else { - hasParam = true - buf.WriteByte('?') - } - - buf.WriteString(param) - buf.WriteByte('=') - buf.WriteString(url.QueryEscape(cfg.Params[param])) + writeDSNParam(&buf, &hasParam, param, url.QueryEscape(cfg.Params[param])) } } @@ -452,6 +407,14 @@ func parseDSNParams(cfg *Config, params string) (err error) { return errors.New("invalid bool value: " + value) } + // Check connections for Liveness before using them + case "checkConnLiveness": + var isBool bool + cfg.CheckConnLiveness, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + // Switch "rowsAffected" mode case "clientFoundRows": var isBool bool @@ -531,13 +494,7 @@ func parseDSNParams(cfg *Config, params string) (err error) { if err != nil { return fmt.Errorf("invalid value for server pub key name: %v", err) } - - if pubKey := getServerPubKey(name); pubKey != nil { - cfg.ServerPubKey = name - cfg.pubKey = pubKey - } else { - return errors.New("invalid value / unknown server pub key name: " + name) - } + cfg.ServerPubKey = name // Strict mode case "strict": @@ -556,25 +513,17 @@ func parseDSNParams(cfg *Config, params string) (err error) { if isBool { if boolValue { cfg.TLSConfig = "true" - cfg.tls = &tls.Config{} } else { cfg.TLSConfig = "false" } } else if vl := strings.ToLower(value); vl == "skip-verify" || vl == "preferred" { cfg.TLSConfig = vl - cfg.tls = &tls.Config{InsecureSkipVerify: true} } else { name, err := url.QueryUnescape(value) if err != nil { return fmt.Errorf("invalid value for TLS config name: %v", err) } - - if tlsConfig := getTLSConfigClone(name); tlsConfig != nil { - cfg.TLSConfig = name - cfg.tls = tlsConfig - } else { - return errors.New("invalid value / unknown config name: " + name) - } + cfg.TLSConfig = name } // I/O write Timeout diff --git a/github.com/go-sql-driver/mysql/go.mod b/github.com/go-sql-driver/mysql/go.mod new file mode 100644 index 0000000000..fffbf6a909 --- /dev/null +++ b/github.com/go-sql-driver/mysql/go.mod @@ -0,0 +1,3 @@ +module github.com/go-sql-driver/mysql + +go 1.10 diff --git a/github.com/go-sql-driver/mysql/nulltime.go b/github.com/go-sql-driver/mysql/nulltime.go new file mode 100644 index 0000000000..afa8a89e9a --- /dev/null +++ b/github.com/go-sql-driver/mysql/nulltime.go @@ -0,0 +1,50 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "database/sql/driver" + "fmt" + "time" +) + +// Scan implements the Scanner interface. +// The value type must be time.Time or string / []byte (formatted time-string), +// otherwise Scan fails. +func (nt *NullTime) Scan(value interface{}) (err error) { + if value == nil { + nt.Time, nt.Valid = time.Time{}, false + return + } + + switch v := value.(type) { + case time.Time: + nt.Time, nt.Valid = v, true + return + case []byte: + nt.Time, err = parseDateTime(string(v), time.UTC) + nt.Valid = (err == nil) + return + case string: + nt.Time, err = parseDateTime(v, time.UTC) + nt.Valid = (err == nil) + return + } + + nt.Valid = false + return fmt.Errorf("Can't convert %T to time.Time", value) +} + +// Value implements the driver Valuer interface. +func (nt NullTime) Value() (driver.Value, error) { + if !nt.Valid { + return nil, nil + } + return nt.Time, nil +} diff --git a/github.com/go-sql-driver/mysql/nulltime_go113.go b/github.com/go-sql-driver/mysql/nulltime_go113.go new file mode 100644 index 0000000000..c392594dd4 --- /dev/null +++ b/github.com/go-sql-driver/mysql/nulltime_go113.go @@ -0,0 +1,31 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build go1.13 + +package mysql + +import ( + "database/sql" +) + +// NullTime represents a time.Time that may be NULL. +// NullTime implements the Scanner interface so +// it can be used as a scan destination: +// +// var nt NullTime +// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt) +// ... +// if nt.Valid { +// // use nt.Time +// } else { +// // NULL value +// } +// +// This NullTime implementation is not driver-specific +type NullTime sql.NullTime diff --git a/github.com/go-sql-driver/mysql/nulltime_legacy.go b/github.com/go-sql-driver/mysql/nulltime_legacy.go new file mode 100644 index 0000000000..86d159d441 --- /dev/null +++ b/github.com/go-sql-driver/mysql/nulltime_legacy.go @@ -0,0 +1,34 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build !go1.13 + +package mysql + +import ( + "time" +) + +// NullTime represents a time.Time that may be NULL. +// NullTime implements the Scanner interface so +// it can be used as a scan destination: +// +// var nt NullTime +// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt) +// ... +// if nt.Valid { +// // use nt.Time +// } else { +// // NULL value +// } +// +// This NullTime implementation is not driver-specific +type NullTime struct { + Time time.Time + Valid bool // Valid is true if Time is not NULL +} diff --git a/github.com/go-sql-driver/mysql/packets.go b/github.com/go-sql-driver/mysql/packets.go index 5e0853767d..82ad7a2009 100644 --- a/github.com/go-sql-driver/mysql/packets.go +++ b/github.com/go-sql-driver/mysql/packets.go @@ -96,6 +96,35 @@ func (mc *mysqlConn) writePacket(data []byte) error { return ErrPktTooLarge } + // Perform a stale connection check. We only perform this check for + // the first query on a connection that has been checked out of the + // connection pool: a fresh connection from the pool is more likely + // to be stale, and it has not performed any previous writes that + // could cause data corruption, so it's safe to return ErrBadConn + // if the check fails. + if mc.reset { + mc.reset = false + conn := mc.netConn + if mc.rawConn != nil { + conn = mc.rawConn + } + var err error + // If this connection has a ReadTimeout which we've been setting on + // reads, reset it to its default value before we attempt a non-blocking + // read, otherwise the scheduler will just time us out before we can read + if mc.cfg.ReadTimeout != 0 { + err = conn.SetReadDeadline(time.Time{}) + } + if err == nil && mc.cfg.CheckConnLiveness { + err = connCheck(conn) + } + if err != nil { + errLog.Print("closing bad idle connection: ", err) + mc.Close() + return driver.ErrBadConn + } + } + for { var size int if pktLen >= maxPacketSize { @@ -332,6 +361,7 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string if err := tlsConn.Handshake(); err != nil { return err } + mc.rawConn = mc.netConn mc.netConn = tlsConn mc.buf.nc = tlsConn } @@ -991,6 +1021,22 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { ) } + case uint64: + paramTypes[i+i] = byte(fieldTypeLongLong) + paramTypes[i+i+1] = 0x80 // type is unsigned + + if cap(paramValues)-len(paramValues)-8 >= 0 { + paramValues = paramValues[:len(paramValues)+8] + binary.LittleEndian.PutUint64( + paramValues[len(paramValues)-8:], + uint64(v), + ) + } else { + paramValues = append(paramValues, + uint64ToBytes(uint64(v))..., + ) + } + case float64: paramTypes[i+i] = byte(fieldTypeDouble) paramTypes[i+i+1] = 0x00 diff --git a/github.com/go-sql-driver/mysql/rows.go b/github.com/go-sql-driver/mysql/rows.go index d3b1e28221..888bdb5f0a 100644 --- a/github.com/go-sql-driver/mysql/rows.go +++ b/github.com/go-sql-driver/mysql/rows.go @@ -111,6 +111,13 @@ func (rows *mysqlRows) Close() (err error) { return err } + // flip the buffer for this connection if we need to drain it. + // note that for a successful query (i.e. one where rows.next() + // has been called until it returns false), `rows.mc` will be nil + // by the time the user calls `(*Rows).Close`, so we won't reach this + // see: https://github.com/golang/go/commit/651ddbdb5056ded455f47f9c494c67b389622a47 + mc.buf.flip() + // Remove unread packets from stream if !rows.rs.done { err = mc.readUntilEOF() diff --git a/github.com/go-sql-driver/mysql/statement.go b/github.com/go-sql-driver/mysql/statement.go index ce7fe4cd04..f7e370939a 100644 --- a/github.com/go-sql-driver/mysql/statement.go +++ b/github.com/go-sql-driver/mysql/statement.go @@ -13,7 +13,6 @@ import ( "fmt" "io" "reflect" - "strconv" ) type mysqlStmt struct { @@ -164,14 +163,8 @@ func (c converter) ConvertValue(v interface{}) (driver.Value, error) { } case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: return rv.Int(), nil - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32: - return int64(rv.Uint()), nil - case reflect.Uint64: - u64 := rv.Uint() - if u64 >= 1<<63 { - return strconv.FormatUint(u64, 10), nil - } - return int64(u64), nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return rv.Uint(), nil case reflect.Float32, reflect.Float64: return rv.Float(), nil case reflect.Bool: diff --git a/github.com/go-sql-driver/mysql/utils.go b/github.com/go-sql-driver/mysql/utils.go index cb3650bb9b..9552e80b5a 100644 --- a/github.com/go-sql-driver/mysql/utils.go +++ b/github.com/go-sql-driver/mysql/utils.go @@ -56,7 +56,7 @@ var ( // db, err := sql.Open("mysql", "user@tcp(localhost:3306)/test?tls=custom") // func RegisterTLSConfig(key string, config *tls.Config) error { - if _, isBool := readBool(key); isBool || strings.ToLower(key) == "skip-verify" { + if _, isBool := readBool(key); isBool || strings.ToLower(key) == "skip-verify" || strings.ToLower(key) == "preferred" { return fmt.Errorf("key '%s' is reserved", key) } @@ -106,60 +106,6 @@ func readBool(input string) (value bool, valid bool) { * Time related utils * ******************************************************************************/ -// NullTime represents a time.Time that may be NULL. -// NullTime implements the Scanner interface so -// it can be used as a scan destination: -// -// var nt NullTime -// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt) -// ... -// if nt.Valid { -// // use nt.Time -// } else { -// // NULL value -// } -// -// This NullTime implementation is not driver-specific -type NullTime struct { - Time time.Time - Valid bool // Valid is true if Time is not NULL -} - -// Scan implements the Scanner interface. -// The value type must be time.Time or string / []byte (formatted time-string), -// otherwise Scan fails. -func (nt *NullTime) Scan(value interface{}) (err error) { - if value == nil { - nt.Time, nt.Valid = time.Time{}, false - return - } - - switch v := value.(type) { - case time.Time: - nt.Time, nt.Valid = v, true - return - case []byte: - nt.Time, err = parseDateTime(string(v), time.UTC) - nt.Valid = (err == nil) - return - case string: - nt.Time, err = parseDateTime(v, time.UTC) - nt.Valid = (err == nil) - return - } - - nt.Valid = false - return fmt.Errorf("Can't convert %T to time.Time", value) -} - -// Value implements the driver Valuer interface. -func (nt NullTime) Value() (driver.Value, error) { - if !nt.Valid { - return nil, nil - } - return nt.Time, nil -} - func parseDateTime(str string, loc *time.Location) (t time.Time, err error) { base := "0000-00-00 00:00:00.0000000" switch len(str) { @@ -684,7 +630,7 @@ type atomicBool struct { value uint32 } -// IsSet returns wether the current boolean value is true +// IsSet returns whether the current boolean value is true func (ab *atomicBool) IsSet() bool { return atomic.LoadUint32(&ab.value) > 0 } @@ -698,7 +644,7 @@ func (ab *atomicBool) Set(value bool) { } } -// TrySet sets the value of the bool and returns wether the value changed +// TrySet sets the value of the bool and returns whether the value changed func (ab *atomicBool) TrySet(value bool) bool { if value { return atomic.SwapUint32(&ab.value, 1) == 0 diff --git a/github.com/jmespath/go-jmespath/.travis.yml b/github.com/jmespath/go-jmespath/.travis.yml index 1f98077570..730c7fa51b 100644 --- a/github.com/jmespath/go-jmespath/.travis.yml +++ b/github.com/jmespath/go-jmespath/.travis.yml @@ -3,7 +3,15 @@ language: go sudo: false go: - - 1.4 + - 1.5.x + - 1.6.x + - 1.7.x + - 1.8.x + - 1.9.x + - 1.10.x + - 1.11.x + - 1.12.x + - 1.13.x install: go get -v -t ./... script: make test diff --git a/github.com/jmespath/go-jmespath/README.md b/github.com/jmespath/go-jmespath/README.md index 187ef676dc..110ad79997 100644 --- a/github.com/jmespath/go-jmespath/README.md +++ b/github.com/jmespath/go-jmespath/README.md @@ -4,4 +4,84 @@ -See http://jmespath.org for more info. +go-jmespath is a GO implementation of JMESPath, +which is a query language for JSON. It will take a JSON +document and transform it into another JSON document +through a JMESPath expression. + +Using go-jmespath is really easy. There's a single function +you use, `jmespath.search`: + + +```go +> import "github.com/jmespath/go-jmespath" +> +> var jsondata = []byte(`{"foo": {"bar": {"baz": [0, 1, 2, 3, 4]}}}`) // your data +> var data interface{} +> err := json.Unmarshal(jsondata, &data) +> result, err := jmespath.Search("foo.bar.baz[2]", data) +result = 2 +``` + +In the example we gave the ``search`` function input data of +`{"foo": {"bar": {"baz": [0, 1, 2, 3, 4]}}}` as well as the JMESPath +expression `foo.bar.baz[2]`, and the `search` function evaluated +the expression against the input data to produce the result ``2``. + +The JMESPath language can do a lot more than select an element +from a list. Here are a few more examples: + +```go +> var jsondata = []byte(`{"foo": {"bar": {"baz": [0, 1, 2, 3, 4]}}}`) // your data +> var data interface{} +> err := json.Unmarshal(jsondata, &data) +> result, err := jmespath.search("foo.bar", data) +result = { "baz": [ 0, 1, 2, 3, 4 ] } + + +> var jsondata = []byte(`{"foo": [{"first": "a", "last": "b"}, + {"first": "c", "last": "d"}]}`) // your data +> var data interface{} +> err := json.Unmarshal(jsondata, &data) +> result, err := jmespath.search({"foo[*].first", data) +result [ 'a', 'c' ] + + +> var jsondata = []byte(`{"foo": [{"age": 20}, {"age": 25}, + {"age": 30}, {"age": 35}, + {"age": 40}]}`) // your data +> var data interface{} +> err := json.Unmarshal(jsondata, &data) +> result, err := jmespath.search("foo[?age > `30`]") +result = [ { age: 35 }, { age: 40 } ] +``` + +You can also pre-compile your query. This is usefull if +you are going to run multiple searches with it: + +```go + > var jsondata = []byte(`{"foo": "bar"}`) + > var data interface{} + > err := json.Unmarshal(jsondata, &data) + > precompiled, err := Compile("foo") + > if err != nil{ + > // ... handle the error + > } + > result, err := precompiled.Search(data) + result = "bar" +``` + +## More Resources + +The example above only show a small amount of what +a JMESPath expression can do. If you want to take a +tour of the language, the *best* place to go is the +[JMESPath Tutorial](http://jmespath.org/tutorial.html). + +One of the best things about JMESPath is that it is +implemented in many different programming languages including +python, ruby, php, lua, etc. To see a complete list of libraries, +check out the [JMESPath libraries page](http://jmespath.org/libraries.html). + +And finally, the full JMESPath specification can be found +on the [JMESPath site](http://jmespath.org/specification.html). diff --git a/github.com/jmespath/go-jmespath/api.go b/github.com/jmespath/go-jmespath/api.go index 8e26ffeecf..010efe9bfb 100644 --- a/github.com/jmespath/go-jmespath/api.go +++ b/github.com/jmespath/go-jmespath/api.go @@ -2,7 +2,7 @@ package jmespath import "strconv" -// JMESPath is the epresentation of a compiled JMES path query. A JMESPath is +// JMESPath is the representation of a compiled JMES path query. A JMESPath is // safe for concurrent use by multiple goroutines. type JMESPath struct { ast ASTNode diff --git a/github.com/jmespath/go-jmespath/go.mod b/github.com/jmespath/go-jmespath/go.mod new file mode 100644 index 0000000000..aa1e3f1c9f --- /dev/null +++ b/github.com/jmespath/go-jmespath/go.mod @@ -0,0 +1,5 @@ +module github.com/jmespath/go-jmespath + +go 1.14 + +require github.com/stretchr/testify v1.5.1 diff --git a/github.com/jmespath/go-jmespath/go.sum b/github.com/jmespath/go-jmespath/go.sum new file mode 100644 index 0000000000..331fa69822 --- /dev/null +++ b/github.com/jmespath/go-jmespath/go.sum @@ -0,0 +1,11 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/github.com/jmespath/go-jmespath/parser.go b/github.com/jmespath/go-jmespath/parser.go index 1240a17552..4abc303ab4 100644 --- a/github.com/jmespath/go-jmespath/parser.go +++ b/github.com/jmespath/go-jmespath/parser.go @@ -137,7 +137,7 @@ func (p *Parser) Parse(expression string) (ASTNode, error) { } if p.current() != tEOF { return ASTNode{}, p.syntaxError(fmt.Sprintf( - "Unexpected token at the end of the expresssion: %s", p.current())) + "Unexpected token at the end of the expression: %s", p.current())) } return parsed, nil } diff --git a/google.golang.org/appengine/cloudsql/cloudsql.go b/google.golang.org/appengine/cloudsql/cloudsql.go deleted file mode 100644 index 7b27e6b12d..0000000000 --- a/google.golang.org/appengine/cloudsql/cloudsql.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2013 Google Inc. All rights reserved. -// Use of this source code is governed by the Apache 2.0 -// license that can be found in the LICENSE file. - -/* -Package cloudsql exposes access to Google Cloud SQL databases. - -This package does not work in App Engine "flexible environment". - -This package is intended for MySQL drivers to make App Engine-specific -connections. Applications should use this package through database/sql: -Select a pure Go MySQL driver that supports this package, and use sql.Open -with protocol "cloudsql" and an address of the Cloud SQL instance. - -A Go MySQL driver that has been tested to work well with Cloud SQL -is the go-sql-driver: - import "database/sql" - import _ "github.com/go-sql-driver/mysql" - - db, err := sql.Open("mysql", "user@cloudsql(project-id:instance-name)/dbname") - - -Another driver that works well with Cloud SQL is the mymysql driver: - import "database/sql" - import _ "github.com/ziutek/mymysql/godrv" - - db, err := sql.Open("mymysql", "cloudsql:instance-name*dbname/user/password") - - -Using either of these drivers, you can perform a standard SQL query. -This example assumes there is a table named 'users' with -columns 'first_name' and 'last_name': - - rows, err := db.Query("SELECT first_name, last_name FROM users") - if err != nil { - log.Errorf(ctx, "db.Query: %v", err) - } - defer rows.Close() - - for rows.Next() { - var firstName string - var lastName string - if err := rows.Scan(&firstName, &lastName); err != nil { - log.Errorf(ctx, "rows.Scan: %v", err) - continue - } - log.Infof(ctx, "First: %v - Last: %v", firstName, lastName) - } - if err := rows.Err(); err != nil { - log.Errorf(ctx, "Row error: %v", err) - } -*/ -package cloudsql - -import ( - "net" -) - -// Dial connects to the named Cloud SQL instance. -func Dial(instance string) (net.Conn, error) { - return connect(instance) -} diff --git a/google.golang.org/appengine/cloudsql/cloudsql_classic.go b/google.golang.org/appengine/cloudsql/cloudsql_classic.go deleted file mode 100644 index af62dba146..0000000000 --- a/google.golang.org/appengine/cloudsql/cloudsql_classic.go +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2013 Google Inc. All rights reserved. -// Use of this source code is governed by the Apache 2.0 -// license that can be found in the LICENSE file. - -// +build appengine - -package cloudsql - -import ( - "net" - - "appengine/cloudsql" -) - -func connect(instance string) (net.Conn, error) { - return cloudsql.Dial(instance) -} diff --git a/google.golang.org/appengine/cloudsql/cloudsql_vm.go b/google.golang.org/appengine/cloudsql/cloudsql_vm.go deleted file mode 100644 index 90fa7b31e0..0000000000 --- a/google.golang.org/appengine/cloudsql/cloudsql_vm.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2013 Google Inc. All rights reserved. -// Use of this source code is governed by the Apache 2.0 -// license that can be found in the LICENSE file. - -// +build !appengine - -package cloudsql - -import ( - "errors" - "net" -) - -func connect(instance string) (net.Conn, error) { - return nil, errors.New(`cloudsql: not supported in App Engine "flexible environment"`) -} diff --git a/modules.txt b/modules.txt index 183acec842..64e6eae711 100644 --- a/modules.txt +++ b/modules.txt @@ -107,9 +107,10 @@ github.com/apache/thrift/lib/go/thrift # github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e ## explicit github.com/armon/circbuf -# github.com/aws/aws-sdk-go v1.16.19 +# github.com/aws/aws-sdk-go v1.33.8 ## explicit github.com/aws/aws-sdk-go/aws +github.com/aws/aws-sdk-go/aws/arn github.com/aws/aws-sdk-go/aws/awserr github.com/aws/aws-sdk-go/aws/awsutil github.com/aws/aws-sdk-go/aws/client @@ -127,24 +128,34 @@ github.com/aws/aws-sdk-go/aws/endpoints github.com/aws/aws-sdk-go/aws/request github.com/aws/aws-sdk-go/aws/session github.com/aws/aws-sdk-go/aws/signer/v4 +github.com/aws/aws-sdk-go/internal/context github.com/aws/aws-sdk-go/internal/ini github.com/aws/aws-sdk-go/internal/s3err github.com/aws/aws-sdk-go/internal/sdkio +github.com/aws/aws-sdk-go/internal/sdkmath github.com/aws/aws-sdk-go/internal/sdkrand github.com/aws/aws-sdk-go/internal/sdkuri github.com/aws/aws-sdk-go/internal/shareddefaults +github.com/aws/aws-sdk-go/internal/strings +github.com/aws/aws-sdk-go/internal/sync/singleflight +github.com/aws/aws-sdk-go/private/checksum github.com/aws/aws-sdk-go/private/protocol github.com/aws/aws-sdk-go/private/protocol/eventstream github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi +github.com/aws/aws-sdk-go/private/protocol/json/jsonutil +github.com/aws/aws-sdk-go/private/protocol/jsonrpc github.com/aws/aws-sdk-go/private/protocol/query github.com/aws/aws-sdk-go/private/protocol/query/queryutil github.com/aws/aws-sdk-go/private/protocol/rest github.com/aws/aws-sdk-go/private/protocol/restxml github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil +github.com/aws/aws-sdk-go/service/kms github.com/aws/aws-sdk-go/service/s3 +github.com/aws/aws-sdk-go/service/s3/internal/arn github.com/aws/aws-sdk-go/service/s3/s3iface github.com/aws/aws-sdk-go/service/s3/s3manager github.com/aws/aws-sdk-go/service/sts +github.com/aws/aws-sdk-go/service/sts/stsiface # github.com/axiomhq/hyperloglog v0.0.0-20181223111420-4b99d0c2c99e ## explicit github.com/axiomhq/hyperloglog @@ -358,7 +369,7 @@ github.com/go-logfmt/logfmt ## explicit github.com/go-ole/go-ole github.com/go-ole/go-ole/oleutil -# github.com/go-sql-driver/mysql v1.4.1-0.20181218123637-c45f530f8e7f +# github.com/go-sql-driver/mysql v1.5.0 ## explicit github.com/go-sql-driver/mysql # github.com/gofrs/uuid v3.3.0+incompatible @@ -587,7 +598,7 @@ github.com/jcmturner/gokrb5/v8/types # github.com/jcmturner/rpc/v2 v2.0.2 github.com/jcmturner/rpc/v2/mstypes github.com/jcmturner/rpc/v2/ndr -# github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af +# github.com/jmespath/go-jmespath v0.3.0 github.com/jmespath/go-jmespath # github.com/jordanlewis/gcassert v0.0.0-20200706043056-bf61eb72ee48 ## explicit @@ -1006,7 +1017,6 @@ google.golang.org/api/transport/http google.golang.org/api/transport/http/internal/propagation # google.golang.org/appengine v1.4.0 google.golang.org/appengine -google.golang.org/appengine/cloudsql google.golang.org/appengine/internal google.golang.org/appengine/internal/app_identity google.golang.org/appengine/internal/base From a94eda3e9b77ae2487ad2e1238528191cb248c0b Mon Sep 17 00:00:00 2001 From: Matt Jibson Date: Mon, 27 Jul 2020 16:37:38 -0600 Subject: [PATCH 18/49] lib/pq 1.8.0 --- github.com/lib/pq/README.md | 5 +---- github.com/lib/pq/conn.go | 8 ++++---- github.com/lib/pq/doc.go | 9 +++++++-- modules.txt | 2 +- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/github.com/lib/pq/README.md b/github.com/lib/pq/README.md index ecd01939b5..c972a86a57 100644 --- a/github.com/lib/pq/README.md +++ b/github.com/lib/pq/README.md @@ -19,10 +19,7 @@ * Unix socket support * Notifications: `LISTEN`/`NOTIFY` * pgpass support - -## Optional Features - -* GSS (Kerberos) auth (to use, see GoDoc) +* GSS (Kerberos) auth ## Tests diff --git a/github.com/lib/pq/conn.go b/github.com/lib/pq/conn.go index b3ab14d3cc..f313c14986 100644 --- a/github.com/lib/pq/conn.go +++ b/github.com/lib/pq/conn.go @@ -1074,9 +1074,9 @@ func isDriverSetting(key string) bool { return true case "binary_parameters": return true - case "service": + case "krbsrvname": return true - case "spn": + case "krbspn": return true default: return false @@ -1168,13 +1168,13 @@ func (cn *conn) auth(r *readBuf, o values) { var token []byte - if spn, ok := o["spn"]; ok { + if spn, ok := o["krbspn"]; ok { // Use the supplied SPN if provided.. token, err = cli.GetInitTokenFromSpn(spn) } else { // Allow the kerberos service name to be overridden service := "postgres" - if val, ok := o["service"]; ok { + if val, ok := o["krbsrvname"]; ok { service = val } diff --git a/github.com/lib/pq/doc.go b/github.com/lib/pq/doc.go index 78c670b1d9..b57184801b 100644 --- a/github.com/lib/pq/doc.go +++ b/github.com/lib/pq/doc.go @@ -57,8 +57,6 @@ supported: * sslkey - Key file location. The file must contain PEM encoded data. * sslrootcert - The location of the root certificate file. The file must contain PEM encoded data. - * spn - Configures GSS (Kerberos) SPN. - * service - GSS (Kerberos) service name to use when constructing the SPN (default is `postgres`). Valid values for sslmode are: @@ -259,5 +257,12 @@ package: This package is in a separate module so that users who don't need Kerberos don't have to download unnecessary dependencies. +When imported, additional connection string parameters are supported: + + * krbsrvname - GSS (Kerberos) service name when constructing the + SPN (default is `postgres`). This will be combined with the host + to form the full SPN: `krbsrvname/host`. + * krbspn - GSS (Kerberos) SPN. This takes priority over + `krbsrvname` if present. */ package pq diff --git a/modules.txt b/modules.txt index 64e6eae711..608668c1e9 100644 --- a/modules.txt +++ b/modules.txt @@ -638,7 +638,7 @@ github.com/kr/text ## explicit github.com/leanovate/gopter github.com/leanovate/gopter/prop -# github.com/lib/pq v1.7.1 +# github.com/lib/pq v1.8.0 ## explicit github.com/lib/pq github.com/lib/pq/oid From 009db0bd136451702c0ed95df006c9381aee201a Mon Sep 17 00:00:00 2001 From: Jackson Owens Date: Wed, 29 Jul 2020 16:28:48 -0400 Subject: [PATCH 19/49] bump Pebble to cfd19d33cdca cfd19d33 db: compute file count and size metrics incrementally e34d759c internal/rangedel: format keys in Fragmenter panics 06a3fb73 internal/mkbench: fix dropped walkfunc error 61c629af internal/manifest: unexport LevelMetadata files 8fdce959 vfs: Add FS.GetFreeSpace, IsNoSpaceError 24bd87a5 db: fix tombstone elision, grandparent limit bug a9d18a23 *: Update comment on Ingest() on callers needing to Sync() 42cb6336 internal/manifest: take a LevelSlice in L0Sublevels.PickBaseCompaction 887d40ba db: use manifest.NewVersion constructor in tests 2f3fb56c internal/rangedel: Skip range tombstones outside a file's bounds 1a5f7680 record: propagate WAL sync errors to queued writers cc76c76e db: update L0 compaction-picking to use manifest.LevelIterator 69961893 pebble: fix dropped test error 5edc4f5b db: use LevelSlice.SizeSum where applicable d524d342 db: use manifest.LevelIterator in compaction, levelIter 049c566a db: capture stack trace of closed call for ErrClosed --- github.com/cockroachdb/pebble/batch.go | 21 +- github.com/cockroachdb/pebble/compaction.go | 308 ++++++++++-------- .../cockroachdb/pebble/compaction_picker.go | 217 ++++++------ github.com/cockroachdb/pebble/db.go | 76 ++--- .../cockroachdb/pebble/flush_external.go | 12 +- github.com/cockroachdb/pebble/get_iter.go | 7 +- github.com/cockroachdb/pebble/ingest.go | 23 +- .../pebble/internal/manifest/l0_sublevels.go | 36 +- .../internal/manifest/level_metadata.go | 59 +++- .../pebble/internal/manifest/version.go | 61 +++- .../pebble/internal/manifest/version_edit.go | 28 +- .../pebble/internal/rangedel/fragmenter.go | 19 +- .../pebble/internal/rangedel/truncate.go | 16 +- .../pebble/internal/record/log_writer.go | 26 +- .../cockroachdb/pebble/level_checker.go | 67 ++-- github.com/cockroachdb/pebble/level_iter.go | 107 +++--- github.com/cockroachdb/pebble/mem_table.go | 5 +- github.com/cockroachdb/pebble/metrics.go | 10 +- github.com/cockroachdb/pebble/open.go | 5 +- .../cockroachdb/pebble/sstable/reader.go | 8 +- github.com/cockroachdb/pebble/table_cache.go | 9 +- github.com/cockroachdb/pebble/table_stats.go | 18 +- github.com/cockroachdb/pebble/version_set.go | 39 ++- .../cockroachdb/pebble/vfs/disk_usage_unix.go | 19 ++ .../pebble/vfs/disk_usage_windows.go | 21 ++ .../cockroachdb/pebble/vfs/errors_unix.go | 19 ++ .../cockroachdb/pebble/vfs/errors_windows.go | 20 ++ github.com/cockroachdb/pebble/vfs/mem_fs.go | 5 + github.com/cockroachdb/pebble/vfs/vfs.go | 4 + golang.org/x/sys/unix/mkerrors.sh | 5 + golang.org/x/sys/unix/syscall_bsd.go | 17 + golang.org/x/sys/unix/syscall_linux.go | 12 + golang.org/x/sys/unix/zerrors_freebsd_386.go | 6 + .../x/sys/unix/zerrors_freebsd_amd64.go | 6 + golang.org/x/sys/unix/zerrors_freebsd_arm.go | 6 + .../x/sys/unix/zerrors_freebsd_arm64.go | 6 + golang.org/x/sys/unix/zerrors_netbsd_386.go | 6 + golang.org/x/sys/unix/zerrors_netbsd_amd64.go | 6 + golang.org/x/sys/unix/zerrors_netbsd_arm.go | 6 + golang.org/x/sys/unix/zerrors_netbsd_arm64.go | 6 + golang.org/x/sys/unix/zerrors_openbsd_386.go | 7 + .../x/sys/unix/zerrors_openbsd_amd64.go | 7 + golang.org/x/sys/unix/zerrors_openbsd_arm.go | 7 + .../x/sys/unix/zerrors_openbsd_arm64.go | 7 + golang.org/x/sys/unix/zsyscall_linux.go | 46 +++ modules.txt | 4 +- 46 files changed, 911 insertions(+), 514 deletions(-) create mode 100644 github.com/cockroachdb/pebble/vfs/disk_usage_unix.go create mode 100644 github.com/cockroachdb/pebble/vfs/disk_usage_windows.go create mode 100644 github.com/cockroachdb/pebble/vfs/errors_unix.go create mode 100644 github.com/cockroachdb/pebble/vfs/errors_windows.go diff --git a/github.com/cockroachdb/pebble/batch.go b/github.com/cockroachdb/pebble/batch.go index 299fd83291..60db3725ab 100644 --- a/github.com/cockroachdb/pebble/batch.go +++ b/github.com/cockroachdb/pebble/batch.go @@ -184,6 +184,7 @@ type Batch struct { // memtables. data []byte cmp Compare + formatKey base.FormatKey abbreviatedKey AbbreviatedKey memTableSize uint32 @@ -251,6 +252,7 @@ func newBatch(db *DB) *Batch { func newIndexedBatch(db *DB, comparer *Comparer) *Batch { i := indexedBatchPool.Get().(*indexedBatch) i.batch.cmp = comparer.Compare + i.batch.formatKey = comparer.FormatKey i.batch.abbreviatedKey = comparer.AbbreviatedKey i.batch.db = db i.batch.index = &i.index @@ -275,6 +277,7 @@ func (b *Batch) release() { // complains. b.Reset() b.cmp = nil + b.formatKey = nil b.abbreviatedKey = nil b.memTableSize = 0 @@ -702,7 +705,8 @@ func (b *Batch) newRangeDelIter(o *IterOptions) internalIterator { // tombstone is added to the batch. if b.tombstones == nil { frag := &rangedel.Fragmenter{ - Cmp: b.cmp, + Cmp: b.cmp, + Format: b.formatKey, Emit: func(fragmented []rangedel.Tombstone) { b.tombstones = append(b.tombstones, fragmented...) }, @@ -1029,8 +1033,9 @@ type flushableBatchEntry struct { // flushableBatch wraps an existing batch and provides the interfaces needed // for making the batch flushable (i.e. able to mimic a memtable). type flushableBatch struct { - cmp Compare - data []byte + cmp Compare + formatKey base.FormatKey + data []byte // The base sequence number for the entries in the batch. This is the same // value as Batch.seqNum() and is cached here for performance. @@ -1059,9 +1064,10 @@ var _ flushable = (*flushableBatch)(nil) // of the batch data. func newFlushableBatch(batch *Batch, comparer *Comparer) *flushableBatch { b := &flushableBatch{ - data: batch.data, - cmp: comparer.Compare, - offsets: make([]flushableBatchEntry, 0, batch.Count()), + data: batch.data, + cmp: comparer.Compare, + formatKey: comparer.FormatKey, + offsets: make([]flushableBatchEntry, 0, batch.Count()), } if b.data != nil { // Note that this sequence number is not correct when this batch has not @@ -1110,7 +1116,8 @@ func newFlushableBatch(batch *Batch, comparer *Comparer) *flushableBatch { if len(rangeDelOffsets) > 0 { frag := &rangedel.Fragmenter{ - Cmp: b.cmp, + Cmp: b.cmp, + Format: b.formatKey, Emit: func(fragmented []rangedel.Tombstone) { b.tombstones = append(b.tombstones, fragmented...) }, diff --git a/github.com/cockroachdb/pebble/compaction.go b/github.com/cockroachdb/pebble/compaction.go index 5ddbf5e18b..776f79bf90 100644 --- a/github.com/cockroachdb/pebble/compaction.go +++ b/github.com/cockroachdb/pebble/compaction.go @@ -46,14 +46,6 @@ func maxGrandparentOverlapBytes(opts *Options, level int) uint64 { return uint64(10 * opts.Level(level).TargetFileSize) } -// totalSize returns the total size of all the files in f. -func totalSize(f []*fileMetadata) (size uint64) { - for _, x := range f { - size += x.Size - } - return size -} - // noCloseIter wraps around an internal iterator, intercepting and eliding // calls to Close. It is used during compaction to ensure that rangeDelIters // are not closed prematurely. @@ -71,7 +63,7 @@ type userKeyRange struct { type compactionLevel struct { level int - files []*fileMetadata + files manifest.LevelSlice } type compactionKind string @@ -175,8 +167,9 @@ func (c *compaction) makeInfo(jobID int) CompactionInfo { Input: make([]LevelInfo, 0, len(c.inputs)), } for _, cl := range c.inputs { - inputInfo := LevelInfo{Level: cl.level, Tables: make([]TableInfo, 0, len(cl.files))} - for _, m := range cl.files { + inputInfo := LevelInfo{Level: cl.level, Tables: nil} + iter := cl.files.Iter() + for m := iter.First(); m != nil; m = iter.Next() { inputInfo.Tables = append(inputInfo.Tables, m.TableInfo()) } info.Input = append(info.Input, inputInfo) @@ -198,14 +191,13 @@ func (c *compaction) makeInfo(jobID int) CompactionInfo { return info } -func newCompaction( - pc *pickedCompaction, opts *Options, bytesCompacted *uint64, -) *compaction { +func newCompaction(pc *pickedCompaction, opts *Options, bytesCompacted *uint64) *compaction { c := &compaction{ kind: compactionKindDefault, cmp: pc.cmp, formatKey: opts.Comparer.FormatKey, score: pc.score, + inputs: pc.inputs, smallest: pc.smallest, largest: pc.largest, logger: opts.Logger, @@ -215,13 +207,6 @@ func newCompaction( maxExpandedBytes: pc.maxOverlapBytes, atomicBytesIterated: bytesCompacted, } - for _, in := range pc.inputs { - c.inputs = append(c.inputs, compactionLevel{ - level: in.level, - files: in.files.Collect(), - }) - } - c.startLevel = &c.inputs[0] c.outputLevel = &c.inputs[1] @@ -237,16 +222,14 @@ func newCompaction( // level to the next. We avoid such a move if there is lots of overlapping // grandparent data. Otherwise, the move could create a parent file that // will require a very expensive merge later on. - if len(c.startLevel.files) == 1 && len(c.outputLevel.files) == 0 && + if c.outputLevel.files.Empty() && c.startLevel.files.Len() == 1 && c.grandparents.SizeSum() <= c.maxOverlapBytes { c.kind = compactionKindMove } return c } -func newDeleteOnlyCompaction( - opts *Options, cur *version, inputs []compactionLevel, -) *compaction { +func newDeleteOnlyCompaction(opts *Options, cur *version, inputs []compactionLevel) *compaction { c := &compaction{ kind: compactionKindDeleteOnly, cmp: opts.Comparer.Compare, @@ -259,7 +242,7 @@ func newDeleteOnlyCompaction( // Set c.smallest, c.largest. files := make([]manifest.LevelIterator, 0, len(inputs)) for _, in := range inputs { - files = append(files, manifest.NewLevelSlice(in.files).Iter()) + files = append(files, in.files.Iter()) } c.smallest, c.largest = manifest.KeyRange(opts.Comparer.Compare, files...) return c @@ -505,67 +488,6 @@ func (c *compaction) elideRangeTombstone(start, end []byte) bool { return lower >= upper } -// atomicUnitBounds returns the bounds of the atomic compaction unit containing -// the specified sstable (identified by a pointer to its fileMetadata). -func (c *compaction) atomicUnitBounds(f *fileMetadata) (lower, upper []byte) { - for i := range c.inputs { - files := c.inputs[i].files - for j := range files { - if f == files[j] { - // Note that if this file is in a multi-file atomic compaction unit, this file - // may not be the first file in that unit. An example in Pebble would be a - // preceding file with Largest c#12,1 and this file with Smallest c#9,1 and - // containing a range tombstone [c, g)#11,15. The start of the range tombstone - // is already truncated to this file's Smallest.UserKey, due to the code in - // rangedel.Fragmenter.FlushTo(), so this walking back should not be necessary - // (also see range_deletions.md for more details). - // - // We do this walking back to be extra cautious, in case it helps with RocksDB - // compatibility. - lowerBound := f.Smallest.UserKey - for k := j; k > 0; k-- { - cur := files[k] - prev := files[k-1] - if c.cmp(prev.Largest.UserKey, cur.Smallest.UserKey) < 0 { - break - } - if prev.Largest.Trailer == InternalKeyRangeDeleteSentinel { - // The range deletion sentinel key is set for the largest key in a - // table when a range deletion tombstone straddles a table. It - // isn't necessary to include the prev table in the atomic - // compaction unit as prev.largest.UserKey does not actually exist - // in the prev table. - break - } - lowerBound = prev.Smallest.UserKey - } - - upperBound := f.Largest.UserKey - for k := j + 1; k < len(files); k++ { - cur := files[k-1] - next := files[k] - if c.cmp(cur.Largest.UserKey, next.Smallest.UserKey) < 0 { - break - } - if cur.Largest.Trailer == InternalKeyRangeDeleteSentinel { - // The range deletion sentinel key is set for the largest key in a - // table when a range deletion tombstone straddles a table. It - // isn't necessary to include the next table in the atomic - // compaction unit as cur.largest.UserKey does not actually exist - // in the current table. - break - } - // cur.largest.UserKey == next.largest.UserKey, so next is part of - // the atomic compaction unit. - upperBound = next.Largest.UserKey - } - return lowerBound, upperBound - } - } - } - return nil, nil -} - // newInputIter returns an iterator over all the input tables in a compaction. func (c *compaction) newInputIter(newIters tableNewIters) (_ internalIterator, retErr error) { if len(c.flushing) != 0 { @@ -592,19 +514,19 @@ func (c *compaction) newInputIter(newIters tableNewIters) (_ internalIterator, r // Check that the LSM ordering invariants are ok in order to prevent // generating corrupted sstables due to a violation of those invariants. if c.startLevel.level >= 0 { - err := manifest.CheckOrdering(c.cmp, c.formatKey, manifest.Level(c.startLevel.level), - manifest.NewLevelSlice(c.startLevel.files).Iter()) + err := manifest.CheckOrdering(c.cmp, c.formatKey, + manifest.Level(c.startLevel.level), c.startLevel.files.Iter()) if err != nil { c.logger.Fatalf("%s", err) } } - err := manifest.CheckOrdering(c.cmp, c.formatKey, manifest.Level(c.outputLevel.level), - manifest.NewLevelSlice(c.outputLevel.files).Iter()) + err := manifest.CheckOrdering(c.cmp, c.formatKey, + manifest.Level(c.outputLevel.level), c.outputLevel.files.Iter()) if err != nil { c.logger.Fatalf("%s", err) } - iters := make([]internalIterator, 0, 2*len(c.startLevel.files)+1) + iters := make([]internalIterator, 0, 2*c.startLevel.files.Len()+1) defer func() { if retErr != nil { for _, iter := range iters { @@ -623,7 +545,7 @@ func (c *compaction) newInputIter(newIters tableNewIters) (_ internalIterator, r // one which iterates over the range deletions. These two iterators are // combined with a mergingIter. newRangeDelIter := func( - f *fileMetadata, _ *IterOptions, bytesIterated *uint64, + f manifest.LevelFile, _ *IterOptions, bytesIterated *uint64, ) (internalIterator, internalIterator, error) { iter, rangeDelIter, err := newIters(f, nil /* iter options */, &c.bytesIterated) if err == nil { @@ -652,10 +574,16 @@ func (c *compaction) newInputIter(newIters tableNewIters) (_ internalIterator, r // boundaries at write time. Because we're doing the truncation at read // time, we follow RocksDB's lead and do not truncate tombstones to // atomic unit boundaries at compaction time. - lowerBound, upperBound := c.atomicUnitBounds(f) - if lowerBound != nil || upperBound != nil { - rangeDelIter = rangedel.Truncate(c.cmp, rangeDelIter, lowerBound, upperBound) - } + atomicUnit := expandToAtomicUnit(c.cmp, f.Slice()) + lowerBound, upperBound := manifest.KeyRange(c.cmp, atomicUnit.Iter()) + // Range deletion tombstones are often written to sstables + // untruncated on the end key side. However, they are still only + // valid within a given file's bounds. The logic for writing range + // tombstones to an output file sometimes has an incomplete view + // of range tombstones outside the file's internal key bounds. Skip + // any range tombstones completely outside file bounds. + rangeDelIter = rangedel.Truncate( + c.cmp, rangeDelIter, lowerBound.UserKey, upperBound.UserKey, &f.Smallest, &f.Largest) } if rangeDelIter == nil { rangeDelIter = emptyIter @@ -664,15 +592,61 @@ func (c *compaction) newInputIter(newIters tableNewIters) (_ internalIterator, r } iterOpts := IterOptions{logger: c.logger} + addItersForLevel := func(iters []internalIterator, level *compactionLevel) ([]internalIterator, error) { + iters = append(iters, newLevelIter(iterOpts, c.cmp, newIters, level.files.Iter(), + manifest.Level(level.level), &c.bytesIterated)) + // Add the range deletion iterator for each file as an independent level + // in mergingIter, as opposed to making a levelIter out of those. This + // is safer as levelIter expects all keys coming from underlying + // iterators to be in order. Due to compaction / tombstone writing + // logic in finishOutput(), it is possible for range tombstones to not + // be strictly ordered across all files in one level. + // + // Consider this example from the metamorphic tests (also repeated in + // finishOutput()), consisting of three L3 files with their bounds + // specified in square brackets next to the file name: + // + // ./000240.sst [tmgc#391,MERGE-tmgc#391,MERGE] + // tmgc#391,MERGE [786e627a] + // tmgc-udkatvs#331,RANGEDEL + // + // ./000241.sst [tmgc#384,MERGE-tmgc#384,MERGE] + // tmgc#384,MERGE [666c7070] + // tmgc-tvsalezade#383,RANGEDEL + // tmgc-tvsalezade#331,RANGEDEL + // + // ./000242.sst [tmgc#383,RANGEDEL-tvsalezade#72057594037927935,RANGEDEL] + // tmgc-tvsalezade#383,RANGEDEL + // tmgc#375,SET [72646c78766965616c72776865676e79] + // tmgc-tvsalezade#356,RANGEDEL + // + // Here, the range tombstone in 000240.sst falls "after" one in + // 000241.sst, despite 000240.sst being ordered "before" 000241.sst for + // levelIter's purposes. While each file is still consistent before its + // bounds, it's safer to have all rangedel iterators be visible to + // mergingIter. + iter := level.files.Iter() + for f := iter.First(); f != nil; f = iter.Next() { + rangeDelIter, _, err := newRangeDelIter(iter.Take(), nil, &c.bytesIterated) + if err != nil { + return nil, errors.Wrapf(err, "pebble: could not open table %s", errors.Safe(f.FileNum)) + } + if rangeDelIter != emptyIter { + iters = append(iters, rangeDelIter) + } + } + return iters, nil + } + if c.startLevel.level != 0 { - iters = append(iters, newLevelIter(iterOpts, c.cmp, newIters, c.startLevel.files, - manifest.Level(c.startLevel.level), &c.bytesIterated)) - iters = append(iters, newLevelIter(iterOpts, c.cmp, newRangeDelIter, c.startLevel.files, - manifest.Level(c.startLevel.level), &c.bytesIterated)) + iters, err = addItersForLevel(iters, c.startLevel) + if err != nil { + return nil, err + } } else { - for i := range c.startLevel.files { - f := c.startLevel.files[i] - iter, rangeDelIter, err := newIters(f, nil /* iter options */, &c.bytesIterated) + iter := c.startLevel.files.Iter() + for f := iter.First(); f != nil; f = iter.Next() { + iter, rangeDelIter, err := newIters(iter.Take(), nil /* iter options */, &c.bytesIterated) if err != nil { return nil, errors.Wrapf(err, "pebble: could not open table %s", errors.Safe(f.FileNum)) } @@ -683,10 +657,10 @@ func (c *compaction) newInputIter(newIters tableNewIters) (_ internalIterator, r } } - iters = append(iters, newLevelIter(iterOpts, c.cmp, newIters, c.outputLevel.files, - manifest.Level(c.outputLevel.level), &c.bytesIterated)) - iters = append(iters, newLevelIter(iterOpts, c.cmp, newRangeDelIter, c.outputLevel.files, - manifest.Level(c.outputLevel.level), &c.bytesIterated)) + iters, err = addItersForLevel(iters, c.outputLevel) + if err != nil { + return nil, err + } return newMergingIter(c.logger, c.cmp, iters...), nil } @@ -702,7 +676,8 @@ func (c *compaction) String() string { level = c.outputLevel.level } fmt.Fprintf(&buf, "%d:", level) - for _, f := range c.inputs[i].files { + iter := c.inputs[i].files.Iter() + for f := iter.First(); f != nil; f = iter.Next() { fmt.Fprintf(&buf, " %s:%s-%s", f.FileNum, f.Smallest, f.Largest) } fmt.Fprintf(&buf, "\n") @@ -725,7 +700,8 @@ func (d *DB) addInProgressCompaction(c *compaction) { d.mu.compact.inProgress[c] = struct{}{} var isBase, isIntraL0 bool for _, cl := range c.inputs { - for _, f := range cl.files { + iter := cl.files.Iter() + for f := iter.First(); f != nil; f = iter.Next() { if f.Compacting { d.opts.Logger.Fatalf("L%d->L%d: %s already being compacted", c.startLevel.level, c.outputLevel.level, f.FileNum) } @@ -742,7 +718,7 @@ func (d *DB) addInProgressCompaction(c *compaction) { } if (isIntraL0 || isBase) && c.version.L0Sublevels != nil { - l0Inputs := [][]*fileMetadata{c.startLevel.files} + l0Inputs := []manifest.LevelSlice{c.startLevel.files} if isIntraL0 { l0Inputs = append(l0Inputs, c.outputLevel.files) } @@ -783,7 +759,8 @@ func (d *DB) addInProgressCompaction(c *compaction) { // for this compaction should have completed by this point. func (d *DB) removeInProgressCompaction(c *compaction) { for _, cl := range c.inputs { - for _, f := range cl.files { + iter := cl.files.Iter() + for f := iter.First(); f != nil; f = iter.Next() { if !f.Compacting { d.opts.Logger.Fatalf("L%d->L%d: %s already being compacted", c.startLevel.level, c.outputLevel.level, f.FileNum) } @@ -826,7 +803,7 @@ func (d *DB) getFlushPacerInfo() flushPacerInfo { // // d.mu must be held when calling this. func (d *DB) maybeScheduleFlush() { - if d.mu.compact.flushing || atomic.LoadInt32(&d.closed) != 0 || d.opts.ReadOnly { + if d.mu.compact.flushing || d.closed.Load() != nil || d.opts.ReadOnly { return } if len(d.mu.mem.queue) <= 1 { @@ -898,7 +875,7 @@ func (d *DB) maybeScheduleDelayedFlush(tbl *memTable) { // block on locking d.mu until we've finished scheduling the flush // and set `d.mu.compact.flushing` to true. Close will wait for // the current flush to complete. - if atomic.LoadInt32(&d.closed) != 0 { + if d.closed.Load() != nil { return } @@ -1056,7 +1033,7 @@ func (d *DB) flush1() error { // // d.mu must be held when calling this. func (d *DB) maybeScheduleCompaction() { - if atomic.LoadInt32(&d.closed) != 0 || d.opts.ReadOnly { + if d.closed.Load() != nil || d.opts.ReadOnly { return } if d.mu.compact.compactingCount >= d.opts.MaxConcurrentCompactions { @@ -1076,7 +1053,7 @@ func (d *DB) maybeScheduleCompaction() { // Check for the closed flag again, in case the DB was closed while we were // waiting for logLock(). - if atomic.LoadInt32(&d.closed) != 0 { + if d.closed.Load() != nil { return } @@ -1229,7 +1206,8 @@ func (d *DB) maybeUpdateDeleteCompactionHints(c *compaction) { // tombstones. inputsOlder := true for _, in := range c.inputs { - for _, f := range in.files { + iter := in.files.Iter() + for f := iter.First(); f != nil; f = iter.Next() { inputsOlder = inputsOlder && f.LargestSeqNum < h.tombstoneSmallestSeqNum } } @@ -1327,7 +1305,7 @@ func checkDeleteCompactionHints( } compactLevels = append(compactLevels, compactionLevel{ level: l, - files: files, + files: manifest.NewLevelSlice(files), }) } return compactLevels, unresolvedHints @@ -1434,13 +1412,17 @@ func (d *DB) runCompaction( DeletedFiles: map[deletedFileEntry]bool{}, } for _, cl := range c.inputs { - c.metrics[cl.level] = &LevelMetrics{} - for _, f := range cl.files { + levelMetrics := &LevelMetrics{} + iter := cl.files.Iter() + for f := iter.First(); f != nil; f = iter.Next() { + levelMetrics.NumFiles-- + levelMetrics.Size -= int64(f.Size) ve.DeletedFiles[deletedFileEntry{ Level: cl.level, FileNum: f.FileNum, }] = true } + c.metrics[cl.level] = levelMetrics } return ve, nil, nil } @@ -1450,9 +1432,16 @@ func (d *DB) runCompaction( // the move could create a parent file that will require a very expensive // merge later on. if c.kind == compactionKindMove { - meta := c.startLevel.files[0] + iter := c.startLevel.files.Iter() + meta := iter.First() c.metrics = map[int]*LevelMetrics{ + c.startLevel.level: &LevelMetrics{ + NumFiles: -1, + Size: -int64(meta.Size), + }, c.outputLevel.level: &LevelMetrics{ + NumFiles: 1, + Size: int64(meta.Size), BytesMoved: meta.Size, TablesMoved: 1, }, @@ -1514,13 +1503,16 @@ func (d *DB) runCompaction( DeletedFiles: map[deletedFileEntry]bool{}, } - metrics := &LevelMetrics{ - BytesIn: totalSize(c.startLevel.files), - BytesRead: totalSize(c.outputLevel.files), + outputMetrics := &LevelMetrics{ + BytesIn: c.startLevel.files.SizeSum(), + BytesRead: c.outputLevel.files.SizeSum(), } - metrics.BytesRead += metrics.BytesIn + outputMetrics.BytesRead += outputMetrics.BytesIn c.metrics = map[int]*LevelMetrics{ - c.outputLevel.level: metrics, + c.outputLevel.level: outputMetrics, + } + if len(c.flushing) == 0 && c.metrics[c.startLevel.level] == nil { + c.metrics[c.startLevel.level] = &LevelMetrics{} } writerOpts := d.opts.MakeWriterOptions(c.outputLevel.level) @@ -1569,6 +1561,26 @@ func (d *DB) runCompaction( // finishOutput is called for an sstable with the first key of the next sstable, and for the // last sstable with an empty key. finishOutput := func(key []byte) error { + // If we haven't output any point records to the sstable (tw == nil) + // then the sstable will only contain range tombstones. The smallest + // key in the sstable will be the start key of the first range + // tombstone added. We need to ensure that this start key is distinct + // from the limit (key) passed to finishOutput, otherwise we would + // generate an sstable where the largest key is smaller than the + // smallest key due to how the largest key boundary is set below. + // TODO: It is unfortunate that we have to do this check here rather + // than when we decide to finish the sstable in the runCompaction + // loop. A better structure currently eludes us. + if tw == nil { + startKey := c.rangeDelFrag.Start() + if len(iter.tombstones) > 0 { + startKey = iter.tombstones[0].Start.UserKey + } + if d.cmp(startKey, key) == 0 { + return nil + } + } + // NB: clone the key because the data can be held on to by the call to // compactionIter.Tombstones via rangedel.Fragmenter.FlushTo. key = append([]byte(nil), key...) @@ -1578,6 +1590,30 @@ func (d *DB) runCompaction( return err } } + // The tombstone being added could be completely outside the + // eventual bounds of the sstable. Consider this example (bounds + // in square brackets next to table filename): + // + // ./000240.sst [tmgc#391,MERGE-tmgc#391,MERGE] + // tmgc#391,MERGE [786e627a] + // tmgc-udkatvs#331,RANGEDEL + // + // ./000241.sst [tmgc#384,MERGE-tmgc#384,MERGE] + // tmgc#384,MERGE [666c7070] + // tmgc-tvsalezade#383,RANGEDEL + // tmgc-tvsalezade#331,RANGEDEL + // + // ./000242.sst [tmgc#383,RANGEDEL-tvsalezade#72057594037927935,RANGEDEL] + // tmgc-tvsalezade#383,RANGEDEL + // tmgc#375,SET [72646c78766965616c72776865676e79] + // tmgc-tvsalezade#356,RANGEDEL + // + // Note that both of the top two SSTables have range tombstones + // that start after the file's end keys. Since the file bound + // computation happens well after all range tombstones have been + // added to the writer, eliding out-of-file range tombstones based + // on sequence number at this stage is difficult, and necessitates + // read-time logic to ignore range tombstones outside file bounds. if err := tw.Add(v.Start, v.End); err != nil { return err } @@ -1612,12 +1648,14 @@ func (d *DB) runCompaction( } if c.flushing == nil { - metrics.TablesCompacted++ - metrics.BytesCompacted += meta.Size + outputMetrics.TablesCompacted++ + outputMetrics.BytesCompacted += meta.Size } else { - metrics.TablesFlushed++ - metrics.BytesFlushed += meta.Size + outputMetrics.TablesFlushed++ + outputMetrics.BytesFlushed += meta.Size } + outputMetrics.Size += int64(meta.Size) + outputMetrics.NumFiles++ // The handling of range boundaries is a bit complicated. if n := len(ve.NewFiles); n > 1 { @@ -1720,6 +1758,9 @@ func (d *DB) runCompaction( c.largest.Pretty(d.opts.Comparer.FormatKey)) } } + if err := meta.Validate(d.cmp, d.opts.Comparer.FormatKey); err != nil { + return err + } return nil } @@ -1759,7 +1800,7 @@ func (d *DB) runCompaction( limit = c.findL0Limit(startKey) } } - } else if c.rangeDelFrag.Empty() { + } else if c.rangeDelFrag.Empty() || len(ve.NewFiles) == 0 { // In this case, `limit` will be a larger user key than `key.UserKey`, or // nil. In either case, the inner loop will execute at least once to // process `key`, and the input iterator will be advanced. @@ -1929,7 +1970,10 @@ func (d *DB) runCompaction( } for _, cl := range c.inputs { - for _, f := range cl.files { + iter := cl.files.Iter() + for f := iter.First(); f != nil; f = iter.Next() { + c.metrics[cl.level].NumFiles-- + c.metrics[cl.level].Size -= int64(f.Size) ve.DeletedFiles[deletedFileEntry{ Level: cl.level, FileNum: f.FileNum, diff --git a/github.com/cockroachdb/pebble/compaction_picker.go b/github.com/cockroachdb/pebble/compaction_picker.go index 873ba34979..8fd7e801ca 100644 --- a/github.com/cockroachdb/pebble/compaction_picker.go +++ b/github.com/cockroachdb/pebble/compaction_picker.go @@ -39,7 +39,7 @@ type compactionPicker interface { // Information about in-progress compactions provided to the compaction picker. These are used to // constrain the new compactions that will be picked. type compactionInfo struct { - inputs []compactionInput + inputs []compactionLevel outputLevel int } @@ -58,13 +58,6 @@ func (s sortCompactionLevelsDecreasingScore) Swap(i, j int) { s[i], s[j] = s[j], s[i] } -// TODO(jackson): Consolidate this type with the compactionLevel type again -// once compactions are refactored to use LevelIterators too. -type compactionInput struct { - level int - files manifest.LevelSlice -} - // pickedCompaction contains information about a compaction that has already // been chosen, and is being constructed. Compaction construction info lives in // this struct, and is copied over into the compaction struct when that's @@ -77,13 +70,13 @@ type pickedCompaction struct { // startLevel is the level that is being compacted. Inputs from startLevel // and outputLevel will be merged to produce a set of outputLevel files. - startLevel *compactionInput + startLevel *compactionLevel // outputLevel is the level that files are being produced in. outputLevel is // equal to startLevel+1 except when startLevel is 0 in which case it is // equal to compactionPicker.baseLevel(). - outputLevel *compactionInput + outputLevel *compactionLevel - inputs []compactionInput + inputs []compactionLevel // L0-specific compaction info. Set to a non-nil value for all compactions // where startLevel == 0 that were generated by L0Sublevels. @@ -130,7 +123,7 @@ func newPickedCompaction( pc := &pickedCompaction{ cmp: opts.Comparer.Compare, version: cur, - inputs: []compactionInput{{level: startLevel}, {level: outputLevel}}, + inputs: []compactionLevel{{level: startLevel}, {level: outputLevel}}, maxOutputFileSize: uint64(opts.Level(adjustedOutputLevel).TargetFileSize), maxOverlapBytes: maxGrandparentOverlapBytes(opts, adjustedOutputLevel), maxExpandedBytes: expandedCompactionByteSizeLimit(opts, adjustedOutputLevel), @@ -153,9 +146,10 @@ func newPickedCompactionFromL0(lcf *manifest.L0CompactionFiles, opts *Options, v // because compactions built by L0SSTables do not necessarily // pick contiguous sequences of files in pc.version.Levels[0]. files := make([]*manifest.FileMetadata, 0, len(lcf.Files)) - for j := range lcf.FilesIncluded { + iter := vers.Levels[0].Iter() + for j, f := 0, iter.First(); f != nil; j, f = j+1, iter.Next() { if lcf.FilesIncluded[j] { - files = append(files, vers.Levels[0][j]) + files = append(files, f) } } pc.startLevel.files = manifest.NewLevelSlice(files) @@ -164,7 +158,7 @@ func newPickedCompactionFromL0(lcf *manifest.L0CompactionFiles, opts *Options, v func (pc *pickedCompaction) setupInputs() { // Expand the initial inputs to a clean cut. - pc.startLevel.files = expandInputs(pc.cmp, pc.startLevel.files) + pc.startLevel.files = expandToAtomicUnit(pc.cmp, pc.startLevel.files) pc.smallest, pc.largest = manifest.KeyRange(pc.cmp, pc.startLevel.files.Iter()) // Determine the sstables in the output level which overlap with the input @@ -172,7 +166,7 @@ func (pc *pickedCompaction) setupInputs() { // this for intra-L0 compactions; outputLevel.files is left empty for those. if pc.startLevel.level != pc.outputLevel.level { pc.outputLevel.files = pc.version.Overlaps(pc.outputLevel.level, pc.cmp, pc.smallest.UserKey, pc.largest.UserKey) - pc.outputLevel.files = expandInputs(pc.cmp, pc.outputLevel.files) + pc.outputLevel.files = expandToAtomicUnit(pc.cmp, pc.outputLevel.files) pc.smallest, pc.largest = manifest.KeyRange(pc.cmp, pc.startLevel.files.Iter(), pc.outputLevel.files.Iter()) } @@ -199,34 +193,40 @@ func (pc *pickedCompaction) setupInputs() { // to include files that "touch" it. smallestBaseKey := base.InvalidInternalKey largestBaseKey := base.InvalidInternalKey - baseFiles := pc.version.Levels[pc.outputLevel.level] - if len(baseFiles) != 0 { - // smallestBaseFileIdx is the index of the smallest - // (by key ordering) base file already in the compaction. - smallestBaseFileIdx := sort.Search(len(baseFiles), func(i int) bool { - return base.InternalCompare(pc.cmp, baseFiles[i].Largest, pc.smallest) >= 0 - }) - if smallestBaseFileIdx > 0 { - smallestBaseKey = baseFiles[smallestBaseFileIdx-1].Largest + { + baseIter := pc.version.Levels[pc.outputLevel.level].Iter() + sm := baseIter.SeekLT(pc.cmp, pc.smallest.UserKey) + if sm != nil { + // NB: In a case like the following example, SeekLT(b) would + // return 000005 which is okay as it does not actually contain the + // user key `b`. + // + // L6: + // 000005: [a#5,SET-b#RANGEDELSENTINEL] + // 000006: [b#4,SET-c#5,SET] + smallestBaseKey = sm.Largest + } + la := baseIter.SeekGE(pc.cmp, pc.largest.UserKey) + for la != nil && pc.cmp(la.Largest.UserKey, pc.largest.UserKey) == 0 { + la = baseIter.Next() } - // largestBaseFileIdx is the index that's one higher than the - // largest base file included in the compaction. - largestBaseFileIdx := sort.Search(len(baseFiles), func(i int) bool { - return base.InternalCompare(pc.cmp, baseFiles[i].Smallest, pc.largest) > 0 - }) - if largestBaseFileIdx < len(baseFiles) { - largestBaseKey = baseFiles[largestBaseFileIdx].Smallest + if la != nil { + largestBaseKey = la.Smallest } } + oldLcf := *pc.lcf if pc.version.L0Sublevels.ExtendL0ForBaseCompactionTo(smallestBaseKey, largestBaseKey, pc.lcf) { var newStartLevelFiles []*fileMetadata - for j := range pc.lcf.FilesIncluded { + iter := pc.version.Levels[0].Iter() + var sizeSum uint64 + for j, f := 0, iter.First(); f != nil; j, f = j+1, iter.Next() { if pc.lcf.FilesIncluded[j] { - newStartLevelFiles = append(newStartLevelFiles, pc.version.Levels[0][j]) + newStartLevelFiles = append(newStartLevelFiles, f) + sizeSum += f.Size } } - if totalSize(newStartLevelFiles)+pc.outputLevel.files.SizeSum() < pc.maxExpandedBytes { + if sizeSum+pc.outputLevel.files.SizeSum() < pc.maxExpandedBytes { pc.startLevel.files = manifest.NewLevelSlice(newStartLevelFiles) pc.smallest, pc.largest = manifest.KeyRange(pc.cmp, pc.startLevel.files.Iter(), pc.outputLevel.files.Iter()) @@ -248,7 +248,7 @@ func (pc *pickedCompaction) grow(sm, la InternalKey) bool { return false } grow0 := pc.version.Overlaps(pc.startLevel.level, pc.cmp, sm.UserKey, la.UserKey) - grow0 = expandInputs(pc.cmp, grow0) + grow0 = expandToAtomicUnit(pc.cmp, grow0) if grow0.Len() <= pc.startLevel.files.Len() { return false } @@ -257,7 +257,7 @@ func (pc *pickedCompaction) grow(sm, la InternalKey) bool { } sm1, la1 := manifest.KeyRange(pc.cmp, grow0.Iter()) grow1 := pc.version.Overlaps(pc.outputLevel.level, pc.cmp, sm1.UserKey, la1.UserKey) - grow1 = expandInputs(pc.cmp, grow1) + grow1 = expandToAtomicUnit(pc.cmp, grow1) if grow1.Len() != pc.outputLevel.files.Len() { return false } @@ -266,20 +266,25 @@ func (pc *pickedCompaction) grow(sm, la InternalKey) bool { return true } -// expandInputs expands the files in inputs in order to maintain the invariant +// expandToAtomicUnit expands the provided level slice within its level both +// forwards and backwards to its "atomic compaction unit" boundaries, if +// necessary. +// +// While picking compaction inputs, this is required to maintain the invariant // that the versions of keys at level+1 are older than the versions of keys at -// level. This is achieved by adding tables to the right of the current input -// tables such that the rightmost table has a "clean cut". A clean cut is -// either a change in user keys, or when the largest key in the left sstable is -// a range tombstone sentinel key (InternalKeyRangeDeleteSentinel). +// level. Tables are added to the right of the current slice tables such that +// the rightmost table has a "clean cut". A clean cut is either a change in +// user keys, or when the largest key in the left sstable is a range tombstone +// sentinel key (InternalKeyRangeDeleteSentinel). // -// In addition to maintaining the seqnum invariant, expandInputs is used to -// provide clean boundaries for range tombstone truncation during -// compaction. In order to achieve these clean boundaries, expandInputs needs -// to find a "clean cut" on the left edge of the compaction as well. This is -// necessary in order for "atomic compaction units" to always be compacted as a -// unit. Failure to do this leads to a subtle bug with truncation of range -// tombstones to atomic compaction unit boundaries. Consider the scenario: +// In addition to maintaining the seqnum invariant, expandToAtomicUnit is used +// to provide clean boundaries for range tombstone truncation during +// compaction. In order to achieve these clean boundaries, expandToAtomicUnit +// needs to find a "clean cut" on the left edge of the compaction as well. +// This is necessary in order for "atomic compaction units" to always be +// compacted as a unit. Failure to do this leads to a subtle bug with +// truncation of range tombstones to atomic compaction unit boundaries. +// Consider the scenario: // // L3: // 12:[a#2,15-b#1,1] @@ -298,7 +303,7 @@ func (pc *pickedCompaction) grow(sm, la InternalKey) bool { // tombstones will be truncated at "b" (the largest key in its atomic // compaction unit). In the scenario here, that could result in b#1 becoming // visible when it should be deleted. -func expandInputs(cmp Compare, inputs manifest.LevelSlice) manifest.LevelSlice { +func expandToAtomicUnit(cmp Compare, inputs manifest.LevelSlice) manifest.LevelSlice { // NB: Inputs for L0 can't be expanded and *version.Overlaps guarantees // that we get a 'clean cut.' For L0, Overlaps will return a slice without // access to the rest of the L0 files, so it's OK to try to reslice. @@ -369,7 +374,7 @@ type candidateLevelInfo struct { // The file in level that will be compacted. Additional files may be // picked by the compaction, and a pickedCompaction created for the // compaction. - file manifest.LevelSlice + file manifest.LevelFile } // compensatedSize returns f's file size, inflated according to compaction @@ -440,8 +445,8 @@ func (p *compactionPickerByScore) estimatedCompactionDebt(l0ExtraSize uint64) ui // We assume that all the bytes in L0 need to be compacted to Lbase. This is // unlike the RocksDB logic that figures out whether L0 needs compaction. - bytesAddedToNextLevel := l0ExtraSize + totalSize(p.vers.Levels[0]) - nextLevelSize := totalSize(p.vers.Levels[p.baseLevel]) + bytesAddedToNextLevel := l0ExtraSize + p.vers.Levels[0].Slice().SizeSum() + nextLevelSize := p.vers.Levels[p.baseLevel].Slice().SizeSum() var compactionDebt uint64 if bytesAddedToNextLevel > 0 && nextLevelSize > 0 { @@ -453,7 +458,7 @@ func (p *compactionPickerByScore) estimatedCompactionDebt(l0ExtraSize uint64) ui for level := p.baseLevel; level < numLevels-1; level++ { levelSize := nextLevelSize + bytesAddedToNextLevel - nextLevelSize = totalSize(p.vers.Levels[level+1]) + nextLevelSize = p.vers.Levels[level+1].Slice().SizeSum() if levelSize > uint64(p.levelMaxBytes[level]) { bytesAddedToNextLevel = levelSize - uint64(p.levelMaxBytes[level]) if nextLevelSize > 0 { @@ -493,7 +498,7 @@ func (p *compactionPickerByScore) initLevelMaxBytes(inProgressCompactions []comp firstNonEmptyLevel := -1 var dbSize int64 for level := 1; level < numLevels; level++ { - levelSize := int64(totalSize(p.vers.Levels[level])) + levelSize := int64(p.vers.Levels[level].Slice().SizeSum()) if levelSize > 0 { if firstNonEmptyLevel == -1 { firstNonEmptyLevel = level @@ -528,7 +533,7 @@ func (p *compactionPickerByScore) initLevelMaxBytes(inProgressCompactions []comp } const levelMultiplier = 10 - dbSize += int64(totalSize(p.vers.Levels[0])) + dbSize += int64(p.vers.Levels[0].Slice().SizeSum()) bottomLevelSize := dbSize - dbSize/levelMultiplier curLevelSize := bottomLevelSize @@ -687,11 +692,16 @@ func (p *compactionPickerByScore) calculateL0Score( // Score an L0->Lbase compaction by counting the number of idle // (non-compacting) files in L0. - var idleL0Count int - for _, f := range p.vers.Levels[0] { - if !f.Compacting { + var idleL0Count, totalL0Count, intraL0Count int + iter := p.vers.Levels[0].Iter() + for f := iter.First(); f != nil; f = iter.Next() { + if f.Compacting { + intraL0Count = 0 + } else { idleL0Count++ + intraL0Count++ } + totalL0Count++ } info.score = float64(idleL0Count) / float64(p.opts.L0CompactionThreshold) @@ -709,21 +719,12 @@ func (p *compactionPickerByScore) calculateL0Score( return info } - l0Files := p.vers.Levels[0] - if len(l0Files) < p.opts.L0CompactionThreshold+2 { + if totalL0Count < p.opts.L0CompactionThreshold+2 { // If L0 isn't accumulating many files beyond the regular L0 trigger, // don't resort to an intra-L0 compaction yet. This matches the RocksDB // heuristic. return info } - var end = len(l0Files) - for ; end >= 1; end-- { - if l0Files[end-1].Compacting { - break - } - } - - intraL0Count := len(l0Files) - end if intraL0Count < minIntraL0Count { // Not enough idle L0 files to perform an intra-L0 compaction. This // matches the RocksDB heuristic. Note that if another file is flushed @@ -739,7 +740,7 @@ func (p *compactionPickerByScore) calculateL0Score( return info } -func (p *compactionPickerByScore) pickFile(level, outputLevel int) manifest.LevelSlice { +func (p *compactionPickerByScore) pickFile(level, outputLevel int) (manifest.LevelFile, bool) { // Select the file within the level to compact. We want to minimize write // amplification, but also ensure that deletes are propagated to the // bottom level in a timely fashion so as to reclaim disk space. A table's @@ -766,7 +767,7 @@ func (p *compactionPickerByScore) pickFile(level, outputLevel int) manifest.Leve startIter := p.vers.Levels[level].Iter() outputIter := p.vers.Levels[outputLevel].Iter() - var file manifest.LevelSlice + var file manifest.LevelFile smallestRatio := uint64(math.MaxUint64) outputFile := outputIter.First() @@ -806,7 +807,7 @@ func (p *compactionPickerByScore) pickFile(level, outputLevel int) manifest.Leve file = startIter.Take() } } - return file + return file, file.FileMetadata != nil } // pickAuto picks the best compaction, if any. @@ -828,7 +829,7 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (pc *pickedCompact if p.opts.Experimental.L0SublevelCompactions { l0ReadAmp = p.vers.L0Sublevels.MaxDepthAfterOngoingCompactions() } else { - l0ReadAmp = len(p.vers.Levels[0]) + l0ReadAmp = p.vers.Levels[0].Slice().Len() } if l0ReadAmp < n*p.opts.Experimental.L0CompactionConcurrency { return nil @@ -914,8 +915,9 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (pc *pickedCompact continue } - info.file = p.pickFile(info.level, info.outputLevel) - if info.file.Empty() { + var ok bool + info.file, ok = p.pickFile(info.level, info.outputLevel) + if !ok { continue } @@ -972,7 +974,7 @@ func pickAutoHelper( if pc.outputLevel.level != cInfo.outputLevel { panic("pebble: compaction picked unexpected output level") } - pc.startLevel.files = cInfo.file + pc.startLevel.files = cInfo.file.Slice() // Files in level 0 may overlap each other, so pick up all overlapping ones. if pc.startLevel.level == 0 { cmp := opts.Comparer.Compare @@ -997,7 +999,7 @@ func pickL0(env compactionEnv, opts *Options, vers *version, baseLevel int) (pc // // TODO(bilal) Remove the minCompactionDepth parameter once fixing it at 1 // has been shown to not cause a performance regression. - lcf, err := vers.L0Sublevels.PickBaseCompaction(1, vers.Levels[baseLevel]) + lcf, err := vers.L0Sublevels.PickBaseCompaction(1, vers.Levels[baseLevel].Slice()) if err != nil { opts.Logger.Infof("error when picking base compaction: %s", err) return @@ -1048,10 +1050,10 @@ func pickL0(env compactionEnv, opts *Options, vers *version, baseLevel int) (pc } func pickIntraL0(env compactionEnv, opts *Options, vers *version) (pc *pickedCompaction) { - l0Files := vers.Levels[0] - end := len(l0Files) - for ; end >= 1; end-- { - m := l0Files[end-1] + l0Files := vers.Levels[0].Slice() + end := l0Files.Iter() + remaining := l0Files.Len() + for m := end.Last(); m != nil; m = end.Prev() { if m.Compacting { return nil } @@ -1059,7 +1061,7 @@ func pickIntraL0(env compactionEnv, opts *Options, vers *version) (pc *pickedCom break } // Don't compact an L0 file which contains a seqnum greater than the - // earliest unflushed seqnum (we continue the loop, rather than existing, + // earliest unflushed seqnum (we continue the loop, rather than exiting, // see conditional above). This can happen when a file is ingested into L0 // yet doesn't overlap with the memtable. Consider the scenario: // @@ -1095,38 +1097,47 @@ func pickIntraL0(env compactionEnv, opts *Options, vers *version) (pc *pickedCom // And now everything is copacetic. // // See https://github.com/facebook/rocksdb/pull/5958. + remaining-- } - if end < minIntraL0Count { + if remaining < minIntraL0Count { return nil } - compactTotalSize := l0Files[end-1].Size + compactTotalSize := end.Current().Size + compactTotalCount := 1 compactSizePerFile := uint64(math.MaxUint64) - // The compaction will be in the range [begin,end). We add files to the - // compaction until the amount of compaction work per file begins increasing. - begin := end - 1 - for ; begin >= 1; begin-- { - m := l0Files[begin-1] - if m.Compacting { - break - } - newCompactTotalSize := compactTotalSize + m.Size - newCompactSizePerFile := newCompactTotalSize / uint64(end-(begin-1)) - if newCompactSizePerFile > compactSizePerFile { - break - } - compactTotalSize = newCompactTotalSize - compactSizePerFile = newCompactSizePerFile - } + // The compaction will be a subslice of l0Files ending with the file + // currently positioned beneath end. We add files to the compaction from + // the left until the amount of compaction work per file begins + // increasing. + compactFiles := end.Take().Slice().Reslice(func(start, end *manifest.LevelIterator) { + for m := start.Prev(); m != nil; m = start.Prev() { + if m.Compacting { + break + } - if end-begin < minIntraL0Count { + newCompactTotalSize := compactTotalSize + m.Size + newCompactSizePerFile := newCompactTotalSize / uint64(compactTotalCount+1) + if newCompactSizePerFile > compactSizePerFile { + break + } + compactTotalCount++ + compactTotalSize = newCompactTotalSize + compactSizePerFile = newCompactSizePerFile + } + // We advanced the iterator one too far, either beyond the beginning + // of the level's files or to a file that triggered an early break. + // Move the start back over the last file that was okay. + start.Next() + }) + if compactTotalCount < minIntraL0Count { return nil } pc = newPickedCompaction(opts, vers, 0, 0) - pc.startLevel.files = manifest.NewLevelSlice(l0Files[begin:end]) - pc.smallest, pc.largest = manifest.KeyRange(pc.cmp, pc.startLevel.files.Iter()) + pc.startLevel.files = compactFiles + pc.smallest, pc.largest = manifest.KeyRange(pc.cmp, compactFiles.Iter()) // Output only a single sstable for intra-L0 compactions. There is no current // benefit to outputting multiple tables, because other parts of the code // (i.e. iterators and comapction) expect L0 sstables to overlap and will diff --git a/github.com/cockroachdb/pebble/db.go b/github.com/cockroachdb/pebble/db.go index b9b8ff91b7..4cc57eab49 100644 --- a/github.com/cockroachdb/pebble/db.go +++ b/github.com/cockroachdb/pebble/db.go @@ -38,7 +38,7 @@ var ( // key. ErrNotFound = base.ErrNotFound // ErrClosed is returned when an operation is performed on a closed snapshot - // or DB. + // or DB. Use errors.Is(err, ErrClosed) to check for this error. ErrClosed = errors.New("pebble: closed") // ErrReadOnly is returned when a write operation is performed on a read-only // database. @@ -197,7 +197,7 @@ type DB struct { // updates. logRecycler logRecycler - closed int32 // updated atomically + closed atomic.Value closedCh chan struct{} // The count and size of referenced memtables. This includes memtables @@ -357,8 +357,8 @@ func (d *DB) Get(key []byte) ([]byte, io.Closer, error) { } func (d *DB) getInternal(key []byte, b *Batch, s *Snapshot) ([]byte, io.Closer, error) { - if atomic.LoadInt32(&d.closed) != 0 { - panic(ErrClosed) + if err := d.closed.Load(); err != nil { + panic(err) } // Grab and reference the current readState. This prevents the underlying @@ -520,8 +520,8 @@ func (d *DB) LogData(data []byte, opts *WriteOptions) error { // // It is safe to modify the contents of the arguments after Apply returns. func (d *DB) Apply(batch *Batch, opts *WriteOptions) error { - if atomic.LoadInt32(&d.closed) != 0 { - panic(ErrClosed) + if err := d.closed.Load(); err != nil { + panic(err) } if d.opts.ReadOnly { return ErrReadOnly @@ -664,8 +664,8 @@ var iterAllocPool = sync.Pool{ func (d *DB) newIterInternal( batchIter internalIterator, batchRangeDelIter internalIterator, s *Snapshot, o *IterOptions, ) *Iterator { - if atomic.LoadInt32(&d.closed) != 0 { - panic(ErrClosed) + if err := d.closed.Load(); err != nil { + panic(err) } // Grab and reference the current readState. This prevents the underlying @@ -743,8 +743,8 @@ func (d *DB) newIterInternal( mlevels = mlevels[start:] levels := buf.levels[:] - addLevelIterForFiles := func(files []*manifest.FileMetadata, level manifest.Level) { - if len(files) == 0 { + addLevelIterForFiles := func(files manifest.LevelIterator, level manifest.Level) { + if files.Empty() { return } var li *levelIter @@ -766,12 +766,13 @@ func (d *DB) newIterInternal( // Add level iterators for the L0 sublevels, iterating from newest to // oldest. for i := len(current.L0Sublevels.Levels) - 1; i >= 0; i-- { - addLevelIterForFiles(current.L0Sublevels.Levels[i], manifest.L0Sublevel(i)) + iter := manifest.NewLevelSlice(current.L0Sublevels.Levels[i]).Iter() + addLevelIterForFiles(iter, manifest.L0Sublevel(i)) } // Add level iterators for the non-empty non-L0 levels. for level := 1; level < len(current.Levels); level++ { - addLevelIterForFiles(current.Levels[level], manifest.Level(level)) + addLevelIterForFiles(current.Levels[level].Iter(), manifest.Level(level)) } buf.merging.init(&dbi.opts, d.cmp, finalMLevels...) @@ -817,8 +818,8 @@ func (d *DB) NewIter(o *IterOptions) *Iterator { // deleted. Instead, a snapshot prevents deletion of sequence numbers // referenced by the snapshot. func (d *DB) NewSnapshot() *Snapshot { - if atomic.LoadInt32(&d.closed) != 0 { - panic(ErrClosed) + if err := d.closed.Load(); err != nil { + panic(err) } d.mu.Lock() @@ -839,10 +840,10 @@ func (d *DB) NewSnapshot() *Snapshot { func (d *DB) Close() error { d.mu.Lock() defer d.mu.Unlock() - if atomic.LoadInt32(&d.closed) != 0 { - panic(ErrClosed) + if err := d.closed.Load(); err != nil { + panic(err) } - atomic.StoreInt32(&d.closed, 1) + d.closed.Store(errors.WithStack(ErrClosed)) close(d.closedCh) defer d.opts.Cache.Unref() @@ -917,8 +918,8 @@ func (d *DB) Close() error { func (d *DB) Compact( start, end []byte, /* CompactionOptions */ ) error { - if atomic.LoadInt32(&d.closed) != 0 { - panic(ErrClosed) + if err := d.closed.Load(); err != nil { + panic(err) } if d.opts.ReadOnly { return ErrReadOnly @@ -1021,8 +1022,8 @@ func (d *DB) Flush() error { // If no error is returned, the caller can receive from the returned channel in // order to wait for the flush to complete. func (d *DB) AsyncFlush() (<-chan struct{}, error) { - if atomic.LoadInt32(&d.closed) != 0 { - panic(ErrClosed) + if err := d.closed.Load(); err != nil { + panic(err) } if d.opts.ReadOnly { return nil, ErrReadOnly @@ -1095,19 +1096,22 @@ func (d *DB) SSTables() [][]TableInfo { srcLevels := readState.current.Levels var totalTables int for i := range srcLevels { - totalTables += len(srcLevels[i]) + // TODO(jackson): Use metrics on the LevelMetadata once available + // rather than Slice().Len(). + totalTables += srcLevels[i].Slice().Len() } destTables := make([]TableInfo, totalTables) destLevels := make([][]TableInfo, len(srcLevels)) for i := range destLevels { - srcLevel := srcLevels[i] - destLevel := destTables[:len(srcLevel):len(srcLevel)] - destTables = destTables[len(srcLevel):] - for j := range destLevel { - destLevel[j] = srcLevel[j].TableInfo() + iter := srcLevels[i].Iter() + j := 0 + for m := iter.First(); m != nil; m = iter.Next() { + destTables[j] = m.TableInfo() + j++ } - destLevels[i] = destLevel + destLevels[i] = destTables[:j] + destTables = destTables[j:] } return destLevels } @@ -1124,8 +1128,8 @@ func (d *DB) SSTables() [][]TableInfo { // - There may also exist WAL entries for unflushed keys in this range. This // estimation currently excludes space used for the range in the WAL. func (d *DB) EstimateDiskUsage(start, end []byte) (uint64, error) { - if atomic.LoadInt32(&d.closed) != 0 { - panic(ErrClosed) + if err := d.closed.Load(); err != nil { + panic(err) } if d.opts.Comparer.Compare(start, end) > 0 { return 0, errors.New("invalid key-range specified (start > end)") @@ -1278,9 +1282,11 @@ func (d *DB) makeRoomForWrite(b *Batch) error { continue } } - l0ReadAmp := len(d.mu.versions.currentVersion().Levels[0]) + var l0ReadAmp int if d.opts.Experimental.L0SublevelCompactions { l0ReadAmp = d.mu.versions.currentVersion().L0Sublevels.ReadAmplification() + } else { + l0ReadAmp = d.mu.versions.currentVersion().Levels[0].Slice().Len() } if l0ReadAmp >= d.opts.L0StopWritesThreshold { // There are too many level-0 files, so we wait. @@ -1457,15 +1463,9 @@ func (d *DB) getInProgressCompactionInfoLocked(finishing *compaction) (rv []comp for c := range d.mu.compact.inProgress { if len(c.flushing) == 0 && (finishing == nil || c != finishing) { info := compactionInfo{ - inputs: make([]compactionInput, 0, len(c.inputs)), + inputs: c.inputs, outputLevel: -1, } - for _, in := range c.inputs { - info.inputs = append(info.inputs, compactionInput{ - level: in.level, - files: manifest.NewLevelSlice(in.files), - }) - } if c.outputLevel != nil { info.outputLevel = c.outputLevel.level } diff --git a/github.com/cockroachdb/pebble/flush_external.go b/github.com/cockroachdb/pebble/flush_external.go index a5a5fb6d21..9836065097 100644 --- a/github.com/cockroachdb/pebble/flush_external.go +++ b/github.com/cockroachdb/pebble/flush_external.go @@ -5,7 +5,6 @@ package pebble import ( - "sync/atomic" "time" "github.com/cockroachdb/pebble/internal/private" @@ -17,8 +16,8 @@ import ( // replay package rather than calling this private hook directly. func flushExternalTable(untypedDB interface{}, path string, originalMeta *fileMetadata) error { d := untypedDB.(*DB) - if atomic.LoadInt32(&d.closed) != 0 { - panic(ErrClosed) + if err := d.closed.Load(); err != nil { + panic(err) } if d.opts.ReadOnly { return ErrReadOnly @@ -64,7 +63,12 @@ func flushExternalTable(untypedDB interface{}, path string, originalMeta *fileMe NewFiles: []newFileEntry{newFileEntry{Level: 0, Meta: m}}, } metrics := map[int]*LevelMetrics{ - 0: &LevelMetrics{BytesIngested: m.Size, TablesIngested: 1}, + 0: &LevelMetrics{ + NumFiles: 1, + Size: int64(m.Size), + BytesIngested: m.Size, + TablesIngested: 1, + }, } err := d.mu.versions.logAndApply(jobID, ve, metrics, d.dataDir, func() []compactionInfo { return d.getInProgressCompactionInfoLocked(nil) diff --git a/github.com/cockroachdb/pebble/get_iter.go b/github.com/cockroachdb/pebble/get_iter.go index 37dbac5e87..1c3e451a2e 100644 --- a/github.com/cockroachdb/pebble/get_iter.go +++ b/github.com/cockroachdb/pebble/get_iter.go @@ -140,7 +140,7 @@ func (g *getIter) Next() (*InternalKey, []byte) { if g.level == 0 { // Create iterators from L0 from newest to oldest. if n := len(g.l0); n > 0 { - files := g.l0[n-1] + files := manifest.NewLevelSlice(g.l0[n-1]).Iter() g.l0 = g.l0[:n-1] iterOpts := IterOptions{logger: g.logger} g.levelIter.init(iterOpts, g.cmp, g.newIters, files, manifest.L0Sublevel(n), nil) @@ -155,14 +155,15 @@ func (g *getIter) Next() (*InternalKey, []byte) { if g.level >= numLevels { return nil, nil } - if g.version.Levels[g.level].Iter().Empty() { + metadataIter := g.version.Levels[g.level].Iter() + if metadataIter.Empty() { g.level++ continue } iterOpts := IterOptions{logger: g.logger} g.levelIter.init(iterOpts, g.cmp, g.newIters, - g.version.Levels[g.level], manifest.Level(g.level), nil) + metadataIter, manifest.Level(g.level), nil) g.levelIter.initRangeDel(&g.rangeDelIter) g.level++ g.iter = &g.levelIter diff --git a/github.com/cockroachdb/pebble/ingest.go b/github.com/cockroachdb/pebble/ingest.go index b58b8f7ab8..45ae3bfe87 100644 --- a/github.com/cockroachdb/pebble/ingest.go +++ b/github.com/cockroachdb/pebble/ingest.go @@ -6,7 +6,6 @@ package pebble import ( "sort" - "sync/atomic" "time" "github.com/cockroachdb/errors" @@ -397,15 +396,15 @@ func ingestTargetLevel( targetLevel := 0 // Do we overlap with keys in L0? - for i := 0; i < len(v.Levels[0]); i++ { - meta0 := v.Levels[0][i] + iter := v.Levels[0].Iter() + for meta0 := iter.First(); meta0 != nil; meta0 = iter.Next() { c1 := sstableKeyCompare(cmp, meta.Smallest, meta0.Largest) c2 := sstableKeyCompare(cmp, meta.Largest, meta0.Smallest) if c1 > 0 || c2 < 0 { continue } - iter, rangeDelIter, err := newIters(meta0, nil, nil) + iter, rangeDelIter, err := newIters(iter.Take(), nil, nil) if err != nil { return 0, err } @@ -421,7 +420,7 @@ func ingestTargetLevel( level := baseLevel for ; level < numLevels; level++ { - levelIter := newLevelIter(iterOps, cmp, newIters, v.Levels[level], manifest.Level(level), nil) + levelIter := newLevelIter(iterOps, cmp, newIters, v.Levels[level].Iter(), manifest.Level(level), nil) var rangeDelIter internalIterator // Pass in a non-nil pointer to rangeDelIter so that levelIter.findFileGE sets it up for the target file. levelIter.initRangeDel(&rangeDelIter) @@ -468,6 +467,14 @@ func ingestTargetLevel( // the same filesystem as the DB. Sstables can be created for ingestion using // sstable.Writer. On success, Ingest removes the input paths. // +// All sstables *must* be Sync()'d by the caller after all bytes are written +// and before its file handle is closed; failure to do so could violate +// durability or lead to corrupted on-disk state. This method cannot, in a +// platform-and-FS-agnostic way, ensure that all sstables in the input are +// properly synced to disk. Opening new file handles and Sync()-ing them +// does not always guarantee durability; see the discussion here on that: +// https://github.com/cockroachdb/pebble/pull/835#issuecomment-663075379 +// // Ingestion loads each sstable into the lowest level of the LSM which it // doesn't overlap (see ingestTargetLevel). If an sstable overlaps a memtable, // ingestion forces the memtable to flush, and then waits for the flush to @@ -496,8 +503,8 @@ func ingestTargetLevel( // https://github.com/cockroachdb/pebble/issues/25 for an idea for how to fix // this hiccup. func (d *DB) Ingest(paths []string) error { - if atomic.LoadInt32(&d.closed) != 0 { - panic(ErrClosed) + if err := d.closed.Load(); err != nil { + panic(err) } if d.opts.ReadOnly { return ErrReadOnly @@ -669,6 +676,8 @@ func (d *DB) ingestApply(jobID int, meta []*fileMetadata) (*versionEdit, error) levelMetrics = &LevelMetrics{} metrics[f.Level] = levelMetrics } + levelMetrics.NumFiles++ + levelMetrics.Size += int64(m.Size) levelMetrics.BytesIngested += m.Size levelMetrics.TablesIngested++ } diff --git a/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go b/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go index e448150695..b174df930d 100644 --- a/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go +++ b/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go @@ -239,7 +239,7 @@ func NewL0Sublevels( f.l0Index = i keys = append(keys, intervalKey{key: f.Smallest.UserKey}) keys = append(keys, intervalKey{ - key: f.Largest.UserKey, + key: f.Largest.UserKey, isLargest: f.Largest.Trailer != base.InternalKeyRangeDeleteSentinel, }) } @@ -599,11 +599,12 @@ func (s *L0Sublevels) checkCompaction(c *L0CompactionFiles) error { // compaction must be involving L0 SSTables. It's assumed that the Compacting // and IsIntraL0Compacting fields are already set on all FileMetadatas passed // in. -func (s *L0Sublevels) UpdateStateForStartedCompaction(inputs [][]*FileMetadata, isBase bool) error { +func (s *L0Sublevels) UpdateStateForStartedCompaction(inputs []LevelSlice, isBase bool) error { minIntervalIndex := -1 maxIntervalIndex := 0 for i := range inputs { - for _, f := range inputs[i] { + iter := inputs[i].Iter() + for f := iter.First(); f != nil; f = iter.Next() { for i := f.minIntervalIndex; i <= f.maxIntervalIndex; i++ { interval := &s.orderedIntervals[i] interval.compactingFileCount++ @@ -840,7 +841,7 @@ func (is intervalSorterByDecreasingScore) Swap(i, j int) { // files that can be selected for compaction. Returns nil if no compaction is // possible. func (s *L0Sublevels) PickBaseCompaction( - minCompactionDepth int, baseFiles []*FileMetadata, + minCompactionDepth int, baseFiles LevelSlice, ) (*L0CompactionFiles, error) { // For LBase compactions, we consider intervals in a greedy manner in the // following order: @@ -913,27 +914,20 @@ func (s *L0Sublevels) PickBaseCompaction( // Check if the chosen compaction overlaps with any files // in Lbase that have Compacting = true. If that's the case, // this compaction cannot be chosen. - firstBaseIndex := sort.Search(len(baseFiles), func(i int) bool { - // An interval starting at ImmediateSuccessor(key) can never be the - // first interval of a compaction since no file can start at that - // interval. - return s.cmp(baseFiles[i].Largest.UserKey, s.orderedIntervals[c.minIntervalIndex].startKey.key) >= 0 - }) - // Exclusive - lastBaseIndex := sort.Search(len(baseFiles), func(i int) bool { - cmp := s.cmp(baseFiles[i].Smallest.UserKey, s.orderedIntervals[c.maxIntervalIndex+1].startKey.key) + baseIter := baseFiles.Iter() + // An interval starting at ImmediateSuccessor(key) can never be the + // first interval of a compaction since no file can start at that + // interval. + m := baseIter.SeekGE(s.cmp, s.orderedIntervals[c.minIntervalIndex].startKey.key) + + var baseCompacting bool + for ; m != nil && !baseCompacting; m = baseIter.Next() { + cmp := s.cmp(m.Smallest.UserKey, s.orderedIntervals[c.maxIntervalIndex+1].startKey.key) // Compaction is ending at exclusive bound of c.maxIntervalIndex+1 if cmp > 0 || (cmp == 0 && !s.orderedIntervals[c.maxIntervalIndex+1].startKey.isLargest) { - return true - } - return false - }) - baseCompacting := false - for j := firstBaseIndex; j < lastBaseIndex; j++ { - if baseFiles[j].Compacting { - baseCompacting = true break } + baseCompacting = baseCompacting || m.Compacting } if baseCompacting { continue diff --git a/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go b/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go index 079c7083ce..f735c08b89 100644 --- a/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go +++ b/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go @@ -8,17 +8,30 @@ import "sort" // LevelMetadata contains metadata for all of the files within // a level of the LSM. -// TODO(jackson): Convert to an opaque struct. -type LevelMetadata []*FileMetadata +type LevelMetadata struct { + files []*FileMetadata +} // Iter constructs a LevelIterator over the entire level. func (lm LevelMetadata) Iter() LevelIterator { - return LevelIterator{files: lm, end: len(lm)} + return LevelIterator{files: lm.files, end: len(lm.files)} } // Slice constructs a slice containing the entire level. func (lm LevelMetadata) Slice() LevelSlice { - return LevelSlice{files: lm, end: len(lm)} + return LevelSlice{files: lm.files, end: len(lm.files)} +} + +// LevelFile holds a file's metadata along with its position +// within a level of the LSM. +type LevelFile struct { + *FileMetadata + slice LevelSlice +} + +// Slice constructs a LevelSlice containing only this file. +func (lf LevelFile) Slice() LevelSlice { + return lf.slice } // NewLevelSlice constructs a LevelSlice over the provided files. This @@ -191,13 +204,35 @@ func (i *LevelIterator) SeekGE(cmp Compare, userKey []byte) *FileMetadata { return i.files[i.cur] } -// Take constructs a new LevelSlice containing only the file at the iterator's -// current position. If the iterator is not currently positioned over a file, -// the returned iterator is empty. -func (i LevelIterator) Take() LevelSlice { - return LevelSlice{ - files: i.files, - start: i.cur, - end: i.cur + 1, +// SeekLT seeks to the last file in the iterator's file set with a smallest +// user key less than the provided user key. The iterator must have been +// constructed from L1+, because it requries the underlying files to be sorted +// by user keys and non-overlapping. +func (i *LevelIterator) SeekLT(cmp Compare, userKey []byte) *FileMetadata { + files := i.files[i.start:i.end] + i.cur = i.start + sort.Search(len(files), func(j int) bool { + return cmp(files[j].Smallest.UserKey, userKey) >= 0 + }) + if i.cur < i.start { + return nil + } + return i.Prev() +} + +// Take constructs a LevelFile containing the file at the iterator's current +// position. Take panics if the iterator is not currently positioned over a +// file. +func (i LevelIterator) Take() LevelFile { + m := i.Current() + if m == nil { + panic("Take called on invalid LevelIterator") + } + return LevelFile{ + FileMetadata: m, + slice: LevelSlice{ + files: i.files, + start: i.cur, + end: i.cur + 1, + }, } } diff --git a/github.com/cockroachdb/pebble/internal/manifest/version.go b/github.com/cockroachdb/pebble/internal/manifest/version.go index 39c6f3bcef..393923902d 100644 --- a/github.com/cockroachdb/pebble/internal/manifest/version.go +++ b/github.com/cockroachdb/pebble/internal/manifest/version.go @@ -93,10 +93,25 @@ type FileMetadata struct { maxIntervalIndex int } -func (m FileMetadata) String() string { +func (m *FileMetadata) String() string { return fmt.Sprintf("%s:%s-%s", m.FileNum, m.Smallest, m.Largest) } +// Validate validates the metadata for consistency with itself, returning an +// error if inconsistent. +func (m *FileMetadata) Validate(cmp Compare, formatKey base.FormatKey) error { + if base.InternalCompare(cmp, m.Smallest, m.Largest) > 0 { + return errors.Errorf("file %s has inconsistent bounds: %s vs %s", + errors.Safe(m.FileNum), m.Smallest.Pretty(formatKey), + m.Largest.Pretty(formatKey)) + } + if m.SmallestSeqNum > m.LargestSeqNum { + return errors.Errorf("file %s has inconsistent seqnum bounds: %d vs %d", + errors.Safe(m.FileNum), m.SmallestSeqNum, m.LargestSeqNum) + } + return nil +} + // TableInfo returns a subset of the FileMetadata state formatted as a // TableInfo. func (m *FileMetadata) TableInfo() TableInfo { @@ -196,6 +211,24 @@ func overlaps(files []*FileMetadata, cmp Compare, start, end []byte) (lower, upp // NumLevels is the number of levels a Version contains. const NumLevels = 7 +// NewVersion constructs a new Version with the provided files. It assumes +// the provided files are already well-ordered. It's intended for testing. +func NewVersion( + cmp Compare, + formatKey base.FormatKey, + flushSplitBytes int64, + files [NumLevels][]*FileMetadata, +) *Version { + var v Version + for i := range files { + v.Levels[i].files = files[i] + } + if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil { + panic(err) + } + return &v +} + // Version is a collection of file metadata for on-disk tables at various // levels. In-memory DBs are written to level-0 tables, and compactions // migrate data from level N to level N+1. The tables map internal keys (which @@ -367,7 +400,7 @@ func (v *Version) InitL0Sublevels( cmp Compare, formatKey base.FormatKey, flushSplitBytes int64, ) error { var err error - v.L0Sublevels, err = NewL0Sublevels(v.Levels[0], cmp, formatKey, flushSplitBytes) + v.L0Sublevels, err = NewL0Sublevels(v.Levels[0].Slice().Collect(), cmp, formatKey, flushSplitBytes) return err } @@ -399,16 +432,18 @@ func (v *Version) Contains(level int, cmp Compare, m *FileMetadata) bool { func (v *Version) Overlaps(level int, cmp Compare, start, end []byte) LevelSlice { if level == 0 { // Indices that have been selected as overlapping. - selectedIndices := make([]bool, len(v.Levels[level])) + l0 := v.Levels[level].Slice() + l0Iter := l0.Iter() + selectedIndices := make([]bool, l0.Len()) numSelected := 0 var slice LevelSlice for { restart := false - for i, selected := range selectedIndices { + for i, meta := 0, l0Iter.First(); meta != nil; i, meta = i+1, l0Iter.Next() { + selected := selectedIndices[i] if selected { continue } - meta := v.Levels[level][i] smallest := meta.Smallest.UserKey largest := meta.Largest.UserKey if cmp(largest, start) < 0 { @@ -440,9 +475,9 @@ func (v *Version) Overlaps(level int, cmp Compare, start, end []byte) LevelSlice if !restart { slice.files = make([]*FileMetadata, 0, numSelected) - for i, selected := range selectedIndices { - if selected { - slice.files = append(slice.files, v.Levels[level][i]) + for i, meta := 0, l0Iter.First(); meta != nil; i, meta = i+1, l0Iter.Next() { + if selectedIndices[i] { + slice.files = append(slice.files, meta) } } slice.end = len(slice.files) @@ -454,9 +489,9 @@ func (v *Version) Overlaps(level int, cmp Compare, start, end []byte) LevelSlice } var slice LevelSlice - lower, upper := overlaps(v.Levels[level], cmp, start, end) + lower, upper := overlaps(v.Levels[level].files, cmp, start, end) if lower < upper { - slice.files = v.Levels[level] + slice.files = v.Levels[level].files slice.start = lower slice.end = upper } @@ -643,10 +678,8 @@ func CheckOrdering(cmp Compare, format base.FormatKey, level Level, files LevelI } else { var prev *FileMetadata for f := files.First(); f != nil; f, prev = files.Next(), f { - if base.InternalCompare(cmp, f.Smallest, f.Largest) > 0 { - return errors.Errorf("%s file %s has inconsistent bounds: %s vs %s", - errors.Safe(level), errors.Safe(f.FileNum), - f.Smallest.Pretty(format), f.Largest.Pretty(format)) + if err := f.Validate(cmp, format); err != nil { + return errors.Wrapf(err, "%s ", level) } if prev != nil { if !prev.lessSmallestKey(f, cmp) { diff --git a/github.com/cockroachdb/pebble/internal/manifest/version_edit.go b/github.com/cockroachdb/pebble/internal/manifest/version_edit.go index 9bb01c7cfa..e47f4638c0 100644 --- a/github.com/cockroachdb/pebble/internal/manifest/version_edit.go +++ b/github.com/cockroachdb/pebble/internal/manifest/version_edit.go @@ -501,11 +501,11 @@ func (b *BulkVersionEdit) Apply( if curr == nil { continue } - files := curr.Levels[level] - v.Levels[level] = files + levelMetadata := curr.Levels[level] + v.Levels[level] = levelMetadata // We still have to bump the ref count for all files. - for i := range files { - atomic.AddInt32(&files[i].refs, 1) + for i := range levelMetadata.files { + atomic.AddInt32(&levelMetadata.files[i].refs, 1) } continue } @@ -513,7 +513,7 @@ func (b *BulkVersionEdit) Apply( // Some edits on this level. var currFiles []*FileMetadata if curr != nil { - currFiles = curr.Levels[level] + currFiles = curr.Levels[level].files } addedFiles := b.Added[level] deletedMap := b.Deleted[level] @@ -523,7 +523,7 @@ func (b *BulkVersionEdit) Apply( "pebble: internal error: No current or added files but have deleted files: %d", errors.Safe(len(deletedMap))) } - v.Levels[level] = make([]*FileMetadata, 0, n) + v.Levels[level].files = make([]*FileMetadata, 0, n) // We have 2 lists of files, currFiles and addedFiles either of which (but not both) can // be empty. // - currFiles is internally consistent, since it comes from curr. @@ -553,10 +553,10 @@ func (b *BulkVersionEdit) Apply( continue } atomic.AddInt32(&f.refs, 1) - v.Levels[level] = append(v.Levels[level], f) + v.Levels[level].files = append(v.Levels[level].files, f) } } - SortBySeqNum(v.Levels[level]) + SortBySeqNum(v.Levels[level].files) if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil { return nil, nil, errors.Wrap(err, "pebble: internal error") } @@ -599,10 +599,10 @@ func (b *BulkVersionEdit) Apply( } removeZombie(cf.FileNum) atomic.AddInt32(&cf.refs, 1) - v.Levels[level] = append(v.Levels[level], cf) + v.Levels[level].files = append(v.Levels[level].files, cf) } currFiles = currFiles[j:] - numFiles := len(v.Levels[level]) + numFiles := len(v.Levels[level].files) if numFiles > 0 { // We expect k to typically be large, and we can avoid doing consistency // checks of the files within that set of k, since they are already mutually @@ -612,8 +612,8 @@ func (b *BulkVersionEdit) Apply( // its predecessor either came from currFiles or addedFiles, and both are ones // which we need to check against f for consistency (since we have not checked // addedFiles for internal consistency). - if base.InternalCompare(cmp, v.Levels[level][numFiles-1].Largest, f.Smallest) >= 0 { - cf := v.Levels[level][numFiles-1] + if base.InternalCompare(cmp, v.Levels[level].files[numFiles-1].Largest, f.Smallest) >= 0 { + cf := v.Levels[level].files[numFiles-1] return nil, nil, errors.Errorf( "pebble: internal error: L%d files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]", errors.Safe(level), errors.Safe(cf.FileNum), errors.Safe(f.FileNum), @@ -621,7 +621,7 @@ func (b *BulkVersionEdit) Apply( f.Smallest.Pretty(formatKey), f.Largest.Pretty(formatKey)) } } - v.Levels[level] = append(v.Levels[level], f) + v.Levels[level].files = append(v.Levels[level].files, f) } // Add any remaining files in currFiles that are after all the added files. for i := range currFiles { @@ -632,7 +632,7 @@ func (b *BulkVersionEdit) Apply( } removeZombie(f.FileNum) atomic.AddInt32(&f.refs, 1) - v.Levels[level] = append(v.Levels[level], f) + v.Levels[level].files = append(v.Levels[level].files, f) } } return v, zombies, nil diff --git a/github.com/cockroachdb/pebble/internal/rangedel/fragmenter.go b/github.com/cockroachdb/pebble/internal/rangedel/fragmenter.go index 9d1e258ad9..dec8898d35 100644 --- a/github.com/cockroachdb/pebble/internal/rangedel/fragmenter.go +++ b/github.com/cockroachdb/pebble/internal/rangedel/fragmenter.go @@ -63,7 +63,8 @@ func Sort(cmp base.Compare, tombstones []Tombstone) { // tombstones are split at their overlap points. The fragmented tombstones are // output to the supplied Output function. type Fragmenter struct { - Cmp base.Compare + Cmp base.Compare + Format base.FormatKey // Emit is called to emit a chunk of tombstone fragments. Every tombstone // within the chunk has the same start and end key and are in decreasing // order of their sequence numbers. @@ -95,7 +96,7 @@ func (f *Fragmenter) checkInvariants(buf []Tombstone) { } if f.Cmp(buf[i-1].Start.UserKey, buf[i].Start.UserKey) != 0 { panic(fmt.Sprintf("pebble: pending tombstone invariant violated: %s %s", - buf[i-1].Start, buf[i].Start)) + buf[i-1].Start.Pretty(f.Format), buf[i].Start.Pretty(f.Format))) } } } @@ -182,7 +183,7 @@ func (f *Fragmenter) Add(start base.InternalKey, end []byte) { switch c := f.Cmp(start.UserKey, f.flushedKey); { case c < 0: panic(fmt.Sprintf("pebble: start key (%s) < flushed key (%s)", - start.UserKey, f.flushedKey)) + f.Format(start.UserKey), f.Format(f.flushedKey))) } } if f.Cmp(start.UserKey, end) >= 0 { @@ -200,7 +201,7 @@ func (f *Fragmenter) Add(start base.InternalKey, end []byte) { switch c := f.Cmp(f.pending[0].Start.UserKey, start.UserKey); { case c > 0: panic(fmt.Sprintf("pebble: keys must be added in order: %s > %s", - f.pending[0].Start, start)) + f.pending[0].Start.Pretty(f.Format), start.Pretty(f.Format))) case c == 0: // The new tombstone has the same start key as the existing pending // tombstones. Add it to the pending buffer. @@ -236,7 +237,7 @@ func (f *Fragmenter) Deleted(key base.InternalKey, snapshot uint64) bool { if f.Cmp(f.pending[0].Start.UserKey, key.UserKey) > 0 { panic(fmt.Sprintf("pebble: keys must be in order: %s > %s", - f.pending[0].Start, key)) + f.pending[0].Start.Pretty(f.Format), key.Pretty(f.Format))) } seqNum := key.SeqNum() @@ -283,7 +284,7 @@ func (f *Fragmenter) FlushTo(key []byte) { switch c := f.Cmp(key, f.flushedKey); { case c < 0: panic(fmt.Sprintf("pebble: flush-to key (%s) < flushed key (%s)", - key, f.flushedKey)) + f.Format(key), f.Format(f.flushedKey))) } } f.flushedKey = append(f.flushedKey[:0], key...) @@ -294,7 +295,7 @@ func (f *Fragmenter) FlushTo(key []byte) { switch c := f.Cmp(f.pending[0].Start.UserKey, key); { case c > 0: panic(fmt.Sprintf("pebble: keys must be in order: %s > %s", - f.pending[0].Start, key)) + f.Format(f.pending[0].Start.UserKey), f.Format(key))) } // Note that we explicitly do not return early here if Start.UserKey == // key. Similar to the scenario described above, consider: @@ -360,7 +361,7 @@ func (f *Fragmenter) TruncateAndFlushTo(key []byte) { switch c := f.Cmp(key, f.flushedKey); { case c < 0: panic(fmt.Sprintf("pebble: start key (%s) < flushed key (%s)", - key, f.flushedKey)) + f.Format(key), f.Format(f.flushedKey))) } } if invariants.RaceEnabled { @@ -373,7 +374,7 @@ func (f *Fragmenter) TruncateAndFlushTo(key []byte) { switch c := f.Cmp(f.pending[0].Start.UserKey, key); { case c > 0: panic(fmt.Sprintf("pebble: keys must be added in order: %s > %s", - f.pending[0].Start, key)) + f.Format(f.pending[0].Start.UserKey), f.Format(key))) case c == 0: return } diff --git a/github.com/cockroachdb/pebble/internal/rangedel/truncate.go b/github.com/cockroachdb/pebble/internal/rangedel/truncate.go index d328dce7b9..66c86c7065 100644 --- a/github.com/cockroachdb/pebble/internal/rangedel/truncate.go +++ b/github.com/cockroachdb/pebble/internal/rangedel/truncate.go @@ -8,13 +8,27 @@ import "github.com/cockroachdb/pebble/internal/base" // Truncate creates a new iterator where every tombstone in the supplied // iterator is truncated to be contained within the range [lower, upper). -func Truncate(cmp base.Compare, iter base.InternalIterator, lower, upper []byte) *Iter { +// If start and end are specified, filter out any range tombstones that +// are completely outside those bounds. +func Truncate(cmp base.Compare, iter base.InternalIterator, lower, upper []byte, start, end *base.InternalKey) *Iter { var tombstones []Tombstone for key, value := iter.First(); key != nil; key, value = iter.Next() { t := Tombstone{ Start: *key, End: value, } + // Ignore this tombstone if it lies completely outside [start, end]. + // The comparison between t.End and start is by user key only, as + // the range tombstone is exclusive at t.End, so comparing by user keys + // is sufficient. Alternatively, the below comparison can be seen to + // be logically equivalent to: + // InternalKey{UserKey: t.End, SeqNum: SeqNumMax} < start + if start != nil && cmp(t.End, start.UserKey) <= 0 { + continue + } + if end != nil && base.InternalCompare(cmp, t.Start, *end) > 0 { + continue + } if cmp(t.Start.UserKey, lower) < 0 { t.Start.UserKey = lower } diff --git a/github.com/cockroachdb/pebble/internal/record/log_writer.go b/github.com/cockroachdb/pebble/internal/record/log_writer.go index 7ce8dc1d6d..e3ce92af0d 100644 --- a/github.com/cockroachdb/pebble/internal/record/log_writer.go +++ b/github.com/cockroachdb/pebble/internal/record/log_writer.go @@ -255,6 +255,7 @@ type LogWriter struct { block *block free struct { sync.Mutex + // Condition variable used to signal a block is freed. cond sync.Cond blocks []*block allocated int @@ -415,11 +416,20 @@ func (w *LogWriter) flushLoop(context.Context) { data := w.block.buf[w.block.flushed:written] w.block.flushed = written + // If flusher has an error, we propagate it to waiters. Note in spite of + // error we consume the pending list above to free blocks for writers. + if f.err != nil { + f.syncQ.pop(head, tail, f.err) + continue + } f.Unlock() - synced, err := w.flushPending(data, pending, head, tail) - f.Lock() + f.err = err + if f.err != nil { + f.syncQ.clearBlocked() + continue + } if synced && f.minSyncInterval != nil { // A sync was performed. Make sure we've waited for the min sync @@ -436,14 +446,6 @@ func (w *LogWriter) flushLoop(context.Context) { } } } - - f.err = err - if f.err != nil { - // TODO(peter): There might be new waiters that we should propagate f.err - // to. Because f.err is now set, we only have to perform a single extra - // clearing of those waiters as no new ones can arrive via SyncRecord(). - return - } } } @@ -528,6 +530,7 @@ func (w *LogWriter) queueBlock() { } // Close flushes and syncs any unwritten data and closes the writer. +// Where required, external synchronisation is provided by commitPipeline.mu. func (w *LogWriter) Close() error { f := &w.flusher @@ -562,6 +565,7 @@ func (w *LogWriter) Close() error { // WriteRecord writes a complete record. Returns the offset just past the end // of the record. +// External synchronisation provided by commitPipeline.mu. func (w *LogWriter) WriteRecord(p []byte) (int64, error) { return w.SyncRecord(p, nil, nil) } @@ -570,6 +574,7 @@ func (w *LogWriter) WriteRecord(p []byte) (int64, error) { // asynchronously persisted to the underlying writer and done will be called on // the wait group upon completion. Returns the offset just past the end of the // record. +// External synchronisation provided by commitPipeline.mu. func (w *LogWriter) SyncRecord(p []byte, wg *sync.WaitGroup, err *error) (int64, error) { if w.err != nil { return -1, w.err @@ -603,6 +608,7 @@ func (w *LogWriter) SyncRecord(p []byte, wg *sync.WaitGroup, err *error) (int64, } // Size returns the current size of the file. +// External synchronisation provided by commitPipeline.mu. func (w *LogWriter) Size() int64 { return w.blockNum*blockSize + int64(w.block.written) } diff --git a/github.com/cockroachdb/pebble/level_checker.go b/github.com/cockroachdb/pebble/level_checker.go index 29062c4524..199ce2bbe2 100644 --- a/github.com/cockroachdb/pebble/level_checker.go +++ b/github.com/cockroachdb/pebble/level_checker.go @@ -368,11 +368,12 @@ func checkRangeTombstones(c *checkConfig) error { } current := c.readState.current - addTombstonesFromLevel := func(files []*fileMetadata, lsmLevel int) error { - for j := 0; j < len(files); j++ { - lower, upper := getAtomicUnitBounds(c.cmp, files, j) - f := files[j] - iterToClose, iter, err := c.newIters(f, nil, nil) + addTombstonesFromLevel := func(files manifest.LevelIterator, lsmLevel int) error { + for f := files.First(); f != nil; f = files.Next() { + lf := files.Take() + atomicUnit := expandToAtomicUnit(c.cmp, lf.Slice()) + lower, upper := manifest.KeyRange(c.cmp, atomicUnit.Iter()) + iterToClose, iter, err := c.newIters(lf, nil, nil) if err != nil { return err } @@ -382,11 +383,11 @@ func checkRangeTombstones(c *checkConfig) error { } truncate := func(t rangedel.Tombstone) rangedel.Tombstone { // Same checks as in rangedel.Truncate. - if c.cmp(t.Start.UserKey, lower) < 0 { - t.Start.UserKey = lower + if c.cmp(t.Start.UserKey, lower.UserKey) < 0 { + t.Start.UserKey = lower.UserKey } - if c.cmp(t.End, upper) > 0 { - t.End = upper + if c.cmp(t.End, upper.UserKey) > 0 { + t.End = upper.UserKey } if c.cmp(t.Start.UserKey, t.End) >= 0 { t.Start.SetKind(InternalKeyKindInvalid) @@ -405,17 +406,14 @@ func checkRangeTombstones(c *checkConfig) error { if len(current.L0Sublevels.Levels[i]) == 0 { continue } - if err := addTombstonesFromLevel(current.L0Sublevels.Levels[i], 0); err != nil { + err := addTombstonesFromLevel(manifest.NewLevelSlice(current.L0Sublevels.Levels[i]).Iter(), 0) + if err != nil { return err } level++ } for i := 1; i < len(current.Levels); i++ { - if len(current.Levels[i]) == 0 { - continue - } - files := current.Levels[i] - if err := addTombstonesFromLevel(files, i); err != nil { + if err := addTombstonesFromLevel(current.Levels[i].Iter(), i); err != nil { return err } level++ @@ -431,35 +429,6 @@ func checkRangeTombstones(c *checkConfig) error { return iterateAndCheckTombstones(c.cmp, c.formatKey, tombstones) } -// TODO(sbhola): mostly copied from compaction.go: refactor and reuse? -func getAtomicUnitBounds(cmp Compare, files []*fileMetadata, index int) (lower, upper []byte) { - lower = files[index].Smallest.UserKey - upper = files[index].Largest.UserKey - for i := index; i > 0; i-- { - cur := files[i] - prev := files[i-1] - if cmp(prev.Largest.UserKey, cur.Smallest.UserKey) < 0 { - break - } - if prev.Largest.Trailer == InternalKeyRangeDeleteSentinel { - break - } - lower = prev.Smallest.UserKey - } - for i := index + 1; i < len(files); i++ { - cur := files[i-1] - next := files[i] - if cmp(cur.Largest.UserKey, next.Smallest.UserKey) < 0 { - break - } - if cur.Largest.Trailer == InternalKeyRangeDeleteSentinel { - break - } - upper = next.Largest.UserKey - } - return -} - func levelOrMemtable(lsmLevel int, fileNum FileNum) string { if lsmLevel == -1 { return "memtable" @@ -654,7 +623,7 @@ func checkLevelsInternal(c *checkConfig) (err error) { mlevels = append(mlevels, simpleMergingIterLevel{}) } for level := 1; level < len(current.Levels); level++ { - if len(current.Levels[level]) == 0 { + if current.Levels[level].Slice().Empty() { continue } mlevels = append(mlevels, simpleMergingIterLevel{}) @@ -665,9 +634,10 @@ func checkLevelsInternal(c *checkConfig) (err error) { if len(current.L0Sublevels.Levels[sublevel]) == 0 { continue } + manifestIter := manifest.NewLevelSlice(current.L0Sublevels.Levels[sublevel]).Iter() iterOpts := IterOptions{logger: c.logger} li := &levelIter{} - li.init(iterOpts, c.cmp, c.newIters, current.L0Sublevels.Levels[sublevel], + li.init(iterOpts, c.cmp, c.newIters, manifestIter, manifest.L0Sublevel(sublevel), nil) li.initRangeDel(&mlevelAlloc[0].rangeDelIter) li.initSmallestLargestUserKey(&mlevelAlloc[0].smallestUserKey, nil, nil) @@ -675,12 +645,13 @@ func checkLevelsInternal(c *checkConfig) (err error) { mlevelAlloc = mlevelAlloc[1:] } for level := 1; level < len(current.Levels); level++ { - if len(current.Levels[level]) == 0 { + manifestIter := current.Levels[level].Iter() + if manifestIter.Empty() { continue } iterOpts := IterOptions{logger: c.logger} li := &levelIter{} - li.init(iterOpts, c.cmp, c.newIters, current.Levels[level], manifest.Level(level), nil) + li.init(iterOpts, c.cmp, c.newIters, manifestIter, manifest.Level(level), nil) li.initRangeDel(&mlevelAlloc[0].rangeDelIter) li.initSmallestLargestUserKey(&mlevelAlloc[0].smallestUserKey, nil, nil) mlevelAlloc[0].iter = li diff --git a/github.com/cockroachdb/pebble/level_iter.go b/github.com/cockroachdb/pebble/level_iter.go index cb83586032..c464dee8d5 100644 --- a/github.com/cockroachdb/pebble/level_iter.go +++ b/github.com/cockroachdb/pebble/level_iter.go @@ -7,7 +7,6 @@ package pebble import ( "fmt" "runtime/debug" - "sort" "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/invariants" @@ -18,7 +17,7 @@ import ( // number. If bytesIterated is specified, it is incremented as the given file is // iterated through. type tableNewIters func( - meta *fileMetadata, opts *IterOptions, bytesIterated *uint64, + file manifest.LevelFile, opts *IterOptions, bytesIterated *uint64, ) (internalIterator, internalIterator, error) // levelIter provides a merged view of the sstables in a level. @@ -56,8 +55,6 @@ type levelIter struct { tableOpts IterOptions // The LSM level this levelIter is initialized for. level manifest.Level - // The current file wrt the iterator position. - index int // The keys to return when iterating past an sstable boundary and that // boundary is a range deletion tombstone. The boundary could be smallest // (i.e. arrived at with Prev), or largest (arrived at with Next). @@ -67,15 +64,16 @@ type levelIter struct { // which doesn't contain the search key, but which does contain range // tombstones. syntheticBoundary InternalKey - // The iter for the current index. It is nil under any of the following conditions: - // - index < 0 or index > len(files) + // The iter for the current file. It is nil under any of the following conditions: + // - files.Current() == nil // - err != nil // - some other constraint, like the bounds in opts, caused the file at index to not // be relevant to the iteration. iter internalIterator + iterFile *fileMetadata newIters tableNewIters rangeDelIter *internalIterator - files []*fileMetadata + files manifest.LevelIterator err error // Pointer into this level's entry in `mergingIterLevel::smallestUserKey,largestUserKey`. @@ -143,7 +141,7 @@ func newLevelIter( opts IterOptions, cmp Compare, newIters tableNewIters, - files []*fileMetadata, + files manifest.LevelIterator, level manifest.Level, bytesIterated *uint64, ) *levelIter { @@ -156,7 +154,7 @@ func (l *levelIter) init( opts IterOptions, cmp Compare, newIters tableNewIters, - files []*fileMetadata, + files manifest.LevelIterator, level manifest.Level, bytesIterated *uint64, ) { @@ -167,7 +165,7 @@ func (l *levelIter) init( l.upper = opts.UpperBound l.tableOpts.TableFilter = opts.TableFilter l.cmp = cmp - l.index = -1 + l.iterFile = nil l.newIters = newIters l.files = files l.bytesIterated = bytesIterated @@ -185,7 +183,7 @@ func (l *levelIter) initSmallestLargestUserKey( l.isLargestUserKeyRangeDelSentinel = isLargestUserKeyRangeDelSentinel } -func (l *levelIter) findFileGE(key []byte) int { +func (l *levelIter) findFileGE(key []byte) *fileMetadata { // Find the earliest file whose largest key is >= ikey. // // If the earliest file has its largest key == ikey and that largest key is a @@ -198,24 +196,18 @@ func (l *levelIter) findFileGE(key []byte) int { // Additionally, this prevents loading untruncated range deletions from a table which can't // possibly contain the target key and is required for correctness by mergingIter.SeekGE // (see the comment in that function). - // - // TODO(peter): inline the binary search. - return sort.Search(len(l.files), func(i int) bool { - largest := &l.files[i].Largest - c := l.cmp(largest.UserKey, key) - if c > 0 { - return true - } - return c == 0 && largest.Trailer != InternalKeyRangeDeleteSentinel - }) + + m := l.files.SeekGE(l.cmp, key) + for m != nil && m.Largest.Trailer == InternalKeyRangeDeleteSentinel && + l.cmp(m.Largest.UserKey, key) == 0 { + m = l.files.Next() + } + return m } -func (l *levelIter) findFileLT(key []byte) int { +func (l *levelIter) findFileLT(key []byte) *fileMetadata { // Find the last file whose smallest key is < ikey. - index := sort.Search(len(l.files), func(i int) bool { - return l.cmp(l.files[i].Smallest.UserKey, key) >= 0 - }) - return index - 1 + return l.files.SeekLT(l.cmp, key) } // Init the iteration bounds for the current table. Returns -1 if the table @@ -252,10 +244,10 @@ func (l *levelIter) initTableBounds(f *fileMetadata) int { return 0 } -func (l *levelIter) loadFile(index, dir int) bool { +func (l *levelIter) loadFile(file *fileMetadata, dir int) bool { l.smallestBoundary = nil l.largestBoundary = nil - if l.index == index { + if l.iterFile == file { if l.err != nil { return false } @@ -265,7 +257,7 @@ func (l *levelIter) loadFile(index, dir int) bool { // current iteration bounds, but it knows those bounds, so it will enforce them. return true } - // We were already at index, but don't have an iterator, probably because the file was + // We were already at file, but don't have an iterator, probably because the file was // beyond the iteration bounds. It may still be, but it is also possible that the bounds // have changed. We handle that below. } @@ -278,19 +270,19 @@ func (l *levelIter) loadFile(index, dir int) bool { return false } - for ; ; index += dir { - l.index = index - if l.index < 0 || l.index >= len(l.files) { + for { + l.iterFile = file + if file == nil { return false } - f := l.files[l.index] - switch l.initTableBounds(f) { + switch l.initTableBounds(file) { case -1: // The largest key in the sstable is smaller than the lower bound. if dir < 0 { return false } + file = l.files.Next() continue case +1: // The smallest key in the sstable is greater than or equal to the upper @@ -298,11 +290,12 @@ func (l *levelIter) loadFile(index, dir int) bool { if dir > 0 { return false } + file = l.files.Prev() continue } var rangeDelIter internalIterator - l.iter, rangeDelIter, l.err = l.newIters(f, &l.tableOpts, l.bytesIterated) + l.iter, rangeDelIter, l.err = l.newIters(l.files.Take(), &l.tableOpts, l.bytesIterated) if l.err != nil { return false } @@ -312,13 +305,13 @@ func (l *levelIter) loadFile(index, dir int) bool { rangeDelIter.Close() } if l.smallestUserKey != nil { - *l.smallestUserKey = f.Smallest.UserKey + *l.smallestUserKey = file.Smallest.UserKey } if l.largestUserKey != nil { - *l.largestUserKey = f.Largest.UserKey + *l.largestUserKey = file.Largest.UserKey } if l.isLargestUserKeyRangeDelSentinel != nil { - *l.isLargestUserKeyRangeDelSentinel = f.Largest.Trailer == InternalKeyRangeDeleteSentinel + *l.isLargestUserKeyRangeDelSentinel = file.Largest.Trailer == InternalKeyRangeDeleteSentinel } return true } @@ -349,7 +342,7 @@ func (l *levelIter) SeekGE(key []byte) (*InternalKey, []byte) { // NB: the top-level Iterator has already adjusted key based on // IterOptions.LowerBound. - if !l.loadFile(l.findFileGE(key), 1) { + if !l.loadFile(l.findFileGE(key), +1) { return nil, nil } if ikey, val := l.iter.SeekGE(key); ikey != nil { @@ -363,7 +356,7 @@ func (l *levelIter) SeekPrefixGE(prefix, key []byte) (*InternalKey, []byte) { // NB: the top-level Iterator has already adjusted key based on // IterOptions.LowerBound. - if !l.loadFile(l.findFileGE(key), 1) { + if !l.loadFile(l.findFileGE(key), +1) { return nil, nil } if key, val := l.iter.SeekPrefixGE(prefix, key); key != nil { @@ -376,8 +369,7 @@ func (l *levelIter) SeekPrefixGE(prefix, key []byte) (*InternalKey, []byte) { // this case the same as SeekGE where an upper-bound resides within the // sstable and generate a synthetic boundary key. if l.rangeDelIter != nil { - f := l.files[l.index] - l.syntheticBoundary = f.Largest + l.syntheticBoundary = l.iterFile.Largest l.syntheticBoundary.SetKind(InternalKeyKindRangeDelete) l.largestBoundary = &l.syntheticBoundary return l.verify(l.largestBoundary, nil) @@ -404,7 +396,7 @@ func (l *levelIter) First() (*InternalKey, []byte) { // NB: the top-level Iterator will call SeekGE if IterOptions.LowerBound is // set. - if !l.loadFile(0, 1) { + if !l.loadFile(l.files.First(), +1) { return nil, nil } if key, val := l.iter.First(); key != nil { @@ -418,7 +410,7 @@ func (l *levelIter) Last() (*InternalKey, []byte) { // NB: the top-level Iterator will call SeekLT if IterOptions.UpperBound is // set. - if !l.loadFile(len(l.files)-1, -1) { + if !l.loadFile(l.files.Last(), -1) { return nil, nil } if key, val := l.iter.Last(); key != nil { @@ -435,7 +427,7 @@ func (l *levelIter) Next() (*InternalKey, []byte) { switch { case l.largestBoundary != nil: // We're stepping past the boundary key, so now we can load the next file. - if l.loadFile(l.index+1, 1) { + if l.loadFile(l.files.Next(), +1) { if key, val := l.iter.First(); key != nil { return l.verify(key, val) } @@ -461,7 +453,7 @@ func (l *levelIter) Prev() (*InternalKey, []byte) { switch { case l.smallestBoundary != nil: // We're stepping past the boundary key, so now we can load the prev file. - if l.loadFile(l.index-1, -1) { + if l.loadFile(l.files.Prev(), -1) { if key, val := l.iter.Last(); key != nil { return l.verify(key, val) } @@ -508,7 +500,6 @@ func (l *levelIter) skipEmptyFileForward() (*InternalKey, []byte) { // It is safe to set the boundary key kind to RANGEDEL because we're // never going to look at subsequent sstables (we've reached the upper // bound). - f := l.files[l.index] if l.tableOpts.UpperBound != nil { // TODO(peter): Rather than using f.Largest, can we use // l.tableOpts.UpperBound and set the seqnum to 0? We know the upper @@ -516,20 +507,20 @@ func (l *levelIter) skipEmptyFileForward() (*InternalKey, []byte) { // kosher with respect to the invariant that only one record for a // given user key will have seqnum 0. See Iterator.nextUserKey for an // optimization that requires this. - l.syntheticBoundary = f.Largest + l.syntheticBoundary = l.iterFile.Largest l.syntheticBoundary.SetKind(InternalKeyKindRangeDelete) l.largestBoundary = &l.syntheticBoundary return l.largestBoundary, nil } // If the boundary is a range deletion tombstone, return that key. - if f.Largest.Kind() == InternalKeyKindRangeDelete { - l.largestBoundary = &f.Largest + if l.iterFile.Largest.Kind() == InternalKeyKindRangeDelete { + l.largestBoundary = &l.iterFile.Largest return l.largestBoundary, nil } } // Current file was exhausted. Move to the next file. - if !l.loadFile(l.index+1, 1) { + if !l.loadFile(l.files.Next(), +1) { return nil, nil } } @@ -565,25 +556,24 @@ func (l *levelIter) skipEmptyFileBackward() (*InternalKey, []byte) { // It is safe to set the boundary key kind to RANGEDEL because we're // never going to look at earlier sstables (we've reached the lower // bound). - f := l.files[l.index] if l.tableOpts.LowerBound != nil { // TODO(peter): Rather than using f.Smallest, can we use // l.tableOpts.LowerBound and set the seqnum to InternalKeySeqNumMax? // We know the lower bound resides within the table boundaries. - l.syntheticBoundary = f.Smallest + l.syntheticBoundary = l.iterFile.Smallest l.syntheticBoundary.SetKind(InternalKeyKindRangeDelete) l.smallestBoundary = &l.syntheticBoundary return l.smallestBoundary, nil } // If the boundary is a range deletion tombstone, return that key. - if f.Smallest.Kind() == InternalKeyKindRangeDelete { - l.smallestBoundary = &f.Smallest + if l.iterFile.Smallest.Kind() == InternalKeyKindRangeDelete { + l.smallestBoundary = &l.iterFile.Smallest return l.smallestBoundary, nil } } // Current file was exhausted. Move to the previous file. - if !l.loadFile(l.index-1, -1) { + if !l.loadFile(l.files.Prev(), -1) { return nil, nil } } @@ -621,8 +611,7 @@ func (l *levelIter) SetBounds(lower, upper []byte) { // Update tableOpts.{Lower,Upper}Bound in case the new boundaries fall within // the boundaries of the current table. - f := l.files[l.index] - if l.initTableBounds(f) != 0 { + if l.initTableBounds(l.iterFile) != 0 { // The table does not overlap the bounds. Close() will set levelIter.err if // an error occurs. _ = l.Close() @@ -633,7 +622,7 @@ func (l *levelIter) SetBounds(lower, upper []byte) { } func (l *levelIter) String() string { - if l.index >= 0 && l.index < len(l.files) { + if l.iterFile != nil { return fmt.Sprintf("%s: fileNum=%s", l.level, l.iter.String()) } return fmt.Sprintf("%s: fileNum=", l.level) diff --git a/github.com/cockroachdb/pebble/mem_table.go b/github.com/cockroachdb/pebble/mem_table.go index 92542012ac..7f1ccac0e1 100644 --- a/github.com/cockroachdb/pebble/mem_table.go +++ b/github.com/cockroachdb/pebble/mem_table.go @@ -60,6 +60,7 @@ var memTableEmptySize = func() uint32 { // It is safe to call get, apply, newIter, and newRangeDelIter concurrently. type memTable struct { cmp Compare + formatKey base.FormatKey equal Equal arenaBuf []byte skl arenaskl.Skiplist @@ -109,6 +110,7 @@ func newMemTable(opts memTableOptions) *memTable { m := &memTable{ cmp: opts.Comparer.Compare, + formatKey: opts.Comparer.FormatKey, equal: opts.Comparer.Equal, arenaBuf: opts.arenaBuf, writerRefs: 1, @@ -291,7 +293,8 @@ type rangeTombstoneFrags struct { func (f *rangeTombstoneFrags) get(m *memTable) []rangedel.Tombstone { f.once.Do(func() { frag := &rangedel.Fragmenter{ - Cmp: m.cmp, + Cmp: m.cmp, + Format: m.formatKey, Emit: func(fragmented []rangedel.Tombstone) { f.tombstones = append(f.tombstones, fragmented...) }, diff --git a/github.com/cockroachdb/pebble/metrics.go b/github.com/cockroachdb/pebble/metrics.go index 98f6e0d405..f004c364b3 100644 --- a/github.com/cockroachdb/pebble/metrics.go +++ b/github.com/cockroachdb/pebble/metrics.go @@ -38,7 +38,7 @@ type LevelMetrics struct { // The total number of files in the level. NumFiles int64 // The total size in bytes of the files in the level. - Size uint64 + Size int64 // The level's compaction score. Score float64 // The number of incoming bytes from other levels read during @@ -74,6 +74,8 @@ type LevelMetrics struct { // Add updates the counter metrics for the level. func (m *LevelMetrics) Add(u *LevelMetrics) { + m.NumFiles += u.NumFiles + m.Size += u.Size m.BytesIn += u.BytesIn m.BytesIngested += u.BytesIngested m.BytesMoved += u.BytesMoved @@ -100,7 +102,7 @@ func (m *LevelMetrics) WriteAmp() float64 { func (m *LevelMetrics) format(buf *bytes.Buffer, score string) { fmt.Fprintf(buf, "%9d %7s %7s %7s %7s %7s %7s %7s %7s %7s %7s %7d %7.1f\n", m.NumFiles, - humanize.IEC.Uint64(m.Size), + humanize.IEC.Int64(m.Size), score, humanize.IEC.Uint64(m.BytesIn), humanize.IEC.Uint64(m.BytesIngested), @@ -199,8 +201,6 @@ func (m *Metrics) Total() LevelMetrics { l := &m.Levels[level] total.Add(l) total.Sublevels += l.Sublevels - total.NumFiles += l.NumFiles - total.Size += l.Size } // Compute total bytes-in as the bytes written to the WAL + bytes ingested. total.BytesIn = m.WAL.BytesWritten + total.BytesIngested @@ -278,8 +278,6 @@ func (m *Metrics) String() string { l.format(&buf, score) total.Add(l) total.Sublevels += l.Sublevels - total.NumFiles += l.NumFiles - total.Size += l.Size } // Compute total bytes-in as the bytes written to the WAL + bytes ingested. total.BytesIn = m.WAL.BytesWritten + total.BytesIngested diff --git a/github.com/cockroachdb/pebble/open.go b/github.com/cockroachdb/pebble/open.go index 77f72a6c03..dd7fd88570 100644 --- a/github.com/cockroachdb/pebble/open.go +++ b/github.com/cockroachdb/pebble/open.go @@ -12,7 +12,6 @@ import ( "os" "runtime" "sort" - "sync/atomic" "time" "github.com/cockroachdb/errors" @@ -311,7 +310,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { // newLogNum. There should be no difference in using either value. ve.MinUnflushedLogNum = newLogNum d.mu.versions.logLock() - if err := d.mu.versions.logAndApply(jobID, &ve, nil, d.dataDir, func() []compactionInfo { + if err := d.mu.versions.logAndApply(jobID, &ve, newFileMetrics(ve.NewFiles), d.dataDir, func() []compactionInfo { return nil }); err != nil { return nil, err @@ -354,7 +353,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { if invariants.Enabled { runtime.SetFinalizer(d, func(obj interface{}) { d := obj.(*DB) - if atomic.LoadInt32(&d.closed) == 0 { + if err := d.closed.Load(); err == nil { fmt.Fprintf(os.Stderr, "%p: unreferenced DB not closed\n", d) os.Exit(1) } diff --git a/github.com/cockroachdb/pebble/sstable/reader.go b/github.com/cockroachdb/pebble/sstable/reader.go index 3f8c0e66f4..775aaaa664 100644 --- a/github.com/cockroachdb/pebble/sstable/reader.go +++ b/github.com/cockroachdb/pebble/sstable/reader.go @@ -1157,6 +1157,7 @@ func (c Comparers) readerApply(r *Reader) { } if comparer, ok := c[r.Properties.ComparerName]; ok { r.Compare = comparer.Compare + r.FormatKey = comparer.FormatKey r.Split = comparer.Split } } @@ -1206,7 +1207,7 @@ func (c *cacheOpts) writerApply(w *Writer) { // FileReopenOpt is specified if this reader is allowed to reopen additional // file descriptors for this file. Used to take advantage of OS-level readahead. -type FileReopenOpt struct{ +type FileReopenOpt struct { FS vfs.FS Filename string } @@ -1255,6 +1256,7 @@ type Reader struct { footerBH BlockHandle opts ReaderOptions Compare Compare + FormatKey base.FormatKey Split Split mergerOK bool tableFilter *tableFilterReader @@ -1542,7 +1544,8 @@ func (r *Reader) transformRangeDelV1(b []byte) ([]byte, error) { restartInterval: 1, } frag := rangedel.Fragmenter{ - Cmp: r.Compare, + Cmp: r.Compare, + Format: r.FormatKey, Emit: func(fragmented []rangedel.Tombstone) { for i := range fragmented { t := &fragmented[i] @@ -1860,6 +1863,7 @@ func NewReader(f vfs.File, o ReaderOptions, extraOpts ...ReaderOption) (*Reader, if r.Properties.ComparerName == "" || o.Comparer.Name == r.Properties.ComparerName { r.Compare = o.Comparer.Compare + r.FormatKey = o.Comparer.FormatKey r.Split = o.Comparer.Split } diff --git a/github.com/cockroachdb/pebble/table_cache.go b/github.com/cockroachdb/pebble/table_cache.go index 901518d921..2c4c0b9cf6 100644 --- a/github.com/cockroachdb/pebble/table_cache.go +++ b/github.com/cockroachdb/pebble/table_cache.go @@ -18,6 +18,7 @@ import ( "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/invariants" + "github.com/cockroachdb/pebble/internal/manifest" "github.com/cockroachdb/pebble/internal/private" "github.com/cockroachdb/pebble/sstable" "github.com/cockroachdb/pebble/vfs" @@ -49,9 +50,9 @@ func (c *tableCache) getShard(fileNum FileNum) *tableCacheShard { } func (c *tableCache) newIters( - meta *fileMetadata, opts *IterOptions, bytesIterated *uint64, + file manifest.LevelFile, opts *IterOptions, bytesIterated *uint64, ) (internalIterator, internalIterator, error) { - return c.getShard(meta.FileNum).newIters(meta, opts, bytesIterated) + return c.getShard(file.FileNum).newIters(file, opts, bytesIterated) } func (c *tableCache) evict(fileNum FileNum) { @@ -162,13 +163,13 @@ func (c *tableCacheShard) releaseLoop() { } func (c *tableCacheShard) newIters( - meta *fileMetadata, opts *IterOptions, bytesIterated *uint64, + file manifest.LevelFile, opts *IterOptions, bytesIterated *uint64, ) (internalIterator, internalIterator, error) { // Calling findNode gives us the responsibility of decrementing v's // refCount. If opening the underlying table resulted in error, then we // decrement this straight away. Otherwise, we pass that responsibility to // the sstable iterator, which decrements when it is closed. - v := c.findNode(meta) + v := c.findNode(file.FileMetadata) if v.err != nil { c.unrefValue(v) return nil, nil, v.err diff --git a/github.com/cockroachdb/pebble/table_stats.go b/github.com/cockroachdb/pebble/table_stats.go index c9156ef944..c2500d0bff 100644 --- a/github.com/cockroachdb/pebble/table_stats.go +++ b/github.com/cockroachdb/pebble/table_stats.go @@ -6,7 +6,6 @@ package pebble import ( "math" - "sync/atomic" "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/manifest" @@ -68,7 +67,7 @@ func (d *DB) updateTableStatsLocked(newFiles []manifest.NewFileEntry) { func (d *DB) shouldCollectTableStats() bool { ok := !d.mu.tableStats.loading - ok = ok && atomic.LoadInt32(&d.closed) == 0 + ok = ok && d.closed.Load() == nil ok = ok && !d.opts.private.disableTableStats ok = ok && (len(d.mu.tableStats.pending) > 0 || !d.mu.tableStats.loadedInitial) return ok @@ -159,7 +158,9 @@ type collectedStats struct { manifest.TableStats } -func (d *DB) loadNewFileStats(rs *readState, pending []manifest.NewFileEntry) ([]collectedStats, []deleteCompactionHint) { +func (d *DB) loadNewFileStats( + rs *readState, pending []manifest.NewFileEntry, +) ([]collectedStats, []deleteCompactionHint) { var hints []deleteCompactionHint collected := make([]collectedStats, 0, len(pending)) for _, nf := range pending { @@ -200,7 +201,9 @@ func (d *DB) loadNewFileStats(rs *readState, pending []manifest.NewFileEntry) ([ // scanReadStateTableStats is run by an active stat collection job when there // are no pending new files, but there might be files that existed at Open for // which we haven't loaded table stats. -func (d *DB) scanReadStateTableStats(rs *readState, fill []collectedStats) ([]collectedStats, []deleteCompactionHint, bool) { +func (d *DB) scanReadStateTableStats( + rs *readState, fill []collectedStats, +) ([]collectedStats, []deleteCompactionHint, bool) { moreRemain := false var hints []deleteCompactionHint for l, levelMetadata := range rs.current.Levels { @@ -264,7 +267,8 @@ func (d *DB) loadTableStats( defer rangeDelIter.Close() // Truncate tombstones to the containing file's bounds if necessary. // See docs/range_deletions.md for why this is necessary. - rangeDelIter = rangedel.Truncate(d.cmp, rangeDelIter, meta.Smallest.UserKey, meta.Largest.UserKey) + rangeDelIter = rangedel.Truncate( + d.cmp, rangeDelIter, meta.Smallest.UserKey, meta.Largest.UserKey, nil, nil) err = foreachDefragmentedTombstone(rangeDelIter, d.cmp, func(startUserKey, endUserKey []byte, smallestSeqNum, largestSeqNum uint64) error { estimate, hintSeqNum, err := d.estimateSizeBeneath(v, level, meta, startUserKey, endUserKey) @@ -344,7 +348,9 @@ func (d *DB) estimateSizeBeneath( } func foreachDefragmentedTombstone( - rangeDelIter base.InternalIterator, cmp base.Compare, fn func([]byte, []byte, uint64, uint64) error, + rangeDelIter base.InternalIterator, + cmp base.Compare, + fn func([]byte, []byte, uint64, uint64) error, ) error { var startUserKey, endUserKey []byte var smallestSeqNum, largestSeqNum uint64 diff --git a/github.com/cockroachdb/pebble/version_set.go b/github.com/cockroachdb/pebble/version_set.go index bce21e4ac5..6d5cfca011 100644 --- a/github.com/cockroachdb/pebble/version_set.go +++ b/github.com/cockroachdb/pebble/version_set.go @@ -13,6 +13,7 @@ import ( "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/invariants" "github.com/cockroachdb/pebble/internal/manifest" "github.com/cockroachdb/pebble/internal/record" "github.com/cockroachdb/pebble/vfs" @@ -266,8 +267,8 @@ func (vs *versionSet) load(dirname string, opts *Options, mu *sync.Mutex) error for i := range vs.metrics.Levels { l := &vs.metrics.Levels[i] - l.NumFiles = int64(len(newVersion.Levels[i])) - l.Size = uint64(totalSize(newVersion.Levels[i])) + l.NumFiles = int64(newVersion.Levels[i].Slice().Len()) + l.Size = int64(newVersion.Levels[i].Slice().SizeSum()) } return nil } @@ -476,12 +477,18 @@ func (vs *versionSet) logAndApply( } for i := range vs.metrics.Levels { l := &vs.metrics.Levels[i] - l.NumFiles = int64(len(newVersion.Levels[i])) - l.Size = uint64(totalSize(newVersion.Levels[i])) l.Sublevels = 0 if l.NumFiles > 0 { l.Sublevels = 1 } + if invariants.Enabled { + if count := int64(newVersion.Levels[i].Slice().Len()); l.NumFiles != count { + vs.opts.Logger.Fatalf("versionSet metrics L%d NumFiles = %d, actual count = %d", i, l.NumFiles, count) + } + if size := int64(newVersion.Levels[i].Slice().SizeSum()); l.Size != size { + vs.opts.Logger.Fatalf("versionSet metrics L%d Size = %d, actual size = %d", i, l.Size, size) + } + } } vs.metrics.Levels[0].Sublevels = int32(len(newVersion.L0Sublevels.Levels)) return nil @@ -524,8 +531,9 @@ func (vs *versionSet) createManifest( snapshot := versionEdit{ ComparerName: vs.cmpName, } - for level, fileMetadata := range vs.currentVersion().Levels { - for _, meta := range fileMetadata { + for level, levelMetadata := range vs.currentVersion().Levels { + iter := levelMetadata.Iter() + for meta := iter.First(); meta != nil; meta = iter.Next() { snapshot.NewFiles = append(snapshot.NewFiles, newFileEntry{ Level: level, Meta: meta, @@ -597,8 +605,9 @@ func (vs *versionSet) currentVersion() *version { func (vs *versionSet) addLiveFileNums(m map[FileNum]struct{}) { current := vs.currentVersion() for v := vs.versions.Front(); true; v = v.Next() { - for _, ff := range v.Levels { - for _, f := range ff { + for _, lm := range v.Levels { + iter := lm.Iter() + for f := iter.First(); f != nil; f = iter.Next() { m[f.FileNum] = struct{}{} } } @@ -619,3 +628,17 @@ func (vs *versionSet) addObsoleteLocked(obsolete []FileNum) { } vs.obsoleteTables = append(vs.obsoleteTables, obsolete...) } + +func newFileMetrics(newFiles []manifest.NewFileEntry) map[int]*LevelMetrics { + m := map[int]*LevelMetrics{} + for _, nf := range newFiles { + lm := m[nf.Level] + if lm == nil { + lm = &LevelMetrics{} + m[nf.Level] = lm + } + lm.NumFiles++ + lm.Size += int64(nf.Meta.Size) + } + return m +} diff --git a/github.com/cockroachdb/pebble/vfs/disk_usage_unix.go b/github.com/cockroachdb/pebble/vfs/disk_usage_unix.go new file mode 100644 index 0000000000..ffc3325dff --- /dev/null +++ b/github.com/cockroachdb/pebble/vfs/disk_usage_unix.go @@ -0,0 +1,19 @@ +// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +// +build darwin linux openbsd dragonfly freebsd + +package vfs + +import ( + "golang.org/x/sys/unix" +) + +func (defaultFS) GetFreeSpace(path string) (uint64, error) { + stat := unix.Statfs_t{} + if err := unix.Statfs(path, &stat); err != nil { + return 0, err + } + return uint64(stat.Bsize) * stat.Bfree, nil +} diff --git a/github.com/cockroachdb/pebble/vfs/disk_usage_windows.go b/github.com/cockroachdb/pebble/vfs/disk_usage_windows.go new file mode 100644 index 0000000000..6c34c689f4 --- /dev/null +++ b/github.com/cockroachdb/pebble/vfs/disk_usage_windows.go @@ -0,0 +1,21 @@ +// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +// +build windows + +package vfs + +import ( + "golang.org/x/sys/windows" +) + +func (defaultFS) GetFreeSpace(path string) (uint64, error) { + var freeSpace uint64 + p, err := windows.UTF16PtrFromString(path) + if err != nil { + return 0, err + } + err = windows.GetDiskFreeSpaceEx(p, &freeSpace, nil, nil) + return freeSpace, err +} diff --git a/github.com/cockroachdb/pebble/vfs/errors_unix.go b/github.com/cockroachdb/pebble/vfs/errors_unix.go new file mode 100644 index 0000000000..25453918b8 --- /dev/null +++ b/github.com/cockroachdb/pebble/vfs/errors_unix.go @@ -0,0 +1,19 @@ +// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +// +build darwin dragonfly freebsd linux openbsd + +package vfs + +import ( + "github.com/cockroachdb/errors" + + "golang.org/x/sys/unix" +) + +// IsNoSpaceError returns true if the given error indicates that the disk is +// out of space. +func IsNoSpaceError(err error) bool { + return errors.Is(err, unix.ENOSPC) +} diff --git a/github.com/cockroachdb/pebble/vfs/errors_windows.go b/github.com/cockroachdb/pebble/vfs/errors_windows.go new file mode 100644 index 0000000000..b65c31af65 --- /dev/null +++ b/github.com/cockroachdb/pebble/vfs/errors_windows.go @@ -0,0 +1,20 @@ +// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +// +build windows + +package vfs + +import ( + "github.com/cockroachdb/errors" + + "golang.org/x/sys/windows" +) + +// IsNoSpaceError returns true if the given error indicates that the disk is +// out of space. +func IsNoSpaceError(err error) bool { + return errors.Is(err, windows.ERROR_DISK_FULL) || + errors.Is(err, windows.ERROR_HANDLE_DISK_FULL) +} diff --git a/github.com/cockroachdb/pebble/vfs/mem_fs.go b/github.com/cockroachdb/pebble/vfs/mem_fs.go index 6b496cf632..bcd799b8c4 100644 --- a/github.com/cockroachdb/pebble/vfs/mem_fs.go +++ b/github.com/cockroachdb/pebble/vfs/mem_fs.go @@ -492,6 +492,11 @@ func (*MemFS) PathDir(p string) string { return path.Dir(p) } +// GetFreeSpace implements FS.GetFreeSpace. +func (*MemFS) GetFreeSpace(string) (uint64, error) { + return 0, errors.New("pebble: not supported") +} + // memNode holds a file's data or a directory's children, and implements os.FileInfo. type memNode struct { name string diff --git a/github.com/cockroachdb/pebble/vfs/vfs.go b/github.com/cockroachdb/pebble/vfs/vfs.go index 17de88c953..6a0609ccb8 100644 --- a/github.com/cockroachdb/pebble/vfs/vfs.go +++ b/github.com/cockroachdb/pebble/vfs/vfs.go @@ -112,6 +112,10 @@ type FS interface { // PathDir returns all but the last element of path, typically the path's directory. PathDir(path string) string + + // GetFreeSpace returns the amount of free disk space for the filesystem + // where path is any file or directory within that filesystem. + GetFreeSpace(path string) (uint64, error) } // Default is a FS implementation backed by the underlying operating system's diff --git a/golang.org/x/sys/unix/mkerrors.sh b/golang.org/x/sys/unix/mkerrors.sh index 08f8230d6d..53a2493126 100644 --- a/golang.org/x/sys/unix/mkerrors.sh +++ b/golang.org/x/sys/unix/mkerrors.sh @@ -107,6 +107,7 @@ includes_FreeBSD=' #include #include #include +#include #include #include #include @@ -297,6 +298,7 @@ includes_NetBSD=' #include #include #include +#include #include #include #include @@ -325,6 +327,7 @@ includes_OpenBSD=' #include #include #include +#include #include #include #include @@ -507,6 +510,8 @@ ccflags="$@" $2 ~ /^(CLOCK|TIMER)_/ || $2 ~ /^CAN_/ || $2 ~ /^CAP_/ || + $2 ~ /^CP_/ || + $2 ~ /^CPUSTATES$/ || $2 ~ /^ALG_/ || $2 ~ /^FS_(POLICY_FLAGS|KEY_DESC|ENCRYPTION_MODE|[A-Z0-9_]+_KEY_SIZE)/ || $2 ~ /^FS_IOC_.*(ENCRYPTION|VERITY|[GS]ETFLAGS)/ || diff --git a/golang.org/x/sys/unix/syscall_bsd.go b/golang.org/x/sys/unix/syscall_bsd.go index 68605db624..60bbe10adf 100644 --- a/golang.org/x/sys/unix/syscall_bsd.go +++ b/golang.org/x/sys/unix/syscall_bsd.go @@ -527,6 +527,23 @@ func SysctlClockinfo(name string) (*Clockinfo, error) { return &ci, nil } +func SysctlTimeval(name string) (*Timeval, error) { + mib, err := sysctlmib(name) + if err != nil { + return nil, err + } + + var tv Timeval + n := uintptr(unsafe.Sizeof(tv)) + if err := sysctl(mib, (*byte)(unsafe.Pointer(&tv)), &n, nil, 0); err != nil { + return nil, err + } + if n != unsafe.Sizeof(tv) { + return nil, EIO + } + return &tv, nil +} + //sys utimes(path string, timeval *[2]Timeval) (err error) func Utimes(path string, tv []Timeval) error { diff --git a/golang.org/x/sys/unix/syscall_linux.go b/golang.org/x/sys/unix/syscall_linux.go index e50e4cb276..fad483bb9d 100644 --- a/golang.org/x/sys/unix/syscall_linux.go +++ b/golang.org/x/sys/unix/syscall_linux.go @@ -2122,6 +2122,18 @@ func Klogset(typ int, arg int) (err error) { return nil } +// RemoteIovec is Iovec with the pointer replaced with an integer. +// It is used for ProcessVMReadv and ProcessVMWritev, where the pointer +// refers to a location in a different process' address space, which +// would confuse the Go garbage collector. +type RemoteIovec struct { + Base uintptr + Len int +} + +//sys ProcessVMReadv(pid int, localIov []Iovec, remoteIov []RemoteIovec, flags uint) (n int, err error) = SYS_PROCESS_VM_READV +//sys ProcessVMWritev(pid int, localIov []Iovec, remoteIov []RemoteIovec, flags uint) (n int, err error) = SYS_PROCESS_VM_WRITEV + /* * Unimplemented */ diff --git a/golang.org/x/sys/unix/zerrors_freebsd_386.go b/golang.org/x/sys/unix/zerrors_freebsd_386.go index 8482458734..3689c80848 100644 --- a/golang.org/x/sys/unix/zerrors_freebsd_386.go +++ b/golang.org/x/sys/unix/zerrors_freebsd_386.go @@ -339,6 +339,12 @@ const ( CLOCK_UPTIME_FAST = 0x8 CLOCK_UPTIME_PRECISE = 0x7 CLOCK_VIRTUAL = 0x1 + CPUSTATES = 0x5 + CP_IDLE = 0x4 + CP_INTR = 0x3 + CP_NICE = 0x1 + CP_SYS = 0x2 + CP_USER = 0x0 CREAD = 0x800 CRTSCTS = 0x30000 CS5 = 0x0 diff --git a/golang.org/x/sys/unix/zerrors_freebsd_amd64.go b/golang.org/x/sys/unix/zerrors_freebsd_amd64.go index 4acd101c3e..b8f7c3c930 100644 --- a/golang.org/x/sys/unix/zerrors_freebsd_amd64.go +++ b/golang.org/x/sys/unix/zerrors_freebsd_amd64.go @@ -339,6 +339,12 @@ const ( CLOCK_UPTIME_FAST = 0x8 CLOCK_UPTIME_PRECISE = 0x7 CLOCK_VIRTUAL = 0x1 + CPUSTATES = 0x5 + CP_IDLE = 0x4 + CP_INTR = 0x3 + CP_NICE = 0x1 + CP_SYS = 0x2 + CP_USER = 0x0 CREAD = 0x800 CRTSCTS = 0x30000 CS5 = 0x0 diff --git a/golang.org/x/sys/unix/zerrors_freebsd_arm.go b/golang.org/x/sys/unix/zerrors_freebsd_arm.go index e4719873b9..be14bb1a4c 100644 --- a/golang.org/x/sys/unix/zerrors_freebsd_arm.go +++ b/golang.org/x/sys/unix/zerrors_freebsd_arm.go @@ -339,6 +339,12 @@ const ( CLOCK_UPTIME_FAST = 0x8 CLOCK_UPTIME_PRECISE = 0x7 CLOCK_VIRTUAL = 0x1 + CPUSTATES = 0x5 + CP_IDLE = 0x4 + CP_INTR = 0x3 + CP_NICE = 0x1 + CP_SYS = 0x2 + CP_USER = 0x0 CREAD = 0x800 CRTSCTS = 0x30000 CS5 = 0x0 diff --git a/golang.org/x/sys/unix/zerrors_freebsd_arm64.go b/golang.org/x/sys/unix/zerrors_freebsd_arm64.go index 5e49769d96..7ce9c0081a 100644 --- a/golang.org/x/sys/unix/zerrors_freebsd_arm64.go +++ b/golang.org/x/sys/unix/zerrors_freebsd_arm64.go @@ -339,6 +339,12 @@ const ( CLOCK_UPTIME_FAST = 0x8 CLOCK_UPTIME_PRECISE = 0x7 CLOCK_VIRTUAL = 0x1 + CPUSTATES = 0x5 + CP_IDLE = 0x4 + CP_INTR = 0x3 + CP_NICE = 0x1 + CP_SYS = 0x2 + CP_USER = 0x0 CREAD = 0x800 CRTSCTS = 0x30000 CS5 = 0x0 diff --git a/golang.org/x/sys/unix/zerrors_netbsd_386.go b/golang.org/x/sys/unix/zerrors_netbsd_386.go index 96b9b8ab30..20f3a5799f 100644 --- a/golang.org/x/sys/unix/zerrors_netbsd_386.go +++ b/golang.org/x/sys/unix/zerrors_netbsd_386.go @@ -158,6 +158,12 @@ const ( CLONE_SIGHAND = 0x800 CLONE_VFORK = 0x4000 CLONE_VM = 0x100 + CPUSTATES = 0x5 + CP_IDLE = 0x4 + CP_INTR = 0x3 + CP_NICE = 0x1 + CP_SYS = 0x2 + CP_USER = 0x0 CREAD = 0x800 CRTSCTS = 0x10000 CS5 = 0x0 diff --git a/golang.org/x/sys/unix/zerrors_netbsd_amd64.go b/golang.org/x/sys/unix/zerrors_netbsd_amd64.go index ed522a84e8..90b8fcd29c 100644 --- a/golang.org/x/sys/unix/zerrors_netbsd_amd64.go +++ b/golang.org/x/sys/unix/zerrors_netbsd_amd64.go @@ -158,6 +158,12 @@ const ( CLONE_SIGHAND = 0x800 CLONE_VFORK = 0x4000 CLONE_VM = 0x100 + CPUSTATES = 0x5 + CP_IDLE = 0x4 + CP_INTR = 0x3 + CP_NICE = 0x1 + CP_SYS = 0x2 + CP_USER = 0x0 CREAD = 0x800 CRTSCTS = 0x10000 CS5 = 0x0 diff --git a/golang.org/x/sys/unix/zerrors_netbsd_arm.go b/golang.org/x/sys/unix/zerrors_netbsd_arm.go index c8d36fe998..c5c03993b6 100644 --- a/golang.org/x/sys/unix/zerrors_netbsd_arm.go +++ b/golang.org/x/sys/unix/zerrors_netbsd_arm.go @@ -150,6 +150,12 @@ const ( BRKINT = 0x2 CFLUSH = 0xf CLOCAL = 0x8000 + CPUSTATES = 0x5 + CP_IDLE = 0x4 + CP_INTR = 0x3 + CP_NICE = 0x1 + CP_SYS = 0x2 + CP_USER = 0x0 CREAD = 0x800 CRTSCTS = 0x10000 CS5 = 0x0 diff --git a/golang.org/x/sys/unix/zerrors_netbsd_arm64.go b/golang.org/x/sys/unix/zerrors_netbsd_arm64.go index f1c146a74c..14dd3c1d1e 100644 --- a/golang.org/x/sys/unix/zerrors_netbsd_arm64.go +++ b/golang.org/x/sys/unix/zerrors_netbsd_arm64.go @@ -158,6 +158,12 @@ const ( CLONE_SIGHAND = 0x800 CLONE_VFORK = 0x4000 CLONE_VM = 0x100 + CPUSTATES = 0x5 + CP_IDLE = 0x4 + CP_INTR = 0x3 + CP_NICE = 0x1 + CP_SYS = 0x2 + CP_USER = 0x0 CREAD = 0x800 CRTSCTS = 0x10000 CS5 = 0x0 diff --git a/golang.org/x/sys/unix/zerrors_openbsd_386.go b/golang.org/x/sys/unix/zerrors_openbsd_386.go index 5402bd55ce..c865a10df4 100644 --- a/golang.org/x/sys/unix/zerrors_openbsd_386.go +++ b/golang.org/x/sys/unix/zerrors_openbsd_386.go @@ -146,6 +146,13 @@ const ( BRKINT = 0x2 CFLUSH = 0xf CLOCAL = 0x8000 + CPUSTATES = 0x6 + CP_IDLE = 0x5 + CP_INTR = 0x4 + CP_NICE = 0x1 + CP_SPIN = 0x3 + CP_SYS = 0x2 + CP_USER = 0x0 CREAD = 0x800 CRTSCTS = 0x10000 CS5 = 0x0 diff --git a/golang.org/x/sys/unix/zerrors_openbsd_amd64.go b/golang.org/x/sys/unix/zerrors_openbsd_amd64.go index ffaf2d2f9f..9db6b2fb6e 100644 --- a/golang.org/x/sys/unix/zerrors_openbsd_amd64.go +++ b/golang.org/x/sys/unix/zerrors_openbsd_amd64.go @@ -153,6 +153,13 @@ const ( CLOCK_REALTIME = 0x0 CLOCK_THREAD_CPUTIME_ID = 0x4 CLOCK_UPTIME = 0x5 + CPUSTATES = 0x6 + CP_IDLE = 0x5 + CP_INTR = 0x4 + CP_NICE = 0x1 + CP_SPIN = 0x3 + CP_SYS = 0x2 + CP_USER = 0x0 CREAD = 0x800 CRTSCTS = 0x10000 CS5 = 0x0 diff --git a/golang.org/x/sys/unix/zerrors_openbsd_arm.go b/golang.org/x/sys/unix/zerrors_openbsd_arm.go index 7aa796a642..7072526a64 100644 --- a/golang.org/x/sys/unix/zerrors_openbsd_arm.go +++ b/golang.org/x/sys/unix/zerrors_openbsd_arm.go @@ -146,6 +146,13 @@ const ( BRKINT = 0x2 CFLUSH = 0xf CLOCAL = 0x8000 + CPUSTATES = 0x6 + CP_IDLE = 0x5 + CP_INTR = 0x4 + CP_NICE = 0x1 + CP_SPIN = 0x3 + CP_SYS = 0x2 + CP_USER = 0x0 CREAD = 0x800 CRTSCTS = 0x10000 CS5 = 0x0 diff --git a/golang.org/x/sys/unix/zerrors_openbsd_arm64.go b/golang.org/x/sys/unix/zerrors_openbsd_arm64.go index 1792d3f13e..ac5efbe5ac 100644 --- a/golang.org/x/sys/unix/zerrors_openbsd_arm64.go +++ b/golang.org/x/sys/unix/zerrors_openbsd_arm64.go @@ -156,6 +156,13 @@ const ( CLOCK_REALTIME = 0x0 CLOCK_THREAD_CPUTIME_ID = 0x4 CLOCK_UPTIME = 0x5 + CPUSTATES = 0x6 + CP_IDLE = 0x5 + CP_INTR = 0x4 + CP_NICE = 0x1 + CP_SPIN = 0x3 + CP_SYS = 0x2 + CP_USER = 0x0 CREAD = 0x800 CRTSCTS = 0x10000 CS5 = 0x0 diff --git a/golang.org/x/sys/unix/zsyscall_linux.go b/golang.org/x/sys/unix/zsyscall_linux.go index df217825f0..f6603de4f5 100644 --- a/golang.org/x/sys/unix/zsyscall_linux.go +++ b/golang.org/x/sys/unix/zsyscall_linux.go @@ -1847,6 +1847,52 @@ func openByHandleAt(mountFD int, fh *fileHandle, flags int) (fd int, err error) // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ProcessVMReadv(pid int, localIov []Iovec, remoteIov []RemoteIovec, flags uint) (n int, err error) { + var _p0 unsafe.Pointer + if len(localIov) > 0 { + _p0 = unsafe.Pointer(&localIov[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + var _p1 unsafe.Pointer + if len(remoteIov) > 0 { + _p1 = unsafe.Pointer(&remoteIov[0]) + } else { + _p1 = unsafe.Pointer(&_zero) + } + r0, _, e1 := Syscall6(SYS_PROCESS_VM_READV, uintptr(pid), uintptr(_p0), uintptr(len(localIov)), uintptr(_p1), uintptr(len(remoteIov)), uintptr(flags)) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func ProcessVMWritev(pid int, localIov []Iovec, remoteIov []RemoteIovec, flags uint) (n int, err error) { + var _p0 unsafe.Pointer + if len(localIov) > 0 { + _p0 = unsafe.Pointer(&localIov[0]) + } else { + _p0 = unsafe.Pointer(&_zero) + } + var _p1 unsafe.Pointer + if len(remoteIov) > 0 { + _p1 = unsafe.Pointer(&remoteIov[0]) + } else { + _p1 = unsafe.Pointer(&_zero) + } + r0, _, e1 := Syscall6(SYS_PROCESS_VM_WRITEV, uintptr(pid), uintptr(_p0), uintptr(len(localIov)), uintptr(_p1), uintptr(len(remoteIov)), uintptr(flags)) + n = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func pipe2(p *[2]_C_int, flags int) (err error) { _, _, e1 := RawSyscall(SYS_PIPE2, uintptr(unsafe.Pointer(p)), uintptr(flags), 0) if e1 != 0 { diff --git a/modules.txt b/modules.txt index 608668c1e9..14d7d9a89e 100644 --- a/modules.txt +++ b/modules.txt @@ -236,7 +236,7 @@ github.com/cockroachdb/gostdlib/x/tools/internal/span # github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f ## explicit github.com/cockroachdb/logtags -# github.com/cockroachdb/pebble v0.0.0-20200716211707-073e15ee2ccf +# github.com/cockroachdb/pebble v0.0.0-20200729202514-cfd19d33cdca ## explicit github.com/cockroachdb/pebble github.com/cockroachdb/pebble/bloom @@ -912,7 +912,7 @@ golang.org/x/perf/storage/benchfmt ## explicit golang.org/x/sync/errgroup golang.org/x/sync/syncmap -# golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae +# golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1 ## explicit golang.org/x/sys/cpu golang.org/x/sys/internal/unsafeheader From e7b77c675795b893d0e3177bef804424ab6d6b5e Mon Sep 17 00:00:00 2001 From: Matt Jibson Date: Tue, 4 Aug 2020 16:04:02 -0600 Subject: [PATCH 20/49] protoc-gen-doc --- github.com/Masterminds/goutils/.travis.yml | 18 + github.com/Masterminds/goutils/CHANGELOG.md | 8 + github.com/Masterminds/goutils/LICENSE.txt | 202 + github.com/Masterminds/goutils/README.md | 70 + github.com/Masterminds/goutils/appveyor.yml | 21 + .../goutils/cryptorandomstringutils.go | 251 ++ .../Masterminds/goutils/randomstringutils.go | 268 ++ github.com/Masterminds/goutils/stringutils.go | 224 ++ github.com/Masterminds/goutils/wordutils.go | 357 ++ github.com/Masterminds/semver/.travis.yml | 29 + github.com/Masterminds/semver/CHANGELOG.md | 109 + github.com/Masterminds/semver/LICENSE.txt | 19 + github.com/Masterminds/semver/Makefile | 36 + github.com/Masterminds/semver/README.md | 194 + github.com/Masterminds/semver/appveyor.yml | 44 + github.com/Masterminds/semver/collection.go | 24 + github.com/Masterminds/semver/constraints.go | 423 ++ github.com/Masterminds/semver/doc.go | 115 + github.com/Masterminds/semver/version.go | 425 ++ github.com/Masterminds/semver/version_fuzz.go | 10 + github.com/Masterminds/sprig/.gitignore | 2 + github.com/Masterminds/sprig/.travis.yml | 26 + github.com/Masterminds/sprig/CHANGELOG.md | 282 ++ github.com/Masterminds/sprig/LICENSE.txt | 20 + github.com/Masterminds/sprig/Makefile | 13 + github.com/Masterminds/sprig/README.md | 78 + github.com/Masterminds/sprig/appveyor.yml | 26 + github.com/Masterminds/sprig/crypto.go | 502 +++ github.com/Masterminds/sprig/date.go | 83 + github.com/Masterminds/sprig/defaults.go | 83 + github.com/Masterminds/sprig/dict.go | 119 + github.com/Masterminds/sprig/doc.go | 19 + github.com/Masterminds/sprig/functions.go | 306 ++ github.com/Masterminds/sprig/glide.yaml | 19 + github.com/Masterminds/sprig/list.go | 311 ++ github.com/Masterminds/sprig/network.go | 12 + github.com/Masterminds/sprig/numeric.go | 169 + github.com/Masterminds/sprig/reflect.go | 28 + github.com/Masterminds/sprig/regex.go | 35 + github.com/Masterminds/sprig/semver.go | 23 + github.com/Masterminds/sprig/strings.go | 233 ++ github.com/Masterminds/sprig/url.go | 66 + .../envoyproxy/protoc-gen-validate/LICENSE | 202 + .../envoyproxy/protoc-gen-validate/NOTICE | 4 + .../protoc-gen-validate/validate/BUILD | 49 + .../protoc-gen-validate/validate/validate.h | 124 + .../validate/validate.pb.go | 3575 +++++++++++++++++ .../validate/validate.proto | 768 ++++ github.com/google/uuid/.travis.yml | 9 + github.com/google/uuid/CONTRIBUTING.md | 10 + github.com/google/uuid/CONTRIBUTORS | 9 + github.com/google/uuid/LICENSE | 27 + github.com/google/uuid/README.md | 19 + github.com/google/uuid/dce.go | 80 + github.com/google/uuid/doc.go | 12 + github.com/google/uuid/go.mod | 1 + github.com/google/uuid/hash.go | 53 + github.com/google/uuid/marshal.go | 37 + github.com/google/uuid/node.go | 90 + github.com/google/uuid/node_js.go | 12 + github.com/google/uuid/node_net.go | 33 + github.com/google/uuid/sql.go | 59 + github.com/google/uuid/time.go | 123 + github.com/google/uuid/util.go | 43 + github.com/google/uuid/uuid.go | 245 ++ github.com/google/uuid/version1.go | 44 + github.com/google/uuid/version4.go | 38 + github.com/huandu/xstrings/.gitignore | 24 + github.com/huandu/xstrings/.travis.yml | 7 + github.com/huandu/xstrings/CONTRIBUTING.md | 23 + github.com/huandu/xstrings/LICENSE | 22 + github.com/huandu/xstrings/README.md | 117 + github.com/huandu/xstrings/common.go | 25 + github.com/huandu/xstrings/convert.go | 404 ++ github.com/huandu/xstrings/count.go | 120 + github.com/huandu/xstrings/doc.go | 8 + github.com/huandu/xstrings/format.go | 170 + github.com/huandu/xstrings/go.mod | 3 + github.com/huandu/xstrings/manipulate.go | 217 + github.com/huandu/xstrings/translate.go | 547 +++ github.com/imdario/mergo/.deepsource.toml | 12 + github.com/imdario/mergo/.gitignore | 33 + github.com/imdario/mergo/.travis.yml | 9 + github.com/imdario/mergo/CODE_OF_CONDUCT.md | 46 + github.com/imdario/mergo/LICENSE | 28 + github.com/imdario/mergo/README.md | 238 ++ github.com/imdario/mergo/doc.go | 44 + github.com/imdario/mergo/map.go | 176 + github.com/imdario/mergo/merge.go | 338 ++ github.com/imdario/mergo/mergo.go | 97 + .../mitchellh/copystructure/.travis.yml | 12 + github.com/mitchellh/copystructure/LICENSE | 21 + github.com/mitchellh/copystructure/README.md | 21 + .../mitchellh/copystructure/copier_time.go | 15 + .../mitchellh/copystructure/copystructure.go | 548 +++ github.com/mitchellh/copystructure/go.mod | 3 + github.com/mitchellh/copystructure/go.sum | 2 + .../mwitkow/go-proto-validators/.gitignore | 89 + .../mwitkow/go-proto-validators/.travis.yml | 18 + .../mwitkow/go-proto-validators/LICENSE.txt | 201 + .../mwitkow/go-proto-validators/Makefile | 45 + .../mwitkow/go-proto-validators/README.md | 132 + .../go-proto-validators/examples/nested.proto | 15 + .../mwitkow/go-proto-validators/helper.go | 39 + .../go-proto-validators/install_protoc.sh | 22 + .../test/validator_proto2.proto | 78 + .../test/validator_proto3.proto | 72 + .../test/validator_proto3_map.proto | 26 + .../test/validator_proto3_oneof.proto | 28 + .../go-proto-validators/validator.pb.go | 238 ++ .../go-proto-validators/validator.proto | 66 + .../pseudomuto/protoc-gen-doc/.dockerignore | 2 + .../pseudomuto/protoc-gen-doc/.gitignore | 8 + .../pseudomuto/protoc-gen-doc/.gofmtignore | 2 + .../pseudomuto/protoc-gen-doc/.travis.yml | 44 + .../pseudomuto/protoc-gen-doc/CHANGELOG.md | 189 + .../pseudomuto/protoc-gen-doc/CONTRIBUTING.md | 79 + .../pseudomuto/protoc-gen-doc/Dockerfile | 21 + .../pseudomuto/protoc-gen-doc/LICENSE.md | 21 + github.com/pseudomuto/protoc-gen-doc/Makefile | 81 + .../pseudomuto/protoc-gen-doc/README.md | 203 + .../cmd/protoc-gen-doc/flags.go | 98 + .../protoc-gen-doc/cmd/protoc-gen-doc/main.go | 53 + github.com/pseudomuto/protoc-gen-doc/doc.go | 19 + .../examples/proto/Booking.proto | 64 + .../examples/proto/Customer.proto | 37 + .../examples/proto/IgnoreMe.proto | 14 + .../examples/proto/Vehicle.proto | 88 + .../envoyproxy_validate.go | 91 + .../protoc-gen-doc/extensions/extensions.go | 35 + .../google_api_http/google_api_http.go | 65 + .../extensions/lyft_validate/alias.go | 9 + .../validator_field/validator_field.go | 83 + .../pseudomuto/protoc-gen-doc/filters.go | 38 + .../protoc-gen-doc/fixtures/Booking.proto | 84 + .../protoc-gen-doc/fixtures/Vehicle.proto | 147 + github.com/pseudomuto/protoc-gen-doc/go.mod | 22 + github.com/pseudomuto/protoc-gen-doc/go.sum | 38 + .../pseudomuto/protoc-gen-doc/plugin.go | 129 + .../pseudomuto/protoc-gen-doc/renderer.go | 150 + .../pseudomuto/protoc-gen-doc/resources.go | 44 + .../pseudomuto/protoc-gen-doc/revive.toml | 51 + .../pseudomuto/protoc-gen-doc/template.go | 592 +++ .../validate/validate.proto | 763 ++++ .../go-proto-validators/validator.proto | 67 + .../pseudomuto/protokit/fixtures/extend.proto | 54 + .../pseudomuto/protoc-gen-doc/version.go | 4 + github.com/pseudomuto/protokit/.gitignore | 3 + github.com/pseudomuto/protokit/.gofmtignore | 2 + github.com/pseudomuto/protokit/.travis.yml | 32 + github.com/pseudomuto/protokit/CHANGELOG.md | 41 + github.com/pseudomuto/protokit/Gopkg.lock | 46 + github.com/pseudomuto/protokit/Gopkg.toml | 37 + github.com/pseudomuto/protokit/LICENSE | 21 + github.com/pseudomuto/protokit/Makefile | 29 + github.com/pseudomuto/protokit/README.md | 71 + github.com/pseudomuto/protokit/comments.go | 93 + github.com/pseudomuto/protokit/context.go | 58 + github.com/pseudomuto/protokit/doc.go | 13 + .../protokit/examples/jsonator/sample.proto | 19 + .../protokit/examples/jsonator/sample2.proto | 19 + .../protokit/fixtures/booking.proto | 89 + .../pseudomuto/protokit/fixtures/extend.proto | 54 + .../pseudomuto/protokit/fixtures/todo.proto | 101 + .../protokit/fixtures/todo_import.proto | 14 + github.com/pseudomuto/protokit/parser.go | 289 ++ github.com/pseudomuto/protokit/plugin.go | 67 + github.com/pseudomuto/protokit/tools.json | 13 + github.com/pseudomuto/protokit/types.go | 350 ++ github.com/pseudomuto/protokit/version.go | 4 + golang.org/x/crypto/scrypt/scrypt.go | 213 + modules.txt | 30 + 172 files changed, 20564 insertions(+) create mode 100644 github.com/Masterminds/goutils/.travis.yml create mode 100644 github.com/Masterminds/goutils/CHANGELOG.md create mode 100644 github.com/Masterminds/goutils/LICENSE.txt create mode 100644 github.com/Masterminds/goutils/README.md create mode 100644 github.com/Masterminds/goutils/appveyor.yml create mode 100644 github.com/Masterminds/goutils/cryptorandomstringutils.go create mode 100644 github.com/Masterminds/goutils/randomstringutils.go create mode 100644 github.com/Masterminds/goutils/stringutils.go create mode 100644 github.com/Masterminds/goutils/wordutils.go create mode 100644 github.com/Masterminds/semver/.travis.yml create mode 100644 github.com/Masterminds/semver/CHANGELOG.md create mode 100644 github.com/Masterminds/semver/LICENSE.txt create mode 100644 github.com/Masterminds/semver/Makefile create mode 100644 github.com/Masterminds/semver/README.md create mode 100644 github.com/Masterminds/semver/appveyor.yml create mode 100644 github.com/Masterminds/semver/collection.go create mode 100644 github.com/Masterminds/semver/constraints.go create mode 100644 github.com/Masterminds/semver/doc.go create mode 100644 github.com/Masterminds/semver/version.go create mode 100644 github.com/Masterminds/semver/version_fuzz.go create mode 100644 github.com/Masterminds/sprig/.gitignore create mode 100644 github.com/Masterminds/sprig/.travis.yml create mode 100644 github.com/Masterminds/sprig/CHANGELOG.md create mode 100644 github.com/Masterminds/sprig/LICENSE.txt create mode 100644 github.com/Masterminds/sprig/Makefile create mode 100644 github.com/Masterminds/sprig/README.md create mode 100644 github.com/Masterminds/sprig/appveyor.yml create mode 100644 github.com/Masterminds/sprig/crypto.go create mode 100644 github.com/Masterminds/sprig/date.go create mode 100644 github.com/Masterminds/sprig/defaults.go create mode 100644 github.com/Masterminds/sprig/dict.go create mode 100644 github.com/Masterminds/sprig/doc.go create mode 100644 github.com/Masterminds/sprig/functions.go create mode 100644 github.com/Masterminds/sprig/glide.yaml create mode 100644 github.com/Masterminds/sprig/list.go create mode 100644 github.com/Masterminds/sprig/network.go create mode 100644 github.com/Masterminds/sprig/numeric.go create mode 100644 github.com/Masterminds/sprig/reflect.go create mode 100644 github.com/Masterminds/sprig/regex.go create mode 100644 github.com/Masterminds/sprig/semver.go create mode 100644 github.com/Masterminds/sprig/strings.go create mode 100644 github.com/Masterminds/sprig/url.go create mode 100644 github.com/envoyproxy/protoc-gen-validate/LICENSE create mode 100644 github.com/envoyproxy/protoc-gen-validate/NOTICE create mode 100644 github.com/envoyproxy/protoc-gen-validate/validate/BUILD create mode 100644 github.com/envoyproxy/protoc-gen-validate/validate/validate.h create mode 100644 github.com/envoyproxy/protoc-gen-validate/validate/validate.pb.go create mode 100644 github.com/envoyproxy/protoc-gen-validate/validate/validate.proto create mode 100644 github.com/google/uuid/.travis.yml create mode 100644 github.com/google/uuid/CONTRIBUTING.md create mode 100644 github.com/google/uuid/CONTRIBUTORS create mode 100644 github.com/google/uuid/LICENSE create mode 100644 github.com/google/uuid/README.md create mode 100644 github.com/google/uuid/dce.go create mode 100644 github.com/google/uuid/doc.go create mode 100644 github.com/google/uuid/go.mod create mode 100644 github.com/google/uuid/hash.go create mode 100644 github.com/google/uuid/marshal.go create mode 100644 github.com/google/uuid/node.go create mode 100644 github.com/google/uuid/node_js.go create mode 100644 github.com/google/uuid/node_net.go create mode 100644 github.com/google/uuid/sql.go create mode 100644 github.com/google/uuid/time.go create mode 100644 github.com/google/uuid/util.go create mode 100644 github.com/google/uuid/uuid.go create mode 100644 github.com/google/uuid/version1.go create mode 100644 github.com/google/uuid/version4.go create mode 100644 github.com/huandu/xstrings/.gitignore create mode 100644 github.com/huandu/xstrings/.travis.yml create mode 100644 github.com/huandu/xstrings/CONTRIBUTING.md create mode 100644 github.com/huandu/xstrings/LICENSE create mode 100644 github.com/huandu/xstrings/README.md create mode 100644 github.com/huandu/xstrings/common.go create mode 100644 github.com/huandu/xstrings/convert.go create mode 100644 github.com/huandu/xstrings/count.go create mode 100644 github.com/huandu/xstrings/doc.go create mode 100644 github.com/huandu/xstrings/format.go create mode 100644 github.com/huandu/xstrings/go.mod create mode 100644 github.com/huandu/xstrings/manipulate.go create mode 100644 github.com/huandu/xstrings/translate.go create mode 100644 github.com/imdario/mergo/.deepsource.toml create mode 100644 github.com/imdario/mergo/.gitignore create mode 100644 github.com/imdario/mergo/.travis.yml create mode 100644 github.com/imdario/mergo/CODE_OF_CONDUCT.md create mode 100644 github.com/imdario/mergo/LICENSE create mode 100644 github.com/imdario/mergo/README.md create mode 100644 github.com/imdario/mergo/doc.go create mode 100644 github.com/imdario/mergo/map.go create mode 100644 github.com/imdario/mergo/merge.go create mode 100644 github.com/imdario/mergo/mergo.go create mode 100644 github.com/mitchellh/copystructure/.travis.yml create mode 100644 github.com/mitchellh/copystructure/LICENSE create mode 100644 github.com/mitchellh/copystructure/README.md create mode 100644 github.com/mitchellh/copystructure/copier_time.go create mode 100644 github.com/mitchellh/copystructure/copystructure.go create mode 100644 github.com/mitchellh/copystructure/go.mod create mode 100644 github.com/mitchellh/copystructure/go.sum create mode 100644 github.com/mwitkow/go-proto-validators/.gitignore create mode 100644 github.com/mwitkow/go-proto-validators/.travis.yml create mode 100644 github.com/mwitkow/go-proto-validators/LICENSE.txt create mode 100644 github.com/mwitkow/go-proto-validators/Makefile create mode 100644 github.com/mwitkow/go-proto-validators/README.md create mode 100644 github.com/mwitkow/go-proto-validators/examples/nested.proto create mode 100644 github.com/mwitkow/go-proto-validators/helper.go create mode 100644 github.com/mwitkow/go-proto-validators/install_protoc.sh create mode 100644 github.com/mwitkow/go-proto-validators/test/validator_proto2.proto create mode 100644 github.com/mwitkow/go-proto-validators/test/validator_proto3.proto create mode 100644 github.com/mwitkow/go-proto-validators/test/validator_proto3_map.proto create mode 100644 github.com/mwitkow/go-proto-validators/test/validator_proto3_oneof.proto create mode 100644 github.com/mwitkow/go-proto-validators/validator.pb.go create mode 100644 github.com/mwitkow/go-proto-validators/validator.proto create mode 100644 github.com/pseudomuto/protoc-gen-doc/.dockerignore create mode 100644 github.com/pseudomuto/protoc-gen-doc/.gitignore create mode 100644 github.com/pseudomuto/protoc-gen-doc/.gofmtignore create mode 100644 github.com/pseudomuto/protoc-gen-doc/.travis.yml create mode 100644 github.com/pseudomuto/protoc-gen-doc/CHANGELOG.md create mode 100644 github.com/pseudomuto/protoc-gen-doc/CONTRIBUTING.md create mode 100644 github.com/pseudomuto/protoc-gen-doc/Dockerfile create mode 100644 github.com/pseudomuto/protoc-gen-doc/LICENSE.md create mode 100644 github.com/pseudomuto/protoc-gen-doc/Makefile create mode 100644 github.com/pseudomuto/protoc-gen-doc/README.md create mode 100644 github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc/flags.go create mode 100644 github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc/main.go create mode 100644 github.com/pseudomuto/protoc-gen-doc/doc.go create mode 100644 github.com/pseudomuto/protoc-gen-doc/examples/proto/Booking.proto create mode 100644 github.com/pseudomuto/protoc-gen-doc/examples/proto/Customer.proto create mode 100644 github.com/pseudomuto/protoc-gen-doc/examples/proto/IgnoreMe.proto create mode 100644 github.com/pseudomuto/protoc-gen-doc/examples/proto/Vehicle.proto create mode 100644 github.com/pseudomuto/protoc-gen-doc/extensions/envoyproxy_validate/envoyproxy_validate.go create mode 100644 github.com/pseudomuto/protoc-gen-doc/extensions/extensions.go create mode 100644 github.com/pseudomuto/protoc-gen-doc/extensions/google_api_http/google_api_http.go create mode 100644 github.com/pseudomuto/protoc-gen-doc/extensions/lyft_validate/alias.go create mode 100644 github.com/pseudomuto/protoc-gen-doc/extensions/validator_field/validator_field.go create mode 100644 github.com/pseudomuto/protoc-gen-doc/filters.go create mode 100644 github.com/pseudomuto/protoc-gen-doc/fixtures/Booking.proto create mode 100644 github.com/pseudomuto/protoc-gen-doc/fixtures/Vehicle.proto create mode 100644 github.com/pseudomuto/protoc-gen-doc/go.mod create mode 100644 github.com/pseudomuto/protoc-gen-doc/go.sum create mode 100644 github.com/pseudomuto/protoc-gen-doc/plugin.go create mode 100644 github.com/pseudomuto/protoc-gen-doc/renderer.go create mode 100644 github.com/pseudomuto/protoc-gen-doc/resources.go create mode 100644 github.com/pseudomuto/protoc-gen-doc/revive.toml create mode 100644 github.com/pseudomuto/protoc-gen-doc/template.go create mode 100644 github.com/pseudomuto/protoc-gen-doc/thirdparty/github.com/envoyproxy/protoc-gen-validate/validate/validate.proto create mode 100644 github.com/pseudomuto/protoc-gen-doc/thirdparty/github.com/mwitkow/go-proto-validators/validator.proto create mode 100644 github.com/pseudomuto/protoc-gen-doc/thirdparty/github.com/pseudomuto/protokit/fixtures/extend.proto create mode 100644 github.com/pseudomuto/protoc-gen-doc/version.go create mode 100644 github.com/pseudomuto/protokit/.gitignore create mode 100644 github.com/pseudomuto/protokit/.gofmtignore create mode 100644 github.com/pseudomuto/protokit/.travis.yml create mode 100644 github.com/pseudomuto/protokit/CHANGELOG.md create mode 100644 github.com/pseudomuto/protokit/Gopkg.lock create mode 100644 github.com/pseudomuto/protokit/Gopkg.toml create mode 100644 github.com/pseudomuto/protokit/LICENSE create mode 100644 github.com/pseudomuto/protokit/Makefile create mode 100644 github.com/pseudomuto/protokit/README.md create mode 100644 github.com/pseudomuto/protokit/comments.go create mode 100644 github.com/pseudomuto/protokit/context.go create mode 100644 github.com/pseudomuto/protokit/doc.go create mode 100644 github.com/pseudomuto/protokit/examples/jsonator/sample.proto create mode 100644 github.com/pseudomuto/protokit/examples/jsonator/sample2.proto create mode 100644 github.com/pseudomuto/protokit/fixtures/booking.proto create mode 100644 github.com/pseudomuto/protokit/fixtures/extend.proto create mode 100644 github.com/pseudomuto/protokit/fixtures/todo.proto create mode 100644 github.com/pseudomuto/protokit/fixtures/todo_import.proto create mode 100644 github.com/pseudomuto/protokit/parser.go create mode 100644 github.com/pseudomuto/protokit/plugin.go create mode 100644 github.com/pseudomuto/protokit/tools.json create mode 100644 github.com/pseudomuto/protokit/types.go create mode 100644 github.com/pseudomuto/protokit/version.go create mode 100644 golang.org/x/crypto/scrypt/scrypt.go diff --git a/github.com/Masterminds/goutils/.travis.yml b/github.com/Masterminds/goutils/.travis.yml new file mode 100644 index 0000000000..4025e01ec4 --- /dev/null +++ b/github.com/Masterminds/goutils/.travis.yml @@ -0,0 +1,18 @@ +language: go + +go: + - 1.6 + - 1.7 + - 1.8 + - tip + +script: + - go test -v + +notifications: + webhooks: + urls: + - https://webhooks.gitter.im/e/06e3328629952dabe3e0 + on_success: change # options: [always|never|change] default: always + on_failure: always # options: [always|never|change] default: always + on_start: never # options: [always|never|change] default: always diff --git a/github.com/Masterminds/goutils/CHANGELOG.md b/github.com/Masterminds/goutils/CHANGELOG.md new file mode 100644 index 0000000000..d700ec47f2 --- /dev/null +++ b/github.com/Masterminds/goutils/CHANGELOG.md @@ -0,0 +1,8 @@ +# 1.0.1 (2017-05-31) + +## Fixed +- #21: Fix generation of alphanumeric strings (thanks @dbarranco) + +# 1.0.0 (2014-04-30) + +- Initial release. diff --git a/github.com/Masterminds/goutils/LICENSE.txt b/github.com/Masterminds/goutils/LICENSE.txt new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/github.com/Masterminds/goutils/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/github.com/Masterminds/goutils/README.md b/github.com/Masterminds/goutils/README.md new file mode 100644 index 0000000000..163ffe72a8 --- /dev/null +++ b/github.com/Masterminds/goutils/README.md @@ -0,0 +1,70 @@ +GoUtils +=========== +[![Stability: Maintenance](https://masterminds.github.io/stability/maintenance.svg)](https://masterminds.github.io/stability/maintenance.html) +[![GoDoc](https://godoc.org/github.com/Masterminds/goutils?status.png)](https://godoc.org/github.com/Masterminds/goutils) [![Build Status](https://travis-ci.org/Masterminds/goutils.svg?branch=master)](https://travis-ci.org/Masterminds/goutils) [![Build status](https://ci.appveyor.com/api/projects/status/sc2b1ew0m7f0aiju?svg=true)](https://ci.appveyor.com/project/mattfarina/goutils) + + +GoUtils provides users with utility functions to manipulate strings in various ways. It is a Go implementation of some +string manipulation libraries of Java Apache Commons. GoUtils includes the following Java Apache Commons classes: +* WordUtils +* RandomStringUtils +* StringUtils (partial implementation) + +## Installation +If you have Go set up on your system, from the GOPATH directory within the command line/terminal, enter this: + + go get github.com/Masterminds/goutils + +If you do not have Go set up on your system, please follow the [Go installation directions from the documenation](http://golang.org/doc/install), and then follow the instructions above to install GoUtils. + + +## Documentation +GoUtils doc is available here: [![GoDoc](https://godoc.org/github.com/Masterminds/goutils?status.png)](https://godoc.org/github.com/Masterminds/goutils) + + +## Usage +The code snippets below show examples of how to use GoUtils. Some functions return errors while others do not. The first instance below, which does not return an error, is the `Initials` function (located within the `wordutils.go` file). + + package main + + import ( + "fmt" + "github.com/Masterminds/goutils" + ) + + func main() { + + // EXAMPLE 1: A goutils function which returns no errors + fmt.Println (goutils.Initials("John Doe Foo")) // Prints out "JDF" + + } +Some functions return errors mainly due to illegal arguements used as parameters. The code example below illustrates how to deal with function that returns an error. In this instance, the function is the `Random` function (located within the `randomstringutils.go` file). + + package main + + import ( + "fmt" + "github.com/Masterminds/goutils" + ) + + func main() { + + // EXAMPLE 2: A goutils function which returns an error + rand1, err1 := goutils.Random (-1, 0, 0, true, true) + + if err1 != nil { + fmt.Println(err1) // Prints out error message because -1 was entered as the first parameter in goutils.Random(...) + } else { + fmt.Println(rand1) + } + + } + +## License +GoUtils is licensed under the Apache License, Version 2.0. Please check the LICENSE.txt file or visit http://www.apache.org/licenses/LICENSE-2.0 for a copy of the license. + +## Issue Reporting +Make suggestions or report issues using the Git issue tracker: https://github.com/Masterminds/goutils/issues + +## Website +* [GoUtils webpage](http://Masterminds.github.io/goutils/) diff --git a/github.com/Masterminds/goutils/appveyor.yml b/github.com/Masterminds/goutils/appveyor.yml new file mode 100644 index 0000000000..657564a847 --- /dev/null +++ b/github.com/Masterminds/goutils/appveyor.yml @@ -0,0 +1,21 @@ +version: build-{build}.{branch} + +clone_folder: C:\gopath\src\github.com\Masterminds\goutils +shallow_clone: true + +environment: + GOPATH: C:\gopath + +platform: + - x64 + +build: off + +install: + - go version + - go env + +test_script: + - go test -v + +deploy: off diff --git a/github.com/Masterminds/goutils/cryptorandomstringutils.go b/github.com/Masterminds/goutils/cryptorandomstringutils.go new file mode 100644 index 0000000000..177dd86584 --- /dev/null +++ b/github.com/Masterminds/goutils/cryptorandomstringutils.go @@ -0,0 +1,251 @@ +/* +Copyright 2014 Alexander Okoli + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package goutils + +import ( + "crypto/rand" + "fmt" + "math" + "math/big" + "regexp" + "unicode" +) + +/* +CryptoRandomNonAlphaNumeric creates a random string whose length is the number of characters specified. +Characters will be chosen from the set of all characters (ASCII/Unicode values between 0 to 2,147,483,647 (math.MaxInt32)). + +Parameter: + count - the length of random string to create + +Returns: + string - the random string + error - an error stemming from an invalid parameter within underlying function, CryptoRandom(...) +*/ +func CryptoRandomNonAlphaNumeric(count int) (string, error) { + return CryptoRandomAlphaNumericCustom(count, false, false) +} + +/* +CryptoRandomAscii creates a random string whose length is the number of characters specified. +Characters will be chosen from the set of characters whose ASCII value is between 32 and 126 (inclusive). + +Parameter: + count - the length of random string to create + +Returns: + string - the random string + error - an error stemming from an invalid parameter within underlying function, CryptoRandom(...) +*/ +func CryptoRandomAscii(count int) (string, error) { + return CryptoRandom(count, 32, 127, false, false) +} + +/* +CryptoRandomNumeric creates a random string whose length is the number of characters specified. +Characters will be chosen from the set of numeric characters. + +Parameter: + count - the length of random string to create + +Returns: + string - the random string + error - an error stemming from an invalid parameter within underlying function, CryptoRandom(...) +*/ +func CryptoRandomNumeric(count int) (string, error) { + return CryptoRandom(count, 0, 0, false, true) +} + +/* +CryptoRandomAlphabetic creates a random string whose length is the number of characters specified. +Characters will be chosen from the set of alpha-numeric characters as indicated by the arguments. + +Parameters: + count - the length of random string to create + letters - if true, generated string may include alphabetic characters + numbers - if true, generated string may include numeric characters + +Returns: + string - the random string + error - an error stemming from an invalid parameter within underlying function, CryptoRandom(...) +*/ +func CryptoRandomAlphabetic(count int) (string, error) { + return CryptoRandom(count, 0, 0, true, false) +} + +/* +CryptoRandomAlphaNumeric creates a random string whose length is the number of characters specified. +Characters will be chosen from the set of alpha-numeric characters. + +Parameter: + count - the length of random string to create + +Returns: + string - the random string + error - an error stemming from an invalid parameter within underlying function, CryptoRandom(...) +*/ +func CryptoRandomAlphaNumeric(count int) (string, error) { + if count == 0 { + return "", nil + } + RandomString, err := CryptoRandom(count, 0, 0, true, true) + if err != nil { + return "", fmt.Errorf("Error: %s", err) + } + match, err := regexp.MatchString("([0-9]+)", RandomString) + if err != nil { + panic(err) + } + + if !match { + //Get the position between 0 and the length of the string-1 to insert a random number + position := getCryptoRandomInt(count) + //Insert a random number between [0-9] in the position + RandomString = RandomString[:position] + string('0' + getCryptoRandomInt(10)) + RandomString[position + 1:] + return RandomString, err + } + return RandomString, err + +} + +/* +CryptoRandomAlphaNumericCustom creates a random string whose length is the number of characters specified. +Characters will be chosen from the set of alpha-numeric characters as indicated by the arguments. + +Parameters: + count - the length of random string to create + letters - if true, generated string may include alphabetic characters + numbers - if true, generated string may include numeric characters + +Returns: + string - the random string + error - an error stemming from an invalid parameter within underlying function, CryptoRandom(...) +*/ +func CryptoRandomAlphaNumericCustom(count int, letters bool, numbers bool) (string, error) { + return CryptoRandom(count, 0, 0, letters, numbers) +} + +/* +CryptoRandom creates a random string based on a variety of options, using using golang's crypto/rand source of randomness. +If the parameters start and end are both 0, start and end are set to ' ' and 'z', the ASCII printable characters, will be used, +unless letters and numbers are both false, in which case, start and end are set to 0 and math.MaxInt32, respectively. +If chars is not nil, characters stored in chars that are between start and end are chosen. + +Parameters: + count - the length of random string to create + start - the position in set of chars (ASCII/Unicode int) to start at + end - the position in set of chars (ASCII/Unicode int) to end before + letters - if true, generated string may include alphabetic characters + numbers - if true, generated string may include numeric characters + chars - the set of chars to choose randoms from. If nil, then it will use the set of all chars. + +Returns: + string - the random string + error - an error stemming from invalid parameters: if count < 0; or the provided chars array is empty; or end <= start; or end > len(chars) +*/ +func CryptoRandom(count int, start int, end int, letters bool, numbers bool, chars ...rune) (string, error) { + if count == 0 { + return "", nil + } else if count < 0 { + err := fmt.Errorf("randomstringutils illegal argument: Requested random string length %v is less than 0.", count) // equiv to err := errors.New("...") + return "", err + } + if chars != nil && len(chars) == 0 { + err := fmt.Errorf("randomstringutils illegal argument: The chars array must not be empty") + return "", err + } + + if start == 0 && end == 0 { + if chars != nil { + end = len(chars) + } else { + if !letters && !numbers { + end = math.MaxInt32 + } else { + end = 'z' + 1 + start = ' ' + } + } + } else { + if end <= start { + err := fmt.Errorf("randomstringutils illegal argument: Parameter end (%v) must be greater than start (%v)", end, start) + return "", err + } + + if chars != nil && end > len(chars) { + err := fmt.Errorf("randomstringutils illegal argument: Parameter end (%v) cannot be greater than len(chars) (%v)", end, len(chars)) + return "", err + } + } + + buffer := make([]rune, count) + gap := end - start + + // high-surrogates range, (\uD800-\uDBFF) = 55296 - 56319 + // low-surrogates range, (\uDC00-\uDFFF) = 56320 - 57343 + + for count != 0 { + count-- + var ch rune + if chars == nil { + ch = rune(getCryptoRandomInt(gap) + int64(start)) + } else { + ch = chars[getCryptoRandomInt(gap) + int64(start)] + } + + if letters && unicode.IsLetter(ch) || numbers && unicode.IsDigit(ch) || !letters && !numbers { + if ch >= 56320 && ch <= 57343 { // low surrogate range + if count == 0 { + count++ + } else { + // Insert low surrogate + buffer[count] = ch + count-- + // Insert high surrogate + buffer[count] = rune(55296 + getCryptoRandomInt(128)) + } + } else if ch >= 55296 && ch <= 56191 { // High surrogates range (Partial) + if count == 0 { + count++ + } else { + // Insert low surrogate + buffer[count] = rune(56320 + getCryptoRandomInt(128)) + count-- + // Insert high surrogate + buffer[count] = ch + } + } else if ch >= 56192 && ch <= 56319 { + // private high surrogate, skip it + count++ + } else { + // not one of the surrogates* + buffer[count] = ch + } + } else { + count++ + } + } + return string(buffer), nil +} + +func getCryptoRandomInt(count int) int64 { + nBig, err := rand.Int(rand.Reader, big.NewInt(int64(count))) + if err != nil { + panic(err) + } + return nBig.Int64() +} diff --git a/github.com/Masterminds/goutils/randomstringutils.go b/github.com/Masterminds/goutils/randomstringutils.go new file mode 100644 index 0000000000..1364e0cafd --- /dev/null +++ b/github.com/Masterminds/goutils/randomstringutils.go @@ -0,0 +1,268 @@ +/* +Copyright 2014 Alexander Okoli + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package goutils + +import ( + "fmt" + "math" + "math/rand" + "regexp" + "time" + "unicode" +) + +// RANDOM provides the time-based seed used to generate random numbers +var RANDOM = rand.New(rand.NewSource(time.Now().UnixNano())) + +/* +RandomNonAlphaNumeric creates a random string whose length is the number of characters specified. +Characters will be chosen from the set of all characters (ASCII/Unicode values between 0 to 2,147,483,647 (math.MaxInt32)). + +Parameter: + count - the length of random string to create + +Returns: + string - the random string + error - an error stemming from an invalid parameter within underlying function, RandomSeed(...) +*/ +func RandomNonAlphaNumeric(count int) (string, error) { + return RandomAlphaNumericCustom(count, false, false) +} + +/* +RandomAscii creates a random string whose length is the number of characters specified. +Characters will be chosen from the set of characters whose ASCII value is between 32 and 126 (inclusive). + +Parameter: + count - the length of random string to create + +Returns: + string - the random string + error - an error stemming from an invalid parameter within underlying function, RandomSeed(...) +*/ +func RandomAscii(count int) (string, error) { + return Random(count, 32, 127, false, false) +} + +/* +RandomNumeric creates a random string whose length is the number of characters specified. +Characters will be chosen from the set of numeric characters. + +Parameter: + count - the length of random string to create + +Returns: + string - the random string + error - an error stemming from an invalid parameter within underlying function, RandomSeed(...) +*/ +func RandomNumeric(count int) (string, error) { + return Random(count, 0, 0, false, true) +} + +/* +RandomAlphabetic creates a random string whose length is the number of characters specified. +Characters will be chosen from the set of alpha-numeric characters as indicated by the arguments. + +Parameters: + count - the length of random string to create + letters - if true, generated string may include alphabetic characters + numbers - if true, generated string may include numeric characters + +Returns: + string - the random string + error - an error stemming from an invalid parameter within underlying function, RandomSeed(...) +*/ +func RandomAlphabetic(count int) (string, error) { + return Random(count, 0, 0, true, false) +} + +/* +RandomAlphaNumeric creates a random string whose length is the number of characters specified. +Characters will be chosen from the set of alpha-numeric characters. + +Parameter: + count - the length of random string to create + +Returns: + string - the random string + error - an error stemming from an invalid parameter within underlying function, RandomSeed(...) +*/ +func RandomAlphaNumeric(count int) (string, error) { + RandomString, err := Random(count, 0, 0, true, true) + if err != nil { + return "", fmt.Errorf("Error: %s", err) + } + match, err := regexp.MatchString("([0-9]+)", RandomString) + if err != nil { + panic(err) + } + + if !match { + //Get the position between 0 and the length of the string-1 to insert a random number + position := rand.Intn(count) + //Insert a random number between [0-9] in the position + RandomString = RandomString[:position] + string('0'+rand.Intn(10)) + RandomString[position+1:] + return RandomString, err + } + return RandomString, err + +} + +/* +RandomAlphaNumericCustom creates a random string whose length is the number of characters specified. +Characters will be chosen from the set of alpha-numeric characters as indicated by the arguments. + +Parameters: + count - the length of random string to create + letters - if true, generated string may include alphabetic characters + numbers - if true, generated string may include numeric characters + +Returns: + string - the random string + error - an error stemming from an invalid parameter within underlying function, RandomSeed(...) +*/ +func RandomAlphaNumericCustom(count int, letters bool, numbers bool) (string, error) { + return Random(count, 0, 0, letters, numbers) +} + +/* +Random creates a random string based on a variety of options, using default source of randomness. +This method has exactly the same semantics as RandomSeed(int, int, int, bool, bool, []char, *rand.Rand), but +instead of using an externally supplied source of randomness, it uses the internal *rand.Rand instance. + +Parameters: + count - the length of random string to create + start - the position in set of chars (ASCII/Unicode int) to start at + end - the position in set of chars (ASCII/Unicode int) to end before + letters - if true, generated string may include alphabetic characters + numbers - if true, generated string may include numeric characters + chars - the set of chars to choose randoms from. If nil, then it will use the set of all chars. + +Returns: + string - the random string + error - an error stemming from an invalid parameter within underlying function, RandomSeed(...) +*/ +func Random(count int, start int, end int, letters bool, numbers bool, chars ...rune) (string, error) { + return RandomSeed(count, start, end, letters, numbers, chars, RANDOM) +} + +/* +RandomSeed creates a random string based on a variety of options, using supplied source of randomness. +If the parameters start and end are both 0, start and end are set to ' ' and 'z', the ASCII printable characters, will be used, +unless letters and numbers are both false, in which case, start and end are set to 0 and math.MaxInt32, respectively. +If chars is not nil, characters stored in chars that are between start and end are chosen. +This method accepts a user-supplied *rand.Rand instance to use as a source of randomness. By seeding a single *rand.Rand instance +with a fixed seed and using it for each call, the same random sequence of strings can be generated repeatedly and predictably. + +Parameters: + count - the length of random string to create + start - the position in set of chars (ASCII/Unicode decimals) to start at + end - the position in set of chars (ASCII/Unicode decimals) to end before + letters - if true, generated string may include alphabetic characters + numbers - if true, generated string may include numeric characters + chars - the set of chars to choose randoms from. If nil, then it will use the set of all chars. + random - a source of randomness. + +Returns: + string - the random string + error - an error stemming from invalid parameters: if count < 0; or the provided chars array is empty; or end <= start; or end > len(chars) +*/ +func RandomSeed(count int, start int, end int, letters bool, numbers bool, chars []rune, random *rand.Rand) (string, error) { + + if count == 0 { + return "", nil + } else if count < 0 { + err := fmt.Errorf("randomstringutils illegal argument: Requested random string length %v is less than 0.", count) // equiv to err := errors.New("...") + return "", err + } + if chars != nil && len(chars) == 0 { + err := fmt.Errorf("randomstringutils illegal argument: The chars array must not be empty") + return "", err + } + + if start == 0 && end == 0 { + if chars != nil { + end = len(chars) + } else { + if !letters && !numbers { + end = math.MaxInt32 + } else { + end = 'z' + 1 + start = ' ' + } + } + } else { + if end <= start { + err := fmt.Errorf("randomstringutils illegal argument: Parameter end (%v) must be greater than start (%v)", end, start) + return "", err + } + + if chars != nil && end > len(chars) { + err := fmt.Errorf("randomstringutils illegal argument: Parameter end (%v) cannot be greater than len(chars) (%v)", end, len(chars)) + return "", err + } + } + + buffer := make([]rune, count) + gap := end - start + + // high-surrogates range, (\uD800-\uDBFF) = 55296 - 56319 + // low-surrogates range, (\uDC00-\uDFFF) = 56320 - 57343 + + for count != 0 { + count-- + var ch rune + if chars == nil { + ch = rune(random.Intn(gap) + start) + } else { + ch = chars[random.Intn(gap)+start] + } + + if letters && unicode.IsLetter(ch) || numbers && unicode.IsDigit(ch) || !letters && !numbers { + if ch >= 56320 && ch <= 57343 { // low surrogate range + if count == 0 { + count++ + } else { + // Insert low surrogate + buffer[count] = ch + count-- + // Insert high surrogate + buffer[count] = rune(55296 + random.Intn(128)) + } + } else if ch >= 55296 && ch <= 56191 { // High surrogates range (Partial) + if count == 0 { + count++ + } else { + // Insert low surrogate + buffer[count] = rune(56320 + random.Intn(128)) + count-- + // Insert high surrogate + buffer[count] = ch + } + } else if ch >= 56192 && ch <= 56319 { + // private high surrogate, skip it + count++ + } else { + // not one of the surrogates* + buffer[count] = ch + } + } else { + count++ + } + } + return string(buffer), nil +} diff --git a/github.com/Masterminds/goutils/stringutils.go b/github.com/Masterminds/goutils/stringutils.go new file mode 100644 index 0000000000..5037c4516b --- /dev/null +++ b/github.com/Masterminds/goutils/stringutils.go @@ -0,0 +1,224 @@ +/* +Copyright 2014 Alexander Okoli + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package goutils + +import ( + "bytes" + "fmt" + "strings" + "unicode" +) + +// Typically returned by functions where a searched item cannot be found +const INDEX_NOT_FOUND = -1 + +/* +Abbreviate abbreviates a string using ellipses. This will turn the string "Now is the time for all good men" into "Now is the time for..." + +Specifically, the algorithm is as follows: + + - If str is less than maxWidth characters long, return it. + - Else abbreviate it to (str[0:maxWidth - 3] + "..."). + - If maxWidth is less than 4, return an illegal argument error. + - In no case will it return a string of length greater than maxWidth. + +Parameters: + str - the string to check + maxWidth - maximum length of result string, must be at least 4 + +Returns: + string - abbreviated string + error - if the width is too small +*/ +func Abbreviate(str string, maxWidth int) (string, error) { + return AbbreviateFull(str, 0, maxWidth) +} + +/* +AbbreviateFull abbreviates a string using ellipses. This will turn the string "Now is the time for all good men" into "...is the time for..." +This function works like Abbreviate(string, int), but allows you to specify a "left edge" offset. Note that this left edge is not +necessarily going to be the leftmost character in the result, or the first character following the ellipses, but it will appear +somewhere in the result. +In no case will it return a string of length greater than maxWidth. + +Parameters: + str - the string to check + offset - left edge of source string + maxWidth - maximum length of result string, must be at least 4 + +Returns: + string - abbreviated string + error - if the width is too small +*/ +func AbbreviateFull(str string, offset int, maxWidth int) (string, error) { + if str == "" { + return "", nil + } + if maxWidth < 4 { + err := fmt.Errorf("stringutils illegal argument: Minimum abbreviation width is 4") + return "", err + } + if len(str) <= maxWidth { + return str, nil + } + if offset > len(str) { + offset = len(str) + } + if len(str)-offset < (maxWidth - 3) { // 15 - 5 < 10 - 3 = 10 < 7 + offset = len(str) - (maxWidth - 3) + } + abrevMarker := "..." + if offset <= 4 { + return str[0:maxWidth-3] + abrevMarker, nil // str.substring(0, maxWidth - 3) + abrevMarker; + } + if maxWidth < 7 { + err := fmt.Errorf("stringutils illegal argument: Minimum abbreviation width with offset is 7") + return "", err + } + if (offset + maxWidth - 3) < len(str) { // 5 + (10-3) < 15 = 12 < 15 + abrevStr, _ := Abbreviate(str[offset:len(str)], (maxWidth - 3)) + return abrevMarker + abrevStr, nil // abrevMarker + abbreviate(str.substring(offset), maxWidth - 3); + } + return abrevMarker + str[(len(str)-(maxWidth-3)):len(str)], nil // abrevMarker + str.substring(str.length() - (maxWidth - 3)); +} + +/* +DeleteWhiteSpace deletes all whitespaces from a string as defined by unicode.IsSpace(rune). +It returns the string without whitespaces. + +Parameter: + str - the string to delete whitespace from, may be nil + +Returns: + the string without whitespaces +*/ +func DeleteWhiteSpace(str string) string { + if str == "" { + return str + } + sz := len(str) + var chs bytes.Buffer + count := 0 + for i := 0; i < sz; i++ { + ch := rune(str[i]) + if !unicode.IsSpace(ch) { + chs.WriteRune(ch) + count++ + } + } + if count == sz { + return str + } + return chs.String() +} + +/* +IndexOfDifference compares two strings, and returns the index at which the strings begin to differ. + +Parameters: + str1 - the first string + str2 - the second string + +Returns: + the index where str1 and str2 begin to differ; -1 if they are equal +*/ +func IndexOfDifference(str1 string, str2 string) int { + if str1 == str2 { + return INDEX_NOT_FOUND + } + if IsEmpty(str1) || IsEmpty(str2) { + return 0 + } + var i int + for i = 0; i < len(str1) && i < len(str2); i++ { + if rune(str1[i]) != rune(str2[i]) { + break + } + } + if i < len(str2) || i < len(str1) { + return i + } + return INDEX_NOT_FOUND +} + +/* +IsBlank checks if a string is whitespace or empty (""). Observe the following behavior: + + goutils.IsBlank("") = true + goutils.IsBlank(" ") = true + goutils.IsBlank("bob") = false + goutils.IsBlank(" bob ") = false + +Parameter: + str - the string to check + +Returns: + true - if the string is whitespace or empty ("") +*/ +func IsBlank(str string) bool { + strLen := len(str) + if str == "" || strLen == 0 { + return true + } + for i := 0; i < strLen; i++ { + if unicode.IsSpace(rune(str[i])) == false { + return false + } + } + return true +} + +/* +IndexOf returns the index of the first instance of sub in str, with the search beginning from the +index start point specified. -1 is returned if sub is not present in str. + +An empty string ("") will return -1 (INDEX_NOT_FOUND). A negative start position is treated as zero. +A start position greater than the string length returns -1. + +Parameters: + str - the string to check + sub - the substring to find + start - the start position; negative treated as zero + +Returns: + the first index where the sub string was found (always >= start) +*/ +func IndexOf(str string, sub string, start int) int { + + if start < 0 { + start = 0 + } + + if len(str) < start { + return INDEX_NOT_FOUND + } + + if IsEmpty(str) || IsEmpty(sub) { + return INDEX_NOT_FOUND + } + + partialIndex := strings.Index(str[start:len(str)], sub) + if partialIndex == -1 { + return INDEX_NOT_FOUND + } + return partialIndex + start +} + +// IsEmpty checks if a string is empty (""). Returns true if empty, and false otherwise. +func IsEmpty(str string) bool { + return len(str) == 0 +} diff --git a/github.com/Masterminds/goutils/wordutils.go b/github.com/Masterminds/goutils/wordutils.go new file mode 100644 index 0000000000..034cad8e21 --- /dev/null +++ b/github.com/Masterminds/goutils/wordutils.go @@ -0,0 +1,357 @@ +/* +Copyright 2014 Alexander Okoli + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Package goutils provides utility functions to manipulate strings in various ways. +The code snippets below show examples of how to use goutils. Some functions return +errors while others do not, so usage would vary as a result. + +Example: + + package main + + import ( + "fmt" + "github.com/aokoli/goutils" + ) + + func main() { + + // EXAMPLE 1: A goutils function which returns no errors + fmt.Println (goutils.Initials("John Doe Foo")) // Prints out "JDF" + + + + // EXAMPLE 2: A goutils function which returns an error + rand1, err1 := goutils.Random (-1, 0, 0, true, true) + + if err1 != nil { + fmt.Println(err1) // Prints out error message because -1 was entered as the first parameter in goutils.Random(...) + } else { + fmt.Println(rand1) + } + } +*/ +package goutils + +import ( + "bytes" + "strings" + "unicode" +) + +// VERSION indicates the current version of goutils +const VERSION = "1.0.0" + +/* +Wrap wraps a single line of text, identifying words by ' '. +New lines will be separated by '\n'. Very long words, such as URLs will not be wrapped. +Leading spaces on a new line are stripped. Trailing spaces are not stripped. + +Parameters: + str - the string to be word wrapped + wrapLength - the column (a column can fit only one character) to wrap the words at, less than 1 is treated as 1 + +Returns: + a line with newlines inserted +*/ +func Wrap(str string, wrapLength int) string { + return WrapCustom(str, wrapLength, "", false) +} + +/* +WrapCustom wraps a single line of text, identifying words by ' '. +Leading spaces on a new line are stripped. Trailing spaces are not stripped. + +Parameters: + str - the string to be word wrapped + wrapLength - the column number (a column can fit only one character) to wrap the words at, less than 1 is treated as 1 + newLineStr - the string to insert for a new line, "" uses '\n' + wrapLongWords - true if long words (such as URLs) should be wrapped + +Returns: + a line with newlines inserted +*/ +func WrapCustom(str string, wrapLength int, newLineStr string, wrapLongWords bool) string { + + if str == "" { + return "" + } + if newLineStr == "" { + newLineStr = "\n" // TODO Assumes "\n" is seperator. Explore SystemUtils.LINE_SEPARATOR from Apache Commons + } + if wrapLength < 1 { + wrapLength = 1 + } + + inputLineLength := len(str) + offset := 0 + + var wrappedLine bytes.Buffer + + for inputLineLength-offset > wrapLength { + + if rune(str[offset]) == ' ' { + offset++ + continue + } + + end := wrapLength + offset + 1 + spaceToWrapAt := strings.LastIndex(str[offset:end], " ") + offset + + if spaceToWrapAt >= offset { + // normal word (not longer than wrapLength) + wrappedLine.WriteString(str[offset:spaceToWrapAt]) + wrappedLine.WriteString(newLineStr) + offset = spaceToWrapAt + 1 + + } else { + // long word or URL + if wrapLongWords { + end := wrapLength + offset + // long words are wrapped one line at a time + wrappedLine.WriteString(str[offset:end]) + wrappedLine.WriteString(newLineStr) + offset += wrapLength + } else { + // long words aren't wrapped, just extended beyond limit + end := wrapLength + offset + index := strings.IndexRune(str[end:len(str)], ' ') + if index == -1 { + wrappedLine.WriteString(str[offset:len(str)]) + offset = inputLineLength + } else { + spaceToWrapAt = index + end + wrappedLine.WriteString(str[offset:spaceToWrapAt]) + wrappedLine.WriteString(newLineStr) + offset = spaceToWrapAt + 1 + } + } + } + } + + wrappedLine.WriteString(str[offset:len(str)]) + + return wrappedLine.String() + +} + +/* +Capitalize capitalizes all the delimiter separated words in a string. Only the first letter of each word is changed. +To convert the rest of each word to lowercase at the same time, use CapitalizeFully(str string, delimiters ...rune). +The delimiters represent a set of characters understood to separate words. The first string character +and the first non-delimiter character after a delimiter will be capitalized. A "" input string returns "". +Capitalization uses the Unicode title case, normally equivalent to upper case. + +Parameters: + str - the string to capitalize + delimiters - set of characters to determine capitalization, exclusion of this parameter means whitespace would be delimeter + +Returns: + capitalized string +*/ +func Capitalize(str string, delimiters ...rune) string { + + var delimLen int + + if delimiters == nil { + delimLen = -1 + } else { + delimLen = len(delimiters) + } + + if str == "" || delimLen == 0 { + return str + } + + buffer := []rune(str) + capitalizeNext := true + for i := 0; i < len(buffer); i++ { + ch := buffer[i] + if isDelimiter(ch, delimiters...) { + capitalizeNext = true + } else if capitalizeNext { + buffer[i] = unicode.ToTitle(ch) + capitalizeNext = false + } + } + return string(buffer) + +} + +/* +CapitalizeFully converts all the delimiter separated words in a string into capitalized words, that is each word is made up of a +titlecase character and then a series of lowercase characters. The delimiters represent a set of characters understood +to separate words. The first string character and the first non-delimiter character after a delimiter will be capitalized. +Capitalization uses the Unicode title case, normally equivalent to upper case. + +Parameters: + str - the string to capitalize fully + delimiters - set of characters to determine capitalization, exclusion of this parameter means whitespace would be delimeter + +Returns: + capitalized string +*/ +func CapitalizeFully(str string, delimiters ...rune) string { + + var delimLen int + + if delimiters == nil { + delimLen = -1 + } else { + delimLen = len(delimiters) + } + + if str == "" || delimLen == 0 { + return str + } + str = strings.ToLower(str) + return Capitalize(str, delimiters...) +} + +/* +Uncapitalize uncapitalizes all the whitespace separated words in a string. Only the first letter of each word is changed. +The delimiters represent a set of characters understood to separate words. The first string character and the first non-delimiter +character after a delimiter will be uncapitalized. Whitespace is defined by unicode.IsSpace(char). + +Parameters: + str - the string to uncapitalize fully + delimiters - set of characters to determine capitalization, exclusion of this parameter means whitespace would be delimeter + +Returns: + uncapitalized string +*/ +func Uncapitalize(str string, delimiters ...rune) string { + + var delimLen int + + if delimiters == nil { + delimLen = -1 + } else { + delimLen = len(delimiters) + } + + if str == "" || delimLen == 0 { + return str + } + + buffer := []rune(str) + uncapitalizeNext := true // TODO Always makes capitalize/un apply to first char. + for i := 0; i < len(buffer); i++ { + ch := buffer[i] + if isDelimiter(ch, delimiters...) { + uncapitalizeNext = true + } else if uncapitalizeNext { + buffer[i] = unicode.ToLower(ch) + uncapitalizeNext = false + } + } + return string(buffer) +} + +/* +SwapCase swaps the case of a string using a word based algorithm. + +Conversion algorithm: + + Upper case character converts to Lower case + Title case character converts to Lower case + Lower case character after Whitespace or at start converts to Title case + Other Lower case character converts to Upper case + Whitespace is defined by unicode.IsSpace(char). + +Parameters: + str - the string to swap case + +Returns: + the changed string +*/ +func SwapCase(str string) string { + if str == "" { + return str + } + buffer := []rune(str) + + whitespace := true + + for i := 0; i < len(buffer); i++ { + ch := buffer[i] + if unicode.IsUpper(ch) { + buffer[i] = unicode.ToLower(ch) + whitespace = false + } else if unicode.IsTitle(ch) { + buffer[i] = unicode.ToLower(ch) + whitespace = false + } else if unicode.IsLower(ch) { + if whitespace { + buffer[i] = unicode.ToTitle(ch) + whitespace = false + } else { + buffer[i] = unicode.ToUpper(ch) + } + } else { + whitespace = unicode.IsSpace(ch) + } + } + return string(buffer) +} + +/* +Initials extracts the initial letters from each word in the string. The first letter of the string and all first +letters after the defined delimiters are returned as a new string. Their case is not changed. If the delimiters +parameter is excluded, then Whitespace is used. Whitespace is defined by unicode.IsSpacea(char). An empty delimiter array returns an empty string. + +Parameters: + str - the string to get initials from + delimiters - set of characters to determine words, exclusion of this parameter means whitespace would be delimeter +Returns: + string of initial letters +*/ +func Initials(str string, delimiters ...rune) string { + if str == "" { + return str + } + if delimiters != nil && len(delimiters) == 0 { + return "" + } + strLen := len(str) + var buf bytes.Buffer + lastWasGap := true + for i := 0; i < strLen; i++ { + ch := rune(str[i]) + + if isDelimiter(ch, delimiters...) { + lastWasGap = true + } else if lastWasGap { + buf.WriteRune(ch) + lastWasGap = false + } + } + return buf.String() +} + +// private function (lower case func name) +func isDelimiter(ch rune, delimiters ...rune) bool { + if delimiters == nil { + return unicode.IsSpace(ch) + } + for _, delimiter := range delimiters { + if ch == delimiter { + return true + } + } + return false +} diff --git a/github.com/Masterminds/semver/.travis.yml b/github.com/Masterminds/semver/.travis.yml new file mode 100644 index 0000000000..096369d44d --- /dev/null +++ b/github.com/Masterminds/semver/.travis.yml @@ -0,0 +1,29 @@ +language: go + +go: + - 1.6.x + - 1.7.x + - 1.8.x + - 1.9.x + - 1.10.x + - 1.11.x + - 1.12.x + - tip + +# Setting sudo access to false will let Travis CI use containers rather than +# VMs to run the tests. For more details see: +# - http://docs.travis-ci.com/user/workers/container-based-infrastructure/ +# - http://docs.travis-ci.com/user/workers/standard-infrastructure/ +sudo: false + +script: + - make setup + - make test + +notifications: + webhooks: + urls: + - https://webhooks.gitter.im/e/06e3328629952dabe3e0 + on_success: change # options: [always|never|change] default: always + on_failure: always # options: [always|never|change] default: always + on_start: never # options: [always|never|change] default: always diff --git a/github.com/Masterminds/semver/CHANGELOG.md b/github.com/Masterminds/semver/CHANGELOG.md new file mode 100644 index 0000000000..e405c9a84d --- /dev/null +++ b/github.com/Masterminds/semver/CHANGELOG.md @@ -0,0 +1,109 @@ +# 1.5.0 (2019-09-11) + +## Added + +- #103: Add basic fuzzing for `NewVersion()` (thanks @jesse-c) + +## Changed + +- #82: Clarify wildcard meaning in range constraints and update tests for it (thanks @greysteil) +- #83: Clarify caret operator range for pre-1.0.0 dependencies (thanks @greysteil) +- #72: Adding docs comment pointing to vert for a cli +- #71: Update the docs on pre-release comparator handling +- #89: Test with new go versions (thanks @thedevsaddam) +- #87: Added $ to ValidPrerelease for better validation (thanks @jeremycarroll) + +## Fixed + +- #78: Fix unchecked error in example code (thanks @ravron) +- #70: Fix the handling of pre-releases and the 0.0.0 release edge case +- #97: Fixed copyright file for proper display on GitHub +- #107: Fix handling prerelease when sorting alphanum and num +- #109: Fixed where Validate sometimes returns wrong message on error + +# 1.4.2 (2018-04-10) + +## Changed +- #72: Updated the docs to point to vert for a console appliaction +- #71: Update the docs on pre-release comparator handling + +## Fixed +- #70: Fix the handling of pre-releases and the 0.0.0 release edge case + +# 1.4.1 (2018-04-02) + +## Fixed +- Fixed #64: Fix pre-release precedence issue (thanks @uudashr) + +# 1.4.0 (2017-10-04) + +## Changed +- #61: Update NewVersion to parse ints with a 64bit int size (thanks @zknill) + +# 1.3.1 (2017-07-10) + +## Fixed +- Fixed #57: number comparisons in prerelease sometimes inaccurate + +# 1.3.0 (2017-05-02) + +## Added +- #45: Added json (un)marshaling support (thanks @mh-cbon) +- Stability marker. See https://masterminds.github.io/stability/ + +## Fixed +- #51: Fix handling of single digit tilde constraint (thanks @dgodd) + +## Changed +- #55: The godoc icon moved from png to svg + +# 1.2.3 (2017-04-03) + +## Fixed +- #46: Fixed 0.x.x and 0.0.x in constraints being treated as * + +# Release 1.2.2 (2016-12-13) + +## Fixed +- #34: Fixed issue where hyphen range was not working with pre-release parsing. + +# Release 1.2.1 (2016-11-28) + +## Fixed +- #24: Fixed edge case issue where constraint "> 0" does not handle "0.0.1-alpha" + properly. + +# Release 1.2.0 (2016-11-04) + +## Added +- #20: Added MustParse function for versions (thanks @adamreese) +- #15: Added increment methods on versions (thanks @mh-cbon) + +## Fixed +- Issue #21: Per the SemVer spec (section 9) a pre-release is unstable and + might not satisfy the intended compatibility. The change here ignores pre-releases + on constraint checks (e.g., ~ or ^) when a pre-release is not part of the + constraint. For example, `^1.2.3` will ignore pre-releases while + `^1.2.3-alpha` will include them. + +# Release 1.1.1 (2016-06-30) + +## Changed +- Issue #9: Speed up version comparison performance (thanks @sdboyer) +- Issue #8: Added benchmarks (thanks @sdboyer) +- Updated Go Report Card URL to new location +- Updated Readme to add code snippet formatting (thanks @mh-cbon) +- Updating tagging to v[SemVer] structure for compatibility with other tools. + +# Release 1.1.0 (2016-03-11) + +- Issue #2: Implemented validation to provide reasons a versions failed a + constraint. + +# Release 1.0.1 (2015-12-31) + +- Fixed #1: * constraint failing on valid versions. + +# Release 1.0.0 (2015-10-20) + +- Initial release diff --git a/github.com/Masterminds/semver/LICENSE.txt b/github.com/Masterminds/semver/LICENSE.txt new file mode 100644 index 0000000000..9ff7da9c48 --- /dev/null +++ b/github.com/Masterminds/semver/LICENSE.txt @@ -0,0 +1,19 @@ +Copyright (C) 2014-2019, Matt Butcher and Matt Farina + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/github.com/Masterminds/semver/Makefile b/github.com/Masterminds/semver/Makefile new file mode 100644 index 0000000000..a7a1b4e36d --- /dev/null +++ b/github.com/Masterminds/semver/Makefile @@ -0,0 +1,36 @@ +.PHONY: setup +setup: + go get -u gopkg.in/alecthomas/gometalinter.v1 + gometalinter.v1 --install + +.PHONY: test +test: validate lint + @echo "==> Running tests" + go test -v + +.PHONY: validate +validate: + @echo "==> Running static validations" + @gometalinter.v1 \ + --disable-all \ + --enable deadcode \ + --severity deadcode:error \ + --enable gofmt \ + --enable gosimple \ + --enable ineffassign \ + --enable misspell \ + --enable vet \ + --tests \ + --vendor \ + --deadline 60s \ + ./... || exit_code=1 + +.PHONY: lint +lint: + @echo "==> Running linters" + @gometalinter.v1 \ + --disable-all \ + --enable golint \ + --vendor \ + --deadline 60s \ + ./... || : diff --git a/github.com/Masterminds/semver/README.md b/github.com/Masterminds/semver/README.md new file mode 100644 index 0000000000..1b52d2f436 --- /dev/null +++ b/github.com/Masterminds/semver/README.md @@ -0,0 +1,194 @@ +# SemVer + +The `semver` package provides the ability to work with [Semantic Versions](http://semver.org) in Go. Specifically it provides the ability to: + +* Parse semantic versions +* Sort semantic versions +* Check if a semantic version fits within a set of constraints +* Optionally work with a `v` prefix + +[![Stability: +Active](https://masterminds.github.io/stability/active.svg)](https://masterminds.github.io/stability/active.html) +[![Build Status](https://travis-ci.org/Masterminds/semver.svg)](https://travis-ci.org/Masterminds/semver) [![Build status](https://ci.appveyor.com/api/projects/status/jfk66lib7hb985k8/branch/master?svg=true&passingText=windows%20build%20passing&failingText=windows%20build%20failing)](https://ci.appveyor.com/project/mattfarina/semver/branch/master) [![GoDoc](https://godoc.org/github.com/Masterminds/semver?status.svg)](https://godoc.org/github.com/Masterminds/semver) [![Go Report Card](https://goreportcard.com/badge/github.com/Masterminds/semver)](https://goreportcard.com/report/github.com/Masterminds/semver) + +If you are looking for a command line tool for version comparisons please see +[vert](https://github.com/Masterminds/vert) which uses this library. + +## Parsing Semantic Versions + +To parse a semantic version use the `NewVersion` function. For example, + +```go + v, err := semver.NewVersion("1.2.3-beta.1+build345") +``` + +If there is an error the version wasn't parseable. The version object has methods +to get the parts of the version, compare it to other versions, convert the +version back into a string, and get the original string. For more details +please see the [documentation](https://godoc.org/github.com/Masterminds/semver). + +## Sorting Semantic Versions + +A set of versions can be sorted using the [`sort`](https://golang.org/pkg/sort/) +package from the standard library. For example, + +```go + raw := []string{"1.2.3", "1.0", "1.3", "2", "0.4.2",} + vs := make([]*semver.Version, len(raw)) + for i, r := range raw { + v, err := semver.NewVersion(r) + if err != nil { + t.Errorf("Error parsing version: %s", err) + } + + vs[i] = v + } + + sort.Sort(semver.Collection(vs)) +``` + +## Checking Version Constraints + +Checking a version against version constraints is one of the most featureful +parts of the package. + +```go + c, err := semver.NewConstraint(">= 1.2.3") + if err != nil { + // Handle constraint not being parseable. + } + + v, _ := semver.NewVersion("1.3") + if err != nil { + // Handle version not being parseable. + } + // Check if the version meets the constraints. The a variable will be true. + a := c.Check(v) +``` + +## Basic Comparisons + +There are two elements to the comparisons. First, a comparison string is a list +of comma separated and comparisons. These are then separated by || separated or +comparisons. For example, `">= 1.2, < 3.0.0 || >= 4.2.3"` is looking for a +comparison that's greater than or equal to 1.2 and less than 3.0.0 or is +greater than or equal to 4.2.3. + +The basic comparisons are: + +* `=`: equal (aliased to no operator) +* `!=`: not equal +* `>`: greater than +* `<`: less than +* `>=`: greater than or equal to +* `<=`: less than or equal to + +## Working With Pre-release Versions + +Pre-releases, for those not familiar with them, are used for software releases +prior to stable or generally available releases. Examples of pre-releases include +development, alpha, beta, and release candidate releases. A pre-release may be +a version such as `1.2.3-beta.1` while the stable release would be `1.2.3`. In the +order of precidence, pre-releases come before their associated releases. In this +example `1.2.3-beta.1 < 1.2.3`. + +According to the Semantic Version specification pre-releases may not be +API compliant with their release counterpart. It says, + +> A pre-release version indicates that the version is unstable and might not satisfy the intended compatibility requirements as denoted by its associated normal version. + +SemVer comparisons without a pre-release comparator will skip pre-release versions. +For example, `>=1.2.3` will skip pre-releases when looking at a list of releases +while `>=1.2.3-0` will evaluate and find pre-releases. + +The reason for the `0` as a pre-release version in the example comparison is +because pre-releases can only contain ASCII alphanumerics and hyphens (along with +`.` separators), per the spec. Sorting happens in ASCII sort order, again per the spec. The lowest character is a `0` in ASCII sort order (see an [ASCII Table](http://www.asciitable.com/)) + +Understanding ASCII sort ordering is important because A-Z comes before a-z. That +means `>=1.2.3-BETA` will return `1.2.3-alpha`. What you might expect from case +sensitivity doesn't apply here. This is due to ASCII sort ordering which is what +the spec specifies. + +## Hyphen Range Comparisons + +There are multiple methods to handle ranges and the first is hyphens ranges. +These look like: + +* `1.2 - 1.4.5` which is equivalent to `>= 1.2, <= 1.4.5` +* `2.3.4 - 4.5` which is equivalent to `>= 2.3.4, <= 4.5` + +## Wildcards In Comparisons + +The `x`, `X`, and `*` characters can be used as a wildcard character. This works +for all comparison operators. When used on the `=` operator it falls +back to the pack level comparison (see tilde below). For example, + +* `1.2.x` is equivalent to `>= 1.2.0, < 1.3.0` +* `>= 1.2.x` is equivalent to `>= 1.2.0` +* `<= 2.x` is equivalent to `< 3` +* `*` is equivalent to `>= 0.0.0` + +## Tilde Range Comparisons (Patch) + +The tilde (`~`) comparison operator is for patch level ranges when a minor +version is specified and major level changes when the minor number is missing. +For example, + +* `~1.2.3` is equivalent to `>= 1.2.3, < 1.3.0` +* `~1` is equivalent to `>= 1, < 2` +* `~2.3` is equivalent to `>= 2.3, < 2.4` +* `~1.2.x` is equivalent to `>= 1.2.0, < 1.3.0` +* `~1.x` is equivalent to `>= 1, < 2` + +## Caret Range Comparisons (Major) + +The caret (`^`) comparison operator is for major level changes. This is useful +when comparisons of API versions as a major change is API breaking. For example, + +* `^1.2.3` is equivalent to `>= 1.2.3, < 2.0.0` +* `^0.0.1` is equivalent to `>= 0.0.1, < 1.0.0` +* `^1.2.x` is equivalent to `>= 1.2.0, < 2.0.0` +* `^2.3` is equivalent to `>= 2.3, < 3` +* `^2.x` is equivalent to `>= 2.0.0, < 3` + +# Validation + +In addition to testing a version against a constraint, a version can be validated +against a constraint. When validation fails a slice of errors containing why a +version didn't meet the constraint is returned. For example, + +```go + c, err := semver.NewConstraint("<= 1.2.3, >= 1.4") + if err != nil { + // Handle constraint not being parseable. + } + + v, _ := semver.NewVersion("1.3") + if err != nil { + // Handle version not being parseable. + } + + // Validate a version against a constraint. + a, msgs := c.Validate(v) + // a is false + for _, m := range msgs { + fmt.Println(m) + + // Loops over the errors which would read + // "1.3 is greater than 1.2.3" + // "1.3 is less than 1.4" + } +``` + +# Fuzzing + + [dvyukov/go-fuzz](https://github.com/dvyukov/go-fuzz) is used for fuzzing. + +1. `go-fuzz-build` +2. `go-fuzz -workdir=fuzz` + +# Contribute + +If you find an issue or want to contribute please file an [issue](https://github.com/Masterminds/semver/issues) +or [create a pull request](https://github.com/Masterminds/semver/pulls). diff --git a/github.com/Masterminds/semver/appveyor.yml b/github.com/Masterminds/semver/appveyor.yml new file mode 100644 index 0000000000..b2778df15a --- /dev/null +++ b/github.com/Masterminds/semver/appveyor.yml @@ -0,0 +1,44 @@ +version: build-{build}.{branch} + +clone_folder: C:\gopath\src\github.com\Masterminds\semver +shallow_clone: true + +environment: + GOPATH: C:\gopath + +platform: + - x64 + +install: + - go version + - go env + - go get -u gopkg.in/alecthomas/gometalinter.v1 + - set PATH=%PATH%;%GOPATH%\bin + - gometalinter.v1.exe --install + +build_script: + - go install -v ./... + +test_script: + - "gometalinter.v1 \ + --disable-all \ + --enable deadcode \ + --severity deadcode:error \ + --enable gofmt \ + --enable gosimple \ + --enable ineffassign \ + --enable misspell \ + --enable vet \ + --tests \ + --vendor \ + --deadline 60s \ + ./... || exit_code=1" + - "gometalinter.v1 \ + --disable-all \ + --enable golint \ + --vendor \ + --deadline 60s \ + ./... || :" + - go test -v + +deploy: off diff --git a/github.com/Masterminds/semver/collection.go b/github.com/Masterminds/semver/collection.go new file mode 100644 index 0000000000..a78235895f --- /dev/null +++ b/github.com/Masterminds/semver/collection.go @@ -0,0 +1,24 @@ +package semver + +// Collection is a collection of Version instances and implements the sort +// interface. See the sort package for more details. +// https://golang.org/pkg/sort/ +type Collection []*Version + +// Len returns the length of a collection. The number of Version instances +// on the slice. +func (c Collection) Len() int { + return len(c) +} + +// Less is needed for the sort interface to compare two Version objects on the +// slice. If checks if one is less than the other. +func (c Collection) Less(i, j int) bool { + return c[i].LessThan(c[j]) +} + +// Swap is needed for the sort interface to replace the Version objects +// at two different positions in the slice. +func (c Collection) Swap(i, j int) { + c[i], c[j] = c[j], c[i] +} diff --git a/github.com/Masterminds/semver/constraints.go b/github.com/Masterminds/semver/constraints.go new file mode 100644 index 0000000000..b94b93413f --- /dev/null +++ b/github.com/Masterminds/semver/constraints.go @@ -0,0 +1,423 @@ +package semver + +import ( + "errors" + "fmt" + "regexp" + "strings" +) + +// Constraints is one or more constraint that a semantic version can be +// checked against. +type Constraints struct { + constraints [][]*constraint +} + +// NewConstraint returns a Constraints instance that a Version instance can +// be checked against. If there is a parse error it will be returned. +func NewConstraint(c string) (*Constraints, error) { + + // Rewrite - ranges into a comparison operation. + c = rewriteRange(c) + + ors := strings.Split(c, "||") + or := make([][]*constraint, len(ors)) + for k, v := range ors { + cs := strings.Split(v, ",") + result := make([]*constraint, len(cs)) + for i, s := range cs { + pc, err := parseConstraint(s) + if err != nil { + return nil, err + } + + result[i] = pc + } + or[k] = result + } + + o := &Constraints{constraints: or} + return o, nil +} + +// Check tests if a version satisfies the constraints. +func (cs Constraints) Check(v *Version) bool { + // loop over the ORs and check the inner ANDs + for _, o := range cs.constraints { + joy := true + for _, c := range o { + if !c.check(v) { + joy = false + break + } + } + + if joy { + return true + } + } + + return false +} + +// Validate checks if a version satisfies a constraint. If not a slice of +// reasons for the failure are returned in addition to a bool. +func (cs Constraints) Validate(v *Version) (bool, []error) { + // loop over the ORs and check the inner ANDs + var e []error + + // Capture the prerelease message only once. When it happens the first time + // this var is marked + var prerelesase bool + for _, o := range cs.constraints { + joy := true + for _, c := range o { + // Before running the check handle the case there the version is + // a prerelease and the check is not searching for prereleases. + if c.con.pre == "" && v.pre != "" { + if !prerelesase { + em := fmt.Errorf("%s is a prerelease version and the constraint is only looking for release versions", v) + e = append(e, em) + prerelesase = true + } + joy = false + + } else { + + if !c.check(v) { + em := fmt.Errorf(c.msg, v, c.orig) + e = append(e, em) + joy = false + } + } + } + + if joy { + return true, []error{} + } + } + + return false, e +} + +var constraintOps map[string]cfunc +var constraintMsg map[string]string +var constraintRegex *regexp.Regexp + +func init() { + constraintOps = map[string]cfunc{ + "": constraintTildeOrEqual, + "=": constraintTildeOrEqual, + "!=": constraintNotEqual, + ">": constraintGreaterThan, + "<": constraintLessThan, + ">=": constraintGreaterThanEqual, + "=>": constraintGreaterThanEqual, + "<=": constraintLessThanEqual, + "=<": constraintLessThanEqual, + "~": constraintTilde, + "~>": constraintTilde, + "^": constraintCaret, + } + + constraintMsg = map[string]string{ + "": "%s is not equal to %s", + "=": "%s is not equal to %s", + "!=": "%s is equal to %s", + ">": "%s is less than or equal to %s", + "<": "%s is greater than or equal to %s", + ">=": "%s is less than %s", + "=>": "%s is less than %s", + "<=": "%s is greater than %s", + "=<": "%s is greater than %s", + "~": "%s does not have same major and minor version as %s", + "~>": "%s does not have same major and minor version as %s", + "^": "%s does not have same major version as %s", + } + + ops := make([]string, 0, len(constraintOps)) + for k := range constraintOps { + ops = append(ops, regexp.QuoteMeta(k)) + } + + constraintRegex = regexp.MustCompile(fmt.Sprintf( + `^\s*(%s)\s*(%s)\s*$`, + strings.Join(ops, "|"), + cvRegex)) + + constraintRangeRegex = regexp.MustCompile(fmt.Sprintf( + `\s*(%s)\s+-\s+(%s)\s*`, + cvRegex, cvRegex)) +} + +// An individual constraint +type constraint struct { + // The callback function for the restraint. It performs the logic for + // the constraint. + function cfunc + + msg string + + // The version used in the constraint check. For example, if a constraint + // is '<= 2.0.0' the con a version instance representing 2.0.0. + con *Version + + // The original parsed version (e.g., 4.x from != 4.x) + orig string + + // When an x is used as part of the version (e.g., 1.x) + minorDirty bool + dirty bool + patchDirty bool +} + +// Check if a version meets the constraint +func (c *constraint) check(v *Version) bool { + return c.function(v, c) +} + +type cfunc func(v *Version, c *constraint) bool + +func parseConstraint(c string) (*constraint, error) { + m := constraintRegex.FindStringSubmatch(c) + if m == nil { + return nil, fmt.Errorf("improper constraint: %s", c) + } + + ver := m[2] + orig := ver + minorDirty := false + patchDirty := false + dirty := false + if isX(m[3]) { + ver = "0.0.0" + dirty = true + } else if isX(strings.TrimPrefix(m[4], ".")) || m[4] == "" { + minorDirty = true + dirty = true + ver = fmt.Sprintf("%s.0.0%s", m[3], m[6]) + } else if isX(strings.TrimPrefix(m[5], ".")) { + dirty = true + patchDirty = true + ver = fmt.Sprintf("%s%s.0%s", m[3], m[4], m[6]) + } + + con, err := NewVersion(ver) + if err != nil { + + // The constraintRegex should catch any regex parsing errors. So, + // we should never get here. + return nil, errors.New("constraint Parser Error") + } + + cs := &constraint{ + function: constraintOps[m[1]], + msg: constraintMsg[m[1]], + con: con, + orig: orig, + minorDirty: minorDirty, + patchDirty: patchDirty, + dirty: dirty, + } + return cs, nil +} + +// Constraint functions +func constraintNotEqual(v *Version, c *constraint) bool { + if c.dirty { + + // If there is a pre-release on the version but the constraint isn't looking + // for them assume that pre-releases are not compatible. See issue 21 for + // more details. + if v.Prerelease() != "" && c.con.Prerelease() == "" { + return false + } + + if c.con.Major() != v.Major() { + return true + } + if c.con.Minor() != v.Minor() && !c.minorDirty { + return true + } else if c.minorDirty { + return false + } + + return false + } + + return !v.Equal(c.con) +} + +func constraintGreaterThan(v *Version, c *constraint) bool { + + // If there is a pre-release on the version but the constraint isn't looking + // for them assume that pre-releases are not compatible. See issue 21 for + // more details. + if v.Prerelease() != "" && c.con.Prerelease() == "" { + return false + } + + return v.Compare(c.con) == 1 +} + +func constraintLessThan(v *Version, c *constraint) bool { + // If there is a pre-release on the version but the constraint isn't looking + // for them assume that pre-releases are not compatible. See issue 21 for + // more details. + if v.Prerelease() != "" && c.con.Prerelease() == "" { + return false + } + + if !c.dirty { + return v.Compare(c.con) < 0 + } + + if v.Major() > c.con.Major() { + return false + } else if v.Minor() > c.con.Minor() && !c.minorDirty { + return false + } + + return true +} + +func constraintGreaterThanEqual(v *Version, c *constraint) bool { + + // If there is a pre-release on the version but the constraint isn't looking + // for them assume that pre-releases are not compatible. See issue 21 for + // more details. + if v.Prerelease() != "" && c.con.Prerelease() == "" { + return false + } + + return v.Compare(c.con) >= 0 +} + +func constraintLessThanEqual(v *Version, c *constraint) bool { + // If there is a pre-release on the version but the constraint isn't looking + // for them assume that pre-releases are not compatible. See issue 21 for + // more details. + if v.Prerelease() != "" && c.con.Prerelease() == "" { + return false + } + + if !c.dirty { + return v.Compare(c.con) <= 0 + } + + if v.Major() > c.con.Major() { + return false + } else if v.Minor() > c.con.Minor() && !c.minorDirty { + return false + } + + return true +} + +// ~*, ~>* --> >= 0.0.0 (any) +// ~2, ~2.x, ~2.x.x, ~>2, ~>2.x ~>2.x.x --> >=2.0.0, <3.0.0 +// ~2.0, ~2.0.x, ~>2.0, ~>2.0.x --> >=2.0.0, <2.1.0 +// ~1.2, ~1.2.x, ~>1.2, ~>1.2.x --> >=1.2.0, <1.3.0 +// ~1.2.3, ~>1.2.3 --> >=1.2.3, <1.3.0 +// ~1.2.0, ~>1.2.0 --> >=1.2.0, <1.3.0 +func constraintTilde(v *Version, c *constraint) bool { + // If there is a pre-release on the version but the constraint isn't looking + // for them assume that pre-releases are not compatible. See issue 21 for + // more details. + if v.Prerelease() != "" && c.con.Prerelease() == "" { + return false + } + + if v.LessThan(c.con) { + return false + } + + // ~0.0.0 is a special case where all constraints are accepted. It's + // equivalent to >= 0.0.0. + if c.con.Major() == 0 && c.con.Minor() == 0 && c.con.Patch() == 0 && + !c.minorDirty && !c.patchDirty { + return true + } + + if v.Major() != c.con.Major() { + return false + } + + if v.Minor() != c.con.Minor() && !c.minorDirty { + return false + } + + return true +} + +// When there is a .x (dirty) status it automatically opts in to ~. Otherwise +// it's a straight = +func constraintTildeOrEqual(v *Version, c *constraint) bool { + // If there is a pre-release on the version but the constraint isn't looking + // for them assume that pre-releases are not compatible. See issue 21 for + // more details. + if v.Prerelease() != "" && c.con.Prerelease() == "" { + return false + } + + if c.dirty { + c.msg = constraintMsg["~"] + return constraintTilde(v, c) + } + + return v.Equal(c.con) +} + +// ^* --> (any) +// ^2, ^2.x, ^2.x.x --> >=2.0.0, <3.0.0 +// ^2.0, ^2.0.x --> >=2.0.0, <3.0.0 +// ^1.2, ^1.2.x --> >=1.2.0, <2.0.0 +// ^1.2.3 --> >=1.2.3, <2.0.0 +// ^1.2.0 --> >=1.2.0, <2.0.0 +func constraintCaret(v *Version, c *constraint) bool { + // If there is a pre-release on the version but the constraint isn't looking + // for them assume that pre-releases are not compatible. See issue 21 for + // more details. + if v.Prerelease() != "" && c.con.Prerelease() == "" { + return false + } + + if v.LessThan(c.con) { + return false + } + + if v.Major() != c.con.Major() { + return false + } + + return true +} + +var constraintRangeRegex *regexp.Regexp + +const cvRegex string = `v?([0-9|x|X|\*]+)(\.[0-9|x|X|\*]+)?(\.[0-9|x|X|\*]+)?` + + `(-([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?` + + `(\+([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?` + +func isX(x string) bool { + switch x { + case "x", "*", "X": + return true + default: + return false + } +} + +func rewriteRange(i string) string { + m := constraintRangeRegex.FindAllStringSubmatch(i, -1) + if m == nil { + return i + } + o := i + for _, v := range m { + t := fmt.Sprintf(">= %s, <= %s", v[1], v[11]) + o = strings.Replace(o, v[0], t, 1) + } + + return o +} diff --git a/github.com/Masterminds/semver/doc.go b/github.com/Masterminds/semver/doc.go new file mode 100644 index 0000000000..6a6c24c6d6 --- /dev/null +++ b/github.com/Masterminds/semver/doc.go @@ -0,0 +1,115 @@ +/* +Package semver provides the ability to work with Semantic Versions (http://semver.org) in Go. + +Specifically it provides the ability to: + + * Parse semantic versions + * Sort semantic versions + * Check if a semantic version fits within a set of constraints + * Optionally work with a `v` prefix + +Parsing Semantic Versions + +To parse a semantic version use the `NewVersion` function. For example, + + v, err := semver.NewVersion("1.2.3-beta.1+build345") + +If there is an error the version wasn't parseable. The version object has methods +to get the parts of the version, compare it to other versions, convert the +version back into a string, and get the original string. For more details +please see the documentation at https://godoc.org/github.com/Masterminds/semver. + +Sorting Semantic Versions + +A set of versions can be sorted using the `sort` package from the standard library. +For example, + + raw := []string{"1.2.3", "1.0", "1.3", "2", "0.4.2",} + vs := make([]*semver.Version, len(raw)) + for i, r := range raw { + v, err := semver.NewVersion(r) + if err != nil { + t.Errorf("Error parsing version: %s", err) + } + + vs[i] = v + } + + sort.Sort(semver.Collection(vs)) + +Checking Version Constraints + +Checking a version against version constraints is one of the most featureful +parts of the package. + + c, err := semver.NewConstraint(">= 1.2.3") + if err != nil { + // Handle constraint not being parseable. + } + + v, err := semver.NewVersion("1.3") + if err != nil { + // Handle version not being parseable. + } + // Check if the version meets the constraints. The a variable will be true. + a := c.Check(v) + +Basic Comparisons + +There are two elements to the comparisons. First, a comparison string is a list +of comma separated and comparisons. These are then separated by || separated or +comparisons. For example, `">= 1.2, < 3.0.0 || >= 4.2.3"` is looking for a +comparison that's greater than or equal to 1.2 and less than 3.0.0 or is +greater than or equal to 4.2.3. + +The basic comparisons are: + + * `=`: equal (aliased to no operator) + * `!=`: not equal + * `>`: greater than + * `<`: less than + * `>=`: greater than or equal to + * `<=`: less than or equal to + +Hyphen Range Comparisons + +There are multiple methods to handle ranges and the first is hyphens ranges. +These look like: + + * `1.2 - 1.4.5` which is equivalent to `>= 1.2, <= 1.4.5` + * `2.3.4 - 4.5` which is equivalent to `>= 2.3.4, <= 4.5` + +Wildcards In Comparisons + +The `x`, `X`, and `*` characters can be used as a wildcard character. This works +for all comparison operators. When used on the `=` operator it falls +back to the pack level comparison (see tilde below). For example, + + * `1.2.x` is equivalent to `>= 1.2.0, < 1.3.0` + * `>= 1.2.x` is equivalent to `>= 1.2.0` + * `<= 2.x` is equivalent to `<= 3` + * `*` is equivalent to `>= 0.0.0` + +Tilde Range Comparisons (Patch) + +The tilde (`~`) comparison operator is for patch level ranges when a minor +version is specified and major level changes when the minor number is missing. +For example, + + * `~1.2.3` is equivalent to `>= 1.2.3, < 1.3.0` + * `~1` is equivalent to `>= 1, < 2` + * `~2.3` is equivalent to `>= 2.3, < 2.4` + * `~1.2.x` is equivalent to `>= 1.2.0, < 1.3.0` + * `~1.x` is equivalent to `>= 1, < 2` + +Caret Range Comparisons (Major) + +The caret (`^`) comparison operator is for major level changes. This is useful +when comparisons of API versions as a major change is API breaking. For example, + + * `^1.2.3` is equivalent to `>= 1.2.3, < 2.0.0` + * `^1.2.x` is equivalent to `>= 1.2.0, < 2.0.0` + * `^2.3` is equivalent to `>= 2.3, < 3` + * `^2.x` is equivalent to `>= 2.0.0, < 3` +*/ +package semver diff --git a/github.com/Masterminds/semver/version.go b/github.com/Masterminds/semver/version.go new file mode 100644 index 0000000000..400d4f9341 --- /dev/null +++ b/github.com/Masterminds/semver/version.go @@ -0,0 +1,425 @@ +package semver + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "regexp" + "strconv" + "strings" +) + +// The compiled version of the regex created at init() is cached here so it +// only needs to be created once. +var versionRegex *regexp.Regexp +var validPrereleaseRegex *regexp.Regexp + +var ( + // ErrInvalidSemVer is returned a version is found to be invalid when + // being parsed. + ErrInvalidSemVer = errors.New("Invalid Semantic Version") + + // ErrInvalidMetadata is returned when the metadata is an invalid format + ErrInvalidMetadata = errors.New("Invalid Metadata string") + + // ErrInvalidPrerelease is returned when the pre-release is an invalid format + ErrInvalidPrerelease = errors.New("Invalid Prerelease string") +) + +// SemVerRegex is the regular expression used to parse a semantic version. +const SemVerRegex string = `v?([0-9]+)(\.[0-9]+)?(\.[0-9]+)?` + + `(-([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?` + + `(\+([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?` + +// ValidPrerelease is the regular expression which validates +// both prerelease and metadata values. +const ValidPrerelease string = `^([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*)$` + +// Version represents a single semantic version. +type Version struct { + major, minor, patch int64 + pre string + metadata string + original string +} + +func init() { + versionRegex = regexp.MustCompile("^" + SemVerRegex + "$") + validPrereleaseRegex = regexp.MustCompile(ValidPrerelease) +} + +// NewVersion parses a given version and returns an instance of Version or +// an error if unable to parse the version. +func NewVersion(v string) (*Version, error) { + m := versionRegex.FindStringSubmatch(v) + if m == nil { + return nil, ErrInvalidSemVer + } + + sv := &Version{ + metadata: m[8], + pre: m[5], + original: v, + } + + var temp int64 + temp, err := strconv.ParseInt(m[1], 10, 64) + if err != nil { + return nil, fmt.Errorf("Error parsing version segment: %s", err) + } + sv.major = temp + + if m[2] != "" { + temp, err = strconv.ParseInt(strings.TrimPrefix(m[2], "."), 10, 64) + if err != nil { + return nil, fmt.Errorf("Error parsing version segment: %s", err) + } + sv.minor = temp + } else { + sv.minor = 0 + } + + if m[3] != "" { + temp, err = strconv.ParseInt(strings.TrimPrefix(m[3], "."), 10, 64) + if err != nil { + return nil, fmt.Errorf("Error parsing version segment: %s", err) + } + sv.patch = temp + } else { + sv.patch = 0 + } + + return sv, nil +} + +// MustParse parses a given version and panics on error. +func MustParse(v string) *Version { + sv, err := NewVersion(v) + if err != nil { + panic(err) + } + return sv +} + +// String converts a Version object to a string. +// Note, if the original version contained a leading v this version will not. +// See the Original() method to retrieve the original value. Semantic Versions +// don't contain a leading v per the spec. Instead it's optional on +// implementation. +func (v *Version) String() string { + var buf bytes.Buffer + + fmt.Fprintf(&buf, "%d.%d.%d", v.major, v.minor, v.patch) + if v.pre != "" { + fmt.Fprintf(&buf, "-%s", v.pre) + } + if v.metadata != "" { + fmt.Fprintf(&buf, "+%s", v.metadata) + } + + return buf.String() +} + +// Original returns the original value passed in to be parsed. +func (v *Version) Original() string { + return v.original +} + +// Major returns the major version. +func (v *Version) Major() int64 { + return v.major +} + +// Minor returns the minor version. +func (v *Version) Minor() int64 { + return v.minor +} + +// Patch returns the patch version. +func (v *Version) Patch() int64 { + return v.patch +} + +// Prerelease returns the pre-release version. +func (v *Version) Prerelease() string { + return v.pre +} + +// Metadata returns the metadata on the version. +func (v *Version) Metadata() string { + return v.metadata +} + +// originalVPrefix returns the original 'v' prefix if any. +func (v *Version) originalVPrefix() string { + + // Note, only lowercase v is supported as a prefix by the parser. + if v.original != "" && v.original[:1] == "v" { + return v.original[:1] + } + return "" +} + +// IncPatch produces the next patch version. +// If the current version does not have prerelease/metadata information, +// it unsets metadata and prerelease values, increments patch number. +// If the current version has any of prerelease or metadata information, +// it unsets both values and keeps curent patch value +func (v Version) IncPatch() Version { + vNext := v + // according to http://semver.org/#spec-item-9 + // Pre-release versions have a lower precedence than the associated normal version. + // according to http://semver.org/#spec-item-10 + // Build metadata SHOULD be ignored when determining version precedence. + if v.pre != "" { + vNext.metadata = "" + vNext.pre = "" + } else { + vNext.metadata = "" + vNext.pre = "" + vNext.patch = v.patch + 1 + } + vNext.original = v.originalVPrefix() + "" + vNext.String() + return vNext +} + +// IncMinor produces the next minor version. +// Sets patch to 0. +// Increments minor number. +// Unsets metadata. +// Unsets prerelease status. +func (v Version) IncMinor() Version { + vNext := v + vNext.metadata = "" + vNext.pre = "" + vNext.patch = 0 + vNext.minor = v.minor + 1 + vNext.original = v.originalVPrefix() + "" + vNext.String() + return vNext +} + +// IncMajor produces the next major version. +// Sets patch to 0. +// Sets minor to 0. +// Increments major number. +// Unsets metadata. +// Unsets prerelease status. +func (v Version) IncMajor() Version { + vNext := v + vNext.metadata = "" + vNext.pre = "" + vNext.patch = 0 + vNext.minor = 0 + vNext.major = v.major + 1 + vNext.original = v.originalVPrefix() + "" + vNext.String() + return vNext +} + +// SetPrerelease defines the prerelease value. +// Value must not include the required 'hypen' prefix. +func (v Version) SetPrerelease(prerelease string) (Version, error) { + vNext := v + if len(prerelease) > 0 && !validPrereleaseRegex.MatchString(prerelease) { + return vNext, ErrInvalidPrerelease + } + vNext.pre = prerelease + vNext.original = v.originalVPrefix() + "" + vNext.String() + return vNext, nil +} + +// SetMetadata defines metadata value. +// Value must not include the required 'plus' prefix. +func (v Version) SetMetadata(metadata string) (Version, error) { + vNext := v + if len(metadata) > 0 && !validPrereleaseRegex.MatchString(metadata) { + return vNext, ErrInvalidMetadata + } + vNext.metadata = metadata + vNext.original = v.originalVPrefix() + "" + vNext.String() + return vNext, nil +} + +// LessThan tests if one version is less than another one. +func (v *Version) LessThan(o *Version) bool { + return v.Compare(o) < 0 +} + +// GreaterThan tests if one version is greater than another one. +func (v *Version) GreaterThan(o *Version) bool { + return v.Compare(o) > 0 +} + +// Equal tests if two versions are equal to each other. +// Note, versions can be equal with different metadata since metadata +// is not considered part of the comparable version. +func (v *Version) Equal(o *Version) bool { + return v.Compare(o) == 0 +} + +// Compare compares this version to another one. It returns -1, 0, or 1 if +// the version smaller, equal, or larger than the other version. +// +// Versions are compared by X.Y.Z. Build metadata is ignored. Prerelease is +// lower than the version without a prerelease. +func (v *Version) Compare(o *Version) int { + // Compare the major, minor, and patch version for differences. If a + // difference is found return the comparison. + if d := compareSegment(v.Major(), o.Major()); d != 0 { + return d + } + if d := compareSegment(v.Minor(), o.Minor()); d != 0 { + return d + } + if d := compareSegment(v.Patch(), o.Patch()); d != 0 { + return d + } + + // At this point the major, minor, and patch versions are the same. + ps := v.pre + po := o.Prerelease() + + if ps == "" && po == "" { + return 0 + } + if ps == "" { + return 1 + } + if po == "" { + return -1 + } + + return comparePrerelease(ps, po) +} + +// UnmarshalJSON implements JSON.Unmarshaler interface. +func (v *Version) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + temp, err := NewVersion(s) + if err != nil { + return err + } + v.major = temp.major + v.minor = temp.minor + v.patch = temp.patch + v.pre = temp.pre + v.metadata = temp.metadata + v.original = temp.original + temp = nil + return nil +} + +// MarshalJSON implements JSON.Marshaler interface. +func (v *Version) MarshalJSON() ([]byte, error) { + return json.Marshal(v.String()) +} + +func compareSegment(v, o int64) int { + if v < o { + return -1 + } + if v > o { + return 1 + } + + return 0 +} + +func comparePrerelease(v, o string) int { + + // split the prelease versions by their part. The separator, per the spec, + // is a . + sparts := strings.Split(v, ".") + oparts := strings.Split(o, ".") + + // Find the longer length of the parts to know how many loop iterations to + // go through. + slen := len(sparts) + olen := len(oparts) + + l := slen + if olen > slen { + l = olen + } + + // Iterate over each part of the prereleases to compare the differences. + for i := 0; i < l; i++ { + // Since the lentgh of the parts can be different we need to create + // a placeholder. This is to avoid out of bounds issues. + stemp := "" + if i < slen { + stemp = sparts[i] + } + + otemp := "" + if i < olen { + otemp = oparts[i] + } + + d := comparePrePart(stemp, otemp) + if d != 0 { + return d + } + } + + // Reaching here means two versions are of equal value but have different + // metadata (the part following a +). They are not identical in string form + // but the version comparison finds them to be equal. + return 0 +} + +func comparePrePart(s, o string) int { + // Fastpath if they are equal + if s == o { + return 0 + } + + // When s or o are empty we can use the other in an attempt to determine + // the response. + if s == "" { + if o != "" { + return -1 + } + return 1 + } + + if o == "" { + if s != "" { + return 1 + } + return -1 + } + + // When comparing strings "99" is greater than "103". To handle + // cases like this we need to detect numbers and compare them. According + // to the semver spec, numbers are always positive. If there is a - at the + // start like -99 this is to be evaluated as an alphanum. numbers always + // have precedence over alphanum. Parsing as Uints because negative numbers + // are ignored. + + oi, n1 := strconv.ParseUint(o, 10, 64) + si, n2 := strconv.ParseUint(s, 10, 64) + + // The case where both are strings compare the strings + if n1 != nil && n2 != nil { + if s > o { + return 1 + } + return -1 + } else if n1 != nil { + // o is a string and s is a number + return -1 + } else if n2 != nil { + // s is a string and o is a number + return 1 + } + // Both are numbers + if si > oi { + return 1 + } + return -1 + +} diff --git a/github.com/Masterminds/semver/version_fuzz.go b/github.com/Masterminds/semver/version_fuzz.go new file mode 100644 index 0000000000..b42bcd62b9 --- /dev/null +++ b/github.com/Masterminds/semver/version_fuzz.go @@ -0,0 +1,10 @@ +// +build gofuzz + +package semver + +func Fuzz(data []byte) int { + if _, err := NewVersion(string(data)); err != nil { + return 0 + } + return 1 +} diff --git a/github.com/Masterminds/sprig/.gitignore b/github.com/Masterminds/sprig/.gitignore new file mode 100644 index 0000000000..5e3002f88f --- /dev/null +++ b/github.com/Masterminds/sprig/.gitignore @@ -0,0 +1,2 @@ +vendor/ +/.glide diff --git a/github.com/Masterminds/sprig/.travis.yml b/github.com/Masterminds/sprig/.travis.yml new file mode 100644 index 0000000000..b9da8b825b --- /dev/null +++ b/github.com/Masterminds/sprig/.travis.yml @@ -0,0 +1,26 @@ +language: go + +go: + - 1.9.x + - 1.10.x + - 1.11.x + - 1.12.x + - 1.13.x + - tip + +# Setting sudo access to false will let Travis CI use containers rather than +# VMs to run the tests. For more details see: +# - http://docs.travis-ci.com/user/workers/container-based-infrastructure/ +# - http://docs.travis-ci.com/user/workers/standard-infrastructure/ +sudo: false + +script: + - make setup test + +notifications: + webhooks: + urls: + - https://webhooks.gitter.im/e/06e3328629952dabe3e0 + on_success: change # options: [always|never|change] default: always + on_failure: always # options: [always|never|change] default: always + on_start: never # options: [always|never|change] default: always diff --git a/github.com/Masterminds/sprig/CHANGELOG.md b/github.com/Masterminds/sprig/CHANGELOG.md new file mode 100644 index 0000000000..6a79fbde46 --- /dev/null +++ b/github.com/Masterminds/sprig/CHANGELOG.md @@ -0,0 +1,282 @@ +# Changelog + +## Release 2.22.0 (2019-10-02) + +### Added + +- #173: Added getHostByName function to resolve dns names to ips (thanks @fcgravalos) +- #195: Added deepCopy function for use with dicts + +### Changed + +- Updated merge and mergeOverwrite documentation to explain copying and how to + use deepCopy with it + +## Release 2.21.0 (2019-09-18) + +### Added + +- #122: Added encryptAES/decryptAES functions (thanks @n0madic) +- #128: Added toDecimal support (thanks @Dean-Coakley) +- #169: Added list contcat (thanks @astorath) +- #174: Added deepEqual function (thanks @bonifaido) +- #170: Added url parse and join functions (thanks @astorath) + +### Changed + +- #171: Updated glide config for Google UUID to v1 and to add ranges to semver and testify + +### Fixed + +- #172: Fix semver wildcard example (thanks @piepmatz) +- #175: Fix dateInZone doc example (thanks @s3than) + +## Release 2.20.0 (2019-06-18) + +### Added + +- #164: Adding function to get unix epoch for a time (@mattfarina) +- #166: Adding tests for date_in_zone (@mattfarina) + +### Changed + +- #144: Fix function comments based on best practices from Effective Go (@CodeLingoTeam) +- #150: Handles pointer type for time.Time in "htmlDate" (@mapreal19) +- #161, #157, #160, #153, #158, #156, #155, #159, #152 documentation updates (@badeadan) + +### Fixed + +## Release 2.19.0 (2019-03-02) + +IMPORTANT: This release reverts a change from 2.18.0 + +In the previous release (2.18), we prematurely merged a partial change to the crypto functions that led to creating two sets of crypto functions (I blame @technosophos -- since that's me). This release rolls back that change, and does what was originally intended: It alters the existing crypto functions to use secure random. + +We debated whether this classifies as a change worthy of major revision, but given the proximity to the last release, we have decided that treating 2.18 as a faulty release is the correct course of action. We apologize for any inconvenience. + +### Changed + +- Fix substr panic 35fb796 (Alexey igrychev) +- Remove extra period 1eb7729 (Matthew Lorimor) +- Make random string functions use crypto by default 6ceff26 (Matthew Lorimor) +- README edits/fixes/suggestions 08fe136 (Lauri Apple) + + +## Release 2.18.0 (2019-02-12) + +### Added + +- Added mergeOverwrite function +- cryptographic functions that use secure random (see fe1de12) + +### Changed + +- Improve documentation of regexMatch function, resolves #139 90b89ce (Jan Tagscherer) +- Handle has for nil list 9c10885 (Daniel Cohen) +- Document behaviour of mergeOverwrite fe0dbe9 (Lukas Rieder) +- doc: adds missing documentation. 4b871e6 (Fernandez Ludovic) +- Replace outdated goutils imports 01893d2 (Matthew Lorimor) +- Surface crypto secure random strings from goutils fe1de12 (Matthew Lorimor) +- Handle untyped nil values as paramters to string functions 2b2ec8f (Morten Torkildsen) + +### Fixed + +- Fix dict merge issue and provide mergeOverwrite .dst .src1 to overwrite from src -> dst 4c59c12 (Lukas Rieder) +- Fix substr var names and comments d581f80 (Dean Coakley) +- Fix substr documentation 2737203 (Dean Coakley) + +## Release 2.17.1 (2019-01-03) + +### Fixed + +The 2.17.0 release did not have a version pinned for xstrings, which caused compilation failures when xstrings < 1.2 was used. This adds the correct version string to glide.yaml. + +## Release 2.17.0 (2019-01-03) + +### Added + +- adds alder32sum function and test 6908fc2 (marshallford) +- Added kebabcase function ca331a1 (Ilyes512) + +### Changed + +- Update goutils to 1.1.0 4e1125d (Matt Butcher) + +### Fixed + +- Fix 'has' documentation e3f2a85 (dean-coakley) +- docs(dict): fix typo in pick example dc424f9 (Dustin Specker) +- fixes spelling errors... not sure how that happened 4cf188a (marshallford) + +## Release 2.16.0 (2018-08-13) + +### Added + +- add splitn function fccb0b0 (Helgi Þorbjörnsson) +- Add slice func df28ca7 (gongdo) +- Generate serial number a3bdffd (Cody Coons) +- Extract values of dict with values function df39312 (Lawrence Jones) + +### Changed + +- Modify panic message for list.slice ae38335 (gongdo) +- Minor improvement in code quality - Removed an unreachable piece of code at defaults.go#L26:6 - Resolve formatting issues. 5834241 (Abhishek Kashyap) +- Remove duplicated documentation 1d97af1 (Matthew Fisher) +- Test on go 1.11 49df809 (Helgi Þormar Þorbjörnsson) + +### Fixed + +- Fix file permissions c5f40b5 (gongdo) +- Fix example for buildCustomCert 7779e0d (Tin Lam) + +## Release 2.15.0 (2018-04-02) + +### Added + +- #68 and #69: Add json helpers to docs (thanks @arunvelsriram) +- #66: Add ternary function (thanks @binoculars) +- #67: Allow keys function to take multiple dicts (thanks @binoculars) +- #89: Added sha1sum to crypto function (thanks @benkeil) +- #81: Allow customizing Root CA that used by genSignedCert (thanks @chenzhiwei) +- #92: Add travis testing for go 1.10 +- #93: Adding appveyor config for windows testing + +### Changed + +- #90: Updating to more recent dependencies +- #73: replace satori/go.uuid with google/uuid (thanks @petterw) + +### Fixed + +- #76: Fixed documentation typos (thanks @Thiht) +- Fixed rounding issue on the `ago` function. Note, the removes support for Go 1.8 and older + +## Release 2.14.1 (2017-12-01) + +### Fixed + +- #60: Fix typo in function name documentation (thanks @neil-ca-moore) +- #61: Removing line with {{ due to blocking github pages genertion +- #64: Update the list functions to handle int, string, and other slices for compatibility + +## Release 2.14.0 (2017-10-06) + +This new version of Sprig adds a set of functions for generating and working with SSL certificates. + +- `genCA` generates an SSL Certificate Authority +- `genSelfSignedCert` generates an SSL self-signed certificate +- `genSignedCert` generates an SSL certificate and key based on a given CA + +## Release 2.13.0 (2017-09-18) + +This release adds new functions, including: + +- `regexMatch`, `regexFindAll`, `regexFind`, `regexReplaceAll`, `regexReplaceAllLiteral`, and `regexSplit` to work with regular expressions +- `floor`, `ceil`, and `round` math functions +- `toDate` converts a string to a date +- `nindent` is just like `indent` but also prepends a new line +- `ago` returns the time from `time.Now` + +### Added + +- #40: Added basic regex functionality (thanks @alanquillin) +- #41: Added ceil floor and round functions (thanks @alanquillin) +- #48: Added toDate function (thanks @andreynering) +- #50: Added nindent function (thanks @binoculars) +- #46: Added ago function (thanks @slayer) + +### Changed + +- #51: Updated godocs to include new string functions (thanks @curtisallen) +- #49: Added ability to merge multiple dicts (thanks @binoculars) + +## Release 2.12.0 (2017-05-17) + +- `snakecase`, `camelcase`, and `shuffle` are three new string functions +- `fail` allows you to bail out of a template render when conditions are not met + +## Release 2.11.0 (2017-05-02) + +- Added `toJson` and `toPrettyJson` +- Added `merge` +- Refactored documentation + +## Release 2.10.0 (2017-03-15) + +- Added `semver` and `semverCompare` for Semantic Versions +- `list` replaces `tuple` +- Fixed issue with `join` +- Added `first`, `last`, `intial`, `rest`, `prepend`, `append`, `toString`, `toStrings`, `sortAlpha`, `reverse`, `coalesce`, `pluck`, `pick`, `compact`, `keys`, `omit`, `uniq`, `has`, `without` + +## Release 2.9.0 (2017-02-23) + +- Added `splitList` to split a list +- Added crypto functions of `genPrivateKey` and `derivePassword` + +## Release 2.8.0 (2016-12-21) + +- Added access to several path functions (`base`, `dir`, `clean`, `ext`, and `abs`) +- Added functions for _mutating_ dictionaries (`set`, `unset`, `hasKey`) + +## Release 2.7.0 (2016-12-01) + +- Added `sha256sum` to generate a hash of an input +- Added functions to convert a numeric or string to `int`, `int64`, `float64` + +## Release 2.6.0 (2016-10-03) + +- Added a `uuidv4` template function for generating UUIDs inside of a template. + +## Release 2.5.0 (2016-08-19) + +- New `trimSuffix`, `trimPrefix`, `hasSuffix`, and `hasPrefix` functions +- New aliases have been added for a few functions that didn't follow the naming conventions (`trimAll` and `abbrevBoth`) +- `trimall` and `abbrevboth` (notice the case) are deprecated and will be removed in 3.0.0 + +## Release 2.4.0 (2016-08-16) + +- Adds two functions: `until` and `untilStep` + +## Release 2.3.0 (2016-06-21) + +- cat: Concatenate strings with whitespace separators. +- replace: Replace parts of a string: `replace " " "-" "Me First"` renders "Me-First" +- plural: Format plurals: `len "foo" | plural "one foo" "many foos"` renders "many foos" +- indent: Indent blocks of text in a way that is sensitive to "\n" characters. + +## Release 2.2.0 (2016-04-21) + +- Added a `genPrivateKey` function (Thanks @bacongobbler) + +## Release 2.1.0 (2016-03-30) + +- `default` now prints the default value when it does not receive a value down the pipeline. It is much safer now to do `{{.Foo | default "bar"}}`. +- Added accessors for "hermetic" functions. These return only functions that, when given the same input, produce the same output. + +## Release 2.0.0 (2016-03-29) + +Because we switched from `int` to `int64` as the return value for all integer math functions, the library's major version number has been incremented. + +- `min` complements `max` (formerly `biggest`) +- `empty` indicates that a value is the empty value for its type +- `tuple` creates a tuple inside of a template: `{{$t := tuple "a", "b" "c"}}` +- `dict` creates a dictionary inside of a template `{{$d := dict "key1" "val1" "key2" "val2"}}` +- Date formatters have been added for HTML dates (as used in `date` input fields) +- Integer math functions can convert from a number of types, including `string` (via `strconv.ParseInt`). + +## Release 1.2.0 (2016-02-01) + +- Added quote and squote +- Added b32enc and b32dec +- add now takes varargs +- biggest now takes varargs + +## Release 1.1.0 (2015-12-29) + +- Added #4: Added contains function. strings.Contains, but with the arguments + switched to simplify common pipelines. (thanks krancour) +- Added Travis-CI testing support + +## Release 1.0.0 (2015-12-23) + +- Initial release diff --git a/github.com/Masterminds/sprig/LICENSE.txt b/github.com/Masterminds/sprig/LICENSE.txt new file mode 100644 index 0000000000..5c95accc2e --- /dev/null +++ b/github.com/Masterminds/sprig/LICENSE.txt @@ -0,0 +1,20 @@ +Sprig +Copyright (C) 2013 Masterminds + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/github.com/Masterminds/sprig/Makefile b/github.com/Masterminds/sprig/Makefile new file mode 100644 index 0000000000..63a93fdf79 --- /dev/null +++ b/github.com/Masterminds/sprig/Makefile @@ -0,0 +1,13 @@ + +HAS_GLIDE := $(shell command -v glide;) + +.PHONY: test +test: + go test -v . + +.PHONY: setup +setup: +ifndef HAS_GLIDE + go get -u github.com/Masterminds/glide +endif + glide install diff --git a/github.com/Masterminds/sprig/README.md b/github.com/Masterminds/sprig/README.md new file mode 100644 index 0000000000..b70569585f --- /dev/null +++ b/github.com/Masterminds/sprig/README.md @@ -0,0 +1,78 @@ +# Sprig: Template functions for Go templates +[![Stability: Sustained](https://masterminds.github.io/stability/sustained.svg)](https://masterminds.github.io/stability/sustained.html) +[![Build Status](https://travis-ci.org/Masterminds/sprig.svg?branch=master)](https://travis-ci.org/Masterminds/sprig) + +The Go language comes with a [built-in template +language](http://golang.org/pkg/text/template/), but not +very many template functions. Sprig is a library that provides more than 100 commonly +used template functions. + +It is inspired by the template functions found in +[Twig](http://twig.sensiolabs.org/documentation) and in various +JavaScript libraries, such as [underscore.js](http://underscorejs.org/). + +## Usage + +**Template developers**: Please use Sprig's [function documentation](http://masterminds.github.io/sprig/) for +detailed instructions and code snippets for the >100 template functions available. + +**Go developers**: If you'd like to include Sprig as a library in your program, +our API documentation is available [at GoDoc.org](http://godoc.org/github.com/Masterminds/sprig). + +For standard usage, read on. + +### Load the Sprig library + +To load the Sprig `FuncMap`: + +```go + +import ( + "github.com/Masterminds/sprig" + "html/template" +) + +// This example illustrates that the FuncMap *must* be set before the +// templates themselves are loaded. +tpl := template.Must( + template.New("base").Funcs(sprig.FuncMap()).ParseGlob("*.html") +) + + +``` + +### Calling the functions inside of templates + +By convention, all functions are lowercase. This seems to follow the Go +idiom for template functions (as opposed to template methods, which are +TitleCase). For example, this: + +``` +{{ "hello!" | upper | repeat 5 }} +``` + +produces this: + +``` +HELLO!HELLO!HELLO!HELLO!HELLO! +``` + +## Principles Driving Our Function Selection + +We followed these principles to decide which functions to add and how to implement them: + +- Use template functions to build layout. The following + types of operations are within the domain of template functions: + - Formatting + - Layout + - Simple type conversions + - Utilities that assist in handling common formatting and layout needs (e.g. arithmetic) +- Template functions should not return errors unless there is no way to print + a sensible value. For example, converting a string to an integer should not + produce an error if conversion fails. Instead, it should display a default + value. +- Simple math is necessary for grid layouts, pagers, and so on. Complex math + (anything other than arithmetic) should be done outside of templates. +- Template functions only deal with the data passed into them. They never retrieve + data from a source. +- Finally, do not override core Go template functions. diff --git a/github.com/Masterminds/sprig/appveyor.yml b/github.com/Masterminds/sprig/appveyor.yml new file mode 100644 index 0000000000..d545a987a3 --- /dev/null +++ b/github.com/Masterminds/sprig/appveyor.yml @@ -0,0 +1,26 @@ + +version: build-{build}.{branch} + +clone_folder: C:\gopath\src\github.com\Masterminds\sprig +shallow_clone: true + +environment: + GOPATH: C:\gopath + +platform: + - x64 + +install: + - go get -u github.com/Masterminds/glide + - set PATH=%GOPATH%\bin;%PATH% + - go version + - go env + +build_script: + - glide install + - go install ./... + +test_script: + - go test -v + +deploy: off diff --git a/github.com/Masterminds/sprig/crypto.go b/github.com/Masterminds/sprig/crypto.go new file mode 100644 index 0000000000..7a418ba88d --- /dev/null +++ b/github.com/Masterminds/sprig/crypto.go @@ -0,0 +1,502 @@ +package sprig + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/dsa" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/hmac" + "crypto/rand" + "crypto/rsa" + "crypto/sha1" + "crypto/sha256" + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" + "encoding/base64" + "encoding/binary" + "encoding/hex" + "encoding/pem" + "errors" + "fmt" + "io" + "hash/adler32" + "math/big" + "net" + "time" + + "github.com/google/uuid" + "golang.org/x/crypto/scrypt" +) + +func sha256sum(input string) string { + hash := sha256.Sum256([]byte(input)) + return hex.EncodeToString(hash[:]) +} + +func sha1sum(input string) string { + hash := sha1.Sum([]byte(input)) + return hex.EncodeToString(hash[:]) +} + +func adler32sum(input string) string { + hash := adler32.Checksum([]byte(input)) + return fmt.Sprintf("%d", hash) +} + +// uuidv4 provides a safe and secure UUID v4 implementation +func uuidv4() string { + return fmt.Sprintf("%s", uuid.New()) +} + +var master_password_seed = "com.lyndir.masterpassword" + +var password_type_templates = map[string][][]byte{ + "maximum": {[]byte("anoxxxxxxxxxxxxxxxxx"), []byte("axxxxxxxxxxxxxxxxxno")}, + "long": {[]byte("CvcvnoCvcvCvcv"), []byte("CvcvCvcvnoCvcv"), []byte("CvcvCvcvCvcvno"), []byte("CvccnoCvcvCvcv"), []byte("CvccCvcvnoCvcv"), + []byte("CvccCvcvCvcvno"), []byte("CvcvnoCvccCvcv"), []byte("CvcvCvccnoCvcv"), []byte("CvcvCvccCvcvno"), []byte("CvcvnoCvcvCvcc"), + []byte("CvcvCvcvnoCvcc"), []byte("CvcvCvcvCvccno"), []byte("CvccnoCvccCvcv"), []byte("CvccCvccnoCvcv"), []byte("CvccCvccCvcvno"), + []byte("CvcvnoCvccCvcc"), []byte("CvcvCvccnoCvcc"), []byte("CvcvCvccCvccno"), []byte("CvccnoCvcvCvcc"), []byte("CvccCvcvnoCvcc"), + []byte("CvccCvcvCvccno")}, + "medium": {[]byte("CvcnoCvc"), []byte("CvcCvcno")}, + "short": {[]byte("Cvcn")}, + "basic": {[]byte("aaanaaan"), []byte("aannaaan"), []byte("aaannaaa")}, + "pin": {[]byte("nnnn")}, +} + +var template_characters = map[byte]string{ + 'V': "AEIOU", + 'C': "BCDFGHJKLMNPQRSTVWXYZ", + 'v': "aeiou", + 'c': "bcdfghjklmnpqrstvwxyz", + 'A': "AEIOUBCDFGHJKLMNPQRSTVWXYZ", + 'a': "AEIOUaeiouBCDFGHJKLMNPQRSTVWXYZbcdfghjklmnpqrstvwxyz", + 'n': "0123456789", + 'o': "@&%?,=[]_:-+*$#!'^~;()/.", + 'x': "AEIOUaeiouBCDFGHJKLMNPQRSTVWXYZbcdfghjklmnpqrstvwxyz0123456789!@#$%^&*()", +} + +func derivePassword(counter uint32, password_type, password, user, site string) string { + var templates = password_type_templates[password_type] + if templates == nil { + return fmt.Sprintf("cannot find password template %s", password_type) + } + + var buffer bytes.Buffer + buffer.WriteString(master_password_seed) + binary.Write(&buffer, binary.BigEndian, uint32(len(user))) + buffer.WriteString(user) + + salt := buffer.Bytes() + key, err := scrypt.Key([]byte(password), salt, 32768, 8, 2, 64) + if err != nil { + return fmt.Sprintf("failed to derive password: %s", err) + } + + buffer.Truncate(len(master_password_seed)) + binary.Write(&buffer, binary.BigEndian, uint32(len(site))) + buffer.WriteString(site) + binary.Write(&buffer, binary.BigEndian, counter) + + var hmacv = hmac.New(sha256.New, key) + hmacv.Write(buffer.Bytes()) + var seed = hmacv.Sum(nil) + var temp = templates[int(seed[0])%len(templates)] + + buffer.Truncate(0) + for i, element := range temp { + pass_chars := template_characters[element] + pass_char := pass_chars[int(seed[i+1])%len(pass_chars)] + buffer.WriteByte(pass_char) + } + + return buffer.String() +} + +func generatePrivateKey(typ string) string { + var priv interface{} + var err error + switch typ { + case "", "rsa": + // good enough for government work + priv, err = rsa.GenerateKey(rand.Reader, 4096) + case "dsa": + key := new(dsa.PrivateKey) + // again, good enough for government work + if err = dsa.GenerateParameters(&key.Parameters, rand.Reader, dsa.L2048N256); err != nil { + return fmt.Sprintf("failed to generate dsa params: %s", err) + } + err = dsa.GenerateKey(key, rand.Reader) + priv = key + case "ecdsa": + // again, good enough for government work + priv, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + default: + return "Unknown type " + typ + } + if err != nil { + return fmt.Sprintf("failed to generate private key: %s", err) + } + + return string(pem.EncodeToMemory(pemBlockForKey(priv))) +} + +type DSAKeyFormat struct { + Version int + P, Q, G, Y, X *big.Int +} + +func pemBlockForKey(priv interface{}) *pem.Block { + switch k := priv.(type) { + case *rsa.PrivateKey: + return &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(k)} + case *dsa.PrivateKey: + val := DSAKeyFormat{ + P: k.P, Q: k.Q, G: k.G, + Y: k.Y, X: k.X, + } + bytes, _ := asn1.Marshal(val) + return &pem.Block{Type: "DSA PRIVATE KEY", Bytes: bytes} + case *ecdsa.PrivateKey: + b, _ := x509.MarshalECPrivateKey(k) + return &pem.Block{Type: "EC PRIVATE KEY", Bytes: b} + default: + return nil + } +} + +type certificate struct { + Cert string + Key string +} + +func buildCustomCertificate(b64cert string, b64key string) (certificate, error) { + crt := certificate{} + + cert, err := base64.StdEncoding.DecodeString(b64cert) + if err != nil { + return crt, errors.New("unable to decode base64 certificate") + } + + key, err := base64.StdEncoding.DecodeString(b64key) + if err != nil { + return crt, errors.New("unable to decode base64 private key") + } + + decodedCert, _ := pem.Decode(cert) + if decodedCert == nil { + return crt, errors.New("unable to decode certificate") + } + _, err = x509.ParseCertificate(decodedCert.Bytes) + if err != nil { + return crt, fmt.Errorf( + "error parsing certificate: decodedCert.Bytes: %s", + err, + ) + } + + decodedKey, _ := pem.Decode(key) + if decodedKey == nil { + return crt, errors.New("unable to decode key") + } + _, err = x509.ParsePKCS1PrivateKey(decodedKey.Bytes) + if err != nil { + return crt, fmt.Errorf( + "error parsing prive key: decodedKey.Bytes: %s", + err, + ) + } + + crt.Cert = string(cert) + crt.Key = string(key) + + return crt, nil +} + +func generateCertificateAuthority( + cn string, + daysValid int, +) (certificate, error) { + ca := certificate{} + + template, err := getBaseCertTemplate(cn, nil, nil, daysValid) + if err != nil { + return ca, err + } + // Override KeyUsage and IsCA + template.KeyUsage = x509.KeyUsageKeyEncipherment | + x509.KeyUsageDigitalSignature | + x509.KeyUsageCertSign + template.IsCA = true + + priv, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return ca, fmt.Errorf("error generating rsa key: %s", err) + } + + ca.Cert, ca.Key, err = getCertAndKey(template, priv, template, priv) + if err != nil { + return ca, err + } + + return ca, nil +} + +func generateSelfSignedCertificate( + cn string, + ips []interface{}, + alternateDNS []interface{}, + daysValid int, +) (certificate, error) { + cert := certificate{} + + template, err := getBaseCertTemplate(cn, ips, alternateDNS, daysValid) + if err != nil { + return cert, err + } + + priv, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return cert, fmt.Errorf("error generating rsa key: %s", err) + } + + cert.Cert, cert.Key, err = getCertAndKey(template, priv, template, priv) + if err != nil { + return cert, err + } + + return cert, nil +} + +func generateSignedCertificate( + cn string, + ips []interface{}, + alternateDNS []interface{}, + daysValid int, + ca certificate, +) (certificate, error) { + cert := certificate{} + + decodedSignerCert, _ := pem.Decode([]byte(ca.Cert)) + if decodedSignerCert == nil { + return cert, errors.New("unable to decode certificate") + } + signerCert, err := x509.ParseCertificate(decodedSignerCert.Bytes) + if err != nil { + return cert, fmt.Errorf( + "error parsing certificate: decodedSignerCert.Bytes: %s", + err, + ) + } + decodedSignerKey, _ := pem.Decode([]byte(ca.Key)) + if decodedSignerKey == nil { + return cert, errors.New("unable to decode key") + } + signerKey, err := x509.ParsePKCS1PrivateKey(decodedSignerKey.Bytes) + if err != nil { + return cert, fmt.Errorf( + "error parsing prive key: decodedSignerKey.Bytes: %s", + err, + ) + } + + template, err := getBaseCertTemplate(cn, ips, alternateDNS, daysValid) + if err != nil { + return cert, err + } + + priv, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return cert, fmt.Errorf("error generating rsa key: %s", err) + } + + cert.Cert, cert.Key, err = getCertAndKey( + template, + priv, + signerCert, + signerKey, + ) + if err != nil { + return cert, err + } + + return cert, nil +} + +func getCertAndKey( + template *x509.Certificate, + signeeKey *rsa.PrivateKey, + parent *x509.Certificate, + signingKey *rsa.PrivateKey, +) (string, string, error) { + derBytes, err := x509.CreateCertificate( + rand.Reader, + template, + parent, + &signeeKey.PublicKey, + signingKey, + ) + if err != nil { + return "", "", fmt.Errorf("error creating certificate: %s", err) + } + + certBuffer := bytes.Buffer{} + if err := pem.Encode( + &certBuffer, + &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}, + ); err != nil { + return "", "", fmt.Errorf("error pem-encoding certificate: %s", err) + } + + keyBuffer := bytes.Buffer{} + if err := pem.Encode( + &keyBuffer, + &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(signeeKey), + }, + ); err != nil { + return "", "", fmt.Errorf("error pem-encoding key: %s", err) + } + + return string(certBuffer.Bytes()), string(keyBuffer.Bytes()), nil +} + +func getBaseCertTemplate( + cn string, + ips []interface{}, + alternateDNS []interface{}, + daysValid int, +) (*x509.Certificate, error) { + ipAddresses, err := getNetIPs(ips) + if err != nil { + return nil, err + } + dnsNames, err := getAlternateDNSStrs(alternateDNS) + if err != nil { + return nil, err + } + serialNumberUpperBound := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberUpperBound) + if err != nil { + return nil, err + } + return &x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + CommonName: cn, + }, + IPAddresses: ipAddresses, + DNSNames: dnsNames, + NotBefore: time.Now(), + NotAfter: time.Now().Add(time.Hour * 24 * time.Duration(daysValid)), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{ + x509.ExtKeyUsageServerAuth, + x509.ExtKeyUsageClientAuth, + }, + BasicConstraintsValid: true, + }, nil +} + +func getNetIPs(ips []interface{}) ([]net.IP, error) { + if ips == nil { + return []net.IP{}, nil + } + var ipStr string + var ok bool + var netIP net.IP + netIPs := make([]net.IP, len(ips)) + for i, ip := range ips { + ipStr, ok = ip.(string) + if !ok { + return nil, fmt.Errorf("error parsing ip: %v is not a string", ip) + } + netIP = net.ParseIP(ipStr) + if netIP == nil { + return nil, fmt.Errorf("error parsing ip: %s", ipStr) + } + netIPs[i] = netIP + } + return netIPs, nil +} + +func getAlternateDNSStrs(alternateDNS []interface{}) ([]string, error) { + if alternateDNS == nil { + return []string{}, nil + } + var dnsStr string + var ok bool + alternateDNSStrs := make([]string, len(alternateDNS)) + for i, dns := range alternateDNS { + dnsStr, ok = dns.(string) + if !ok { + return nil, fmt.Errorf( + "error processing alternate dns name: %v is not a string", + dns, + ) + } + alternateDNSStrs[i] = dnsStr + } + return alternateDNSStrs, nil +} + +func encryptAES(password string, plaintext string) (string, error) { + if plaintext == "" { + return "", nil + } + + key := make([]byte, 32) + copy(key, []byte(password)) + block, err := aes.NewCipher(key) + if err != nil { + return "", err + } + + content := []byte(plaintext) + blockSize := block.BlockSize() + padding := blockSize - len(content)%blockSize + padtext := bytes.Repeat([]byte{byte(padding)}, padding) + content = append(content, padtext...) + + ciphertext := make([]byte, aes.BlockSize+len(content)) + + iv := ciphertext[:aes.BlockSize] + if _, err := io.ReadFull(rand.Reader, iv); err != nil { + return "", err + } + + mode := cipher.NewCBCEncrypter(block, iv) + mode.CryptBlocks(ciphertext[aes.BlockSize:], content) + + return base64.StdEncoding.EncodeToString(ciphertext), nil +} + +func decryptAES(password string, crypt64 string) (string, error) { + if crypt64 == "" { + return "", nil + } + + key := make([]byte, 32) + copy(key, []byte(password)) + + crypt, err := base64.StdEncoding.DecodeString(crypt64) + if err != nil { + return "", err + } + + block, err := aes.NewCipher(key) + if err != nil { + return "", err + } + + iv := crypt[:aes.BlockSize] + crypt = crypt[aes.BlockSize:] + decrypted := make([]byte, len(crypt)) + mode := cipher.NewCBCDecrypter(block, iv) + mode.CryptBlocks(decrypted, crypt) + + return string(decrypted[:len(decrypted)-int(decrypted[len(decrypted)-1])]), nil +} diff --git a/github.com/Masterminds/sprig/date.go b/github.com/Masterminds/sprig/date.go new file mode 100644 index 0000000000..d1d6155d72 --- /dev/null +++ b/github.com/Masterminds/sprig/date.go @@ -0,0 +1,83 @@ +package sprig + +import ( + "strconv" + "time" +) + +// Given a format and a date, format the date string. +// +// Date can be a `time.Time` or an `int, int32, int64`. +// In the later case, it is treated as seconds since UNIX +// epoch. +func date(fmt string, date interface{}) string { + return dateInZone(fmt, date, "Local") +} + +func htmlDate(date interface{}) string { + return dateInZone("2006-01-02", date, "Local") +} + +func htmlDateInZone(date interface{}, zone string) string { + return dateInZone("2006-01-02", date, zone) +} + +func dateInZone(fmt string, date interface{}, zone string) string { + var t time.Time + switch date := date.(type) { + default: + t = time.Now() + case time.Time: + t = date + case *time.Time: + t = *date + case int64: + t = time.Unix(date, 0) + case int: + t = time.Unix(int64(date), 0) + case int32: + t = time.Unix(int64(date), 0) + } + + loc, err := time.LoadLocation(zone) + if err != nil { + loc, _ = time.LoadLocation("UTC") + } + + return t.In(loc).Format(fmt) +} + +func dateModify(fmt string, date time.Time) time.Time { + d, err := time.ParseDuration(fmt) + if err != nil { + return date + } + return date.Add(d) +} + +func dateAgo(date interface{}) string { + var t time.Time + + switch date := date.(type) { + default: + t = time.Now() + case time.Time: + t = date + case int64: + t = time.Unix(date, 0) + case int: + t = time.Unix(int64(date), 0) + } + // Drop resolution to seconds + duration := time.Since(t).Round(time.Second) + return duration.String() +} + +func toDate(fmt, str string) time.Time { + t, _ := time.ParseInLocation(fmt, str, time.Local) + return t +} + +func unixEpoch(date time.Time) string { + return strconv.FormatInt(date.Unix(), 10) +} diff --git a/github.com/Masterminds/sprig/defaults.go b/github.com/Masterminds/sprig/defaults.go new file mode 100644 index 0000000000..ed6a8ab291 --- /dev/null +++ b/github.com/Masterminds/sprig/defaults.go @@ -0,0 +1,83 @@ +package sprig + +import ( + "encoding/json" + "reflect" +) + +// dfault checks whether `given` is set, and returns default if not set. +// +// This returns `d` if `given` appears not to be set, and `given` otherwise. +// +// For numeric types 0 is unset. +// For strings, maps, arrays, and slices, len() = 0 is considered unset. +// For bool, false is unset. +// Structs are never considered unset. +// +// For everything else, including pointers, a nil value is unset. +func dfault(d interface{}, given ...interface{}) interface{} { + + if empty(given) || empty(given[0]) { + return d + } + return given[0] +} + +// empty returns true if the given value has the zero value for its type. +func empty(given interface{}) bool { + g := reflect.ValueOf(given) + if !g.IsValid() { + return true + } + + // Basically adapted from text/template.isTrue + switch g.Kind() { + default: + return g.IsNil() + case reflect.Array, reflect.Slice, reflect.Map, reflect.String: + return g.Len() == 0 + case reflect.Bool: + return g.Bool() == false + case reflect.Complex64, reflect.Complex128: + return g.Complex() == 0 + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return g.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return g.Uint() == 0 + case reflect.Float32, reflect.Float64: + return g.Float() == 0 + case reflect.Struct: + return false + } +} + +// coalesce returns the first non-empty value. +func coalesce(v ...interface{}) interface{} { + for _, val := range v { + if !empty(val) { + return val + } + } + return nil +} + +// toJson encodes an item into a JSON string +func toJson(v interface{}) string { + output, _ := json.Marshal(v) + return string(output) +} + +// toPrettyJson encodes an item into a pretty (indented) JSON string +func toPrettyJson(v interface{}) string { + output, _ := json.MarshalIndent(v, "", " ") + return string(output) +} + +// ternary returns the first value if the last value is true, otherwise returns the second value. +func ternary(vt interface{}, vf interface{}, v bool) interface{} { + if v { + return vt + } + + return vf +} diff --git a/github.com/Masterminds/sprig/dict.go b/github.com/Masterminds/sprig/dict.go new file mode 100644 index 0000000000..738405b433 --- /dev/null +++ b/github.com/Masterminds/sprig/dict.go @@ -0,0 +1,119 @@ +package sprig + +import ( + "github.com/imdario/mergo" + "github.com/mitchellh/copystructure" +) + +func set(d map[string]interface{}, key string, value interface{}) map[string]interface{} { + d[key] = value + return d +} + +func unset(d map[string]interface{}, key string) map[string]interface{} { + delete(d, key) + return d +} + +func hasKey(d map[string]interface{}, key string) bool { + _, ok := d[key] + return ok +} + +func pluck(key string, d ...map[string]interface{}) []interface{} { + res := []interface{}{} + for _, dict := range d { + if val, ok := dict[key]; ok { + res = append(res, val) + } + } + return res +} + +func keys(dicts ...map[string]interface{}) []string { + k := []string{} + for _, dict := range dicts { + for key := range dict { + k = append(k, key) + } + } + return k +} + +func pick(dict map[string]interface{}, keys ...string) map[string]interface{} { + res := map[string]interface{}{} + for _, k := range keys { + if v, ok := dict[k]; ok { + res[k] = v + } + } + return res +} + +func omit(dict map[string]interface{}, keys ...string) map[string]interface{} { + res := map[string]interface{}{} + + omit := make(map[string]bool, len(keys)) + for _, k := range keys { + omit[k] = true + } + + for k, v := range dict { + if _, ok := omit[k]; !ok { + res[k] = v + } + } + return res +} + +func dict(v ...interface{}) map[string]interface{} { + dict := map[string]interface{}{} + lenv := len(v) + for i := 0; i < lenv; i += 2 { + key := strval(v[i]) + if i+1 >= lenv { + dict[key] = "" + continue + } + dict[key] = v[i+1] + } + return dict +} + +func merge(dst map[string]interface{}, srcs ...map[string]interface{}) interface{} { + for _, src := range srcs { + if err := mergo.Merge(&dst, src); err != nil { + // Swallow errors inside of a template. + return "" + } + } + return dst +} + +func mergeOverwrite(dst map[string]interface{}, srcs ...map[string]interface{}) interface{} { + for _, src := range srcs { + if err := mergo.MergeWithOverwrite(&dst, src); err != nil { + // Swallow errors inside of a template. + return "" + } + } + return dst +} + +func values(dict map[string]interface{}) []interface{} { + values := []interface{}{} + for _, value := range dict { + values = append(values, value) + } + + return values +} + +func deepCopy(i interface{}) interface{} { + c, err := copystructure.Copy(i) + if err != nil { + panic("deepCopy error: " + err.Error()) + } + + return c +} diff --git a/github.com/Masterminds/sprig/doc.go b/github.com/Masterminds/sprig/doc.go new file mode 100644 index 0000000000..8f8f1d7370 --- /dev/null +++ b/github.com/Masterminds/sprig/doc.go @@ -0,0 +1,19 @@ +/* +Sprig: Template functions for Go. + +This package contains a number of utility functions for working with data +inside of Go `html/template` and `text/template` files. + +To add these functions, use the `template.Funcs()` method: + + t := templates.New("foo").Funcs(sprig.FuncMap()) + +Note that you should add the function map before you parse any template files. + + In several cases, Sprig reverses the order of arguments from the way they + appear in the standard library. This is to make it easier to pipe + arguments into functions. + +See http://masterminds.github.io/sprig/ for more detailed documentation on each of the available functions. +*/ +package sprig diff --git a/github.com/Masterminds/sprig/functions.go b/github.com/Masterminds/sprig/functions.go new file mode 100644 index 0000000000..7b5b0af86c --- /dev/null +++ b/github.com/Masterminds/sprig/functions.go @@ -0,0 +1,306 @@ +package sprig + +import ( + "errors" + "html/template" + "os" + "path" + "reflect" + "strconv" + "strings" + ttemplate "text/template" + "time" + + util "github.com/Masterminds/goutils" + "github.com/huandu/xstrings" +) + +// Produce the function map. +// +// Use this to pass the functions into the template engine: +// +// tpl := template.New("foo").Funcs(sprig.FuncMap())) +// +func FuncMap() template.FuncMap { + return HtmlFuncMap() +} + +// HermeticTxtFuncMap returns a 'text/template'.FuncMap with only repeatable functions. +func HermeticTxtFuncMap() ttemplate.FuncMap { + r := TxtFuncMap() + for _, name := range nonhermeticFunctions { + delete(r, name) + } + return r +} + +// HermeticHtmlFuncMap returns an 'html/template'.Funcmap with only repeatable functions. +func HermeticHtmlFuncMap() template.FuncMap { + r := HtmlFuncMap() + for _, name := range nonhermeticFunctions { + delete(r, name) + } + return r +} + +// TxtFuncMap returns a 'text/template'.FuncMap +func TxtFuncMap() ttemplate.FuncMap { + return ttemplate.FuncMap(GenericFuncMap()) +} + +// HtmlFuncMap returns an 'html/template'.Funcmap +func HtmlFuncMap() template.FuncMap { + return template.FuncMap(GenericFuncMap()) +} + +// GenericFuncMap returns a copy of the basic function map as a map[string]interface{}. +func GenericFuncMap() map[string]interface{} { + gfm := make(map[string]interface{}, len(genericMap)) + for k, v := range genericMap { + gfm[k] = v + } + return gfm +} + +// These functions are not guaranteed to evaluate to the same result for given input, because they +// refer to the environemnt or global state. +var nonhermeticFunctions = []string{ + // Date functions + "date", + "date_in_zone", + "date_modify", + "now", + "htmlDate", + "htmlDateInZone", + "dateInZone", + "dateModify", + + // Strings + "randAlphaNum", + "randAlpha", + "randAscii", + "randNumeric", + "uuidv4", + + // OS + "env", + "expandenv", + + // Network + "getHostByName", +} + +var genericMap = map[string]interface{}{ + "hello": func() string { return "Hello!" }, + + // Date functions + "date": date, + "date_in_zone": dateInZone, + "date_modify": dateModify, + "now": func() time.Time { return time.Now() }, + "htmlDate": htmlDate, + "htmlDateInZone": htmlDateInZone, + "dateInZone": dateInZone, + "dateModify": dateModify, + "ago": dateAgo, + "toDate": toDate, + "unixEpoch": unixEpoch, + + // Strings + "abbrev": abbrev, + "abbrevboth": abbrevboth, + "trunc": trunc, + "trim": strings.TrimSpace, + "upper": strings.ToUpper, + "lower": strings.ToLower, + "title": strings.Title, + "untitle": untitle, + "substr": substring, + // Switch order so that "foo" | repeat 5 + "repeat": func(count int, str string) string { return strings.Repeat(str, count) }, + // Deprecated: Use trimAll. + "trimall": func(a, b string) string { return strings.Trim(b, a) }, + // Switch order so that "$foo" | trimall "$" + "trimAll": func(a, b string) string { return strings.Trim(b, a) }, + "trimSuffix": func(a, b string) string { return strings.TrimSuffix(b, a) }, + "trimPrefix": func(a, b string) string { return strings.TrimPrefix(b, a) }, + "nospace": util.DeleteWhiteSpace, + "initials": initials, + "randAlphaNum": randAlphaNumeric, + "randAlpha": randAlpha, + "randAscii": randAscii, + "randNumeric": randNumeric, + "swapcase": util.SwapCase, + "shuffle": xstrings.Shuffle, + "snakecase": xstrings.ToSnakeCase, + "camelcase": xstrings.ToCamelCase, + "kebabcase": xstrings.ToKebabCase, + "wrap": func(l int, s string) string { return util.Wrap(s, l) }, + "wrapWith": func(l int, sep, str string) string { return util.WrapCustom(str, l, sep, true) }, + // Switch order so that "foobar" | contains "foo" + "contains": func(substr string, str string) bool { return strings.Contains(str, substr) }, + "hasPrefix": func(substr string, str string) bool { return strings.HasPrefix(str, substr) }, + "hasSuffix": func(substr string, str string) bool { return strings.HasSuffix(str, substr) }, + "quote": quote, + "squote": squote, + "cat": cat, + "indent": indent, + "nindent": nindent, + "replace": replace, + "plural": plural, + "sha1sum": sha1sum, + "sha256sum": sha256sum, + "adler32sum": adler32sum, + "toString": strval, + + // Wrap Atoi to stop errors. + "atoi": func(a string) int { i, _ := strconv.Atoi(a); return i }, + "int64": toInt64, + "int": toInt, + "float64": toFloat64, + "toDecimal": toDecimal, + + //"gt": func(a, b int) bool {return a > b}, + //"gte": func(a, b int) bool {return a >= b}, + //"lt": func(a, b int) bool {return a < b}, + //"lte": func(a, b int) bool {return a <= b}, + + // split "/" foo/bar returns map[int]string{0: foo, 1: bar} + "split": split, + "splitList": func(sep, orig string) []string { return strings.Split(orig, sep) }, + // splitn "/" foo/bar/fuu returns map[int]string{0: foo, 1: bar/fuu} + "splitn": splitn, + "toStrings": strslice, + + "until": until, + "untilStep": untilStep, + + // VERY basic arithmetic. + "add1": func(i interface{}) int64 { return toInt64(i) + 1 }, + "add": func(i ...interface{}) int64 { + var a int64 = 0 + for _, b := range i { + a += toInt64(b) + } + return a + }, + "sub": func(a, b interface{}) int64 { return toInt64(a) - toInt64(b) }, + "div": func(a, b interface{}) int64 { return toInt64(a) / toInt64(b) }, + "mod": func(a, b interface{}) int64 { return toInt64(a) % toInt64(b) }, + "mul": func(a interface{}, v ...interface{}) int64 { + val := toInt64(a) + for _, b := range v { + val = val * toInt64(b) + } + return val + }, + "biggest": max, + "max": max, + "min": min, + "ceil": ceil, + "floor": floor, + "round": round, + + // string slices. Note that we reverse the order b/c that's better + // for template processing. + "join": join, + "sortAlpha": sortAlpha, + + // Defaults + "default": dfault, + "empty": empty, + "coalesce": coalesce, + "compact": compact, + "deepCopy": deepCopy, + "toJson": toJson, + "toPrettyJson": toPrettyJson, + "ternary": ternary, + + // Reflection + "typeOf": typeOf, + "typeIs": typeIs, + "typeIsLike": typeIsLike, + "kindOf": kindOf, + "kindIs": kindIs, + "deepEqual": reflect.DeepEqual, + + // OS: + "env": func(s string) string { return os.Getenv(s) }, + "expandenv": func(s string) string { return os.ExpandEnv(s) }, + + // Network: + "getHostByName": getHostByName, + + // File Paths: + "base": path.Base, + "dir": path.Dir, + "clean": path.Clean, + "ext": path.Ext, + "isAbs": path.IsAbs, + + // Encoding: + "b64enc": base64encode, + "b64dec": base64decode, + "b32enc": base32encode, + "b32dec": base32decode, + + // Data Structures: + "tuple": list, // FIXME: with the addition of append/prepend these are no longer immutable. + "list": list, + "dict": dict, + "set": set, + "unset": unset, + "hasKey": hasKey, + "pluck": pluck, + "keys": keys, + "pick": pick, + "omit": omit, + "merge": merge, + "mergeOverwrite": mergeOverwrite, + "values": values, + + "append": push, "push": push, + "prepend": prepend, + "first": first, + "rest": rest, + "last": last, + "initial": initial, + "reverse": reverse, + "uniq": uniq, + "without": without, + "has": has, + "slice": slice, + "concat": concat, + + // Crypto: + "genPrivateKey": generatePrivateKey, + "derivePassword": derivePassword, + "buildCustomCert": buildCustomCertificate, + "genCA": generateCertificateAuthority, + "genSelfSignedCert": generateSelfSignedCertificate, + "genSignedCert": generateSignedCertificate, + "encryptAES": encryptAES, + "decryptAES": decryptAES, + + // UUIDs: + "uuidv4": uuidv4, + + // SemVer: + "semver": semver, + "semverCompare": semverCompare, + + // Flow Control: + "fail": func(msg string) (string, error) { return "", errors.New(msg) }, + + // Regex + "regexMatch": regexMatch, + "regexFindAll": regexFindAll, + "regexFind": regexFind, + "regexReplaceAll": regexReplaceAll, + "regexReplaceAllLiteral": regexReplaceAllLiteral, + "regexSplit": regexSplit, + + // URLs: + "urlParse": urlParse, + "urlJoin": urlJoin, +} diff --git a/github.com/Masterminds/sprig/glide.yaml b/github.com/Masterminds/sprig/glide.yaml new file mode 100644 index 0000000000..f317d2b2b1 --- /dev/null +++ b/github.com/Masterminds/sprig/glide.yaml @@ -0,0 +1,19 @@ +package: github.com/Masterminds/sprig +import: +- package: github.com/Masterminds/goutils + version: ^1.0.0 +- package: github.com/google/uuid + version: ^1.0.0 +- package: golang.org/x/crypto + subpackages: + - scrypt +- package: github.com/Masterminds/semver + version: ^v1.2.2 +- package: github.com/stretchr/testify + version: ^v1.2.2 +- package: github.com/imdario/mergo + version: ~0.3.7 +- package: github.com/huandu/xstrings + version: ^1.2 +- package: github.com/mitchellh/copystructure + version: ^1.0.0 diff --git a/github.com/Masterminds/sprig/list.go b/github.com/Masterminds/sprig/list.go new file mode 100644 index 0000000000..c0381bbb65 --- /dev/null +++ b/github.com/Masterminds/sprig/list.go @@ -0,0 +1,311 @@ +package sprig + +import ( + "fmt" + "reflect" + "sort" +) + +// Reflection is used in these functions so that slices and arrays of strings, +// ints, and other types not implementing []interface{} can be worked with. +// For example, this is useful if you need to work on the output of regexs. + +func list(v ...interface{}) []interface{} { + return v +} + +func push(list interface{}, v interface{}) []interface{} { + tp := reflect.TypeOf(list).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(list) + + l := l2.Len() + nl := make([]interface{}, l) + for i := 0; i < l; i++ { + nl[i] = l2.Index(i).Interface() + } + + return append(nl, v) + + default: + panic(fmt.Sprintf("Cannot push on type %s", tp)) + } +} + +func prepend(list interface{}, v interface{}) []interface{} { + //return append([]interface{}{v}, list...) + + tp := reflect.TypeOf(list).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(list) + + l := l2.Len() + nl := make([]interface{}, l) + for i := 0; i < l; i++ { + nl[i] = l2.Index(i).Interface() + } + + return append([]interface{}{v}, nl...) + + default: + panic(fmt.Sprintf("Cannot prepend on type %s", tp)) + } +} + +func last(list interface{}) interface{} { + tp := reflect.TypeOf(list).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(list) + + l := l2.Len() + if l == 0 { + return nil + } + + return l2.Index(l - 1).Interface() + default: + panic(fmt.Sprintf("Cannot find last on type %s", tp)) + } +} + +func first(list interface{}) interface{} { + tp := reflect.TypeOf(list).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(list) + + l := l2.Len() + if l == 0 { + return nil + } + + return l2.Index(0).Interface() + default: + panic(fmt.Sprintf("Cannot find first on type %s", tp)) + } +} + +func rest(list interface{}) []interface{} { + tp := reflect.TypeOf(list).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(list) + + l := l2.Len() + if l == 0 { + return nil + } + + nl := make([]interface{}, l-1) + for i := 1; i < l; i++ { + nl[i-1] = l2.Index(i).Interface() + } + + return nl + default: + panic(fmt.Sprintf("Cannot find rest on type %s", tp)) + } +} + +func initial(list interface{}) []interface{} { + tp := reflect.TypeOf(list).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(list) + + l := l2.Len() + if l == 0 { + return nil + } + + nl := make([]interface{}, l-1) + for i := 0; i < l-1; i++ { + nl[i] = l2.Index(i).Interface() + } + + return nl + default: + panic(fmt.Sprintf("Cannot find initial on type %s", tp)) + } +} + +func sortAlpha(list interface{}) []string { + k := reflect.Indirect(reflect.ValueOf(list)).Kind() + switch k { + case reflect.Slice, reflect.Array: + a := strslice(list) + s := sort.StringSlice(a) + s.Sort() + return s + } + return []string{strval(list)} +} + +func reverse(v interface{}) []interface{} { + tp := reflect.TypeOf(v).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(v) + + l := l2.Len() + // We do not sort in place because the incoming array should not be altered. + nl := make([]interface{}, l) + for i := 0; i < l; i++ { + nl[l-i-1] = l2.Index(i).Interface() + } + + return nl + default: + panic(fmt.Sprintf("Cannot find reverse on type %s", tp)) + } +} + +func compact(list interface{}) []interface{} { + tp := reflect.TypeOf(list).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(list) + + l := l2.Len() + nl := []interface{}{} + var item interface{} + for i := 0; i < l; i++ { + item = l2.Index(i).Interface() + if !empty(item) { + nl = append(nl, item) + } + } + + return nl + default: + panic(fmt.Sprintf("Cannot compact on type %s", tp)) + } +} + +func uniq(list interface{}) []interface{} { + tp := reflect.TypeOf(list).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(list) + + l := l2.Len() + dest := []interface{}{} + var item interface{} + for i := 0; i < l; i++ { + item = l2.Index(i).Interface() + if !inList(dest, item) { + dest = append(dest, item) + } + } + + return dest + default: + panic(fmt.Sprintf("Cannot find uniq on type %s", tp)) + } +} + +func inList(haystack []interface{}, needle interface{}) bool { + for _, h := range haystack { + if reflect.DeepEqual(needle, h) { + return true + } + } + return false +} + +func without(list interface{}, omit ...interface{}) []interface{} { + tp := reflect.TypeOf(list).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(list) + + l := l2.Len() + res := []interface{}{} + var item interface{} + for i := 0; i < l; i++ { + item = l2.Index(i).Interface() + if !inList(omit, item) { + res = append(res, item) + } + } + + return res + default: + panic(fmt.Sprintf("Cannot find without on type %s", tp)) + } +} + +func has(needle interface{}, haystack interface{}) bool { + if haystack == nil { + return false + } + tp := reflect.TypeOf(haystack).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(haystack) + var item interface{} + l := l2.Len() + for i := 0; i < l; i++ { + item = l2.Index(i).Interface() + if reflect.DeepEqual(needle, item) { + return true + } + } + + return false + default: + panic(fmt.Sprintf("Cannot find has on type %s", tp)) + } +} + +// $list := [1, 2, 3, 4, 5] +// slice $list -> list[0:5] = list[:] +// slice $list 0 3 -> list[0:3] = list[:3] +// slice $list 3 5 -> list[3:5] +// slice $list 3 -> list[3:5] = list[3:] +func slice(list interface{}, indices ...interface{}) interface{} { + tp := reflect.TypeOf(list).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(list) + + l := l2.Len() + if l == 0 { + return nil + } + + var start, end int + if len(indices) > 0 { + start = toInt(indices[0]) + } + if len(indices) < 2 { + end = l + } else { + end = toInt(indices[1]) + } + + return l2.Slice(start, end).Interface() + default: + panic(fmt.Sprintf("list should be type of slice or array but %s", tp)) + } +} + +func concat(lists ...interface{}) interface{} { + var res []interface{} + for _, list := range lists { + tp := reflect.TypeOf(list).Kind() + switch tp { + case reflect.Slice, reflect.Array: + l2 := reflect.ValueOf(list) + for i := 0; i < l2.Len(); i++ { + res = append(res, l2.Index(i).Interface()) + } + default: + panic(fmt.Sprintf("Cannot concat type %s as list", tp)) + } + } + return res +} diff --git a/github.com/Masterminds/sprig/network.go b/github.com/Masterminds/sprig/network.go new file mode 100644 index 0000000000..d786cc7363 --- /dev/null +++ b/github.com/Masterminds/sprig/network.go @@ -0,0 +1,12 @@ +package sprig + +import ( + "math/rand" + "net" +) + +func getHostByName(name string) string { + addrs, _ := net.LookupHost(name) + //TODO: add error handing when release v3 cames out + return addrs[rand.Intn(len(addrs))] +} diff --git a/github.com/Masterminds/sprig/numeric.go b/github.com/Masterminds/sprig/numeric.go new file mode 100644 index 0000000000..f4af4af2a7 --- /dev/null +++ b/github.com/Masterminds/sprig/numeric.go @@ -0,0 +1,169 @@ +package sprig + +import ( + "fmt" + "math" + "reflect" + "strconv" +) + +// toFloat64 converts 64-bit floats +func toFloat64(v interface{}) float64 { + if str, ok := v.(string); ok { + iv, err := strconv.ParseFloat(str, 64) + if err != nil { + return 0 + } + return iv + } + + val := reflect.Indirect(reflect.ValueOf(v)) + switch val.Kind() { + case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int: + return float64(val.Int()) + case reflect.Uint8, reflect.Uint16, reflect.Uint32: + return float64(val.Uint()) + case reflect.Uint, reflect.Uint64: + return float64(val.Uint()) + case reflect.Float32, reflect.Float64: + return val.Float() + case reflect.Bool: + if val.Bool() == true { + return 1 + } + return 0 + default: + return 0 + } +} + +func toInt(v interface{}) int { + //It's not optimal. Bud I don't want duplicate toInt64 code. + return int(toInt64(v)) +} + +// toInt64 converts integer types to 64-bit integers +func toInt64(v interface{}) int64 { + if str, ok := v.(string); ok { + iv, err := strconv.ParseInt(str, 10, 64) + if err != nil { + return 0 + } + return iv + } + + val := reflect.Indirect(reflect.ValueOf(v)) + switch val.Kind() { + case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int: + return val.Int() + case reflect.Uint8, reflect.Uint16, reflect.Uint32: + return int64(val.Uint()) + case reflect.Uint, reflect.Uint64: + tv := val.Uint() + if tv <= math.MaxInt64 { + return int64(tv) + } + // TODO: What is the sensible thing to do here? + return math.MaxInt64 + case reflect.Float32, reflect.Float64: + return int64(val.Float()) + case reflect.Bool: + if val.Bool() == true { + return 1 + } + return 0 + default: + return 0 + } +} + +func max(a interface{}, i ...interface{}) int64 { + aa := toInt64(a) + for _, b := range i { + bb := toInt64(b) + if bb > aa { + aa = bb + } + } + return aa +} + +func min(a interface{}, i ...interface{}) int64 { + aa := toInt64(a) + for _, b := range i { + bb := toInt64(b) + if bb < aa { + aa = bb + } + } + return aa +} + +func until(count int) []int { + step := 1 + if count < 0 { + step = -1 + } + return untilStep(0, count, step) +} + +func untilStep(start, stop, step int) []int { + v := []int{} + + if stop < start { + if step >= 0 { + return v + } + for i := start; i > stop; i += step { + v = append(v, i) + } + return v + } + + if step <= 0 { + return v + } + for i := start; i < stop; i += step { + v = append(v, i) + } + return v +} + +func floor(a interface{}) float64 { + aa := toFloat64(a) + return math.Floor(aa) +} + +func ceil(a interface{}) float64 { + aa := toFloat64(a) + return math.Ceil(aa) +} + +func round(a interface{}, p int, r_opt ...float64) float64 { + roundOn := .5 + if len(r_opt) > 0 { + roundOn = r_opt[0] + } + val := toFloat64(a) + places := toFloat64(p) + + var round float64 + pow := math.Pow(10, places) + digit := pow * val + _, div := math.Modf(digit) + if div >= roundOn { + round = math.Ceil(digit) + } else { + round = math.Floor(digit) + } + return round / pow +} + +// converts unix octal to decimal +func toDecimal(v interface{}) int64 { + result, err := strconv.ParseInt(fmt.Sprint(v), 8, 64) + if err != nil { + return 0 + } + return result +} diff --git a/github.com/Masterminds/sprig/reflect.go b/github.com/Masterminds/sprig/reflect.go new file mode 100644 index 0000000000..8a65c132f0 --- /dev/null +++ b/github.com/Masterminds/sprig/reflect.go @@ -0,0 +1,28 @@ +package sprig + +import ( + "fmt" + "reflect" +) + +// typeIs returns true if the src is the type named in target. +func typeIs(target string, src interface{}) bool { + return target == typeOf(src) +} + +func typeIsLike(target string, src interface{}) bool { + t := typeOf(src) + return target == t || "*"+target == t +} + +func typeOf(src interface{}) string { + return fmt.Sprintf("%T", src) +} + +func kindIs(target string, src interface{}) bool { + return target == kindOf(src) +} + +func kindOf(src interface{}) string { + return reflect.ValueOf(src).Kind().String() +} diff --git a/github.com/Masterminds/sprig/regex.go b/github.com/Masterminds/sprig/regex.go new file mode 100644 index 0000000000..2016f66336 --- /dev/null +++ b/github.com/Masterminds/sprig/regex.go @@ -0,0 +1,35 @@ +package sprig + +import ( + "regexp" +) + +func regexMatch(regex string, s string) bool { + match, _ := regexp.MatchString(regex, s) + return match +} + +func regexFindAll(regex string, s string, n int) []string { + r := regexp.MustCompile(regex) + return r.FindAllString(s, n) +} + +func regexFind(regex string, s string) string { + r := regexp.MustCompile(regex) + return r.FindString(s) +} + +func regexReplaceAll(regex string, s string, repl string) string { + r := regexp.MustCompile(regex) + return r.ReplaceAllString(s, repl) +} + +func regexReplaceAllLiteral(regex string, s string, repl string) string { + r := regexp.MustCompile(regex) + return r.ReplaceAllLiteralString(s, repl) +} + +func regexSplit(regex string, s string, n int) []string { + r := regexp.MustCompile(regex) + return r.Split(s, n) +} diff --git a/github.com/Masterminds/sprig/semver.go b/github.com/Masterminds/sprig/semver.go new file mode 100644 index 0000000000..c2bf8a1fdf --- /dev/null +++ b/github.com/Masterminds/sprig/semver.go @@ -0,0 +1,23 @@ +package sprig + +import ( + sv2 "github.com/Masterminds/semver" +) + +func semverCompare(constraint, version string) (bool, error) { + c, err := sv2.NewConstraint(constraint) + if err != nil { + return false, err + } + + v, err := sv2.NewVersion(version) + if err != nil { + return false, err + } + + return c.Check(v), nil +} + +func semver(version string) (*sv2.Version, error) { + return sv2.NewVersion(version) +} diff --git a/github.com/Masterminds/sprig/strings.go b/github.com/Masterminds/sprig/strings.go new file mode 100644 index 0000000000..943fa3e8ad --- /dev/null +++ b/github.com/Masterminds/sprig/strings.go @@ -0,0 +1,233 @@ +package sprig + +import ( + "encoding/base32" + "encoding/base64" + "fmt" + "reflect" + "strconv" + "strings" + + util "github.com/Masterminds/goutils" +) + +func base64encode(v string) string { + return base64.StdEncoding.EncodeToString([]byte(v)) +} + +func base64decode(v string) string { + data, err := base64.StdEncoding.DecodeString(v) + if err != nil { + return err.Error() + } + return string(data) +} + +func base32encode(v string) string { + return base32.StdEncoding.EncodeToString([]byte(v)) +} + +func base32decode(v string) string { + data, err := base32.StdEncoding.DecodeString(v) + if err != nil { + return err.Error() + } + return string(data) +} + +func abbrev(width int, s string) string { + if width < 4 { + return s + } + r, _ := util.Abbreviate(s, width) + return r +} + +func abbrevboth(left, right int, s string) string { + if right < 4 || left > 0 && right < 7 { + return s + } + r, _ := util.AbbreviateFull(s, left, right) + return r +} +func initials(s string) string { + // Wrap this just to eliminate the var args, which templates don't do well. + return util.Initials(s) +} + +func randAlphaNumeric(count int) string { + // It is not possible, it appears, to actually generate an error here. + r, _ := util.CryptoRandomAlphaNumeric(count) + return r +} + +func randAlpha(count int) string { + r, _ := util.CryptoRandomAlphabetic(count) + return r +} + +func randAscii(count int) string { + r, _ := util.CryptoRandomAscii(count) + return r +} + +func randNumeric(count int) string { + r, _ := util.CryptoRandomNumeric(count) + return r +} + +func untitle(str string) string { + return util.Uncapitalize(str) +} + +func quote(str ...interface{}) string { + out := make([]string, 0, len(str)) + for _, s := range str { + if s != nil { + out = append(out, fmt.Sprintf("%q", strval(s))) + } + } + return strings.Join(out, " ") +} + +func squote(str ...interface{}) string { + out := make([]string, 0, len(str)) + for _, s := range str { + if s != nil { + out = append(out, fmt.Sprintf("'%v'", s)) + } + } + return strings.Join(out, " ") +} + +func cat(v ...interface{}) string { + v = removeNilElements(v) + r := strings.TrimSpace(strings.Repeat("%v ", len(v))) + return fmt.Sprintf(r, v...) +} + +func indent(spaces int, v string) string { + pad := strings.Repeat(" ", spaces) + return pad + strings.Replace(v, "\n", "\n"+pad, -1) +} + +func nindent(spaces int, v string) string { + return "\n" + indent(spaces, v) +} + +func replace(old, new, src string) string { + return strings.Replace(src, old, new, -1) +} + +func plural(one, many string, count int) string { + if count == 1 { + return one + } + return many +} + +func strslice(v interface{}) []string { + switch v := v.(type) { + case []string: + return v + case []interface{}: + b := make([]string, 0, len(v)) + for _, s := range v { + if s != nil { + b = append(b, strval(s)) + } + } + return b + default: + val := reflect.ValueOf(v) + switch val.Kind() { + case reflect.Array, reflect.Slice: + l := val.Len() + b := make([]string, 0, l) + for i := 0; i < l; i++ { + value := val.Index(i).Interface() + if value != nil { + b = append(b, strval(value)) + } + } + return b + default: + if v == nil { + return []string{} + } else { + return []string{strval(v)} + } + } + } +} + +func removeNilElements(v []interface{}) []interface{} { + newSlice := make([]interface{}, 0, len(v)) + for _, i := range v { + if i != nil { + newSlice = append(newSlice, i) + } + } + return newSlice +} + +func strval(v interface{}) string { + switch v := v.(type) { + case string: + return v + case []byte: + return string(v) + case error: + return v.Error() + case fmt.Stringer: + return v.String() + default: + return fmt.Sprintf("%v", v) + } +} + +func trunc(c int, s string) string { + if len(s) <= c { + return s + } + return s[0:c] +} + +func join(sep string, v interface{}) string { + return strings.Join(strslice(v), sep) +} + +func split(sep, orig string) map[string]string { + parts := strings.Split(orig, sep) + res := make(map[string]string, len(parts)) + for i, v := range parts { + res["_"+strconv.Itoa(i)] = v + } + return res +} + +func splitn(sep string, n int, orig string) map[string]string { + parts := strings.SplitN(orig, sep, n) + res := make(map[string]string, len(parts)) + for i, v := range parts { + res["_"+strconv.Itoa(i)] = v + } + return res +} + +// substring creates a substring of the given string. +// +// If start is < 0, this calls string[:end]. +// +// If start is >= 0 and end < 0 or end bigger than s length, this calls string[start:] +// +// Otherwise, this calls string[start, end]. +func substring(start, end int, s string) string { + if start < 0 { + return s[:end] + } + if end < 0 || end > len(s) { + return s[start:] + } + return s[start:end] +} diff --git a/github.com/Masterminds/sprig/url.go b/github.com/Masterminds/sprig/url.go new file mode 100644 index 0000000000..5f22d801f9 --- /dev/null +++ b/github.com/Masterminds/sprig/url.go @@ -0,0 +1,66 @@ +package sprig + +import ( + "fmt" + "net/url" + "reflect" +) + +func dictGetOrEmpty(dict map[string]interface{}, key string) string { + value, ok := dict[key]; if !ok { + return "" + } + tp := reflect.TypeOf(value).Kind() + if tp != reflect.String { + panic(fmt.Sprintf("unable to parse %s key, must be of type string, but %s found", key, tp.String())) + } + return reflect.ValueOf(value).String() +} + +// parses given URL to return dict object +func urlParse(v string) map[string]interface{} { + dict := map[string]interface{}{} + parsedUrl, err := url.Parse(v) + if err != nil { + panic(fmt.Sprintf("unable to parse url: %s", err)) + } + dict["scheme"] = parsedUrl.Scheme + dict["host"] = parsedUrl.Host + dict["hostname"] = parsedUrl.Hostname() + dict["path"] = parsedUrl.Path + dict["query"] = parsedUrl.RawQuery + dict["opaque"] = parsedUrl.Opaque + dict["fragment"] = parsedUrl.Fragment + if parsedUrl.User != nil { + dict["userinfo"] = parsedUrl.User.String() + } else { + dict["userinfo"] = "" + } + + return dict +} + +// join given dict to URL string +func urlJoin(d map[string]interface{}) string { + resUrl := url.URL{ + Scheme: dictGetOrEmpty(d, "scheme"), + Host: dictGetOrEmpty(d, "host"), + Path: dictGetOrEmpty(d, "path"), + RawQuery: dictGetOrEmpty(d, "query"), + Opaque: dictGetOrEmpty(d, "opaque"), + Fragment: dictGetOrEmpty(d, "fragment"), + + } + userinfo := dictGetOrEmpty(d, "userinfo") + var user *url.Userinfo = nil + if userinfo != "" { + tempUrl, err := url.Parse(fmt.Sprintf("proto://%s@host", userinfo)) + if err != nil { + panic(fmt.Sprintf("unable to parse userinfo in dict: %s", err)) + } + user = tempUrl.User + } + + resUrl.User = user + return resUrl.String() +} diff --git a/github.com/envoyproxy/protoc-gen-validate/LICENSE b/github.com/envoyproxy/protoc-gen-validate/LICENSE new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/github.com/envoyproxy/protoc-gen-validate/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/github.com/envoyproxy/protoc-gen-validate/NOTICE b/github.com/envoyproxy/protoc-gen-validate/NOTICE new file mode 100644 index 0000000000..60884a0590 --- /dev/null +++ b/github.com/envoyproxy/protoc-gen-validate/NOTICE @@ -0,0 +1,4 @@ +protoc-gen-validate +Copyright 2019 Envoy Project Authors + +Licensed under Apache License 2.0. See LICENSE for terms. diff --git a/github.com/envoyproxy/protoc-gen-validate/validate/BUILD b/github.com/envoyproxy/protoc-gen-validate/validate/BUILD new file mode 100644 index 0000000000..8a3d063d62 --- /dev/null +++ b/github.com/envoyproxy/protoc-gen-validate/validate/BUILD @@ -0,0 +1,49 @@ +# gazelle:exclude validate.pb.go + +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") +load("@com_google_protobuf//:protobuf.bzl", "py_proto_library") + +proto_library( + name = "validate_proto", + srcs = ["validate.proto"], + visibility = ["//visibility:public"], + deps = [ + "@com_google_protobuf//:descriptor_proto", + "@com_google_protobuf//:duration_proto", + "@com_google_protobuf//:timestamp_proto", + ], +) + +cc_proto_library( + name = "validate_cc", + deps = [":validate_proto"], + visibility = ["//visibility:public"], +) + +py_proto_library( + name = "validate_py", + srcs = ["validate.proto"], + protoc = "@com_google_protobuf//:protoc", + default_runtime = "@com_google_protobuf//:protobuf_python", + deps = ["@com_google_protobuf//:protobuf_python"], + visibility = ["//visibility:public"], +) + +go_proto_library( + name = "go_default_library", + importpath = "github.com/envoyproxy/protoc-gen-validate/validate", + proto = ":validate_proto", + visibility = ["//visibility:public"], +) + +cc_library( + name = "cc_validate", + hdrs = ["validate.h"], + visibility = ["//visibility:public"], +) + +java_proto_library( + name = "validate_java", + deps = [":validate_proto"], + visibility=["//visibility:public"], +) diff --git a/github.com/envoyproxy/protoc-gen-validate/validate/validate.h b/github.com/envoyproxy/protoc-gen-validate/validate/validate.h new file mode 100644 index 0000000000..b309c2ef3b --- /dev/null +++ b/github.com/envoyproxy/protoc-gen-validate/validate/validate.h @@ -0,0 +1,124 @@ +#ifndef _VALIDATE_H +#define _VALIDATE_H + +#include +#include +#include +#include +#include +#include +#include + +#if !defined(_WIN32) +#include +#else +#include +#include +#endif + +namespace pgv { +using std::string; + +class UnimplementedException : public std::runtime_error { +public: + UnimplementedException() : std::runtime_error("not yet implemented") {} + // Thrown by C++ validation code that is not yet implemented. +}; + +using ValidationMsg = std::string; + +class BaseValidator { +protected: + static std::unordered_map& validators() { + static auto* validator_map = new std::unordered_map(); + return *validator_map; + } +}; + +template +class Validator : public BaseValidator { +public: + Validator(std::function check) : check_(check) + { + validators()[std::type_index(typeid(T))] = this; + } + + static bool CheckMessage(const T& m, ValidationMsg* err) + { + auto val = static_cast*>(validators()[std::type_index(typeid(T))]); + if (val) { + return val->check_(m, err); + } + return true; + } + +private: + std::function check_; +}; + +static inline std::string String(const ValidationMsg& msg) +{ + return std::string(msg); +} + +static inline bool IsPrefix(const string& maybe_prefix, const string& search_in) +{ + return search_in.compare(0, maybe_prefix.size(), maybe_prefix) == 0; +} + +static inline bool IsSuffix(const string& maybe_suffix, const string& search_in) +{ + return maybe_suffix.size() <= search_in.size() && search_in.compare(search_in.size() - maybe_suffix.size(), maybe_suffix.size(), maybe_suffix) == 0; +} + +static inline bool Contains(const string& search_in, const string& to_find) +{ + return search_in.find(to_find) != string::npos; +} + +static inline bool IsIpv4(const string& to_validate) { + struct sockaddr_in sa; + return !(inet_pton(AF_INET, to_validate.c_str(), &sa.sin_addr) < 1); +} + +static inline bool IsIpv6(const string& to_validate) { + struct sockaddr_in6 sa_six; + return !(inet_pton(AF_INET6, to_validate.c_str(), &sa_six.sin6_addr) < 1); +} + +static inline bool IsIp(const string& to_validate) { + return IsIpv4(to_validate) || IsIpv6(to_validate); +} + +static inline bool IsHostname(const string& to_validate) { + if (to_validate.length() > 253) { + return false; + } + + const std::regex dot_regex{"\\."}; + const auto iter_end = std::sregex_token_iterator(); + auto iter = std::sregex_token_iterator(to_validate.begin(), to_validate.end(), dot_regex, -1); + for (; iter != iter_end; ++iter) { + const std::string &part = *iter; + if (part.empty() || part.length() > 63) { + return false; + } + if (part.at(0) == '-') { + return false; + } + if (part.at(part.length() - 1) == '-') { + return false; + } + for (const auto &character : part) { + if ((character < 'A' || character > 'Z') && (character < 'a' || character > 'z') && (character < '0' || character > '9') && character != '-') { + return false; + } + } + } + + return true; +} + +} // namespace pgv + +#endif // _VALIDATE_H diff --git a/github.com/envoyproxy/protoc-gen-validate/validate/validate.pb.go b/github.com/envoyproxy/protoc-gen-validate/validate/validate.pb.go new file mode 100644 index 0000000000..255dc5ec0e --- /dev/null +++ b/github.com/envoyproxy/protoc-gen-validate/validate/validate.pb.go @@ -0,0 +1,3575 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: validate/validate.proto + +package validate // import "github.com/envoyproxy/protoc-gen-validate/validate" + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" +import descriptor "github.com/golang/protobuf/protoc-gen-go/descriptor" +import duration "github.com/golang/protobuf/ptypes/duration" +import timestamp "github.com/golang/protobuf/ptypes/timestamp" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +// FieldRules encapsulates the rules for each type of field. Depending on the +// field, the correct set should be used to ensure proper validations. +type FieldRules struct { + // Types that are valid to be assigned to Type: + // *FieldRules_Float + // *FieldRules_Double + // *FieldRules_Int32 + // *FieldRules_Int64 + // *FieldRules_Uint32 + // *FieldRules_Uint64 + // *FieldRules_Sint32 + // *FieldRules_Sint64 + // *FieldRules_Fixed32 + // *FieldRules_Fixed64 + // *FieldRules_Sfixed32 + // *FieldRules_Sfixed64 + // *FieldRules_Bool + // *FieldRules_String_ + // *FieldRules_Bytes + // *FieldRules_Enum + // *FieldRules_Message + // *FieldRules_Repeated + // *FieldRules_Map + // *FieldRules_Any + // *FieldRules_Duration + // *FieldRules_Timestamp + Type isFieldRules_Type `protobuf_oneof:"type"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *FieldRules) Reset() { *m = FieldRules{} } +func (m *FieldRules) String() string { return proto.CompactTextString(m) } +func (*FieldRules) ProtoMessage() {} +func (*FieldRules) Descriptor() ([]byte, []int) { + return fileDescriptor_validate_c8f5e113dd6422a8, []int{0} +} +func (m *FieldRules) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_FieldRules.Unmarshal(m, b) +} +func (m *FieldRules) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_FieldRules.Marshal(b, m, deterministic) +} +func (dst *FieldRules) XXX_Merge(src proto.Message) { + xxx_messageInfo_FieldRules.Merge(dst, src) +} +func (m *FieldRules) XXX_Size() int { + return xxx_messageInfo_FieldRules.Size(m) +} +func (m *FieldRules) XXX_DiscardUnknown() { + xxx_messageInfo_FieldRules.DiscardUnknown(m) +} + +var xxx_messageInfo_FieldRules proto.InternalMessageInfo + +type isFieldRules_Type interface { + isFieldRules_Type() +} + +type FieldRules_Float struct { + Float *FloatRules `protobuf:"bytes,1,opt,name=float,oneof"` +} + +type FieldRules_Double struct { + Double *DoubleRules `protobuf:"bytes,2,opt,name=double,oneof"` +} + +type FieldRules_Int32 struct { + Int32 *Int32Rules `protobuf:"bytes,3,opt,name=int32,oneof"` +} + +type FieldRules_Int64 struct { + Int64 *Int64Rules `protobuf:"bytes,4,opt,name=int64,oneof"` +} + +type FieldRules_Uint32 struct { + Uint32 *UInt32Rules `protobuf:"bytes,5,opt,name=uint32,oneof"` +} + +type FieldRules_Uint64 struct { + Uint64 *UInt64Rules `protobuf:"bytes,6,opt,name=uint64,oneof"` +} + +type FieldRules_Sint32 struct { + Sint32 *SInt32Rules `protobuf:"bytes,7,opt,name=sint32,oneof"` +} + +type FieldRules_Sint64 struct { + Sint64 *SInt64Rules `protobuf:"bytes,8,opt,name=sint64,oneof"` +} + +type FieldRules_Fixed32 struct { + Fixed32 *Fixed32Rules `protobuf:"bytes,9,opt,name=fixed32,oneof"` +} + +type FieldRules_Fixed64 struct { + Fixed64 *Fixed64Rules `protobuf:"bytes,10,opt,name=fixed64,oneof"` +} + +type FieldRules_Sfixed32 struct { + Sfixed32 *SFixed32Rules `protobuf:"bytes,11,opt,name=sfixed32,oneof"` +} + +type FieldRules_Sfixed64 struct { + Sfixed64 *SFixed64Rules `protobuf:"bytes,12,opt,name=sfixed64,oneof"` +} + +type FieldRules_Bool struct { + Bool *BoolRules `protobuf:"bytes,13,opt,name=bool,oneof"` +} + +type FieldRules_String_ struct { + String_ *StringRules `protobuf:"bytes,14,opt,name=string,oneof"` +} + +type FieldRules_Bytes struct { + Bytes *BytesRules `protobuf:"bytes,15,opt,name=bytes,oneof"` +} + +type FieldRules_Enum struct { + Enum *EnumRules `protobuf:"bytes,16,opt,name=enum,oneof"` +} + +type FieldRules_Message struct { + Message *MessageRules `protobuf:"bytes,17,opt,name=message,oneof"` +} + +type FieldRules_Repeated struct { + Repeated *RepeatedRules `protobuf:"bytes,18,opt,name=repeated,oneof"` +} + +type FieldRules_Map struct { + Map *MapRules `protobuf:"bytes,19,opt,name=map,oneof"` +} + +type FieldRules_Any struct { + Any *AnyRules `protobuf:"bytes,20,opt,name=any,oneof"` +} + +type FieldRules_Duration struct { + Duration *DurationRules `protobuf:"bytes,21,opt,name=duration,oneof"` +} + +type FieldRules_Timestamp struct { + Timestamp *TimestampRules `protobuf:"bytes,22,opt,name=timestamp,oneof"` +} + +func (*FieldRules_Float) isFieldRules_Type() {} + +func (*FieldRules_Double) isFieldRules_Type() {} + +func (*FieldRules_Int32) isFieldRules_Type() {} + +func (*FieldRules_Int64) isFieldRules_Type() {} + +func (*FieldRules_Uint32) isFieldRules_Type() {} + +func (*FieldRules_Uint64) isFieldRules_Type() {} + +func (*FieldRules_Sint32) isFieldRules_Type() {} + +func (*FieldRules_Sint64) isFieldRules_Type() {} + +func (*FieldRules_Fixed32) isFieldRules_Type() {} + +func (*FieldRules_Fixed64) isFieldRules_Type() {} + +func (*FieldRules_Sfixed32) isFieldRules_Type() {} + +func (*FieldRules_Sfixed64) isFieldRules_Type() {} + +func (*FieldRules_Bool) isFieldRules_Type() {} + +func (*FieldRules_String_) isFieldRules_Type() {} + +func (*FieldRules_Bytes) isFieldRules_Type() {} + +func (*FieldRules_Enum) isFieldRules_Type() {} + +func (*FieldRules_Message) isFieldRules_Type() {} + +func (*FieldRules_Repeated) isFieldRules_Type() {} + +func (*FieldRules_Map) isFieldRules_Type() {} + +func (*FieldRules_Any) isFieldRules_Type() {} + +func (*FieldRules_Duration) isFieldRules_Type() {} + +func (*FieldRules_Timestamp) isFieldRules_Type() {} + +func (m *FieldRules) GetType() isFieldRules_Type { + if m != nil { + return m.Type + } + return nil +} + +func (m *FieldRules) GetFloat() *FloatRules { + if x, ok := m.GetType().(*FieldRules_Float); ok { + return x.Float + } + return nil +} + +func (m *FieldRules) GetDouble() *DoubleRules { + if x, ok := m.GetType().(*FieldRules_Double); ok { + return x.Double + } + return nil +} + +func (m *FieldRules) GetInt32() *Int32Rules { + if x, ok := m.GetType().(*FieldRules_Int32); ok { + return x.Int32 + } + return nil +} + +func (m *FieldRules) GetInt64() *Int64Rules { + if x, ok := m.GetType().(*FieldRules_Int64); ok { + return x.Int64 + } + return nil +} + +func (m *FieldRules) GetUint32() *UInt32Rules { + if x, ok := m.GetType().(*FieldRules_Uint32); ok { + return x.Uint32 + } + return nil +} + +func (m *FieldRules) GetUint64() *UInt64Rules { + if x, ok := m.GetType().(*FieldRules_Uint64); ok { + return x.Uint64 + } + return nil +} + +func (m *FieldRules) GetSint32() *SInt32Rules { + if x, ok := m.GetType().(*FieldRules_Sint32); ok { + return x.Sint32 + } + return nil +} + +func (m *FieldRules) GetSint64() *SInt64Rules { + if x, ok := m.GetType().(*FieldRules_Sint64); ok { + return x.Sint64 + } + return nil +} + +func (m *FieldRules) GetFixed32() *Fixed32Rules { + if x, ok := m.GetType().(*FieldRules_Fixed32); ok { + return x.Fixed32 + } + return nil +} + +func (m *FieldRules) GetFixed64() *Fixed64Rules { + if x, ok := m.GetType().(*FieldRules_Fixed64); ok { + return x.Fixed64 + } + return nil +} + +func (m *FieldRules) GetSfixed32() *SFixed32Rules { + if x, ok := m.GetType().(*FieldRules_Sfixed32); ok { + return x.Sfixed32 + } + return nil +} + +func (m *FieldRules) GetSfixed64() *SFixed64Rules { + if x, ok := m.GetType().(*FieldRules_Sfixed64); ok { + return x.Sfixed64 + } + return nil +} + +func (m *FieldRules) GetBool() *BoolRules { + if x, ok := m.GetType().(*FieldRules_Bool); ok { + return x.Bool + } + return nil +} + +func (m *FieldRules) GetString_() *StringRules { + if x, ok := m.GetType().(*FieldRules_String_); ok { + return x.String_ + } + return nil +} + +func (m *FieldRules) GetBytes() *BytesRules { + if x, ok := m.GetType().(*FieldRules_Bytes); ok { + return x.Bytes + } + return nil +} + +func (m *FieldRules) GetEnum() *EnumRules { + if x, ok := m.GetType().(*FieldRules_Enum); ok { + return x.Enum + } + return nil +} + +func (m *FieldRules) GetMessage() *MessageRules { + if x, ok := m.GetType().(*FieldRules_Message); ok { + return x.Message + } + return nil +} + +func (m *FieldRules) GetRepeated() *RepeatedRules { + if x, ok := m.GetType().(*FieldRules_Repeated); ok { + return x.Repeated + } + return nil +} + +func (m *FieldRules) GetMap() *MapRules { + if x, ok := m.GetType().(*FieldRules_Map); ok { + return x.Map + } + return nil +} + +func (m *FieldRules) GetAny() *AnyRules { + if x, ok := m.GetType().(*FieldRules_Any); ok { + return x.Any + } + return nil +} + +func (m *FieldRules) GetDuration() *DurationRules { + if x, ok := m.GetType().(*FieldRules_Duration); ok { + return x.Duration + } + return nil +} + +func (m *FieldRules) GetTimestamp() *TimestampRules { + if x, ok := m.GetType().(*FieldRules_Timestamp); ok { + return x.Timestamp + } + return nil +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*FieldRules) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { + return _FieldRules_OneofMarshaler, _FieldRules_OneofUnmarshaler, _FieldRules_OneofSizer, []interface{}{ + (*FieldRules_Float)(nil), + (*FieldRules_Double)(nil), + (*FieldRules_Int32)(nil), + (*FieldRules_Int64)(nil), + (*FieldRules_Uint32)(nil), + (*FieldRules_Uint64)(nil), + (*FieldRules_Sint32)(nil), + (*FieldRules_Sint64)(nil), + (*FieldRules_Fixed32)(nil), + (*FieldRules_Fixed64)(nil), + (*FieldRules_Sfixed32)(nil), + (*FieldRules_Sfixed64)(nil), + (*FieldRules_Bool)(nil), + (*FieldRules_String_)(nil), + (*FieldRules_Bytes)(nil), + (*FieldRules_Enum)(nil), + (*FieldRules_Message)(nil), + (*FieldRules_Repeated)(nil), + (*FieldRules_Map)(nil), + (*FieldRules_Any)(nil), + (*FieldRules_Duration)(nil), + (*FieldRules_Timestamp)(nil), + } +} + +func _FieldRules_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*FieldRules) + // type + switch x := m.Type.(type) { + case *FieldRules_Float: + b.EncodeVarint(1<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Float); err != nil { + return err + } + case *FieldRules_Double: + b.EncodeVarint(2<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Double); err != nil { + return err + } + case *FieldRules_Int32: + b.EncodeVarint(3<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Int32); err != nil { + return err + } + case *FieldRules_Int64: + b.EncodeVarint(4<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Int64); err != nil { + return err + } + case *FieldRules_Uint32: + b.EncodeVarint(5<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Uint32); err != nil { + return err + } + case *FieldRules_Uint64: + b.EncodeVarint(6<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Uint64); err != nil { + return err + } + case *FieldRules_Sint32: + b.EncodeVarint(7<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Sint32); err != nil { + return err + } + case *FieldRules_Sint64: + b.EncodeVarint(8<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Sint64); err != nil { + return err + } + case *FieldRules_Fixed32: + b.EncodeVarint(9<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Fixed32); err != nil { + return err + } + case *FieldRules_Fixed64: + b.EncodeVarint(10<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Fixed64); err != nil { + return err + } + case *FieldRules_Sfixed32: + b.EncodeVarint(11<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Sfixed32); err != nil { + return err + } + case *FieldRules_Sfixed64: + b.EncodeVarint(12<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Sfixed64); err != nil { + return err + } + case *FieldRules_Bool: + b.EncodeVarint(13<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Bool); err != nil { + return err + } + case *FieldRules_String_: + b.EncodeVarint(14<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.String_); err != nil { + return err + } + case *FieldRules_Bytes: + b.EncodeVarint(15<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Bytes); err != nil { + return err + } + case *FieldRules_Enum: + b.EncodeVarint(16<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Enum); err != nil { + return err + } + case *FieldRules_Message: + b.EncodeVarint(17<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Message); err != nil { + return err + } + case *FieldRules_Repeated: + b.EncodeVarint(18<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Repeated); err != nil { + return err + } + case *FieldRules_Map: + b.EncodeVarint(19<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Map); err != nil { + return err + } + case *FieldRules_Any: + b.EncodeVarint(20<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Any); err != nil { + return err + } + case *FieldRules_Duration: + b.EncodeVarint(21<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Duration); err != nil { + return err + } + case *FieldRules_Timestamp: + b.EncodeVarint(22<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Timestamp); err != nil { + return err + } + case nil: + default: + return fmt.Errorf("FieldRules.Type has unexpected type %T", x) + } + return nil +} + +func _FieldRules_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*FieldRules) + switch tag { + case 1: // type.float + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(FloatRules) + err := b.DecodeMessage(msg) + m.Type = &FieldRules_Float{msg} + return true, err + case 2: // type.double + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(DoubleRules) + err := b.DecodeMessage(msg) + m.Type = &FieldRules_Double{msg} + return true, err + case 3: // type.int32 + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(Int32Rules) + err := b.DecodeMessage(msg) + m.Type = &FieldRules_Int32{msg} + return true, err + case 4: // type.int64 + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(Int64Rules) + err := b.DecodeMessage(msg) + m.Type = &FieldRules_Int64{msg} + return true, err + case 5: // type.uint32 + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(UInt32Rules) + err := b.DecodeMessage(msg) + m.Type = &FieldRules_Uint32{msg} + return true, err + case 6: // type.uint64 + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(UInt64Rules) + err := b.DecodeMessage(msg) + m.Type = &FieldRules_Uint64{msg} + return true, err + case 7: // type.sint32 + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(SInt32Rules) + err := b.DecodeMessage(msg) + m.Type = &FieldRules_Sint32{msg} + return true, err + case 8: // type.sint64 + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(SInt64Rules) + err := b.DecodeMessage(msg) + m.Type = &FieldRules_Sint64{msg} + return true, err + case 9: // type.fixed32 + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(Fixed32Rules) + err := b.DecodeMessage(msg) + m.Type = &FieldRules_Fixed32{msg} + return true, err + case 10: // type.fixed64 + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(Fixed64Rules) + err := b.DecodeMessage(msg) + m.Type = &FieldRules_Fixed64{msg} + return true, err + case 11: // type.sfixed32 + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(SFixed32Rules) + err := b.DecodeMessage(msg) + m.Type = &FieldRules_Sfixed32{msg} + return true, err + case 12: // type.sfixed64 + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(SFixed64Rules) + err := b.DecodeMessage(msg) + m.Type = &FieldRules_Sfixed64{msg} + return true, err + case 13: // type.bool + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(BoolRules) + err := b.DecodeMessage(msg) + m.Type = &FieldRules_Bool{msg} + return true, err + case 14: // type.string + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(StringRules) + err := b.DecodeMessage(msg) + m.Type = &FieldRules_String_{msg} + return true, err + case 15: // type.bytes + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(BytesRules) + err := b.DecodeMessage(msg) + m.Type = &FieldRules_Bytes{msg} + return true, err + case 16: // type.enum + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(EnumRules) + err := b.DecodeMessage(msg) + m.Type = &FieldRules_Enum{msg} + return true, err + case 17: // type.message + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(MessageRules) + err := b.DecodeMessage(msg) + m.Type = &FieldRules_Message{msg} + return true, err + case 18: // type.repeated + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(RepeatedRules) + err := b.DecodeMessage(msg) + m.Type = &FieldRules_Repeated{msg} + return true, err + case 19: // type.map + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(MapRules) + err := b.DecodeMessage(msg) + m.Type = &FieldRules_Map{msg} + return true, err + case 20: // type.any + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(AnyRules) + err := b.DecodeMessage(msg) + m.Type = &FieldRules_Any{msg} + return true, err + case 21: // type.duration + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(DurationRules) + err := b.DecodeMessage(msg) + m.Type = &FieldRules_Duration{msg} + return true, err + case 22: // type.timestamp + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(TimestampRules) + err := b.DecodeMessage(msg) + m.Type = &FieldRules_Timestamp{msg} + return true, err + default: + return false, nil + } +} + +func _FieldRules_OneofSizer(msg proto.Message) (n int) { + m := msg.(*FieldRules) + // type + switch x := m.Type.(type) { + case *FieldRules_Float: + s := proto.Size(x.Float) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *FieldRules_Double: + s := proto.Size(x.Double) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *FieldRules_Int32: + s := proto.Size(x.Int32) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *FieldRules_Int64: + s := proto.Size(x.Int64) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *FieldRules_Uint32: + s := proto.Size(x.Uint32) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *FieldRules_Uint64: + s := proto.Size(x.Uint64) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *FieldRules_Sint32: + s := proto.Size(x.Sint32) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *FieldRules_Sint64: + s := proto.Size(x.Sint64) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *FieldRules_Fixed32: + s := proto.Size(x.Fixed32) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *FieldRules_Fixed64: + s := proto.Size(x.Fixed64) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *FieldRules_Sfixed32: + s := proto.Size(x.Sfixed32) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *FieldRules_Sfixed64: + s := proto.Size(x.Sfixed64) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *FieldRules_Bool: + s := proto.Size(x.Bool) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *FieldRules_String_: + s := proto.Size(x.String_) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *FieldRules_Bytes: + s := proto.Size(x.Bytes) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *FieldRules_Enum: + s := proto.Size(x.Enum) + n += 2 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *FieldRules_Message: + s := proto.Size(x.Message) + n += 2 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *FieldRules_Repeated: + s := proto.Size(x.Repeated) + n += 2 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *FieldRules_Map: + s := proto.Size(x.Map) + n += 2 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *FieldRules_Any: + s := proto.Size(x.Any) + n += 2 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *FieldRules_Duration: + s := proto.Size(x.Duration) + n += 2 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *FieldRules_Timestamp: + s := proto.Size(x.Timestamp) + n += 2 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case nil: + default: + panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) + } + return n +} + +// FloatRules describes the constraints applied to `float` values +type FloatRules struct { + // Const specifies that this field must be exactly the specified value + Const *float32 `protobuf:"fixed32,1,opt,name=const" json:"const,omitempty"` + // Lt specifies that this field must be less than the specified value, + // exclusive + Lt *float32 `protobuf:"fixed32,2,opt,name=lt" json:"lt,omitempty"` + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + Lte *float32 `protobuf:"fixed32,3,opt,name=lte" json:"lte,omitempty"` + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + Gt *float32 `protobuf:"fixed32,4,opt,name=gt" json:"gt,omitempty"` + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + Gte *float32 `protobuf:"fixed32,5,opt,name=gte" json:"gte,omitempty"` + // In specifies that this field must be equal to one of the specified + // values + In []float32 `protobuf:"fixed32,6,rep,name=in" json:"in,omitempty"` + // NotIn specifies that this field cannot be equal to one of the specified + // values + NotIn []float32 `protobuf:"fixed32,7,rep,name=not_in,json=notIn" json:"not_in,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *FloatRules) Reset() { *m = FloatRules{} } +func (m *FloatRules) String() string { return proto.CompactTextString(m) } +func (*FloatRules) ProtoMessage() {} +func (*FloatRules) Descriptor() ([]byte, []int) { + return fileDescriptor_validate_c8f5e113dd6422a8, []int{1} +} +func (m *FloatRules) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_FloatRules.Unmarshal(m, b) +} +func (m *FloatRules) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_FloatRules.Marshal(b, m, deterministic) +} +func (dst *FloatRules) XXX_Merge(src proto.Message) { + xxx_messageInfo_FloatRules.Merge(dst, src) +} +func (m *FloatRules) XXX_Size() int { + return xxx_messageInfo_FloatRules.Size(m) +} +func (m *FloatRules) XXX_DiscardUnknown() { + xxx_messageInfo_FloatRules.DiscardUnknown(m) +} + +var xxx_messageInfo_FloatRules proto.InternalMessageInfo + +func (m *FloatRules) GetConst() float32 { + if m != nil && m.Const != nil { + return *m.Const + } + return 0 +} + +func (m *FloatRules) GetLt() float32 { + if m != nil && m.Lt != nil { + return *m.Lt + } + return 0 +} + +func (m *FloatRules) GetLte() float32 { + if m != nil && m.Lte != nil { + return *m.Lte + } + return 0 +} + +func (m *FloatRules) GetGt() float32 { + if m != nil && m.Gt != nil { + return *m.Gt + } + return 0 +} + +func (m *FloatRules) GetGte() float32 { + if m != nil && m.Gte != nil { + return *m.Gte + } + return 0 +} + +func (m *FloatRules) GetIn() []float32 { + if m != nil { + return m.In + } + return nil +} + +func (m *FloatRules) GetNotIn() []float32 { + if m != nil { + return m.NotIn + } + return nil +} + +// DoubleRules describes the constraints applied to `double` values +type DoubleRules struct { + // Const specifies that this field must be exactly the specified value + Const *float64 `protobuf:"fixed64,1,opt,name=const" json:"const,omitempty"` + // Lt specifies that this field must be less than the specified value, + // exclusive + Lt *float64 `protobuf:"fixed64,2,opt,name=lt" json:"lt,omitempty"` + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + Lte *float64 `protobuf:"fixed64,3,opt,name=lte" json:"lte,omitempty"` + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + Gt *float64 `protobuf:"fixed64,4,opt,name=gt" json:"gt,omitempty"` + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + Gte *float64 `protobuf:"fixed64,5,opt,name=gte" json:"gte,omitempty"` + // In specifies that this field must be equal to one of the specified + // values + In []float64 `protobuf:"fixed64,6,rep,name=in" json:"in,omitempty"` + // NotIn specifies that this field cannot be equal to one of the specified + // values + NotIn []float64 `protobuf:"fixed64,7,rep,name=not_in,json=notIn" json:"not_in,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DoubleRules) Reset() { *m = DoubleRules{} } +func (m *DoubleRules) String() string { return proto.CompactTextString(m) } +func (*DoubleRules) ProtoMessage() {} +func (*DoubleRules) Descriptor() ([]byte, []int) { + return fileDescriptor_validate_c8f5e113dd6422a8, []int{2} +} +func (m *DoubleRules) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DoubleRules.Unmarshal(m, b) +} +func (m *DoubleRules) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DoubleRules.Marshal(b, m, deterministic) +} +func (dst *DoubleRules) XXX_Merge(src proto.Message) { + xxx_messageInfo_DoubleRules.Merge(dst, src) +} +func (m *DoubleRules) XXX_Size() int { + return xxx_messageInfo_DoubleRules.Size(m) +} +func (m *DoubleRules) XXX_DiscardUnknown() { + xxx_messageInfo_DoubleRules.DiscardUnknown(m) +} + +var xxx_messageInfo_DoubleRules proto.InternalMessageInfo + +func (m *DoubleRules) GetConst() float64 { + if m != nil && m.Const != nil { + return *m.Const + } + return 0 +} + +func (m *DoubleRules) GetLt() float64 { + if m != nil && m.Lt != nil { + return *m.Lt + } + return 0 +} + +func (m *DoubleRules) GetLte() float64 { + if m != nil && m.Lte != nil { + return *m.Lte + } + return 0 +} + +func (m *DoubleRules) GetGt() float64 { + if m != nil && m.Gt != nil { + return *m.Gt + } + return 0 +} + +func (m *DoubleRules) GetGte() float64 { + if m != nil && m.Gte != nil { + return *m.Gte + } + return 0 +} + +func (m *DoubleRules) GetIn() []float64 { + if m != nil { + return m.In + } + return nil +} + +func (m *DoubleRules) GetNotIn() []float64 { + if m != nil { + return m.NotIn + } + return nil +} + +// Int32Rules describes the constraints applied to `int32` values +type Int32Rules struct { + // Const specifies that this field must be exactly the specified value + Const *int32 `protobuf:"varint,1,opt,name=const" json:"const,omitempty"` + // Lt specifies that this field must be less than the specified value, + // exclusive + Lt *int32 `protobuf:"varint,2,opt,name=lt" json:"lt,omitempty"` + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + Lte *int32 `protobuf:"varint,3,opt,name=lte" json:"lte,omitempty"` + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + Gt *int32 `protobuf:"varint,4,opt,name=gt" json:"gt,omitempty"` + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + Gte *int32 `protobuf:"varint,5,opt,name=gte" json:"gte,omitempty"` + // In specifies that this field must be equal to one of the specified + // values + In []int32 `protobuf:"varint,6,rep,name=in" json:"in,omitempty"` + // NotIn specifies that this field cannot be equal to one of the specified + // values + NotIn []int32 `protobuf:"varint,7,rep,name=not_in,json=notIn" json:"not_in,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Int32Rules) Reset() { *m = Int32Rules{} } +func (m *Int32Rules) String() string { return proto.CompactTextString(m) } +func (*Int32Rules) ProtoMessage() {} +func (*Int32Rules) Descriptor() ([]byte, []int) { + return fileDescriptor_validate_c8f5e113dd6422a8, []int{3} +} +func (m *Int32Rules) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Int32Rules.Unmarshal(m, b) +} +func (m *Int32Rules) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Int32Rules.Marshal(b, m, deterministic) +} +func (dst *Int32Rules) XXX_Merge(src proto.Message) { + xxx_messageInfo_Int32Rules.Merge(dst, src) +} +func (m *Int32Rules) XXX_Size() int { + return xxx_messageInfo_Int32Rules.Size(m) +} +func (m *Int32Rules) XXX_DiscardUnknown() { + xxx_messageInfo_Int32Rules.DiscardUnknown(m) +} + +var xxx_messageInfo_Int32Rules proto.InternalMessageInfo + +func (m *Int32Rules) GetConst() int32 { + if m != nil && m.Const != nil { + return *m.Const + } + return 0 +} + +func (m *Int32Rules) GetLt() int32 { + if m != nil && m.Lt != nil { + return *m.Lt + } + return 0 +} + +func (m *Int32Rules) GetLte() int32 { + if m != nil && m.Lte != nil { + return *m.Lte + } + return 0 +} + +func (m *Int32Rules) GetGt() int32 { + if m != nil && m.Gt != nil { + return *m.Gt + } + return 0 +} + +func (m *Int32Rules) GetGte() int32 { + if m != nil && m.Gte != nil { + return *m.Gte + } + return 0 +} + +func (m *Int32Rules) GetIn() []int32 { + if m != nil { + return m.In + } + return nil +} + +func (m *Int32Rules) GetNotIn() []int32 { + if m != nil { + return m.NotIn + } + return nil +} + +// Int64Rules describes the constraints applied to `int64` values +type Int64Rules struct { + // Const specifies that this field must be exactly the specified value + Const *int64 `protobuf:"varint,1,opt,name=const" json:"const,omitempty"` + // Lt specifies that this field must be less than the specified value, + // exclusive + Lt *int64 `protobuf:"varint,2,opt,name=lt" json:"lt,omitempty"` + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + Lte *int64 `protobuf:"varint,3,opt,name=lte" json:"lte,omitempty"` + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + Gt *int64 `protobuf:"varint,4,opt,name=gt" json:"gt,omitempty"` + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + Gte *int64 `protobuf:"varint,5,opt,name=gte" json:"gte,omitempty"` + // In specifies that this field must be equal to one of the specified + // values + In []int64 `protobuf:"varint,6,rep,name=in" json:"in,omitempty"` + // NotIn specifies that this field cannot be equal to one of the specified + // values + NotIn []int64 `protobuf:"varint,7,rep,name=not_in,json=notIn" json:"not_in,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Int64Rules) Reset() { *m = Int64Rules{} } +func (m *Int64Rules) String() string { return proto.CompactTextString(m) } +func (*Int64Rules) ProtoMessage() {} +func (*Int64Rules) Descriptor() ([]byte, []int) { + return fileDescriptor_validate_c8f5e113dd6422a8, []int{4} +} +func (m *Int64Rules) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Int64Rules.Unmarshal(m, b) +} +func (m *Int64Rules) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Int64Rules.Marshal(b, m, deterministic) +} +func (dst *Int64Rules) XXX_Merge(src proto.Message) { + xxx_messageInfo_Int64Rules.Merge(dst, src) +} +func (m *Int64Rules) XXX_Size() int { + return xxx_messageInfo_Int64Rules.Size(m) +} +func (m *Int64Rules) XXX_DiscardUnknown() { + xxx_messageInfo_Int64Rules.DiscardUnknown(m) +} + +var xxx_messageInfo_Int64Rules proto.InternalMessageInfo + +func (m *Int64Rules) GetConst() int64 { + if m != nil && m.Const != nil { + return *m.Const + } + return 0 +} + +func (m *Int64Rules) GetLt() int64 { + if m != nil && m.Lt != nil { + return *m.Lt + } + return 0 +} + +func (m *Int64Rules) GetLte() int64 { + if m != nil && m.Lte != nil { + return *m.Lte + } + return 0 +} + +func (m *Int64Rules) GetGt() int64 { + if m != nil && m.Gt != nil { + return *m.Gt + } + return 0 +} + +func (m *Int64Rules) GetGte() int64 { + if m != nil && m.Gte != nil { + return *m.Gte + } + return 0 +} + +func (m *Int64Rules) GetIn() []int64 { + if m != nil { + return m.In + } + return nil +} + +func (m *Int64Rules) GetNotIn() []int64 { + if m != nil { + return m.NotIn + } + return nil +} + +// UInt32Rules describes the constraints applied to `uint32` values +type UInt32Rules struct { + // Const specifies that this field must be exactly the specified value + Const *uint32 `protobuf:"varint,1,opt,name=const" json:"const,omitempty"` + // Lt specifies that this field must be less than the specified value, + // exclusive + Lt *uint32 `protobuf:"varint,2,opt,name=lt" json:"lt,omitempty"` + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + Lte *uint32 `protobuf:"varint,3,opt,name=lte" json:"lte,omitempty"` + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + Gt *uint32 `protobuf:"varint,4,opt,name=gt" json:"gt,omitempty"` + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + Gte *uint32 `protobuf:"varint,5,opt,name=gte" json:"gte,omitempty"` + // In specifies that this field must be equal to one of the specified + // values + In []uint32 `protobuf:"varint,6,rep,name=in" json:"in,omitempty"` + // NotIn specifies that this field cannot be equal to one of the specified + // values + NotIn []uint32 `protobuf:"varint,7,rep,name=not_in,json=notIn" json:"not_in,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *UInt32Rules) Reset() { *m = UInt32Rules{} } +func (m *UInt32Rules) String() string { return proto.CompactTextString(m) } +func (*UInt32Rules) ProtoMessage() {} +func (*UInt32Rules) Descriptor() ([]byte, []int) { + return fileDescriptor_validate_c8f5e113dd6422a8, []int{5} +} +func (m *UInt32Rules) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_UInt32Rules.Unmarshal(m, b) +} +func (m *UInt32Rules) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_UInt32Rules.Marshal(b, m, deterministic) +} +func (dst *UInt32Rules) XXX_Merge(src proto.Message) { + xxx_messageInfo_UInt32Rules.Merge(dst, src) +} +func (m *UInt32Rules) XXX_Size() int { + return xxx_messageInfo_UInt32Rules.Size(m) +} +func (m *UInt32Rules) XXX_DiscardUnknown() { + xxx_messageInfo_UInt32Rules.DiscardUnknown(m) +} + +var xxx_messageInfo_UInt32Rules proto.InternalMessageInfo + +func (m *UInt32Rules) GetConst() uint32 { + if m != nil && m.Const != nil { + return *m.Const + } + return 0 +} + +func (m *UInt32Rules) GetLt() uint32 { + if m != nil && m.Lt != nil { + return *m.Lt + } + return 0 +} + +func (m *UInt32Rules) GetLte() uint32 { + if m != nil && m.Lte != nil { + return *m.Lte + } + return 0 +} + +func (m *UInt32Rules) GetGt() uint32 { + if m != nil && m.Gt != nil { + return *m.Gt + } + return 0 +} + +func (m *UInt32Rules) GetGte() uint32 { + if m != nil && m.Gte != nil { + return *m.Gte + } + return 0 +} + +func (m *UInt32Rules) GetIn() []uint32 { + if m != nil { + return m.In + } + return nil +} + +func (m *UInt32Rules) GetNotIn() []uint32 { + if m != nil { + return m.NotIn + } + return nil +} + +// UInt64Rules describes the constraints applied to `uint64` values +type UInt64Rules struct { + // Const specifies that this field must be exactly the specified value + Const *uint64 `protobuf:"varint,1,opt,name=const" json:"const,omitempty"` + // Lt specifies that this field must be less than the specified value, + // exclusive + Lt *uint64 `protobuf:"varint,2,opt,name=lt" json:"lt,omitempty"` + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + Lte *uint64 `protobuf:"varint,3,opt,name=lte" json:"lte,omitempty"` + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + Gt *uint64 `protobuf:"varint,4,opt,name=gt" json:"gt,omitempty"` + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + Gte *uint64 `protobuf:"varint,5,opt,name=gte" json:"gte,omitempty"` + // In specifies that this field must be equal to one of the specified + // values + In []uint64 `protobuf:"varint,6,rep,name=in" json:"in,omitempty"` + // NotIn specifies that this field cannot be equal to one of the specified + // values + NotIn []uint64 `protobuf:"varint,7,rep,name=not_in,json=notIn" json:"not_in,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *UInt64Rules) Reset() { *m = UInt64Rules{} } +func (m *UInt64Rules) String() string { return proto.CompactTextString(m) } +func (*UInt64Rules) ProtoMessage() {} +func (*UInt64Rules) Descriptor() ([]byte, []int) { + return fileDescriptor_validate_c8f5e113dd6422a8, []int{6} +} +func (m *UInt64Rules) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_UInt64Rules.Unmarshal(m, b) +} +func (m *UInt64Rules) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_UInt64Rules.Marshal(b, m, deterministic) +} +func (dst *UInt64Rules) XXX_Merge(src proto.Message) { + xxx_messageInfo_UInt64Rules.Merge(dst, src) +} +func (m *UInt64Rules) XXX_Size() int { + return xxx_messageInfo_UInt64Rules.Size(m) +} +func (m *UInt64Rules) XXX_DiscardUnknown() { + xxx_messageInfo_UInt64Rules.DiscardUnknown(m) +} + +var xxx_messageInfo_UInt64Rules proto.InternalMessageInfo + +func (m *UInt64Rules) GetConst() uint64 { + if m != nil && m.Const != nil { + return *m.Const + } + return 0 +} + +func (m *UInt64Rules) GetLt() uint64 { + if m != nil && m.Lt != nil { + return *m.Lt + } + return 0 +} + +func (m *UInt64Rules) GetLte() uint64 { + if m != nil && m.Lte != nil { + return *m.Lte + } + return 0 +} + +func (m *UInt64Rules) GetGt() uint64 { + if m != nil && m.Gt != nil { + return *m.Gt + } + return 0 +} + +func (m *UInt64Rules) GetGte() uint64 { + if m != nil && m.Gte != nil { + return *m.Gte + } + return 0 +} + +func (m *UInt64Rules) GetIn() []uint64 { + if m != nil { + return m.In + } + return nil +} + +func (m *UInt64Rules) GetNotIn() []uint64 { + if m != nil { + return m.NotIn + } + return nil +} + +// SInt32Rules describes the constraints applied to `sint32` values +type SInt32Rules struct { + // Const specifies that this field must be exactly the specified value + Const *int32 `protobuf:"zigzag32,1,opt,name=const" json:"const,omitempty"` + // Lt specifies that this field must be less than the specified value, + // exclusive + Lt *int32 `protobuf:"zigzag32,2,opt,name=lt" json:"lt,omitempty"` + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + Lte *int32 `protobuf:"zigzag32,3,opt,name=lte" json:"lte,omitempty"` + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + Gt *int32 `protobuf:"zigzag32,4,opt,name=gt" json:"gt,omitempty"` + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + Gte *int32 `protobuf:"zigzag32,5,opt,name=gte" json:"gte,omitempty"` + // In specifies that this field must be equal to one of the specified + // values + In []int32 `protobuf:"zigzag32,6,rep,name=in" json:"in,omitempty"` + // NotIn specifies that this field cannot be equal to one of the specified + // values + NotIn []int32 `protobuf:"zigzag32,7,rep,name=not_in,json=notIn" json:"not_in,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SInt32Rules) Reset() { *m = SInt32Rules{} } +func (m *SInt32Rules) String() string { return proto.CompactTextString(m) } +func (*SInt32Rules) ProtoMessage() {} +func (*SInt32Rules) Descriptor() ([]byte, []int) { + return fileDescriptor_validate_c8f5e113dd6422a8, []int{7} +} +func (m *SInt32Rules) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_SInt32Rules.Unmarshal(m, b) +} +func (m *SInt32Rules) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_SInt32Rules.Marshal(b, m, deterministic) +} +func (dst *SInt32Rules) XXX_Merge(src proto.Message) { + xxx_messageInfo_SInt32Rules.Merge(dst, src) +} +func (m *SInt32Rules) XXX_Size() int { + return xxx_messageInfo_SInt32Rules.Size(m) +} +func (m *SInt32Rules) XXX_DiscardUnknown() { + xxx_messageInfo_SInt32Rules.DiscardUnknown(m) +} + +var xxx_messageInfo_SInt32Rules proto.InternalMessageInfo + +func (m *SInt32Rules) GetConst() int32 { + if m != nil && m.Const != nil { + return *m.Const + } + return 0 +} + +func (m *SInt32Rules) GetLt() int32 { + if m != nil && m.Lt != nil { + return *m.Lt + } + return 0 +} + +func (m *SInt32Rules) GetLte() int32 { + if m != nil && m.Lte != nil { + return *m.Lte + } + return 0 +} + +func (m *SInt32Rules) GetGt() int32 { + if m != nil && m.Gt != nil { + return *m.Gt + } + return 0 +} + +func (m *SInt32Rules) GetGte() int32 { + if m != nil && m.Gte != nil { + return *m.Gte + } + return 0 +} + +func (m *SInt32Rules) GetIn() []int32 { + if m != nil { + return m.In + } + return nil +} + +func (m *SInt32Rules) GetNotIn() []int32 { + if m != nil { + return m.NotIn + } + return nil +} + +// SInt64Rules describes the constraints applied to `sint64` values +type SInt64Rules struct { + // Const specifies that this field must be exactly the specified value + Const *int64 `protobuf:"zigzag64,1,opt,name=const" json:"const,omitempty"` + // Lt specifies that this field must be less than the specified value, + // exclusive + Lt *int64 `protobuf:"zigzag64,2,opt,name=lt" json:"lt,omitempty"` + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + Lte *int64 `protobuf:"zigzag64,3,opt,name=lte" json:"lte,omitempty"` + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + Gt *int64 `protobuf:"zigzag64,4,opt,name=gt" json:"gt,omitempty"` + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + Gte *int64 `protobuf:"zigzag64,5,opt,name=gte" json:"gte,omitempty"` + // In specifies that this field must be equal to one of the specified + // values + In []int64 `protobuf:"zigzag64,6,rep,name=in" json:"in,omitempty"` + // NotIn specifies that this field cannot be equal to one of the specified + // values + NotIn []int64 `protobuf:"zigzag64,7,rep,name=not_in,json=notIn" json:"not_in,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SInt64Rules) Reset() { *m = SInt64Rules{} } +func (m *SInt64Rules) String() string { return proto.CompactTextString(m) } +func (*SInt64Rules) ProtoMessage() {} +func (*SInt64Rules) Descriptor() ([]byte, []int) { + return fileDescriptor_validate_c8f5e113dd6422a8, []int{8} +} +func (m *SInt64Rules) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_SInt64Rules.Unmarshal(m, b) +} +func (m *SInt64Rules) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_SInt64Rules.Marshal(b, m, deterministic) +} +func (dst *SInt64Rules) XXX_Merge(src proto.Message) { + xxx_messageInfo_SInt64Rules.Merge(dst, src) +} +func (m *SInt64Rules) XXX_Size() int { + return xxx_messageInfo_SInt64Rules.Size(m) +} +func (m *SInt64Rules) XXX_DiscardUnknown() { + xxx_messageInfo_SInt64Rules.DiscardUnknown(m) +} + +var xxx_messageInfo_SInt64Rules proto.InternalMessageInfo + +func (m *SInt64Rules) GetConst() int64 { + if m != nil && m.Const != nil { + return *m.Const + } + return 0 +} + +func (m *SInt64Rules) GetLt() int64 { + if m != nil && m.Lt != nil { + return *m.Lt + } + return 0 +} + +func (m *SInt64Rules) GetLte() int64 { + if m != nil && m.Lte != nil { + return *m.Lte + } + return 0 +} + +func (m *SInt64Rules) GetGt() int64 { + if m != nil && m.Gt != nil { + return *m.Gt + } + return 0 +} + +func (m *SInt64Rules) GetGte() int64 { + if m != nil && m.Gte != nil { + return *m.Gte + } + return 0 +} + +func (m *SInt64Rules) GetIn() []int64 { + if m != nil { + return m.In + } + return nil +} + +func (m *SInt64Rules) GetNotIn() []int64 { + if m != nil { + return m.NotIn + } + return nil +} + +// Fixed32Rules describes the constraints applied to `fixed32` values +type Fixed32Rules struct { + // Const specifies that this field must be exactly the specified value + Const *uint32 `protobuf:"fixed32,1,opt,name=const" json:"const,omitempty"` + // Lt specifies that this field must be less than the specified value, + // exclusive + Lt *uint32 `protobuf:"fixed32,2,opt,name=lt" json:"lt,omitempty"` + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + Lte *uint32 `protobuf:"fixed32,3,opt,name=lte" json:"lte,omitempty"` + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + Gt *uint32 `protobuf:"fixed32,4,opt,name=gt" json:"gt,omitempty"` + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + Gte *uint32 `protobuf:"fixed32,5,opt,name=gte" json:"gte,omitempty"` + // In specifies that this field must be equal to one of the specified + // values + In []uint32 `protobuf:"fixed32,6,rep,name=in" json:"in,omitempty"` + // NotIn specifies that this field cannot be equal to one of the specified + // values + NotIn []uint32 `protobuf:"fixed32,7,rep,name=not_in,json=notIn" json:"not_in,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Fixed32Rules) Reset() { *m = Fixed32Rules{} } +func (m *Fixed32Rules) String() string { return proto.CompactTextString(m) } +func (*Fixed32Rules) ProtoMessage() {} +func (*Fixed32Rules) Descriptor() ([]byte, []int) { + return fileDescriptor_validate_c8f5e113dd6422a8, []int{9} +} +func (m *Fixed32Rules) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Fixed32Rules.Unmarshal(m, b) +} +func (m *Fixed32Rules) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Fixed32Rules.Marshal(b, m, deterministic) +} +func (dst *Fixed32Rules) XXX_Merge(src proto.Message) { + xxx_messageInfo_Fixed32Rules.Merge(dst, src) +} +func (m *Fixed32Rules) XXX_Size() int { + return xxx_messageInfo_Fixed32Rules.Size(m) +} +func (m *Fixed32Rules) XXX_DiscardUnknown() { + xxx_messageInfo_Fixed32Rules.DiscardUnknown(m) +} + +var xxx_messageInfo_Fixed32Rules proto.InternalMessageInfo + +func (m *Fixed32Rules) GetConst() uint32 { + if m != nil && m.Const != nil { + return *m.Const + } + return 0 +} + +func (m *Fixed32Rules) GetLt() uint32 { + if m != nil && m.Lt != nil { + return *m.Lt + } + return 0 +} + +func (m *Fixed32Rules) GetLte() uint32 { + if m != nil && m.Lte != nil { + return *m.Lte + } + return 0 +} + +func (m *Fixed32Rules) GetGt() uint32 { + if m != nil && m.Gt != nil { + return *m.Gt + } + return 0 +} + +func (m *Fixed32Rules) GetGte() uint32 { + if m != nil && m.Gte != nil { + return *m.Gte + } + return 0 +} + +func (m *Fixed32Rules) GetIn() []uint32 { + if m != nil { + return m.In + } + return nil +} + +func (m *Fixed32Rules) GetNotIn() []uint32 { + if m != nil { + return m.NotIn + } + return nil +} + +// Fixed64Rules describes the constraints applied to `fixed64` values +type Fixed64Rules struct { + // Const specifies that this field must be exactly the specified value + Const *uint64 `protobuf:"fixed64,1,opt,name=const" json:"const,omitempty"` + // Lt specifies that this field must be less than the specified value, + // exclusive + Lt *uint64 `protobuf:"fixed64,2,opt,name=lt" json:"lt,omitempty"` + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + Lte *uint64 `protobuf:"fixed64,3,opt,name=lte" json:"lte,omitempty"` + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + Gt *uint64 `protobuf:"fixed64,4,opt,name=gt" json:"gt,omitempty"` + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + Gte *uint64 `protobuf:"fixed64,5,opt,name=gte" json:"gte,omitempty"` + // In specifies that this field must be equal to one of the specified + // values + In []uint64 `protobuf:"fixed64,6,rep,name=in" json:"in,omitempty"` + // NotIn specifies that this field cannot be equal to one of the specified + // values + NotIn []uint64 `protobuf:"fixed64,7,rep,name=not_in,json=notIn" json:"not_in,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Fixed64Rules) Reset() { *m = Fixed64Rules{} } +func (m *Fixed64Rules) String() string { return proto.CompactTextString(m) } +func (*Fixed64Rules) ProtoMessage() {} +func (*Fixed64Rules) Descriptor() ([]byte, []int) { + return fileDescriptor_validate_c8f5e113dd6422a8, []int{10} +} +func (m *Fixed64Rules) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Fixed64Rules.Unmarshal(m, b) +} +func (m *Fixed64Rules) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Fixed64Rules.Marshal(b, m, deterministic) +} +func (dst *Fixed64Rules) XXX_Merge(src proto.Message) { + xxx_messageInfo_Fixed64Rules.Merge(dst, src) +} +func (m *Fixed64Rules) XXX_Size() int { + return xxx_messageInfo_Fixed64Rules.Size(m) +} +func (m *Fixed64Rules) XXX_DiscardUnknown() { + xxx_messageInfo_Fixed64Rules.DiscardUnknown(m) +} + +var xxx_messageInfo_Fixed64Rules proto.InternalMessageInfo + +func (m *Fixed64Rules) GetConst() uint64 { + if m != nil && m.Const != nil { + return *m.Const + } + return 0 +} + +func (m *Fixed64Rules) GetLt() uint64 { + if m != nil && m.Lt != nil { + return *m.Lt + } + return 0 +} + +func (m *Fixed64Rules) GetLte() uint64 { + if m != nil && m.Lte != nil { + return *m.Lte + } + return 0 +} + +func (m *Fixed64Rules) GetGt() uint64 { + if m != nil && m.Gt != nil { + return *m.Gt + } + return 0 +} + +func (m *Fixed64Rules) GetGte() uint64 { + if m != nil && m.Gte != nil { + return *m.Gte + } + return 0 +} + +func (m *Fixed64Rules) GetIn() []uint64 { + if m != nil { + return m.In + } + return nil +} + +func (m *Fixed64Rules) GetNotIn() []uint64 { + if m != nil { + return m.NotIn + } + return nil +} + +// SFixed32Rules describes the constraints applied to `sfixed32` values +type SFixed32Rules struct { + // Const specifies that this field must be exactly the specified value + Const *int32 `protobuf:"fixed32,1,opt,name=const" json:"const,omitempty"` + // Lt specifies that this field must be less than the specified value, + // exclusive + Lt *int32 `protobuf:"fixed32,2,opt,name=lt" json:"lt,omitempty"` + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + Lte *int32 `protobuf:"fixed32,3,opt,name=lte" json:"lte,omitempty"` + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + Gt *int32 `protobuf:"fixed32,4,opt,name=gt" json:"gt,omitempty"` + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + Gte *int32 `protobuf:"fixed32,5,opt,name=gte" json:"gte,omitempty"` + // In specifies that this field must be equal to one of the specified + // values + In []int32 `protobuf:"fixed32,6,rep,name=in" json:"in,omitempty"` + // NotIn specifies that this field cannot be equal to one of the specified + // values + NotIn []int32 `protobuf:"fixed32,7,rep,name=not_in,json=notIn" json:"not_in,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SFixed32Rules) Reset() { *m = SFixed32Rules{} } +func (m *SFixed32Rules) String() string { return proto.CompactTextString(m) } +func (*SFixed32Rules) ProtoMessage() {} +func (*SFixed32Rules) Descriptor() ([]byte, []int) { + return fileDescriptor_validate_c8f5e113dd6422a8, []int{11} +} +func (m *SFixed32Rules) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_SFixed32Rules.Unmarshal(m, b) +} +func (m *SFixed32Rules) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_SFixed32Rules.Marshal(b, m, deterministic) +} +func (dst *SFixed32Rules) XXX_Merge(src proto.Message) { + xxx_messageInfo_SFixed32Rules.Merge(dst, src) +} +func (m *SFixed32Rules) XXX_Size() int { + return xxx_messageInfo_SFixed32Rules.Size(m) +} +func (m *SFixed32Rules) XXX_DiscardUnknown() { + xxx_messageInfo_SFixed32Rules.DiscardUnknown(m) +} + +var xxx_messageInfo_SFixed32Rules proto.InternalMessageInfo + +func (m *SFixed32Rules) GetConst() int32 { + if m != nil && m.Const != nil { + return *m.Const + } + return 0 +} + +func (m *SFixed32Rules) GetLt() int32 { + if m != nil && m.Lt != nil { + return *m.Lt + } + return 0 +} + +func (m *SFixed32Rules) GetLte() int32 { + if m != nil && m.Lte != nil { + return *m.Lte + } + return 0 +} + +func (m *SFixed32Rules) GetGt() int32 { + if m != nil && m.Gt != nil { + return *m.Gt + } + return 0 +} + +func (m *SFixed32Rules) GetGte() int32 { + if m != nil && m.Gte != nil { + return *m.Gte + } + return 0 +} + +func (m *SFixed32Rules) GetIn() []int32 { + if m != nil { + return m.In + } + return nil +} + +func (m *SFixed32Rules) GetNotIn() []int32 { + if m != nil { + return m.NotIn + } + return nil +} + +// SFixed64Rules describes the constraints applied to `sfixed64` values +type SFixed64Rules struct { + // Const specifies that this field must be exactly the specified value + Const *int64 `protobuf:"fixed64,1,opt,name=const" json:"const,omitempty"` + // Lt specifies that this field must be less than the specified value, + // exclusive + Lt *int64 `protobuf:"fixed64,2,opt,name=lt" json:"lt,omitempty"` + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + Lte *int64 `protobuf:"fixed64,3,opt,name=lte" json:"lte,omitempty"` + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + Gt *int64 `protobuf:"fixed64,4,opt,name=gt" json:"gt,omitempty"` + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + Gte *int64 `protobuf:"fixed64,5,opt,name=gte" json:"gte,omitempty"` + // In specifies that this field must be equal to one of the specified + // values + In []int64 `protobuf:"fixed64,6,rep,name=in" json:"in,omitempty"` + // NotIn specifies that this field cannot be equal to one of the specified + // values + NotIn []int64 `protobuf:"fixed64,7,rep,name=not_in,json=notIn" json:"not_in,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SFixed64Rules) Reset() { *m = SFixed64Rules{} } +func (m *SFixed64Rules) String() string { return proto.CompactTextString(m) } +func (*SFixed64Rules) ProtoMessage() {} +func (*SFixed64Rules) Descriptor() ([]byte, []int) { + return fileDescriptor_validate_c8f5e113dd6422a8, []int{12} +} +func (m *SFixed64Rules) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_SFixed64Rules.Unmarshal(m, b) +} +func (m *SFixed64Rules) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_SFixed64Rules.Marshal(b, m, deterministic) +} +func (dst *SFixed64Rules) XXX_Merge(src proto.Message) { + xxx_messageInfo_SFixed64Rules.Merge(dst, src) +} +func (m *SFixed64Rules) XXX_Size() int { + return xxx_messageInfo_SFixed64Rules.Size(m) +} +func (m *SFixed64Rules) XXX_DiscardUnknown() { + xxx_messageInfo_SFixed64Rules.DiscardUnknown(m) +} + +var xxx_messageInfo_SFixed64Rules proto.InternalMessageInfo + +func (m *SFixed64Rules) GetConst() int64 { + if m != nil && m.Const != nil { + return *m.Const + } + return 0 +} + +func (m *SFixed64Rules) GetLt() int64 { + if m != nil && m.Lt != nil { + return *m.Lt + } + return 0 +} + +func (m *SFixed64Rules) GetLte() int64 { + if m != nil && m.Lte != nil { + return *m.Lte + } + return 0 +} + +func (m *SFixed64Rules) GetGt() int64 { + if m != nil && m.Gt != nil { + return *m.Gt + } + return 0 +} + +func (m *SFixed64Rules) GetGte() int64 { + if m != nil && m.Gte != nil { + return *m.Gte + } + return 0 +} + +func (m *SFixed64Rules) GetIn() []int64 { + if m != nil { + return m.In + } + return nil +} + +func (m *SFixed64Rules) GetNotIn() []int64 { + if m != nil { + return m.NotIn + } + return nil +} + +// BoolRules describes the constraints applied to `bool` values +type BoolRules struct { + // Const specifies that this field must be exactly the specified value + Const *bool `protobuf:"varint,1,opt,name=const" json:"const,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *BoolRules) Reset() { *m = BoolRules{} } +func (m *BoolRules) String() string { return proto.CompactTextString(m) } +func (*BoolRules) ProtoMessage() {} +func (*BoolRules) Descriptor() ([]byte, []int) { + return fileDescriptor_validate_c8f5e113dd6422a8, []int{13} +} +func (m *BoolRules) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_BoolRules.Unmarshal(m, b) +} +func (m *BoolRules) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_BoolRules.Marshal(b, m, deterministic) +} +func (dst *BoolRules) XXX_Merge(src proto.Message) { + xxx_messageInfo_BoolRules.Merge(dst, src) +} +func (m *BoolRules) XXX_Size() int { + return xxx_messageInfo_BoolRules.Size(m) +} +func (m *BoolRules) XXX_DiscardUnknown() { + xxx_messageInfo_BoolRules.DiscardUnknown(m) +} + +var xxx_messageInfo_BoolRules proto.InternalMessageInfo + +func (m *BoolRules) GetConst() bool { + if m != nil && m.Const != nil { + return *m.Const + } + return false +} + +// StringRules describe the constraints applied to `string` values +type StringRules struct { + // Const specifies that this field must be exactly the specified value + Const *string `protobuf:"bytes,1,opt,name=const" json:"const,omitempty"` + // Len specifies that this field must be the specified number of + // characters (Unicode code points). Note that the number of + // characters may differ from the number of bytes in the string. + Len *uint64 `protobuf:"varint,19,opt,name=len" json:"len,omitempty"` + // MinLen specifies that this field must be the specified number of + // characters (Unicode code points) at a minimum. Note that the number of + // characters may differ from the number of bytes in the string. + MinLen *uint64 `protobuf:"varint,2,opt,name=min_len,json=minLen" json:"min_len,omitempty"` + // MaxLen specifies that this field must be the specified number of + // characters (Unicode code points) at a maximum. Note that the number of + // characters may differ from the number of bytes in the string. + MaxLen *uint64 `protobuf:"varint,3,opt,name=max_len,json=maxLen" json:"max_len,omitempty"` + // LenBytes specifies that this field must be the specified number of bytes + // at a minimum + LenBytes *uint64 `protobuf:"varint,20,opt,name=len_bytes,json=lenBytes" json:"len_bytes,omitempty"` + // MinBytes specifies that this field must be the specified number of bytes + // at a minimum + MinBytes *uint64 `protobuf:"varint,4,opt,name=min_bytes,json=minBytes" json:"min_bytes,omitempty"` + // MaxBytes specifies that this field must be the specified number of bytes + // at a maximum + MaxBytes *uint64 `protobuf:"varint,5,opt,name=max_bytes,json=maxBytes" json:"max_bytes,omitempty"` + // Pattern specifes that this field must match against the specified + // regular expression (RE2 syntax). The included expression should elide + // any delimiters. + Pattern *string `protobuf:"bytes,6,opt,name=pattern" json:"pattern,omitempty"` + // Prefix specifies that this field must have the specified substring at + // the beginning of the string. + Prefix *string `protobuf:"bytes,7,opt,name=prefix" json:"prefix,omitempty"` + // Suffix specifies that this field must have the specified substring at + // the end of the string. + Suffix *string `protobuf:"bytes,8,opt,name=suffix" json:"suffix,omitempty"` + // Contains specifies that this field must have the specified substring + // anywhere in the string. + Contains *string `protobuf:"bytes,9,opt,name=contains" json:"contains,omitempty"` + // In specifies that this field must be equal to one of the specified + // values + In []string `protobuf:"bytes,10,rep,name=in" json:"in,omitempty"` + // NotIn specifies that this field cannot be equal to one of the specified + // values + NotIn []string `protobuf:"bytes,11,rep,name=not_in,json=notIn" json:"not_in,omitempty"` + // WellKnown rules provide advanced constraints against common string + // patterns + // + // Types that are valid to be assigned to WellKnown: + // *StringRules_Email + // *StringRules_Hostname + // *StringRules_Ip + // *StringRules_Ipv4 + // *StringRules_Ipv6 + // *StringRules_Uri + // *StringRules_UriRef + // *StringRules_Address + WellKnown isStringRules_WellKnown `protobuf_oneof:"well_known"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *StringRules) Reset() { *m = StringRules{} } +func (m *StringRules) String() string { return proto.CompactTextString(m) } +func (*StringRules) ProtoMessage() {} +func (*StringRules) Descriptor() ([]byte, []int) { + return fileDescriptor_validate_c8f5e113dd6422a8, []int{14} +} +func (m *StringRules) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_StringRules.Unmarshal(m, b) +} +func (m *StringRules) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_StringRules.Marshal(b, m, deterministic) +} +func (dst *StringRules) XXX_Merge(src proto.Message) { + xxx_messageInfo_StringRules.Merge(dst, src) +} +func (m *StringRules) XXX_Size() int { + return xxx_messageInfo_StringRules.Size(m) +} +func (m *StringRules) XXX_DiscardUnknown() { + xxx_messageInfo_StringRules.DiscardUnknown(m) +} + +var xxx_messageInfo_StringRules proto.InternalMessageInfo + +func (m *StringRules) GetConst() string { + if m != nil && m.Const != nil { + return *m.Const + } + return "" +} + +func (m *StringRules) GetLen() uint64 { + if m != nil && m.Len != nil { + return *m.Len + } + return 0 +} + +func (m *StringRules) GetMinLen() uint64 { + if m != nil && m.MinLen != nil { + return *m.MinLen + } + return 0 +} + +func (m *StringRules) GetMaxLen() uint64 { + if m != nil && m.MaxLen != nil { + return *m.MaxLen + } + return 0 +} + +func (m *StringRules) GetLenBytes() uint64 { + if m != nil && m.LenBytes != nil { + return *m.LenBytes + } + return 0 +} + +func (m *StringRules) GetMinBytes() uint64 { + if m != nil && m.MinBytes != nil { + return *m.MinBytes + } + return 0 +} + +func (m *StringRules) GetMaxBytes() uint64 { + if m != nil && m.MaxBytes != nil { + return *m.MaxBytes + } + return 0 +} + +func (m *StringRules) GetPattern() string { + if m != nil && m.Pattern != nil { + return *m.Pattern + } + return "" +} + +func (m *StringRules) GetPrefix() string { + if m != nil && m.Prefix != nil { + return *m.Prefix + } + return "" +} + +func (m *StringRules) GetSuffix() string { + if m != nil && m.Suffix != nil { + return *m.Suffix + } + return "" +} + +func (m *StringRules) GetContains() string { + if m != nil && m.Contains != nil { + return *m.Contains + } + return "" +} + +func (m *StringRules) GetIn() []string { + if m != nil { + return m.In + } + return nil +} + +func (m *StringRules) GetNotIn() []string { + if m != nil { + return m.NotIn + } + return nil +} + +type isStringRules_WellKnown interface { + isStringRules_WellKnown() +} + +type StringRules_Email struct { + Email bool `protobuf:"varint,12,opt,name=email,oneof"` +} + +type StringRules_Hostname struct { + Hostname bool `protobuf:"varint,13,opt,name=hostname,oneof"` +} + +type StringRules_Ip struct { + Ip bool `protobuf:"varint,14,opt,name=ip,oneof"` +} + +type StringRules_Ipv4 struct { + Ipv4 bool `protobuf:"varint,15,opt,name=ipv4,oneof"` +} + +type StringRules_Ipv6 struct { + Ipv6 bool `protobuf:"varint,16,opt,name=ipv6,oneof"` +} + +type StringRules_Uri struct { + Uri bool `protobuf:"varint,17,opt,name=uri,oneof"` +} + +type StringRules_UriRef struct { + UriRef bool `protobuf:"varint,18,opt,name=uri_ref,json=uriRef,oneof"` +} + +type StringRules_Address struct { + Address bool `protobuf:"varint,21,opt,name=address,oneof"` +} + +func (*StringRules_Email) isStringRules_WellKnown() {} + +func (*StringRules_Hostname) isStringRules_WellKnown() {} + +func (*StringRules_Ip) isStringRules_WellKnown() {} + +func (*StringRules_Ipv4) isStringRules_WellKnown() {} + +func (*StringRules_Ipv6) isStringRules_WellKnown() {} + +func (*StringRules_Uri) isStringRules_WellKnown() {} + +func (*StringRules_UriRef) isStringRules_WellKnown() {} + +func (*StringRules_Address) isStringRules_WellKnown() {} + +func (m *StringRules) GetWellKnown() isStringRules_WellKnown { + if m != nil { + return m.WellKnown + } + return nil +} + +func (m *StringRules) GetEmail() bool { + if x, ok := m.GetWellKnown().(*StringRules_Email); ok { + return x.Email + } + return false +} + +func (m *StringRules) GetHostname() bool { + if x, ok := m.GetWellKnown().(*StringRules_Hostname); ok { + return x.Hostname + } + return false +} + +func (m *StringRules) GetIp() bool { + if x, ok := m.GetWellKnown().(*StringRules_Ip); ok { + return x.Ip + } + return false +} + +func (m *StringRules) GetIpv4() bool { + if x, ok := m.GetWellKnown().(*StringRules_Ipv4); ok { + return x.Ipv4 + } + return false +} + +func (m *StringRules) GetIpv6() bool { + if x, ok := m.GetWellKnown().(*StringRules_Ipv6); ok { + return x.Ipv6 + } + return false +} + +func (m *StringRules) GetUri() bool { + if x, ok := m.GetWellKnown().(*StringRules_Uri); ok { + return x.Uri + } + return false +} + +func (m *StringRules) GetUriRef() bool { + if x, ok := m.GetWellKnown().(*StringRules_UriRef); ok { + return x.UriRef + } + return false +} + +func (m *StringRules) GetAddress() bool { + if x, ok := m.GetWellKnown().(*StringRules_Address); ok { + return x.Address + } + return false +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*StringRules) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { + return _StringRules_OneofMarshaler, _StringRules_OneofUnmarshaler, _StringRules_OneofSizer, []interface{}{ + (*StringRules_Email)(nil), + (*StringRules_Hostname)(nil), + (*StringRules_Ip)(nil), + (*StringRules_Ipv4)(nil), + (*StringRules_Ipv6)(nil), + (*StringRules_Uri)(nil), + (*StringRules_UriRef)(nil), + (*StringRules_Address)(nil), + } +} + +func _StringRules_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*StringRules) + // well_known + switch x := m.WellKnown.(type) { + case *StringRules_Email: + t := uint64(0) + if x.Email { + t = 1 + } + b.EncodeVarint(12<<3 | proto.WireVarint) + b.EncodeVarint(t) + case *StringRules_Hostname: + t := uint64(0) + if x.Hostname { + t = 1 + } + b.EncodeVarint(13<<3 | proto.WireVarint) + b.EncodeVarint(t) + case *StringRules_Ip: + t := uint64(0) + if x.Ip { + t = 1 + } + b.EncodeVarint(14<<3 | proto.WireVarint) + b.EncodeVarint(t) + case *StringRules_Ipv4: + t := uint64(0) + if x.Ipv4 { + t = 1 + } + b.EncodeVarint(15<<3 | proto.WireVarint) + b.EncodeVarint(t) + case *StringRules_Ipv6: + t := uint64(0) + if x.Ipv6 { + t = 1 + } + b.EncodeVarint(16<<3 | proto.WireVarint) + b.EncodeVarint(t) + case *StringRules_Uri: + t := uint64(0) + if x.Uri { + t = 1 + } + b.EncodeVarint(17<<3 | proto.WireVarint) + b.EncodeVarint(t) + case *StringRules_UriRef: + t := uint64(0) + if x.UriRef { + t = 1 + } + b.EncodeVarint(18<<3 | proto.WireVarint) + b.EncodeVarint(t) + case *StringRules_Address: + t := uint64(0) + if x.Address { + t = 1 + } + b.EncodeVarint(21<<3 | proto.WireVarint) + b.EncodeVarint(t) + case nil: + default: + return fmt.Errorf("StringRules.WellKnown has unexpected type %T", x) + } + return nil +} + +func _StringRules_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*StringRules) + switch tag { + case 12: // well_known.email + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.WellKnown = &StringRules_Email{x != 0} + return true, err + case 13: // well_known.hostname + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.WellKnown = &StringRules_Hostname{x != 0} + return true, err + case 14: // well_known.ip + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.WellKnown = &StringRules_Ip{x != 0} + return true, err + case 15: // well_known.ipv4 + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.WellKnown = &StringRules_Ipv4{x != 0} + return true, err + case 16: // well_known.ipv6 + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.WellKnown = &StringRules_Ipv6{x != 0} + return true, err + case 17: // well_known.uri + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.WellKnown = &StringRules_Uri{x != 0} + return true, err + case 18: // well_known.uri_ref + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.WellKnown = &StringRules_UriRef{x != 0} + return true, err + case 21: // well_known.address + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.WellKnown = &StringRules_Address{x != 0} + return true, err + default: + return false, nil + } +} + +func _StringRules_OneofSizer(msg proto.Message) (n int) { + m := msg.(*StringRules) + // well_known + switch x := m.WellKnown.(type) { + case *StringRules_Email: + n += 1 // tag and wire + n += 1 + case *StringRules_Hostname: + n += 1 // tag and wire + n += 1 + case *StringRules_Ip: + n += 1 // tag and wire + n += 1 + case *StringRules_Ipv4: + n += 1 // tag and wire + n += 1 + case *StringRules_Ipv6: + n += 2 // tag and wire + n += 1 + case *StringRules_Uri: + n += 2 // tag and wire + n += 1 + case *StringRules_UriRef: + n += 2 // tag and wire + n += 1 + case *StringRules_Address: + n += 2 // tag and wire + n += 1 + case nil: + default: + panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) + } + return n +} + +// BytesRules describe the constraints applied to `bytes` values +type BytesRules struct { + // Const specifies that this field must be exactly the specified value + Const []byte `protobuf:"bytes,1,opt,name=const" json:"const,omitempty"` + // Len specifies that this field must be the specified number of bytes + Len *uint64 `protobuf:"varint,13,opt,name=len" json:"len,omitempty"` + // MinLen specifies that this field must be the specified number of bytes + // at a minimum + MinLen *uint64 `protobuf:"varint,2,opt,name=min_len,json=minLen" json:"min_len,omitempty"` + // MaxLen specifies that this field must be the specified number of bytes + // at a maximum + MaxLen *uint64 `protobuf:"varint,3,opt,name=max_len,json=maxLen" json:"max_len,omitempty"` + // Pattern specifes that this field must match against the specified + // regular expression (RE2 syntax). The included expression should elide + // any delimiters. + Pattern *string `protobuf:"bytes,4,opt,name=pattern" json:"pattern,omitempty"` + // Prefix specifies that this field must have the specified bytes at the + // beginning of the string. + Prefix []byte `protobuf:"bytes,5,opt,name=prefix" json:"prefix,omitempty"` + // Suffix specifies that this field must have the specified bytes at the + // end of the string. + Suffix []byte `protobuf:"bytes,6,opt,name=suffix" json:"suffix,omitempty"` + // Contains specifies that this field must have the specified bytes + // anywhere in the string. + Contains []byte `protobuf:"bytes,7,opt,name=contains" json:"contains,omitempty"` + // In specifies that this field must be equal to one of the specified + // values + In [][]byte `protobuf:"bytes,8,rep,name=in" json:"in,omitempty"` + // NotIn specifies that this field cannot be equal to one of the specified + // values + NotIn [][]byte `protobuf:"bytes,9,rep,name=not_in,json=notIn" json:"not_in,omitempty"` + // WellKnown rules provide advanced constraints against common byte + // patterns + // + // Types that are valid to be assigned to WellKnown: + // *BytesRules_Ip + // *BytesRules_Ipv4 + // *BytesRules_Ipv6 + WellKnown isBytesRules_WellKnown `protobuf_oneof:"well_known"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *BytesRules) Reset() { *m = BytesRules{} } +func (m *BytesRules) String() string { return proto.CompactTextString(m) } +func (*BytesRules) ProtoMessage() {} +func (*BytesRules) Descriptor() ([]byte, []int) { + return fileDescriptor_validate_c8f5e113dd6422a8, []int{15} +} +func (m *BytesRules) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_BytesRules.Unmarshal(m, b) +} +func (m *BytesRules) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_BytesRules.Marshal(b, m, deterministic) +} +func (dst *BytesRules) XXX_Merge(src proto.Message) { + xxx_messageInfo_BytesRules.Merge(dst, src) +} +func (m *BytesRules) XXX_Size() int { + return xxx_messageInfo_BytesRules.Size(m) +} +func (m *BytesRules) XXX_DiscardUnknown() { + xxx_messageInfo_BytesRules.DiscardUnknown(m) +} + +var xxx_messageInfo_BytesRules proto.InternalMessageInfo + +func (m *BytesRules) GetConst() []byte { + if m != nil { + return m.Const + } + return nil +} + +func (m *BytesRules) GetLen() uint64 { + if m != nil && m.Len != nil { + return *m.Len + } + return 0 +} + +func (m *BytesRules) GetMinLen() uint64 { + if m != nil && m.MinLen != nil { + return *m.MinLen + } + return 0 +} + +func (m *BytesRules) GetMaxLen() uint64 { + if m != nil && m.MaxLen != nil { + return *m.MaxLen + } + return 0 +} + +func (m *BytesRules) GetPattern() string { + if m != nil && m.Pattern != nil { + return *m.Pattern + } + return "" +} + +func (m *BytesRules) GetPrefix() []byte { + if m != nil { + return m.Prefix + } + return nil +} + +func (m *BytesRules) GetSuffix() []byte { + if m != nil { + return m.Suffix + } + return nil +} + +func (m *BytesRules) GetContains() []byte { + if m != nil { + return m.Contains + } + return nil +} + +func (m *BytesRules) GetIn() [][]byte { + if m != nil { + return m.In + } + return nil +} + +func (m *BytesRules) GetNotIn() [][]byte { + if m != nil { + return m.NotIn + } + return nil +} + +type isBytesRules_WellKnown interface { + isBytesRules_WellKnown() +} + +type BytesRules_Ip struct { + Ip bool `protobuf:"varint,10,opt,name=ip,oneof"` +} + +type BytesRules_Ipv4 struct { + Ipv4 bool `protobuf:"varint,11,opt,name=ipv4,oneof"` +} + +type BytesRules_Ipv6 struct { + Ipv6 bool `protobuf:"varint,12,opt,name=ipv6,oneof"` +} + +func (*BytesRules_Ip) isBytesRules_WellKnown() {} + +func (*BytesRules_Ipv4) isBytesRules_WellKnown() {} + +func (*BytesRules_Ipv6) isBytesRules_WellKnown() {} + +func (m *BytesRules) GetWellKnown() isBytesRules_WellKnown { + if m != nil { + return m.WellKnown + } + return nil +} + +func (m *BytesRules) GetIp() bool { + if x, ok := m.GetWellKnown().(*BytesRules_Ip); ok { + return x.Ip + } + return false +} + +func (m *BytesRules) GetIpv4() bool { + if x, ok := m.GetWellKnown().(*BytesRules_Ipv4); ok { + return x.Ipv4 + } + return false +} + +func (m *BytesRules) GetIpv6() bool { + if x, ok := m.GetWellKnown().(*BytesRules_Ipv6); ok { + return x.Ipv6 + } + return false +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*BytesRules) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { + return _BytesRules_OneofMarshaler, _BytesRules_OneofUnmarshaler, _BytesRules_OneofSizer, []interface{}{ + (*BytesRules_Ip)(nil), + (*BytesRules_Ipv4)(nil), + (*BytesRules_Ipv6)(nil), + } +} + +func _BytesRules_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*BytesRules) + // well_known + switch x := m.WellKnown.(type) { + case *BytesRules_Ip: + t := uint64(0) + if x.Ip { + t = 1 + } + b.EncodeVarint(10<<3 | proto.WireVarint) + b.EncodeVarint(t) + case *BytesRules_Ipv4: + t := uint64(0) + if x.Ipv4 { + t = 1 + } + b.EncodeVarint(11<<3 | proto.WireVarint) + b.EncodeVarint(t) + case *BytesRules_Ipv6: + t := uint64(0) + if x.Ipv6 { + t = 1 + } + b.EncodeVarint(12<<3 | proto.WireVarint) + b.EncodeVarint(t) + case nil: + default: + return fmt.Errorf("BytesRules.WellKnown has unexpected type %T", x) + } + return nil +} + +func _BytesRules_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*BytesRules) + switch tag { + case 10: // well_known.ip + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.WellKnown = &BytesRules_Ip{x != 0} + return true, err + case 11: // well_known.ipv4 + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.WellKnown = &BytesRules_Ipv4{x != 0} + return true, err + case 12: // well_known.ipv6 + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.WellKnown = &BytesRules_Ipv6{x != 0} + return true, err + default: + return false, nil + } +} + +func _BytesRules_OneofSizer(msg proto.Message) (n int) { + m := msg.(*BytesRules) + // well_known + switch x := m.WellKnown.(type) { + case *BytesRules_Ip: + n += 1 // tag and wire + n += 1 + case *BytesRules_Ipv4: + n += 1 // tag and wire + n += 1 + case *BytesRules_Ipv6: + n += 1 // tag and wire + n += 1 + case nil: + default: + panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) + } + return n +} + +// EnumRules describe the constraints applied to enum values +type EnumRules struct { + // Const specifies that this field must be exactly the specified value + Const *int32 `protobuf:"varint,1,opt,name=const" json:"const,omitempty"` + // DefinedOnly specifies that this field must be only one of the defined + // values for this enum, failing on any undefined value. + DefinedOnly *bool `protobuf:"varint,2,opt,name=defined_only,json=definedOnly" json:"defined_only,omitempty"` + // In specifies that this field must be equal to one of the specified + // values + In []int32 `protobuf:"varint,3,rep,name=in" json:"in,omitempty"` + // NotIn specifies that this field cannot be equal to one of the specified + // values + NotIn []int32 `protobuf:"varint,4,rep,name=not_in,json=notIn" json:"not_in,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *EnumRules) Reset() { *m = EnumRules{} } +func (m *EnumRules) String() string { return proto.CompactTextString(m) } +func (*EnumRules) ProtoMessage() {} +func (*EnumRules) Descriptor() ([]byte, []int) { + return fileDescriptor_validate_c8f5e113dd6422a8, []int{16} +} +func (m *EnumRules) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_EnumRules.Unmarshal(m, b) +} +func (m *EnumRules) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_EnumRules.Marshal(b, m, deterministic) +} +func (dst *EnumRules) XXX_Merge(src proto.Message) { + xxx_messageInfo_EnumRules.Merge(dst, src) +} +func (m *EnumRules) XXX_Size() int { + return xxx_messageInfo_EnumRules.Size(m) +} +func (m *EnumRules) XXX_DiscardUnknown() { + xxx_messageInfo_EnumRules.DiscardUnknown(m) +} + +var xxx_messageInfo_EnumRules proto.InternalMessageInfo + +func (m *EnumRules) GetConst() int32 { + if m != nil && m.Const != nil { + return *m.Const + } + return 0 +} + +func (m *EnumRules) GetDefinedOnly() bool { + if m != nil && m.DefinedOnly != nil { + return *m.DefinedOnly + } + return false +} + +func (m *EnumRules) GetIn() []int32 { + if m != nil { + return m.In + } + return nil +} + +func (m *EnumRules) GetNotIn() []int32 { + if m != nil { + return m.NotIn + } + return nil +} + +// MessageRules describe the constraints applied to embedded message values. +// For message-type fields, validation is performed recursively. +type MessageRules struct { + // Skip specifies that the validation rules of this field should not be + // evaluated + Skip *bool `protobuf:"varint,1,opt,name=skip" json:"skip,omitempty"` + // Required specifies that this field must be set + Required *bool `protobuf:"varint,2,opt,name=required" json:"required,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *MessageRules) Reset() { *m = MessageRules{} } +func (m *MessageRules) String() string { return proto.CompactTextString(m) } +func (*MessageRules) ProtoMessage() {} +func (*MessageRules) Descriptor() ([]byte, []int) { + return fileDescriptor_validate_c8f5e113dd6422a8, []int{17} +} +func (m *MessageRules) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_MessageRules.Unmarshal(m, b) +} +func (m *MessageRules) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_MessageRules.Marshal(b, m, deterministic) +} +func (dst *MessageRules) XXX_Merge(src proto.Message) { + xxx_messageInfo_MessageRules.Merge(dst, src) +} +func (m *MessageRules) XXX_Size() int { + return xxx_messageInfo_MessageRules.Size(m) +} +func (m *MessageRules) XXX_DiscardUnknown() { + xxx_messageInfo_MessageRules.DiscardUnknown(m) +} + +var xxx_messageInfo_MessageRules proto.InternalMessageInfo + +func (m *MessageRules) GetSkip() bool { + if m != nil && m.Skip != nil { + return *m.Skip + } + return false +} + +func (m *MessageRules) GetRequired() bool { + if m != nil && m.Required != nil { + return *m.Required + } + return false +} + +// RepeatedRules describe the constraints applied to `repeated` values +type RepeatedRules struct { + // MinItems specifies that this field must have the specified number of + // items at a minimum + MinItems *uint64 `protobuf:"varint,1,opt,name=min_items,json=minItems" json:"min_items,omitempty"` + // MaxItems specifies that this field must have the specified number of + // items at a maximum + MaxItems *uint64 `protobuf:"varint,2,opt,name=max_items,json=maxItems" json:"max_items,omitempty"` + // Unique specifies that all elements in this field must be unique. This + // contraint is only applicable to scalar and enum types (messages are not + // supported). + Unique *bool `protobuf:"varint,3,opt,name=unique" json:"unique,omitempty"` + // Items specifies the contraints to be applied to each item in the field. + // Repeated message fields will still execute validation against each item + // unless skip is specified here. + Items *FieldRules `protobuf:"bytes,4,opt,name=items" json:"items,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *RepeatedRules) Reset() { *m = RepeatedRules{} } +func (m *RepeatedRules) String() string { return proto.CompactTextString(m) } +func (*RepeatedRules) ProtoMessage() {} +func (*RepeatedRules) Descriptor() ([]byte, []int) { + return fileDescriptor_validate_c8f5e113dd6422a8, []int{18} +} +func (m *RepeatedRules) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_RepeatedRules.Unmarshal(m, b) +} +func (m *RepeatedRules) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_RepeatedRules.Marshal(b, m, deterministic) +} +func (dst *RepeatedRules) XXX_Merge(src proto.Message) { + xxx_messageInfo_RepeatedRules.Merge(dst, src) +} +func (m *RepeatedRules) XXX_Size() int { + return xxx_messageInfo_RepeatedRules.Size(m) +} +func (m *RepeatedRules) XXX_DiscardUnknown() { + xxx_messageInfo_RepeatedRules.DiscardUnknown(m) +} + +var xxx_messageInfo_RepeatedRules proto.InternalMessageInfo + +func (m *RepeatedRules) GetMinItems() uint64 { + if m != nil && m.MinItems != nil { + return *m.MinItems + } + return 0 +} + +func (m *RepeatedRules) GetMaxItems() uint64 { + if m != nil && m.MaxItems != nil { + return *m.MaxItems + } + return 0 +} + +func (m *RepeatedRules) GetUnique() bool { + if m != nil && m.Unique != nil { + return *m.Unique + } + return false +} + +func (m *RepeatedRules) GetItems() *FieldRules { + if m != nil { + return m.Items + } + return nil +} + +// MapRules describe the constraints applied to `map` values +type MapRules struct { + // MinPairs specifies that this field must have the specified number of + // KVs at a minimum + MinPairs *uint64 `protobuf:"varint,1,opt,name=min_pairs,json=minPairs" json:"min_pairs,omitempty"` + // MaxPairs specifies that this field must have the specified number of + // KVs at a maximum + MaxPairs *uint64 `protobuf:"varint,2,opt,name=max_pairs,json=maxPairs" json:"max_pairs,omitempty"` + // NoSparse specifies values in this field cannot be unset. This only + // applies to map's with message value types. + NoSparse *bool `protobuf:"varint,3,opt,name=no_sparse,json=noSparse" json:"no_sparse,omitempty"` + // Keys specifies the constraints to be applied to each key in the field. + Keys *FieldRules `protobuf:"bytes,4,opt,name=keys" json:"keys,omitempty"` + // Values specifies the constraints to be applied to the value of each key + // in the field. Message values will still have their validations evaluated + // unless skip is specified here. + Values *FieldRules `protobuf:"bytes,5,opt,name=values" json:"values,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *MapRules) Reset() { *m = MapRules{} } +func (m *MapRules) String() string { return proto.CompactTextString(m) } +func (*MapRules) ProtoMessage() {} +func (*MapRules) Descriptor() ([]byte, []int) { + return fileDescriptor_validate_c8f5e113dd6422a8, []int{19} +} +func (m *MapRules) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_MapRules.Unmarshal(m, b) +} +func (m *MapRules) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_MapRules.Marshal(b, m, deterministic) +} +func (dst *MapRules) XXX_Merge(src proto.Message) { + xxx_messageInfo_MapRules.Merge(dst, src) +} +func (m *MapRules) XXX_Size() int { + return xxx_messageInfo_MapRules.Size(m) +} +func (m *MapRules) XXX_DiscardUnknown() { + xxx_messageInfo_MapRules.DiscardUnknown(m) +} + +var xxx_messageInfo_MapRules proto.InternalMessageInfo + +func (m *MapRules) GetMinPairs() uint64 { + if m != nil && m.MinPairs != nil { + return *m.MinPairs + } + return 0 +} + +func (m *MapRules) GetMaxPairs() uint64 { + if m != nil && m.MaxPairs != nil { + return *m.MaxPairs + } + return 0 +} + +func (m *MapRules) GetNoSparse() bool { + if m != nil && m.NoSparse != nil { + return *m.NoSparse + } + return false +} + +func (m *MapRules) GetKeys() *FieldRules { + if m != nil { + return m.Keys + } + return nil +} + +func (m *MapRules) GetValues() *FieldRules { + if m != nil { + return m.Values + } + return nil +} + +// AnyRules describe constraints applied exclusively to the +// `google.protobuf.Any` well-known type +type AnyRules struct { + // Required specifies that this field must be set + Required *bool `protobuf:"varint,1,opt,name=required" json:"required,omitempty"` + // In specifies that this field's `type_url` must be equal to one of the + // specified values. + In []string `protobuf:"bytes,2,rep,name=in" json:"in,omitempty"` + // NotIn specifies that this field's `type_url` must not be equal to any of + // the specified values. + NotIn []string `protobuf:"bytes,3,rep,name=not_in,json=notIn" json:"not_in,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *AnyRules) Reset() { *m = AnyRules{} } +func (m *AnyRules) String() string { return proto.CompactTextString(m) } +func (*AnyRules) ProtoMessage() {} +func (*AnyRules) Descriptor() ([]byte, []int) { + return fileDescriptor_validate_c8f5e113dd6422a8, []int{20} +} +func (m *AnyRules) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_AnyRules.Unmarshal(m, b) +} +func (m *AnyRules) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_AnyRules.Marshal(b, m, deterministic) +} +func (dst *AnyRules) XXX_Merge(src proto.Message) { + xxx_messageInfo_AnyRules.Merge(dst, src) +} +func (m *AnyRules) XXX_Size() int { + return xxx_messageInfo_AnyRules.Size(m) +} +func (m *AnyRules) XXX_DiscardUnknown() { + xxx_messageInfo_AnyRules.DiscardUnknown(m) +} + +var xxx_messageInfo_AnyRules proto.InternalMessageInfo + +func (m *AnyRules) GetRequired() bool { + if m != nil && m.Required != nil { + return *m.Required + } + return false +} + +func (m *AnyRules) GetIn() []string { + if m != nil { + return m.In + } + return nil +} + +func (m *AnyRules) GetNotIn() []string { + if m != nil { + return m.NotIn + } + return nil +} + +// DurationRules describe the constraints applied exclusively to the +// `google.protobuf.Duration` well-known type +type DurationRules struct { + // Required specifies that this field must be set + Required *bool `protobuf:"varint,1,opt,name=required" json:"required,omitempty"` + // Const specifies that this field must be exactly the specified value + Const *duration.Duration `protobuf:"bytes,2,opt,name=const" json:"const,omitempty"` + // Lt specifies that this field must be less than the specified value, + // exclusive + Lt *duration.Duration `protobuf:"bytes,3,opt,name=lt" json:"lt,omitempty"` + // Lt specifies that this field must be less than the specified value, + // inclusive + Lte *duration.Duration `protobuf:"bytes,4,opt,name=lte" json:"lte,omitempty"` + // Gt specifies that this field must be greater than the specified value, + // exclusive + Gt *duration.Duration `protobuf:"bytes,5,opt,name=gt" json:"gt,omitempty"` + // Gte specifies that this field must be greater than the specified value, + // inclusive + Gte *duration.Duration `protobuf:"bytes,6,opt,name=gte" json:"gte,omitempty"` + // In specifies that this field must be equal to one of the specified + // values + In []*duration.Duration `protobuf:"bytes,7,rep,name=in" json:"in,omitempty"` + // NotIn specifies that this field cannot be equal to one of the specified + // values + NotIn []*duration.Duration `protobuf:"bytes,8,rep,name=not_in,json=notIn" json:"not_in,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DurationRules) Reset() { *m = DurationRules{} } +func (m *DurationRules) String() string { return proto.CompactTextString(m) } +func (*DurationRules) ProtoMessage() {} +func (*DurationRules) Descriptor() ([]byte, []int) { + return fileDescriptor_validate_c8f5e113dd6422a8, []int{21} +} +func (m *DurationRules) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DurationRules.Unmarshal(m, b) +} +func (m *DurationRules) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DurationRules.Marshal(b, m, deterministic) +} +func (dst *DurationRules) XXX_Merge(src proto.Message) { + xxx_messageInfo_DurationRules.Merge(dst, src) +} +func (m *DurationRules) XXX_Size() int { + return xxx_messageInfo_DurationRules.Size(m) +} +func (m *DurationRules) XXX_DiscardUnknown() { + xxx_messageInfo_DurationRules.DiscardUnknown(m) +} + +var xxx_messageInfo_DurationRules proto.InternalMessageInfo + +func (m *DurationRules) GetRequired() bool { + if m != nil && m.Required != nil { + return *m.Required + } + return false +} + +func (m *DurationRules) GetConst() *duration.Duration { + if m != nil { + return m.Const + } + return nil +} + +func (m *DurationRules) GetLt() *duration.Duration { + if m != nil { + return m.Lt + } + return nil +} + +func (m *DurationRules) GetLte() *duration.Duration { + if m != nil { + return m.Lte + } + return nil +} + +func (m *DurationRules) GetGt() *duration.Duration { + if m != nil { + return m.Gt + } + return nil +} + +func (m *DurationRules) GetGte() *duration.Duration { + if m != nil { + return m.Gte + } + return nil +} + +func (m *DurationRules) GetIn() []*duration.Duration { + if m != nil { + return m.In + } + return nil +} + +func (m *DurationRules) GetNotIn() []*duration.Duration { + if m != nil { + return m.NotIn + } + return nil +} + +// TimestampRules describe the constraints applied exclusively to the +// `google.protobuf.Timestamp` well-known type +type TimestampRules struct { + // Required specifies that this field must be set + Required *bool `protobuf:"varint,1,opt,name=required" json:"required,omitempty"` + // Const specifies that this field must be exactly the specified value + Const *timestamp.Timestamp `protobuf:"bytes,2,opt,name=const" json:"const,omitempty"` + // Lt specifies that this field must be less than the specified value, + // exclusive + Lt *timestamp.Timestamp `protobuf:"bytes,3,opt,name=lt" json:"lt,omitempty"` + // Lte specifies that this field must be less than the specified value, + // inclusive + Lte *timestamp.Timestamp `protobuf:"bytes,4,opt,name=lte" json:"lte,omitempty"` + // Gt specifies that this field must be greater than the specified value, + // exclusive + Gt *timestamp.Timestamp `protobuf:"bytes,5,opt,name=gt" json:"gt,omitempty"` + // Gte specifies that this field must be greater than the specified value, + // inclusive + Gte *timestamp.Timestamp `protobuf:"bytes,6,opt,name=gte" json:"gte,omitempty"` + // LtNow specifies that this must be less than the current time. LtNow + // can only be used with the Within rule. + LtNow *bool `protobuf:"varint,7,opt,name=lt_now,json=ltNow" json:"lt_now,omitempty"` + // GtNow specifies that this must be greater than the current time. GtNow + // can only be used with the Within rule. + GtNow *bool `protobuf:"varint,8,opt,name=gt_now,json=gtNow" json:"gt_now,omitempty"` + // Within specifies that this field must be within this duration of the + // current time. This constraint can be used alone or with the LtNow and + // GtNow rules. + Within *duration.Duration `protobuf:"bytes,9,opt,name=within" json:"within,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *TimestampRules) Reset() { *m = TimestampRules{} } +func (m *TimestampRules) String() string { return proto.CompactTextString(m) } +func (*TimestampRules) ProtoMessage() {} +func (*TimestampRules) Descriptor() ([]byte, []int) { + return fileDescriptor_validate_c8f5e113dd6422a8, []int{22} +} +func (m *TimestampRules) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_TimestampRules.Unmarshal(m, b) +} +func (m *TimestampRules) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_TimestampRules.Marshal(b, m, deterministic) +} +func (dst *TimestampRules) XXX_Merge(src proto.Message) { + xxx_messageInfo_TimestampRules.Merge(dst, src) +} +func (m *TimestampRules) XXX_Size() int { + return xxx_messageInfo_TimestampRules.Size(m) +} +func (m *TimestampRules) XXX_DiscardUnknown() { + xxx_messageInfo_TimestampRules.DiscardUnknown(m) +} + +var xxx_messageInfo_TimestampRules proto.InternalMessageInfo + +func (m *TimestampRules) GetRequired() bool { + if m != nil && m.Required != nil { + return *m.Required + } + return false +} + +func (m *TimestampRules) GetConst() *timestamp.Timestamp { + if m != nil { + return m.Const + } + return nil +} + +func (m *TimestampRules) GetLt() *timestamp.Timestamp { + if m != nil { + return m.Lt + } + return nil +} + +func (m *TimestampRules) GetLte() *timestamp.Timestamp { + if m != nil { + return m.Lte + } + return nil +} + +func (m *TimestampRules) GetGt() *timestamp.Timestamp { + if m != nil { + return m.Gt + } + return nil +} + +func (m *TimestampRules) GetGte() *timestamp.Timestamp { + if m != nil { + return m.Gte + } + return nil +} + +func (m *TimestampRules) GetLtNow() bool { + if m != nil && m.LtNow != nil { + return *m.LtNow + } + return false +} + +func (m *TimestampRules) GetGtNow() bool { + if m != nil && m.GtNow != nil { + return *m.GtNow + } + return false +} + +func (m *TimestampRules) GetWithin() *duration.Duration { + if m != nil { + return m.Within + } + return nil +} + +var E_Disabled = &proto.ExtensionDesc{ + ExtendedType: (*descriptor.MessageOptions)(nil), + ExtensionType: (*bool)(nil), + Field: 919191, + Name: "validate.disabled", + Tag: "varint,919191,opt,name=disabled", + Filename: "validate/validate.proto", +} + +var E_Required = &proto.ExtensionDesc{ + ExtendedType: (*descriptor.OneofOptions)(nil), + ExtensionType: (*bool)(nil), + Field: 919191, + Name: "validate.required", + Tag: "varint,919191,opt,name=required", + Filename: "validate/validate.proto", +} + +var E_Rules = &proto.ExtensionDesc{ + ExtendedType: (*descriptor.FieldOptions)(nil), + ExtensionType: (*FieldRules)(nil), + Field: 919191, + Name: "validate.rules", + Tag: "bytes,919191,opt,name=rules", + Filename: "validate/validate.proto", +} + +func init() { + proto.RegisterType((*FieldRules)(nil), "validate.FieldRules") + proto.RegisterType((*FloatRules)(nil), "validate.FloatRules") + proto.RegisterType((*DoubleRules)(nil), "validate.DoubleRules") + proto.RegisterType((*Int32Rules)(nil), "validate.Int32Rules") + proto.RegisterType((*Int64Rules)(nil), "validate.Int64Rules") + proto.RegisterType((*UInt32Rules)(nil), "validate.UInt32Rules") + proto.RegisterType((*UInt64Rules)(nil), "validate.UInt64Rules") + proto.RegisterType((*SInt32Rules)(nil), "validate.SInt32Rules") + proto.RegisterType((*SInt64Rules)(nil), "validate.SInt64Rules") + proto.RegisterType((*Fixed32Rules)(nil), "validate.Fixed32Rules") + proto.RegisterType((*Fixed64Rules)(nil), "validate.Fixed64Rules") + proto.RegisterType((*SFixed32Rules)(nil), "validate.SFixed32Rules") + proto.RegisterType((*SFixed64Rules)(nil), "validate.SFixed64Rules") + proto.RegisterType((*BoolRules)(nil), "validate.BoolRules") + proto.RegisterType((*StringRules)(nil), "validate.StringRules") + proto.RegisterType((*BytesRules)(nil), "validate.BytesRules") + proto.RegisterType((*EnumRules)(nil), "validate.EnumRules") + proto.RegisterType((*MessageRules)(nil), "validate.MessageRules") + proto.RegisterType((*RepeatedRules)(nil), "validate.RepeatedRules") + proto.RegisterType((*MapRules)(nil), "validate.MapRules") + proto.RegisterType((*AnyRules)(nil), "validate.AnyRules") + proto.RegisterType((*DurationRules)(nil), "validate.DurationRules") + proto.RegisterType((*TimestampRules)(nil), "validate.TimestampRules") + proto.RegisterExtension(E_Disabled) + proto.RegisterExtension(E_Required) + proto.RegisterExtension(E_Rules) +} + +func init() { proto.RegisterFile("validate/validate.proto", fileDescriptor_validate_c8f5e113dd6422a8) } + +var fileDescriptor_validate_c8f5e113dd6422a8 = []byte{ + // 1640 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x98, 0xcd, 0x6e, 0xdb, 0xc6, + 0x16, 0xc7, 0xaf, 0xf8, 0x25, 0x6a, 0x2c, 0x45, 0xd2, 0xc4, 0x76, 0x18, 0xdf, 0x8f, 0x38, 0x5a, + 0x5c, 0x38, 0xb9, 0x89, 0x9d, 0xeb, 0xb8, 0x42, 0x90, 0xa2, 0x05, 0x6a, 0xa4, 0x41, 0x83, 0x36, + 0x4d, 0x40, 0x37, 0x9b, 0x6e, 0x04, 0xda, 0x1a, 0x29, 0x03, 0x53, 0x43, 0x86, 0xa4, 0x6c, 0xeb, + 0x21, 0xd2, 0x76, 0xd7, 0xbe, 0x4a, 0x57, 0xdd, 0xf7, 0x4d, 0xba, 0xee, 0x0b, 0x14, 0xf3, 0xc5, + 0x8f, 0x43, 0x5a, 0x5e, 0x74, 0xa7, 0x39, 0xe7, 0x7f, 0x66, 0x7e, 0xf8, 0x0f, 0x39, 0x73, 0x28, + 0x74, 0xe7, 0x22, 0x08, 0xe9, 0x34, 0xc8, 0xc8, 0x81, 0xfe, 0xb1, 0x1f, 0x27, 0x51, 0x16, 0x61, + 0x57, 0x8f, 0x77, 0x76, 0xe7, 0x51, 0x34, 0x0f, 0xc9, 0x81, 0x88, 0x9f, 0x2e, 0x67, 0x07, 0x53, + 0x92, 0x9e, 0x25, 0x34, 0xce, 0xa2, 0x44, 0x6a, 0x77, 0xfe, 0x53, 0x53, 0x2c, 0x93, 0x20, 0xa3, + 0x11, 0x53, 0xf9, 0x7b, 0x30, 0x9f, 0xd1, 0x05, 0x49, 0xb3, 0x60, 0x11, 0x4b, 0xc1, 0xe8, 0x77, + 0x17, 0xa1, 0x97, 0x94, 0x84, 0x53, 0x7f, 0x19, 0x92, 0x14, 0x3f, 0x42, 0xf6, 0x2c, 0x8c, 0x82, + 0xcc, 0x6b, 0xed, 0xb6, 0xf6, 0x36, 0x0e, 0x37, 0xf7, 0x73, 0xb6, 0x97, 0x3c, 0x2c, 0x44, 0x5f, + 0xfd, 0xc3, 0x97, 0x22, 0x7c, 0x80, 0x9c, 0x69, 0xb4, 0x3c, 0x0d, 0x89, 0x67, 0x08, 0xf9, 0x56, + 0x21, 0x7f, 0x21, 0xe2, 0x5a, 0xaf, 0x64, 0x7c, 0x7a, 0xca, 0xb2, 0xa7, 0x87, 0x9e, 0x09, 0xa7, + 0x7f, 0xc5, 0xc3, 0xf9, 0xf4, 0x42, 0xa4, 0xd4, 0xe3, 0x23, 0xcf, 0x6a, 0x50, 0x8f, 0x8f, 0xca, + 0xea, 0xf1, 0x11, 0x87, 0x59, 0xca, 0xc9, 0x6d, 0x08, 0xf3, 0xae, 0x32, 0xbb, 0x92, 0xe9, 0x82, + 0xf1, 0x91, 0xe7, 0x34, 0x15, 0x14, 0x0b, 0x28, 0x19, 0x2f, 0x48, 0xe5, 0x0a, 0x6d, 0x58, 0x70, + 0x52, 0x5d, 0x21, 0xcd, 0x57, 0x48, 0xe5, 0x0a, 0x6e, 0x53, 0x41, 0x69, 0x05, 0x29, 0xc3, 0x87, + 0xa8, 0x3d, 0xa3, 0x57, 0x64, 0xfa, 0xf4, 0xd0, 0xeb, 0x88, 0x8a, 0xed, 0xd2, 0x06, 0xc8, 0x84, + 0x2e, 0xd1, 0xc2, 0xbc, 0x66, 0x7c, 0xe4, 0xa1, 0xc6, 0x9a, 0x62, 0x19, 0x2d, 0xc4, 0x9f, 0x20, + 0x37, 0xd5, 0x0b, 0x6d, 0x88, 0xa2, 0x3b, 0x25, 0x34, 0xb0, 0x52, 0x2e, 0x2d, 0xca, 0xc6, 0x47, + 0x5e, 0xb7, 0xb9, 0xac, 0x58, 0x2c, 0x97, 0xe2, 0x07, 0xc8, 0x3a, 0x8d, 0xa2, 0xd0, 0xeb, 0x89, + 0x92, 0xdb, 0x45, 0xc9, 0x71, 0x14, 0x85, 0x5a, 0x2e, 0x24, 0xc2, 0xb1, 0x2c, 0xa1, 0x6c, 0xee, + 0xdd, 0xaa, 0x39, 0x26, 0xe2, 0x85, 0x63, 0x62, 0xc8, 0x9f, 0x91, 0xd3, 0x55, 0x46, 0x52, 0xaf, + 0x0f, 0x9f, 0x91, 0x63, 0x1e, 0xce, 0x9f, 0x11, 0x21, 0xe2, 0x24, 0x84, 0x2d, 0x17, 0xde, 0x00, + 0x92, 0x7c, 0xc9, 0x96, 0x8b, 0x9c, 0x84, 0x4b, 0xb8, 0xad, 0x0b, 0x92, 0xa6, 0xc1, 0x9c, 0x78, + 0x43, 0x68, 0xeb, 0x6b, 0x99, 0xc8, 0x6d, 0x55, 0x42, 0xee, 0x4f, 0x42, 0x62, 0x12, 0x64, 0x64, + 0xea, 0x61, 0xe8, 0x8f, 0xaf, 0x32, 0xb9, 0x3f, 0x5a, 0x8a, 0xff, 0x8b, 0xcc, 0x45, 0x10, 0x7b, + 0xb7, 0x45, 0x05, 0x2e, 0x2d, 0x13, 0xc4, 0x5a, 0xcc, 0x05, 0x5c, 0x17, 0xb0, 0x95, 0xb7, 0x09, + 0x75, 0x5f, 0xb0, 0x55, 0xae, 0x0b, 0xd8, 0x8a, 0x63, 0xe8, 0x63, 0xc0, 0xdb, 0x82, 0x18, 0x2f, + 0x54, 0x26, 0xc7, 0xd0, 0x52, 0xfc, 0x0c, 0x75, 0xf2, 0xd3, 0xc1, 0xdb, 0x16, 0x75, 0x5e, 0x51, + 0xf7, 0x9d, 0x4e, 0xe9, 0xc2, 0x42, 0x7c, 0xec, 0x20, 0x2b, 0x5b, 0xc5, 0x64, 0xf4, 0xb1, 0x85, + 0x50, 0x71, 0x4e, 0xe0, 0x4d, 0x64, 0x9f, 0x45, 0x2c, 0x95, 0x87, 0x89, 0xe1, 0xcb, 0x01, 0xbe, + 0x85, 0x8c, 0x30, 0x13, 0x07, 0x86, 0xe1, 0x1b, 0x61, 0x86, 0x07, 0xc8, 0x0c, 0x33, 0x22, 0x4e, + 0x04, 0xc3, 0xe7, 0x3f, 0xb9, 0x62, 0x9e, 0x89, 0x97, 0xde, 0xf0, 0x8d, 0xb9, 0x50, 0xcc, 0x33, + 0x22, 0x5e, 0x6b, 0xc3, 0xe7, 0x3f, 0xb9, 0x82, 0x32, 0xcf, 0xd9, 0x35, 0xb9, 0x82, 0x32, 0xbc, + 0x85, 0x1c, 0x16, 0x65, 0x13, 0xca, 0xbc, 0xb6, 0x88, 0xd9, 0x2c, 0xca, 0x5e, 0xb1, 0xd1, 0x0f, + 0x2d, 0xb4, 0x51, 0x3a, 0x88, 0xaa, 0x40, 0xad, 0x3a, 0x50, 0x0b, 0x02, 0xb5, 0x20, 0x50, 0x0b, + 0x02, 0xb5, 0x20, 0x50, 0xab, 0x01, 0xa8, 0xa5, 0x81, 0xb8, 0x41, 0xc5, 0x49, 0x51, 0xe5, 0xb1, + 0xeb, 0x3c, 0x36, 0xe4, 0xb1, 0x21, 0x8f, 0x0d, 0x79, 0x6c, 0xc8, 0x63, 0x37, 0xf0, 0xd8, 0x80, + 0x47, 0xbd, 0xb4, 0x55, 0x1e, 0xb3, 0xce, 0x63, 0x42, 0x1e, 0x13, 0xf2, 0x98, 0x90, 0xc7, 0x84, + 0x3c, 0x66, 0x03, 0x8f, 0x59, 0xde, 0xb0, 0x77, 0xd7, 0x19, 0xd4, 0xab, 0x03, 0xf5, 0x20, 0x50, + 0x0f, 0x02, 0xf5, 0x20, 0x50, 0x0f, 0x02, 0xf5, 0x1a, 0x80, 0x7a, 0x10, 0xa8, 0xd1, 0x21, 0xab, + 0x0e, 0x64, 0x41, 0x20, 0x0b, 0x02, 0x59, 0x10, 0xc8, 0x82, 0x40, 0x56, 0x03, 0x90, 0x55, 0x06, + 0x3a, 0xb9, 0xce, 0xa1, 0x61, 0x1d, 0x68, 0x08, 0x81, 0x86, 0x10, 0x68, 0x08, 0x81, 0x86, 0x10, + 0x68, 0xd8, 0x00, 0x34, 0x84, 0x40, 0x8d, 0x0e, 0xe1, 0x3a, 0x10, 0x86, 0x40, 0x18, 0x02, 0x61, + 0x08, 0x84, 0x21, 0x10, 0x6e, 0x00, 0xc2, 0x1a, 0xe8, 0xc7, 0x16, 0xea, 0x96, 0x6f, 0xb0, 0x2a, + 0x51, 0xbb, 0x4e, 0xd4, 0x86, 0x44, 0x6d, 0x48, 0xd4, 0x86, 0x44, 0x6d, 0x48, 0xd4, 0x6e, 0x20, + 0x6a, 0xd7, 0x88, 0x1a, 0x3d, 0x72, 0xea, 0x44, 0x0e, 0x24, 0x72, 0x20, 0x91, 0x03, 0x89, 0x1c, + 0x48, 0xe4, 0x34, 0x10, 0x39, 0x9a, 0xe8, 0xa7, 0x16, 0xea, 0x9d, 0x5c, 0x6f, 0x52, 0xbf, 0x8e, + 0xd4, 0x87, 0x48, 0x7d, 0x88, 0xd4, 0x87, 0x48, 0x7d, 0x88, 0xd4, 0x6f, 0x40, 0xea, 0xd7, 0x91, + 0x1a, 0x5d, 0x1a, 0xd4, 0x91, 0x06, 0x10, 0x69, 0x00, 0x91, 0x06, 0x10, 0x69, 0x00, 0x91, 0x06, + 0x0d, 0x48, 0x03, 0x8d, 0x74, 0x1f, 0x75, 0xf2, 0x0e, 0xa5, 0x4a, 0xe3, 0x2a, 0x9a, 0xd1, 0x2f, + 0x16, 0xda, 0x28, 0x35, 0x26, 0x55, 0x55, 0x47, 0x33, 0x73, 0x46, 0xc2, 0xc4, 0x05, 0xcf, 0xcf, + 0x03, 0xc2, 0xf0, 0x1d, 0xd4, 0x5e, 0x50, 0x36, 0xe1, 0x51, 0x79, 0x6c, 0x38, 0x0b, 0xca, 0xbe, + 0x51, 0x89, 0xe0, 0x4a, 0x24, 0x4c, 0x95, 0x08, 0xae, 0x78, 0xe2, 0x9f, 0xa8, 0x13, 0x12, 0x36, + 0x91, 0xcd, 0xce, 0xa6, 0x48, 0xb9, 0x21, 0x61, 0xa2, 0xcb, 0xe1, 0x49, 0x3e, 0x9d, 0x4c, 0xca, + 0x53, 0xc6, 0x5d, 0xd0, 0x52, 0x32, 0xb8, 0x52, 0x49, 0x5b, 0x25, 0x83, 0x2b, 0x99, 0xf4, 0x50, + 0x3b, 0x0e, 0xb2, 0x8c, 0x24, 0x4c, 0x74, 0xc1, 0x1d, 0x5f, 0x0f, 0xf1, 0x36, 0x72, 0xe2, 0x84, + 0xcc, 0xe8, 0x95, 0xe8, 0x76, 0x3b, 0xbe, 0x1a, 0xf1, 0x78, 0xba, 0x9c, 0xf1, 0xb8, 0x2b, 0xe3, + 0x72, 0x84, 0x77, 0x90, 0x7b, 0x16, 0xb1, 0x2c, 0xa0, 0x2c, 0x15, 0xcd, 0x6b, 0xc7, 0xcf, 0xc7, + 0xca, 0x70, 0xb4, 0x6b, 0xee, 0x75, 0x80, 0xe1, 0x1b, 0x22, 0x26, 0x0d, 0xc7, 0xdb, 0xc8, 0x26, + 0x8b, 0x80, 0x86, 0xa2, 0xb9, 0x74, 0x79, 0xdb, 0x26, 0x86, 0xf8, 0x5f, 0xc8, 0x7d, 0x1f, 0xa5, + 0x19, 0x0b, 0x16, 0x44, 0x34, 0x91, 0x3c, 0x95, 0x47, 0xf0, 0x00, 0x19, 0x34, 0x16, 0xfd, 0x22, + 0x8f, 0x1b, 0x34, 0xc6, 0x9b, 0xc8, 0xa2, 0xf1, 0xc5, 0x91, 0xe8, 0x09, 0x79, 0x4c, 0x8c, 0x54, + 0x74, 0x2c, 0x9a, 0x3f, 0x1d, 0x1d, 0x63, 0x8c, 0xcc, 0x65, 0x42, 0x45, 0x8f, 0xc7, 0x83, 0x7c, + 0x80, 0xef, 0xa2, 0xf6, 0x32, 0xa1, 0x93, 0x84, 0xcc, 0x44, 0x1b, 0xe7, 0x8a, 0x6f, 0x80, 0x84, + 0xfa, 0x64, 0x86, 0x77, 0x50, 0x3b, 0x98, 0x4e, 0x13, 0x92, 0xa6, 0xa2, 0xb5, 0xe2, 0x29, 0x1d, + 0x38, 0xee, 0x22, 0x74, 0x49, 0xc2, 0x70, 0x72, 0xce, 0xa2, 0x4b, 0x36, 0xfa, 0xcd, 0x40, 0xa8, + 0xe8, 0x41, 0xab, 0x4f, 0x46, 0x17, 0x3c, 0x19, 0xbd, 0xbf, 0xf3, 0x64, 0x94, 0xb6, 0xd0, 0xba, + 0x6e, 0x0b, 0x6d, 0xb1, 0x68, 0x7d, 0x0b, 0x1d, 0x19, 0x6f, 0xd8, 0xc2, 0xb6, 0xc8, 0xc0, 0x2d, + 0x74, 0x77, 0xcd, 0xbd, 0x2e, 0xd8, 0xc2, 0x8e, 0x88, 0xa9, 0x2d, 0x94, 0x9b, 0x81, 0x1a, 0x36, + 0x63, 0xa3, 0x71, 0x33, 0xba, 0xe5, 0xcd, 0x00, 0x0e, 0x9e, 0xa3, 0x4e, 0xde, 0x97, 0x5f, 0xd3, + 0x2b, 0xdd, 0x47, 0xdd, 0x29, 0x99, 0x51, 0x46, 0xa6, 0x93, 0x88, 0x85, 0x2b, 0x61, 0x99, 0xeb, + 0x6f, 0xa8, 0xd8, 0x1b, 0x16, 0xae, 0x14, 0xb8, 0xd9, 0xd0, 0x0a, 0x59, 0xe5, 0x56, 0xe8, 0x73, + 0xd4, 0x2d, 0xb7, 0xf5, 0x18, 0x23, 0x2b, 0x3d, 0xa7, 0xb1, 0x7a, 0xdd, 0xc5, 0x6f, 0xee, 0x4f, + 0x42, 0x3e, 0x2c, 0x69, 0x42, 0xa6, 0x6a, 0xa5, 0x7c, 0xcc, 0x5b, 0xa9, 0x5e, 0xa5, 0xc5, 0xd7, + 0x2f, 0x25, 0xcd, 0xc8, 0x22, 0x55, 0xfd, 0x02, 0x7f, 0x29, 0x5f, 0xf1, 0xb1, 0x7e, 0x29, 0x65, + 0xd2, 0xc8, 0x5f, 0x4a, 0x99, 0xdc, 0x46, 0xce, 0x92, 0xd1, 0x0f, 0x4b, 0x79, 0xac, 0xb9, 0xbe, + 0x1a, 0xe1, 0x87, 0xc8, 0x96, 0x05, 0xb5, 0x0f, 0xe2, 0xe2, 0x13, 0xde, 0x97, 0x92, 0xd1, 0xaf, + 0x2d, 0xe4, 0xea, 0x0f, 0x08, 0x8d, 0x12, 0x07, 0x34, 0x29, 0xa3, 0xbc, 0xe5, 0x63, 0x8d, 0x22, + 0x93, 0x05, 0x4a, 0x9e, 0x64, 0xd1, 0x24, 0x8d, 0x83, 0x24, 0xd5, 0x34, 0x2e, 0x8b, 0x4e, 0xc4, + 0x18, 0xef, 0x21, 0xeb, 0x9c, 0xac, 0xd6, 0xe3, 0x08, 0x05, 0x7e, 0x84, 0x9c, 0x8b, 0x20, 0x5c, + 0xaa, 0x03, 0xe8, 0x3a, 0xad, 0xd2, 0x8c, 0x5e, 0x23, 0x57, 0x7f, 0xd3, 0x54, 0x3c, 0x6f, 0x55, + 0x3d, 0x57, 0x5b, 0x6b, 0x34, 0x1c, 0x2b, 0x66, 0xe9, 0x58, 0x19, 0xfd, 0x61, 0xa0, 0x5e, 0xe5, + 0xb3, 0x67, 0xed, 0xa4, 0x07, 0xfa, 0x41, 0x93, 0xff, 0x69, 0xdc, 0xdd, 0x97, 0x7f, 0xa1, 0xec, + 0xeb, 0xbf, 0x50, 0x8a, 0x2f, 0x28, 0xf5, 0x0c, 0x3e, 0x10, 0x37, 0x92, 0x79, 0x93, 0x9a, 0x5f, + 0x56, 0xff, 0x93, 0x97, 0x95, 0x75, 0x93, 0x56, 0xdc, 0x63, 0x0f, 0xc4, 0x3d, 0x66, 0xdf, 0x38, + 0xef, 0x5c, 0xcc, 0xcb, 0xaf, 0x38, 0xe7, 0xc6, 0x79, 0xe7, 0x72, 0x5e, 0x75, 0xd3, 0xad, 0x9f, + 0x97, 0x32, 0xfc, 0x24, 0x37, 0xd4, 0xbd, 0x49, 0xae, 0xbc, 0xfe, 0xd3, 0x40, 0xb7, 0xaa, 0x9f, + 0x8a, 0x6b, 0xcd, 0x7e, 0x52, 0x35, 0x7b, 0xa7, 0x36, 0x7f, 0x31, 0x97, 0x72, 0xfb, 0x61, 0xc9, + 0xed, 0x75, 0x72, 0x6e, 0xf7, 0xa3, 0xb2, 0xdd, 0xeb, 0xc4, 0xc2, 0xef, 0x87, 0x25, 0xbf, 0xd7, + 0xce, 0x3c, 0x17, 0x33, 0x17, 0x86, 0xaf, 0x9d, 0x99, 0x3b, 0xbe, 0x85, 0x9c, 0x30, 0x9b, 0xb0, + 0xe8, 0x52, 0x9c, 0xaa, 0xae, 0x6f, 0x87, 0xd9, 0xb7, 0xd1, 0x25, 0x0f, 0xcf, 0x65, 0xd8, 0x95, + 0xe1, 0xb9, 0x08, 0xff, 0x1f, 0x39, 0x97, 0x34, 0x7b, 0x2f, 0x4e, 0xd6, 0x1b, 0xf6, 0x53, 0x09, + 0x9f, 0x7f, 0x86, 0xdc, 0x29, 0x4d, 0x83, 0xd3, 0x90, 0x4c, 0xf1, 0xbd, 0x9a, 0x5c, 0x9d, 0x6b, + 0x6f, 0x62, 0x5e, 0x93, 0x7a, 0x3f, 0x7f, 0x7c, 0x26, 0x77, 0x41, 0x97, 0x3c, 0xff, 0xb4, 0xd8, + 0x21, 0xfc, 0xef, 0x5a, 0xf9, 0x1b, 0x46, 0xa2, 0x59, 0xad, 0x58, 0x17, 0x3c, 0xff, 0x1a, 0xd9, + 0x89, 0xd8, 0xe7, 0x7a, 0xa5, 0x78, 0xb5, 0xab, 0x95, 0xd7, 0x9e, 0x5a, 0x62, 0x8e, 0xe3, 0xb7, + 0x68, 0x87, 0x46, 0xfb, 0x84, 0x5d, 0x44, 0xab, 0x38, 0x89, 0xae, 0x56, 0xfb, 0xf1, 0xfc, 0x22, + 0xd7, 0x7f, 0x7f, 0x38, 0xa7, 0xd9, 0xfb, 0xe5, 0xe9, 0xfe, 0x59, 0xb4, 0x38, 0x28, 0x34, 0xf2, + 0xcf, 0xcd, 0xb3, 0xc7, 0x73, 0xc2, 0x1e, 0xd7, 0xfe, 0x53, 0xfd, 0x2b, 0x00, 0x00, 0xff, 0xff, + 0xca, 0xcb, 0x15, 0xa8, 0x67, 0x15, 0x00, 0x00, +} diff --git a/github.com/envoyproxy/protoc-gen-validate/validate/validate.proto b/github.com/envoyproxy/protoc-gen-validate/validate/validate.proto new file mode 100644 index 0000000000..e50286e227 --- /dev/null +++ b/github.com/envoyproxy/protoc-gen-validate/validate/validate.proto @@ -0,0 +1,768 @@ +syntax = "proto2"; +package validate; + +option go_package = "github.com/envoyproxy/protoc-gen-validate/validate"; +option java_package = "io.envoyproxy.pgv.validate"; + +import "google/protobuf/descriptor.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; + +// Validation rules applied at the message level +extend google.protobuf.MessageOptions { + // Disabled nullifies any validation rules for this message, including any + // message fields associated with it that do support validation. + optional bool disabled = 919191; +} + +// Validation rules applied at the oneof level +extend google.protobuf.OneofOptions { + // Required ensures that exactly one the field options in a oneof is set; + // validation fails if no fields in the oneof are set. + optional bool required = 919191; +} + +// Validation rules applied at the field level +extend google.protobuf.FieldOptions { + // Rules specify the validations to be performed on this field. By default, + // no validation is performed against a field. + optional FieldRules rules = 919191; +} + +// FieldRules encapsulates the rules for each type of field. Depending on the +// field, the correct set should be used to ensure proper validations. +message FieldRules { + oneof type { + // Scalar Field Types + FloatRules float = 1; + DoubleRules double = 2; + Int32Rules int32 = 3; + Int64Rules int64 = 4; + UInt32Rules uint32 = 5; + UInt64Rules uint64 = 6; + SInt32Rules sint32 = 7; + SInt64Rules sint64 = 8; + Fixed32Rules fixed32 = 9; + Fixed64Rules fixed64 = 10; + SFixed32Rules sfixed32 = 11; + SFixed64Rules sfixed64 = 12; + BoolRules bool = 13; + StringRules string = 14; + BytesRules bytes = 15; + + // Complex Field Types + EnumRules enum = 16; + MessageRules message = 17; + RepeatedRules repeated = 18; + MapRules map = 19; + + // Well-Known Field Types + AnyRules any = 20; + DurationRules duration = 21; + TimestampRules timestamp = 22; + } +} + +// FloatRules describes the constraints applied to `float` values +message FloatRules { + // Const specifies that this field must be exactly the specified value + optional float const = 1; + + // Lt specifies that this field must be less than the specified value, + // exclusive + optional float lt = 2; + + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + optional float lte = 3; + + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + optional float gt = 4; + + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + optional float gte = 5; + + // In specifies that this field must be equal to one of the specified + // values + repeated float in = 6; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated float not_in = 7; +} + +// DoubleRules describes the constraints applied to `double` values +message DoubleRules { + // Const specifies that this field must be exactly the specified value + optional double const = 1; + + // Lt specifies that this field must be less than the specified value, + // exclusive + optional double lt = 2; + + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + optional double lte = 3; + + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + optional double gt = 4; + + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + optional double gte = 5; + + // In specifies that this field must be equal to one of the specified + // values + repeated double in = 6; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated double not_in = 7; +} + +// Int32Rules describes the constraints applied to `int32` values +message Int32Rules { + // Const specifies that this field must be exactly the specified value + optional int32 const = 1; + + // Lt specifies that this field must be less than the specified value, + // exclusive + optional int32 lt = 2; + + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + optional int32 lte = 3; + + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + optional int32 gt = 4; + + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + optional int32 gte = 5; + + // In specifies that this field must be equal to one of the specified + // values + repeated int32 in = 6; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated int32 not_in = 7; +} + +// Int64Rules describes the constraints applied to `int64` values +message Int64Rules { + // Const specifies that this field must be exactly the specified value + optional int64 const = 1; + + // Lt specifies that this field must be less than the specified value, + // exclusive + optional int64 lt = 2; + + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + optional int64 lte = 3; + + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + optional int64 gt = 4; + + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + optional int64 gte = 5; + + // In specifies that this field must be equal to one of the specified + // values + repeated int64 in = 6; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated int64 not_in = 7; +} + +// UInt32Rules describes the constraints applied to `uint32` values +message UInt32Rules { + // Const specifies that this field must be exactly the specified value + optional uint32 const = 1; + + // Lt specifies that this field must be less than the specified value, + // exclusive + optional uint32 lt = 2; + + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + optional uint32 lte = 3; + + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + optional uint32 gt = 4; + + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + optional uint32 gte = 5; + + // In specifies that this field must be equal to one of the specified + // values + repeated uint32 in = 6; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated uint32 not_in = 7; +} + +// UInt64Rules describes the constraints applied to `uint64` values +message UInt64Rules { + // Const specifies that this field must be exactly the specified value + optional uint64 const = 1; + + // Lt specifies that this field must be less than the specified value, + // exclusive + optional uint64 lt = 2; + + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + optional uint64 lte = 3; + + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + optional uint64 gt = 4; + + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + optional uint64 gte = 5; + + // In specifies that this field must be equal to one of the specified + // values + repeated uint64 in = 6; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated uint64 not_in = 7; +} + +// SInt32Rules describes the constraints applied to `sint32` values +message SInt32Rules { + // Const specifies that this field must be exactly the specified value + optional sint32 const = 1; + + // Lt specifies that this field must be less than the specified value, + // exclusive + optional sint32 lt = 2; + + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + optional sint32 lte = 3; + + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + optional sint32 gt = 4; + + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + optional sint32 gte = 5; + + // In specifies that this field must be equal to one of the specified + // values + repeated sint32 in = 6; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated sint32 not_in = 7; +} + +// SInt64Rules describes the constraints applied to `sint64` values +message SInt64Rules { + // Const specifies that this field must be exactly the specified value + optional sint64 const = 1; + + // Lt specifies that this field must be less than the specified value, + // exclusive + optional sint64 lt = 2; + + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + optional sint64 lte = 3; + + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + optional sint64 gt = 4; + + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + optional sint64 gte = 5; + + // In specifies that this field must be equal to one of the specified + // values + repeated sint64 in = 6; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated sint64 not_in = 7; +} + +// Fixed32Rules describes the constraints applied to `fixed32` values +message Fixed32Rules { + // Const specifies that this field must be exactly the specified value + optional fixed32 const = 1; + + // Lt specifies that this field must be less than the specified value, + // exclusive + optional fixed32 lt = 2; + + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + optional fixed32 lte = 3; + + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + optional fixed32 gt = 4; + + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + optional fixed32 gte = 5; + + // In specifies that this field must be equal to one of the specified + // values + repeated fixed32 in = 6; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated fixed32 not_in = 7; +} + +// Fixed64Rules describes the constraints applied to `fixed64` values +message Fixed64Rules { + // Const specifies that this field must be exactly the specified value + optional fixed64 const = 1; + + // Lt specifies that this field must be less than the specified value, + // exclusive + optional fixed64 lt = 2; + + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + optional fixed64 lte = 3; + + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + optional fixed64 gt = 4; + + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + optional fixed64 gte = 5; + + // In specifies that this field must be equal to one of the specified + // values + repeated fixed64 in = 6; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated fixed64 not_in = 7; +} + +// SFixed32Rules describes the constraints applied to `sfixed32` values +message SFixed32Rules { + // Const specifies that this field must be exactly the specified value + optional sfixed32 const = 1; + + // Lt specifies that this field must be less than the specified value, + // exclusive + optional sfixed32 lt = 2; + + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + optional sfixed32 lte = 3; + + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + optional sfixed32 gt = 4; + + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + optional sfixed32 gte = 5; + + // In specifies that this field must be equal to one of the specified + // values + repeated sfixed32 in = 6; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated sfixed32 not_in = 7; +} + +// SFixed64Rules describes the constraints applied to `sfixed64` values +message SFixed64Rules { + // Const specifies that this field must be exactly the specified value + optional sfixed64 const = 1; + + // Lt specifies that this field must be less than the specified value, + // exclusive + optional sfixed64 lt = 2; + + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + optional sfixed64 lte = 3; + + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + optional sfixed64 gt = 4; + + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + optional sfixed64 gte = 5; + + // In specifies that this field must be equal to one of the specified + // values + repeated sfixed64 in = 6; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated sfixed64 not_in = 7; +} + +// BoolRules describes the constraints applied to `bool` values +message BoolRules { + // Const specifies that this field must be exactly the specified value + optional bool const = 1; +} + +// StringRules describe the constraints applied to `string` values +message StringRules { + // Const specifies that this field must be exactly the specified value + optional string const = 1; + + // Len specifies that this field must be the specified number of + // characters (Unicode code points). Note that the number of + // characters may differ from the number of bytes in the string. + optional uint64 len = 19; + + // MinLen specifies that this field must be the specified number of + // characters (Unicode code points) at a minimum. Note that the number of + // characters may differ from the number of bytes in the string. + optional uint64 min_len = 2; + + // MaxLen specifies that this field must be the specified number of + // characters (Unicode code points) at a maximum. Note that the number of + // characters may differ from the number of bytes in the string. + optional uint64 max_len = 3; + + // LenBytes specifies that this field must be the specified number of bytes + // at a minimum + optional uint64 len_bytes = 20; + + // MinBytes specifies that this field must be the specified number of bytes + // at a minimum + optional uint64 min_bytes = 4; + + // MaxBytes specifies that this field must be the specified number of bytes + // at a maximum + optional uint64 max_bytes = 5; + + // Pattern specifes that this field must match against the specified + // regular expression (RE2 syntax). The included expression should elide + // any delimiters. + optional string pattern = 6; + + // Prefix specifies that this field must have the specified substring at + // the beginning of the string. + optional string prefix = 7; + + // Suffix specifies that this field must have the specified substring at + // the end of the string. + optional string suffix = 8; + + // Contains specifies that this field must have the specified substring + // anywhere in the string. + optional string contains = 9; + + // In specifies that this field must be equal to one of the specified + // values + repeated string in = 10; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated string not_in = 11; + + // WellKnown rules provide advanced constraints against common string + // patterns + oneof well_known { + // Email specifies that the field must be a valid email address as + // defined by RFC 5322 + bool email = 12; + + // Hostname specifies that the field must be a valid hostname as + // defined by RFC 1034. This constraint does not support + // internationalized domain names (IDNs). + bool hostname = 13; + + // Ip specifies that the field must be a valid IP (v4 or v6) address. + // Valid IPv6 addresses should not include surrounding square brackets. + bool ip = 14; + + // Ipv4 specifies that the field must be a valid IPv4 address. + bool ipv4 = 15; + + // Ipv6 specifies that the field must be a valid IPv6 address. Valid + // IPv6 addresses should not include surrounding square brackets. + bool ipv6 = 16; + + // Uri specifies that the field must be a valid, absolute URI as defined + // by RFC 3986 + bool uri = 17; + + // UriRef specifies that the field must be a valid URI as defined by RFC + // 3986 and may be relative or absolute. + bool uri_ref = 18; + + // Address specifies that the field must be either a valid hostname as + // defined by RFC 1034 (which does not support internationalized domain + // names or IDNs), or it can be a valid IP (v4 or v6). + bool address = 21; + } +} + +// BytesRules describe the constraints applied to `bytes` values +message BytesRules { + // Const specifies that this field must be exactly the specified value + optional bytes const = 1; + + // Len specifies that this field must be the specified number of bytes + optional uint64 len = 13; + + // MinLen specifies that this field must be the specified number of bytes + // at a minimum + optional uint64 min_len = 2; + + // MaxLen specifies that this field must be the specified number of bytes + // at a maximum + optional uint64 max_len = 3; + + // Pattern specifes that this field must match against the specified + // regular expression (RE2 syntax). The included expression should elide + // any delimiters. + optional string pattern = 4; + + // Prefix specifies that this field must have the specified bytes at the + // beginning of the string. + optional bytes prefix = 5; + + // Suffix specifies that this field must have the specified bytes at the + // end of the string. + optional bytes suffix = 6; + + // Contains specifies that this field must have the specified bytes + // anywhere in the string. + optional bytes contains = 7; + + // In specifies that this field must be equal to one of the specified + // values + repeated bytes in = 8; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated bytes not_in = 9; + + // WellKnown rules provide advanced constraints against common byte + // patterns + oneof well_known { + // Ip specifies that the field must be a valid IP (v4 or v6) address in + // byte format + bool ip = 10; + + // Ipv4 specifies that the field must be a valid IPv4 address in byte + // format + bool ipv4 = 11; + + // Ipv6 specifies that the field must be a valid IPv6 address in byte + // format + bool ipv6 = 12; + } +} + +// EnumRules describe the constraints applied to enum values +message EnumRules { + // Const specifies that this field must be exactly the specified value + optional int32 const = 1; + + // DefinedOnly specifies that this field must be only one of the defined + // values for this enum, failing on any undefined value. + optional bool defined_only = 2; + + // In specifies that this field must be equal to one of the specified + // values + repeated int32 in = 3; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated int32 not_in = 4; +} + +// MessageRules describe the constraints applied to embedded message values. +// For message-type fields, validation is performed recursively. +message MessageRules { + // Skip specifies that the validation rules of this field should not be + // evaluated + optional bool skip = 1; + + // Required specifies that this field must be set + optional bool required = 2; +} + +// RepeatedRules describe the constraints applied to `repeated` values +message RepeatedRules { + // MinItems specifies that this field must have the specified number of + // items at a minimum + optional uint64 min_items = 1; + + // MaxItems specifies that this field must have the specified number of + // items at a maximum + optional uint64 max_items = 2; + + // Unique specifies that all elements in this field must be unique. This + // contraint is only applicable to scalar and enum types (messages are not + // supported). + optional bool unique = 3; + + // Items specifies the contraints to be applied to each item in the field. + // Repeated message fields will still execute validation against each item + // unless skip is specified here. + optional FieldRules items = 4; +} + +// MapRules describe the constraints applied to `map` values +message MapRules { + // MinPairs specifies that this field must have the specified number of + // KVs at a minimum + optional uint64 min_pairs = 1; + + // MaxPairs specifies that this field must have the specified number of + // KVs at a maximum + optional uint64 max_pairs = 2; + + // NoSparse specifies values in this field cannot be unset. This only + // applies to map's with message value types. + optional bool no_sparse = 3; + + // Keys specifies the constraints to be applied to each key in the field. + optional FieldRules keys = 4; + + // Values specifies the constraints to be applied to the value of each key + // in the field. Message values will still have their validations evaluated + // unless skip is specified here. + optional FieldRules values = 5; +} + +// AnyRules describe constraints applied exclusively to the +// `google.protobuf.Any` well-known type +message AnyRules { + // Required specifies that this field must be set + optional bool required = 1; + + // In specifies that this field's `type_url` must be equal to one of the + // specified values. + repeated string in = 2; + + // NotIn specifies that this field's `type_url` must not be equal to any of + // the specified values. + repeated string not_in = 3; +} + +// DurationRules describe the constraints applied exclusively to the +// `google.protobuf.Duration` well-known type +message DurationRules { + // Required specifies that this field must be set + optional bool required = 1; + + // Const specifies that this field must be exactly the specified value + optional google.protobuf.Duration const = 2; + + // Lt specifies that this field must be less than the specified value, + // exclusive + optional google.protobuf.Duration lt = 3; + + // Lt specifies that this field must be less than the specified value, + // inclusive + optional google.protobuf.Duration lte = 4; + + // Gt specifies that this field must be greater than the specified value, + // exclusive + optional google.protobuf.Duration gt = 5; + + // Gte specifies that this field must be greater than the specified value, + // inclusive + optional google.protobuf.Duration gte = 6; + + // In specifies that this field must be equal to one of the specified + // values + repeated google.protobuf.Duration in = 7; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated google.protobuf.Duration not_in = 8; +} + +// TimestampRules describe the constraints applied exclusively to the +// `google.protobuf.Timestamp` well-known type +message TimestampRules { + // Required specifies that this field must be set + optional bool required = 1; + + // Const specifies that this field must be exactly the specified value + optional google.protobuf.Timestamp const = 2; + + // Lt specifies that this field must be less than the specified value, + // exclusive + optional google.protobuf.Timestamp lt = 3; + + // Lte specifies that this field must be less than the specified value, + // inclusive + optional google.protobuf.Timestamp lte = 4; + + // Gt specifies that this field must be greater than the specified value, + // exclusive + optional google.protobuf.Timestamp gt = 5; + + // Gte specifies that this field must be greater than the specified value, + // inclusive + optional google.protobuf.Timestamp gte = 6; + + // LtNow specifies that this must be less than the current time. LtNow + // can only be used with the Within rule. + optional bool lt_now = 7; + + // GtNow specifies that this must be greater than the current time. GtNow + // can only be used with the Within rule. + optional bool gt_now = 8; + + // Within specifies that this field must be within this duration of the + // current time. This constraint can be used alone or with the LtNow and + // GtNow rules. + optional google.protobuf.Duration within = 9; +} diff --git a/github.com/google/uuid/.travis.yml b/github.com/google/uuid/.travis.yml new file mode 100644 index 0000000000..d8156a60ba --- /dev/null +++ b/github.com/google/uuid/.travis.yml @@ -0,0 +1,9 @@ +language: go + +go: + - 1.4.3 + - 1.5.3 + - tip + +script: + - go test -v ./... diff --git a/github.com/google/uuid/CONTRIBUTING.md b/github.com/google/uuid/CONTRIBUTING.md new file mode 100644 index 0000000000..04fdf09f13 --- /dev/null +++ b/github.com/google/uuid/CONTRIBUTING.md @@ -0,0 +1,10 @@ +# How to contribute + +We definitely welcome patches and contribution to this project! + +### Legal requirements + +In order to protect both you and ourselves, you will need to sign the +[Contributor License Agreement](https://cla.developers.google.com/clas). + +You may have already signed it for other Google projects. diff --git a/github.com/google/uuid/CONTRIBUTORS b/github.com/google/uuid/CONTRIBUTORS new file mode 100644 index 0000000000..b4bb97f6bc --- /dev/null +++ b/github.com/google/uuid/CONTRIBUTORS @@ -0,0 +1,9 @@ +Paul Borman +bmatsuo +shawnps +theory +jboverfelt +dsymonds +cd1 +wallclockbuilder +dansouza diff --git a/github.com/google/uuid/LICENSE b/github.com/google/uuid/LICENSE new file mode 100644 index 0000000000..5dc68268d9 --- /dev/null +++ b/github.com/google/uuid/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009,2014 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/github.com/google/uuid/README.md b/github.com/google/uuid/README.md new file mode 100644 index 0000000000..9d92c11f16 --- /dev/null +++ b/github.com/google/uuid/README.md @@ -0,0 +1,19 @@ +# uuid ![build status](https://travis-ci.org/google/uuid.svg?branch=master) +The uuid package generates and inspects UUIDs based on +[RFC 4122](http://tools.ietf.org/html/rfc4122) +and DCE 1.1: Authentication and Security Services. + +This package is based on the github.com/pborman/uuid package (previously named +code.google.com/p/go-uuid). It differs from these earlier packages in that +a UUID is a 16 byte array rather than a byte slice. One loss due to this +change is the ability to represent an invalid UUID (vs a NIL UUID). + +###### Install +`go get github.com/google/uuid` + +###### Documentation +[![GoDoc](https://godoc.org/github.com/google/uuid?status.svg)](http://godoc.org/github.com/google/uuid) + +Full `go doc` style documentation for the package can be viewed online without +installing this package by using the GoDoc site here: +http://godoc.org/github.com/google/uuid diff --git a/github.com/google/uuid/dce.go b/github.com/google/uuid/dce.go new file mode 100644 index 0000000000..fa820b9d30 --- /dev/null +++ b/github.com/google/uuid/dce.go @@ -0,0 +1,80 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "encoding/binary" + "fmt" + "os" +) + +// A Domain represents a Version 2 domain +type Domain byte + +// Domain constants for DCE Security (Version 2) UUIDs. +const ( + Person = Domain(0) + Group = Domain(1) + Org = Domain(2) +) + +// NewDCESecurity returns a DCE Security (Version 2) UUID. +// +// The domain should be one of Person, Group or Org. +// On a POSIX system the id should be the users UID for the Person +// domain and the users GID for the Group. The meaning of id for +// the domain Org or on non-POSIX systems is site defined. +// +// For a given domain/id pair the same token may be returned for up to +// 7 minutes and 10 seconds. +func NewDCESecurity(domain Domain, id uint32) (UUID, error) { + uuid, err := NewUUID() + if err == nil { + uuid[6] = (uuid[6] & 0x0f) | 0x20 // Version 2 + uuid[9] = byte(domain) + binary.BigEndian.PutUint32(uuid[0:], id) + } + return uuid, err +} + +// NewDCEPerson returns a DCE Security (Version 2) UUID in the person +// domain with the id returned by os.Getuid. +// +// NewDCESecurity(Person, uint32(os.Getuid())) +func NewDCEPerson() (UUID, error) { + return NewDCESecurity(Person, uint32(os.Getuid())) +} + +// NewDCEGroup returns a DCE Security (Version 2) UUID in the group +// domain with the id returned by os.Getgid. +// +// NewDCESecurity(Group, uint32(os.Getgid())) +func NewDCEGroup() (UUID, error) { + return NewDCESecurity(Group, uint32(os.Getgid())) +} + +// Domain returns the domain for a Version 2 UUID. Domains are only defined +// for Version 2 UUIDs. +func (uuid UUID) Domain() Domain { + return Domain(uuid[9]) +} + +// ID returns the id for a Version 2 UUID. IDs are only defined for Version 2 +// UUIDs. +func (uuid UUID) ID() uint32 { + return binary.BigEndian.Uint32(uuid[0:4]) +} + +func (d Domain) String() string { + switch d { + case Person: + return "Person" + case Group: + return "Group" + case Org: + return "Org" + } + return fmt.Sprintf("Domain%d", int(d)) +} diff --git a/github.com/google/uuid/doc.go b/github.com/google/uuid/doc.go new file mode 100644 index 0000000000..5b8a4b9af8 --- /dev/null +++ b/github.com/google/uuid/doc.go @@ -0,0 +1,12 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package uuid generates and inspects UUIDs. +// +// UUIDs are based on RFC 4122 and DCE 1.1: Authentication and Security +// Services. +// +// A UUID is a 16 byte (128 bit) array. UUIDs may be used as keys to +// maps or compared directly. +package uuid diff --git a/github.com/google/uuid/go.mod b/github.com/google/uuid/go.mod new file mode 100644 index 0000000000..fc84cd79d4 --- /dev/null +++ b/github.com/google/uuid/go.mod @@ -0,0 +1 @@ +module github.com/google/uuid diff --git a/github.com/google/uuid/hash.go b/github.com/google/uuid/hash.go new file mode 100644 index 0000000000..b174616315 --- /dev/null +++ b/github.com/google/uuid/hash.go @@ -0,0 +1,53 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "crypto/md5" + "crypto/sha1" + "hash" +) + +// Well known namespace IDs and UUIDs +var ( + NameSpaceDNS = Must(Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8")) + NameSpaceURL = Must(Parse("6ba7b811-9dad-11d1-80b4-00c04fd430c8")) + NameSpaceOID = Must(Parse("6ba7b812-9dad-11d1-80b4-00c04fd430c8")) + NameSpaceX500 = Must(Parse("6ba7b814-9dad-11d1-80b4-00c04fd430c8")) + Nil UUID // empty UUID, all zeros +) + +// NewHash returns a new UUID derived from the hash of space concatenated with +// data generated by h. The hash should be at least 16 byte in length. The +// first 16 bytes of the hash are used to form the UUID. The version of the +// UUID will be the lower 4 bits of version. NewHash is used to implement +// NewMD5 and NewSHA1. +func NewHash(h hash.Hash, space UUID, data []byte, version int) UUID { + h.Reset() + h.Write(space[:]) + h.Write(data) + s := h.Sum(nil) + var uuid UUID + copy(uuid[:], s) + uuid[6] = (uuid[6] & 0x0f) | uint8((version&0xf)<<4) + uuid[8] = (uuid[8] & 0x3f) | 0x80 // RFC 4122 variant + return uuid +} + +// NewMD5 returns a new MD5 (Version 3) UUID based on the +// supplied name space and data. It is the same as calling: +// +// NewHash(md5.New(), space, data, 3) +func NewMD5(space UUID, data []byte) UUID { + return NewHash(md5.New(), space, data, 3) +} + +// NewSHA1 returns a new SHA1 (Version 5) UUID based on the +// supplied name space and data. It is the same as calling: +// +// NewHash(sha1.New(), space, data, 5) +func NewSHA1(space UUID, data []byte) UUID { + return NewHash(sha1.New(), space, data, 5) +} diff --git a/github.com/google/uuid/marshal.go b/github.com/google/uuid/marshal.go new file mode 100644 index 0000000000..7f9e0c6c0e --- /dev/null +++ b/github.com/google/uuid/marshal.go @@ -0,0 +1,37 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import "fmt" + +// MarshalText implements encoding.TextMarshaler. +func (uuid UUID) MarshalText() ([]byte, error) { + var js [36]byte + encodeHex(js[:], uuid) + return js[:], nil +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (uuid *UUID) UnmarshalText(data []byte) error { + id, err := ParseBytes(data) + if err == nil { + *uuid = id + } + return err +} + +// MarshalBinary implements encoding.BinaryMarshaler. +func (uuid UUID) MarshalBinary() ([]byte, error) { + return uuid[:], nil +} + +// UnmarshalBinary implements encoding.BinaryUnmarshaler. +func (uuid *UUID) UnmarshalBinary(data []byte) error { + if len(data) != 16 { + return fmt.Errorf("invalid UUID (got %d bytes)", len(data)) + } + copy(uuid[:], data) + return nil +} diff --git a/github.com/google/uuid/node.go b/github.com/google/uuid/node.go new file mode 100644 index 0000000000..d651a2b061 --- /dev/null +++ b/github.com/google/uuid/node.go @@ -0,0 +1,90 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "sync" +) + +var ( + nodeMu sync.Mutex + ifname string // name of interface being used + nodeID [6]byte // hardware for version 1 UUIDs + zeroID [6]byte // nodeID with only 0's +) + +// NodeInterface returns the name of the interface from which the NodeID was +// derived. The interface "user" is returned if the NodeID was set by +// SetNodeID. +func NodeInterface() string { + defer nodeMu.Unlock() + nodeMu.Lock() + return ifname +} + +// SetNodeInterface selects the hardware address to be used for Version 1 UUIDs. +// If name is "" then the first usable interface found will be used or a random +// Node ID will be generated. If a named interface cannot be found then false +// is returned. +// +// SetNodeInterface never fails when name is "". +func SetNodeInterface(name string) bool { + defer nodeMu.Unlock() + nodeMu.Lock() + return setNodeInterface(name) +} + +func setNodeInterface(name string) bool { + iname, addr := getHardwareInterface(name) // null implementation for js + if iname != "" && addr != nil { + ifname = iname + copy(nodeID[:], addr) + return true + } + + // We found no interfaces with a valid hardware address. If name + // does not specify a specific interface generate a random Node ID + // (section 4.1.6) + if name == "" { + ifname = "random" + randomBits(nodeID[:]) + return true + } + return false +} + +// NodeID returns a slice of a copy of the current Node ID, setting the Node ID +// if not already set. +func NodeID() []byte { + defer nodeMu.Unlock() + nodeMu.Lock() + if nodeID == zeroID { + setNodeInterface("") + } + nid := nodeID + return nid[:] +} + +// SetNodeID sets the Node ID to be used for Version 1 UUIDs. The first 6 bytes +// of id are used. If id is less than 6 bytes then false is returned and the +// Node ID is not set. +func SetNodeID(id []byte) bool { + if len(id) < 6 { + return false + } + defer nodeMu.Unlock() + nodeMu.Lock() + copy(nodeID[:], id) + ifname = "user" + return true +} + +// NodeID returns the 6 byte node id encoded in uuid. It returns nil if uuid is +// not valid. The NodeID is only well defined for version 1 and 2 UUIDs. +func (uuid UUID) NodeID() []byte { + var node [6]byte + copy(node[:], uuid[10:]) + return node[:] +} diff --git a/github.com/google/uuid/node_js.go b/github.com/google/uuid/node_js.go new file mode 100644 index 0000000000..24b78edc90 --- /dev/null +++ b/github.com/google/uuid/node_js.go @@ -0,0 +1,12 @@ +// Copyright 2017 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build js + +package uuid + +// getHardwareInterface returns nil values for the JS version of the code. +// This remvoves the "net" dependency, because it is not used in the browser. +// Using the "net" library inflates the size of the transpiled JS code by 673k bytes. +func getHardwareInterface(name string) (string, []byte) { return "", nil } diff --git a/github.com/google/uuid/node_net.go b/github.com/google/uuid/node_net.go new file mode 100644 index 0000000000..0cbbcddbd6 --- /dev/null +++ b/github.com/google/uuid/node_net.go @@ -0,0 +1,33 @@ +// Copyright 2017 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !js + +package uuid + +import "net" + +var interfaces []net.Interface // cached list of interfaces + +// getHardwareInterface returns the name and hardware address of interface name. +// If name is "" then the name and hardware address of one of the system's +// interfaces is returned. If no interfaces are found (name does not exist or +// there are no interfaces) then "", nil is returned. +// +// Only addresses of at least 6 bytes are returned. +func getHardwareInterface(name string) (string, []byte) { + if interfaces == nil { + var err error + interfaces, err = net.Interfaces() + if err != nil { + return "", nil + } + } + for _, ifs := range interfaces { + if len(ifs.HardwareAddr) >= 6 && (name == "" || name == ifs.Name) { + return ifs.Name, ifs.HardwareAddr + } + } + return "", nil +} diff --git a/github.com/google/uuid/sql.go b/github.com/google/uuid/sql.go new file mode 100644 index 0000000000..f326b54db3 --- /dev/null +++ b/github.com/google/uuid/sql.go @@ -0,0 +1,59 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "database/sql/driver" + "fmt" +) + +// Scan implements sql.Scanner so UUIDs can be read from databases transparently +// Currently, database types that map to string and []byte are supported. Please +// consult database-specific driver documentation for matching types. +func (uuid *UUID) Scan(src interface{}) error { + switch src := src.(type) { + case nil: + return nil + + case string: + // if an empty UUID comes from a table, we return a null UUID + if src == "" { + return nil + } + + // see Parse for required string format + u, err := Parse(src) + if err != nil { + return fmt.Errorf("Scan: %v", err) + } + + *uuid = u + + case []byte: + // if an empty UUID comes from a table, we return a null UUID + if len(src) == 0 { + return nil + } + + // assumes a simple slice of bytes if 16 bytes + // otherwise attempts to parse + if len(src) != 16 { + return uuid.Scan(string(src)) + } + copy((*uuid)[:], src) + + default: + return fmt.Errorf("Scan: unable to scan type %T into UUID", src) + } + + return nil +} + +// Value implements sql.Valuer so that UUIDs can be written to databases +// transparently. Currently, UUIDs map to strings. Please consult +// database-specific driver documentation for matching types. +func (uuid UUID) Value() (driver.Value, error) { + return uuid.String(), nil +} diff --git a/github.com/google/uuid/time.go b/github.com/google/uuid/time.go new file mode 100644 index 0000000000..e6ef06cdc8 --- /dev/null +++ b/github.com/google/uuid/time.go @@ -0,0 +1,123 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "encoding/binary" + "sync" + "time" +) + +// A Time represents a time as the number of 100's of nanoseconds since 15 Oct +// 1582. +type Time int64 + +const ( + lillian = 2299160 // Julian day of 15 Oct 1582 + unix = 2440587 // Julian day of 1 Jan 1970 + epoch = unix - lillian // Days between epochs + g1582 = epoch * 86400 // seconds between epochs + g1582ns100 = g1582 * 10000000 // 100s of a nanoseconds between epochs +) + +var ( + timeMu sync.Mutex + lasttime uint64 // last time we returned + clockSeq uint16 // clock sequence for this run + + timeNow = time.Now // for testing +) + +// UnixTime converts t the number of seconds and nanoseconds using the Unix +// epoch of 1 Jan 1970. +func (t Time) UnixTime() (sec, nsec int64) { + sec = int64(t - g1582ns100) + nsec = (sec % 10000000) * 100 + sec /= 10000000 + return sec, nsec +} + +// GetTime returns the current Time (100s of nanoseconds since 15 Oct 1582) and +// clock sequence as well as adjusting the clock sequence as needed. An error +// is returned if the current time cannot be determined. +func GetTime() (Time, uint16, error) { + defer timeMu.Unlock() + timeMu.Lock() + return getTime() +} + +func getTime() (Time, uint16, error) { + t := timeNow() + + // If we don't have a clock sequence already, set one. + if clockSeq == 0 { + setClockSequence(-1) + } + now := uint64(t.UnixNano()/100) + g1582ns100 + + // If time has gone backwards with this clock sequence then we + // increment the clock sequence + if now <= lasttime { + clockSeq = ((clockSeq + 1) & 0x3fff) | 0x8000 + } + lasttime = now + return Time(now), clockSeq, nil +} + +// ClockSequence returns the current clock sequence, generating one if not +// already set. The clock sequence is only used for Version 1 UUIDs. +// +// The uuid package does not use global static storage for the clock sequence or +// the last time a UUID was generated. Unless SetClockSequence is used, a new +// random clock sequence is generated the first time a clock sequence is +// requested by ClockSequence, GetTime, or NewUUID. (section 4.2.1.1) +func ClockSequence() int { + defer timeMu.Unlock() + timeMu.Lock() + return clockSequence() +} + +func clockSequence() int { + if clockSeq == 0 { + setClockSequence(-1) + } + return int(clockSeq & 0x3fff) +} + +// SetClockSequence sets the clock sequence to the lower 14 bits of seq. Setting to +// -1 causes a new sequence to be generated. +func SetClockSequence(seq int) { + defer timeMu.Unlock() + timeMu.Lock() + setClockSequence(seq) +} + +func setClockSequence(seq int) { + if seq == -1 { + var b [2]byte + randomBits(b[:]) // clock sequence + seq = int(b[0])<<8 | int(b[1]) + } + oldSeq := clockSeq + clockSeq = uint16(seq&0x3fff) | 0x8000 // Set our variant + if oldSeq != clockSeq { + lasttime = 0 + } +} + +// Time returns the time in 100s of nanoseconds since 15 Oct 1582 encoded in +// uuid. The time is only defined for version 1 and 2 UUIDs. +func (uuid UUID) Time() Time { + time := int64(binary.BigEndian.Uint32(uuid[0:4])) + time |= int64(binary.BigEndian.Uint16(uuid[4:6])) << 32 + time |= int64(binary.BigEndian.Uint16(uuid[6:8])&0xfff) << 48 + return Time(time) +} + +// ClockSequence returns the clock sequence encoded in uuid. +// The clock sequence is only well defined for version 1 and 2 UUIDs. +func (uuid UUID) ClockSequence() int { + return int(binary.BigEndian.Uint16(uuid[8:10])) & 0x3fff +} diff --git a/github.com/google/uuid/util.go b/github.com/google/uuid/util.go new file mode 100644 index 0000000000..5ea6c73780 --- /dev/null +++ b/github.com/google/uuid/util.go @@ -0,0 +1,43 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "io" +) + +// randomBits completely fills slice b with random data. +func randomBits(b []byte) { + if _, err := io.ReadFull(rander, b); err != nil { + panic(err.Error()) // rand should never fail + } +} + +// xvalues returns the value of a byte as a hexadecimal digit or 255. +var xvalues = [256]byte{ + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 255, 255, 255, 255, 255, 255, + 255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +} + +// xtob converts hex characters x1 and x2 into a byte. +func xtob(x1, x2 byte) (byte, bool) { + b1 := xvalues[x1] + b2 := xvalues[x2] + return (b1 << 4) | b2, b1 != 255 && b2 != 255 +} diff --git a/github.com/google/uuid/uuid.go b/github.com/google/uuid/uuid.go new file mode 100644 index 0000000000..524404cc52 --- /dev/null +++ b/github.com/google/uuid/uuid.go @@ -0,0 +1,245 @@ +// Copyright 2018 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "bytes" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "strings" +) + +// A UUID is a 128 bit (16 byte) Universal Unique IDentifier as defined in RFC +// 4122. +type UUID [16]byte + +// A Version represents a UUID's version. +type Version byte + +// A Variant represents a UUID's variant. +type Variant byte + +// Constants returned by Variant. +const ( + Invalid = Variant(iota) // Invalid UUID + RFC4122 // The variant specified in RFC4122 + Reserved // Reserved, NCS backward compatibility. + Microsoft // Reserved, Microsoft Corporation backward compatibility. + Future // Reserved for future definition. +) + +var rander = rand.Reader // random function + +// Parse decodes s into a UUID or returns an error. Both the standard UUID +// forms of xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx and +// urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx are decoded as well as the +// Microsoft encoding {xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx} and the raw hex +// encoding: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx. +func Parse(s string) (UUID, error) { + var uuid UUID + switch len(s) { + // xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + case 36: + + // urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + case 36 + 9: + if strings.ToLower(s[:9]) != "urn:uuid:" { + return uuid, fmt.Errorf("invalid urn prefix: %q", s[:9]) + } + s = s[9:] + + // {xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx} + case 36 + 2: + s = s[1:] + + // xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + case 32: + var ok bool + for i := range uuid { + uuid[i], ok = xtob(s[i*2], s[i*2+1]) + if !ok { + return uuid, errors.New("invalid UUID format") + } + } + return uuid, nil + default: + return uuid, fmt.Errorf("invalid UUID length: %d", len(s)) + } + // s is now at least 36 bytes long + // it must be of the form xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + if s[8] != '-' || s[13] != '-' || s[18] != '-' || s[23] != '-' { + return uuid, errors.New("invalid UUID format") + } + for i, x := range [16]int{ + 0, 2, 4, 6, + 9, 11, + 14, 16, + 19, 21, + 24, 26, 28, 30, 32, 34} { + v, ok := xtob(s[x], s[x+1]) + if !ok { + return uuid, errors.New("invalid UUID format") + } + uuid[i] = v + } + return uuid, nil +} + +// ParseBytes is like Parse, except it parses a byte slice instead of a string. +func ParseBytes(b []byte) (UUID, error) { + var uuid UUID + switch len(b) { + case 36: // xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + case 36 + 9: // urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + if !bytes.Equal(bytes.ToLower(b[:9]), []byte("urn:uuid:")) { + return uuid, fmt.Errorf("invalid urn prefix: %q", b[:9]) + } + b = b[9:] + case 36 + 2: // {xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx} + b = b[1:] + case 32: // xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + var ok bool + for i := 0; i < 32; i += 2 { + uuid[i/2], ok = xtob(b[i], b[i+1]) + if !ok { + return uuid, errors.New("invalid UUID format") + } + } + return uuid, nil + default: + return uuid, fmt.Errorf("invalid UUID length: %d", len(b)) + } + // s is now at least 36 bytes long + // it must be of the form xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + if b[8] != '-' || b[13] != '-' || b[18] != '-' || b[23] != '-' { + return uuid, errors.New("invalid UUID format") + } + for i, x := range [16]int{ + 0, 2, 4, 6, + 9, 11, + 14, 16, + 19, 21, + 24, 26, 28, 30, 32, 34} { + v, ok := xtob(b[x], b[x+1]) + if !ok { + return uuid, errors.New("invalid UUID format") + } + uuid[i] = v + } + return uuid, nil +} + +// MustParse is like Parse but panics if the string cannot be parsed. +// It simplifies safe initialization of global variables holding compiled UUIDs. +func MustParse(s string) UUID { + uuid, err := Parse(s) + if err != nil { + panic(`uuid: Parse(` + s + `): ` + err.Error()) + } + return uuid +} + +// FromBytes creates a new UUID from a byte slice. Returns an error if the slice +// does not have a length of 16. The bytes are copied from the slice. +func FromBytes(b []byte) (uuid UUID, err error) { + err = uuid.UnmarshalBinary(b) + return uuid, err +} + +// Must returns uuid if err is nil and panics otherwise. +func Must(uuid UUID, err error) UUID { + if err != nil { + panic(err) + } + return uuid +} + +// String returns the string form of uuid, xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx +// , or "" if uuid is invalid. +func (uuid UUID) String() string { + var buf [36]byte + encodeHex(buf[:], uuid) + return string(buf[:]) +} + +// URN returns the RFC 2141 URN form of uuid, +// urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx, or "" if uuid is invalid. +func (uuid UUID) URN() string { + var buf [36 + 9]byte + copy(buf[:], "urn:uuid:") + encodeHex(buf[9:], uuid) + return string(buf[:]) +} + +func encodeHex(dst []byte, uuid UUID) { + hex.Encode(dst, uuid[:4]) + dst[8] = '-' + hex.Encode(dst[9:13], uuid[4:6]) + dst[13] = '-' + hex.Encode(dst[14:18], uuid[6:8]) + dst[18] = '-' + hex.Encode(dst[19:23], uuid[8:10]) + dst[23] = '-' + hex.Encode(dst[24:], uuid[10:]) +} + +// Variant returns the variant encoded in uuid. +func (uuid UUID) Variant() Variant { + switch { + case (uuid[8] & 0xc0) == 0x80: + return RFC4122 + case (uuid[8] & 0xe0) == 0xc0: + return Microsoft + case (uuid[8] & 0xe0) == 0xe0: + return Future + default: + return Reserved + } +} + +// Version returns the version of uuid. +func (uuid UUID) Version() Version { + return Version(uuid[6] >> 4) +} + +func (v Version) String() string { + if v > 15 { + return fmt.Sprintf("BAD_VERSION_%d", v) + } + return fmt.Sprintf("VERSION_%d", v) +} + +func (v Variant) String() string { + switch v { + case RFC4122: + return "RFC4122" + case Reserved: + return "Reserved" + case Microsoft: + return "Microsoft" + case Future: + return "Future" + case Invalid: + return "Invalid" + } + return fmt.Sprintf("BadVariant%d", int(v)) +} + +// SetRand sets the random number generator to r, which implements io.Reader. +// If r.Read returns an error when the package requests random data then +// a panic will be issued. +// +// Calling SetRand with nil sets the random number generator to the default +// generator. +func SetRand(r io.Reader) { + if r == nil { + rander = rand.Reader + return + } + rander = r +} diff --git a/github.com/google/uuid/version1.go b/github.com/google/uuid/version1.go new file mode 100644 index 0000000000..199a1ac654 --- /dev/null +++ b/github.com/google/uuid/version1.go @@ -0,0 +1,44 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "encoding/binary" +) + +// NewUUID returns a Version 1 UUID based on the current NodeID and clock +// sequence, and the current time. If the NodeID has not been set by SetNodeID +// or SetNodeInterface then it will be set automatically. If the NodeID cannot +// be set NewUUID returns nil. If clock sequence has not been set by +// SetClockSequence then it will be set automatically. If GetTime fails to +// return the current NewUUID returns nil and an error. +// +// In most cases, New should be used. +func NewUUID() (UUID, error) { + nodeMu.Lock() + if nodeID == zeroID { + setNodeInterface("") + } + nodeMu.Unlock() + + var uuid UUID + now, seq, err := GetTime() + if err != nil { + return uuid, err + } + + timeLow := uint32(now & 0xffffffff) + timeMid := uint16((now >> 32) & 0xffff) + timeHi := uint16((now >> 48) & 0x0fff) + timeHi |= 0x1000 // Version 1 + + binary.BigEndian.PutUint32(uuid[0:], timeLow) + binary.BigEndian.PutUint16(uuid[4:], timeMid) + binary.BigEndian.PutUint16(uuid[6:], timeHi) + binary.BigEndian.PutUint16(uuid[8:], seq) + copy(uuid[10:], nodeID[:]) + + return uuid, nil +} diff --git a/github.com/google/uuid/version4.go b/github.com/google/uuid/version4.go new file mode 100644 index 0000000000..84af91c9f5 --- /dev/null +++ b/github.com/google/uuid/version4.go @@ -0,0 +1,38 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import "io" + +// New creates a new random UUID or panics. New is equivalent to +// the expression +// +// uuid.Must(uuid.NewRandom()) +func New() UUID { + return Must(NewRandom()) +} + +// NewRandom returns a Random (Version 4) UUID. +// +// The strength of the UUIDs is based on the strength of the crypto/rand +// package. +// +// A note about uniqueness derived from the UUID Wikipedia entry: +// +// Randomly generated UUIDs have 122 random bits. One's annual risk of being +// hit by a meteorite is estimated to be one chance in 17 billion, that +// means the probability is about 0.00000000006 (6 × 10−11), +// equivalent to the odds of creating a few tens of trillions of UUIDs in a +// year and having one duplicate. +func NewRandom() (UUID, error) { + var uuid UUID + _, err := io.ReadFull(rander, uuid[:]) + if err != nil { + return Nil, err + } + uuid[6] = (uuid[6] & 0x0f) | 0x40 // Version 4 + uuid[8] = (uuid[8] & 0x3f) | 0x80 // Variant is 10 + return uuid, nil +} diff --git a/github.com/huandu/xstrings/.gitignore b/github.com/huandu/xstrings/.gitignore new file mode 100644 index 0000000000..daf913b1b3 --- /dev/null +++ b/github.com/huandu/xstrings/.gitignore @@ -0,0 +1,24 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof diff --git a/github.com/huandu/xstrings/.travis.yml b/github.com/huandu/xstrings/.travis.yml new file mode 100644 index 0000000000..d6460be411 --- /dev/null +++ b/github.com/huandu/xstrings/.travis.yml @@ -0,0 +1,7 @@ +language: go +install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls +script: + - go test -v -covermode=count -coverprofile=coverage.out + - 'if [ "$TRAVIS_PULL_REQUEST" = "false" ] && [ ! -z "$COVERALLS_TOKEN" ]; then $HOME/gopath/bin/goveralls -coverprofile=coverage.out -service=travis-ci -repotoken $COVERALLS_TOKEN; fi' diff --git a/github.com/huandu/xstrings/CONTRIBUTING.md b/github.com/huandu/xstrings/CONTRIBUTING.md new file mode 100644 index 0000000000..d7b4b8d584 --- /dev/null +++ b/github.com/huandu/xstrings/CONTRIBUTING.md @@ -0,0 +1,23 @@ +# Contributing # + +Thanks for your contribution in advance. No matter what you will contribute to this project, pull request or bug report or feature discussion, it's always highly appreciated. + +## New API or feature ## + +I want to speak more about how to add new functions to this package. + +Package `xstring` is a collection of useful string functions which should be implemented in Go. It's a bit subject to say which function should be included and which should not. I set up following rules in order to make it clear and as objective as possible. + +* Rule 1: Only string algorithm, which takes string as input, can be included. +* Rule 2: If a function has been implemented in package `string`, it must not be included. +* Rule 3: If a function is not language neutral, it must not be included. +* Rule 4: If a function is a part of standard library in other languages, it can be included. +* Rule 5: If a function is quite useful in some famous framework or library, it can be included. + +New function must be discussed in project issues before submitting any code. If a pull request with new functions is sent without any ref issue, it will be rejected. + +## Pull request ## + +Pull request is always welcome. Just make sure you have run `go fmt` and all test cases passed before submit. + +If the pull request is to add a new API or feature, don't forget to update README.md and add new API in function list. diff --git a/github.com/huandu/xstrings/LICENSE b/github.com/huandu/xstrings/LICENSE new file mode 100644 index 0000000000..2701772593 --- /dev/null +++ b/github.com/huandu/xstrings/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2015 Huan Du + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/github.com/huandu/xstrings/README.md b/github.com/huandu/xstrings/README.md new file mode 100644 index 0000000000..292bf2f39e --- /dev/null +++ b/github.com/huandu/xstrings/README.md @@ -0,0 +1,117 @@ +# xstrings # + +[![Build Status](https://travis-ci.org/huandu/xstrings.svg?branch=master)](https://travis-ci.org/huandu/xstrings) +[![GoDoc](https://godoc.org/github.com/huandu/xstrings?status.svg)](https://godoc.org/github.com/huandu/xstrings) +[![Go Report](https://goreportcard.com/badge/github.com/huandu/xstrings)](https://goreportcard.com/report/github.com/huandu/xstrings) +[![Coverage Status](https://coveralls.io/repos/github/huandu/xstrings/badge.svg?branch=master)](https://coveralls.io/github/huandu/xstrings?branch=master) + +Go package [xstrings](https://godoc.org/github.com/huandu/xstrings) is a collection of string functions, which are widely used in other languages but absent in Go package [strings](http://golang.org/pkg/strings). + +All functions are well tested and carefully tuned for performance. + +## Propose a new function ## + +Please review [contributing guideline](CONTRIBUTING.md) and [create new issue](https://github.com/huandu/xstrings/issues) to state why it should be included. + +## Install ## + +Use `go get` to install this library. + + go get github.com/huandu/xstrings + +## API document ## + +See [GoDoc](https://godoc.org/github.com/huandu/xstrings) for full document. + +## Function list ## + +Go functions have a unique naming style. One, who has experience in other language but new in Go, may have difficulties to find out right string function to use. + +Here is a list of functions in [strings](http://golang.org/pkg/strings) and [xstrings](https://godoc.org/github.com/huandu/xstrings) with enough extra information about how to map these functions to their friends in other languages. Hope this list could be helpful for fresh gophers. + +### Package `xstrings` functions ### + +*Keep this table sorted by Function in ascending order.* + +| Function | Friends | # | +| -------- | ------- | --- | +| [Center](https://godoc.org/github.com/huandu/xstrings#Center) | `str.center` in Python; `String#center` in Ruby | [#30](https://github.com/huandu/xstrings/issues/30) | +| [Count](https://godoc.org/github.com/huandu/xstrings#Count) | `String#count` in Ruby | [#16](https://github.com/huandu/xstrings/issues/16) | +| [Delete](https://godoc.org/github.com/huandu/xstrings#Delete) | `String#delete` in Ruby | [#17](https://github.com/huandu/xstrings/issues/17) | +| [ExpandTabs](https://godoc.org/github.com/huandu/xstrings#ExpandTabs) | `str.expandtabs` in Python | [#27](https://github.com/huandu/xstrings/issues/27) | +| [FirstRuneToLower](https://godoc.org/github.com/huandu/xstrings#FirstRuneToLower) | `lcfirst` in PHP or Perl | [#15](https://github.com/huandu/xstrings/issues/15) | +| [FirstRuneToUpper](https://godoc.org/github.com/huandu/xstrings#FirstRuneToUpper) | `String#capitalize` in Ruby; `ucfirst` in PHP or Perl | [#15](https://github.com/huandu/xstrings/issues/15) | +| [Insert](https://godoc.org/github.com/huandu/xstrings#Insert) | `String#insert` in Ruby | [#18](https://github.com/huandu/xstrings/issues/18) | +| [LastPartition](https://godoc.org/github.com/huandu/xstrings#LastPartition) | `str.rpartition` in Python; `String#rpartition` in Ruby | [#19](https://github.com/huandu/xstrings/issues/19) | +| [LeftJustify](https://godoc.org/github.com/huandu/xstrings#LeftJustify) | `str.ljust` in Python; `String#ljust` in Ruby | [#28](https://github.com/huandu/xstrings/issues/28) | +| [Len](https://godoc.org/github.com/huandu/xstrings#Len) | `mb_strlen` in PHP | [#23](https://github.com/huandu/xstrings/issues/23) | +| [Partition](https://godoc.org/github.com/huandu/xstrings#Partition) | `str.partition` in Python; `String#partition` in Ruby | [#10](https://github.com/huandu/xstrings/issues/10) | +| [Reverse](https://godoc.org/github.com/huandu/xstrings#Reverse) | `String#reverse` in Ruby; `strrev` in PHP; `reverse` in Perl | [#7](https://github.com/huandu/xstrings/issues/7) | +| [RightJustify](https://godoc.org/github.com/huandu/xstrings#RightJustify) | `str.rjust` in Python; `String#rjust` in Ruby | [#29](https://github.com/huandu/xstrings/issues/29) | +| [RuneWidth](https://godoc.org/github.com/huandu/xstrings#RuneWidth) | - | [#27](https://github.com/huandu/xstrings/issues/27) | +| [Scrub](https://godoc.org/github.com/huandu/xstrings#Scrub) | `String#scrub` in Ruby | [#20](https://github.com/huandu/xstrings/issues/20) | +| [Shuffle](https://godoc.org/github.com/huandu/xstrings#Shuffle) | `str_shuffle` in PHP | [#13](https://github.com/huandu/xstrings/issues/13) | +| [ShuffleSource](https://godoc.org/github.com/huandu/xstrings#ShuffleSource) | `str_shuffle` in PHP | [#13](https://github.com/huandu/xstrings/issues/13) | +| [Slice](https://godoc.org/github.com/huandu/xstrings#Slice) | `mb_substr` in PHP | [#9](https://github.com/huandu/xstrings/issues/9) | +| [Squeeze](https://godoc.org/github.com/huandu/xstrings#Squeeze) | `String#squeeze` in Ruby | [#11](https://github.com/huandu/xstrings/issues/11) | +| [Successor](https://godoc.org/github.com/huandu/xstrings#Successor) | `String#succ` or `String#next` in Ruby | [#22](https://github.com/huandu/xstrings/issues/22) | +| [SwapCase](https://godoc.org/github.com/huandu/xstrings#SwapCase) | `str.swapcase` in Python; `String#swapcase` in Ruby | [#12](https://github.com/huandu/xstrings/issues/12) | +| [ToCamelCase](https://godoc.org/github.com/huandu/xstrings#ToCamelCase) | `String#camelize` in RoR | [#1](https://github.com/huandu/xstrings/issues/1) | +| [ToKebab](https://godoc.org/github.com/huandu/xstrings#ToKebabCase) | - | [#41](https://github.com/huandu/xstrings/issues/41) | +| [ToSnakeCase](https://godoc.org/github.com/huandu/xstrings#ToSnakeCase) | `String#underscore` in RoR | [#1](https://github.com/huandu/xstrings/issues/1) | +| [Translate](https://godoc.org/github.com/huandu/xstrings#Translate) | `str.translate` in Python; `String#tr` in Ruby; `strtr` in PHP; `tr///` in Perl | [#21](https://github.com/huandu/xstrings/issues/21) | +| [Width](https://godoc.org/github.com/huandu/xstrings#Width) | `mb_strwidth` in PHP | [#26](https://github.com/huandu/xstrings/issues/26) | +| [WordCount](https://godoc.org/github.com/huandu/xstrings#WordCount) | `str_word_count` in PHP | [#14](https://github.com/huandu/xstrings/issues/14) | +| [WordSplit](https://godoc.org/github.com/huandu/xstrings#WordSplit) | - | [#14](https://github.com/huandu/xstrings/issues/14) | + +### Package `strings` functions ### + +*Keep this table sorted by Function in ascending order.* + +| Function | Friends | +| -------- | ------- | +| [Contains](http://golang.org/pkg/strings/#Contains) | `String#include?` in Ruby | +| [ContainsAny](http://golang.org/pkg/strings/#ContainsAny) | - | +| [ContainsRune](http://golang.org/pkg/strings/#ContainsRune) | - | +| [Count](http://golang.org/pkg/strings/#Count) | `str.count` in Python; `substr_count` in PHP | +| [EqualFold](http://golang.org/pkg/strings/#EqualFold) | `stricmp` in PHP; `String#casecmp` in Ruby | +| [Fields](http://golang.org/pkg/strings/#Fields) | `str.split` in Python; `split` in Perl; `String#split` in Ruby | +| [FieldsFunc](http://golang.org/pkg/strings/#FieldsFunc) | - | +| [HasPrefix](http://golang.org/pkg/strings/#HasPrefix) | `str.startswith` in Python; `String#start_with?` in Ruby | +| [HasSuffix](http://golang.org/pkg/strings/#HasSuffix) | `str.endswith` in Python; `String#end_with?` in Ruby | +| [Index](http://golang.org/pkg/strings/#Index) | `str.index` in Python; `String#index` in Ruby; `strpos` in PHP; `index` in Perl | +| [IndexAny](http://golang.org/pkg/strings/#IndexAny) | - | +| [IndexByte](http://golang.org/pkg/strings/#IndexByte) | - | +| [IndexFunc](http://golang.org/pkg/strings/#IndexFunc) | - | +| [IndexRune](http://golang.org/pkg/strings/#IndexRune) | - | +| [Join](http://golang.org/pkg/strings/#Join) | `str.join` in Python; `Array#join` in Ruby; `implode` in PHP; `join` in Perl | +| [LastIndex](http://golang.org/pkg/strings/#LastIndex) | `str.rindex` in Python; `String#rindex`; `strrpos` in PHP; `rindex` in Perl | +| [LastIndexAny](http://golang.org/pkg/strings/#LastIndexAny) | - | +| [LastIndexFunc](http://golang.org/pkg/strings/#LastIndexFunc) | - | +| [Map](http://golang.org/pkg/strings/#Map) | `String#each_codepoint` in Ruby | +| [Repeat](http://golang.org/pkg/strings/#Repeat) | operator `*` in Python and Ruby; `str_repeat` in PHP | +| [Replace](http://golang.org/pkg/strings/#Replace) | `str.replace` in Python; `String#sub` in Ruby; `str_replace` in PHP | +| [Split](http://golang.org/pkg/strings/#Split) | `str.split` in Python; `String#split` in Ruby; `explode` in PHP; `split` in Perl | +| [SplitAfter](http://golang.org/pkg/strings/#SplitAfter) | - | +| [SplitAfterN](http://golang.org/pkg/strings/#SplitAfterN) | - | +| [SplitN](http://golang.org/pkg/strings/#SplitN) | `str.split` in Python; `String#split` in Ruby; `explode` in PHP; `split` in Perl | +| [Title](http://golang.org/pkg/strings/#Title) | `str.title` in Python | +| [ToLower](http://golang.org/pkg/strings/#ToLower) | `str.lower` in Python; `String#downcase` in Ruby; `strtolower` in PHP; `lc` in Perl | +| [ToLowerSpecial](http://golang.org/pkg/strings/#ToLowerSpecial) | - | +| [ToTitle](http://golang.org/pkg/strings/#ToTitle) | - | +| [ToTitleSpecial](http://golang.org/pkg/strings/#ToTitleSpecial) | - | +| [ToUpper](http://golang.org/pkg/strings/#ToUpper) | `str.upper` in Python; `String#upcase` in Ruby; `strtoupper` in PHP; `uc` in Perl | +| [ToUpperSpecial](http://golang.org/pkg/strings/#ToUpperSpecial) | - | +| [Trim](http://golang.org/pkg/strings/#Trim) | `str.strip` in Python; `String#strip` in Ruby; `trim` in PHP | +| [TrimFunc](http://golang.org/pkg/strings/#TrimFunc) | - | +| [TrimLeft](http://golang.org/pkg/strings/#TrimLeft) | `str.lstrip` in Python; `String#lstrip` in Ruby; `ltrim` in PHP | +| [TrimLeftFunc](http://golang.org/pkg/strings/#TrimLeftFunc) | - | +| [TrimPrefix](http://golang.org/pkg/strings/#TrimPrefix) | - | +| [TrimRight](http://golang.org/pkg/strings/#TrimRight) | `str.rstrip` in Python; `String#rstrip` in Ruby; `rtrim` in PHP | +| [TrimRightFunc](http://golang.org/pkg/strings/#TrimRightFunc) | - | +| [TrimSpace](http://golang.org/pkg/strings/#TrimSpace) | `str.strip` in Python; `String#strip` in Ruby; `trim` in PHP | +| [TrimSuffix](http://golang.org/pkg/strings/#TrimSuffix) | `String#chomp` in Ruby; `chomp` in Perl | + +## License ## + +This library is licensed under MIT license. See LICENSE for details. diff --git a/github.com/huandu/xstrings/common.go b/github.com/huandu/xstrings/common.go new file mode 100644 index 0000000000..2aff57aab4 --- /dev/null +++ b/github.com/huandu/xstrings/common.go @@ -0,0 +1,25 @@ +// Copyright 2015 Huan Du. All rights reserved. +// Licensed under the MIT license that can be found in the LICENSE file. + +package xstrings + +import ( + "bytes" +) + +const bufferMaxInitGrowSize = 2048 + +// Lazy initialize a buffer. +func allocBuffer(orig, cur string) *bytes.Buffer { + output := &bytes.Buffer{} + maxSize := len(orig) * 4 + + // Avoid to reserve too much memory at once. + if maxSize > bufferMaxInitGrowSize { + maxSize = bufferMaxInitGrowSize + } + + output.Grow(maxSize) + output.WriteString(orig[:len(orig)-len(cur)]) + return output +} diff --git a/github.com/huandu/xstrings/convert.go b/github.com/huandu/xstrings/convert.go new file mode 100644 index 0000000000..3686780d23 --- /dev/null +++ b/github.com/huandu/xstrings/convert.go @@ -0,0 +1,404 @@ +// Copyright 2015 Huan Du. All rights reserved. +// Licensed under the MIT license that can be found in the LICENSE file. + +package xstrings + +import ( + "bytes" + "math/rand" + "unicode" + "unicode/utf8" +) + +// ToCamelCase is to convert words separated by space, underscore and hyphen to camel case. +// +// Some samples. +// "some_words" => "SomeWords" +// "http_server" => "HttpServer" +// "no_https" => "NoHttps" +// "_complex__case_" => "_Complex_Case_" +// "some words" => "SomeWords" +func ToCamelCase(str string) string { + if len(str) == 0 { + return "" + } + + buf := &bytes.Buffer{} + var r0, r1 rune + var size int + + // leading connector will appear in output. + for len(str) > 0 { + r0, size = utf8.DecodeRuneInString(str) + str = str[size:] + + if !isConnector(r0) { + r0 = unicode.ToUpper(r0) + break + } + + buf.WriteRune(r0) + } + + if len(str) == 0 { + // A special case for a string contains only 1 rune. + if size != 0 { + buf.WriteRune(r0) + } + + return buf.String() + } + + for len(str) > 0 { + r1 = r0 + r0, size = utf8.DecodeRuneInString(str) + str = str[size:] + + if isConnector(r0) && isConnector(r1) { + buf.WriteRune(r1) + continue + } + + if isConnector(r1) { + r0 = unicode.ToUpper(r0) + } else { + r0 = unicode.ToLower(r0) + buf.WriteRune(r1) + } + } + + buf.WriteRune(r0) + return buf.String() +} + +// ToSnakeCase can convert all upper case characters in a string to +// snake case format. +// +// Some samples. +// "FirstName" => "first_name" +// "HTTPServer" => "http_server" +// "NoHTTPS" => "no_https" +// "GO_PATH" => "go_path" +// "GO PATH" => "go_path" // space is converted to underscore. +// "GO-PATH" => "go_path" // hyphen is converted to underscore. +// "HTTP2XX" => "http_2xx" // insert an underscore before a number and after an alphabet. +// "http2xx" => "http_2xx" +// "HTTP20xOK" => "http_20x_ok" +func ToSnakeCase(str string) string { + return camelCaseToLowerCase(str, '_') +} + +// ToKebabCase can convert all upper case characters in a string to +// kebab case format. +// +// Some samples. +// "FirstName" => "first-name" +// "HTTPServer" => "http-server" +// "NoHTTPS" => "no-https" +// "GO_PATH" => "go-path" +// "GO PATH" => "go-path" // space is converted to '-'. +// "GO-PATH" => "go-path" // hyphen is converted to '-'. +// "HTTP2XX" => "http-2xx" // insert a '-' before a number and after an alphabet. +// "http2xx" => "http-2xx" +// "HTTP20xOK" => "http-20x-ok" +func ToKebabCase(str string) string { + return camelCaseToLowerCase(str, '-') +} + +func camelCaseToLowerCase(str string, connector rune) string { + if len(str) == 0 { + return "" + } + + buf := &bytes.Buffer{} + var prev, r0, r1 rune + var size int + + r0 = connector + + for len(str) > 0 { + prev = r0 + r0, size = utf8.DecodeRuneInString(str) + str = str[size:] + + switch { + case r0 == utf8.RuneError: + buf.WriteRune(r0) + + case unicode.IsUpper(r0): + if prev != connector && !unicode.IsNumber(prev) { + buf.WriteRune(connector) + } + + buf.WriteRune(unicode.ToLower(r0)) + + if len(str) == 0 { + break + } + + r0, size = utf8.DecodeRuneInString(str) + str = str[size:] + + if !unicode.IsUpper(r0) { + buf.WriteRune(r0) + break + } + + // find next non-upper-case character and insert connector properly. + // it's designed to convert `HTTPServer` to `http_server`. + // if there are more than 2 adjacent upper case characters in a word, + // treat them as an abbreviation plus a normal word. + for len(str) > 0 { + r1 = r0 + r0, size = utf8.DecodeRuneInString(str) + str = str[size:] + + if r0 == utf8.RuneError { + buf.WriteRune(unicode.ToLower(r1)) + buf.WriteRune(r0) + break + } + + if !unicode.IsUpper(r0) { + if isConnector(r0) { + r0 = connector + + buf.WriteRune(unicode.ToLower(r1)) + } else if unicode.IsNumber(r0) { + // treat a number as an upper case rune + // so that both `http2xx` and `HTTP2XX` can be converted to `http_2xx`. + buf.WriteRune(unicode.ToLower(r1)) + buf.WriteRune(connector) + buf.WriteRune(r0) + } else { + buf.WriteRune(connector) + buf.WriteRune(unicode.ToLower(r1)) + buf.WriteRune(r0) + } + + break + } + + buf.WriteRune(unicode.ToLower(r1)) + } + + if len(str) == 0 || r0 == connector { + buf.WriteRune(unicode.ToLower(r0)) + } + + case unicode.IsNumber(r0): + if prev != connector && !unicode.IsNumber(prev) { + buf.WriteRune(connector) + } + + buf.WriteRune(r0) + + default: + if isConnector(r0) { + r0 = connector + } + + buf.WriteRune(r0) + } + } + + return buf.String() +} + +func isConnector(r rune) bool { + return r == '-' || r == '_' || unicode.IsSpace(r) +} + +// SwapCase will swap characters case from upper to lower or lower to upper. +func SwapCase(str string) string { + var r rune + var size int + + buf := &bytes.Buffer{} + + for len(str) > 0 { + r, size = utf8.DecodeRuneInString(str) + + switch { + case unicode.IsUpper(r): + buf.WriteRune(unicode.ToLower(r)) + + case unicode.IsLower(r): + buf.WriteRune(unicode.ToUpper(r)) + + default: + buf.WriteRune(r) + } + + str = str[size:] + } + + return buf.String() +} + +// FirstRuneToUpper converts first rune to upper case if necessary. +func FirstRuneToUpper(str string) string { + if str == "" { + return str + } + + r, size := utf8.DecodeRuneInString(str) + + if !unicode.IsLower(r) { + return str + } + + buf := &bytes.Buffer{} + buf.WriteRune(unicode.ToUpper(r)) + buf.WriteString(str[size:]) + return buf.String() +} + +// FirstRuneToLower converts first rune to lower case if necessary. +func FirstRuneToLower(str string) string { + if str == "" { + return str + } + + r, size := utf8.DecodeRuneInString(str) + + if !unicode.IsUpper(r) { + return str + } + + buf := &bytes.Buffer{} + buf.WriteRune(unicode.ToLower(r)) + buf.WriteString(str[size:]) + return buf.String() +} + +// Shuffle randomizes runes in a string and returns the result. +// It uses default random source in `math/rand`. +func Shuffle(str string) string { + if str == "" { + return str + } + + runes := []rune(str) + index := 0 + + for i := len(runes) - 1; i > 0; i-- { + index = rand.Intn(i + 1) + + if i != index { + runes[i], runes[index] = runes[index], runes[i] + } + } + + return string(runes) +} + +// ShuffleSource randomizes runes in a string with given random source. +func ShuffleSource(str string, src rand.Source) string { + if str == "" { + return str + } + + runes := []rune(str) + index := 0 + r := rand.New(src) + + for i := len(runes) - 1; i > 0; i-- { + index = r.Intn(i + 1) + + if i != index { + runes[i], runes[index] = runes[index], runes[i] + } + } + + return string(runes) +} + +// Successor returns the successor to string. +// +// If there is one alphanumeric rune is found in string, increase the rune by 1. +// If increment generates a "carry", the rune to the left of it is incremented. +// This process repeats until there is no carry, adding an additional rune if necessary. +// +// If there is no alphanumeric rune, the rightmost rune will be increased by 1 +// regardless whether the result is a valid rune or not. +// +// Only following characters are alphanumeric. +// * a - z +// * A - Z +// * 0 - 9 +// +// Samples (borrowed from ruby's String#succ document): +// "abcd" => "abce" +// "THX1138" => "THX1139" +// "<>" => "<>" +// "1999zzz" => "2000aaa" +// "ZZZ9999" => "AAAA0000" +// "***" => "**+" +func Successor(str string) string { + if str == "" { + return str + } + + var r rune + var i int + carry := ' ' + runes := []rune(str) + l := len(runes) + lastAlphanumeric := l + + for i = l - 1; i >= 0; i-- { + r = runes[i] + + if ('a' <= r && r <= 'y') || + ('A' <= r && r <= 'Y') || + ('0' <= r && r <= '8') { + runes[i]++ + carry = ' ' + lastAlphanumeric = i + break + } + + switch r { + case 'z': + runes[i] = 'a' + carry = 'a' + lastAlphanumeric = i + + case 'Z': + runes[i] = 'A' + carry = 'A' + lastAlphanumeric = i + + case '9': + runes[i] = '0' + carry = '0' + lastAlphanumeric = i + } + } + + // Needs to add one character for carry. + if i < 0 && carry != ' ' { + buf := &bytes.Buffer{} + buf.Grow(l + 4) // Reserve enough space for write. + + if lastAlphanumeric != 0 { + buf.WriteString(str[:lastAlphanumeric]) + } + + buf.WriteRune(carry) + + for _, r = range runes[lastAlphanumeric:] { + buf.WriteRune(r) + } + + return buf.String() + } + + // No alphanumeric character. Simply increase last rune's value. + if lastAlphanumeric == l { + runes[l-1]++ + } + + return string(runes) +} diff --git a/github.com/huandu/xstrings/count.go b/github.com/huandu/xstrings/count.go new file mode 100644 index 0000000000..f96e38703a --- /dev/null +++ b/github.com/huandu/xstrings/count.go @@ -0,0 +1,120 @@ +// Copyright 2015 Huan Du. All rights reserved. +// Licensed under the MIT license that can be found in the LICENSE file. + +package xstrings + +import ( + "unicode" + "unicode/utf8" +) + +// Len returns str's utf8 rune length. +func Len(str string) int { + return utf8.RuneCountInString(str) +} + +// WordCount returns number of words in a string. +// +// Word is defined as a locale dependent string containing alphabetic characters, +// which may also contain but not start with `'` and `-` characters. +func WordCount(str string) int { + var r rune + var size, n int + + inWord := false + + for len(str) > 0 { + r, size = utf8.DecodeRuneInString(str) + + switch { + case isAlphabet(r): + if !inWord { + inWord = true + n++ + } + + case inWord && (r == '\'' || r == '-'): + // Still in word. + + default: + inWord = false + } + + str = str[size:] + } + + return n +} + +const minCJKCharacter = '\u3400' + +// Checks r is a letter but not CJK character. +func isAlphabet(r rune) bool { + if !unicode.IsLetter(r) { + return false + } + + switch { + // Quick check for non-CJK character. + case r < minCJKCharacter: + return true + + // Common CJK characters. + case r >= '\u4E00' && r <= '\u9FCC': + return false + + // Rare CJK characters. + case r >= '\u3400' && r <= '\u4D85': + return false + + // Rare and historic CJK characters. + case r >= '\U00020000' && r <= '\U0002B81D': + return false + } + + return true +} + +// Width returns string width in monotype font. +// Multi-byte characters are usually twice the width of single byte characters. +// +// Algorithm comes from `mb_strwidth` in PHP. +// http://php.net/manual/en/function.mb-strwidth.php +func Width(str string) int { + var r rune + var size, n int + + for len(str) > 0 { + r, size = utf8.DecodeRuneInString(str) + n += RuneWidth(r) + str = str[size:] + } + + return n +} + +// RuneWidth returns character width in monotype font. +// Multi-byte characters are usually twice the width of single byte characters. +// +// Algorithm comes from `mb_strwidth` in PHP. +// http://php.net/manual/en/function.mb-strwidth.php +func RuneWidth(r rune) int { + switch { + case r == utf8.RuneError || r < '\x20': + return 0 + + case '\x20' <= r && r < '\u2000': + return 1 + + case '\u2000' <= r && r < '\uFF61': + return 2 + + case '\uFF61' <= r && r < '\uFFA0': + return 1 + + case '\uFFA0' <= r: + return 2 + } + + return 0 +} diff --git a/github.com/huandu/xstrings/doc.go b/github.com/huandu/xstrings/doc.go new file mode 100644 index 0000000000..1a6ef069f6 --- /dev/null +++ b/github.com/huandu/xstrings/doc.go @@ -0,0 +1,8 @@ +// Copyright 2015 Huan Du. All rights reserved. +// Licensed under the MIT license that can be found in the LICENSE file. + +// Package xstrings is to provide string algorithms which are useful but not included in `strings` package. +// See project home page for details. https://github.com/huandu/xstrings +// +// Package xstrings assumes all strings are encoded in utf8. +package xstrings diff --git a/github.com/huandu/xstrings/format.go b/github.com/huandu/xstrings/format.go new file mode 100644 index 0000000000..2d02df1c04 --- /dev/null +++ b/github.com/huandu/xstrings/format.go @@ -0,0 +1,170 @@ +// Copyright 2015 Huan Du. All rights reserved. +// Licensed under the MIT license that can be found in the LICENSE file. + +package xstrings + +import ( + "bytes" + "unicode/utf8" +) + +// ExpandTabs can expand tabs ('\t') rune in str to one or more spaces dpending on +// current column and tabSize. +// The column number is reset to zero after each newline ('\n') occurring in the str. +// +// ExpandTabs uses RuneWidth to decide rune's width. +// For example, CJK characters will be treated as two characters. +// +// If tabSize <= 0, ExpandTabs panics with error. +// +// Samples: +// ExpandTabs("a\tbc\tdef\tghij\tk", 4) => "a bc def ghij k" +// ExpandTabs("abcdefg\thij\nk\tl", 4) => "abcdefg hij\nk l" +// ExpandTabs("z中\t文\tw", 4) => "z中 文 w" +func ExpandTabs(str string, tabSize int) string { + if tabSize <= 0 { + panic("tab size must be positive") + } + + var r rune + var i, size, column, expand int + var output *bytes.Buffer + + orig := str + + for len(str) > 0 { + r, size = utf8.DecodeRuneInString(str) + + if r == '\t' { + expand = tabSize - column%tabSize + + if output == nil { + output = allocBuffer(orig, str) + } + + for i = 0; i < expand; i++ { + output.WriteByte(byte(' ')) + } + + column += expand + } else { + if r == '\n' { + column = 0 + } else { + column += RuneWidth(r) + } + + if output != nil { + output.WriteRune(r) + } + } + + str = str[size:] + } + + if output == nil { + return orig + } + + return output.String() +} + +// LeftJustify returns a string with pad string at right side if str's rune length is smaller than length. +// If str's rune length is larger than length, str itself will be returned. +// +// If pad is an empty string, str will be returned. +// +// Samples: +// LeftJustify("hello", 4, " ") => "hello" +// LeftJustify("hello", 10, " ") => "hello " +// LeftJustify("hello", 10, "123") => "hello12312" +func LeftJustify(str string, length int, pad string) string { + l := Len(str) + + if l >= length || pad == "" { + return str + } + + remains := length - l + padLen := Len(pad) + + output := &bytes.Buffer{} + output.Grow(len(str) + (remains/padLen+1)*len(pad)) + output.WriteString(str) + writePadString(output, pad, padLen, remains) + return output.String() +} + +// RightJustify returns a string with pad string at left side if str's rune length is smaller than length. +// If str's rune length is larger than length, str itself will be returned. +// +// If pad is an empty string, str will be returned. +// +// Samples: +// RightJustify("hello", 4, " ") => "hello" +// RightJustify("hello", 10, " ") => " hello" +// RightJustify("hello", 10, "123") => "12312hello" +func RightJustify(str string, length int, pad string) string { + l := Len(str) + + if l >= length || pad == "" { + return str + } + + remains := length - l + padLen := Len(pad) + + output := &bytes.Buffer{} + output.Grow(len(str) + (remains/padLen+1)*len(pad)) + writePadString(output, pad, padLen, remains) + output.WriteString(str) + return output.String() +} + +// Center returns a string with pad string at both side if str's rune length is smaller than length. +// If str's rune length is larger than length, str itself will be returned. +// +// If pad is an empty string, str will be returned. +// +// Samples: +// Center("hello", 4, " ") => "hello" +// Center("hello", 10, " ") => " hello " +// Center("hello", 10, "123") => "12hello123" +func Center(str string, length int, pad string) string { + l := Len(str) + + if l >= length || pad == "" { + return str + } + + remains := length - l + padLen := Len(pad) + + output := &bytes.Buffer{} + output.Grow(len(str) + (remains/padLen+1)*len(pad)) + writePadString(output, pad, padLen, remains/2) + output.WriteString(str) + writePadString(output, pad, padLen, (remains+1)/2) + return output.String() +} + +func writePadString(output *bytes.Buffer, pad string, padLen, remains int) { + var r rune + var size int + + repeats := remains / padLen + + for i := 0; i < repeats; i++ { + output.WriteString(pad) + } + + remains = remains % padLen + + if remains != 0 { + for i := 0; i < remains; i++ { + r, size = utf8.DecodeRuneInString(pad) + output.WriteRune(r) + pad = pad[size:] + } + } +} diff --git a/github.com/huandu/xstrings/go.mod b/github.com/huandu/xstrings/go.mod new file mode 100644 index 0000000000..3982c204ca --- /dev/null +++ b/github.com/huandu/xstrings/go.mod @@ -0,0 +1,3 @@ +module github.com/huandu/xstrings + +go 1.12 diff --git a/github.com/huandu/xstrings/manipulate.go b/github.com/huandu/xstrings/manipulate.go new file mode 100644 index 0000000000..0eefb43ed7 --- /dev/null +++ b/github.com/huandu/xstrings/manipulate.go @@ -0,0 +1,217 @@ +// Copyright 2015 Huan Du. All rights reserved. +// Licensed under the MIT license that can be found in the LICENSE file. + +package xstrings + +import ( + "bytes" + "strings" + "unicode/utf8" +) + +// Reverse a utf8 encoded string. +func Reverse(str string) string { + var size int + + tail := len(str) + buf := make([]byte, tail) + s := buf + + for len(str) > 0 { + _, size = utf8.DecodeRuneInString(str) + tail -= size + s = append(s[:tail], []byte(str[:size])...) + str = str[size:] + } + + return string(buf) +} + +// Slice a string by rune. +// +// Start must satisfy 0 <= start <= rune length. +// +// End can be positive, zero or negative. +// If end >= 0, start and end must satisfy start <= end <= rune length. +// If end < 0, it means slice to the end of string. +// +// Otherwise, Slice will panic as out of range. +func Slice(str string, start, end int) string { + var size, startPos, endPos int + + origin := str + + if start < 0 || end > len(str) || (end >= 0 && start > end) { + panic("out of range") + } + + if end >= 0 { + end -= start + } + + for start > 0 && len(str) > 0 { + _, size = utf8.DecodeRuneInString(str) + start-- + startPos += size + str = str[size:] + } + + if end < 0 { + return origin[startPos:] + } + + endPos = startPos + + for end > 0 && len(str) > 0 { + _, size = utf8.DecodeRuneInString(str) + end-- + endPos += size + str = str[size:] + } + + if len(str) == 0 && (start > 0 || end > 0) { + panic("out of range") + } + + return origin[startPos:endPos] +} + +// Partition splits a string by sep into three parts. +// The return value is a slice of strings with head, match and tail. +// +// If str contains sep, for example "hello" and "l", Partition returns +// "he", "l", "lo" +// +// If str doesn't contain sep, for example "hello" and "x", Partition returns +// "hello", "", "" +func Partition(str, sep string) (head, match, tail string) { + index := strings.Index(str, sep) + + if index == -1 { + head = str + return + } + + head = str[:index] + match = str[index : index+len(sep)] + tail = str[index+len(sep):] + return +} + +// LastPartition splits a string by last instance of sep into three parts. +// The return value is a slice of strings with head, match and tail. +// +// If str contains sep, for example "hello" and "l", LastPartition returns +// "hel", "l", "o" +// +// If str doesn't contain sep, for example "hello" and "x", LastPartition returns +// "", "", "hello" +func LastPartition(str, sep string) (head, match, tail string) { + index := strings.LastIndex(str, sep) + + if index == -1 { + tail = str + return + } + + head = str[:index] + match = str[index : index+len(sep)] + tail = str[index+len(sep):] + return +} + +// Insert src into dst at given rune index. +// Index is counted by runes instead of bytes. +// +// If index is out of range of dst, panic with out of range. +func Insert(dst, src string, index int) string { + return Slice(dst, 0, index) + src + Slice(dst, index, -1) +} + +// Scrub scrubs invalid utf8 bytes with repl string. +// Adjacent invalid bytes are replaced only once. +func Scrub(str, repl string) string { + var buf *bytes.Buffer + var r rune + var size, pos int + var hasError bool + + origin := str + + for len(str) > 0 { + r, size = utf8.DecodeRuneInString(str) + + if r == utf8.RuneError { + if !hasError { + if buf == nil { + buf = &bytes.Buffer{} + } + + buf.WriteString(origin[:pos]) + hasError = true + } + } else if hasError { + hasError = false + buf.WriteString(repl) + + origin = origin[pos:] + pos = 0 + } + + pos += size + str = str[size:] + } + + if buf != nil { + buf.WriteString(origin) + return buf.String() + } + + // No invalid byte. + return origin +} + +// WordSplit splits a string into words. Returns a slice of words. +// If there is no word in a string, return nil. +// +// Word is defined as a locale dependent string containing alphabetic characters, +// which may also contain but not start with `'` and `-` characters. +func WordSplit(str string) []string { + var word string + var words []string + var r rune + var size, pos int + + inWord := false + + for len(str) > 0 { + r, size = utf8.DecodeRuneInString(str) + + switch { + case isAlphabet(r): + if !inWord { + inWord = true + word = str + pos = 0 + } + + case inWord && (r == '\'' || r == '-'): + // Still in word. + + default: + if inWord { + inWord = false + words = append(words, word[:pos]) + } + } + + pos += size + str = str[size:] + } + + if inWord { + words = append(words, word[:pos]) + } + + return words +} diff --git a/github.com/huandu/xstrings/translate.go b/github.com/huandu/xstrings/translate.go new file mode 100644 index 0000000000..66e23f86d0 --- /dev/null +++ b/github.com/huandu/xstrings/translate.go @@ -0,0 +1,547 @@ +// Copyright 2015 Huan Du. All rights reserved. +// Licensed under the MIT license that can be found in the LICENSE file. + +package xstrings + +import ( + "bytes" + "unicode" + "unicode/utf8" +) + +type runeRangeMap struct { + FromLo rune // Lower bound of range map. + FromHi rune // An inclusive higher bound of range map. + ToLo rune + ToHi rune +} + +type runeDict struct { + Dict [unicode.MaxASCII + 1]rune +} + +type runeMap map[rune]rune + +// Translator can translate string with pre-compiled from and to patterns. +// If a from/to pattern pair needs to be used more than once, it's recommended +// to create a Translator and reuse it. +type Translator struct { + quickDict *runeDict // A quick dictionary to look up rune by index. Only available for latin runes. + runeMap runeMap // Rune map for translation. + ranges []*runeRangeMap // Ranges of runes. + mappedRune rune // If mappedRune >= 0, all matched runes are translated to the mappedRune. + reverted bool // If to pattern is empty, all matched characters will be deleted. + hasPattern bool +} + +// NewTranslator creates new Translator through a from/to pattern pair. +func NewTranslator(from, to string) *Translator { + tr := &Translator{} + + if from == "" { + return tr + } + + reverted := from[0] == '^' + deletion := len(to) == 0 + + if reverted { + from = from[1:] + } + + var fromStart, fromEnd, fromRangeStep rune + var toStart, toEnd, toRangeStep rune + var fromRangeSize, toRangeSize rune + var singleRunes []rune + + // Update the to rune range. + updateRange := func() { + // No more rune to read in the to rune pattern. + if toEnd == utf8.RuneError { + return + } + + if toRangeStep == 0 { + to, toStart, toEnd, toRangeStep = nextRuneRange(to, toEnd) + return + } + + // Current range is not empty. Consume 1 rune from start. + if toStart != toEnd { + toStart += toRangeStep + return + } + + // No more rune. Repeat the last rune. + if to == "" { + toEnd = utf8.RuneError + return + } + + // Both start and end are used. Read two more runes from the to pattern. + to, toStart, toEnd, toRangeStep = nextRuneRange(to, utf8.RuneError) + } + + if deletion { + toStart = utf8.RuneError + toEnd = utf8.RuneError + } else { + // If from pattern is reverted, only the last rune in the to pattern will be used. + if reverted { + var size int + + for len(to) > 0 { + toStart, size = utf8.DecodeRuneInString(to) + to = to[size:] + } + + toEnd = utf8.RuneError + } else { + to, toStart, toEnd, toRangeStep = nextRuneRange(to, utf8.RuneError) + } + } + + fromEnd = utf8.RuneError + + for len(from) > 0 { + from, fromStart, fromEnd, fromRangeStep = nextRuneRange(from, fromEnd) + + // fromStart is a single character. Just map it with a rune in the to pattern. + if fromRangeStep == 0 { + singleRunes = tr.addRune(fromStart, toStart, singleRunes) + updateRange() + continue + } + + for toEnd != utf8.RuneError && fromStart != fromEnd { + // If mapped rune is a single character instead of a range, simply shift first + // rune in the range. + if toRangeStep == 0 { + singleRunes = tr.addRune(fromStart, toStart, singleRunes) + updateRange() + fromStart += fromRangeStep + continue + } + + fromRangeSize = (fromEnd - fromStart) * fromRangeStep + toRangeSize = (toEnd - toStart) * toRangeStep + + // Not enough runes in the to pattern. Need to read more. + if fromRangeSize > toRangeSize { + fromStart, toStart = tr.addRuneRange(fromStart, fromStart+toRangeSize*fromRangeStep, toStart, toEnd, singleRunes) + fromStart += fromRangeStep + updateRange() + + // Edge case: If fromRangeSize == toRangeSize + 1, the last fromStart value needs be considered + // as a single rune. + if fromStart == fromEnd { + singleRunes = tr.addRune(fromStart, toStart, singleRunes) + updateRange() + } + + continue + } + + fromStart, toStart = tr.addRuneRange(fromStart, fromEnd, toStart, toStart+fromRangeSize*toRangeStep, singleRunes) + updateRange() + break + } + + if fromStart == fromEnd { + fromEnd = utf8.RuneError + continue + } + + fromStart, toStart = tr.addRuneRange(fromStart, fromEnd, toStart, toStart, singleRunes) + fromEnd = utf8.RuneError + } + + if fromEnd != utf8.RuneError { + singleRunes = tr.addRune(fromEnd, toStart, singleRunes) + } + + tr.reverted = reverted + tr.mappedRune = -1 + tr.hasPattern = true + + // Translate RuneError only if in deletion or reverted mode. + if deletion || reverted { + tr.mappedRune = toStart + } + + return tr +} + +func (tr *Translator) addRune(from, to rune, singleRunes []rune) []rune { + if from <= unicode.MaxASCII { + if tr.quickDict == nil { + tr.quickDict = &runeDict{} + } + + tr.quickDict.Dict[from] = to + } else { + if tr.runeMap == nil { + tr.runeMap = make(runeMap) + } + + tr.runeMap[from] = to + } + + singleRunes = append(singleRunes, from) + return singleRunes +} + +func (tr *Translator) addRuneRange(fromLo, fromHi, toLo, toHi rune, singleRunes []rune) (rune, rune) { + var r rune + var rrm *runeRangeMap + + if fromLo < fromHi { + rrm = &runeRangeMap{ + FromLo: fromLo, + FromHi: fromHi, + ToLo: toLo, + ToHi: toHi, + } + } else { + rrm = &runeRangeMap{ + FromLo: fromHi, + FromHi: fromLo, + ToLo: toHi, + ToHi: toLo, + } + } + + // If there is any single rune conflicts with this rune range, clear single rune record. + for _, r = range singleRunes { + if rrm.FromLo <= r && r <= rrm.FromHi { + if r <= unicode.MaxASCII { + tr.quickDict.Dict[r] = 0 + } else { + delete(tr.runeMap, r) + } + } + } + + tr.ranges = append(tr.ranges, rrm) + return fromHi, toHi +} + +func nextRuneRange(str string, last rune) (remaining string, start, end rune, rangeStep rune) { + var r rune + var size int + + remaining = str + escaping := false + isRange := false + + for len(remaining) > 0 { + r, size = utf8.DecodeRuneInString(remaining) + remaining = remaining[size:] + + // Parse special characters. + if !escaping { + if r == '\\' { + escaping = true + continue + } + + if r == '-' { + // Ignore slash at beginning of string. + if last == utf8.RuneError { + continue + } + + start = last + isRange = true + continue + } + } + + escaping = false + + if last != utf8.RuneError { + // This is a range which start and end are the same. + // Considier it as a normal character. + if isRange && last == r { + isRange = false + continue + } + + start = last + end = r + + if isRange { + if start < end { + rangeStep = 1 + } else { + rangeStep = -1 + } + } + + return + } + + last = r + } + + start = last + end = utf8.RuneError + return +} + +// Translate str with a from/to pattern pair. +// +// See comment in Translate function for usage and samples. +func (tr *Translator) Translate(str string) string { + if !tr.hasPattern || str == "" { + return str + } + + var r rune + var size int + var needTr bool + + orig := str + + var output *bytes.Buffer + + for len(str) > 0 { + r, size = utf8.DecodeRuneInString(str) + r, needTr = tr.TranslateRune(r) + + if needTr && output == nil { + output = allocBuffer(orig, str) + } + + if r != utf8.RuneError && output != nil { + output.WriteRune(r) + } + + str = str[size:] + } + + // No character is translated. + if output == nil { + return orig + } + + return output.String() +} + +// TranslateRune return translated rune and true if r matches the from pattern. +// If r doesn't match the pattern, original r is returned and translated is false. +func (tr *Translator) TranslateRune(r rune) (result rune, translated bool) { + switch { + case tr.quickDict != nil: + if r <= unicode.MaxASCII { + result = tr.quickDict.Dict[r] + + if result != 0 { + translated = true + + if tr.mappedRune >= 0 { + result = tr.mappedRune + } + + break + } + } + + fallthrough + + case tr.runeMap != nil: + var ok bool + + if result, ok = tr.runeMap[r]; ok { + translated = true + + if tr.mappedRune >= 0 { + result = tr.mappedRune + } + + break + } + + fallthrough + + default: + var rrm *runeRangeMap + ranges := tr.ranges + + for i := len(ranges) - 1; i >= 0; i-- { + rrm = ranges[i] + + if rrm.FromLo <= r && r <= rrm.FromHi { + translated = true + + if tr.mappedRune >= 0 { + result = tr.mappedRune + break + } + + if rrm.ToLo < rrm.ToHi { + result = rrm.ToLo + r - rrm.FromLo + } else if rrm.ToLo > rrm.ToHi { + // ToHi can be smaller than ToLo if range is from higher to lower. + result = rrm.ToLo - r + rrm.FromLo + } else { + result = rrm.ToLo + } + + break + } + } + } + + if tr.reverted { + if !translated { + result = tr.mappedRune + } + + translated = !translated + } + + if !translated { + result = r + } + + return +} + +// HasPattern returns true if Translator has one pattern at least. +func (tr *Translator) HasPattern() bool { + return tr.hasPattern +} + +// Translate str with the characters defined in from replaced by characters defined in to. +// +// From and to are patterns representing a set of characters. Pattern is defined as following. +// +// * Special characters +// * '-' means a range of runes, e.g. +// * "a-z" means all characters from 'a' to 'z' inclusive; +// * "z-a" means all characters from 'z' to 'a' inclusive. +// * '^' as first character means a set of all runes excepted listed, e.g. +// * "^a-z" means all characters except 'a' to 'z' inclusive. +// * '\' escapes special characters. +// * Normal character represents itself, e.g. "abc" is a set including 'a', 'b' and 'c'. +// +// Translate will try to find a 1:1 mapping from from to to. +// If to is smaller than from, last rune in to will be used to map "out of range" characters in from. +// +// Note that '^' only works in the from pattern. It will be considered as a normal character in the to pattern. +// +// If the to pattern is an empty string, Translate works exactly the same as Delete. +// +// Samples: +// Translate("hello", "aeiou", "12345") => "h2ll4" +// Translate("hello", "a-z", "A-Z") => "HELLO" +// Translate("hello", "z-a", "a-z") => "svool" +// Translate("hello", "aeiou", "*") => "h*ll*" +// Translate("hello", "^l", "*") => "**ll*" +// Translate("hello ^ world", `\^lo`, "*") => "he*** * w*r*d" +func Translate(str, from, to string) string { + tr := NewTranslator(from, to) + return tr.Translate(str) +} + +// Delete runes in str matching the pattern. +// Pattern is defined in Translate function. +// +// Samples: +// Delete("hello", "aeiou") => "hll" +// Delete("hello", "a-k") => "llo" +// Delete("hello", "^a-k") => "he" +func Delete(str, pattern string) string { + tr := NewTranslator(pattern, "") + return tr.Translate(str) +} + +// Count how many runes in str match the pattern. +// Pattern is defined in Translate function. +// +// Samples: +// Count("hello", "aeiou") => 3 +// Count("hello", "a-k") => 3 +// Count("hello", "^a-k") => 2 +func Count(str, pattern string) int { + if pattern == "" || str == "" { + return 0 + } + + var r rune + var size int + var matched bool + + tr := NewTranslator(pattern, "") + cnt := 0 + + for len(str) > 0 { + r, size = utf8.DecodeRuneInString(str) + str = str[size:] + + if _, matched = tr.TranslateRune(r); matched { + cnt++ + } + } + + return cnt +} + +// Squeeze deletes adjacent repeated runes in str. +// If pattern is not empty, only runes matching the pattern will be squeezed. +// +// Samples: +// Squeeze("hello", "") => "helo" +// Squeeze("hello", "m-z") => "hello" +// Squeeze("hello world", " ") => "hello world" +func Squeeze(str, pattern string) string { + var last, r rune + var size int + var skipSqueeze, matched bool + var tr *Translator + var output *bytes.Buffer + + orig := str + last = -1 + + if len(pattern) > 0 { + tr = NewTranslator(pattern, "") + } + + for len(str) > 0 { + r, size = utf8.DecodeRuneInString(str) + + // Need to squeeze the str. + if last == r && !skipSqueeze { + if tr != nil { + if _, matched = tr.TranslateRune(r); !matched { + skipSqueeze = true + } + } + + if output == nil { + output = allocBuffer(orig, str) + } + + if skipSqueeze { + output.WriteRune(r) + } + } else { + if output != nil { + output.WriteRune(r) + } + + last = r + skipSqueeze = false + } + + str = str[size:] + } + + if output == nil { + return orig + } + + return output.String() +} diff --git a/github.com/imdario/mergo/.deepsource.toml b/github.com/imdario/mergo/.deepsource.toml new file mode 100644 index 0000000000..8a0681af85 --- /dev/null +++ b/github.com/imdario/mergo/.deepsource.toml @@ -0,0 +1,12 @@ +version = 1 + +test_patterns = [ + "*_test.go" +] + +[[analyzers]] +name = "go" +enabled = true + + [analyzers.meta] + import_path = "github.com/imdario/mergo" \ No newline at end of file diff --git a/github.com/imdario/mergo/.gitignore b/github.com/imdario/mergo/.gitignore new file mode 100644 index 0000000000..529c3412ba --- /dev/null +++ b/github.com/imdario/mergo/.gitignore @@ -0,0 +1,33 @@ +#### joe made this: http://goel.io/joe + +#### go #### +# Binaries for programs and plugins +*.exe +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 +.glide/ + +#### vim #### +# Swap +[._]*.s[a-v][a-z] +[._]*.sw[a-p] +[._]s[a-v][a-z] +[._]sw[a-p] + +# Session +Session.vim + +# Temporary +.netrwhist +*~ +# Auto-generated tag files +tags diff --git a/github.com/imdario/mergo/.travis.yml b/github.com/imdario/mergo/.travis.yml new file mode 100644 index 0000000000..dad29725f8 --- /dev/null +++ b/github.com/imdario/mergo/.travis.yml @@ -0,0 +1,9 @@ +language: go +install: + - go get -t + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls +script: + - go test -race -v ./... +after_script: + - $HOME/gopath/bin/goveralls -service=travis-ci -repotoken $COVERALLS_TOKEN diff --git a/github.com/imdario/mergo/CODE_OF_CONDUCT.md b/github.com/imdario/mergo/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..469b44907a --- /dev/null +++ b/github.com/imdario/mergo/CODE_OF_CONDUCT.md @@ -0,0 +1,46 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at i@dario.im. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/4/ diff --git a/github.com/imdario/mergo/LICENSE b/github.com/imdario/mergo/LICENSE new file mode 100644 index 0000000000..686680298d --- /dev/null +++ b/github.com/imdario/mergo/LICENSE @@ -0,0 +1,28 @@ +Copyright (c) 2013 Dario Castañé. All rights reserved. +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/github.com/imdario/mergo/README.md b/github.com/imdario/mergo/README.md new file mode 100644 index 0000000000..02fc81e062 --- /dev/null +++ b/github.com/imdario/mergo/README.md @@ -0,0 +1,238 @@ +# Mergo + +A helper to merge structs and maps in Golang. Useful for configuration default values, avoiding messy if-statements. + +Also a lovely [comune](http://en.wikipedia.org/wiki/Mergo) (municipality) in the Province of Ancona in the Italian region of Marche. + +## Status + +It is ready for production use. [It is used in several projects by Docker, Google, The Linux Foundation, VMWare, Shopify, etc](https://github.com/imdario/mergo#mergo-in-the-wild). + +[![GoDoc][3]][4] +[![GoCard][5]][6] +[![Build Status][1]][2] +[![Coverage Status][7]][8] +[![Sourcegraph][9]][10] +[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fimdario%2Fmergo.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2Fimdario%2Fmergo?ref=badge_shield) + +[1]: https://travis-ci.org/imdario/mergo.png +[2]: https://travis-ci.org/imdario/mergo +[3]: https://godoc.org/github.com/imdario/mergo?status.svg +[4]: https://godoc.org/github.com/imdario/mergo +[5]: https://goreportcard.com/badge/imdario/mergo +[6]: https://goreportcard.com/report/github.com/imdario/mergo +[7]: https://coveralls.io/repos/github/imdario/mergo/badge.svg?branch=master +[8]: https://coveralls.io/github/imdario/mergo?branch=master +[9]: https://sourcegraph.com/github.com/imdario/mergo/-/badge.svg +[10]: https://sourcegraph.com/github.com/imdario/mergo?badge + +### Latest release + +[Release v0.3.7](https://github.com/imdario/mergo/releases/tag/v0.3.7). + +### Important note + +Please keep in mind that in [0.3.2](//github.com/imdario/mergo/releases/tag/0.3.2) Mergo changed `Merge()`and `Map()` signatures to support [transformers](#transformers). An optional/variadic argument has been added, so it won't break existing code. + +If you were using Mergo **before** April 6th 2015, please check your project works as intended after updating your local copy with ```go get -u github.com/imdario/mergo```. I apologize for any issue caused by its previous behavior and any future bug that Mergo could cause (I hope it won't!) in existing projects after the change (release 0.2.0). + +### Donations + +If Mergo is useful to you, consider buying me a coffee, a beer or making a monthly donation so I can keep building great free software. :heart_eyes: + +Buy Me a Coffee at ko-fi.com +[![Beerpay](https://beerpay.io/imdario/mergo/badge.svg)](https://beerpay.io/imdario/mergo) +[![Beerpay](https://beerpay.io/imdario/mergo/make-wish.svg)](https://beerpay.io/imdario/mergo) +Donate using Liberapay + +### Mergo in the wild + +- [moby/moby](https://github.com/moby/moby) +- [kubernetes/kubernetes](https://github.com/kubernetes/kubernetes) +- [vmware/dispatch](https://github.com/vmware/dispatch) +- [Shopify/themekit](https://github.com/Shopify/themekit) +- [imdario/zas](https://github.com/imdario/zas) +- [matcornic/hermes](https://github.com/matcornic/hermes) +- [OpenBazaar/openbazaar-go](https://github.com/OpenBazaar/openbazaar-go) +- [kataras/iris](https://github.com/kataras/iris) +- [michaelsauter/crane](https://github.com/michaelsauter/crane) +- [go-task/task](https://github.com/go-task/task) +- [sensu/uchiwa](https://github.com/sensu/uchiwa) +- [ory/hydra](https://github.com/ory/hydra) +- [sisatech/vcli](https://github.com/sisatech/vcli) +- [dairycart/dairycart](https://github.com/dairycart/dairycart) +- [projectcalico/felix](https://github.com/projectcalico/felix) +- [resin-os/balena](https://github.com/resin-os/balena) +- [go-kivik/kivik](https://github.com/go-kivik/kivik) +- [Telefonica/govice](https://github.com/Telefonica/govice) +- [supergiant/supergiant](supergiant/supergiant) +- [SergeyTsalkov/brooce](https://github.com/SergeyTsalkov/brooce) +- [soniah/dnsmadeeasy](https://github.com/soniah/dnsmadeeasy) +- [ohsu-comp-bio/funnel](https://github.com/ohsu-comp-bio/funnel) +- [EagerIO/Stout](https://github.com/EagerIO/Stout) +- [lynndylanhurley/defsynth-api](https://github.com/lynndylanhurley/defsynth-api) +- [russross/canvasassignments](https://github.com/russross/canvasassignments) +- [rdegges/cryptly-api](https://github.com/rdegges/cryptly-api) +- [casualjim/exeggutor](https://github.com/casualjim/exeggutor) +- [divshot/gitling](https://github.com/divshot/gitling) +- [RWJMurphy/gorl](https://github.com/RWJMurphy/gorl) +- [andrerocker/deploy42](https://github.com/andrerocker/deploy42) +- [elwinar/rambler](https://github.com/elwinar/rambler) +- [tmaiaroto/gopartman](https://github.com/tmaiaroto/gopartman) +- [jfbus/impressionist](https://github.com/jfbus/impressionist) +- [Jmeyering/zealot](https://github.com/Jmeyering/zealot) +- [godep-migrator/rigger-host](https://github.com/godep-migrator/rigger-host) +- [Dronevery/MultiwaySwitch-Go](https://github.com/Dronevery/MultiwaySwitch-Go) +- [thoas/picfit](https://github.com/thoas/picfit) +- [mantasmatelis/whooplist-server](https://github.com/mantasmatelis/whooplist-server) +- [jnuthong/item_search](https://github.com/jnuthong/item_search) +- [bukalapak/snowboard](https://github.com/bukalapak/snowboard) + +## Installation + + go get github.com/imdario/mergo + + // use in your .go code + import ( + "github.com/imdario/mergo" + ) + +## Usage + +You can only merge same-type structs with exported fields initialized as zero value of their type and same-types maps. Mergo won't merge unexported (private) fields but will do recursively any exported one. It won't merge empty structs value as [they are not considered zero values](https://golang.org/ref/spec#The_zero_value) either. Also maps will be merged recursively except for structs inside maps (because they are not addressable using Go reflection). + +```go +if err := mergo.Merge(&dst, src); err != nil { + // ... +} +``` + +Also, you can merge overwriting values using the transformer `WithOverride`. + +```go +if err := mergo.Merge(&dst, src, mergo.WithOverride); err != nil { + // ... +} +``` + +Additionally, you can map a `map[string]interface{}` to a struct (and otherwise, from struct to map), following the same restrictions as in `Merge()`. Keys are capitalized to find each corresponding exported field. + +```go +if err := mergo.Map(&dst, srcMap); err != nil { + // ... +} +``` + +Warning: if you map a struct to map, it won't do it recursively. Don't expect Mergo to map struct members of your struct as `map[string]interface{}`. They will be just assigned as values. + +More information and examples in [godoc documentation](http://godoc.org/github.com/imdario/mergo). + +### Nice example + +```go +package main + +import ( + "fmt" + "github.com/imdario/mergo" +) + +type Foo struct { + A string + B int64 +} + +func main() { + src := Foo{ + A: "one", + B: 2, + } + dest := Foo{ + A: "two", + } + mergo.Merge(&dest, src) + fmt.Println(dest) + // Will print + // {two 2} +} +``` + +Note: if test are failing due missing package, please execute: + + go get gopkg.in/yaml.v2 + +### Transformers + +Transformers allow to merge specific types differently than in the default behavior. In other words, now you can customize how some types are merged. For example, `time.Time` is a struct; it doesn't have zero value but IsZero can return true because it has fields with zero value. How can we merge a non-zero `time.Time`? + +```go +package main + +import ( + "fmt" + "github.com/imdario/mergo" + "reflect" + "time" +) + +type timeTransfomer struct { +} + +func (t timeTransfomer) Transformer(typ reflect.Type) func(dst, src reflect.Value) error { + if typ == reflect.TypeOf(time.Time{}) { + return func(dst, src reflect.Value) error { + if dst.CanSet() { + isZero := dst.MethodByName("IsZero") + result := isZero.Call([]reflect.Value{}) + if result[0].Bool() { + dst.Set(src) + } + } + return nil + } + } + return nil +} + +type Snapshot struct { + Time time.Time + // ... +} + +func main() { + src := Snapshot{time.Now()} + dest := Snapshot{} + mergo.Merge(&dest, src, mergo.WithTransformers(timeTransfomer{})) + fmt.Println(dest) + // Will print + // { 2018-01-12 01:15:00 +0000 UTC m=+0.000000001 } +} +``` + + +## Contact me + +If I can help you, you have an idea or you are using Mergo in your projects, don't hesitate to drop me a line (or a pull request): [@im_dario](https://twitter.com/im_dario) + +## About + +Written by [Dario Castañé](http://dario.im). + +## Top Contributors + +[![0](https://sourcerer.io/fame/imdario/imdario/mergo/images/0)](https://sourcerer.io/fame/imdario/imdario/mergo/links/0) +[![1](https://sourcerer.io/fame/imdario/imdario/mergo/images/1)](https://sourcerer.io/fame/imdario/imdario/mergo/links/1) +[![2](https://sourcerer.io/fame/imdario/imdario/mergo/images/2)](https://sourcerer.io/fame/imdario/imdario/mergo/links/2) +[![3](https://sourcerer.io/fame/imdario/imdario/mergo/images/3)](https://sourcerer.io/fame/imdario/imdario/mergo/links/3) +[![4](https://sourcerer.io/fame/imdario/imdario/mergo/images/4)](https://sourcerer.io/fame/imdario/imdario/mergo/links/4) +[![5](https://sourcerer.io/fame/imdario/imdario/mergo/images/5)](https://sourcerer.io/fame/imdario/imdario/mergo/links/5) +[![6](https://sourcerer.io/fame/imdario/imdario/mergo/images/6)](https://sourcerer.io/fame/imdario/imdario/mergo/links/6) +[![7](https://sourcerer.io/fame/imdario/imdario/mergo/images/7)](https://sourcerer.io/fame/imdario/imdario/mergo/links/7) + + +## License + +[BSD 3-Clause](http://opensource.org/licenses/BSD-3-Clause) license, as [Go language](http://golang.org/LICENSE). + + +[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fimdario%2Fmergo.svg?type=large)](https://app.fossa.io/projects/git%2Bgithub.com%2Fimdario%2Fmergo?ref=badge_large) diff --git a/github.com/imdario/mergo/doc.go b/github.com/imdario/mergo/doc.go new file mode 100644 index 0000000000..6e9aa7baf3 --- /dev/null +++ b/github.com/imdario/mergo/doc.go @@ -0,0 +1,44 @@ +// Copyright 2013 Dario Castañé. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package mergo merges same-type structs and maps by setting default values in zero-value fields. + +Mergo won't merge unexported (private) fields but will do recursively any exported one. It also won't merge structs inside maps (because they are not addressable using Go reflection). + +Usage + +From my own work-in-progress project: + + type networkConfig struct { + Protocol string + Address string + ServerType string `json: "server_type"` + Port uint16 + } + + type FssnConfig struct { + Network networkConfig + } + + var fssnDefault = FssnConfig { + networkConfig { + "tcp", + "127.0.0.1", + "http", + 31560, + }, + } + + // Inside a function [...] + + if err := mergo.Merge(&config, fssnDefault); err != nil { + log.Fatal(err) + } + + // More code [...] + +*/ +package mergo diff --git a/github.com/imdario/mergo/map.go b/github.com/imdario/mergo/map.go new file mode 100644 index 0000000000..d83258b4dd --- /dev/null +++ b/github.com/imdario/mergo/map.go @@ -0,0 +1,176 @@ +// Copyright 2014 Dario Castañé. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Based on src/pkg/reflect/deepequal.go from official +// golang's stdlib. + +package mergo + +import ( + "fmt" + "reflect" + "unicode" + "unicode/utf8" +) + +func changeInitialCase(s string, mapper func(rune) rune) string { + if s == "" { + return s + } + r, n := utf8.DecodeRuneInString(s) + return string(mapper(r)) + s[n:] +} + +func isExported(field reflect.StructField) bool { + r, _ := utf8.DecodeRuneInString(field.Name) + return r >= 'A' && r <= 'Z' +} + +// Traverses recursively both values, assigning src's fields values to dst. +// The map argument tracks comparisons that have already been seen, which allows +// short circuiting on recursive types. +func deepMap(dst, src reflect.Value, visited map[uintptr]*visit, depth int, config *Config) (err error) { + overwrite := config.Overwrite + if dst.CanAddr() { + addr := dst.UnsafeAddr() + h := 17 * addr + seen := visited[h] + typ := dst.Type() + for p := seen; p != nil; p = p.next { + if p.ptr == addr && p.typ == typ { + return nil + } + } + // Remember, remember... + visited[h] = &visit{addr, typ, seen} + } + zeroValue := reflect.Value{} + switch dst.Kind() { + case reflect.Map: + dstMap := dst.Interface().(map[string]interface{}) + for i, n := 0, src.NumField(); i < n; i++ { + srcType := src.Type() + field := srcType.Field(i) + if !isExported(field) { + continue + } + fieldName := field.Name + fieldName = changeInitialCase(fieldName, unicode.ToLower) + if v, ok := dstMap[fieldName]; !ok || (isEmptyValue(reflect.ValueOf(v)) || overwrite) { + dstMap[fieldName] = src.Field(i).Interface() + } + } + case reflect.Ptr: + if dst.IsNil() { + v := reflect.New(dst.Type().Elem()) + dst.Set(v) + } + dst = dst.Elem() + fallthrough + case reflect.Struct: + srcMap := src.Interface().(map[string]interface{}) + for key := range srcMap { + config.overwriteWithEmptyValue = true + srcValue := srcMap[key] + fieldName := changeInitialCase(key, unicode.ToUpper) + dstElement := dst.FieldByName(fieldName) + if dstElement == zeroValue { + // We discard it because the field doesn't exist. + continue + } + srcElement := reflect.ValueOf(srcValue) + dstKind := dstElement.Kind() + srcKind := srcElement.Kind() + if srcKind == reflect.Ptr && dstKind != reflect.Ptr { + srcElement = srcElement.Elem() + srcKind = reflect.TypeOf(srcElement.Interface()).Kind() + } else if dstKind == reflect.Ptr { + // Can this work? I guess it can't. + if srcKind != reflect.Ptr && srcElement.CanAddr() { + srcPtr := srcElement.Addr() + srcElement = reflect.ValueOf(srcPtr) + srcKind = reflect.Ptr + } + } + + if !srcElement.IsValid() { + continue + } + if srcKind == dstKind { + if _, err = deepMerge(dstElement, srcElement, visited, depth+1, config); err != nil { + return + } + } else if dstKind == reflect.Interface && dstElement.Kind() == reflect.Interface { + if _, err = deepMerge(dstElement, srcElement, visited, depth+1, config); err != nil { + return + } + } else if srcKind == reflect.Map { + if err = deepMap(dstElement, srcElement, visited, depth+1, config); err != nil { + return + } + } else { + return fmt.Errorf("type mismatch on %s field: found %v, expected %v", fieldName, srcKind, dstKind) + } + } + } + return +} + +// Map sets fields' values in dst from src. +// src can be a map with string keys or a struct. dst must be the opposite: +// if src is a map, dst must be a valid pointer to struct. If src is a struct, +// dst must be map[string]interface{}. +// It won't merge unexported (private) fields and will do recursively +// any exported field. +// If dst is a map, keys will be src fields' names in lower camel case. +// Missing key in src that doesn't match a field in dst will be skipped. This +// doesn't apply if dst is a map. +// This is separated method from Merge because it is cleaner and it keeps sane +// semantics: merging equal types, mapping different (restricted) types. +func Map(dst, src interface{}, opts ...func(*Config)) error { + return _map(dst, src, opts...) +} + +// MapWithOverwrite will do the same as Map except that non-empty dst attributes will be overridden by +// non-empty src attribute values. +// Deprecated: Use Map(…) with WithOverride +func MapWithOverwrite(dst, src interface{}, opts ...func(*Config)) error { + return _map(dst, src, append(opts, WithOverride)...) +} + +func _map(dst, src interface{}, opts ...func(*Config)) error { + var ( + vDst, vSrc reflect.Value + err error + ) + config := &Config{} + + for _, opt := range opts { + opt(config) + } + + if vDst, vSrc, err = resolveValues(dst, src); err != nil { + return err + } + // To be friction-less, we redirect equal-type arguments + // to deepMerge. Only because arguments can be anything. + if vSrc.Kind() == vDst.Kind() { + _, err := deepMerge(vDst, vSrc, make(map[uintptr]*visit), 0, config) + return err + } + switch vSrc.Kind() { + case reflect.Struct: + if vDst.Kind() != reflect.Map { + return ErrExpectedMapAsDestination + } + case reflect.Map: + if vDst.Kind() != reflect.Struct { + return ErrExpectedStructAsDestination + } + default: + return ErrNotSupported + } + return deepMap(vDst, vSrc, make(map[uintptr]*visit), 0, config) +} diff --git a/github.com/imdario/mergo/merge.go b/github.com/imdario/mergo/merge.go new file mode 100644 index 0000000000..3332c9c2a7 --- /dev/null +++ b/github.com/imdario/mergo/merge.go @@ -0,0 +1,338 @@ +// Copyright 2013 Dario Castañé. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Based on src/pkg/reflect/deepequal.go from official +// golang's stdlib. + +package mergo + +import ( + "fmt" + "reflect" + "unsafe" +) + +func hasExportedField(dst reflect.Value) (exported bool) { + for i, n := 0, dst.NumField(); i < n; i++ { + field := dst.Type().Field(i) + if isExportedComponent(&field) { + return true + } + } + return +} + +func isExportedComponent(field *reflect.StructField) bool { + name := field.Name + pkgPath := field.PkgPath + if len(pkgPath) > 0 { + return false + } + c := name[0] + if 'a' <= c && c <= 'z' || c == '_' { + return false + } + return true +} + +type Config struct { + Overwrite bool + AppendSlice bool + TypeCheck bool + Transformers Transformers + overwriteWithEmptyValue bool + overwriteSliceWithEmptyValue bool +} + +type Transformers interface { + Transformer(reflect.Type) func(dst, src reflect.Value) error +} + +// Traverses recursively both values, assigning src's fields values to dst. +// The map argument tracks comparisons that have already been seen, which allows +// short circuiting on recursive types. +func deepMerge(dstIn, src reflect.Value, visited map[uintptr]*visit, depth int, config *Config) (dst reflect.Value, err error) { + dst = dstIn + overwrite := config.Overwrite + typeCheck := config.TypeCheck + overwriteWithEmptySrc := config.overwriteWithEmptyValue + overwriteSliceWithEmptySrc := config.overwriteSliceWithEmptyValue + + if !src.IsValid() { + return + } + + if dst.CanAddr() { + addr := dst.UnsafeAddr() + h := 17 * addr + seen := visited[h] + typ := dst.Type() + for p := seen; p != nil; p = p.next { + if p.ptr == addr && p.typ == typ { + return dst, nil + } + } + // Remember, remember... + visited[h] = &visit{addr, typ, seen} + } + + if config.Transformers != nil && !isEmptyValue(dst) { + if fn := config.Transformers.Transformer(dst.Type()); fn != nil { + err = fn(dst, src) + return + } + } + + if dst.IsValid() && src.IsValid() && src.Type() != dst.Type() { + err = fmt.Errorf("cannot append two different types (%s, %s)", src.Kind(), dst.Kind()) + return + } + + switch dst.Kind() { + case reflect.Struct: + if hasExportedField(dst) { + dstCp := reflect.New(dst.Type()).Elem() + for i, n := 0, dst.NumField(); i < n; i++ { + dstField := dst.Field(i) + structField := dst.Type().Field(i) + // copy un-exported struct fields + if !isExportedComponent(&structField) { + rf := dstCp.Field(i) + rf = reflect.NewAt(rf.Type(), unsafe.Pointer(rf.UnsafeAddr())).Elem() //nolint:gosec + dstRF := dst.Field(i) + if !dst.Field(i).CanAddr() { + continue + } + + dstRF = reflect.NewAt(dstRF.Type(), unsafe.Pointer(dstRF.UnsafeAddr())).Elem() //nolint:gosec + rf.Set(dstRF) + continue + } + dstField, err = deepMerge(dstField, src.Field(i), visited, depth+1, config) + if err != nil { + return + } + dstCp.Field(i).Set(dstField) + } + + if dst.CanSet() { + dst.Set(dstCp) + } else { + dst = dstCp + } + return + } else { + if (isReflectNil(dst) || overwrite) && (!isEmptyValue(src) || overwriteWithEmptySrc) { + dst = src + } + } + + case reflect.Map: + if dst.IsNil() && !src.IsNil() { + if dst.CanSet() { + dst.Set(reflect.MakeMap(dst.Type())) + } else { + dst = src + return + } + } + for _, key := range src.MapKeys() { + srcElement := src.MapIndex(key) + dstElement := dst.MapIndex(key) + if !srcElement.IsValid() { + continue + } + if dst.MapIndex(key).IsValid() { + k := dstElement.Interface() + dstElement = reflect.ValueOf(k) + } + if isReflectNil(srcElement) { + if overwrite || isReflectNil(dstElement) { + dst.SetMapIndex(key, srcElement) + } + continue + } + if !srcElement.CanInterface() { + continue + } + + if srcElement.CanInterface() { + srcElement = reflect.ValueOf(srcElement.Interface()) + if dstElement.IsValid() { + dstElement = reflect.ValueOf(dstElement.Interface()) + } + } + dstElement, err = deepMerge(dstElement, srcElement, visited, depth+1, config) + if err != nil { + return + } + dst.SetMapIndex(key, dstElement) + + } + case reflect.Slice: + newSlice := dst + if (!isEmptyValue(src) || overwriteWithEmptySrc || overwriteSliceWithEmptySrc) && (overwrite || isEmptyValue(dst)) && !config.AppendSlice { + if typeCheck && src.Type() != dst.Type() { + return dst, fmt.Errorf("cannot override two slices with different type (%s, %s)", src.Type(), dst.Type()) + } + newSlice = src + } else if config.AppendSlice { + if typeCheck && src.Type() != dst.Type() { + err = fmt.Errorf("cannot append two slice with different type (%s, %s)", src.Type(), dst.Type()) + return + } + newSlice = reflect.AppendSlice(dst, src) + } + if dst.CanSet() { + dst.Set(newSlice) + } else { + dst = newSlice + } + case reflect.Ptr, reflect.Interface: + if isReflectNil(src) { + break + } + + if dst.Kind() != reflect.Ptr && src.Type().AssignableTo(dst.Type()) { + if dst.IsNil() || overwrite { + if overwrite || isEmptyValue(dst) { + if dst.CanSet() { + dst.Set(src) + } else { + dst = src + } + } + } + break + } + + if src.Kind() != reflect.Interface { + if dst.IsNil() || (src.Kind() != reflect.Ptr && overwrite) { + if dst.CanSet() && (overwrite || isEmptyValue(dst)) { + dst.Set(src) + } + } else if src.Kind() == reflect.Ptr { + if dst, err = deepMerge(dst.Elem(), src.Elem(), visited, depth+1, config); err != nil { + return + } + dst = dst.Addr() + } else if dst.Elem().Type() == src.Type() { + if dst, err = deepMerge(dst.Elem(), src, visited, depth+1, config); err != nil { + return + } + } else { + return dst, ErrDifferentArgumentsTypes + } + break + } + if dst.IsNil() || overwrite { + if (overwrite || isEmptyValue(dst)) && (overwriteWithEmptySrc || !isEmptyValue(src)) { + if dst.CanSet() { + dst.Set(src) + } else { + dst = src + } + } + } else if _, err = deepMerge(dst.Elem(), src.Elem(), visited, depth+1, config); err != nil { + return + } + default: + overwriteFull := (!isEmptyValue(src) || overwriteWithEmptySrc) && (overwrite || isEmptyValue(dst)) + if overwriteFull { + if dst.CanSet() { + dst.Set(src) + } else { + dst = src + } + } + } + + return +} + +// Merge will fill any empty for value type attributes on the dst struct using corresponding +// src attributes if they themselves are not empty. dst and src must be valid same-type structs +// and dst must be a pointer to struct. +// It won't merge unexported (private) fields and will do recursively any exported field. +func Merge(dst, src interface{}, opts ...func(*Config)) error { + return merge(dst, src, opts...) +} + +// MergeWithOverwrite will do the same as Merge except that non-empty dst attributes will be overridden by +// non-empty src attribute values. +// Deprecated: use Merge(…) with WithOverride +func MergeWithOverwrite(dst, src interface{}, opts ...func(*Config)) error { + return merge(dst, src, append(opts, WithOverride)...) +} + +// WithTransformers adds transformers to merge, allowing to customize the merging of some types. +func WithTransformers(transformers Transformers) func(*Config) { + return func(config *Config) { + config.Transformers = transformers + } +} + +// WithOverride will make merge override non-empty dst attributes with non-empty src attributes values. +func WithOverride(config *Config) { + config.Overwrite = true +} + +// WithOverwriteWithEmptyValue will make merge override non empty dst attributes with empty src attributes values. +func WithOverwriteWithEmptyValue(config *Config) { + config.overwriteWithEmptyValue = true +} + +// WithOverrideEmptySlice will make merge override empty dst slice with empty src slice. +func WithOverrideEmptySlice(config *Config) { + config.overwriteSliceWithEmptyValue = true +} + +// WithAppendSlice will make merge append slices instead of overwriting it. +func WithAppendSlice(config *Config) { + config.AppendSlice = true +} + +// WithTypeCheck will make merge check types while overwriting it (must be used with WithOverride). +func WithTypeCheck(config *Config) { + config.TypeCheck = true +} + +func merge(dst, src interface{}, opts ...func(*Config)) error { + var ( + vDst, vSrc reflect.Value + err error + ) + + config := &Config{} + + for _, opt := range opts { + opt(config) + } + + if vDst, vSrc, err = resolveValues(dst, src); err != nil { + return err + } + if !vDst.CanSet() { + return fmt.Errorf("cannot set dst, needs reference") + } + if vDst.Type() != vSrc.Type() { + return ErrDifferentArgumentsTypes + } + _, err = deepMerge(vDst, vSrc, make(map[uintptr]*visit), 0, config) + return err +} + +// IsReflectNil is the reflect value provided nil +func isReflectNil(v reflect.Value) bool { + k := v.Kind() + switch k { + case reflect.Interface, reflect.Slice, reflect.Chan, reflect.Func, reflect.Map, reflect.Ptr: + // Both interface and slice are nil if first word is 0. + // Both are always bigger than a word; assume flagIndir. + return v.IsNil() + default: + return false + } +} diff --git a/github.com/imdario/mergo/mergo.go b/github.com/imdario/mergo/mergo.go new file mode 100644 index 0000000000..a82fea2fdc --- /dev/null +++ b/github.com/imdario/mergo/mergo.go @@ -0,0 +1,97 @@ +// Copyright 2013 Dario Castañé. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Based on src/pkg/reflect/deepequal.go from official +// golang's stdlib. + +package mergo + +import ( + "errors" + "reflect" +) + +// Errors reported by Mergo when it finds invalid arguments. +var ( + ErrNilArguments = errors.New("src and dst must not be nil") + ErrDifferentArgumentsTypes = errors.New("src and dst must be of same type") + ErrNotSupported = errors.New("only structs and maps are supported") + ErrExpectedMapAsDestination = errors.New("dst was expected to be a map") + ErrExpectedStructAsDestination = errors.New("dst was expected to be a struct") +) + +// During deepMerge, must keep track of checks that are +// in progress. The comparison algorithm assumes that all +// checks in progress are true when it reencounters them. +// Visited are stored in a map indexed by 17 * a1 + a2; +type visit struct { + ptr uintptr + typ reflect.Type + next *visit +} + +// From src/pkg/encoding/json/encode.go. +func isEmptyValue(v reflect.Value) bool { + switch v.Kind() { + case reflect.Array, reflect.Map, reflect.Slice, reflect.String: + return v.Len() == 0 + case reflect.Bool: + return !v.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.Interface, reflect.Ptr: + if v.IsNil() { + return true + } + return isEmptyValue(v.Elem()) + case reflect.Func: + return v.IsNil() + case reflect.Invalid: + return true + } + return false +} + +func resolveValues(dst, src interface{}) (vDst, vSrc reflect.Value, err error) { + if dst == nil || src == nil { + err = ErrNilArguments + return + } + vDst = reflect.ValueOf(dst).Elem() + if vDst.Kind() != reflect.Struct && vDst.Kind() != reflect.Map { + err = ErrNotSupported + return + } + vSrc = reflect.ValueOf(src) + // We check if vSrc is a pointer to dereference it. + if vSrc.Kind() == reflect.Ptr { + vSrc = vSrc.Elem() + } + return +} + +// Traverses recursively both values, assigning src's fields values to dst. +// The map argument tracks comparisons that have already been seen, which allows +// short circuiting on recursive types. +func deeper(dst, src reflect.Value, visited map[uintptr]*visit, depth int) (err error) { + if dst.CanAddr() { + addr := dst.UnsafeAddr() + h := 17 * addr + seen := visited[h] + typ := dst.Type() + for p := seen; p != nil; p = p.next { + if p.ptr == addr && p.typ == typ { + return nil + } + } + // Remember, remember... + visited[h] = &visit{addr, typ, seen} + } + return // TODO refactor +} diff --git a/github.com/mitchellh/copystructure/.travis.yml b/github.com/mitchellh/copystructure/.travis.yml new file mode 100644 index 0000000000..d7b9589ab1 --- /dev/null +++ b/github.com/mitchellh/copystructure/.travis.yml @@ -0,0 +1,12 @@ +language: go + +go: + - 1.7 + - tip + +script: + - go test + +matrix: + allow_failures: + - go: tip diff --git a/github.com/mitchellh/copystructure/LICENSE b/github.com/mitchellh/copystructure/LICENSE new file mode 100644 index 0000000000..2298515904 --- /dev/null +++ b/github.com/mitchellh/copystructure/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Mitchell Hashimoto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/github.com/mitchellh/copystructure/README.md b/github.com/mitchellh/copystructure/README.md new file mode 100644 index 0000000000..bcb8c8d2cb --- /dev/null +++ b/github.com/mitchellh/copystructure/README.md @@ -0,0 +1,21 @@ +# copystructure + +copystructure is a Go library for deep copying values in Go. + +This allows you to copy Go values that may contain reference values +such as maps, slices, or pointers, and copy their data as well instead +of just their references. + +## Installation + +Standard `go get`: + +``` +$ go get github.com/mitchellh/copystructure +``` + +## Usage & Example + +For usage and examples see the [Godoc](http://godoc.org/github.com/mitchellh/copystructure). + +The `Copy` function has examples associated with it there. diff --git a/github.com/mitchellh/copystructure/copier_time.go b/github.com/mitchellh/copystructure/copier_time.go new file mode 100644 index 0000000000..db6a6aa1a1 --- /dev/null +++ b/github.com/mitchellh/copystructure/copier_time.go @@ -0,0 +1,15 @@ +package copystructure + +import ( + "reflect" + "time" +) + +func init() { + Copiers[reflect.TypeOf(time.Time{})] = timeCopier +} + +func timeCopier(v interface{}) (interface{}, error) { + // Just... copy it. + return v.(time.Time), nil +} diff --git a/github.com/mitchellh/copystructure/copystructure.go b/github.com/mitchellh/copystructure/copystructure.go new file mode 100644 index 0000000000..140435255e --- /dev/null +++ b/github.com/mitchellh/copystructure/copystructure.go @@ -0,0 +1,548 @@ +package copystructure + +import ( + "errors" + "reflect" + "sync" + + "github.com/mitchellh/reflectwalk" +) + +// Copy returns a deep copy of v. +func Copy(v interface{}) (interface{}, error) { + return Config{}.Copy(v) +} + +// CopierFunc is a function that knows how to deep copy a specific type. +// Register these globally with the Copiers variable. +type CopierFunc func(interface{}) (interface{}, error) + +// Copiers is a map of types that behave specially when they are copied. +// If a type is found in this map while deep copying, this function +// will be called to copy it instead of attempting to copy all fields. +// +// The key should be the type, obtained using: reflect.TypeOf(value with type). +// +// It is unsafe to write to this map after Copies have started. If you +// are writing to this map while also copying, wrap all modifications to +// this map as well as to Copy in a mutex. +var Copiers map[reflect.Type]CopierFunc = make(map[reflect.Type]CopierFunc) + +// Must is a helper that wraps a call to a function returning +// (interface{}, error) and panics if the error is non-nil. It is intended +// for use in variable initializations and should only be used when a copy +// error should be a crashing case. +func Must(v interface{}, err error) interface{} { + if err != nil { + panic("copy error: " + err.Error()) + } + + return v +} + +var errPointerRequired = errors.New("Copy argument must be a pointer when Lock is true") + +type Config struct { + // Lock any types that are a sync.Locker and are not a mutex while copying. + // If there is an RLocker method, use that to get the sync.Locker. + Lock bool + + // Copiers is a map of types associated with a CopierFunc. Use the global + // Copiers map if this is nil. + Copiers map[reflect.Type]CopierFunc +} + +func (c Config) Copy(v interface{}) (interface{}, error) { + if c.Lock && reflect.ValueOf(v).Kind() != reflect.Ptr { + return nil, errPointerRequired + } + + w := new(walker) + if c.Lock { + w.useLocks = true + } + + if c.Copiers == nil { + c.Copiers = Copiers + } + + err := reflectwalk.Walk(v, w) + if err != nil { + return nil, err + } + + // Get the result. If the result is nil, then we want to turn it + // into a typed nil if we can. + result := w.Result + if result == nil { + val := reflect.ValueOf(v) + result = reflect.Indirect(reflect.New(val.Type())).Interface() + } + + return result, nil +} + +// Return the key used to index interfaces types we've seen. Store the number +// of pointers in the upper 32bits, and the depth in the lower 32bits. This is +// easy to calculate, easy to match a key with our current depth, and we don't +// need to deal with initializing and cleaning up nested maps or slices. +func ifaceKey(pointers, depth int) uint64 { + return uint64(pointers)<<32 | uint64(depth) +} + +type walker struct { + Result interface{} + + depth int + ignoreDepth int + vals []reflect.Value + cs []reflect.Value + + // This stores the number of pointers we've walked over, indexed by depth. + ps []int + + // If an interface is indirected by a pointer, we need to know the type of + // interface to create when creating the new value. Store the interface + // types here, indexed by both the walk depth and the number of pointers + // already seen at that depth. Use ifaceKey to calculate the proper uint64 + // value. + ifaceTypes map[uint64]reflect.Type + + // any locks we've taken, indexed by depth + locks []sync.Locker + // take locks while walking the structure + useLocks bool +} + +func (w *walker) Enter(l reflectwalk.Location) error { + w.depth++ + + // ensure we have enough elements to index via w.depth + for w.depth >= len(w.locks) { + w.locks = append(w.locks, nil) + } + + for len(w.ps) < w.depth+1 { + w.ps = append(w.ps, 0) + } + + return nil +} + +func (w *walker) Exit(l reflectwalk.Location) error { + locker := w.locks[w.depth] + w.locks[w.depth] = nil + if locker != nil { + defer locker.Unlock() + } + + // clear out pointers and interfaces as we exit the stack + w.ps[w.depth] = 0 + + for k := range w.ifaceTypes { + mask := uint64(^uint32(0)) + if k&mask == uint64(w.depth) { + delete(w.ifaceTypes, k) + } + } + + w.depth-- + if w.ignoreDepth > w.depth { + w.ignoreDepth = 0 + } + + if w.ignoring() { + return nil + } + + switch l { + case reflectwalk.Array: + fallthrough + case reflectwalk.Map: + fallthrough + case reflectwalk.Slice: + w.replacePointerMaybe() + + // Pop map off our container + w.cs = w.cs[:len(w.cs)-1] + case reflectwalk.MapValue: + // Pop off the key and value + mv := w.valPop() + mk := w.valPop() + m := w.cs[len(w.cs)-1] + + // If mv is the zero value, SetMapIndex deletes the key form the map, + // or in this case never adds it. We need to create a properly typed + // zero value so that this key can be set. + if !mv.IsValid() { + mv = reflect.Zero(m.Elem().Type().Elem()) + } + m.Elem().SetMapIndex(mk, mv) + case reflectwalk.ArrayElem: + // Pop off the value and the index and set it on the array + v := w.valPop() + i := w.valPop().Interface().(int) + if v.IsValid() { + a := w.cs[len(w.cs)-1] + ae := a.Elem().Index(i) // storing array as pointer on stack - so need Elem() call + if ae.CanSet() { + ae.Set(v) + } + } + case reflectwalk.SliceElem: + // Pop off the value and the index and set it on the slice + v := w.valPop() + i := w.valPop().Interface().(int) + if v.IsValid() { + s := w.cs[len(w.cs)-1] + se := s.Elem().Index(i) + if se.CanSet() { + se.Set(v) + } + } + case reflectwalk.Struct: + w.replacePointerMaybe() + + // Remove the struct from the container stack + w.cs = w.cs[:len(w.cs)-1] + case reflectwalk.StructField: + // Pop off the value and the field + v := w.valPop() + f := w.valPop().Interface().(reflect.StructField) + if v.IsValid() { + s := w.cs[len(w.cs)-1] + sf := reflect.Indirect(s).FieldByName(f.Name) + + if sf.CanSet() { + sf.Set(v) + } + } + case reflectwalk.WalkLoc: + // Clear out the slices for GC + w.cs = nil + w.vals = nil + } + + return nil +} + +func (w *walker) Map(m reflect.Value) error { + if w.ignoring() { + return nil + } + w.lock(m) + + // Create the map. If the map itself is nil, then just make a nil map + var newMap reflect.Value + if m.IsNil() { + newMap = reflect.New(m.Type()) + } else { + newMap = wrapPtr(reflect.MakeMap(m.Type())) + } + + w.cs = append(w.cs, newMap) + w.valPush(newMap) + return nil +} + +func (w *walker) MapElem(m, k, v reflect.Value) error { + return nil +} + +func (w *walker) PointerEnter(v bool) error { + if v { + w.ps[w.depth]++ + } + return nil +} + +func (w *walker) PointerExit(v bool) error { + if v { + w.ps[w.depth]-- + } + return nil +} + +func (w *walker) Interface(v reflect.Value) error { + if !v.IsValid() { + return nil + } + if w.ifaceTypes == nil { + w.ifaceTypes = make(map[uint64]reflect.Type) + } + + w.ifaceTypes[ifaceKey(w.ps[w.depth], w.depth)] = v.Type() + return nil +} + +func (w *walker) Primitive(v reflect.Value) error { + if w.ignoring() { + return nil + } + w.lock(v) + + // IsValid verifies the v is non-zero and CanInterface verifies + // that we're allowed to read this value (unexported fields). + var newV reflect.Value + if v.IsValid() && v.CanInterface() { + newV = reflect.New(v.Type()) + newV.Elem().Set(v) + } + + w.valPush(newV) + w.replacePointerMaybe() + return nil +} + +func (w *walker) Slice(s reflect.Value) error { + if w.ignoring() { + return nil + } + w.lock(s) + + var newS reflect.Value + if s.IsNil() { + newS = reflect.New(s.Type()) + } else { + newS = wrapPtr(reflect.MakeSlice(s.Type(), s.Len(), s.Cap())) + } + + w.cs = append(w.cs, newS) + w.valPush(newS) + return nil +} + +func (w *walker) SliceElem(i int, elem reflect.Value) error { + if w.ignoring() { + return nil + } + + // We don't write the slice here because elem might still be + // arbitrarily complex. Just record the index and continue on. + w.valPush(reflect.ValueOf(i)) + + return nil +} + +func (w *walker) Array(a reflect.Value) error { + if w.ignoring() { + return nil + } + w.lock(a) + + newA := reflect.New(a.Type()) + + w.cs = append(w.cs, newA) + w.valPush(newA) + return nil +} + +func (w *walker) ArrayElem(i int, elem reflect.Value) error { + if w.ignoring() { + return nil + } + + // We don't write the array here because elem might still be + // arbitrarily complex. Just record the index and continue on. + w.valPush(reflect.ValueOf(i)) + + return nil +} + +func (w *walker) Struct(s reflect.Value) error { + if w.ignoring() { + return nil + } + w.lock(s) + + var v reflect.Value + if c, ok := Copiers[s.Type()]; ok { + // We have a Copier for this struct, so we use that copier to + // get the copy, and we ignore anything deeper than this. + w.ignoreDepth = w.depth + + dup, err := c(s.Interface()) + if err != nil { + return err + } + + // We need to put a pointer to the value on the value stack, + // so allocate a new pointer and set it. + v = reflect.New(s.Type()) + reflect.Indirect(v).Set(reflect.ValueOf(dup)) + } else { + // No copier, we copy ourselves and allow reflectwalk to guide + // us deeper into the structure for copying. + v = reflect.New(s.Type()) + } + + // Push the value onto the value stack for setting the struct field, + // and add the struct itself to the containers stack in case we walk + // deeper so that its own fields can be modified. + w.valPush(v) + w.cs = append(w.cs, v) + + return nil +} + +func (w *walker) StructField(f reflect.StructField, v reflect.Value) error { + if w.ignoring() { + return nil + } + + // If PkgPath is non-empty, this is a private (unexported) field. + // We do not set this unexported since the Go runtime doesn't allow us. + if f.PkgPath != "" { + return reflectwalk.SkipEntry + } + + // Push the field onto the stack, we'll handle it when we exit + // the struct field in Exit... + w.valPush(reflect.ValueOf(f)) + return nil +} + +// ignore causes the walker to ignore any more values until we exit this on +func (w *walker) ignore() { + w.ignoreDepth = w.depth +} + +func (w *walker) ignoring() bool { + return w.ignoreDepth > 0 && w.depth >= w.ignoreDepth +} + +func (w *walker) pointerPeek() bool { + return w.ps[w.depth] > 0 +} + +func (w *walker) valPop() reflect.Value { + result := w.vals[len(w.vals)-1] + w.vals = w.vals[:len(w.vals)-1] + + // If we're out of values, that means we popped everything off. In + // this case, we reset the result so the next pushed value becomes + // the result. + if len(w.vals) == 0 { + w.Result = nil + } + + return result +} + +func (w *walker) valPush(v reflect.Value) { + w.vals = append(w.vals, v) + + // If we haven't set the result yet, then this is the result since + // it is the first (outermost) value we're seeing. + if w.Result == nil && v.IsValid() { + w.Result = v.Interface() + } +} + +func (w *walker) replacePointerMaybe() { + // Determine the last pointer value. If it is NOT a pointer, then + // we need to push that onto the stack. + if !w.pointerPeek() { + w.valPush(reflect.Indirect(w.valPop())) + return + } + + v := w.valPop() + + // If the expected type is a pointer to an interface of any depth, + // such as *interface{}, **interface{}, etc., then we need to convert + // the value "v" from *CONCRETE to *interface{} so types match for + // Set. + // + // Example if v is type *Foo where Foo is a struct, v would become + // *interface{} instead. This only happens if we have an interface expectation + // at this depth. + // + // For more info, see GH-16 + if iType, ok := w.ifaceTypes[ifaceKey(w.ps[w.depth], w.depth)]; ok && iType.Kind() == reflect.Interface { + y := reflect.New(iType) // Create *interface{} + y.Elem().Set(reflect.Indirect(v)) // Assign "Foo" to interface{} (dereferenced) + v = y // v is now typed *interface{} (where *v = Foo) + } + + for i := 1; i < w.ps[w.depth]; i++ { + if iType, ok := w.ifaceTypes[ifaceKey(w.ps[w.depth]-i, w.depth)]; ok { + iface := reflect.New(iType).Elem() + iface.Set(v) + v = iface + } + + p := reflect.New(v.Type()) + p.Elem().Set(v) + v = p + } + + w.valPush(v) +} + +// if this value is a Locker, lock it and add it to the locks slice +func (w *walker) lock(v reflect.Value) { + if !w.useLocks { + return + } + + if !v.IsValid() || !v.CanInterface() { + return + } + + type rlocker interface { + RLocker() sync.Locker + } + + var locker sync.Locker + + // We can't call Interface() on a value directly, since that requires + // a copy. This is OK, since the pointer to a value which is a sync.Locker + // is also a sync.Locker. + if v.Kind() == reflect.Ptr { + switch l := v.Interface().(type) { + case rlocker: + // don't lock a mutex directly + if _, ok := l.(*sync.RWMutex); !ok { + locker = l.RLocker() + } + case sync.Locker: + locker = l + } + } else if v.CanAddr() { + switch l := v.Addr().Interface().(type) { + case rlocker: + // don't lock a mutex directly + if _, ok := l.(*sync.RWMutex); !ok { + locker = l.RLocker() + } + case sync.Locker: + locker = l + } + } + + // still no callable locker + if locker == nil { + return + } + + // don't lock a mutex directly + switch locker.(type) { + case *sync.Mutex, *sync.RWMutex: + return + } + + locker.Lock() + w.locks[w.depth] = locker +} + +// wrapPtr is a helper that takes v and always make it *v. copystructure +// stores things internally as pointers until the last moment before unwrapping +func wrapPtr(v reflect.Value) reflect.Value { + if !v.IsValid() { + return v + } + vPtr := reflect.New(v.Type()) + vPtr.Elem().Set(v) + return vPtr +} diff --git a/github.com/mitchellh/copystructure/go.mod b/github.com/mitchellh/copystructure/go.mod new file mode 100644 index 0000000000..d01864309b --- /dev/null +++ b/github.com/mitchellh/copystructure/go.mod @@ -0,0 +1,3 @@ +module github.com/mitchellh/copystructure + +require github.com/mitchellh/reflectwalk v1.0.0 diff --git a/github.com/mitchellh/copystructure/go.sum b/github.com/mitchellh/copystructure/go.sum new file mode 100644 index 0000000000..be57245619 --- /dev/null +++ b/github.com/mitchellh/copystructure/go.sum @@ -0,0 +1,2 @@ +github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= diff --git a/github.com/mwitkow/go-proto-validators/.gitignore b/github.com/mwitkow/go-proto-validators/.gitignore new file mode 100644 index 0000000000..5f452f6285 --- /dev/null +++ b/github.com/mwitkow/go-proto-validators/.gitignore @@ -0,0 +1,89 @@ +### JetBrains template +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio + +*.iml + +## Directory-based project format: +.idea/ +# if you remove the above rule, at least ignore the following: + +# User-specific stuff: +# .idea/workspace.xml +# .idea/tasks.xml +# .idea/dictionaries + +# Sensitive or high-churn files: +# .idea/dataSources.ids +# .idea/dataSources.xml +# .idea/sqlDataSources.xml +# .idea/dynamic.xml +# .idea/uiDesigner.xml + +# Gradle: +# .idea/gradle.xml +# .idea/libraries + +# Mongo Explorer plugin: +# .idea/mongoSettings.xml + +## File-based project format: +*.ipr +*.iws + +## Plugin-specific files: + +# IntelliJ +/out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +### Go template +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof +### Vim template +[._]*.s[a-w][a-z] +[._]s[a-w][a-z] +*.un~ +Session.vim +.netrwhist +*~ +### Linux template +*~ + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# Created by .ignore support plugin (hsz.mobi) diff --git a/github.com/mwitkow/go-proto-validators/.travis.yml b/github.com/mwitkow/go-proto-validators/.travis.yml new file mode 100644 index 0000000000..7c43822551 --- /dev/null +++ b/github.com/mwitkow/go-proto-validators/.travis.yml @@ -0,0 +1,18 @@ +language: go +sudo: false + +go: + - 1.5 + - 1.6 + +before_install: + - ./install_protoc.sh + - export PATH=$PATH:$HOME/soft/protobuf + +install: + - go get github.com/stretchr/testify + - go get github.com/gogo/protobuf/protoc-gen-gogo + - go get github.com/golang/protobuf/protoc-gen-go + +script: + - make test diff --git a/github.com/mwitkow/go-proto-validators/LICENSE.txt b/github.com/mwitkow/go-proto-validators/LICENSE.txt new file mode 100644 index 0000000000..b2b065037f --- /dev/null +++ b/github.com/mwitkow/go-proto-validators/LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/github.com/mwitkow/go-proto-validators/Makefile b/github.com/mwitkow/go-proto-validators/Makefile new file mode 100644 index 0000000000..ff1bc0d45d --- /dev/null +++ b/github.com/mwitkow/go-proto-validators/Makefile @@ -0,0 +1,45 @@ +# Copyright 2016 Michal Witkowski. All Rights Reserved. +# See LICENSE for licensing terms. + +export PATH := ${GOPATH}/bin:${PATH} + +install: + @echo "--- Installing govalidators to GOPATH" + go install github.com/mwitkow/go-proto-validators/protoc-gen-govalidators + +regenerate_test_gogo: + @echo "Regenerating test .proto files with gogo imports" + (protoc \ + --proto_path=${GOPATH}/src \ + --proto_path=test \ + --gogo_out=test/gogo \ + --govalidators_out=gogoimport=true:test/gogo test/*.proto) + +regenerate_test_golang: + @echo "--- Regenerating test .proto files with golang imports" + (protoc \ + --proto_path=${GOPATH}/src \ + --proto_path=test \ + --go_out=test/golang \ + --govalidators_out=test/golang test/*.proto) + +regenerate_example: install + @echo "--- Regenerating example directory" + (protoc \ + --proto_path=${GOPATH}/src \ + --proto_path=. \ + --go_out=. \ + --govalidators_out=. examples/*.proto) + +test: install regenerate_test_gogo regenerate_test_golang + @echo "Running tests" + go test -v ./... + +regenerate: + @echo "--- Regenerating validator.proto" + (protoc \ + --proto_path=${GOPATH}/src \ + --proto_path=${GOPATH}/src/github.com/gogo/protobuf/protobuf \ + --proto_path=. \ + --gogo_out=Mgoogle/protobuf/descriptor.proto=github.com/gogo/protobuf/protoc-gen-gogo/descriptor:. \ + validator.proto) diff --git a/github.com/mwitkow/go-proto-validators/README.md b/github.com/mwitkow/go-proto-validators/README.md new file mode 100644 index 0000000000..3df5ea8ac4 --- /dev/null +++ b/github.com/mwitkow/go-proto-validators/README.md @@ -0,0 +1,132 @@ +# Golang ProtoBuf Validator Compiler + +[![Travis Build](https://travis-ci.org/mwitkow/go-proto-validators.svg)](https://travis-ci.org/mwitkow/go-proto-validators) +[![Apache 2.0 License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) + +A `protoc` plugin that generates `Validate() error` functions on Go proto `struct`s based on field options inside `.proto` +files. The validation functions are code-generated and thus don't suffer on performance from tag-based reflection on +deeply-nested messages. + +## Paint me a code picture + +Let's take the following `proto3` snippet: + +```proto +syntax = "proto3"; +package validator.examples; +import "github.com/mwitkow/go-proto-validators/validator.proto"; + +message InnerMessage { + // some_integer can only be in range (1, 100). + int32 some_integer = 1 [(validator.field) = {int_gt: 0, int_lt: 100}]; + // some_float can only be in range (0;1). + double some_float = 2 [(validator.field) = {float_gte: 0, float_lte: 1}]; +} + +message OuterMessage { + // important_string must be a lowercase alpha-numeric of 5 to 30 characters (RE2 syntax). + string important_string = 1 [(validator.field) = {regex: "^[a-z]{2,5}$"}]; + // proto3 doesn't have `required`, the `msg_exist` enforces presence of InnerMessage. + InnerMessage inner = 2 [(validator.field) = {msg_exists : true}]; +} +``` + +First, the **`required` keyword is back** for `proto3`, under the guise of `msg_exists`. The painful `if-nil` checks are taken care of! + +Second, the expected values in fields are now part of the contract `.proto` file. No more hunting down conditions in code! + +Third, the generated code is understandable and has clear understandable error messages. Take a look: + +```go +func (this *InnerMessage) Validate() error { + if !(this.SomeInteger > 0) { + return fmt.Errorf("validation error: InnerMessage.SomeInteger must be greater than '0'") + } + if !(this.SomeInteger < 100) { + return fmt.Errorf("validation error: InnerMessage.SomeInteger must be less than '100'") + } + if !(this.SomeFloat >= 0) { + return fmt.Errorf("validation error: InnerMessage.SomeFloat must be greater than or equal to '0'") + } + if !(this.SomeFloat <= 1) { + return fmt.Errorf("validation error: InnerMessage.SomeFloat must be less than or equal to '1'") + } + return nil +} + +var _regex_OuterMessage_ImportantString = regexp.MustCompile("^[a-z]{2,5}$") + +func (this *OuterMessage) Validate() error { + if !_regex_OuterMessage_ImportantString.MatchString(this.ImportantString) { + return fmt.Errorf("validation error: OuterMessage.ImportantString must conform to regex '^[a-z]{2,5}$'") + } + if nil == this.Inner { + return fmt.Errorf("validation error: OuterMessage.Inner message must exist") + } + if this.Inner != nil { + if err := validators.CallValidatorIfExists(this.Inner); err != nil { + return err + } + } + return nil +} +``` + +## Installing and using + +The `protoc` compiler expects to find plugins named `proto-gen-XYZ` on the execution `$PATH`. So first: + +```sh +export PATH=${PATH}:${GOPATH}/bin +``` + +Then, do the usual + +```sh +go get github.com/mwitkow/go-proto-validators/protoc-gen-govalidators +``` + +Your `protoc` builds probably look very simple like: + +```sh +protoc \ + --proto_path=. \ + --go_out=. \ + *.proto +``` + +That's fine, until you encounter `.proto` includes. Because `go-proto-validators` uses field options inside the `.proto` +files themselves, it's `.proto` definition (and the Google `descriptor.proto` itself) need to on the `protoc` include +path. Hence the above becomes: + +```sh +protoc \ + --proto_path=${GOPATH}/src \ + --proto_path=${GOPATH}/src/github.com/google/protobuf/src \ + --proto_path=. \ + --go_out=. \ + --govalidators_out=. \ + *.proto +``` + +Or with gogo protobufs: + +```sh +protoc \ + --proto_path=${GOPATH}/src \ + --proto_path=${GOPATH}/src/github.com/gogo/protobuf/protobuf \ + --proto_path=. \ + --gogo_out=. \ + --govalidators_out=gogoimport=true:. \ + *.proto +``` + +Basically the magical incantation (apart from includes) is the `--govalidators_out`. That triggers the +`protoc-gen-govalidators` plugin to generate `mymessage.validator.pb.go`. That's it :) + +###License + +`go-proto-validators` is released under the Apache 2.0 license. See the [LICENSE](LICENSE) file for details. + + + diff --git a/github.com/mwitkow/go-proto-validators/examples/nested.proto b/github.com/mwitkow/go-proto-validators/examples/nested.proto new file mode 100644 index 0000000000..5689c03818 --- /dev/null +++ b/github.com/mwitkow/go-proto-validators/examples/nested.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; +package validator.examples; +import "github.com/mwitkow/go-proto-validators/validator.proto"; + +message InnerMessage { + // some_integer can only be in range (1, 100). + int32 some_integer = 1 [(validator.field) = {int_gt: 0, int_lt: 100}]; +} + +message OuterMessage { + // important_string must be a lowercase alpha-numeric of 5 to 30 characters (RE2 syntax). + string important_string = 1 [(validator.field) = {regex: "^[a-z]{2,5}$"}]; + // proto3 doesn't have `required`, the `msg_exist` enforces presence of InnerMessage. + InnerMessage inner = 2 [(validator.field) = {msg_exists : true}]; +} \ No newline at end of file diff --git a/github.com/mwitkow/go-proto-validators/helper.go b/github.com/mwitkow/go-proto-validators/helper.go new file mode 100644 index 0000000000..9a3ee9beb8 --- /dev/null +++ b/github.com/mwitkow/go-proto-validators/helper.go @@ -0,0 +1,39 @@ +// Copyright 2016 Michal Witkowski. All Rights Reserved. +// See LICENSE for licensing terms. + +package validator + +import "strings" + +// Validator is a general interface that allows a message to be validated. +type Validator interface { + Validate() error +} + +func CallValidatorIfExists(candidate interface{}) error { + if validator, ok := candidate.(Validator); ok { + return validator.Validate() + } + return nil +} + +type fieldError struct { + fieldStack []string + nestedErr error +} + +func (f *fieldError) Error() string { + return "invalid field " + strings.Join(f.fieldStack, ".") + ": " + f.nestedErr.Error() +} + +// FieldError wraps a given Validator error providing a message call stack. +func FieldError(fieldName string, err error) error { + if fErr, ok := err.(*fieldError); ok { + fErr.fieldStack = append([]string{fieldName}, fErr.fieldStack...) + return err + } + return &fieldError{ + fieldStack: []string{fieldName}, + nestedErr: err, + } +} diff --git a/github.com/mwitkow/go-proto-validators/install_protoc.sh b/github.com/mwitkow/go-proto-validators/install_protoc.sh new file mode 100644 index 0000000000..d7ec41ca16 --- /dev/null +++ b/github.com/mwitkow/go-proto-validators/install_protoc.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright 2016 Michal Witkowski. All Rights Reserved. +# See LICENSE for licensing terms. +# +# This script installs protobuf compiler `protoc` into PATH. + +version=${PROTOBUF_VERSION:-"3.0.0-beta-2"} +dst_dir="${HOME}/soft/protobuf" + +# Fail on issues. +set -e + +echo "Downloading and installing protoc ${version}" + +mkdir -p ${dst_dir} + +wget https://github.com/google/protobuf/releases/download/v${version}/protoc-${version}-linux-x86_64.zip -O ${dst_dir}/dist.zip + +cd ${dst_dir} +unzip -o dist.zip + +echo "Proto in \$PROTOBUF_DIR=${PROTOBUF_DIR}" diff --git a/github.com/mwitkow/go-proto-validators/test/validator_proto2.proto b/github.com/mwitkow/go-proto-validators/test/validator_proto2.proto new file mode 100644 index 0000000000..e0e95ae09f --- /dev/null +++ b/github.com/mwitkow/go-proto-validators/test/validator_proto2.proto @@ -0,0 +1,78 @@ +// Copyright 2016 Michal Witkowski. All Rights Reserved. +// See LICENSE for licensing terms. + +syntax = "proto2"; +package validatortest; + +import "github.com/gogo/protobuf/gogoproto/gogo.proto"; +import "github.com/mwitkow/go-proto-validators/validator.proto"; + +message ValidatorMessage { + // Embedded message test structure. + message Embedded { + optional string Identifier = 1 [(validator.field) = {regex: "^[a-z]{2,5}$"}]; + required int64 SomeValue = 2 [(validator.field) = {int_gt: 0, int_lt: 100}]; + } + + // String regex constraint tests. + required string StringReq = 1 [(validator.field) = {regex: "^.{2,5}$"}]; + required string StringReqNonNull = 2 [(validator.field) = {regex: "^.{2,5}$"}, (gogoproto.nullable) = false]; + optional string StringOpt = 3 [(validator.field) = {regex: "^.{2,5}$"}]; + optional string StringOptNonNull = 4 [(validator.field) = {regex: "^.{2,5}$"}, (gogoproto.nullable) = false]; + required string StringUnescaped = 5 [(validator.field) = {regex: "[\\p{L}\\p{N}]({\\p{L}\\p{N}_- ]{0,28}[\\p{L}\\p{N}])?."}]; + + // Strict integer inequality constraint tests. + required uint32 IntReq = 6 [(validator.field) = {int_gt: 10}]; + required uint32 IntReqNonNull = 7 [(validator.field) = {int_gt: 0}, (gogoproto.nullable) = false]; + repeated uint32 IntRep = 8 [(validator.field) = {int_gt: 10}]; + repeated uint32 IntRepNonNull = 9 [(validator.field) = {int_gt: 0}]; + + // Embedded message recursive constraint tests. + required Embedded embeddedReq = 10; + required Embedded embeddedNonNull = 11 [(gogoproto.nullable) = false]; + repeated Embedded embeddedRep = 12; + repeated Embedded embeddedRepNonNullable = 13 [(gogoproto.nullable) = false]; + + // Custom error tests. + optional int32 CustomErrorInt = 16 [(validator.field) = {int_gt: 10, human_error: "My Custom Error"}]; + + // Strict floating-point inequality constraint tests. + // With this epsilon value, the limits become + // SomeFloat+0.05 > 0.35 + // SomeFloat-0.05 < 0.65 + required double StrictSomeDoubleReq = 17 [(validator.field) = {float_gt: 0.35, float_lt: 0.65, float_epsilon: 0.05}]; + required double StrictSomeDoubleReqNonNull = 18 [(validator.field) = {float_gt: 0.35, float_lt: 0.65, float_epsilon: 0.05}, (gogoproto.nullable) = false]; + repeated double StrictSomeDoubleRep = 19 [(validator.field) = {float_gt: 0.35, float_lt: 0.65, float_epsilon: 0.05}]; + repeated double StrictSomeDoubleRepNonNull = 20 [(validator.field) = {float_gt: 0.35, float_lt: 0.65, float_epsilon: 0.05}]; + required float StrictSomeFloatReq = 21 [(validator.field) = {float_gt: 0.35, float_lt: 0.65, float_epsilon: 0.05}]; + required float StrictSomeFloatReqNonNull = 22 [(validator.field) = {float_gt: 0.35, float_lt: 0.65, float_epsilon: 0.05}, (gogoproto.nullable) = false]; + repeated float StrictSomeFloatRep = 23 [(validator.field) = {float_gt: 0.35, float_lt: 0.65, float_epsilon: 0.05}]; + repeated float StrictSomeFloatRepNonNull = 24 [(validator.field) = {float_gt: 0.35, float_lt: 0.65, float_epsilon: 0.05}]; + + // Non-strict floating-point inequality constraint tests. + required double SomeDoubleReq = 25 [(validator.field) = {float_gte: 0.25, float_lte: 0.75}]; + required double SomeDoubleReqNonNull = 26 [(validator.field) = {float_gte: 0.25, float_lte: 0.75}, (gogoproto.nullable) = false]; + repeated double SomeDoubleRep = 27 [(validator.field) = {float_gte: 0.25, float_lte: 0.75}]; + repeated double SomeDoubleRepNonNull = 28 [(validator.field) = {float_gte: 0.25, float_lte: 0.75}]; + required float SomeFloatReq = 29 [(validator.field) = {float_gte: 0.25, float_lte: 0.75}]; + required float SomeFloatReqNonNull = 30 [(validator.field) = {float_gte: 0.25, float_lte: 0.75}, (gogoproto.nullable) = false]; + repeated float SomeFloatRep = 31 [(validator.field) = {float_gte: 0.25, float_lte: 0.75}]; + repeated float SomeFloatRepNonNull = 32 [(validator.field) = {float_gte: 0.25, float_lte: 0.75}]; + + // String not-empty constraint tests. + required string SomeNonEmptyString = 33 [(validator.field) = {string_not_empty: true}]; + + // Repeated base-type without constraint tests. + repeated int32 RepeatedBaseType = 34; + + // Repeated element count constraint tests. + repeated int32 Repeated = 35 [(validator.field) = {repeated_count_min: 2, repeated_count_max: 5}]; + + optional string SomeStringLtReq = 36 [(validator.field) = {length_gt: 2}]; + optional string SomeStringGtReq = 37 [(validator.field) = {length_lt: 12}]; + optional string SomeStringEqReq = 38 [(validator.field) = {length_eq: 10}]; + optional bytes SomeBytesLtReq = 39 [(validator.field) = {length_gt: 5}]; + optional bytes SomeBytesGtReq = 40 [(validator.field) = {length_lt: 20}]; + optional bytes SomeBytesEqReq = 41 [(validator.field) = {length_eq: 12}]; + +} diff --git a/github.com/mwitkow/go-proto-validators/test/validator_proto3.proto b/github.com/mwitkow/go-proto-validators/test/validator_proto3.proto new file mode 100644 index 0000000000..dd0b9dddbc --- /dev/null +++ b/github.com/mwitkow/go-proto-validators/test/validator_proto3.proto @@ -0,0 +1,72 @@ +// Copyright 2016 Michal Witkowski. All Rights Reserved. +// See LICENSE for licensing terms. + +syntax = "proto3"; +package validatortest; + +import "github.com/gogo/protobuf/gogoproto/gogo.proto"; +import "github.com/mwitkow/go-proto-validators/validator.proto"; + +message ValidatorMessage3 { + // Embedded message test structure. + message Embedded { + string Identifier = 1 [(validator.field) = {regex: "^[a-z]{2,5}$"}]; + int64 SomeValue = 2 [(validator.field) = {int_gt: 0, int_lt: 100}]; + } + + // String regex constraint tests. + string SomeString = 1 [(validator.field) = {regex: "^.{2,5}$"}]; + repeated string SomeStringRep = 2 [(validator.field) = {regex: "^.{2,5}$"}]; + string SomeStringNoQuotes = 3 [(validator.field) = {regex: "^[^\"]{2,5}$"}]; + string SomeStringUnescaped = 4 [(validator.field) = {regex: "[\\p{L}\\p{N}]({\\p{L}\\p{N}_- ]{0,28}[\\p{L}\\p{N}])?."}]; + + // Strict integer inequality constraint tests. + uint32 SomeInt = 6 [(validator.field) = {int_gt: 10}]; + repeated uint32 SomeIntRep = 7 [(validator.field) = {int_gt: 10}]; + repeated uint32 SomeIntRepNonNull = 8 [(validator.field) = {int_gt: 10}]; + + // Embedded message existence and recursive constraint tests. + Embedded someEmbedded = 10; + Embedded someEmbeddedNonNullable = 11 [(gogoproto.nullable) = false]; + Embedded someEmbeddedExists = 12 [(validator.field) = {msg_exists : true}]; + repeated Embedded someEmbeddedRep = 14; + repeated Embedded someEmbeddedRepNonNullable = 15 [(gogoproto.nullable) = false]; + + // Custom error tests. + int32 CustomErrorInt = 16 [(validator.field) = {int_lt: 10, human_error: "My Custom Error"}]; + + // Strict floating-point inequality constraint tests. + // With this epsilon value, the limits become + // SomeFloat+0.05 > 0.35 + // SomeFloat-0.05 < 0.65 + double StrictSomeDouble = 17 [(validator.field) = {float_gt: 0.35, float_lt: 0.65, float_epsilon: 0.05}]; + repeated double StrictSomeDoubleRep = 19 [(validator.field) = {float_gt: 0.35, float_lt: 0.65, float_epsilon: 0.05}]; + repeated double StrictSomeDoubleRepNonNull = 20 [(validator.field) = {float_gt: 0.35, float_lt: 0.65, float_epsilon: 0.05}]; + float StrictSomeFloat = 21 [(validator.field) = {float_gt: 0.35, float_lt: 0.65, float_epsilon: 0.05}]; + repeated float StrictSomeFloatRep = 22 [(validator.field) = {float_gt: 0.35, float_lt: 0.65, float_epsilon: 0.05}]; + repeated float StrictSomeFloatRepNonNull = 23 [(validator.field) = {float_gt: 0.35, float_lt: 0.65, float_epsilon: 0.05}]; + + // Non-strict floating-point inequality constraint tests. + double SomeDouble = 24 [(validator.field) = {float_gte: 0.25, float_lte: 0.75}]; + repeated double SomeDoubleRep = 25 [(validator.field) = {float_gte: 0.25, float_lte: 0.75}]; + repeated double SomeDoubleRepNonNull = 26 [(validator.field) = {float_gte: 0.25, float_lte: 0.75}]; + float SomeFloat = 27 [(validator.field) = {float_gte: 0.25, float_lte: 0.75}]; + repeated float SomeFloatRep = 28 [(validator.field) = {float_gte: 0.25, float_lte: 0.75}]; + repeated float SomeFloatRepNonNull = 30 [(validator.field) = {float_gte: 0.25, float_lte: 0.75}]; + + // String not-empty constraint tests. + string SomeNonEmptyString = 31 [(validator.field) = {string_not_empty: true}]; + + // Repeated base-type without constraint tests. + repeated int32 RepeatedBaseType = 32; + + // Repeated element count constraint tests. + repeated int32 Repeated = 33 [(validator.field) = {repeated_count_min: 2, repeated_count_max: 5}]; + string SomeStringLtReq = 36 [(validator.field) = {length_gt: 2}]; + string SomeStringGtReq = 37 [(validator.field) = {length_lt: 12}]; + string SomeStringEqReq = 38 [(validator.field) = {length_eq: 10}]; + + bytes SomeBytesLtReq = 39 [(validator.field) = {length_gt: 5}]; + bytes SomeBytesGtReq = 40 [(validator.field) = {length_lt: 20}]; + bytes SomeBytesEqReq = 41 [(validator.field) = {length_eq: 12}]; +} diff --git a/github.com/mwitkow/go-proto-validators/test/validator_proto3_map.proto b/github.com/mwitkow/go-proto-validators/test/validator_proto3_map.proto new file mode 100644 index 0000000000..f8799b13e8 --- /dev/null +++ b/github.com/mwitkow/go-proto-validators/test/validator_proto3_map.proto @@ -0,0 +1,26 @@ +// Copyright 2016 Michal Witkowski. All Rights Reserved. +// See LICENSE for licensing terms. + +syntax = "proto3"; +package validatortest; + +import "github.com/gogo/protobuf/gogoproto/gogo.proto"; +import "github.com/mwitkow/go-proto-validators/validator.proto"; + +message ValueType { + string something = 1 ; +} + +// This needs to be able to compile. Fixes https://github.com/mwitkow/go-proto-validators/issues/1 +message ValidatorMapMessage3 { + map SomeStringMap = 1; + + message NestedType { + string something = 1 ; + } + + map SomeExtMap = 2; + map SomeNestedMap = 3; +} + + diff --git a/github.com/mwitkow/go-proto-validators/test/validator_proto3_oneof.proto b/github.com/mwitkow/go-proto-validators/test/validator_proto3_oneof.proto new file mode 100644 index 0000000000..2170ed5dd8 --- /dev/null +++ b/github.com/mwitkow/go-proto-validators/test/validator_proto3_oneof.proto @@ -0,0 +1,28 @@ +// Copyright 2016 Michal Witkowski. All Rights Reserved. +// See LICENSE for licensing terms. + +syntax = "proto3"; +package validatortest; + +import "github.com/gogo/protobuf/gogoproto/gogo.proto"; +import "github.com/mwitkow/go-proto-validators/validator.proto"; + +message ExternalMsg { + string Identifier = 1 [(validator.field) = {regex: "^[a-z]{2,5}$"}]; + int64 SomeValue = 2 [(validator.field) = {int_gt: 0, int_lt: 100}]; +} + +message OneOfMessage3 { + uint32 SomeInt = 1 [(validator.field) = {int_gt: 10}]; + + oneof type { + ExternalMsg one_msg = 2; + uint32 one_int = 3 [(validator.field) = {int_gt: 20}]; + uint32 two_int = 4 [(validator.field) = {int_gt: 100}]; + } + + oneof something { + uint32 three_int = 5 [(validator.field) = {int_gt: 20}]; + uint32 four_int = 6 [(validator.field) = {int_gt: 100}]; + } +} \ No newline at end of file diff --git a/github.com/mwitkow/go-proto-validators/validator.pb.go b/github.com/mwitkow/go-proto-validators/validator.pb.go new file mode 100644 index 0000000000..755e6ed060 --- /dev/null +++ b/github.com/mwitkow/go-proto-validators/validator.pb.go @@ -0,0 +1,238 @@ +// Code generated by protoc-gen-gogo. +// source: validator.proto +// DO NOT EDIT! + +/* +Package validator is a generated protocol buffer package. + +It is generated from these files: + validator.proto + +It has these top-level messages: + FieldValidator +*/ +package validator + +import proto "github.com/gogo/protobuf/proto" +import fmt "fmt" +import math "math" +import google_protobuf "github.com/gogo/protobuf/protoc-gen-gogo/descriptor" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package + +type FieldValidator struct { + // Uses a Golang RE2-syntax regex to match the field contents. + Regex *string `protobuf:"bytes,1,opt,name=regex" json:"regex,omitempty"` + // Field value of integer strictly greater than this value. + IntGt *int64 `protobuf:"varint,2,opt,name=int_gt,json=intGt" json:"int_gt,omitempty"` + // Field value of integer strictly smaller than this value. + IntLt *int64 `protobuf:"varint,3,opt,name=int_lt,json=intLt" json:"int_lt,omitempty"` + // Used for nested message types, requires that the message type exists. + MsgExists *bool `protobuf:"varint,4,opt,name=msg_exists,json=msgExists" json:"msg_exists,omitempty"` + // Human error specifies a user-customizable error that is visible to the user. + HumanError *string `protobuf:"bytes,5,opt,name=human_error,json=humanError" json:"human_error,omitempty"` + // Field value of double strictly greater than this value. + // Note that this value can only take on a valid floating point + // value. Use together with float_epsilon if you need something more specific. + FloatGt *float64 `protobuf:"fixed64,6,opt,name=float_gt,json=floatGt" json:"float_gt,omitempty"` + // Field value of double strictly smaller than this value. + // Note that this value can only take on a valid floating point + // value. Use together with float_epsilon if you need something more specific. + FloatLt *float64 `protobuf:"fixed64,7,opt,name=float_lt,json=floatLt" json:"float_lt,omitempty"` + // Field value of double describing the epsilon within which + // any comparison should be considered to be true. For example, + // when using float_gt = 0.35, using a float_epsilon of 0.05 + // would mean that any value above 0.30 is acceptable. It can be + // thought of as a {float_value_condition} +- {float_epsilon}. + // If unset, no correction for floating point inaccuracies in + // comparisons will be attempted. + FloatEpsilon *float64 `protobuf:"fixed64,8,opt,name=float_epsilon,json=floatEpsilon" json:"float_epsilon,omitempty"` + // Floating-point value compared to which the field content should be greater or equal. + FloatGte *float64 `protobuf:"fixed64,9,opt,name=float_gte,json=floatGte" json:"float_gte,omitempty"` + // Floating-point value compared to which the field content should be smaller or equal. + FloatLte *float64 `protobuf:"fixed64,10,opt,name=float_lte,json=floatLte" json:"float_lte,omitempty"` + // Used for string fields, requires the string to be not empty (i.e different from ""). + StringNotEmpty *bool `protobuf:"varint,11,opt,name=string_not_empty,json=stringNotEmpty" json:"string_not_empty,omitempty"` + // Repeated field with at least this number of elements. + RepeatedCountMin *int64 `protobuf:"varint,12,opt,name=repeated_count_min,json=repeatedCountMin" json:"repeated_count_min,omitempty"` + // Repeated field with at most this number of elements. + RepeatedCountMax *int64 `protobuf:"varint,13,opt,name=repeated_count_max,json=repeatedCountMax" json:"repeated_count_max,omitempty"` + // Field value of length greater than this value. + LengthGt *int64 `protobuf:"varint,14,opt,name=length_gt,json=lengthGt" json:"length_gt,omitempty"` + // Field value of length smaller than this value. + LengthLt *int64 `protobuf:"varint,15,opt,name=length_lt,json=lengthLt" json:"length_lt,omitempty"` + // Field value of integer strictly equal this value. + LengthEq *int64 `protobuf:"varint,16,opt,name=length_eq,json=lengthEq" json:"length_eq,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *FieldValidator) Reset() { *m = FieldValidator{} } +func (m *FieldValidator) String() string { return proto.CompactTextString(m) } +func (*FieldValidator) ProtoMessage() {} +func (*FieldValidator) Descriptor() ([]byte, []int) { return fileDescriptorValidator, []int{0} } + +func (m *FieldValidator) GetRegex() string { + if m != nil && m.Regex != nil { + return *m.Regex + } + return "" +} + +func (m *FieldValidator) GetIntGt() int64 { + if m != nil && m.IntGt != nil { + return *m.IntGt + } + return 0 +} + +func (m *FieldValidator) GetIntLt() int64 { + if m != nil && m.IntLt != nil { + return *m.IntLt + } + return 0 +} + +func (m *FieldValidator) GetMsgExists() bool { + if m != nil && m.MsgExists != nil { + return *m.MsgExists + } + return false +} + +func (m *FieldValidator) GetHumanError() string { + if m != nil && m.HumanError != nil { + return *m.HumanError + } + return "" +} + +func (m *FieldValidator) GetFloatGt() float64 { + if m != nil && m.FloatGt != nil { + return *m.FloatGt + } + return 0 +} + +func (m *FieldValidator) GetFloatLt() float64 { + if m != nil && m.FloatLt != nil { + return *m.FloatLt + } + return 0 +} + +func (m *FieldValidator) GetFloatEpsilon() float64 { + if m != nil && m.FloatEpsilon != nil { + return *m.FloatEpsilon + } + return 0 +} + +func (m *FieldValidator) GetFloatGte() float64 { + if m != nil && m.FloatGte != nil { + return *m.FloatGte + } + return 0 +} + +func (m *FieldValidator) GetFloatLte() float64 { + if m != nil && m.FloatLte != nil { + return *m.FloatLte + } + return 0 +} + +func (m *FieldValidator) GetStringNotEmpty() bool { + if m != nil && m.StringNotEmpty != nil { + return *m.StringNotEmpty + } + return false +} + +func (m *FieldValidator) GetRepeatedCountMin() int64 { + if m != nil && m.RepeatedCountMin != nil { + return *m.RepeatedCountMin + } + return 0 +} + +func (m *FieldValidator) GetRepeatedCountMax() int64 { + if m != nil && m.RepeatedCountMax != nil { + return *m.RepeatedCountMax + } + return 0 +} + +func (m *FieldValidator) GetLengthGt() int64 { + if m != nil && m.LengthGt != nil { + return *m.LengthGt + } + return 0 +} + +func (m *FieldValidator) GetLengthLt() int64 { + if m != nil && m.LengthLt != nil { + return *m.LengthLt + } + return 0 +} + +func (m *FieldValidator) GetLengthEq() int64 { + if m != nil && m.LengthEq != nil { + return *m.LengthEq + } + return 0 +} + +var E_Field = &proto.ExtensionDesc{ + ExtendedType: (*google_protobuf.FieldOptions)(nil), + ExtensionType: (*FieldValidator)(nil), + Field: 65020, + Name: "validator.field", + Tag: "bytes,65020,opt,name=field", + Filename: "validator.proto", +} + +func init() { + proto.RegisterType((*FieldValidator)(nil), "validator.FieldValidator") + proto.RegisterExtension(E_Field) +} + +func init() { proto.RegisterFile("validator.proto", fileDescriptorValidator) } + +var fileDescriptorValidator = []byte{ + // 392 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x6c, 0x92, 0xcf, 0x8f, 0x93, 0x40, + 0x14, 0xc7, 0x83, 0xbb, 0xec, 0xc2, 0xb0, 0xdb, 0x6d, 0x26, 0x9a, 0x4c, 0x35, 0x8d, 0x44, 0x2f, + 0x1c, 0x0c, 0x4d, 0x3c, 0x7a, 0xd4, 0x60, 0x2f, 0xf8, 0x23, 0x1c, 0x3c, 0x78, 0x21, 0x58, 0x5e, + 0xa7, 0x93, 0x0c, 0x33, 0x74, 0xe6, 0xd5, 0xe0, 0x3f, 0xe0, 0x3f, 0xad, 0x07, 0xc3, 0x20, 0x85, + 0x4d, 0x7a, 0x9c, 0xcf, 0xe7, 0xcb, 0x1b, 0x78, 0x7c, 0xc9, 0xc3, 0xcf, 0x4a, 0x8a, 0xba, 0x42, + 0x6d, 0xd2, 0xd6, 0x68, 0xd4, 0x34, 0x3c, 0x83, 0xe7, 0x31, 0xd7, 0x9a, 0x4b, 0xd8, 0x38, 0xf1, + 0xe3, 0xb4, 0xdf, 0xd4, 0x60, 0x77, 0x46, 0xb4, 0xe7, 0xf0, 0xab, 0xdf, 0xd7, 0x64, 0xf1, 0x51, + 0x80, 0xac, 0xbf, 0x8d, 0x0f, 0xd1, 0xa7, 0xc4, 0x37, 0xc0, 0xa1, 0x63, 0x5e, 0xec, 0x25, 0x61, + 0x31, 0x1c, 0xe8, 0x33, 0x72, 0x23, 0x14, 0x96, 0x1c, 0xd9, 0x93, 0xd8, 0x4b, 0xae, 0x0a, 0x5f, + 0x28, 0xdc, 0xe2, 0x88, 0x25, 0xb2, 0xab, 0x33, 0xce, 0x91, 0xae, 0x09, 0x69, 0x2c, 0x2f, 0xa1, + 0x13, 0x16, 0x2d, 0xbb, 0x8e, 0xbd, 0x24, 0x28, 0xc2, 0xc6, 0xf2, 0xcc, 0x01, 0xfa, 0x92, 0x44, + 0x87, 0x53, 0x53, 0xa9, 0x12, 0x8c, 0xd1, 0x86, 0xf9, 0xee, 0x22, 0xe2, 0x50, 0xd6, 0x13, 0xba, + 0x22, 0xc1, 0x5e, 0xea, 0xca, 0xdd, 0x77, 0x13, 0x7b, 0x89, 0x57, 0xdc, 0xba, 0xf3, 0x16, 0x27, + 0x25, 0x91, 0xdd, 0xce, 0x54, 0x8e, 0xf4, 0x35, 0xb9, 0x1f, 0x14, 0xb4, 0x56, 0x48, 0xad, 0x58, + 0xe0, 0xfc, 0x9d, 0x83, 0xd9, 0xc0, 0xe8, 0x0b, 0x12, 0x8e, 0xa3, 0x81, 0x85, 0x2e, 0x10, 0xfc, + 0x9f, 0x0d, 0x93, 0x94, 0x08, 0x8c, 0xcc, 0x64, 0x8e, 0x40, 0x13, 0xb2, 0xb4, 0x68, 0x84, 0xe2, + 0xa5, 0xd2, 0x58, 0x42, 0xd3, 0xe2, 0x2f, 0x16, 0xb9, 0x4f, 0x5b, 0x0c, 0xfc, 0xb3, 0xc6, 0xac, + 0xa7, 0xf4, 0x0d, 0xa1, 0x06, 0x5a, 0xa8, 0x10, 0xea, 0x72, 0xa7, 0x4f, 0x0a, 0xcb, 0x46, 0x28, + 0x76, 0xe7, 0x36, 0xb4, 0x1c, 0xcd, 0x87, 0x5e, 0x7c, 0x12, 0xea, 0x52, 0xba, 0xea, 0xd8, 0xfd, + 0xa5, 0x74, 0xd5, 0xf5, 0xaf, 0x28, 0x41, 0x71, 0x3c, 0xf4, 0xbb, 0x59, 0xb8, 0x50, 0x30, 0x80, + 0x2d, 0xce, 0xa4, 0x44, 0xf6, 0x30, 0x97, 0xf9, 0x5c, 0xc2, 0x91, 0x2d, 0xe7, 0x32, 0x3b, 0xbe, + 0xfb, 0x4a, 0xfc, 0x7d, 0xdf, 0x03, 0xba, 0x4e, 0x87, 0xd2, 0xa4, 0x63, 0x69, 0x52, 0xd7, 0x8f, + 0x2f, 0x2d, 0x0a, 0xad, 0x2c, 0xfb, 0xfb, 0xa7, 0xff, 0xd1, 0xd1, 0xdb, 0x55, 0x3a, 0xf5, 0xee, + 0x71, 0x81, 0x8a, 0x61, 0xd0, 0xfb, 0xe8, 0xfb, 0xd4, 0xc4, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, + 0xec, 0x56, 0x30, 0x88, 0xa6, 0x02, 0x00, 0x00, +} diff --git a/github.com/mwitkow/go-proto-validators/validator.proto b/github.com/mwitkow/go-proto-validators/validator.proto new file mode 100644 index 0000000000..88f19a67c1 --- /dev/null +++ b/github.com/mwitkow/go-proto-validators/validator.proto @@ -0,0 +1,66 @@ +// Copyright 2016 Michal Witkowski. All Rights Reserved. +// See LICENSE for licensing terms. + +// Protocol Buffers extensions for defining auto-generateable validators for messages. + +// TODO(mwitkow): Add example. + + +syntax = "proto2"; +package validator; + +import "google/protobuf/descriptor.proto"; + +option go_package = "validator"; + +// TODO(mwitkow): Email protobuf-global-extension-registry@google.com to get an extension ID. + +extend google.protobuf.FieldOptions { + optional FieldValidator field = 65020; +} + +message FieldValidator { + // Uses a Golang RE2-syntax regex to match the field contents. + optional string regex = 1; + // Field value of integer strictly greater than this value. + optional int64 int_gt = 2; + // Field value of integer strictly smaller than this value. + optional int64 int_lt = 3; + // Used for nested message types, requires that the message type exists. + optional bool msg_exists = 4; + // Human error specifies a user-customizable error that is visible to the user. + optional string human_error = 5; + // Field value of double strictly greater than this value. + // Note that this value can only take on a valid floating point + // value. Use together with float_epsilon if you need something more specific. + optional double float_gt = 6; + // Field value of double strictly smaller than this value. + // Note that this value can only take on a valid floating point + // value. Use together with float_epsilon if you need something more specific. + optional double float_lt = 7; + // Field value of double describing the epsilon within which + // any comparison should be considered to be true. For example, + // when using float_gt = 0.35, using a float_epsilon of 0.05 + // would mean that any value above 0.30 is acceptable. It can be + // thought of as a {float_value_condition} +- {float_epsilon}. + // If unset, no correction for floating point inaccuracies in + // comparisons will be attempted. + optional double float_epsilon = 8; + // Floating-point value compared to which the field content should be greater or equal. + optional double float_gte = 9; + // Floating-point value compared to which the field content should be smaller or equal. + optional double float_lte = 10; + // Used for string fields, requires the string to be not empty (i.e different from ""). + optional bool string_not_empty = 11; + // Repeated field with at least this number of elements. + optional int64 repeated_count_min = 12; + // Repeated field with at most this number of elements. + optional int64 repeated_count_max = 13; + // Field value of length greater than this value. + optional int64 length_gt = 14; + // Field value of length smaller than this value. + optional int64 length_lt = 15; + // Field value of integer strictly equal this value. + optional int64 length_eq = 16; + +} diff --git a/github.com/pseudomuto/protoc-gen-doc/.dockerignore b/github.com/pseudomuto/protoc-gen-doc/.dockerignore new file mode 100644 index 0000000000..ce4f1fc3f6 --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/.dockerignore @@ -0,0 +1,2 @@ +/ +!/script diff --git a/github.com/pseudomuto/protoc-gen-doc/.gitignore b/github.com/pseudomuto/protoc-gen-doc/.gitignore new file mode 100644 index 0000000000..0a89e775c8 --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/.gitignore @@ -0,0 +1,8 @@ +/_tools +/coverage.txt +/dist +/gen_fixtures +/protoc-gen-doc +/test/*.dat +/vendor +/tmp/ diff --git a/github.com/pseudomuto/protoc-gen-doc/.gofmtignore b/github.com/pseudomuto/protoc-gen-doc/.gofmtignore new file mode 100644 index 0000000000..5cecb48392 --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/.gofmtignore @@ -0,0 +1,2 @@ +/_tools/* +/vendor/* diff --git a/github.com/pseudomuto/protoc-gen-doc/.travis.yml b/github.com/pseudomuto/protoc-gen-doc/.travis.yml new file mode 100644 index 0000000000..01bf213355 --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/.travis.yml @@ -0,0 +1,44 @@ +language: go +sudo: required + +services: + - docker + +env: + global: + - GO111MODULE="on" + - PROTOC_RELEASE="https://github.com/google/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip" + - PROTOC_TARGET="${HOME}/protoc" + - PATH="${PROTOC_TARGET}/bin:${PATH}" + - secure: Tbet2rxD8QgjthAo+bxt41qbF2wUPTx0difGK5p4yQISK/njTuT5cqcxnOa4GIbyKtNtx0EgGnyVcQJiQkmZiF6Sabf0mtqU/CQ4PmVV76e9bHwA/CrTtudibMn16ozxuuxvhNxFOMQEhwcQOkW93M/Q9FZUEw9/CGpRGFfSzuA= + +cache: + - "${HOME}/protoc" + +go: + - 1.12.x + - master + +install: + - if [ ! -d "${PROTOC_TARGET}" ]; then curl -fsSL "$PROTOC_RELEASE" > "${PROTOC_TARGET}.zip"; fi + - if [ -f "${PROTOC_TARGET}.zip" ]; then unzip "${PROTOC_TARGET}.zip" -d "${PROTOC_TARGET}"; fi + - go get github.com/golang/protobuf/{proto,protoc-gen-go} github.com/haya14busa/goverage + +script: + - goverage -race -coverprofile=coverage.txt -covermode=atomic + - make bench + - make lint + +after_success: + - bash <(curl -s https://codecov.io/bash) + +deploy: + provider: script + script: script/push_to_docker.sh + skip_cleanup: true + on: + tags: true + all_branches: true + +notifications: + email: false diff --git a/github.com/pseudomuto/protoc-gen-doc/CHANGELOG.md b/github.com/pseudomuto/protoc-gen-doc/CHANGELOG.md new file mode 100644 index 0000000000..88b82cdebd --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/CHANGELOG.md @@ -0,0 +1,189 @@ +# Changelog + +All noteworthy changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) (as of Feb 2018) +and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). + +## [Unreleased](https://github.com/pseudomuto/protoc-gen-doc/compare/v1.3.2...master) + +## [v1.3.2](https://github.com/pseudomuto/protoc-gen-doc/compare/v1.3.1...v1.3.2) - May 7, 2020 + +### Changed + +* Stripped unnecessary whitespace from markdown templates [#410](https://github.com/pseudomuto/protoc-gen-doc/pull/410) +* Renamed `lyft_validate` extension to `envoyproxy_validate` + [#392](https://github.com/pseudomuto/protoc-gen-doc/pull/392) +* Fixed `envoyproxy_validate` rules with slice values such as `in` and `not_in` + [#392](https://github.com/pseudomuto/protoc-gen-doc/pull/392) + +## [v1.3.1](https://github.com/pseudomuto/protoc-gen-doc/compare/v1.3.0...v1.3.1) - March 3, 2020 + +### Changed + +* Switched from dep to go modules + +## [v1.3.0](https://github.com/pseudomuto/protoc-gen-doc/compare/v1.2.0...v1.3.0) - April 15, 2019 + +### Added + +* Support for options and extended options in files, services, methods, enums, enum values, messages and fields [#376](https://github.com/pseudomuto/protoc-gen-doc/pull/376) + +### Changed + +### Fixed + +## [v1.2.0](https://github.com/pseudomuto/protoc-gen-doc/compare/v1.1.0...v1.2.0) - March 13, 2019 + +### Added + +* Added [Sprig](https://github.com/Masterminds/sprig) functions to renderer contexts +* Added `IsMap` to `MessageField` objects indicating whether or not the field is a map field +* Added `RequestStreaming` and `ResponseStreaming` to `ServiceMethod` objects indicating whether the request and/or response are streaming. Updated the templates to add "stream" indicators to method requests and responses. +* Support for recursively adding nested enums and messages. + +### Changed + +* Bumped protobuf to 3.6.1 in docker container + +### Fixed + +* CI issue related to Regexp comparison on Golang master +* Markdown template was incorrectly links response types to request types +* Markdown template anchor tags were self-closing (invalid). + +## [v1.1.0](https://github.com/pseudomuto/protoc-gen-doc/compare/v1.0.0...v1.1.0) - March 13, 2018 + +### Added + +* Switch to using [protokit] to handle parsing and running the plugin +* Default values are not included in the built-in templates +* Added the ability to ignore certain files by adding `:pattern[,pattern]` to `--doc_opt` +* Added `-help` and `-version` flags to the binary (e.g. `./protoc-gen-doc -help`) + +### Changed + +* Dev tooling, now using dep and retool +* CI setup to use go 1.10 (and master) and protoc 3.5.1 +* Docker image updated to use protoc 3.5.1 as well + +## Fixed + +* Several issues with the documentation +* `NrBr` filter was a little too aggressive + +### Removed + +* The entire `parser` package (in favor of [protokit]) + +[protokit]: https://github.com/pseudomuto/protokit + +# v1.0.0 - September 26, 2017 + +This is the tenth official release. (However, it's the first 1.x release!!) + +**New things** + +* Support for both proto2 and proto3! +* Application is now dockerized, no need to install it (unless you want to of course) +* Ported to Go and precompiled for Linux, OSX, and Windows +* Added test coverage to ensure functionality and prevent regressions +* Ignore comments starting with `@exclude` +* Added (and backfilled) CHANGELOG.md +* Added CONTRIBUTING.md + +**Bug fixes** + +* Message enums no longer included in file enums (#288) +* Added top-level `files` key for JSON output and _camelCased_ all fields (#289) +* Fixed issue with TOC in Markdown not nesting correctly (#293) +* No need to worry about expiring apt keys (#295) +* Extra slashes in comment prefix no longer show up in output (#298) +* Added field details for templates (#300) +* Markdown headers include new line so they render correctly (#303) + +**Breaking changes** + +* File-level comments are now attached to the syntax directive +* JSON fields are now _camelCased_ +* Mustache templates replaced with golang templates +* Dropped direct support for PDF generation (still possible with FOP) +* `doc_out` flag removed in favor of `doc_opt` + + +# v0.9 - February 26, 2017 + +This is the ninth official release. + +**changes** + +* Improve installation instructions for macOS (thanks @guozheng) +* Improve installation instructions for Debian/Ubuntu (thanks @mhaberler) +* Add asciidoc.mustache example template (thanks @ArcEye) +* Don't do HTML escaping in Markdown template (thanks @sunfmin) +* Add support for JSON output + +# v0.8 - February 26, 2016 + +This is the eight official release. + +**changes** + +* Add support for documenting files (#9) +* Add support for default values (#11) +* Add no-exclude flag to ignore @exclude directives (#13) +* Add support for RPC services (#14) (thanks to @murph0 !) + +# v0.7 - January 7, 2016 + +This is the seventh official release. + +**changes** + +* Added support for extensions (thanks @masterzen !) +* Added Custom Templates wiki page +* Added additional distro packages for Debian 8, Ubuntu 15.04 + 15.10 and Fedora 22 + 23 + +# v0.6 - April 8, 2015 + +This is the sixth official release. + +No functional changes were made, but Linux distribution package repositories are now provided for Ubuntu, Arch, Fedora +and openSUSE through the Open Build Service, and an RPM for CentOS 7 here below. + +# v0.5 - December 19, 2014 + +This is the fifth official release. + +**changes** + +* Support exclusion also of enum values (accidental omission in 0.4). + +# v0.4 - December 19, 2014 + +This is the fourth official release. + +**changes** + +* Updated to a newer version of qt-mustache. +* Updated Windows zip to libprotobuf/libprotoc 2.6.1. +* Added support for excluding messages/enums/fields. + +# v0.3 - August 19, 2014 + +This is the third official release. + +**changes** + +* Updated to a newer version of qt-mustache which is more spec compliant. +* Added missing documentation for enums to Markdown output. + +# v0.2 - August 14, 2014 + +This is the second official release. + +* No functional changes were made, but the build system was improved. + +# v0.1 - August 6, 2014 + +* Initial release diff --git a/github.com/pseudomuto/protoc-gen-doc/CONTRIBUTING.md b/github.com/pseudomuto/protoc-gen-doc/CONTRIBUTING.md new file mode 100644 index 0000000000..39b268ca7b --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/CONTRIBUTING.md @@ -0,0 +1,79 @@ +# Contributing + +First off, glad you're here and want to contribute! :heart: + +## Getting Started + +In order to work on this project, you'll need to install a few things: + +1. A recent version of [Go](https://golang.org/doc/install) +1. [protoc](https://github.com/google/protobuf#protocol-compiler-installation) - The protobuf compiler +1. [Docker](https://www.docker.com/) (only required to if you need/want to build the docker container via `make docker`) + +Once those are installed, running `make setup` should get you the rest of the way. + +When writing tests, be sure that the package in the test file is suffixed with `_test`. Eg. `protoc_gen_doc_test`. This +ensures that you'll only be testing the public interface. + +## Submitting a PR + +Here are some general guidelines for making PRs for this repo. + +1. [Fork this repo](https://github.com/pseudomuto/protoc-gen-doc/fork) +1. Make a branch off of master (`git checkout -b `) +1. Make focused commits with descriptive messages +1. Add tests that fail without your code, and pass with it +1. GoFmt your code! (see to setup your editor to do this for you) +1. Be sure to run `make examples` so your changes are reflected in the example docs +1. **Ping someone on the PR** (Lots of people, including myself, won't get a notification unless pinged directly) + +Every PR should have a well detailed summary of the changes being made and the reasoning behind them. Make sure to add +at least three sections. + +### What is Changing? + +Make sure you spell out in as much detail as necessary what will happen to which systems when your PR is merged, +what are the expected changes. + +### How is it Changing? + +Include any relevant implementation details, mimize surprises for the reviewers in this section, if you had to take some +unorthodox approaches (read hacks), explain why here. + +### What Could Go Wrong? + +How has this change been tested? In your opinion what is the risk, if any, of merging these changes. + +#### Reviewers should: + +1. Identify anything that the PR author may have missed from above. +2. Test the PR through whatever means necessary, including manually, to verify it is safe to be deployed. +3. Question everything. Never assume that something was tested or fully understood, always question and ask if there is + any uncertainty. +4. Before merging the PR make sure it has _**one**_ of the `Major release`, `Minor release`, or `Patch release` labels + applied to it (useful for the changelog and determining the version for the next release). + +## Release Process + +We follow [Semantic Versioning 2.0.0](http://semver.org/#semantic-versioning-200), and the fact that PRs are tagged with +the type of release makes determining the next version super simple. + +Look through the new (since the last release) PRs that are included in this release to determine the new version. + +* If COUNT(labelled `Major release`) > 0, then it's a MAJOR version bump. +* If COUNT(labelled `Minor release`) > 0, then it's a MINOR version bump. +* PATCH version bump + +### Now that we've got the version: + +* Run `make docker_test` to build the image and generate the examples. There should be no diff after this. +* Update the version in `version.go` +* Update CHANGELOG.md. Be sure to include links to PRs and highlight new features, bug fixes, and any breaking changes. +* Run `make release` + +Now that the tag is on GitHub, we have a couple more things to do: + +* Create a release based on the tag and copy the entry in CHANGELOG.md into the notes. +* Run `make dist` and add the tar files (in `./dist`) to the release. + +Once CI has run for the tag, Travis will push the image to DockerHub. diff --git a/github.com/pseudomuto/protoc-gen-doc/Dockerfile b/github.com/pseudomuto/protoc-gen-doc/Dockerfile new file mode 100644 index 0000000000..6c361727e4 --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/Dockerfile @@ -0,0 +1,21 @@ +FROM debian:jessie-slim +LABEL maintainer="pseudomuto " protoc_version="3.6.1" + +WORKDIR / + +ADD https://github.com/google/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip ./ +RUN apt-get -q -y update && \ + apt-get -q -y install unzip && \ + unzip protoc-3.6.1-linux-x86_64.zip -d ./usr/local && \ + rm protoc-3.6.1-linux-x86_64.zip && \ + apt-get remove --purge -y unzip && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* + +ADD dist/protoc-gen-doc /usr/local/bin/ +ADD script/entrypoint.sh ./ + +VOLUME ["/out", "/protos"] + +ENTRYPOINT ["/entrypoint.sh"] +CMD ["--doc_opt=html,index.html"] diff --git a/github.com/pseudomuto/protoc-gen-doc/LICENSE.md b/github.com/pseudomuto/protoc-gen-doc/LICENSE.md new file mode 100644 index 0000000000..7baa276e49 --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/LICENSE.md @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 David Muto (pseudomuto) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/github.com/pseudomuto/protoc-gen-doc/Makefile b/github.com/pseudomuto/protoc-gen-doc/Makefile new file mode 100644 index 0000000000..093e899225 --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/Makefile @@ -0,0 +1,81 @@ +.PHONY: bench test build dist docker examples release lint + +export GO111MODULE=on + +EXAMPLE_DIR=$(PWD)/examples +DOCS_DIR=$(EXAMPLE_DIR)/doc +PROTOS_DIR=$(EXAMPLE_DIR)/proto + +EXAMPLE_CMD=protoc --plugin=protoc-gen-doc \ + -Ithirdparty -Itmp/googleapis -Iexamples/proto \ + --doc_out=examples/doc + +DOCKER_CMD=docker run --rm \ + -v $(DOCS_DIR):/out:rw \ + -v $(PROTOS_DIR):/protos:ro \ + -v $(EXAMPLE_DIR)/templates:/templates:ro \ + -v $(PWD)/thirdparty/github.com/mwitkow:/usr/local/include/github.com/mwitkow:ro \ + -v $(PWD)/thirdparty/github.com/envoyproxy:/usr/local/include/github.com/envoyproxy:ro \ + -v $(PWD)/tmp/googleapis/google/api:/usr/local/include/google/api:ro \ + pseudomuto/protoc-gen-doc + +VERSION = $(shell cat version.go | sed -n 's/.*const VERSION = "\(.*\)"/\1/p') + +resources.go: resources/*.tmpl resources/*.json + $(info Generating resources...) + @go run resources/main.go -in resources -out resources.go -pkg gendoc + +fixtures/fileset.pb: fixtures/*.proto fixtures/generate.go + $(info Generating fixtures...) + @cd fixtures && go generate + +tmp/googleapis: + rm -rf tmp/googleapis tmp/protocolbuffers + git clone --depth 1 https://github.com/googleapis/googleapis tmp/googleapis + rm -rf tmp/googleapis/.git + git clone --depth 1 https://github.com/protocolbuffers/protobuf tmp/protocolbuffers + cp -r tmp/protocolbuffers/src/* tmp/googleapis/ + rm -rf tmp/protocolbuffers + +test: fixtures/fileset.pb resources.go + @go test -cover -race ./ ./cmd/... ./extensions/... + +bench: + @go test -bench=. + +build: resources.go + @go build ./cmd/... + +dist: + @script/dist.sh + +docker: + @script/push_to_docker.sh + +docker_test: build tmp/googleapis docker + @rm -f examples/doc/* + @$(DOCKER_CMD) --doc_opt=docbook,example.docbook:Ignore* + @$(DOCKER_CMD) --doc_opt=html,example.html:Ignore* + @$(DOCKER_CMD) --doc_opt=json,example.json:Ignore* + @$(DOCKER_CMD) --doc_opt=markdown,example.md:Ignore* + @$(DOCKER_CMD) --doc_opt=/templates/asciidoc.tmpl,example.txt:Ignore* + +examples: build tmp/googleapis examples/proto/*.proto examples/templates/*.tmpl + $(info Making examples...) + @rm -f examples/doc/* + @$(EXAMPLE_CMD) --doc_opt=docbook,example.docbook:Ignore* examples/proto/*.proto + @$(EXAMPLE_CMD) --doc_opt=html,example.html:Ignore* examples/proto/*.proto + @$(EXAMPLE_CMD) --doc_opt=json,example.json:Ignore* examples/proto/*.proto + @$(EXAMPLE_CMD) --doc_opt=markdown,example.md:Ignore* examples/proto/*.proto + @$(EXAMPLE_CMD) --doc_opt=examples/templates/asciidoc.tmpl,example.txt:Ignore* examples/proto/*.proto + +release: + @echo Releasing v${VERSION}... + git add CHANGELOG.md version.go + git commit -m "Bump version to v${VERSION}" + git tag -m "Version ${VERSION}" "v${VERSION}" + git push && git push --tags + +lint: + @which revive >/dev/null || go get github.com/mgechev/revive + revive --config revive.toml ./... diff --git a/github.com/pseudomuto/protoc-gen-doc/README.md b/github.com/pseudomuto/protoc-gen-doc/README.md new file mode 100644 index 0000000000..e63bda8ff6 --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/README.md @@ -0,0 +1,203 @@ +# protoc-gen-doc + +[![Travis Build Status][travis-svg]][travis-ci] +[![codecov][codecov-svg]][codecov-url] +[![GoDoc][godoc-svg]][godoc-url] +[![Go Report Card][goreport-svg]][goreport-url] + +This is a documentation generator plugin for the Google Protocol Buffers compiler (`protoc`). The plugin can generate +HTML, JSON, DocBook and Markdown documentation from comments in your `.proto` files. + +It supports proto2 and proto3, and can handle having both in the same context (see [examples](examples/) for proof). + +## Installation + +There is a Docker image available (`docker pull pseudomuto/protoc-gen-doc`) that has everything you need to generate +documentation from your protos. + +If you'd like to install this locally, you can `go get` it. + +`go get -u github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc` + +## Invoking the Plugin + +The plugin is invoked by passing the `--doc_out`, and `--doc_opt` options to the `protoc` compiler. The option has the +following format: + + --doc_opt=|, + +The format may be one of the built-in ones ( `docbook`, `html`, `markdown` or `json`) +or the name of a file containing a custom [Go template][gotemplate]. + +### Using the Docker Image (Recommended) + +The docker image has two volumes: `/out` and `/protos` which are the directory to write the documentation to and the +directory containing your proto files. + +You could generate HTML docs for the examples by running the following: + +``` +docker run --rm \ + -v $(pwd)/examples/doc:/out \ + -v $(pwd)/examples/proto:/protos \ + pseudomuto/protoc-gen-doc +``` + +By default HTML documentation is generated in `/out/index.html` for all `.proto` files in the `/protos` volume. This can +be changed by passing the `--doc_opt` parameter to the container. + +For example, to generate Markdown for all the examples: + +``` +docker run --rm \ + -v $(pwd)/examples/doc:/out \ + -v $(pwd)/examples/proto:/protos \ + pseudomuto/protoc-gen-doc --doc_opt=markdown,docs.md +``` + +You can also generate documentation for a single file. This can be done by passing the file(s) to the command: + +``` +docker run --rm \ + -v $(pwd)/examples/doc:/out \ + -v $(pwd)/examples/proto:/protos \ + pseudomuto/protoc-gen-doc --doc_opt=markdown,docs.md /protos/Booking.proto [OPTIONALLY LIST MORE FILES] +``` + +You can also exclude proto files that match specific path expressions. This is done by passing a second option delimited by `:`. +For example, you can pass any number of comma separated patterns as the second option: + +``` +docker run --rm \ + -v $(pwd)/examples/doc:/out \ + -v $(pwd)/examples/proto:/protos \ + pseudomuto/protoc-gen-doc --doc_opt=:google/*,somepath/* +``` + +_**Remember**_: Paths should be from within the container, not the host! + +> NOTE: Due to the way wildcard expansion works with docker you cannot use a wildcard path (e.g. `protos/*.proto`) in +the file list. To get around this, if no files are passed, the container will generate docs for `protos/*.proto`, which +can be changed by mounting different volumes. + +### Simple Usage + +For example, to generate HTML documentation for all `.proto` files in the `proto` directory into `doc/index.html`, type: + + protoc --doc_out=./doc --doc_opt=html,index.html proto/*.proto + +The plugin executable must be in `PATH` for this to work. + +### Using a precompiled binary + +Alternatively, you can specify a pre-built/not in `PATH` binary using the `--plugin` option. + + protoc \ + --plugin=protoc-gen-doc=./protoc-gen-doc \ + --doc_out=./doc \ + --doc_opt=html,index.html \ + proto/*.proto + +### With a Custom Template + +If you'd like to use your own template, simply use the path to the template file rather than the type. + + protoc --doc_out=./doc --doc_opt=/path/to/template.tmpl,index.txt proto/*.proto + +For information about the available template arguments and functions, see [Custom Templates][custom]. If you just want +to customize the look of the HTML output, put your CSS in `stylesheet.css` next to the output file and it will be picked +up. + +## Writing Documentation + +Messages, Fields, Services (and their methods), Enums (and their values), Extensions, and Files can be documented. +Generally speaking, comments come in 2 forms: leading and trailing. + +**Leading comments** + +Leading comments can be used everywhere. + +```protobuf +/** + * This is a leading comment for a message +*/ +message SomeMessage { + // this is another leading comment + string value = 1; +} +``` + +> NOTE: File level comments should be leading comments on the syntax directive. + +**Trailing comments** + +Fields, Service Methods, Enum Values and Extensions support trailing comments. + +```protobuf +enum MyEnum { + DEFAULT = 0; // the default value + OTHER = 1; // the other value +} +``` + +**Excluding comments** + +If you want to have some comment in your proto files, but don't want them to be part of the docs, you can simply prefix +the comment with `@exclude`. + +Example: include only the comment for the `id` field + +```protobuf +/** + * @exclude + * This comment won't be rendered + */ +message ExcludedMessage { + string id = 1; // the id of this message. + string name = 2; // @exclude the name of this message + + /* @exclude the value of this message. */ + int32 value = 3; +} +``` + +Check out the [example protos](examples/proto) to see all the options. + +## Output Example + +With the input `.proto` files + +* [Booking.proto](examples/proto/Booking.proto) +* [Customer.proto](examples/proto/Customer.proto) +* [Vehicle.proto](examples/proto/Vehicle.proto) + +the plugin gives the output + +* [Markdown](examples/doc/example.md) +* [HTML][html_preview] +* [DocBook](examples/doc/example.docbook) +* [JSON](examples/doc/example.json) + +Check out the `examples` task in the [Makefile](Makefile) to see how these were generated. + +[gotemplate]: + https://golang.org/pkg/text/template/ + "Template - The Go Programming Language" +[custom]: + https://github.com/pseudomuto/protoc-gen-doc/wiki/Custom-Templates + "Custom templates instructions" +[html_preview]: + https://rawgit.com/pseudomuto/protoc-gen-doc/master/examples/doc/example.html + "HTML Example Output" +[travis-svg]: + https://travis-ci.org/pseudomuto/protoc-gen-doc.svg?branch=master + "Travis CI build status SVG" +[travis-ci]: + https://travis-ci.org/pseudomuto/protoc-gen-doc + "protoc-gen-doc at Travis CI" +[codecov-svg]: https://codecov.io/gh/pseudomuto/protoc-gen-doc/branch/master/graph/badge.svg +[codecov-url]: https://codecov.io/gh/pseudomuto/protoc-gen-doc +[godoc-svg]: https://godoc.org/github.com/pseudomuto/protoc-gen-doc?status.svg +[godoc-url]: https://godoc.org/github.com/pseudomuto/protoc-gen-doc +[goreport-svg]: https://goreportcard.com/badge/github.com/pseudomuto/protoc-gen-doc +[goreport-url]: https://goreportcard.com/report/github.com/pseudomuto/protoc-gen-doc diff --git a/github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc/flags.go b/github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc/flags.go new file mode 100644 index 0000000000..09132e2f33 --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc/flags.go @@ -0,0 +1,98 @@ +package main + +import ( + "flag" + "fmt" + "io" + + gendoc "github.com/pseudomuto/protoc-gen-doc" +) + +const helpMessage = ` +This is a protoc plugin that is used to generate documentation from your protobuf files. Invocation is controlled by +using the doc_opt and doc_out options for protoc. + +EXAMPLE: Generate HTML docs +protoc --doc_out=. --doc_opt=html,index.html protos/*.proto + +EXAMPLE: Exclude file patterns +protoc --doc_out=. --doc_opt=html,index.html:google/*,somedir/* protos/*.proto + +EXAMPLE: Use a custom template +protoc --doc_out=. --doc_opt=custom.tmpl,docs.txt protos/*.proto + +See https://github.com/pseudomuto/protoc-gen-doc for more details. +` + +// Version returns the currently running version of protoc-gen-doc +func Version() string { + return gendoc.VERSION +} + +// Flags contains details about the CLI invocation of protoc-gen-doc +type Flags struct { + appName string + flagSet *flag.FlagSet + err error + showHelp bool + showVersion bool + writer io.Writer +} + +// Code returns the status code to exit with after handling the supplied flags +func (f *Flags) Code() int { + if f.err != nil { + return 1 + } + + return 0 +} + +// HasMatch returns whether or not the supplied args are matches. For example, passing `--help` will match, or some +// unknown parameter, but passing nothing will not. +func (f *Flags) HasMatch() bool { + return f.ShowHelp() || f.ShowVersion() +} + +// ShowHelp determines whether or not to show the help message +func (f *Flags) ShowHelp() bool { + return f.err != nil || f.showHelp +} + +// ShowVersion determines whether or not to show the version message +func (f *Flags) ShowVersion() bool { + return f.showVersion +} + +// PrintHelp prints the usage string including all flags to the `io.Writer` that was supplied to the `Flags` object. +func (f *Flags) PrintHelp() { + fmt.Fprintf(f.writer, "Usage of %s:\n", f.appName) + fmt.Fprintf(f.writer, "%s\n", helpMessage) + fmt.Fprintf(f.writer, "FLAGS\n") + f.flagSet.PrintDefaults() +} + +// PrintVersion prints the version string to the `io.Writer` that was supplied to the `Flags` object. +func (f *Flags) PrintVersion() { + fmt.Fprintf(f.writer, "%s version %s\n", f.appName, Version()) +} + +// ParseFlags parses the supplied options are returns a `Flags` object to the caller. +// +// Parameters: +// * `w` - the `io.Writer` to use for printing messages (help, version, etc.) +// * `args` - the set of args the program was invoked with (typically `os.Args`) +func ParseFlags(w io.Writer, args []string) *Flags { + f := Flags{appName: args[0], writer: w} + + f.flagSet = flag.NewFlagSet(args[0], flag.ContinueOnError) + f.flagSet.BoolVar(&f.showHelp, "help", false, "Show this help message") + f.flagSet.BoolVar(&f.showVersion, "version", false, fmt.Sprintf("Print the current version (%v)", Version())) + f.flagSet.SetOutput(w) + + // prevent showing help on parse error + f.flagSet.Usage = func() {} + + f.err = f.flagSet.Parse(args[1:]) + return &f +} diff --git a/github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc/main.go b/github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc/main.go new file mode 100644 index 0000000000..56e42ce4fc --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc/main.go @@ -0,0 +1,53 @@ +// protoc-gen-doc is used to generate documentation from comments in your proto files. +// +// It is a protoc plugin, and can be invoked by passing `--doc_out` and `--doc_opt` arguments to protoc. +// +// Example: generate HTML documentation +// +// protoc --doc_out=. --doc_opt=html,index.html protos/*.proto +// +// Example: use a custom template +// +// protoc --doc_out=. --doc_opt=custom.tmpl,docs.txt protos/*.proto +// +// For more details, check out the README at https://github.com/pseudomuto/protoc-gen-doc +package main + +import ( + "github.com/pseudomuto/protokit" + + "log" + "os" + + gendoc "github.com/pseudomuto/protoc-gen-doc" + _ "github.com/pseudomuto/protoc-gen-doc/extensions/google_api_http" // imported for side effects + _ "github.com/pseudomuto/protoc-gen-doc/extensions/lyft_validate" // imported for side effects + _ "github.com/pseudomuto/protoc-gen-doc/extensions/validator_field" // imported for side effects +) + +func main() { + if flags := ParseFlags(os.Stdout, os.Args); HandleFlags(flags) { + os.Exit(flags.Code()) + } + + if err := protokit.RunPlugin(new(gendoc.Plugin)); err != nil { + log.Fatal(err) + } +} + +// HandleFlags checks if there's a match and returns true if it was "handled" +func HandleFlags(f *Flags) bool { + if !f.HasMatch() { + return false + } + + if f.ShowHelp() { + f.PrintHelp() + } + + if f.ShowVersion() { + f.PrintVersion() + } + + return true +} diff --git a/github.com/pseudomuto/protoc-gen-doc/doc.go b/github.com/pseudomuto/protoc-gen-doc/doc.go new file mode 100644 index 0000000000..e9bee30cf2 --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/doc.go @@ -0,0 +1,19 @@ +// Package gendoc is a protoc plugin for generating documentation from your proto files. +// +// Typically this will not be used as a library, though nothing prevents that. Normally it'll be invoked by passing +// `--doc_out` and `--doc_opt` values to protoc. +// +// Example: generate HTML documentation +// +// protoc --doc_out=. --doc_opt=html,index.html protos/*.proto +// +// Example: exclude patterns +// +// protoc --doc_out=. --doc_opt=html,index.html:google/*,somedir/* protos/*.proto +// +// Example: use a custom template +// +// protoc --doc_out=. --doc_opt=custom.tmpl,docs.txt protos/*.proto +// +// For more details, check out the README at https://github.com/pseudomuto/protoc-gen-doc +package gendoc diff --git a/github.com/pseudomuto/protoc-gen-doc/examples/proto/Booking.proto b/github.com/pseudomuto/protoc-gen-doc/examples/proto/Booking.proto new file mode 100644 index 0000000000..95cac71f12 --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/examples/proto/Booking.proto @@ -0,0 +1,64 @@ +/** + * Booking related messages. + * + * This file is really just an example. The data model is completely + * fictional. + */ +syntax = "proto3"; + +import "google/api/annotations.proto"; +import "github.com/mwitkow/go-proto-validators/validator.proto"; + +package com.example; + +/** + * Represents the booking status ID. + */ +message BookingStatusID { + int32 id = 1; /// Unique booking status ID. +} + +/** + * Represents the status of a vehicle booking. + */ +message BookingStatus { + int32 id = 1; /// Unique booking status ID. + string description = 2 [(validator.field) = {string_not_empty: true, length_lt: 255}]; /// Booking status description. E.g. "Active". +} + +/** + * Represents the booking of a vehicle. + * + * Vehicles are some cool shit. But drive carefully! + */ +message Booking { + int32 vehicle_id = 1; /// ID of booked vehicle. + int32 customer_id = 2; /// Customer that booked the vehicle. + BookingStatus status = 3; /// Status of the booking. + + /** Has booking confirmation been sent? */ + bool confirmation_sent = 4; + + /** Has payment been received? */ + bool payment_received = 5; +} + +// An empty message for testing +message EmptyBookingMessage { +} + +/** + * Service for handling vehicle bookings. + */ +service BookingService { + /// Used to book a vehicle. Pass in a Booking and a BookingStatus will be returned. + rpc BookVehicle (Booking) returns (BookingStatus) { + option (google.api.http) = { + post: "/api/bookings/vehicle/{vehicle_id}" + body: "*" + }; + } + + /// Used to subscribe to updates of the BookingStatus. + rpc BookingUpdates (BookingStatusID) returns (stream BookingStatus); +} diff --git a/github.com/pseudomuto/protoc-gen-doc/examples/proto/Customer.proto b/github.com/pseudomuto/protoc-gen-doc/examples/proto/Customer.proto new file mode 100644 index 0000000000..d640ac2962 --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/examples/proto/Customer.proto @@ -0,0 +1,37 @@ +/// This file has messages for describing a customer. +syntax = "proto2"; + +import "github.com/envoyproxy/protoc-gen-validate/validate/validate.proto"; + +package com.example; + +option ruby_package = "com.example.ruby"; + +// Use // or /** */ to document messages, fields and enums. + +/** + * Represents a mail address. + */ +message Address { + required string address_line_1 = 1 [(validate.rules).string.min_len = 2]; /** First address line. */ + optional string address_line_2 = 2; /** Second address line. */ + optional string address_line_3 = 3; /** Second address line. */ + + required string town = 4; /// Address town. + optional string county = 5; /// Address county, if applicable. + required string country = 6; /// Address country. +} + +/** + * Represents a customer. + */ +message Customer { + required int32 id = 1; /// Unique customer ID. + required string first_name = 2; /// Customer first name. + required string last_name = 3; /// Customer last name. + optional string details = 4; /// Customer details. + + optional string email_address = 5; /// Customer e-mail address. + repeated string phone_number = 6; /// Customer phone numbers, primary first. + repeated Address mail_addresses = 7; /// Customer mail addresses, primary first. +} diff --git a/github.com/pseudomuto/protoc-gen-doc/examples/proto/IgnoreMe.proto b/github.com/pseudomuto/protoc-gen-doc/examples/proto/IgnoreMe.proto new file mode 100644 index 0000000000..eafe1d5506 --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/examples/proto/IgnoreMe.proto @@ -0,0 +1,14 @@ +/** + * This file should be ignored when making example docs. + * + */ +syntax = "proto3"; + +package com.example; + +/** + * This message should be ignored when making example docs. + */ +message Ignored { + string name = 2; +} diff --git a/github.com/pseudomuto/protoc-gen-doc/examples/proto/Vehicle.proto b/github.com/pseudomuto/protoc-gen-doc/examples/proto/Vehicle.proto new file mode 100644 index 0000000000..db6daac023 --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/examples/proto/Vehicle.proto @@ -0,0 +1,88 @@ +/** Messages describing manufacturers / vehicles. */ +syntax = "proto2"; + +package com.example; + +/** + * Represents a manufacturer of cars. + */ +message Manufacturer { + /** + * Manufacturer category. A manufacturer may be either inhouse or external. + */ + enum Category { + CATEGORY_INHOUSE = 0; /** The manufacturer is inhouse. */ + CATEGORY_EXTERNAL = 1; /** The manufacturer is external. */ + } + + required int32 id = 1; /** The unique manufacturer ID. */ + required string code = 2; /** A manufacturer code, e.g. "DKL4P". */ + optional string details = 3; /** Manufacturer details (minimum orders et.c.). */ + + /** Manufacturer category. */ + optional Category category = 4 [default = CATEGORY_EXTERNAL]; + + extensions 100 to max; +} + +// File-level extensions can also be documented: + +extend Manufacturer { + /** Manufacturer country. */ + optional string country = 100 [default = "China"]; +} + +/** + * Represents a vehicle model. + */ +message Model { + required string id = 1; /** The unique model ID. */ + required string model_code = 2; /** The car model code, e.g. "PZ003". */ + required string model_name = 3; /** The car model name, e.g. "Z3". */ + + required sint32 daily_hire_rate_dollars = 4; /// Dollars per day. + required sint32 daily_hire_rate_cents = 5; /// Cents per day. + + extensions 100 to max; +} + +/** + * Represents a vehicle that can be hired. + */ +message Vehicle { + /** + * Represents a vehicle category. E.g. "Sedan" or "Truck". + */ + message Category { + required string code = 1; /// Category code. E.g. "S". + required string description = 2; /// Category name. E.g. "Sedan". + } + + required int32 id = 1; /** Unique vehicle ID. */ + required Model model = 2; /** Vehicle model. */ + required string reg_number = 3; /** Vehicle registration number. */ + optional sint32 mileage = 4; /** Current vehicle mileage, if known. */ + optional Category category = 5; /** Vehicle category. */ + + // Doc comments for fields can come before or + // after the field definition. And just like + // comments for messages / enums, they can be + // multi-paragraph: + + /** + * Dollars per day. + */ + optional sint32 daily_hire_rate_dollars = 6 [default = 50]; + + /** + * Cents per day. + */ + optional sint32 daily_hire_rate_cents = 7; + + // Nested extensions can also be documented: + + extend Model { + /** Vehicle model series. */ + optional string series = 100; + } +} diff --git a/github.com/pseudomuto/protoc-gen-doc/extensions/envoyproxy_validate/envoyproxy_validate.go b/github.com/pseudomuto/protoc-gen-doc/extensions/envoyproxy_validate/envoyproxy_validate.go new file mode 100644 index 0000000000..70941f654d --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/extensions/envoyproxy_validate/envoyproxy_validate.go @@ -0,0 +1,91 @@ +package extensions + +import ( + "encoding/json" + "reflect" + "strings" + + "github.com/envoyproxy/protoc-gen-validate/validate" + "github.com/pseudomuto/protoc-gen-doc/extensions" +) + +// ValidateRule represents a single validator rule from the (validate.rules) method option extension. +type ValidateRule struct { + Name string `json:"name"` + Value interface{} `json:"value"` +} + +// ValidateExtension contains the rules set by the (validate.rules) method option extension. +type ValidateExtension struct { + *validate.FieldRules + rules []ValidateRule // memoized so that we don't have to use reflection more than we need. +} + +// MarshalJSON implements the json.Marshaler interface. +func (v ValidateExtension) MarshalJSON() ([]byte, error) { return json.Marshal(v.Rules()) } + +// Rules returns the set of rules for this extension. +func (v ValidateExtension) Rules() []ValidateRule { + if v.FieldRules == nil { + return nil + } + if v.rules == nil { + v.rules = flattenRules("", reflect.ValueOf(v.FieldRules)) + } + return v.rules +} + +func flattenRules(prefix string, vv reflect.Value) (rules []ValidateRule) { + vv = reflect.Indirect(vv) + vt := vv.Type() + switch vt.Kind() { + case reflect.Struct: + nextField: + for i := 0; i < vt.NumField(); i++ { + f := vt.Field(i) + ft := f.Type + fv := vv.Field(i) + + for ft.Kind() == reflect.Interface || ft.Kind() == reflect.Ptr { + if fv.IsNil() { + continue nextField + } + fv = fv.Elem() + ft = fv.Type() + } + name := prefix + if tag, ok := f.Tag.Lookup("protobuf"); ok { + for _, opt := range strings.Split(tag, ",") { + if strings.HasPrefix(opt, "name=") { + if name != "" && !strings.HasSuffix(name, ".") { + name += "." + } + name += strings.TrimPrefix(opt, "name=") + break + } + } + } else if _, ok := f.Tag.Lookup("protobuf_oneof"); !ok { + continue nextField + } + rules = append(rules, flattenRules(name, fv)...) + } + case reflect.Slice: + if vv.Len() == 0 { + return nil + } + fallthrough + default: + rules = append(rules, ValidateRule{Name: prefix, Value: vv.Interface()}) + } + return rules +} + +func init() { + extensions.SetTransformer("validate.rules", func(payload interface{}) interface{} { + rules, ok := payload.(*validate.FieldRules) + if !ok { + return nil + } + return ValidateExtension{FieldRules: rules} + }) +} diff --git a/github.com/pseudomuto/protoc-gen-doc/extensions/extensions.go b/github.com/pseudomuto/protoc-gen-doc/extensions/extensions.go new file mode 100644 index 0000000000..5e48c72a26 --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/extensions/extensions.go @@ -0,0 +1,35 @@ +// Package extensions implements a system for working with extended options. +package extensions + +// Transformer functions for transforming payloads of an extension option into +// something that can be rendered by a template. +type Transformer func(payload interface{}) interface{} + +var transformers = make(map[string]Transformer) + +// SetTransformer sets the transformer function for the given extension name +func SetTransformer(extensionName string, f Transformer) { + transformers[extensionName] = f +} + +// Transform the extensions using the registered transformers. +func Transform(extensions map[string]interface{}) map[string]interface{} { + if extensions == nil { + return nil + } + out := make(map[string]interface{}, len(extensions)) + for name, payload := range extensions { + transform, ok := transformers[name] + if !ok { + // No transformer registered, skip. + continue + } + transformedPayload := transform(payload) + if transformedPayload == nil { + // Transformer returned nothing, skip. + continue + } + out[name] = transformedPayload + } + return out +} diff --git a/github.com/pseudomuto/protoc-gen-doc/extensions/google_api_http/google_api_http.go b/github.com/pseudomuto/protoc-gen-doc/extensions/google_api_http/google_api_http.go new file mode 100644 index 0000000000..cbc5ad14cf --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/extensions/google_api_http/google_api_http.go @@ -0,0 +1,65 @@ +package extensions + +import ( + "net/http" + + "github.com/pseudomuto/protoc-gen-doc/extensions" + "google.golang.org/genproto/googleapis/api/annotations" +) + +// HTTPRule represents a single HTTP rule from the (google.api.http) method option extension. +type HTTPRule struct { + Method string `json:"method"` + Pattern string `json:"pattern"` + Body string `json:"body,omitempty"` +} + +// HTTPExtension contains the rules set by the (google.api.http) method option extension. +type HTTPExtension struct { + Rules []HTTPRule `json:"rules"` +} + +func getRule(r *annotations.HttpRule) (rule HTTPRule) { + switch r.GetPattern().(type) { + case *annotations.HttpRule_Get: + rule.Method = http.MethodGet + rule.Pattern = r.GetGet() + case *annotations.HttpRule_Put: + rule.Method = http.MethodPut + rule.Pattern = r.GetPut() + case *annotations.HttpRule_Post: + rule.Method = http.MethodPost + rule.Pattern = r.GetPost() + case *annotations.HttpRule_Delete: + rule.Method = http.MethodDelete + rule.Pattern = r.GetDelete() + case *annotations.HttpRule_Patch: + rule.Method = http.MethodPatch + rule.Pattern = r.GetPatch() + case *annotations.HttpRule_Custom: + custom := r.GetCustom() + rule.Method = custom.GetKind() + rule.Pattern = custom.GetPath() + } + rule.Body = r.GetBody() + return +} + +func init() { + extensions.SetTransformer("google.api.http", func(payload interface{}) interface{} { + var rules []HTTPRule + rule, ok := payload.(*annotations.HttpRule) + if !ok { + return nil + } + + rules = append(rules, getRule(rule)) + + // NOTE: The option can only have one level of nested AdditionalBindings. + for _, rule := range rule.AdditionalBindings { + rules = append(rules, getRule(rule)) + } + + return HTTPExtension{Rules: rules} + }) +} diff --git a/github.com/pseudomuto/protoc-gen-doc/extensions/lyft_validate/alias.go b/github.com/pseudomuto/protoc-gen-doc/extensions/lyft_validate/alias.go new file mode 100644 index 0000000000..985a0cf965 --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/extensions/lyft_validate/alias.go @@ -0,0 +1,9 @@ +package extensions + +import evp "github.com/pseudomuto/protoc-gen-doc/extensions/envoyproxy_validate" + +// ValidateRule represents a single validator rule from the (validate.rules) method option extension. +type ValidateRule = evp.ValidateRule + +// ValidateExtension contains the rules set by the (validate.rules) method option extension. +type ValidateExtension = evp.ValidateExtension diff --git a/github.com/pseudomuto/protoc-gen-doc/extensions/validator_field/validator_field.go b/github.com/pseudomuto/protoc-gen-doc/extensions/validator_field/validator_field.go new file mode 100644 index 0000000000..224e414d58 --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/extensions/validator_field/validator_field.go @@ -0,0 +1,83 @@ +package extensions + +import ( + "encoding/json" + "reflect" + "strings" + + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/protoc-gen-go/descriptor" + validator "github.com/mwitkow/go-proto-validators" + "github.com/pseudomuto/protoc-gen-doc/extensions" +) + +func init() { + // NOTE: mwitkow/go-proto-validators uses gogo/profobuf/proto and therefore + // only registers the extension under gogo. We need to register it under + // golang/protobuf/proto with the same properties, except using the + // golang/protobuf FieldOptions descriptor. + proto.RegisterExtension(&proto.ExtensionDesc{ + ExtendedType: (*descriptor.FieldOptions)(nil), + ExtensionType: validator.E_Field.ExtensionType, + Field: validator.E_Field.Field, + Name: validator.E_Field.Name, + Tag: validator.E_Field.Tag, + Filename: validator.E_Field.Filename, + }) +} + +// ValidatorRule represents a single validator rule from the (validator.field) method option extension. +type ValidatorRule struct { + Name string `json:"name"` + Value interface{} `json:"value"` +} + +// ValidatorExtension contains the rules set by the (validator.field) method option extension. +type ValidatorExtension struct { + *validator.FieldValidator + rules []ValidatorRule // memoized so that we don't have to use reflection more than we need. +} + +// MarshalJSON implements the json.Marshaler interface. +func (v ValidatorExtension) MarshalJSON() ([]byte, error) { return json.Marshal(v.Rules()) } + +// Rules returns all active rules +func (v ValidatorExtension) Rules() []ValidatorRule { + if v.FieldValidator == nil { + return nil + } + if v.rules != nil { + return v.rules + } + vv := reflect.ValueOf(*v.FieldValidator) + vt := vv.Type() + for i := 0; i < vt.NumField(); i++ { + tag, ok := vt.Field(i).Tag.Lookup("protobuf") + if !ok { + continue + } + for _, opt := range strings.Split(tag, ",") { + if strings.HasPrefix(opt, "name=") { + tag = strings.TrimPrefix(opt, "name=") + break + } + } + value := vv.Field(i) + if value.IsNil() { + continue + } + value = reflect.Indirect(value) + v.rules = append(v.rules, ValidatorRule{Name: tag, Value: value.Interface()}) + } + return v.rules +} + +func init() { + extensions.SetTransformer("validator.field", func(payload interface{}) interface{} { + validator, ok := payload.(*validator.FieldValidator) + if !ok { + return nil + } + return ValidatorExtension{FieldValidator: validator} + }) +} diff --git a/github.com/pseudomuto/protoc-gen-doc/filters.go b/github.com/pseudomuto/protoc-gen-doc/filters.go new file mode 100644 index 0000000000..21fc93a4b0 --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/filters.go @@ -0,0 +1,38 @@ +package gendoc + +import ( + "fmt" + "html/template" + "regexp" + "strings" +) + +var ( + paraPattern = regexp.MustCompile(`(\n|\r|\r\n)\s*`) + spacePattern = regexp.MustCompile("( )+") + multiNewlinePattern = regexp.MustCompile(`(\r\n|\r|\n){2,}`) +) + +// PFilter splits the content by new lines and wraps each one in a

tag. +func PFilter(content string) template.HTML { + paragraphs := paraPattern.Split(content, -1) + return template.HTML(fmt.Sprintf("

%s

", strings.Join(paragraphs, "

"))) +} + +// ParaFilter splits the content by new lines and wraps each one in a tag. +func ParaFilter(content string) string { + paragraphs := paraPattern.Split(content, -1) + return fmt.Sprintf("%s", strings.Join(paragraphs, "")) +} + +// NoBrFilter removes single CR and LF from content. +func NoBrFilter(content string) string { + normalized := strings.Replace(content, "\r\n", "\n", -1) + paragraphs := multiNewlinePattern.Split(normalized, -1) + for i, p := range paragraphs { + withoutCR := strings.Replace(p, "\r", " ", -1) + withoutLF := strings.Replace(withoutCR, "\n", " ", -1) + paragraphs[i] = spacePattern.ReplaceAllString(withoutLF, " ") + } + return strings.Join(paragraphs, "\n\n") +} diff --git a/github.com/pseudomuto/protoc-gen-doc/fixtures/Booking.proto b/github.com/pseudomuto/protoc-gen-doc/fixtures/Booking.proto new file mode 100644 index 0000000000..1ea32a4f29 --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/fixtures/Booking.proto @@ -0,0 +1,84 @@ +/** + * Booking related messages. + * + * This file is really just an example. The data model is completely + * fictional. + */ +syntax = "proto2"; + +import "github.com/pseudomuto/protokit/fixtures/extend.proto"; + +package com.example; + +option (com.pseudomuto.protokit.v1.extend_file) = true; + +/** + * Service for handling vehicle bookings. + */ +service BookingService { + option (com.pseudomuto.protokit.v1.extend_service) = true; + + /// Used to book a vehicle. Pass in a Booking and a BookingStatus will be returned. + rpc BookVehicle (Booking) returns (BookingStatus) { + option (com.pseudomuto.protokit.v1.extend_method) = true; + }; +} + +/** + * Represents the status of a vehicle booking. + */ +message BookingStatus { + /** + * A flag for the status result. + */ + enum StatusCode { + OK = 200; // OK result. + BAD_REQUEST = 400; // BAD result. + } + + required int32 id = 1 [(com.pseudomuto.protokit.v1.extend_field) = true]; /// Unique booking status ID. + required string description = 2; /// Booking status description. E.g. "Active". + optional StatusCode status_code = 3; /// The status of this status? + + extensions 100 to max; +} + +extend BookingStatus { + /** The country the booking occurred in. */ + optional string country = 100 [default = "china"]; +} + +/** + * The type of booking. + */ +enum BookingType { + option (com.pseudomuto.protokit.v1.extend_enum) = true; + + IMMEDIATE = 100; // Immediate booking. + FUTURE = 101 [(com.pseudomuto.protokit.v1.extend_enum_value) = true]; // Future booking. +} + +/** + * Represents the booking of a vehicle. + * + * Vehicles are some cool shit. But drive carefully! + */ +message Booking { + option (com.pseudomuto.protokit.v1.extend_message) = true; + + required int32 vehicle_id = 1; /// ID of booked vehicle. + required int32 customer_id = 2; /// Customer that booked the vehicle. + required BookingStatus status = 3; /// Status of the booking. + + /** Has booking confirmation been sent? */ + required bool confirmation_sent = 4; + + /** Has payment been received? */ + optional bool payment_received = 5 [default = false, (com.pseudomuto.protokit.v1.extend_field) = true]; + + // Nested extentions are also a thing. + + extend BookingStatus { + optional string optional_field_1 = 101; // An optional field to be used however you please. + } +} diff --git a/github.com/pseudomuto/protoc-gen-doc/fixtures/Vehicle.proto b/github.com/pseudomuto/protoc-gen-doc/fixtures/Vehicle.proto new file mode 100644 index 0000000000..561599da40 --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/fixtures/Vehicle.proto @@ -0,0 +1,147 @@ +/** + * Messages describing manufacturers / vehicles. + */ +syntax = "proto3"; + +import "github.com/pseudomuto/protokit/fixtures/extend.proto"; + +package com.example; + +option (com.pseudomuto.protokit.v1.extend_file) = true; + +/** + * The vehicle service. + * + * Manages vehicles and such... + */ +service VehicleService { + option (com.pseudomuto.protokit.v1.extend_service) = true; + + // Returns the set of models. + rpc GetModels(EmptyMessage) returns (stream Model); + + rpc AddModels(stream Model) returns (stream Model); // creates models + + /** + * Looks up a vehicle by id. + */ + rpc GetVehicle(FindVehicleById) returns (Vehicle) { + option (com.pseudomuto.protokit.v1.extend_method) = true; + }; +} + +/** + * A request message for finding vehicles. + */ +message FindVehicleById { + int32 id = 1; // The id of the vehicle to find. +} + +/** + * Represents a vehicle model. + */ +message Model { + string id = 1; // The unique model ID. + string model_code = 2; // The car model code, e.g. "PZ003". + string model_name = 3; // The car model name, e.g. "Z3". + + sint32 daily_hire_rate_dollars = 4; // Dollars per day. + sint32 daily_hire_rate_cents = 5; // Cents per day. + + Type type = 6; // The type of this model +} + +// An empty message. +message EmptyMessage { +} + +/** + * @exclude + * This comment won't be rendered + */ +message ExcludedMessage { + string id = 1; // the id of this message. + string name = 2; // @exclude the name of this message + + /* @exclude the value of this message. */ + int32 value = 3; +} + +// The type of model. +enum Type { + option (com.pseudomuto.protokit.v1.extend_enum) = true; + + COUPE = 0; // The type is coupe. + SEDAN = 1 [(com.pseudomuto.protokit.v1.extend_enum_value) = true]; // The type is sedan. +} + +/** + * Represents a manufacturer of cars. + */ +message Manufacturer { + /** + * Manufacturer category. A manufacturer may be either inhouse or external. + */ + enum Category { + CATEGORY_INHOUSE = 0; // The manufacturer is inhouse. + CATEGORY_EXTERNAL = 1; // The manufacturer is external. + } + + int32 id = 1; /** The unique manufacturer ID. */ + string code = 2; // A manufacturer code, e.g. "DKL4P". + string details = 3; // Manufacturer details (minimum orders etc.). + + /** Manufacturer category. */ + Category category = 4; +} + +/** + * Represents a vehicle that can be hired. + */ +message Vehicle { + option (com.pseudomuto.protokit.v1.extend_message) = true; + + /** + * Represents a vehicle category. E.g. "Sedan" or "Truck". + */ + message Category { + string code = 1; /// Category code. E.g. "S". + string description = 2; /// Category name. E.g. "Sedan". + } + + message Engine { + enum FuelType { + FUEL_TYPE_UNSPECIFIED = 0; + PETROL = 1; + DIESEL = 2; + ELECTRIC = 3; + } + message Stats { + sint32 mpg = 1; + sint32 bhp = 2; + double zero_to_sixty_secs = 3; + } + FuelType fuel_type = 1; + sint32 size_cc = 2; /** Size in cubic centimetres, if applicable. */ + Stats stats = 3; + } + + int32 id = 1; /** Unique vehicle ID. */ + Model model = 2; /** Vehicle model. */ + string reg_number = 3 [(com.pseudomuto.protokit.v1.extend_field) = true]; /** Vehicle registration number. */ + sint32 mileage = 4; /** Current vehicle mileage, if known. */ + Category category = 5; /** Vehicle category. */ + Engine engine = 9; /** Vehicle engine. */ + + + // Doc comments for fields can come before or + // after the field definition. And just like + // comments for messages / enums, they can be + // multi-paragraph: + + + // rates + repeated sint32 rates = 6; + + map properties = 7; // bag of properties related to the vehicle. +} diff --git a/github.com/pseudomuto/protoc-gen-doc/go.mod b/github.com/pseudomuto/protoc-gen-doc/go.mod new file mode 100644 index 0000000000..985fb13c51 --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/go.mod @@ -0,0 +1,22 @@ +module github.com/pseudomuto/protoc-gen-doc + +require ( + github.com/Masterminds/semver v1.4.2 // indirect + github.com/Masterminds/sprig v2.15.0+incompatible + github.com/aokoli/goutils v1.0.1 // indirect + github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b // indirect + github.com/envoyproxy/protoc-gen-validate v0.0.14 + github.com/gogo/protobuf v1.1.1 // indirect + github.com/golang/protobuf v1.1.0 + github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c // indirect + github.com/huandu/xstrings v1.0.0 // indirect + github.com/imdario/mergo v0.3.4 // indirect + github.com/mwitkow/go-proto-validators v0.0.0-20180403085117-0950a7990007 + github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0 // indirect + github.com/pseudomuto/protokit v0.2.0 + github.com/stretchr/testify v0.0.0-20170130113145-4d4bfba8f1d1 + golang.org/x/crypto v0.0.0-20180501155221-613d6eafa307 // indirect + golang.org/x/sync v0.0.0-20190412183630-56d357773e84 // indirect + google.golang.org/genproto v0.0.0-20181107211654-5fc9ac540362 + gopkg.in/yaml.v2 v2.2.2 // indirect +) diff --git a/github.com/pseudomuto/protoc-gen-doc/go.sum b/github.com/pseudomuto/protoc-gen-doc/go.sum new file mode 100644 index 0000000000..89b4598039 --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/go.sum @@ -0,0 +1,38 @@ +github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc= +github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/sprig v2.15.0+incompatible h1:0gSxPGWS9PAr7U2NsQ2YQg6juRDINkUyuvbb4b2Xm8w= +github.com/Masterminds/sprig v2.15.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= +github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg= +github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ= +github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b h1:XxMZvQZtTXpWMNWK82vdjCLCe7uGMFXdTsJH0v3Hkvw= +github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/protoc-gen-validate v0.0.14 h1:YBW6/cKy9prEGRYLnaGa4IDhzxZhRCtKsax8srGKDnM= +github.com/envoyproxy/protoc-gen-validate v0.0.14/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/protobuf v1.1.0 h1:0iH4Ffd/meGoXqF2lSAhZHt8X+cPgkfn/cb6Cce5Vpc= +github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c h1:jWtZjFEUE/Bz0IeIhqCnyZ3HG6KRXSntXe4SjtuTH7c= +github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk= +github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= +github.com/imdario/mergo v0.3.4 h1:mKkfHkZWD8dC7WxKx3N9WCF0Y+dLau45704YQmY6H94= +github.com/imdario/mergo v0.3.4/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/mwitkow/go-proto-validators v0.0.0-20180403085117-0950a7990007 h1:28i1IjGcx8AofiB4N3q5Yls55VEaitzuEPkFJEVgGkA= +github.com/mwitkow/go-proto-validators v0.0.0-20180403085117-0950a7990007/go.mod h1:m2XC9Qq0AlmmVksL6FktJCdTYyLk7V3fKyp0sl1yWQo= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0 h1:GD+A8+e+wFkqje55/2fOVnZPkoDIu1VooBWfNrnY8Uo= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pseudomuto/protokit v0.2.0 h1:hlnBDcy3YEDXH7kc9gV+NLaN0cDzhDvD1s7Y6FZ8RpM= +github.com/pseudomuto/protokit v0.2.0/go.mod h1:2PdH30hxVHsup8KpBTOXTBeMVhJZVio3Q8ViKSAXT0Q= +github.com/stretchr/testify v0.0.0-20170130113145-4d4bfba8f1d1 h1:Zx8Rp9ozC4FPFxfEKRSUu8+Ay3sZxEUZ7JrCWMbGgvE= +github.com/stretchr/testify v0.0.0-20170130113145-4d4bfba8f1d1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +golang.org/x/crypto v0.0.0-20180501155221-613d6eafa307 h1:O5C+XK++apFo5B+Vq4ujc/LkLwHxg9fDdgjgoIikBdA= +golang.org/x/crypto v0.0.0-20180501155221-613d6eafa307/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84 h1:IqXQ59gzdXv58Jmm2xn0tSOR9i6HqroaOFRQ3wR/dJQ= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +google.golang.org/genproto v0.0.0-20181107211654-5fc9ac540362 h1:b69RmkJsx8NyRJsKF2mQ/AF8s4BNxwNsT4rQ3wON1U0= +google.golang.org/genproto v0.0.0-20181107211654-5fc9ac540362/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/github.com/pseudomuto/protoc-gen-doc/plugin.go b/github.com/pseudomuto/protoc-gen-doc/plugin.go new file mode 100644 index 0000000000..d37a57268e --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/plugin.go @@ -0,0 +1,129 @@ +package gendoc + +import ( + "fmt" + "io/ioutil" + "path" + "regexp" + "strings" + + "github.com/golang/protobuf/proto" + plugin_go "github.com/golang/protobuf/protoc-gen-go/plugin" + "github.com/pseudomuto/protokit" +) + +// PluginOptions encapsulates options for the plugin. The type of renderer, template file, and the name of the output +// file are included. +type PluginOptions struct { + Type RenderType + TemplateFile string + OutputFile string + ExcludePatterns []*regexp.Regexp +} + +// Plugin describes a protoc code generate plugin. It's an implementation of Plugin from github.com/pseudomuto/protokit +type Plugin struct{} + +// Generate compiles the documentation and generates the CodeGeneratorResponse to send back to protoc. It does this +// by rendering a template based on the options parsed from the CodeGeneratorRequest. +func (p *Plugin) Generate(r *plugin_go.CodeGeneratorRequest) (*plugin_go.CodeGeneratorResponse, error) { + options, err := ParseOptions(r) + if err != nil { + return nil, err + } + + result := excludeUnwantedProtos(protokit.ParseCodeGenRequest(r), options.ExcludePatterns) + template := NewTemplate(result) + + customTemplate := "" + + if options.TemplateFile != "" { + data, err := ioutil.ReadFile(options.TemplateFile) + if err != nil { + return nil, err + } + + customTemplate = string(data) + } + + output, err := RenderTemplate(options.Type, template, customTemplate) + if err != nil { + return nil, err + } + + resp := new(plugin_go.CodeGeneratorResponse) + resp.File = append(resp.File, &plugin_go.CodeGeneratorResponse_File{ + Name: proto.String(options.OutputFile), + Content: proto.String(string(output)), + }) + + return resp, nil +} + +func excludeUnwantedProtos(fds []*protokit.FileDescriptor, excludePatterns []*regexp.Regexp) []*protokit.FileDescriptor { + descs := make([]*protokit.FileDescriptor, 0) + +OUTER: + for _, d := range fds { + for _, p := range excludePatterns { + if p.MatchString(d.GetName()) { + continue OUTER + } + } + + descs = append(descs, d) + } + + return descs +} + +// ParseOptions parses plugin options from a CodeGeneratorRequest. It does this by splitting the `Parameter` field from +// the request object and parsing out the type of renderer to use and the name of the file to be generated. +// +// The parameter (`--doc_opt`) must be of the format ,:,*. +// The file will be written to the directory specified with the `--doc_out` argument to protoc. +func ParseOptions(req *plugin_go.CodeGeneratorRequest) (*PluginOptions, error) { + options := &PluginOptions{ + Type: RenderTypeHTML, + TemplateFile: "", + OutputFile: "index.html", + } + + params := req.GetParameter() + if strings.Contains(params, ":") { + // Parse out exclude patterns if any + parts := strings.Split(params, ":") + for _, pattern := range strings.Split(parts[1], ",") { + r, err := regexp.Compile(pattern) + if err != nil { + return nil, err + } + options.ExcludePatterns = append(options.ExcludePatterns, r) + } + // The first part is parsed below + params = parts[0] + } + if params == "" { + return options, nil + } + + if !strings.Contains(params, ",") { + return nil, fmt.Errorf("Invalid parameter: %s", params) + } + + parts := strings.Split(params, ",") + if len(parts) != 2 { + return nil, fmt.Errorf("Invalid parameter: %s", params) + } + + options.TemplateFile = parts[0] + options.OutputFile = path.Base(parts[1]) + + renderType, err := NewRenderType(options.TemplateFile) + if err == nil { + options.Type = renderType + options.TemplateFile = "" + } + + return options, nil +} diff --git a/github.com/pseudomuto/protoc-gen-doc/renderer.go b/github.com/pseudomuto/protoc-gen-doc/renderer.go new file mode 100644 index 0000000000..950f48d2a2 --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/renderer.go @@ -0,0 +1,150 @@ +package gendoc + +import ( + "bytes" + "encoding/json" + "errors" + html_template "html/template" + text_template "text/template" + + "github.com/Masterminds/sprig" +) + +// RenderType is an "enum" for which type of renderer to use. +type RenderType int8 + +// Available render types. +const ( + _ RenderType = iota + RenderTypeDocBook + RenderTypeHTML + RenderTypeJSON + RenderTypeMarkdown +) + +// NewRenderType creates a RenderType from the supplied string. If the type is not known, (0, error) is returned. It is +// assumed (by the plugin) that invalid render type simply means that the path to a custom template was supplied. +func NewRenderType(renderType string) (RenderType, error) { + switch renderType { + case "docbook": + return RenderTypeDocBook, nil + case "html": + return RenderTypeHTML, nil + case "json": + return RenderTypeJSON, nil + case "markdown": + return RenderTypeMarkdown, nil + } + + return 0, errors.New("Invalid render type") +} + +func (rt RenderType) renderer() (Processor, error) { + tmpl, err := rt.template() + if err != nil { + return nil, err + } + + switch rt { + case RenderTypeDocBook: + return &textRenderer{string(tmpl)}, nil + case RenderTypeHTML: + return &htmlRenderer{string(tmpl)}, nil + case RenderTypeJSON: + return new(jsonRenderer), nil + case RenderTypeMarkdown: + return &htmlRenderer{string(tmpl)}, nil + } + + return nil, errors.New("Unable to create a processor") +} + +func (rt RenderType) template() ([]byte, error) { + switch rt { + case RenderTypeDocBook: + return fetchResource("docbook.tmpl") + case RenderTypeHTML: + return fetchResource("html.tmpl") + case RenderTypeJSON: + return nil, nil + case RenderTypeMarkdown: + return fetchResource("markdown.tmpl") + } + + return nil, errors.New("Couldn't find template for render type") +} + +var funcMap = map[string]interface{}{ + "p": PFilter, + "para": ParaFilter, + "nobr": NoBrFilter, +} + +// Processor is an interface that is satisfied by all built-in processors (text, html, and json). +type Processor interface { + Apply(template *Template) ([]byte, error) +} + +// RenderTemplate renders the template based on the render type. It supports overriding the default input templates by +// supplying a non-empty string as the last parameter. +// +// Example: generating an HTML template (assuming you've got a Template object) +// data, err := RenderTemplate(RenderTypeHTML, &template, "") +// +// Example: generating a custom template (assuming you've got a Template object) +// data, err := RenderTemplate(RenderTypeHTML, &template, "{{range .Files}}{{.Name}}{{end}}") +func RenderTemplate(kind RenderType, template *Template, inputTemplate string) ([]byte, error) { + if inputTemplate != "" { + processor := &textRenderer{inputTemplate} + return processor.Apply(template) + } + + processor, err := kind.renderer() + if err != nil { + return nil, err + } + + return processor.Apply(template) +} + +type textRenderer struct { + inputTemplate string +} + +func (mr *textRenderer) Apply(template *Template) ([]byte, error) { + tmpl, err := text_template.New("Text Template").Funcs(funcMap).Funcs(sprig.TxtFuncMap()).Parse(mr.inputTemplate) + if err != nil { + return nil, err + } + + var buf bytes.Buffer + if err = tmpl.Execute(&buf, template); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +type htmlRenderer struct { + inputTemplate string +} + +func (mr *htmlRenderer) Apply(template *Template) ([]byte, error) { + tmpl, err := html_template.New("Text Template").Funcs(funcMap).Funcs(sprig.HtmlFuncMap()).Parse(mr.inputTemplate) + if err != nil { + return nil, err + } + + var buf bytes.Buffer + if err = tmpl.Execute(&buf, template); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +type jsonRenderer struct{} + +func (r *jsonRenderer) Apply(template *Template) ([]byte, error) { + return json.MarshalIndent(template, "", " ") +} diff --git a/github.com/pseudomuto/protoc-gen-doc/resources.go b/github.com/pseudomuto/protoc-gen-doc/resources.go new file mode 100644 index 0000000000..73ab609ec9 --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/resources.go @@ -0,0 +1,44 @@ +// AUTOGENERATED CODE. DO NOT EDIT. + +package gendoc + +import ( + "bytes" + "compress/gzip" + "encoding/base64" + "fmt" + "io" +) + +var embeddedResources = map[string]string{ + "docbook.tmpl": "H4sIAAAAAAAA/+xZ70/zNhD+3r/Cyr7t1Zu8E0OaJrdIKytoAoQo23c3ubbWHDuznUIV9X+fHIf8/tHRUtjGF0R8T+7su3uec1t88RwytAGpqOBj5wf3m4OA+yKgfDV2fn+cff3JuZiMMJGa+gwmI4SwpprB5F4KLXzB0KXw4xC4JpoKjj1rHSGUJJLwFSB3Rhmo3c68qsA3KGPOHSWJe0dC2O1K75q3IyIJci9B+ZJG5q3URcnvLShFVpnrwjmiwdhJEncWM2YdO9ZlOeKN4KuWqH1xjY0ukXtN1IwCC1S+jjVZMEBLSUIYO4SxPGAeEvuMKMVJ2IheGJB1W9uQcbGSIo6QL5gaOz+WnCOEzWIEvjE+0UCvx873jncw4pt7Pgw6q0P0GkhQXkEIS/FUXUEIA9dyO0lPiz370A553EbQj7ghC2D9kFIlW4HYq+0Re42DYL0QQe29Un9XumHw4KWG79s3ZpT/icwf4EVHm5SYjs66yD5iz8Am/f7MGyZbQ3FbGWBb/xKWJGb6D8JiE9XgJtnazyhJ6nYvBSQJ8KAjaCP3Jq0pvFqPavaxZxmR09pLCVhQuOwhJ+2vzxq4kbnjE/cOlIYAFREGOHx+Cg5/DJbnOTmU6b8QNYC4i8MFyHcWg5YuG8zROwlC099UcE0op3xV81wY/rno2LL8r1QHe5WLTtlUNAqPw9NdXY6ldGmSh+Tt7BTydqAumbP9C+TE5vvoUnIYLU9Fqg4aZQ9dY736ISNvb/Mp5CuDDbDuOY0pXwoZEtbLls9J/jnJPyf5f2qSV3i/j/hkPTIHuaH+676CePsZ3jK/b0Gvxcf4juHNBcueFQ1P+gf4Kwal0bB0PYCKBFewB/TN9Skr5QnEKctPTUmy1YaMWKpn1rmWQELKV7sdUun/fXTu3YPNfGMTdrlzF9b8ym18xNtPzVDSpdZvV+c+YUTae3vatVXuD994uu87Q5xusZ8PAY5rbxKqUaWs0m4khRbdtH65mAhtEtgNmH750mf+jWxIn/1+q9ddemHXrkRv+O96vV/f95kf4sW2xV5r7YZMNUWqGI9p81UlqqsELzMz/V2hxPDSc9/mjbiZ4gyiplG0lzdTqr2AtmZ7Qa/2O8h0viYyGg683u8kpq6dwIZw1WWrJlpVyWq5OZXUaYS9/HejvwMAAP//LC5x+GkaAAA=", + "html.tmpl": "H4sIAAAAAAAA/9xaX2/bOBJ/z6eYVbvIbltZjpO0PVfxAZu2Wxy2bdCke3tPB1qiLaI0qRWptDnD3/1AUn8oiZKdxOkeDn2oRI5mhjO/Gf5IJ/zh9cfzq39dvIFErujs4CA0/wOECUaxegAIJZEUzy4yLnnEKbzmUb7CTCJJOAsDM2skV1giiBKUCSzPvM9Xb/2XXjFFCfsCGaZnnpA3FIsEY+mBvEnxmSfxNxlEQniQZHhx5iVSpmIaBAvOpBgtOV9SjFIiRhFfKbm/L9CK0Juzz/OcyXx6Mh4/ezEePzsZj4lElEReUBjVpswzwJzHN7AuXgC+klgmU3g+xqtX1eAKZUvCpnCEV4ByyeuZiFOeTeHRZDKpB5WDvnFmCp5xx3sGAjHhC5yRRS2aojgmbOnPuZR8NYWT2uzmoHhIjiz/tO6vmCwTOQXGsxWitbY5z2KcVcqO0m8gOCUxPEII9Rsdj07xt67ZiWV2H5qtOI5O8QrGXZPHf8lKkWVVgc6PccQzDWRlmeFuvk+fv8CT044mieYUd9F0NB7/2IKHIP/BU3hpjxdrijilKBV4CuVT14wqw75QvRiPLZ0o+rLMeM5iv3Q9jtS/rk5dCDKbMpn4UUJo/BO+xuxnGwRdZYu5+tdVFnew00hSFEWdJBXZgYkjQzKGtJ0kwmLMpC7KLsK62FIqrLUd/dynb/wKgifwgYMZAM5gQTIhIQXClJonQVt38ASudOb5AhYE01jUQiM94BtkyLjlgvr0rRKoP7BQYzeDbdomhbarmxTfW9lxoew3NMfUoe35bZSdFMpeYxFlJFVl5VBp91VnYPE3iZkgnNnBrQaHAvymFNo1LoNa7xLoQYVlsH9BYj8Ky4B/yFdznDlUnt5W4+meUsjyFVwjmmMxspPI8tVQ/j6g1e6B6dE12RaTW2k73k88RIQoykxENOlphMXM+nrW17OlK5nVu5Ki7R87mINtK+JMYkWcaguPJI98NY4Iwxnk1FJLiZC+JkradHsfLDdWihftFkwJw37p1VFjh3N059oTmAElMGvsxo2Nbc5p7FriW0IxqB2RsCXE5LrRe6nyxUxt2ZZjIlKKbqZmE7811SjXdqKYTZfhuBxyMKx2nJtO+RGmdFhnh8sgSpZsCpmK4Y56LfQkGA7fHz6DwzeHgFgMh38cwhzFSyz0ZphguOLnVsD1nCPSo+c2RCp0NIcrpwjTIJpTHn15ddCDrOa39lojzCTOXm1HUYOLPVdg6BC9l3+bo5OXw4RqsRhHL61vK5hrPqMODebJb9SJgxY12VQFvQzFJBeqzL41kx8GxVHGvP3g+/BZ4AyiXEi+gvPLS/D9O5y0aomRGtXnpjAwZz/1qKhiaTQ5AhKfefq85/UeB5OjSn4yq3rSedGTwiCZlPOqgLVCuzd55WktzGk5W40BrNcZYksMI9UKxGZTTaipx6o+/s3UHjI9g5HaTBoSISUz6xUgREUYHq3Xhbg3qx7DALXEc9ocsPx5j4VAy5ZLPWYdxt/mlJYOhCJFDCKKhDjzdJl5s/dhoEaVc79xtuxx0CCla269xizueFb5/oblq4dy/M2DOl4Rxbt5XwNms/Fr1uleyR/FShTyfIqvMa3pptjXii5xdk2iB4PRZZ2NPWQiDJoF0fyu/YXyv3a2S3m82aUhSb9rkqRItw6rrbW2GAYxuS46SU9TGG4Iuv0U0bH3VavZhMlEtyB3c0gm1nKKpnjFUyuihY+lNymMLBa5qXbfoR4SJselC3ZuW9WUHNth77Oj5sgCRu+Q0AfRJshCwziriFRHPK/VBGV9M2iPZrNQxjOtOAxkrN9UDqsXfcKs3iwPzVggs5ahwGEplGZHaoOzAkBnXbV/ruKRsZ1S2VlXKdSpMrU0KxPm1cC1X4sSVlHYYitVgo0UFpl7jRcop1IXyGZTvE1BS9szRemFQdrjTjfafRXeiXcYaFTMDppb78pgV9dZa1v2wc7OR72kdo7W68fcHK46CgrY4j9hBN41oiRGkmfmEsKrRvAoyykWHrRXkJzMfi9EYnMBo0jISRtWZk3tZLigPgimGv89AoUvZs/YNTXOUthWDGVKiqL4J5GJiX0nvlvWtFOBOIadhKnp40+jwqUi+z+PPuVtXtdQSEntjgZ+gXjXllWacm24hbrA7eD9isZZNlbhtL6nAjswa5IGX4lM1DI3G+BFs3ww7KqoDqX4Y92t/19Qq1stpBlhcgHej0+vvS4k99FHbwmJ1vd6BHxrrJTp7uw9lDicN/xsbvatC8FbbfiVPfem/wsS9Yu5kXtgCjB8JvifoAENLefm5EvYsqWvnrgVwTBBvjXDcBEM+IsZhvu7FvZ7T7F7oc511BoFU10We82qcqHV1EnVXO9QCI4ycBVBFQmdoy78neDfAfo7AasHVn0A6cKjC44ONFrA6ABhqAXWaOg99Pcc7G2E7N43h7DwgD3ztlAZ6Jb3gct9++RDdcn7QHm/HfJBCqD/Eqm/GX7vRvgey4TH0OiHn/CfORYSGmXwCYuUM4Gbo/suAOPOA6K/WFsLtsVoE7MGYcXUpcwwWhG23GxA6OcKUTvaNeHrGDbDbstm7i6mv1frtzD6WBict68MrBsHk13XlcPAhYN13WD+9GyEUjJKpEy9ppPJSYHm4pz27urqAuaExYQtO5cMrmNaP7V2BrldOgNC/fMXSEqc9R3j1PbD45vd8rYzPa8Od0XGyqobPN2t14/7f8qBO1wiDJT0Y7ad/xiftwgV0d0ipULsFtmVNt+VSjtvHDpAHrpw+F447r9t+N5AvNdWcK8Lhl1a5S3y3viyfalgzzeIRfkryK6/2CQTY7bJFPr+xKX+vbeVwpIvjNKMS94kAR+4xKJ6O3/6tHr+B7pG1cvFjUwskv0rrz95VAu9u6gpRz6/6bCKFrjasKpZl15h+0eVrKRd+vfyciM+cEDJEuhioYSbWvjA/HmabtGgArRFxIRti9Cv21w9v0xQlg6ZSbb5qtLhFmkWRhPajXKwCiEMzHAYFH/r/t8AAAD//y1zzmD9LgAA", + "markdown.tmpl": "H4sIAAAAAAAA/+RWTU/bTBC++1fMG7+HIuRwR0kOhQKqACGCekFV2SSTxJKz63rXUZF3/3u1H/au7RhSCfVSDtgzY8/H8zw7cQwPBRNsyTK4ZMtyh1QQkTIaTQhQssPpSLB8NJuckVkUxTE8kUWGwNZwwahAKnhUVQWhG4TxVZohVyqqqv/XaYY/9OtwPoXxPdmhUgk8V5W7//4pbu5PIoCqSiBdw/gOOScb5KCU8brMtVspAJvmltFNmOqqzLIwHdKVS5EA0hUkjaXLfKHlrlvD+D6swC+BlKeM9qo0AVdKg5ZkuMcMfMyU9CAqlWATO6b8HIt9uuzBWLv9lH86Ye1N4Hm+JBkp4BvJSoSn1xx119w4k712JkI7T6KjBeIl13TmhDfJgWTphk5HRbrZitFsQmBb4Ho6io06n1iun5uc5VakzftRVY0vkS+LNNeiViroxouqVdjD4FVvMnpFHMyqcb8h/CrFbKVzSjC3IA04IOGWLDADCcGbICMJif4De4W26f5Ahhjq/I5v6ScF2chW1wtJtfaJfdp0YR6vKsoWBbQnsXNc4pqUmTDcKgXOPAczdxhyejANWmXUV49IKPlIepV7ZD4Tri/35W6BxRBCfZQakIbRCs7hIcRagFlbrzWS0pRuuhHb3t+BzoYm/yX26O3qtZgks0DBbmV9gHwl6NB7HBwBuBluEOz3IAzA6IOAZmk7BA5pK4BhYHeGkBxcvP+uPl96An0ZVKjnpEVBR53+1+Y4gb4hzjsUW7aqNfqIP0vkombnEXnOKMfaHmSnS0TX7Noy/K3QDQyvXddSd/s6d7CELQnOPxcFkl1KN0oBN/cN5C6rnayf1voP5LWBtxK/d/B6HPP6Y8LR685nHEP/E0CTNc71B2XNxj0TyEHCxekpSPhK9gQkPLyKrTlg10yHYu26edBclovXIdLs9eDZ8v98PBCiadOTF6rRfPxaDEdwNoO2yzGtR2jOX56HMT1QaNvJQs91K9fFfEuKvHl620qmp6/tBujfAQAA//+3qHlyoQsAAA==", + "scalars.json": "H4sIAAAAAAAA/9yXzW4aMRDH7zzFiFMqBZDSlEa9JZGQOOQEOUWp5GVnvW6NTewxzaqq1HfoG/ZJqt0F1gYvpChVk9zQfHg9v/nPWNx1AL53AAC6C6NJT4sFdj9BN9Uukdg9rV1KE9rSvDbMFot45MzG7XxzciY1o+H52vGFLVk8ZZEvgpyNvaBcq6jLuKRYO0aVowPw47SlxiB1X4lBYFNhYN4q8P1ZrMCwjn9dn1DeNTb13Vq0sGRGsERiT6LilAOqmU6F4n0YK8wyMROoCDJtNh5QyBmJJYJy8wSNhd8/f4HIoNDOQCZQpiAsSPEVZQGkIWdl7DppyaRDewrOItjqYiCUJWRpPwI8uHkDXKgI7iDWh+1Fe6iFIuRo4rC9FB/1leDKzUEbGInH8tcJs2DwwQmD6btDPWi0/sJ6MDw/0IPm5k0PpFY82oT4SPvhu10YWDLCC9huxsBP3+3IXvLuGPlHOLgWMbqoGt0zy/EAgaM06Y4RZQuZmERcXCPu/4jkKET2qN05EVxhCkJRPWt9mOZoEebaIGxGWhZ1Cu6OM+VMgUHuJDNQXcG+7fVoj9qPz855eN7G+VWvwEw8YhqR8aX8xgoLWflqJAWh7cNNgK4G5FbvdLZ6OYAZBJ0RKuAGGaGp484+n128vM35zEqtWEakumKJgud0CGb54B6G+WH4epft/mF/uh7f9tL7Cy29wa2UaC2f8q/Lj2vK9K08eqZfZWlHpiKFbnuCCrecfoFT4/BaMmsHIyZt/XN/twOSTa+hdsDcWQJWd36mFTGh4HY66l2sXq+01NjHXiIILifX4zEQPlJMF+GHGmKhnbdczKc2CZuft3wiZGbJDJwS5ZVj3Ooz4aQqbf98VMrfAXbDig0fpgpgJhFkmCnA4oNDNSvXafvUtNG5KggnLYTu7svjYoR2s55OaR+dqsO9i6vxtEbUue/8CQAA//8z+wC/ohEAAA==", +} + +func fetchResource(name string) ([]byte, error) { + raw, ok := embeddedResources[name] + if !ok { + return nil, fmt.Errorf("Could not find resource for '%s'", name) + } + + compressed, err := base64.StdEncoding.DecodeString(raw) + if err != nil { + return nil, err + } + + var out bytes.Buffer + buf := bytes.NewBuffer(compressed) + + r, err := gzip.NewReader(buf) + if err != nil { + return nil, err + } + + if _, err := io.Copy(&out, r); err != nil { + return nil, err + } + + return out.Bytes(), nil +} diff --git a/github.com/pseudomuto/protoc-gen-doc/revive.toml b/github.com/pseudomuto/protoc-gen-doc/revive.toml new file mode 100644 index 0000000000..8d5bea1837 --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/revive.toml @@ -0,0 +1,51 @@ +ignoreGeneratedHeader = false +severity = "warning" +confidence = 0.8 +errorCode = 1 +warningCode = 1 + +[rule.blank-imports] +[rule.context-as-argument] +[rule.context-keys-type] +[rule.dot-imports] +[rule.error-return] +[rule.error-strings] +[rule.error-naming] +[rule.exported] +[rule.if-return] +[rule.increment-decrement] +[rule.var-naming] +[rule.var-declaration] +[rule.package-comments] +[rule.range] +[rule.receiver-naming] +[rule.time-naming] +[rule.unexported-return] +[rule.indent-error-flow] +[rule.errorf] +[rule.empty-block] +[rule.superfluous-else] +[rule.unused-parameter] +[rule.unreachable-code] +[rule.redefines-builtin-id] + +# The White Box Testing Blog suggests acceptable metrics for Cyclomatic Complexity, +# 1-10 - simple, not much risk +# 11-20 - complex, low risk +# 21-50 - too complex, medium risk, attention +# More than 50 - too complex, can't test , high risk + +# Configuration of the `cyclomatic` rule. Here we specify that +# the rule should fail if it detects code with higher complexity than 20. +[rule.cyclomatic] + arguments = [20] + +# Configuration of the `argument-limit` rule. Here we specify that +# the rule should fail if the input arguments exceed 6. +[rule.argument-limit] + arguments = [6] + +# Configuration of the `function-result-limit` rule. Here we specify that +# the rule should fail if the returned results exceed 3. +[rule.function-result-limit] + arguments = [3] diff --git a/github.com/pseudomuto/protoc-gen-doc/template.go b/github.com/pseudomuto/protoc-gen-doc/template.go new file mode 100644 index 0000000000..a3a05834ba --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/template.go @@ -0,0 +1,592 @@ +package gendoc + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + + "github.com/golang/protobuf/protoc-gen-go/descriptor" + "github.com/pseudomuto/protoc-gen-doc/extensions" + "github.com/pseudomuto/protokit" +) + +// Template is a type for encapsulating all the parsed files, messages, fields, enums, services, extensions, etc. into +// an object that will be supplied to a go template. +type Template struct { + // The files that were parsed + Files []*File `json:"files"` + // Details about the scalar values and their respective types in supported languages. + Scalars []*ScalarValue `json:"scalarValueTypes"` +} + +// NewTemplate creates a Template object from a set of descriptors. +func NewTemplate(descs []*protokit.FileDescriptor) *Template { + files := make([]*File, 0, len(descs)) + + for _, f := range descs { + file := &File{ + Name: f.GetName(), + Description: description(f.GetSyntaxComments().String()), + Package: f.GetPackage(), + HasEnums: len(f.Enums) > 0, + HasExtensions: len(f.Extensions) > 0, + HasMessages: len(f.Messages) > 0, + HasServices: len(f.Services) > 0, + Enums: make(orderedEnums, 0, len(f.Enums)), + Extensions: make(orderedExtensions, 0, len(f.Extensions)), + Messages: make(orderedMessages, 0, len(f.Messages)), + Services: make(orderedServices, 0, len(f.Services)), + Options: mergeOptions(extractOptions(f.GetOptions()), extensions.Transform(f.OptionExtensions)), + } + + for _, e := range f.Enums { + file.Enums = append(file.Enums, parseEnum(e)) + } + + for _, e := range f.Extensions { + file.Extensions = append(file.Extensions, parseFileExtension(e)) + } + + // Recursively add nested types from messages + var addFromMessage func(*protokit.Descriptor) + addFromMessage = func(m *protokit.Descriptor) { + file.Messages = append(file.Messages, parseMessage(m)) + for _, e := range m.Enums { + file.Enums = append(file.Enums, parseEnum(e)) + } + for _, n := range m.Messages { + addFromMessage(n) + } + } + for _, m := range f.Messages { + addFromMessage(m) + } + + for _, s := range f.Services { + file.Services = append(file.Services, parseService(s)) + } + + sort.Sort(file.Enums) + sort.Sort(file.Extensions) + sort.Sort(file.Messages) + sort.Sort(file.Services) + + files = append(files, file) + } + + return &Template{Files: files, Scalars: makeScalars()} +} + +func makeScalars() []*ScalarValue { + data, _ := fetchResource("scalars.json") + + var scalars []*ScalarValue + json.Unmarshal(data, &scalars) + + return scalars +} + +func mergeOptions(opts ...map[string]interface{}) map[string]interface{} { + out := make(map[string]interface{}) + for _, opts := range opts { + for k, v := range opts { + if _, ok := out[k]; ok { + continue + } + out[k] = v + } + } + if len(out) == 0 { + return nil + } + return out +} + +// CommonOptions are options common to all descriptor types. +type commonOptions interface { + GetDeprecated() bool +} + +func extractOptions(opts commonOptions) map[string]interface{} { + out := make(map[string]interface{}) + if opts.GetDeprecated() { + out["deprecated"] = true + } + switch opts := opts.(type) { + case *descriptor.MethodOptions: + if opts != nil && opts.IdempotencyLevel != nil { + out["idempotency_level"] = opts.IdempotencyLevel.String() + } + } + return out +} + +// File wraps all the relevant parsed info about a proto file. File objects guarantee that their top-level enums, +// extensions, messages, and services are sorted alphabetically based on their "long name". Other values (enum values, +// fields, service methods) will be in the order that they're defined within their respective proto files. +// +// In the case of proto3 files, HasExtensions will always be false, and Extensions will be empty. +type File struct { + Name string `json:"name"` + Description string `json:"description"` + Package string `json:"package"` + + HasEnums bool `json:"hasEnums"` + HasExtensions bool `json:"hasExtensions"` + HasMessages bool `json:"hasMessages"` + HasServices bool `json:"hasServices"` + + Enums orderedEnums `json:"enums"` + Extensions orderedExtensions `json:"extensions"` + Messages orderedMessages `json:"messages"` + Services orderedServices `json:"services"` + + Options map[string]interface{} `json:"options,omitempty"` +} + +// Option returns the named option. +func (f File) Option(name string) interface{} { return f.Options[name] } + +// FileExtension contains details about top-level extensions within a proto(2) file. +type FileExtension struct { + Name string `json:"name"` + LongName string `json:"longName"` + FullName string `json:"fullName"` + Description string `json:"description"` + Label string `json:"label"` + Type string `json:"type"` + LongType string `json:"longType"` + FullType string `json:"fullType"` + Number int `json:"number"` + DefaultValue string `json:"defaultValue"` + ContainingType string `json:"containingType"` + ContainingLongType string `json:"containingLongType"` + ContainingFullType string `json:"containingFullType"` +} + +// Message contains details about a protobuf message. +// +// In the case of proto3 files, HasExtensions will always be false, and Extensions will be empty. +type Message struct { + Name string `json:"name"` + LongName string `json:"longName"` + FullName string `json:"fullName"` + Description string `json:"description"` + + HasExtensions bool `json:"hasExtensions"` + HasFields bool `json:"hasFields"` + + Extensions []*MessageExtension `json:"extensions"` + Fields []*MessageField `json:"fields"` + + Options map[string]interface{} `json:"options,omitempty"` +} + +// Option returns the named option. +func (m Message) Option(name string) interface{} { return m.Options[name] } + +// FieldOptions returns all options that are set on the fields in this message. +func (m Message) FieldOptions() []string { + optionSet := make(map[string]struct{}) + for _, field := range m.Fields { + for option := range field.Options { + optionSet[option] = struct{}{} + } + } + if len(optionSet) == 0 { + return nil + } + options := make([]string, 0, len(optionSet)) + for option := range optionSet { + options = append(options, option) + } + sort.Strings(options) + return options +} + +// FieldsWithOption returns all fields that have the given option set. +// If no single value has the option set, this returns nil. +func (m Message) FieldsWithOption(optionName string) []*MessageField { + fields := make([]*MessageField, 0, len(m.Fields)) + for _, field := range m.Fields { + if _, ok := field.Options[optionName]; ok { + fields = append(fields, field) + } + } + if len(fields) > 0 { + return fields + } + return nil +} + +// MessageField contains details about an individual field within a message. +// +// In the case of proto3 files, DefaultValue will always be empty. Similarly, label will be empty unless the field is +// repeated (in which case it'll be "repeated"). +type MessageField struct { + Name string `json:"name"` + Description string `json:"description"` + Label string `json:"label"` + Type string `json:"type"` + LongType string `json:"longType"` + FullType string `json:"fullType"` + IsMap bool `json:"ismap"` + DefaultValue string `json:"defaultValue"` + + Options map[string]interface{} `json:"options,omitempty"` +} + +// Option returns the named option. +func (f MessageField) Option(name string) interface{} { return f.Options[name] } + +// MessageExtension contains details about message-scoped extensions in proto(2) files. +type MessageExtension struct { + FileExtension + + ScopeType string `json:"scopeType"` + ScopeLongType string `json:"scopeLongType"` + ScopeFullType string `json:"scopeFullType"` +} + +// Enum contains details about enumerations. These can be either top level enums, or nested (defined within a message). +type Enum struct { + Name string `json:"name"` + LongName string `json:"longName"` + FullName string `json:"fullName"` + Description string `json:"description"` + Values []*EnumValue `json:"values"` + + Options map[string]interface{} `json:"options,omitempty"` +} + +// Option returns the named option. +func (e Enum) Option(name string) interface{} { return e.Options[name] } + +// ValueOptions returns all options that are set on the values in this enum. +func (e Enum) ValueOptions() []string { + optionSet := make(map[string]struct{}) + for _, value := range e.Values { + for option := range value.Options { + optionSet[option] = struct{}{} + } + } + if len(optionSet) == 0 { + return nil + } + options := make([]string, 0, len(optionSet)) + for option := range optionSet { + options = append(options, option) + } + sort.Strings(options) + return options +} + +// ValuesWithOption returns all values that have the given option set. +// If no single value has the option set, this returns nil. +func (e Enum) ValuesWithOption(optionName string) []*EnumValue { + values := make([]*EnumValue, 0, len(e.Values)) + for _, value := range e.Values { + if _, ok := value.Options[optionName]; ok { + values = append(values, value) + } + } + if len(values) > 0 { + return values + } + return nil +} + +// EnumValue contains details about an individual value within an enumeration. +type EnumValue struct { + Name string `json:"name"` + Number string `json:"number"` + Description string `json:"description"` + + Options map[string]interface{} `json:"options,omitempty"` +} + +// Option returns the named option. +func (v EnumValue) Option(name string) interface{} { return v.Options[name] } + +// Service contains details about a service definition within a proto file. +type Service struct { + Name string `json:"name"` + LongName string `json:"longName"` + FullName string `json:"fullName"` + Description string `json:"description"` + Methods []*ServiceMethod `json:"methods"` + + Options map[string]interface{} `json:"options,omitempty"` +} + +// Option returns the named option. +func (s Service) Option(name string) interface{} { return s.Options[name] } + +// MethodOptions returns all options that are set on the methods in this service. +func (s Service) MethodOptions() []string { + optionSet := make(map[string]struct{}) + for _, method := range s.Methods { + for option := range method.Options { + optionSet[option] = struct{}{} + } + } + if len(optionSet) == 0 { + return nil + } + options := make([]string, 0, len(optionSet)) + for option := range optionSet { + options = append(options, option) + } + sort.Strings(options) + return options +} + +// MethodsWithOption returns all methods that have the given option set. +// If no single method has the option set, this returns nil. +func (s Service) MethodsWithOption(optionName string) []*ServiceMethod { + methods := make([]*ServiceMethod, 0, len(s.Methods)) + for _, method := range s.Methods { + if _, ok := method.Options[optionName]; ok { + methods = append(methods, method) + } + } + if len(methods) > 0 { + return methods + } + return nil +} + +// ServiceMethod contains details about an individual method within a service. +type ServiceMethod struct { + Name string `json:"name"` + Description string `json:"description"` + RequestType string `json:"requestType"` + RequestLongType string `json:"requestLongType"` + RequestFullType string `json:"requestFullType"` + RequestStreaming bool `json:"requestStreaming"` + ResponseType string `json:"responseType"` + ResponseLongType string `json:"responseLongType"` + ResponseFullType string `json:"responseFullType"` + ResponseStreaming bool `json:"responseStreaming"` + + Options map[string]interface{} `json:"options,omitempty"` +} + +// Option returns the named option. +func (m ServiceMethod) Option(name string) interface{} { return m.Options[name] } + +// ScalarValue contains information about scalar value types in protobuf. The common use case for this type is to know +// which language specific type maps to the protobuf type. +// +// For example, the protobuf type `int64` maps to `long` in C#, and `Bignum` in Ruby. For the full list, take a look at +// https://developers.google.com/protocol-buffers/docs/proto3#scalar +type ScalarValue struct { + ProtoType string `json:"protoType"` + Notes string `json:"notes"` + CppType string `json:"cppType"` + CSharp string `json:"csType"` + GoType string `json:"goType"` + JavaType string `json:"javaType"` + PhpType string `json:"phpType"` + PythonType string `json:"pythonType"` + RubyType string `json:"rubyType"` +} + +func parseEnum(pe *protokit.EnumDescriptor) *Enum { + enum := &Enum{ + Name: pe.GetName(), + LongName: pe.GetLongName(), + FullName: pe.GetFullName(), + Description: description(pe.GetComments().String()), + Options: mergeOptions(extractOptions(pe.GetOptions()), extensions.Transform(pe.OptionExtensions)), + } + + for _, val := range pe.GetValues() { + enum.Values = append(enum.Values, &EnumValue{ + Name: val.GetName(), + Number: fmt.Sprint(val.GetNumber()), + Description: description(val.GetComments().String()), + Options: mergeOptions(extractOptions(val.GetOptions()), extensions.Transform(val.OptionExtensions)), + }) + } + + return enum +} + +func parseFileExtension(pe *protokit.ExtensionDescriptor) *FileExtension { + t, lt, ft := parseType(pe) + + return &FileExtension{ + Name: pe.GetName(), + LongName: pe.GetLongName(), + FullName: pe.GetFullName(), + Description: description(pe.GetComments().String()), + Label: labelName(pe.GetLabel(), pe.IsProto3()), + Type: t, + LongType: lt, + FullType: ft, + Number: int(pe.GetNumber()), + DefaultValue: pe.GetDefaultValue(), + ContainingType: baseName(pe.GetExtendee()), + ContainingLongType: strings.TrimPrefix(pe.GetExtendee(), "."+pe.GetPackage()+"."), + ContainingFullType: strings.TrimPrefix(pe.GetExtendee(), "."), + } +} + +func parseMessage(pm *protokit.Descriptor) *Message { + msg := &Message{ + Name: pm.GetName(), + LongName: pm.GetLongName(), + FullName: pm.GetFullName(), + Description: description(pm.GetComments().String()), + HasExtensions: len(pm.GetExtensions()) > 0, + HasFields: len(pm.GetMessageFields()) > 0, + Extensions: make([]*MessageExtension, 0, len(pm.Extensions)), + Fields: make([]*MessageField, 0, len(pm.Fields)), + Options: mergeOptions(extractOptions(pm.GetOptions()), extensions.Transform(pm.OptionExtensions)), + } + + for _, ext := range pm.Extensions { + msg.Extensions = append(msg.Extensions, parseMessageExtension(ext)) + } + + for _, f := range pm.Fields { + msg.Fields = append(msg.Fields, parseMessageField(f)) + } + + return msg +} + +func parseMessageExtension(pe *protokit.ExtensionDescriptor) *MessageExtension { + return &MessageExtension{ + FileExtension: *parseFileExtension(pe), + ScopeType: pe.GetParent().GetName(), + ScopeLongType: pe.GetParent().GetLongName(), + ScopeFullType: pe.GetParent().GetFullName(), + } +} + +func parseMessageField(pf *protokit.FieldDescriptor) *MessageField { + t, lt, ft := parseType(pf) + + m := &MessageField{ + Name: pf.GetName(), + Description: description(pf.GetComments().String()), + Label: labelName(pf.GetLabel(), pf.IsProto3()), + Type: t, + LongType: lt, + FullType: ft, + DefaultValue: pf.GetDefaultValue(), + Options: mergeOptions(extractOptions(pf.GetOptions()), extensions.Transform(pf.OptionExtensions)), + } + + // Check if this is a map. + // See https://github.com/golang/protobuf/blob/master/protoc-gen-go/descriptor/descriptor.pb.go#L1556 + // for more information + if m.Label == "repeated" && + strings.Contains(m.LongType, ".") && + strings.HasSuffix(m.Type, "Entry") && + strings.HasSuffix(m.LongType, "Entry") && + strings.HasSuffix(m.FullType, "Entry") { + m.IsMap = true + } + + return m +} + +func parseService(ps *protokit.ServiceDescriptor) *Service { + service := &Service{ + Name: ps.GetName(), + LongName: ps.GetLongName(), + FullName: ps.GetFullName(), + Description: description(ps.GetComments().String()), + Options: mergeOptions(extractOptions(ps.GetOptions()), extensions.Transform(ps.OptionExtensions)), + } + + for _, sm := range ps.Methods { + service.Methods = append(service.Methods, parseServiceMethod(sm)) + } + + return service +} + +func parseServiceMethod(pm *protokit.MethodDescriptor) *ServiceMethod { + return &ServiceMethod{ + Name: pm.GetName(), + Description: description(pm.GetComments().String()), + RequestType: baseName(pm.GetInputType()), + RequestLongType: strings.TrimPrefix(pm.GetInputType(), "."+pm.GetPackage()+"."), + RequestFullType: strings.TrimPrefix(pm.GetInputType(), "."), + RequestStreaming: pm.GetClientStreaming(), + ResponseType: baseName(pm.GetOutputType()), + ResponseLongType: strings.TrimPrefix(pm.GetOutputType(), "."+pm.GetPackage()+"."), + ResponseFullType: strings.TrimPrefix(pm.GetOutputType(), "."), + ResponseStreaming: pm.GetServerStreaming(), + Options: mergeOptions(extractOptions(pm.GetOptions()), extensions.Transform(pm.OptionExtensions)), + } +} + +func baseName(name string) string { + parts := strings.Split(name, ".") + return parts[len(parts)-1] +} + +func labelName(lbl descriptor.FieldDescriptorProto_Label, proto3 bool) string { + if proto3 && lbl != descriptor.FieldDescriptorProto_LABEL_REPEATED { + return "" + } + + return strings.ToLower(strings.TrimPrefix(lbl.String(), "LABEL_")) +} + +type typeContainer interface { + GetType() descriptor.FieldDescriptorProto_Type + GetTypeName() string + GetPackage() string +} + +func parseType(tc typeContainer) (string, string, string) { + name := tc.GetTypeName() + + if strings.HasPrefix(name, ".") { + name = strings.TrimPrefix(name, ".") + return baseName(name), strings.TrimPrefix(name, tc.GetPackage()+"."), name + } + + name = strings.ToLower(strings.TrimPrefix(tc.GetType().String(), "TYPE_")) + return name, name, name +} + +func description(comment string) string { + val := strings.TrimLeft(comment, "*/\n ") + if strings.HasPrefix(val, "@exclude") { + return "" + } + + return val +} + +type orderedEnums []*Enum + +func (oe orderedEnums) Len() int { return len(oe) } +func (oe orderedEnums) Swap(i, j int) { oe[i], oe[j] = oe[j], oe[i] } +func (oe orderedEnums) Less(i, j int) bool { return oe[i].LongName < oe[j].LongName } + +type orderedExtensions []*FileExtension + +func (oe orderedExtensions) Len() int { return len(oe) } +func (oe orderedExtensions) Swap(i, j int) { oe[i], oe[j] = oe[j], oe[i] } +func (oe orderedExtensions) Less(i, j int) bool { return oe[i].LongName < oe[j].LongName } + +type orderedMessages []*Message + +func (om orderedMessages) Len() int { return len(om) } +func (om orderedMessages) Swap(i, j int) { om[i], om[j] = om[j], om[i] } +func (om orderedMessages) Less(i, j int) bool { return om[i].LongName < om[j].LongName } + +type orderedServices []*Service + +func (os orderedServices) Len() int { return len(os) } +func (os orderedServices) Swap(i, j int) { os[i], os[j] = os[j], os[i] } +func (os orderedServices) Less(i, j int) bool { return os[i].LongName < os[j].LongName } diff --git a/github.com/pseudomuto/protoc-gen-doc/thirdparty/github.com/envoyproxy/protoc-gen-validate/validate/validate.proto b/github.com/pseudomuto/protoc-gen-doc/thirdparty/github.com/envoyproxy/protoc-gen-validate/validate/validate.proto new file mode 100644 index 0000000000..05cce2cbd3 --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/thirdparty/github.com/envoyproxy/protoc-gen-validate/validate/validate.proto @@ -0,0 +1,763 @@ +syntax = "proto2"; +package validate; + +option go_package = "github.com/envoyproxy/protoc-gen-validate/validate"; +option java_package = "com.lyft.pgv.validate"; + +import "google/protobuf/descriptor.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; + +// Validation rules applied at the message level +extend google.protobuf.MessageOptions { + // Disabled nullifies any validation rules for this message, including any + // message fields associated with it that do support validation. + optional bool disabled = 919191; +} + +// Validation rules applied at the oneof level +extend google.protobuf.OneofOptions { + // Required ensures that exactly one the field options in a oneof is set; + // validation fails if no fields in the oneof are set. + optional bool required = 919191; +} + +// Validation rules applied at the field level +extend google.protobuf.FieldOptions { + // Rules specify the validations to be performed on this field. By default, + // no validation is performed against a field. + optional FieldRules rules = 919191; +} + +// FieldRules encapsulates the rules for each type of field. Depending on the +// field, the correct set should be used to ensure proper validations. +message FieldRules { + oneof type { + // Scalar Field Types + FloatRules float = 1; + DoubleRules double = 2; + Int32Rules int32 = 3; + Int64Rules int64 = 4; + UInt32Rules uint32 = 5; + UInt64Rules uint64 = 6; + SInt32Rules sint32 = 7; + SInt64Rules sint64 = 8; + Fixed32Rules fixed32 = 9; + Fixed64Rules fixed64 = 10; + SFixed32Rules sfixed32 = 11; + SFixed64Rules sfixed64 = 12; + BoolRules bool = 13; + StringRules string = 14; + BytesRules bytes = 15; + + // Complex Field Types + EnumRules enum = 16; + MessageRules message = 17; + RepeatedRules repeated = 18; + MapRules map = 19; + + // Well-Known Field Types + AnyRules any = 20; + DurationRules duration = 21; + TimestampRules timestamp = 22; + } +} + +// FloatRules describes the constraints applied to `float` values +message FloatRules { + // Const specifies that this field must be exactly the specified value + optional float const = 1; + + // Lt specifies that this field must be less than the specified value, + // exclusive + optional float lt = 2; + + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + optional float lte = 3; + + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + optional float gt = 4; + + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + optional float gte = 5; + + // In specifies that this field must be equal to one of the specified + // values + repeated float in = 6; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated float not_in = 7; +} + +// DoubleRules describes the constraints applied to `double` values +message DoubleRules { + // Const specifies that this field must be exactly the specified value + optional double const = 1; + + // Lt specifies that this field must be less than the specified value, + // exclusive + optional double lt = 2; + + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + optional double lte = 3; + + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + optional double gt = 4; + + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + optional double gte = 5; + + // In specifies that this field must be equal to one of the specified + // values + repeated double in = 6; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated double not_in = 7; +} + +// Int32Rules describes the constraints applied to `int32` values +message Int32Rules { + // Const specifies that this field must be exactly the specified value + optional int32 const = 1; + + // Lt specifies that this field must be less than the specified value, + // exclusive + optional int32 lt = 2; + + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + optional int32 lte = 3; + + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + optional int32 gt = 4; + + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + optional int32 gte = 5; + + // In specifies that this field must be equal to one of the specified + // values + repeated int32 in = 6; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated int32 not_in = 7; +} + +// Int64Rules describes the constraints applied to `int64` values +message Int64Rules { + // Const specifies that this field must be exactly the specified value + optional int64 const = 1; + + // Lt specifies that this field must be less than the specified value, + // exclusive + optional int64 lt = 2; + + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + optional int64 lte = 3; + + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + optional int64 gt = 4; + + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + optional int64 gte = 5; + + // In specifies that this field must be equal to one of the specified + // values + repeated int64 in = 6; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated int64 not_in = 7; +} + +// UInt32Rules describes the constraints applied to `uint32` values +message UInt32Rules { + // Const specifies that this field must be exactly the specified value + optional uint32 const = 1; + + // Lt specifies that this field must be less than the specified value, + // exclusive + optional uint32 lt = 2; + + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + optional uint32 lte = 3; + + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + optional uint32 gt = 4; + + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + optional uint32 gte = 5; + + // In specifies that this field must be equal to one of the specified + // values + repeated uint32 in = 6; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated uint32 not_in = 7; +} + +// UInt64Rules describes the constraints applied to `uint64` values +message UInt64Rules { + // Const specifies that this field must be exactly the specified value + optional uint64 const = 1; + + // Lt specifies that this field must be less than the specified value, + // exclusive + optional uint64 lt = 2; + + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + optional uint64 lte = 3; + + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + optional uint64 gt = 4; + + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + optional uint64 gte = 5; + + // In specifies that this field must be equal to one of the specified + // values + repeated uint64 in = 6; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated uint64 not_in = 7; +} + +// SInt32Rules describes the constraints applied to `sint32` values +message SInt32Rules { + // Const specifies that this field must be exactly the specified value + optional sint32 const = 1; + + // Lt specifies that this field must be less than the specified value, + // exclusive + optional sint32 lt = 2; + + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + optional sint32 lte = 3; + + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + optional sint32 gt = 4; + + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + optional sint32 gte = 5; + + // In specifies that this field must be equal to one of the specified + // values + repeated sint32 in = 6; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated sint32 not_in = 7; +} + +// SInt64Rules describes the constraints applied to `sint64` values +message SInt64Rules { + // Const specifies that this field must be exactly the specified value + optional sint64 const = 1; + + // Lt specifies that this field must be less than the specified value, + // exclusive + optional sint64 lt = 2; + + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + optional sint64 lte = 3; + + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + optional sint64 gt = 4; + + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + optional sint64 gte = 5; + + // In specifies that this field must be equal to one of the specified + // values + repeated sint64 in = 6; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated sint64 not_in = 7; +} + +// Fixed32Rules describes the constraints applied to `fixed32` values +message Fixed32Rules { + // Const specifies that this field must be exactly the specified value + optional fixed32 const = 1; + + // Lt specifies that this field must be less than the specified value, + // exclusive + optional fixed32 lt = 2; + + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + optional fixed32 lte = 3; + + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + optional fixed32 gt = 4; + + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + optional fixed32 gte = 5; + + // In specifies that this field must be equal to one of the specified + // values + repeated fixed32 in = 6; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated fixed32 not_in = 7; +} + +// Fixed64Rules describes the constraints applied to `fixed64` values +message Fixed64Rules { + // Const specifies that this field must be exactly the specified value + optional fixed64 const = 1; + + // Lt specifies that this field must be less than the specified value, + // exclusive + optional fixed64 lt = 2; + + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + optional fixed64 lte = 3; + + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + optional fixed64 gt = 4; + + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + optional fixed64 gte = 5; + + // In specifies that this field must be equal to one of the specified + // values + repeated fixed64 in = 6; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated fixed64 not_in = 7; +} + +// SFixed32Rules describes the constraints applied to `sfixed32` values +message SFixed32Rules { + // Const specifies that this field must be exactly the specified value + optional sfixed32 const = 1; + + // Lt specifies that this field must be less than the specified value, + // exclusive + optional sfixed32 lt = 2; + + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + optional sfixed32 lte = 3; + + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + optional sfixed32 gt = 4; + + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + optional sfixed32 gte = 5; + + // In specifies that this field must be equal to one of the specified + // values + repeated sfixed32 in = 6; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated sfixed32 not_in = 7; +} + +// SFixed64Rules describes the constraints applied to `sfixed64` values +message SFixed64Rules { + // Const specifies that this field must be exactly the specified value + optional sfixed64 const = 1; + + // Lt specifies that this field must be less than the specified value, + // exclusive + optional sfixed64 lt = 2; + + // Lte specifies that this field must be less than or equal to the + // specified value, inclusive + optional sfixed64 lte = 3; + + // Gt specifies that this field must be greater than the specified value, + // exclusive. If the value of Gt is larger than a specified Lt or Lte, the + // range is reversed. + optional sfixed64 gt = 4; + + // Gte specifies that this field must be greater than or equal to the + // specified value, inclusive. If the value of Gte is larger than a + // specified Lt or Lte, the range is reversed. + optional sfixed64 gte = 5; + + // In specifies that this field must be equal to one of the specified + // values + repeated sfixed64 in = 6; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated sfixed64 not_in = 7; +} + +// BoolRules describes the constraints applied to `bool` values +message BoolRules { + // Const specifies that this field must be exactly the specified value + optional bool const = 1; +} + +// StringRules describe the constraints applied to `string` values +message StringRules { + // Const specifies that this field must be exactly the specified value + optional string const = 1; + + // Len specifies that this field must be the specified number of + // characters (Unicode code points). Note that the number of + // characters may differ from the number of bytes in the string. + optional uint64 len = 19; + + // MinLen specifies that this field must be the specified number of + // characters (Unicode code points) at a minimum. Note that the number of + // characters may differ from the number of bytes in the string. + optional uint64 min_len = 2; + + // MaxLen specifies that this field must be the specified number of + // characters (Unicode code points) at a maximum. Note that the number of + // characters may differ from the number of bytes in the string. + optional uint64 max_len = 3; + + // LenBytes specifies that this field must be the specified number of bytes + // at a minimum + optional uint64 len_bytes = 20; + + // MinBytes specifies that this field must be the specified number of bytes + // at a minimum + optional uint64 min_bytes = 4; + + // MaxBytes specifies that this field must be the specified number of bytes + // at a maximum + optional uint64 max_bytes = 5; + + // Pattern specifes that this field must match against the specified + // regular expression (RE2 syntax). The included expression should elide + // any delimiters. + optional string pattern = 6; + + // Prefix specifies that this field must have the specified substring at + // the beginning of the string. + optional string prefix = 7; + + // Suffix specifies that this field must have the specified substring at + // the end of the string. + optional string suffix = 8; + + // Contains specifies that this field must have the specified substring + // anywhere in the string. + optional string contains = 9; + + // In specifies that this field must be equal to one of the specified + // values + repeated string in = 10; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated string not_in = 11; + + // WellKnown rules provide advanced constraints against common string + // patterns + oneof well_known { + // Email specifies that the field must be a valid email address as + // defined by RFC 5322 + bool email = 12; + + // Hostname specifies that the field must be a valid hostname as + // defined by RFC 1034. This constraint does not support + // internationalized domain names (IDNs). + bool hostname = 13; + + // Ip specifies that the field must be a valid IP (v4 or v6) address. + // Valid IPv6 addresses should not include surrounding square brackets. + bool ip = 14; + + // Ipv4 specifies that the field must be a valid IPv4 address. + bool ipv4 = 15; + + // Ipv6 specifies that the field must be a valid IPv6 address. Valid + // IPv6 addresses should not include surrounding square brackets. + bool ipv6 = 16; + + // Uri specifies that the field must be a valid, absolute URI as defined + // by RFC 3986 + bool uri = 17; + + // UriRef specifies that the field must be a valid URI as defined by RFC + // 3986 and may be relative or absolute. + bool uri_ref = 18; + } +} + +// BytesRules describe the constraints applied to `bytes` values +message BytesRules { + // Const specifies that this field must be exactly the specified value + optional bytes const = 1; + + // Len specifies that this field must be the specified number of bytes + optional uint64 len = 13; + + // MinLen specifies that this field must be the specified number of bytes + // at a minimum + optional uint64 min_len = 2; + + // MaxLen specifies that this field must be the specified number of bytes + // at a maximum + optional uint64 max_len = 3; + + // Pattern specifes that this field must match against the specified + // regular expression (RE2 syntax). The included expression should elide + // any delimiters. + optional string pattern = 4; + + // Prefix specifies that this field must have the specified bytes at the + // beginning of the string. + optional bytes prefix = 5; + + // Suffix specifies that this field must have the specified bytes at the + // end of the string. + optional bytes suffix = 6; + + // Contains specifies that this field must have the specified bytes + // anywhere in the string. + optional bytes contains = 7; + + // In specifies that this field must be equal to one of the specified + // values + repeated bytes in = 8; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated bytes not_in = 9; + + // WellKnown rules provide advanced constraints against common byte + // patterns + oneof well_known { + // Ip specifies that the field must be a valid IP (v4 or v6) address in + // byte format + bool ip = 10; + + // Ipv4 specifies that the field must be a valid IPv4 address in byte + // format + bool ipv4 = 11; + + // Ipv6 specifies that the field must be a valid IPv6 address in byte + // format + bool ipv6 = 12; + } +} + +// EnumRules describe the constraints applied to enum values +message EnumRules { + // Const specifies that this field must be exactly the specified value + optional int32 const = 1; + + // DefinedOnly specifies that this field must be only one of the defined + // values for this enum, failing on any undefined value. + optional bool defined_only = 2; + + // In specifies that this field must be equal to one of the specified + // values + repeated int32 in = 3; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated int32 not_in = 4; +} + +// MessageRules describe the constraints applied to embedded message values. +// For message-type fields, validation is performed recursively. +message MessageRules { + // Skip specifies that the validation rules of this field should not be + // evaluated + optional bool skip = 1; + + // Required specifies that this field must be set + optional bool required = 2; +} + +// RepeatedRules describe the constraints applied to `repeated` values +message RepeatedRules { + // MinItems specifies that this field must have the specified number of + // items at a minimum + optional uint64 min_items = 1; + + // MaxItems specifies that this field must have the specified number of + // items at a maximum + optional uint64 max_items = 2; + + // Unique specifies that all elements in this field must be unique. This + // contraint is only applicable to scalar and enum types (messages are not + // supported). + optional bool unique = 3; + + // Items specifies the contraints to be applied to each item in the field. + // Repeated message fields will still execute validation against each item + // unless skip is specified here. + optional FieldRules items = 4; +} + +// MapRules describe the constraints applied to `map` values +message MapRules { + // MinPairs specifies that this field must have the specified number of + // KVs at a minimum + optional uint64 min_pairs = 1; + + // MaxPairs specifies that this field must have the specified number of + // KVs at a maximum + optional uint64 max_pairs = 2; + + // NoSparse specifies values in this field cannot be unset. This only + // applies to map's with message value types. + optional bool no_sparse = 3; + + // Keys specifies the constraints to be applied to each key in the field. + optional FieldRules keys = 4; + + // Values specifies the constraints to be applied to the value of each key + // in the field. Message values will still have their validations evaluated + // unless skip is specified here. + optional FieldRules values = 5; +} + +// AnyRules describe constraints applied exclusively to the +// `google.protobuf.Any` well-known type +message AnyRules { + // Required specifies that this field must be set + optional bool required = 1; + + // In specifies that this field's `type_url` must be equal to one of the + // specified values. + repeated string in = 2; + + // NotIn specifies that this field's `type_url` must not be equal to any of + // the specified values. + repeated string not_in = 3; +} + +// DurationRules describe the constraints applied exclusively to the +// `google.protobuf.Duration` well-known type +message DurationRules { + // Required specifies that this field must be set + optional bool required = 1; + + // Const specifies that this field must be exactly the specified value + optional google.protobuf.Duration const = 2; + + // Lt specifies that this field must be less than the specified value, + // exclusive + optional google.protobuf.Duration lt = 3; + + // Lt specifies that this field must be less than the specified value, + // inclusive + optional google.protobuf.Duration lte = 4; + + // Gt specifies that this field must be greater than the specified value, + // exclusive + optional google.protobuf.Duration gt = 5; + + // Gte specifies that this field must be greater than the specified value, + // inclusive + optional google.protobuf.Duration gte = 6; + + // In specifies that this field must be equal to one of the specified + // values + repeated google.protobuf.Duration in = 7; + + // NotIn specifies that this field cannot be equal to one of the specified + // values + repeated google.protobuf.Duration not_in = 8; +} + +// TimestampRules describe the constraints applied exclusively to the +// `google.protobuf.Timestamp` well-known type +message TimestampRules { + // Required specifies that this field must be set + optional bool required = 1; + + // Const specifies that this field must be exactly the specified value + optional google.protobuf.Timestamp const = 2; + + // Lt specifies that this field must be less than the specified value, + // exclusive + optional google.protobuf.Timestamp lt = 3; + + // Lte specifies that this field must be less than the specified value, + // inclusive + optional google.protobuf.Timestamp lte = 4; + + // Gt specifies that this field must be greater than the specified value, + // exclusive + optional google.protobuf.Timestamp gt = 5; + + // Gte specifies that this field must be greater than the specified value, + // inclusive + optional google.protobuf.Timestamp gte = 6; + + // LtNow specifies that this must be less than the current time. LtNow + // can only be used with the Within rule. + optional bool lt_now = 7; + + // GtNow specifies that this must be greater than the current time. GtNow + // can only be used with the Within rule. + optional bool gt_now = 8; + + // Within specifies that this field must be within this duration of the + // current time. This constraint can be used alone or with the LtNow and + // GtNow rules. + optional google.protobuf.Duration within = 9; +} diff --git a/github.com/pseudomuto/protoc-gen-doc/thirdparty/github.com/mwitkow/go-proto-validators/validator.proto b/github.com/pseudomuto/protoc-gen-doc/thirdparty/github.com/mwitkow/go-proto-validators/validator.proto new file mode 100644 index 0000000000..13d0f59596 --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/thirdparty/github.com/mwitkow/go-proto-validators/validator.proto @@ -0,0 +1,67 @@ +// Copyright 2016 Michal Witkowski. All Rights Reserved. +// See LICENSE for licensing terms. + +// Protocol Buffers extensions for defining auto-generateable validators for messages. + +// TODO(mwitkow): Add example. + + +syntax = "proto2"; +package validator; + +import "google/protobuf/descriptor.proto"; + +option go_package = "validator"; + +// TODO(mwitkow): Email protobuf-global-extension-registry@google.com to get an extension ID. + +extend google.protobuf.FieldOptions { + optional FieldValidator field = 65020; +} + +message FieldValidator { + // Uses a Golang RE2-syntax regex to match the field contents. + optional string regex = 1; + // Field value of integer strictly greater than this value. + optional int64 int_gt = 2; + // Field value of integer strictly smaller than this value. + optional int64 int_lt = 3; + // Used for nested message types, requires that the message type exists. + optional bool msg_exists = 4; + // Human error specifies a user-customizable error that is visible to the user. + optional string human_error = 5; + // Field value of double strictly greater than this value. + // Note that this value can only take on a valid floating point + // value. Use together with float_epsilon if you need something more specific. + optional double float_gt = 6; + // Field value of double strictly smaller than this value. + // Note that this value can only take on a valid floating point + // value. Use together with float_epsilon if you need something more specific. + optional double float_lt = 7; + // Field value of double describing the epsilon within which + // any comparison should be considered to be true. For example, + // when using float_gt = 0.35, using a float_epsilon of 0.05 + // would mean that any value above 0.30 is acceptable. It can be + // thought of as a {float_value_condition} +- {float_epsilon}. + // If unset, no correction for floating point inaccuracies in + // comparisons will be attempted. + optional double float_epsilon = 8; + // Floating-point value compared to which the field content should be greater or equal. + optional double float_gte = 9; + // Floating-point value compared to which the field content should be smaller or equal. + optional double float_lte = 10; + // Used for string fields, requires the string to be not empty (i.e different from ""). + optional bool string_not_empty = 11; + // Repeated field with at least this number of elements. + optional int64 repeated_count_min = 12; + // Repeated field with at most this number of elements. + optional int64 repeated_count_max = 13; + // Field value of length greater than this value. + optional int64 length_gt = 14; + // Field value of length smaller than this value. + optional int64 length_lt = 15; + // Field value of integer strictly equal this value. + optional int64 length_eq = 16; + // Requires that the value is in the enum. + optional bool is_in_enum = 17; +} diff --git a/github.com/pseudomuto/protoc-gen-doc/thirdparty/github.com/pseudomuto/protokit/fixtures/extend.proto b/github.com/pseudomuto/protoc-gen-doc/thirdparty/github.com/pseudomuto/protokit/fixtures/extend.proto new file mode 100644 index 0000000000..c0260241fb --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/thirdparty/github.com/pseudomuto/protokit/fixtures/extend.proto @@ -0,0 +1,54 @@ +syntax = "proto2"; + +import "google/protobuf/descriptor.proto"; + +package com.pseudomuto.protokit.v1; + +/** + * Extension of protobuf file options. + */ + extend google.protobuf.FileOptions { + optional bool extend_file = 20000; +} + +/** + * Extension of protobuf service options. + */ + extend google.protobuf.ServiceOptions { + optional bool extend_service = 20000; +} + +/** + * Extension of protobuf method options. + */ +extend google.protobuf.MethodOptions { + optional bool extend_method = 20000; +} + +/** + * Extension of protobuf enum options. + */ +extend google.protobuf.EnumOptions { + optional bool extend_enum = 20000; +} + +/** + * Extension of protobuf enum value options. + */ +extend google.protobuf.EnumValueOptions { + optional bool extend_enum_value = 20000; +} + +/** + * Extension of protobuf message options. + */ +extend google.protobuf.MessageOptions { + optional bool extend_message = 20000; +} + +/** + * Extension of protobuf field options. + */ +extend google.protobuf.FieldOptions { + optional bool extend_field = 20000; +} diff --git a/github.com/pseudomuto/protoc-gen-doc/version.go b/github.com/pseudomuto/protoc-gen-doc/version.go new file mode 100644 index 0000000000..39beb18938 --- /dev/null +++ b/github.com/pseudomuto/protoc-gen-doc/version.go @@ -0,0 +1,4 @@ +package gendoc + +// VERSION is the version of protoc-gen-doc being used. +const VERSION = "1.3.2" diff --git a/github.com/pseudomuto/protokit/.gitignore b/github.com/pseudomuto/protokit/.gitignore new file mode 100644 index 0000000000..76893a1b76 --- /dev/null +++ b/github.com/pseudomuto/protokit/.gitignore @@ -0,0 +1,3 @@ +/_tools +/coverage.txt +/vendor diff --git a/github.com/pseudomuto/protokit/.gofmtignore b/github.com/pseudomuto/protokit/.gofmtignore new file mode 100644 index 0000000000..5cecb48392 --- /dev/null +++ b/github.com/pseudomuto/protokit/.gofmtignore @@ -0,0 +1,2 @@ +/_tools/* +/vendor/* diff --git a/github.com/pseudomuto/protokit/.travis.yml b/github.com/pseudomuto/protokit/.travis.yml new file mode 100644 index 0000000000..b8cc9cff06 --- /dev/null +++ b/github.com/pseudomuto/protokit/.travis.yml @@ -0,0 +1,32 @@ +language: go + +env: + global: + - PROTOC_RELEASE="https://github.com/google/protobuf/releases/download/v3.5.1/protoc-3.5.1-linux-x86_64.zip" + - PROTOC_TARGET="${HOME}/protoc" + - PATH="${PROTOC_TARGET}/bin:${PATH}" + +go: + - 1.10.x + - master + +cache: + - "${HOME}/protoc" + - "${GOPATH}/src/github.com/pseudomuto/protokit/_tools" + - "${GOPATH}/src/github.com/pseudomuto/protokit/vendor" + +install: + - if [ ! -d "${PROTOC_TARGET}" ]; then curl -fsSL "${PROTOC_RELEASE}" > "${PROTOC_TARGET}.zip"; fi + - if [ -f "${PROTOC_TARGET}.zip" ]; then unzip "${PROTOC_TARGET}.zip" -d "${PROTOC_TARGET}"; fi + - go get github.com/golang/protobuf/{proto,protoc-gen-go} + - go get github.com/twitchtv/retool + - make setup + +script: + - make test-ci + +after_success: + - bash <(curl -s https://codecov.io/bash) + +notifications: + email: false diff --git a/github.com/pseudomuto/protokit/CHANGELOG.md b/github.com/pseudomuto/protokit/CHANGELOG.md new file mode 100644 index 0000000000..875b9f64ac --- /dev/null +++ b/github.com/pseudomuto/protokit/CHANGELOG.md @@ -0,0 +1,41 @@ +# Changelog + +All noteworthy changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) +and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). + +## [Unreleased](https://github.com/pseudomuto/protokit/compare/v0.2.0...master) + +## [0.2.0](https://github.com/pseudomuto/protokit/compare/v0.1.0...v0.2.0) + +### Added + +* Support for extended options in files, services, methods, enums, enum values, messages and fields. For example, the [`google.api.http` method option](https://cloud.google.com/service-infrastructure/docs/service-management/reference/rpc/google.api#httprule). + +## [0.1.0](https://github.com/pseudomuto/protokit/compare/v0.1.0-pre3...v0.1.0) + +### Added + +* `(Get)PackageComments` and `(Get)SyntaxComments` to `FileDescriptor` + +### Deprecated + +* `Comments` and `GetComments()` from `FileDescriptor` + +## [0.1.0-pre3](https://github.com/pseudomuto/protokit/compare/v0.1.0-pre2...v0.1.0-pre3) + +### Added + +* `FileDescriptor{}.Imported` for referencing publically imported descriptors, enums, and extensions +* `Comments{}.Get` to create empty comments (rather than `nil`) for all objects when a comment isn't defined. + +## [0.1.0-pre2](https://github.com/pseudomuto/protokit/compare/v0.1.0-pre...v0.1.0-pre2) - 2018-02-22 + +### Fixed + +* CreateGenRequest should filter for FileToGenerate using the `GetName()` rather than getting the base name + +## [0.1.0-pre](https://github.com/pseudomuto/protokit/compare/c3aa082037b33bcd713106641e88afba846d003d...v0.1.0-pre) - 2018-02-21 + +The initial alpha release of protokit. diff --git a/github.com/pseudomuto/protokit/Gopkg.lock b/github.com/pseudomuto/protokit/Gopkg.lock new file mode 100644 index 0000000000..93e7fcbe93 --- /dev/null +++ b/github.com/pseudomuto/protokit/Gopkg.lock @@ -0,0 +1,46 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + name = "github.com/davecgh/go-spew" + packages = ["spew"] + revision = "346938d642f2ec3594ed81d874461961cd0faa76" + version = "v1.1.0" + +[[projects]] + name = "github.com/golang/protobuf" + packages = [ + "proto", + "protoc-gen-go/descriptor", + "protoc-gen-go/plugin" + ] + revision = "925541529c1fa6821df4e44ce2723319eb2be768" + version = "v1.0.0" + +[[projects]] + name = "github.com/pmezard/go-difflib" + packages = ["difflib"] + revision = "792786c7400a136282c1664665ae0a8db921c6c2" + version = "v1.0.0" + +[[projects]] + name = "github.com/stretchr/testify" + packages = [ + "assert", + "require", + "suite" + ] + revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71" + version = "v1.2.1" + +[[projects]] + name = "google.golang.org/genproto" + packages = ["googleapis/api/annotations"] + revision = "86e600f69ee4704c6efbf6a2a40a5c10700e76c2" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "c73f089a26a0388685f79df3c11d454b1504724a48125fe54eb144d498aece1c" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/github.com/pseudomuto/protokit/Gopkg.toml b/github.com/pseudomuto/protokit/Gopkg.toml new file mode 100644 index 0000000000..27ba0d23c0 --- /dev/null +++ b/github.com/pseudomuto/protokit/Gopkg.toml @@ -0,0 +1,37 @@ +# Gopkg.toml example +# +# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" +# +# [prune] +# non-go = false +# go-tests = true +# unused-packages = true + +[[constraint]] + name = "github.com/golang/protobuf" + version = "^1.0.0" + +[[constraint]] + name = "github.com/stretchr/testify" + version = "^1.2.0" + +[prune] + go-tests = true + unused-packages = true diff --git a/github.com/pseudomuto/protokit/LICENSE b/github.com/pseudomuto/protokit/LICENSE new file mode 100644 index 0000000000..c88b7b4130 --- /dev/null +++ b/github.com/pseudomuto/protokit/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 David Muto (pseudomuto) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/github.com/pseudomuto/protokit/Makefile b/github.com/pseudomuto/protokit/Makefile new file mode 100644 index 0000000000..4cdf958c90 --- /dev/null +++ b/github.com/pseudomuto/protokit/Makefile @@ -0,0 +1,29 @@ +.PHONY: bench release setup test + +VERSION = $(shell cat version.go | sed -n 's/.*const Version = "\(.*\)"/\1/p') + +setup: + $(info Synching dev tools and dependencies...) + @if test -z $(which retool); then go get github.com/twitchtv/retool; fi + @retool sync + @retool do dep ensure + +fixtures/fileset.pb: fixtures/*.proto + $(info Generating fixtures...) + @cd fixtures && go generate + +bench: + go test -bench=. + +test: fixtures/fileset.pb + @go test -race -cover ./ ./utils + +test-ci: fixtures/fileset.pb bench + @retool do goverage -race -coverprofile=coverage.txt -covermode=atomic ./ ./utils + +release: + @echo Releasing v${VERSION}... + git add CHANGELOG.md version.go + git commit -m "Bump version to v${VERSION}" + git tag -m "Version ${VERSION}" "v${VERSION}" + git push && git push --tags diff --git a/github.com/pseudomuto/protokit/README.md b/github.com/pseudomuto/protokit/README.md new file mode 100644 index 0000000000..54fd787e29 --- /dev/null +++ b/github.com/pseudomuto/protokit/README.md @@ -0,0 +1,71 @@ +# protokit + +[![Travis Build Status][travis-svg]][travis-ci] +[![codecov][codecov-svg]][codecov-url] +[![GoDoc][godoc-svg]][godoc-url] +[![Go Report Card][goreport-svg]][goreport-url] + +A starter kit for building protoc-plugins. Rather than write your own, you can just use an existing one. + +See the [examples](examples/) directory for uh...examples. + +## Getting Started + +```golang +package main + +import ( + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/protoc-gen-go/plugin" + "github.com/pseudomuto/protokit" + _ "google.golang.org/genproto/googleapis/api/annotations" // Support (google.api.http) option (from google/api/annotations.proto). + + "log" +) + +func main() { + // all the heavy lifting done for you! + if err := protokit.RunPlugin(new(plugin)); err != nil { + log.Fatal(err) + } +} + +// plugin is an implementation of protokit.Plugin +type plugin struct{} + +func (p *plugin) Generate(in *plugin_go.CodeGeneratorRequest) (*plugin_go.CodeGeneratorResponse, error) { + descriptors := protokit.ParseCodeGenRequest(req) + + resp := new(plugin_go.CodeGeneratorResponse) + + for _, d := range descriptors { + // TODO: YOUR WORK HERE + fileName := // generate a file name based on d.GetName() + content := // generate content for the output file + + resp.File = append(resp.File, &plugin_go.CodeGeneratorResponse_File{ + Name: proto.String(fileName), + Content: proto.String(content), + }) + } + + return resp, nil +} +``` + +Then invoke your plugin via `protoc`. For example (assuming your app is called `thingy`): + +`protoc --plugin=protoc-gen-thingy=./thingy -I. --thingy_out=. rpc/*.proto` + +[travis-svg]: + https://travis-ci.org/pseudomuto/protokit.svg?branch=master + "Travis CI build status SVG" +[travis-ci]: + https://travis-ci.org/pseudomuto/protokit + "protoc-gen-twagger at Travis CI" +[codecov-svg]: https://codecov.io/gh/pseudomuto/protokit/branch/master/graph/badge.svg +[codecov-url]: https://codecov.io/gh/pseudomuto/protokit +[godoc-svg]: https://godoc.org/github.com/pseudomuto/protokit?status.svg +[godoc-url]: https://godoc.org/github.com/pseudomuto/protokit +[goreport-svg]: https://goreportcard.com/badge/github.com/pseudomuto/protokit +[goreport-url]: https://goreportcard.com/report/github.com/pseudomuto/protokit diff --git a/github.com/pseudomuto/protokit/comments.go b/github.com/pseudomuto/protokit/comments.go new file mode 100644 index 0000000000..5f0b9e6864 --- /dev/null +++ b/github.com/pseudomuto/protokit/comments.go @@ -0,0 +1,93 @@ +package protokit + +import ( + "github.com/golang/protobuf/protoc-gen-go/descriptor" + + "bytes" + "strconv" + "strings" +) + +// A Comment describes the leading, trailing, and detached comments for a proto object. See `SourceCodeInfo_Location` in +// descriptor.proto for details on what those terms mean +type Comment struct { + Leading string + Trailing string + Detached []string +} + +// String returns the leading and trailing comments joined by 2 line breaks (`\n\n`). If either are empty, the line +// breaks are removed. +func (c *Comment) String() string { + b := new(bytes.Buffer) + if c.GetLeading() != "" { + b.WriteString(c.GetLeading()) + b.WriteString("\n\n") + } + + b.WriteString(c.GetTrailing()) + + return strings.TrimSpace(b.String()) +} + +func newComment(loc *descriptor.SourceCodeInfo_Location) *Comment { + detached := make([]string, len(loc.GetLeadingDetachedComments())) + for i, c := range loc.GetLeadingDetachedComments() { + detached[i] = scrub(c) + } + + return &Comment{ + Leading: scrub(loc.GetLeadingComments()), + Trailing: scrub(loc.GetTrailingComments()), + Detached: detached, + } +} + +// GetLeading returns the leading comments +func (c *Comment) GetLeading() string { return c.Leading } + +// GetTrailing returns the leading comments +func (c *Comment) GetTrailing() string { return c.Trailing } + +// GetDetached returns the detached leading comments +func (c *Comment) GetDetached() []string { return c.Detached } + +// Comments is a map of source location paths to values. +type Comments map[string]*Comment + +// ParseComments parses all comments within a proto file. The locations are encoded into the map by joining the paths +// with a "." character. E.g. `4.2.3.0`. +// +// Leading/trailing spaces are trimmed for each comment type (leading, trailing, detached) +func ParseComments(fd *descriptor.FileDescriptorProto) Comments { + comments := make(Comments) + + for _, loc := range fd.GetSourceCodeInfo().GetLocation() { + if loc.GetLeadingComments() == "" && loc.GetTrailingComments() == "" && len(loc.GetLeadingDetachedComments()) == 0 { + continue + } + + path := loc.GetPath() + key := make([]string, len(path)) + for idx, p := range path { + key[idx] = strconv.Itoa(int(p)) + } + + comments[strings.Join(key, ".")] = newComment(loc) + } + + return comments +} + +func (c Comments) Get(path string) *Comment { + if val, ok := c[path]; ok { + return val + } + + // return an empty comment + return &Comment{Detached: make([]string, 0)} +} + +func scrub(str string) string { + return strings.TrimSpace(strings.Replace(str, "\n ", "\n", -1)) +} diff --git a/github.com/pseudomuto/protokit/context.go b/github.com/pseudomuto/protokit/context.go new file mode 100644 index 0000000000..a9ea0e5b55 --- /dev/null +++ b/github.com/pseudomuto/protokit/context.go @@ -0,0 +1,58 @@ +package protokit + +import ( + "context" +) + +type contextKey string + +const ( + fileContextKey = contextKey("file") + descriptorContextKey = contextKey("descriptor") + enumContextKey = contextKey("enum") + serviceContextKey = contextKey("service") +) + +// ContextWithFileDescriptor returns a new context with the attached `FileDescriptor` +func ContextWithFileDescriptor(ctx context.Context, fd *FileDescriptor) context.Context { + return context.WithValue(ctx, fileContextKey, fd) +} + +// FileDescriptorFromContext returns the `FileDescriptor` from the context and whether or not the key was found. +func FileDescriptorFromContext(ctx context.Context) (*FileDescriptor, bool) { + val, ok := ctx.Value(fileContextKey).(*FileDescriptor) + return val, ok +} + +// ContextWithDescriptor returns a new context with the specified `Descriptor` +func ContextWithDescriptor(ctx context.Context, d *Descriptor) context.Context { + return context.WithValue(ctx, descriptorContextKey, d) +} + +// DescriptorFromContext returns the associated `Descriptor` for the context and whether or not it was found +func DescriptorFromContext(ctx context.Context) (*Descriptor, bool) { + val, ok := ctx.Value(descriptorContextKey).(*Descriptor) + return val, ok +} + +// ContextWithEnumDescriptor returns a new context with the specified `EnumDescriptor` +func ContextWithEnumDescriptor(ctx context.Context, d *EnumDescriptor) context.Context { + return context.WithValue(ctx, enumContextKey, d) +} + +// EnumDescriptorFromContext returns the associated `EnumDescriptor` for the context and whether or not it was found +func EnumDescriptorFromContext(ctx context.Context) (*EnumDescriptor, bool) { + val, ok := ctx.Value(enumContextKey).(*EnumDescriptor) + return val, ok +} + +// ContextWithServiceDescriptor returns a new context with `service` +func ContextWithServiceDescriptor(ctx context.Context, service *ServiceDescriptor) context.Context { + return context.WithValue(ctx, serviceContextKey, service) +} + +// ServiceDescriptorFromContext returns the `Service` from the context and whether or not the key was found. +func ServiceDescriptorFromContext(ctx context.Context) (*ServiceDescriptor, bool) { + val, ok := ctx.Value(serviceContextKey).(*ServiceDescriptor) + return val, ok +} diff --git a/github.com/pseudomuto/protokit/doc.go b/github.com/pseudomuto/protokit/doc.go new file mode 100644 index 0000000000..19f2247b4f --- /dev/null +++ b/github.com/pseudomuto/protokit/doc.go @@ -0,0 +1,13 @@ +// Package protokit is a library that makes it easy to create your own protoc plugins. It has excellent test coverage, +// and saves you so much time! +// +// There are two main things this library provides; a parser for parsing protobuf files into some well-defined structs, +// and an abstraction to make it simple to write your own protoc plugins. +// +// Getting Started +// +// For a quick view of how to get started, see https://godoc.org/github.com/pseudomuto/protokit#example-RunPlugin +// +// If you want see/try a working example, check out the examples in +// https://github.com/pseudomuto/protokit/tree/master/examples. +package protokit diff --git a/github.com/pseudomuto/protokit/examples/jsonator/sample.proto b/github.com/pseudomuto/protokit/examples/jsonator/sample.proto new file mode 100644 index 0000000000..be9f4bd074 --- /dev/null +++ b/github.com/pseudomuto/protokit/examples/jsonator/sample.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +import "google/api/annotations.proto"; +import "google/protobuf/empty.proto"; + +// This is just a sample proto for demonstrating how to use this library. +// +// There's nothing "fancy" here. +package com.jsonator.v1; + +service SampleService { + rpc RandomSample(google.protobuf.Empty) returns (Sample) { + option (google.api.http).get = "/v1/sample"; + } +} + +message Sample { + int64 id = 1; +} diff --git a/github.com/pseudomuto/protokit/examples/jsonator/sample2.proto b/github.com/pseudomuto/protokit/examples/jsonator/sample2.proto new file mode 100644 index 0000000000..c87ae4950b --- /dev/null +++ b/github.com/pseudomuto/protokit/examples/jsonator/sample2.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +import "google/api/annotations.proto"; +import "google/protobuf/empty.proto"; + +// This is just another sample proto for demonstrating how to use this library. +// +// There's also nothing "fancy" here. +package com.jsonator.v2; + +service SampleService { + rpc RandomSample(google.protobuf.Empty) returns (Sample) { + option (google.api.http).get = "/v2/sample"; + } +} + +message Sample { + int64 id = 1; +} diff --git a/github.com/pseudomuto/protokit/fixtures/booking.proto b/github.com/pseudomuto/protokit/fixtures/booking.proto new file mode 100644 index 0000000000..38101f65e8 --- /dev/null +++ b/github.com/pseudomuto/protokit/fixtures/booking.proto @@ -0,0 +1,89 @@ +syntax = "proto2"; + +import "extend.proto"; + +/** + * Booking related messages. + * + * This file is really just an example. The data model is completely fictional. + */ +package com.pseudomuto.protokit.v1; + +option (com.pseudomuto.protokit.v1.extend_file) = true; + +/** + * Service for handling vehicle bookings. + */ +service BookingService { + option (com.pseudomuto.protokit.v1.extend_service) = true; + + /// Used to book a vehicle. Pass in a Booking and a BookingStatus will be returned. + rpc BookVehicle (Booking) returns (BookingStatus) { + option (com.pseudomuto.protokit.v1.extend_method) = true; + }; +} + +/** + * Represents the status of a vehicle booking. + */ +message BookingStatus { + /** + * A flag for the status result. + */ + enum StatusCode { + OK = 200; // OK result. + BAD_REQUEST = 400; // BAD result. + } + + required int32 id = 1; /// Unique booking status ID. + required string description = 2; /// Booking status description. E.g. "Active". + optional StatusCode status_code = 3; /// The status of this status? + + extensions 100 to max; +} + +// File-level extension +extend BookingStatus { + /* The country the booking occurred in. */ + optional string country = 100 [default = "china", (com.pseudomuto.protokit.v1.extend_field) = true]; +} + +/** + * The type of booking. + */ +enum BookingType { + option (com.pseudomuto.protokit.v1.extend_enum) = true; + + IMMEDIATE = 100; // Immediate booking. + FUTURE = 101 [(com.pseudomuto.protokit.v1.extend_enum_value) = true]; // Future booking. +} + +/** + * Represents the booking of a vehicle. + * + * Vehicles are some cool shit. But drive carefully! + */ +message Booking { + option (com.pseudomuto.protokit.v1.extend_message) = true; + + required int32 vehicle_id = 1; /// ID of booked vehicle. + required int32 customer_id = 2; /// Customer that booked the vehicle. + required BookingStatus status = 3; /// Status of the booking. + + /** Has booking confirmation been sent? */ + required bool confirmation_sent = 4; + + /** Has payment been received? */ + optional bool payment_received = 5 [default = true, (com.pseudomuto.protokit.v1.extend_field) = true]; + + oneof things { + int32 reference_num = 6; // the numeric reference number + string reference_tag = 7; // the reference tag (string) + } + + // Nested extentions are also a thing. + + extend BookingStatus { + optional string optional_field_1 = 101; // An optional field to be used however you please. + } +} diff --git a/github.com/pseudomuto/protokit/fixtures/extend.proto b/github.com/pseudomuto/protokit/fixtures/extend.proto new file mode 100644 index 0000000000..c0260241fb --- /dev/null +++ b/github.com/pseudomuto/protokit/fixtures/extend.proto @@ -0,0 +1,54 @@ +syntax = "proto2"; + +import "google/protobuf/descriptor.proto"; + +package com.pseudomuto.protokit.v1; + +/** + * Extension of protobuf file options. + */ + extend google.protobuf.FileOptions { + optional bool extend_file = 20000; +} + +/** + * Extension of protobuf service options. + */ + extend google.protobuf.ServiceOptions { + optional bool extend_service = 20000; +} + +/** + * Extension of protobuf method options. + */ +extend google.protobuf.MethodOptions { + optional bool extend_method = 20000; +} + +/** + * Extension of protobuf enum options. + */ +extend google.protobuf.EnumOptions { + optional bool extend_enum = 20000; +} + +/** + * Extension of protobuf enum value options. + */ +extend google.protobuf.EnumValueOptions { + optional bool extend_enum_value = 20000; +} + +/** + * Extension of protobuf message options. + */ +extend google.protobuf.MessageOptions { + optional bool extend_message = 20000; +} + +/** + * Extension of protobuf field options. + */ +extend google.protobuf.FieldOptions { + optional bool extend_field = 20000; +} diff --git a/github.com/pseudomuto/protokit/fixtures/todo.proto b/github.com/pseudomuto/protokit/fixtures/todo.proto new file mode 100644 index 0000000000..95a5831828 --- /dev/null +++ b/github.com/pseudomuto/protokit/fixtures/todo.proto @@ -0,0 +1,101 @@ +// Top-level comments are attached to the syntax directive. +syntax = "proto3"; + +import "google/protobuf/any.proto"; +import "google/protobuf/timestamp.proto"; +import "extend.proto"; +import public "todo_import.proto"; +option go_package = "todo"; + +// The official documentation for the Todo API. +// +// Some parts of this file are unnecessarily complicated. In order to have a test for nested messages, enums, etc. I've +// added some odd looking implementation details. So you know, don't use this in real life for a todo service. +// +// The get started run the following: +// +// * `make setup` +// * `make test` +package com.pseudomuto.protokit.v1; + +option (com.pseudomuto.protokit.v1.extend_file) = true; + +// A service for managing "todo" items. +// +// Add, complete, and remove your items on your todo lists. +service Todo { + option (com.pseudomuto.protokit.v1.extend_service) = true; + + // Create a new todo list + rpc CreateList(CreateListRequest) returns (CreateListResponse) { + option (com.pseudomuto.protokit.v1.extend_method) = true; + } + + // Add an item to your list + // + // Adds a new item to the specified list. + rpc AddItem(AddItemRequest) returns (AddItemResponse); +} + +// An enumeration of list types +enum ListType { + option (com.pseudomuto.protokit.v1.extend_enum) = true; + + REMINDERS = 0; // The reminders type. + CHECKLIST = 1 [(com.pseudomuto.protokit.v1.extend_enum_value) = true]; // The checklist type. +} + +// A list object. +message List { + option (com.pseudomuto.protokit.v1.extend_message) = true; + + int64 id = 1; // The id of the list. + string name = 2 [(com.pseudomuto.protokit.v1.extend_field) = true]; // The name of the list. + ListType type = 3; // The type of list + google.protobuf.Timestamp created_at = 4; // The timestamp for creation. + google.protobuf.Any details = 5; // Some arbitrary list details. +} + +// A request object for creating todo lists. +message CreateListRequest { + // The name of the list. + string name = 1; +} + +// A successfully created list response. +message CreateListResponse { + // An internal status message + message Status { + sint32 code = 1; // The status code. + } + + List list = 1; // The list that was created. + Status status = 2; // The status for the response. +} + +// A list item +message Item { + // An enumeration of possible statuses + enum Status { + PENDING = 0; // The pending status. + COMPLETED = 1; // The completed status. + } + + int64 id = 1; // The id of the item. + string title = 2; // The title of the item. + Status completed = 3; // The current status of the item. + google.protobuf.Timestamp created_at = 4; // The timestamp for creation. + ListItemDetails details = 5; // Item details. +} + +// A request message for adding new items. +message AddItemRequest { + int64 list_id = 1; // The id of the list to add to. + string title = 2; // The title of the item. + bool completed = 3; // Whether or not the item is completed. +} + +// A successfully added item response. +message AddItemResponse { + Item item = 1; // The list item that was added. +} diff --git a/github.com/pseudomuto/protokit/fixtures/todo_import.proto b/github.com/pseudomuto/protokit/fixtures/todo_import.proto new file mode 100644 index 0000000000..8d745e656f --- /dev/null +++ b/github.com/pseudomuto/protokit/fixtures/todo_import.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +// This is really just in place to make sure imported files are included in the parsed result. +package com.pseudomuto.protokit.v1; + +// Details for list items +message ListItemDetails { + string notes = 1; // Some notes for the item +} + +// A dummy enum to ensure importing works. +enum ListItemDetailEnum { + DEFAULT = 0; // The default value. +} diff --git a/github.com/pseudomuto/protokit/parser.go b/github.com/pseudomuto/protokit/parser.go new file mode 100644 index 0000000000..3f2c987ced --- /dev/null +++ b/github.com/pseudomuto/protokit/parser.go @@ -0,0 +1,289 @@ +package protokit + +import ( + "github.com/golang/protobuf/protoc-gen-go/descriptor" + "github.com/golang/protobuf/protoc-gen-go/plugin" + + "context" + "fmt" + "strings" +) + +const ( + // tag numbers in FileDescriptorProto + packageCommentPath = 2 + messageCommentPath = 4 + enumCommentPath = 5 + serviceCommentPath = 6 + extensionCommentPath = 7 + syntaxCommentPath = 12 + + // tag numbers in DescriptorProto + messageFieldCommentPath = 2 // field + messageMessageCommentPath = 3 // nested_type + messageEnumCommentPath = 4 // enum_type + messageExtensionCommentPath = 6 // extension + + // tag numbers in EnumDescriptorProto + enumValueCommentPath = 2 // value + + // tag numbers in ServiceDescriptorProto + serviceMethodCommentPath = 2 +) + +// ParseCodeGenRequest parses the given request into `FileDescriptor` objects. Only the `req.FilesToGenerate` will be +// returned. +// +// For example, given the following invocation, only booking.proto will be returned even if it imports other protos: +// +// protoc --plugin=protoc-gen-test=./test -I. protos/booking.proto +func ParseCodeGenRequest(req *plugin_go.CodeGeneratorRequest) []*FileDescriptor { + allFiles := make(map[string]*FileDescriptor) + genFiles := make([]*FileDescriptor, len(req.GetFileToGenerate())) + + for _, pf := range req.GetProtoFile() { + allFiles[pf.GetName()] = parseFile(context.Background(), pf) + } + + for i, f := range req.GetFileToGenerate() { + genFiles[i] = allFiles[f] + parseImports(genFiles[i], allFiles) + } + + return genFiles +} + +func parseFile(ctx context.Context, fd *descriptor.FileDescriptorProto) *FileDescriptor { + comments := ParseComments(fd) + + file := &FileDescriptor{ + comments: comments, + FileDescriptorProto: fd, + Comments: comments.Get(fmt.Sprintf("%d", packageCommentPath)), + PackageComments: comments.Get(fmt.Sprintf("%d", packageCommentPath)), + SyntaxComments: comments.Get(fmt.Sprintf("%d", syntaxCommentPath)), + } + + if fd.Options != nil { + file.setOptions(fd.Options) + } + + fileCtx := ContextWithFileDescriptor(ctx, file) + file.Enums = parseEnums(fileCtx, fd.GetEnumType()) + file.Extensions = parseExtensions(fileCtx, fd.GetExtension()) + file.Messages = parseMessages(fileCtx, fd.GetMessageType()) + file.Services = parseServices(fileCtx, fd.GetService()) + + return file +} + +func parseEnums(ctx context.Context, protos []*descriptor.EnumDescriptorProto) []*EnumDescriptor { + enums := make([]*EnumDescriptor, len(protos)) + file, _ := FileDescriptorFromContext(ctx) + parent, hasParent := DescriptorFromContext(ctx) + + for i, ed := range protos { + longName := ed.GetName() + commentPath := fmt.Sprintf("%d.%d", enumCommentPath, i) + + if hasParent { + longName = fmt.Sprintf("%s.%s", parent.GetLongName(), longName) + commentPath = fmt.Sprintf("%s.%d.%d", parent.path, messageEnumCommentPath, i) + } + + enums[i] = &EnumDescriptor{ + common: newCommon(file, commentPath, longName), + EnumDescriptorProto: ed, + Comments: file.comments.Get(commentPath), + Parent: parent, + } + if ed.Options != nil { + enums[i].setOptions(ed.Options) + } + + subCtx := ContextWithEnumDescriptor(ctx, enums[i]) + enums[i].Values = parseEnumValues(subCtx, ed.GetValue()) + } + + return enums +} + +func parseEnumValues(ctx context.Context, protos []*descriptor.EnumValueDescriptorProto) []*EnumValueDescriptor { + values := make([]*EnumValueDescriptor, len(protos)) + file, _ := FileDescriptorFromContext(ctx) + enum, _ := EnumDescriptorFromContext(ctx) + + for i, vd := range protos { + longName := fmt.Sprintf("%s.%s", enum.GetLongName(), vd.GetName()) + + values[i] = &EnumValueDescriptor{ + common: newCommon(file, "", longName), + EnumValueDescriptorProto: vd, + Enum: enum, + Comments: file.comments.Get(fmt.Sprintf("%s.%d.%d", enum.path, enumValueCommentPath, i)), + } + if vd.Options != nil { + values[i].setOptions(vd.Options) + } + } + + return values +} + +func parseExtensions(ctx context.Context, protos []*descriptor.FieldDescriptorProto) []*ExtensionDescriptor { + exts := make([]*ExtensionDescriptor, len(protos)) + file, _ := FileDescriptorFromContext(ctx) + parent, hasParent := DescriptorFromContext(ctx) + + for i, ext := range protos { + commentPath := fmt.Sprintf("%d.%d", extensionCommentPath, i) + longName := fmt.Sprintf("%s.%s", ext.GetExtendee(), ext.GetName()) + + if strings.Contains(longName, file.GetPackage()) { + parts := strings.Split(ext.GetExtendee(), ".") + longName = fmt.Sprintf("%s.%s", parts[len(parts)-1], ext.GetName()) + } + + if hasParent { + commentPath = fmt.Sprintf("%s.%d.%d", parent.path, messageExtensionCommentPath, i) + } + + exts[i] = &ExtensionDescriptor{ + common: newCommon(file, commentPath, longName), + FieldDescriptorProto: ext, + Comments: file.comments.Get(commentPath), + Parent: parent, + } + if ext.Options != nil { + exts[i].setOptions(ext.Options) + } + } + + return exts +} + +func parseImports(fd *FileDescriptor, allFiles map[string]*FileDescriptor) { + fd.Imports = make([]*ImportedDescriptor, 0) + + for _, index := range fd.GetPublicDependency() { + file := allFiles[fd.GetDependency()[index]] + + for _, d := range file.GetMessages() { + // skip map entry objects + if !d.GetOptions().GetMapEntry() { + fd.Imports = append(fd.Imports, &ImportedDescriptor{d.common}) + } + } + + for _, e := range file.GetEnums() { + fd.Imports = append(fd.Imports, &ImportedDescriptor{e.common}) + } + + for _, ext := range file.GetExtensions() { + fd.Imports = append(fd.Imports, &ImportedDescriptor{ext.common}) + } + } +} + +func parseMessages(ctx context.Context, protos []*descriptor.DescriptorProto) []*Descriptor { + msgs := make([]*Descriptor, len(protos)) + file, _ := FileDescriptorFromContext(ctx) + parent, hasParent := DescriptorFromContext(ctx) + + for i, md := range protos { + longName := md.GetName() + commentPath := fmt.Sprintf("%d.%d", messageCommentPath, i) + + if hasParent { + longName = fmt.Sprintf("%s.%s", parent.GetLongName(), longName) + commentPath = fmt.Sprintf("%s.%d.%d", parent.path, messageMessageCommentPath, i) + } + + msgs[i] = &Descriptor{ + common: newCommon(file, commentPath, longName), + DescriptorProto: md, + Comments: file.comments.Get(commentPath), + Parent: parent, + } + if md.Options != nil { + msgs[i].setOptions(md.Options) + } + + msgCtx := ContextWithDescriptor(ctx, msgs[i]) + msgs[i].Enums = parseEnums(msgCtx, md.GetEnumType()) + msgs[i].Extensions = parseExtensions(msgCtx, md.GetExtension()) + msgs[i].Fields = parseMessageFields(msgCtx, md.GetField()) + msgs[i].Messages = parseMessages(msgCtx, md.GetNestedType()) + } + + return msgs +} + +func parseMessageFields(ctx context.Context, protos []*descriptor.FieldDescriptorProto) []*FieldDescriptor { + fields := make([]*FieldDescriptor, len(protos)) + file, _ := FileDescriptorFromContext(ctx) + message, _ := DescriptorFromContext(ctx) + + for i, fd := range protos { + longName := fmt.Sprintf("%s.%s", message.GetLongName(), fd.GetName()) + + fields[i] = &FieldDescriptor{ + common: newCommon(file, "", longName), + FieldDescriptorProto: fd, + Comments: file.comments.Get(fmt.Sprintf("%s.%d.%d", message.path, messageFieldCommentPath, i)), + Message: message, + } + if fd.Options != nil { + fields[i].setOptions(fd.Options) + } + } + + return fields +} + +func parseServices(ctx context.Context, protos []*descriptor.ServiceDescriptorProto) []*ServiceDescriptor { + svcs := make([]*ServiceDescriptor, len(protos)) + file, _ := FileDescriptorFromContext(ctx) + + for i, sd := range protos { + longName := sd.GetName() + commentPath := fmt.Sprintf("%d.%d", serviceCommentPath, i) + + svcs[i] = &ServiceDescriptor{ + common: newCommon(file, commentPath, longName), + ServiceDescriptorProto: sd, + Comments: file.comments.Get(commentPath), + } + if sd.Options != nil { + svcs[i].setOptions(sd.Options) + } + + svcCtx := ContextWithServiceDescriptor(ctx, svcs[i]) + svcs[i].Methods = parseServiceMethods(svcCtx, sd.GetMethod()) + } + + return svcs +} + +func parseServiceMethods(ctx context.Context, protos []*descriptor.MethodDescriptorProto) []*MethodDescriptor { + methods := make([]*MethodDescriptor, len(protos)) + + file, _ := FileDescriptorFromContext(ctx) + svc, _ := ServiceDescriptorFromContext(ctx) + + for i, md := range protos { + longName := fmt.Sprintf("%s.%s", svc.GetLongName(), md.GetName()) + + methods[i] = &MethodDescriptor{ + common: newCommon(file, "", longName), + MethodDescriptorProto: md, + Service: svc, + Comments: file.comments.Get(fmt.Sprintf("%s.%d.%d", svc.path, serviceMethodCommentPath, i)), + } + if md.Options != nil { + methods[i].setOptions(md.Options) + } + } + + return methods +} diff --git a/github.com/pseudomuto/protokit/plugin.go b/github.com/pseudomuto/protokit/plugin.go new file mode 100644 index 0000000000..f68684bace --- /dev/null +++ b/github.com/pseudomuto/protokit/plugin.go @@ -0,0 +1,67 @@ +package protokit + +import ( + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/protoc-gen-go/plugin" + + "fmt" + "io" + "io/ioutil" + "os" +) + +// Plugin describes an interface for running protoc code generator plugins +type Plugin interface { + Generate(req *plugin_go.CodeGeneratorRequest) (*plugin_go.CodeGeneratorResponse, error) +} + +// RunPlugin runs the supplied plugin by reading input from stdin and generating output to stdout. +func RunPlugin(p Plugin) error { + return RunPluginWithIO(p, os.Stdin, os.Stdout) +} + +// RunPluginWithIO runs the supplied plugin using the supplied reader and writer for IO. +func RunPluginWithIO(p Plugin, r io.Reader, w io.Writer) error { + req, err := readRequest(r) + if err != nil { + return err + } + + resp, err := p.Generate(req) + if err != nil { + return err + } + + return writeResponse(w, resp) +} + +func readRequest(r io.Reader) (*plugin_go.CodeGeneratorRequest, error) { + data, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + + req := new(plugin_go.CodeGeneratorRequest) + if err = proto.Unmarshal(data, req); err != nil { + return nil, err + } + + if len(req.GetFileToGenerate()) == 0 { + return nil, fmt.Errorf("no files were supplied to the generator") + } + + return req, nil +} + +func writeResponse(w io.Writer, resp *plugin_go.CodeGeneratorResponse) error { + data, err := proto.Marshal(resp) + if err != nil { + return err + } + + if _, err := w.Write(data); err != nil { + return err + } + + return nil +} diff --git a/github.com/pseudomuto/protokit/tools.json b/github.com/pseudomuto/protokit/tools.json new file mode 100644 index 0000000000..6047e87a16 --- /dev/null +++ b/github.com/pseudomuto/protokit/tools.json @@ -0,0 +1,13 @@ +{ + "Tools": [ + { + "Repository": "github.com/golang/dep/cmd/dep", + "Commit": "37d9ea0ac16f0e0a05afc3b60e1ac8c364b6c329" + }, + { + "Repository": "github.com/haya14busa/goverage", + "Commit": "eec3514a20b55dca311300eeb08fe5c8ab9ab781" + } + ], + "RetoolVersion": "1.3.5" +} \ No newline at end of file diff --git a/github.com/pseudomuto/protokit/types.go b/github.com/pseudomuto/protokit/types.go new file mode 100644 index 0000000000..a9ecd93c65 --- /dev/null +++ b/github.com/pseudomuto/protokit/types.go @@ -0,0 +1,350 @@ +package protokit + +import ( + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/protoc-gen-go/descriptor" + + "fmt" + "strings" +) + +type common struct { + file *FileDescriptor + path string + LongName string + FullName string + + OptionExtensions map[string]interface{} +} + +func newCommon(f *FileDescriptor, path, longName string) common { + fn := longName + if !strings.HasPrefix(fn, ".") { + fn = fmt.Sprintf("%s.%s", f.GetPackage(), longName) + } + + return common{ + file: f, + path: path, + LongName: longName, + FullName: fn, + } +} + +// GetFile returns the FileDescriptor that contains this object +func (c *common) GetFile() *FileDescriptor { return c.file } + +// GetPackage returns the package this object is in +func (c *common) GetPackage() string { return c.file.GetPackage() } + +// GetLongName returns the name prefixed with the dot-separated parent descriptor's name (if any) +func (c *common) GetLongName() string { return c.LongName } + +// GetFullName returns the `LongName` prefixed with the package this object is in +func (c *common) GetFullName() string { return c.FullName } + +// IsProto3 returns whether or not this is a proto3 object +func (c *common) IsProto3() bool { return c.file.GetSyntax() == "proto3" } + +func getOptions(options proto.Message) (m map[string]interface{}) { + for _, extension := range proto.RegisteredExtensions(options) { + if !proto.HasExtension(options, extension) { + continue + } + ext, err := proto.GetExtension(options, extension) + if err != nil { + continue + } + if m == nil { + m = make(map[string]interface{}) + } + m[extension.Name] = ext + } + return m +} + +func (c *common) setOptions(options proto.Message) { + if opts := getOptions(options); len(opts) > 0 { + if c.OptionExtensions == nil { + c.OptionExtensions = opts + return + } + for k, v := range opts { + c.OptionExtensions[k] = v + } + } +} + +// An ImportedDescriptor describes a type that was imported by a FileDescriptor. +type ImportedDescriptor struct { + common +} + +// A FileDescriptor describes a single proto file with all of its messages, enums, services, etc. +type FileDescriptor struct { + comments Comments + *descriptor.FileDescriptorProto + + Comments *Comment // Deprecated: see PackageComments + PackageComments *Comment + SyntaxComments *Comment + + Enums []*EnumDescriptor + Extensions []*ExtensionDescriptor + Imports []*ImportedDescriptor + Messages []*Descriptor + Services []*ServiceDescriptor + + OptionExtensions map[string]interface{} +} + +// IsProto3 returns whether or not this file is a proto3 file +func (f *FileDescriptor) IsProto3() bool { return f.GetSyntax() == "proto3" } + +// GetComments returns the file's package comments. +// +// Deprecated: please see GetPackageComments +func (f *FileDescriptor) GetComments() *Comment { return f.Comments } + +// GetPackageComments returns the file's package comments +func (f *FileDescriptor) GetPackageComments() *Comment { return f.PackageComments } + +// GetSyntaxComments returns the file's syntax comments +func (f *FileDescriptor) GetSyntaxComments() *Comment { return f.SyntaxComments } + +// GetEnums returns the top-level enumerations defined in this file +func (f *FileDescriptor) GetEnums() []*EnumDescriptor { return f.Enums } + +// GetExtensions returns the top-level (file) extensions defined in this file +func (f *FileDescriptor) GetExtensions() []*ExtensionDescriptor { return f.Extensions } + +// GetImports returns the proto files imported by this file +func (f *FileDescriptor) GetImports() []*ImportedDescriptor { return f.Imports } + +// GetMessages returns the top-level messages defined in this file +func (f *FileDescriptor) GetMessages() []*Descriptor { return f.Messages } + +// GetServices returns the services defined in this file +func (f *FileDescriptor) GetServices() []*ServiceDescriptor { return f.Services } + +// GetEnum returns the enumeration with the specified name (returns `nil` if not found) +func (f *FileDescriptor) GetEnum(name string) *EnumDescriptor { + for _, e := range f.GetEnums() { + if e.GetName() == name || e.GetLongName() == name { + return e + } + } + + return nil +} + +// GetMessage returns the message with the specified name (returns `nil` if not found) +func (f *FileDescriptor) GetMessage(name string) *Descriptor { + for _, m := range f.GetMessages() { + if m.GetName() == name || m.GetLongName() == name { + return m + } + } + + return nil +} + +// GetService returns the service with the specified name (returns `nil` if not found) +func (f *FileDescriptor) GetService(name string) *ServiceDescriptor { + for _, s := range f.GetServices() { + if s.GetName() == name || s.GetLongName() == name { + return s + } + } + + return nil +} + +func (f *FileDescriptor) setOptions(options proto.Message) { + if opts := getOptions(options); len(opts) > 0 { + if f.OptionExtensions == nil { + f.OptionExtensions = opts + return + } + for k, v := range opts { + f.OptionExtensions[k] = v + } + } +} + +// An EnumDescriptor describe an enum type +type EnumDescriptor struct { + common + *descriptor.EnumDescriptorProto + Parent *Descriptor + Values []*EnumValueDescriptor + Comments *Comment +} + +// GetComments returns a description of this enum +func (e *EnumDescriptor) GetComments() *Comment { return e.Comments } + +// GetParent returns the parent message (if any) that contains this enum +func (e *EnumDescriptor) GetParent() *Descriptor { return e.Parent } + +// GetValues returns the available values for this enum +func (e *EnumDescriptor) GetValues() []*EnumValueDescriptor { return e.Values } + +// GetNamedValue returns the value with the specified name (returns `nil` if not found) +func (e *EnumDescriptor) GetNamedValue(name string) *EnumValueDescriptor { + for _, v := range e.GetValues() { + if v.GetName() == name { + return v + } + } + + return nil +} + +// An EnumValueDescriptor describes an enum value +type EnumValueDescriptor struct { + common + *descriptor.EnumValueDescriptorProto + Enum *EnumDescriptor + Comments *Comment +} + +// GetComments returns a description of the value +func (v *EnumValueDescriptor) GetComments() *Comment { return v.Comments } + +// GetEnum returns the parent enumeration that contains this value +func (v *EnumValueDescriptor) GetEnum() *EnumDescriptor { return v.Enum } + +// An ExtensionDescriptor describes a protobuf extension. If it's a top-level extension it's parent will be `nil` +type ExtensionDescriptor struct { + common + *descriptor.FieldDescriptorProto + Parent *Descriptor + Comments *Comment +} + +// GetComments returns a description of the extension +func (e *ExtensionDescriptor) GetComments() *Comment { return e.Comments } + +// GetParent returns the descriptor that defined this extension (if any) +func (e *ExtensionDescriptor) GetParent() *Descriptor { return e.Parent } + +// A Descriptor describes a message +type Descriptor struct { + common + *descriptor.DescriptorProto + Parent *Descriptor + Comments *Comment + Enums []*EnumDescriptor + Extensions []*ExtensionDescriptor + Fields []*FieldDescriptor + Messages []*Descriptor +} + +// GetComments returns a description of the message +func (m *Descriptor) GetComments() *Comment { return m.Comments } + +// GetParent returns the parent descriptor (if any) that defines this descriptor +func (m *Descriptor) GetParent() *Descriptor { return m.Parent } + +// GetEnums returns the nested enumerations within the message +func (m *Descriptor) GetEnums() []*EnumDescriptor { return m.Enums } + +// GetExtensions returns the message-level extensions defined by this message +func (m *Descriptor) GetExtensions() []*ExtensionDescriptor { return m.Extensions } + +// GetMessages returns the nested messages within the message +func (m *Descriptor) GetMessages() []*Descriptor { return m.Messages } + +// GetMessageFields returns the message fields +func (m *Descriptor) GetMessageFields() []*FieldDescriptor { return m.Fields } + +// GetEnum returns the enum with the specified name. The name can be either simple, or fully qualified (returns `nil` if +// not found) +func (m *Descriptor) GetEnum(name string) *EnumDescriptor { + for _, e := range m.GetEnums() { + // can lookup by name or message prefixed name (qualified) + if e.GetName() == name || e.GetLongName() == name { + return e + } + } + + return nil +} + +// GetMessage returns the nested message with the specified name. The name can be simple or fully qualified (returns +// `nil` if not found) +func (m *Descriptor) GetMessage(name string) *Descriptor { + for _, msg := range m.GetMessages() { + // can lookup by name or message prefixed name (qualified) + if msg.GetName() == name || msg.GetLongName() == name { + return msg + } + } + + return nil +} + +// GetMessageField returns the field with the specified name (returns `nil` if not found) +func (m *Descriptor) GetMessageField(name string) *FieldDescriptor { + for _, f := range m.GetMessageFields() { + if f.GetName() == name || f.GetLongName() == name { + return f + } + } + + return nil +} + +// A FieldDescriptor describes a message field +type FieldDescriptor struct { + common + *descriptor.FieldDescriptorProto + Comments *Comment + Message *Descriptor +} + +// GetComments returns a description of the field +func (mf *FieldDescriptor) GetComments() *Comment { return mf.Comments } + +// GetMessage returns the descriptor that defines this field +func (mf *FieldDescriptor) GetMessage() *Descriptor { return mf.Message } + +// A ServiceDescriptor describes a service +type ServiceDescriptor struct { + common + *descriptor.ServiceDescriptorProto + Comments *Comment + Methods []*MethodDescriptor +} + +// GetComments returns a description of the service +func (s *ServiceDescriptor) GetComments() *Comment { return s.Comments } + +// GetMethods returns the methods for the service +func (s *ServiceDescriptor) GetMethods() []*MethodDescriptor { return s.Methods } + +// GetNamedMethod returns the method with the specified name (if found) +func (s *ServiceDescriptor) GetNamedMethod(name string) *MethodDescriptor { + for _, m := range s.GetMethods() { + if m.GetName() == name || m.GetLongName() == name { + return m + } + } + + return nil +} + +// A MethodDescriptor describes a method in a service +type MethodDescriptor struct { + common + *descriptor.MethodDescriptorProto + Comments *Comment + Service *ServiceDescriptor +} + +// GetComments returns a description of the method +func (m *MethodDescriptor) GetComments() *Comment { return m.Comments } + +// GetService returns the service descriptor that defines this method +func (m *MethodDescriptor) GetService() *ServiceDescriptor { return m.Service } diff --git a/github.com/pseudomuto/protokit/version.go b/github.com/pseudomuto/protokit/version.go new file mode 100644 index 0000000000..cf6b6a2c31 --- /dev/null +++ b/github.com/pseudomuto/protokit/version.go @@ -0,0 +1,4 @@ +package protokit + +// Version describes the current version of protokit being used +const Version = "0.2.0" diff --git a/golang.org/x/crypto/scrypt/scrypt.go b/golang.org/x/crypto/scrypt/scrypt.go new file mode 100644 index 0000000000..2f81fe4148 --- /dev/null +++ b/golang.org/x/crypto/scrypt/scrypt.go @@ -0,0 +1,213 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package scrypt implements the scrypt key derivation function as defined in +// Colin Percival's paper "Stronger Key Derivation via Sequential Memory-Hard +// Functions" (https://www.tarsnap.com/scrypt/scrypt.pdf). +package scrypt // import "golang.org/x/crypto/scrypt" + +import ( + "crypto/sha256" + "errors" + "math/bits" + + "golang.org/x/crypto/pbkdf2" +) + +const maxInt = int(^uint(0) >> 1) + +// blockCopy copies n numbers from src into dst. +func blockCopy(dst, src []uint32, n int) { + copy(dst, src[:n]) +} + +// blockXOR XORs numbers from dst with n numbers from src. +func blockXOR(dst, src []uint32, n int) { + for i, v := range src[:n] { + dst[i] ^= v + } +} + +// salsaXOR applies Salsa20/8 to the XOR of 16 numbers from tmp and in, +// and puts the result into both tmp and out. +func salsaXOR(tmp *[16]uint32, in, out []uint32) { + w0 := tmp[0] ^ in[0] + w1 := tmp[1] ^ in[1] + w2 := tmp[2] ^ in[2] + w3 := tmp[3] ^ in[3] + w4 := tmp[4] ^ in[4] + w5 := tmp[5] ^ in[5] + w6 := tmp[6] ^ in[6] + w7 := tmp[7] ^ in[7] + w8 := tmp[8] ^ in[8] + w9 := tmp[9] ^ in[9] + w10 := tmp[10] ^ in[10] + w11 := tmp[11] ^ in[11] + w12 := tmp[12] ^ in[12] + w13 := tmp[13] ^ in[13] + w14 := tmp[14] ^ in[14] + w15 := tmp[15] ^ in[15] + + x0, x1, x2, x3, x4, x5, x6, x7, x8 := w0, w1, w2, w3, w4, w5, w6, w7, w8 + x9, x10, x11, x12, x13, x14, x15 := w9, w10, w11, w12, w13, w14, w15 + + for i := 0; i < 8; i += 2 { + x4 ^= bits.RotateLeft32(x0+x12, 7) + x8 ^= bits.RotateLeft32(x4+x0, 9) + x12 ^= bits.RotateLeft32(x8+x4, 13) + x0 ^= bits.RotateLeft32(x12+x8, 18) + + x9 ^= bits.RotateLeft32(x5+x1, 7) + x13 ^= bits.RotateLeft32(x9+x5, 9) + x1 ^= bits.RotateLeft32(x13+x9, 13) + x5 ^= bits.RotateLeft32(x1+x13, 18) + + x14 ^= bits.RotateLeft32(x10+x6, 7) + x2 ^= bits.RotateLeft32(x14+x10, 9) + x6 ^= bits.RotateLeft32(x2+x14, 13) + x10 ^= bits.RotateLeft32(x6+x2, 18) + + x3 ^= bits.RotateLeft32(x15+x11, 7) + x7 ^= bits.RotateLeft32(x3+x15, 9) + x11 ^= bits.RotateLeft32(x7+x3, 13) + x15 ^= bits.RotateLeft32(x11+x7, 18) + + x1 ^= bits.RotateLeft32(x0+x3, 7) + x2 ^= bits.RotateLeft32(x1+x0, 9) + x3 ^= bits.RotateLeft32(x2+x1, 13) + x0 ^= bits.RotateLeft32(x3+x2, 18) + + x6 ^= bits.RotateLeft32(x5+x4, 7) + x7 ^= bits.RotateLeft32(x6+x5, 9) + x4 ^= bits.RotateLeft32(x7+x6, 13) + x5 ^= bits.RotateLeft32(x4+x7, 18) + + x11 ^= bits.RotateLeft32(x10+x9, 7) + x8 ^= bits.RotateLeft32(x11+x10, 9) + x9 ^= bits.RotateLeft32(x8+x11, 13) + x10 ^= bits.RotateLeft32(x9+x8, 18) + + x12 ^= bits.RotateLeft32(x15+x14, 7) + x13 ^= bits.RotateLeft32(x12+x15, 9) + x14 ^= bits.RotateLeft32(x13+x12, 13) + x15 ^= bits.RotateLeft32(x14+x13, 18) + } + x0 += w0 + x1 += w1 + x2 += w2 + x3 += w3 + x4 += w4 + x5 += w5 + x6 += w6 + x7 += w7 + x8 += w8 + x9 += w9 + x10 += w10 + x11 += w11 + x12 += w12 + x13 += w13 + x14 += w14 + x15 += w15 + + out[0], tmp[0] = x0, x0 + out[1], tmp[1] = x1, x1 + out[2], tmp[2] = x2, x2 + out[3], tmp[3] = x3, x3 + out[4], tmp[4] = x4, x4 + out[5], tmp[5] = x5, x5 + out[6], tmp[6] = x6, x6 + out[7], tmp[7] = x7, x7 + out[8], tmp[8] = x8, x8 + out[9], tmp[9] = x9, x9 + out[10], tmp[10] = x10, x10 + out[11], tmp[11] = x11, x11 + out[12], tmp[12] = x12, x12 + out[13], tmp[13] = x13, x13 + out[14], tmp[14] = x14, x14 + out[15], tmp[15] = x15, x15 +} + +func blockMix(tmp *[16]uint32, in, out []uint32, r int) { + blockCopy(tmp[:], in[(2*r-1)*16:], 16) + for i := 0; i < 2*r; i += 2 { + salsaXOR(tmp, in[i*16:], out[i*8:]) + salsaXOR(tmp, in[i*16+16:], out[i*8+r*16:]) + } +} + +func integer(b []uint32, r int) uint64 { + j := (2*r - 1) * 16 + return uint64(b[j]) | uint64(b[j+1])<<32 +} + +func smix(b []byte, r, N int, v, xy []uint32) { + var tmp [16]uint32 + x := xy + y := xy[32*r:] + + j := 0 + for i := 0; i < 32*r; i++ { + x[i] = uint32(b[j]) | uint32(b[j+1])<<8 | uint32(b[j+2])<<16 | uint32(b[j+3])<<24 + j += 4 + } + for i := 0; i < N; i += 2 { + blockCopy(v[i*(32*r):], x, 32*r) + blockMix(&tmp, x, y, r) + + blockCopy(v[(i+1)*(32*r):], y, 32*r) + blockMix(&tmp, y, x, r) + } + for i := 0; i < N; i += 2 { + j := int(integer(x, r) & uint64(N-1)) + blockXOR(x, v[j*(32*r):], 32*r) + blockMix(&tmp, x, y, r) + + j = int(integer(y, r) & uint64(N-1)) + blockXOR(y, v[j*(32*r):], 32*r) + blockMix(&tmp, y, x, r) + } + j = 0 + for _, v := range x[:32*r] { + b[j+0] = byte(v >> 0) + b[j+1] = byte(v >> 8) + b[j+2] = byte(v >> 16) + b[j+3] = byte(v >> 24) + j += 4 + } +} + +// Key derives a key from the password, salt, and cost parameters, returning +// a byte slice of length keyLen that can be used as cryptographic key. +// +// N is a CPU/memory cost parameter, which must be a power of two greater than 1. +// r and p must satisfy r * p < 2³â°. If the parameters do not satisfy the +// limits, the function returns a nil byte slice and an error. +// +// For example, you can get a derived key for e.g. AES-256 (which needs a +// 32-byte key) by doing: +// +// dk, err := scrypt.Key([]byte("some password"), salt, 32768, 8, 1, 32) +// +// The recommended parameters for interactive logins as of 2017 are N=32768, r=8 +// and p=1. The parameters N, r, and p should be increased as memory latency and +// CPU parallelism increases; consider setting N to the highest power of 2 you +// can derive within 100 milliseconds. Remember to get a good random salt. +func Key(password, salt []byte, N, r, p, keyLen int) ([]byte, error) { + if N <= 1 || N&(N-1) != 0 { + return nil, errors.New("scrypt: N must be > 1 and a power of 2") + } + if uint64(r)*uint64(p) >= 1<<30 || r > maxInt/128/p || r > maxInt/256 || N > maxInt/128/r { + return nil, errors.New("scrypt: parameters are too large") + } + + xy := make([]uint32, 64*r) + v := make([]uint32, 32*N*r) + b := pbkdf2.Key(password, salt, 1, p*128*r, sha256.New) + + for i := 0; i < p; i++ { + smix(b[i*128*r:], r, N, v, xy) + } + + return pbkdf2.Key(password, b, 1, keyLen, sha256.New), nil +} diff --git a/modules.txt b/modules.txt index 14d7d9a89e..5067fbfdb3 100644 --- a/modules.txt +++ b/modules.txt @@ -55,6 +55,12 @@ github.com/BurntSushi/toml # github.com/DataDog/zstd v1.4.4 ## explicit github.com/DataDog/zstd +# github.com/Masterminds/goutils v1.1.0 +github.com/Masterminds/goutils +# github.com/Masterminds/semver v1.5.0 +github.com/Masterminds/semver +# github.com/Masterminds/sprig v2.22.0+incompatible +github.com/Masterminds/sprig # github.com/MichaelTJones/walk v0.0.0-20161122175330-4748e29d5718 ## explicit github.com/MichaelTJones/walk @@ -353,6 +359,8 @@ github.com/elastic/gosigar/sys/windows # github.com/elazarl/go-bindata-assetfs v1.0.0 ## explicit github.com/elazarl/go-bindata-assetfs +# github.com/envoyproxy/protoc-gen-validate v0.1.0 +github.com/envoyproxy/protoc-gen-validate/validate # github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a ## explicit github.com/facebookgo/clock @@ -481,6 +489,8 @@ github.com/google/pprof/profile github.com/google/pprof/third_party/d3 github.com/google/pprof/third_party/d3flamegraph github.com/google/pprof/third_party/svgpan +# github.com/google/uuid v1.1.1 +github.com/google/uuid # github.com/googleapis/gax-go v2.0.2+incompatible ## explicit github.com/googleapis/gax-go @@ -511,11 +521,15 @@ github.com/grpc-ecosystem/grpc-gateway/utilities github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc # github.com/hashicorp/go-uuid v1.0.2 github.com/hashicorp/go-uuid +# github.com/huandu/xstrings v1.3.0 +github.com/huandu/xstrings # github.com/ianlancetaylor/cgosymbolizer v0.0.0-20170921033129-f5072df9c550 ## explicit github.com/ianlancetaylor/cgosymbolizer # github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 github.com/ianlancetaylor/demangle +# github.com/imdario/mergo v0.3.9 +github.com/imdario/mergo # github.com/inconshreveable/mousetrap v1.0.0 github.com/inconshreveable/mousetrap # github.com/jackc/chunkreader/v2 v2.0.1 @@ -691,6 +705,8 @@ github.com/mibk/dupl/output github.com/mibk/dupl/suffixtree github.com/mibk/dupl/syntax github.com/mibk/dupl/syntax/golang +# github.com/mitchellh/copystructure v1.0.0 +github.com/mitchellh/copystructure # github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/go-homedir # github.com/mitchellh/reflectwalk v1.0.0 @@ -703,6 +719,8 @@ github.com/mmatczuk/go_generics/globals # github.com/montanaflynn/stats v0.4.1-0.20190115100425-713f2944833c ## explicit github.com/montanaflynn/stats +# github.com/mwitkow/go-proto-validators v0.0.0-20180403085117-0950a7990007 +github.com/mwitkow/go-proto-validators # github.com/nlopes/slack v0.4.0 ## explicit github.com/nlopes/slack @@ -773,6 +791,17 @@ github.com/prometheus/common/model github.com/prometheus/procfs github.com/prometheus/procfs/internal/fs github.com/prometheus/procfs/internal/util +# github.com/pseudomuto/protoc-gen-doc v1.3.2 +## explicit +github.com/pseudomuto/protoc-gen-doc +github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc +github.com/pseudomuto/protoc-gen-doc/extensions +github.com/pseudomuto/protoc-gen-doc/extensions/envoyproxy_validate +github.com/pseudomuto/protoc-gen-doc/extensions/google_api_http +github.com/pseudomuto/protoc-gen-doc/extensions/lyft_validate +github.com/pseudomuto/protoc-gen-doc/extensions/validator_field +# github.com/pseudomuto/protokit v0.2.0 +github.com/pseudomuto/protokit # github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 ## explicit github.com/rcrowley/go-metrics @@ -866,6 +895,7 @@ golang.org/x/crypto/pbkdf2 golang.org/x/crypto/pkcs12 golang.org/x/crypto/pkcs12/internal/rc2 golang.org/x/crypto/poly1305 +golang.org/x/crypto/scrypt golang.org/x/crypto/ssh golang.org/x/crypto/ssh/agent golang.org/x/crypto/ssh/internal/bcrypt_pbkdf From 48dac659f42ce6d7176148092968a915fade0aa2 Mon Sep 17 00:00:00 2001 From: Nathan VanBenschoten Date: Tue, 4 Aug 2020 10:21:54 -0400 Subject: [PATCH 21/49] downgrade to go1.13 --- modules.txt | 180 ---------------------------------------------------- 1 file changed, 180 deletions(-) diff --git a/modules.txt b/modules.txt index 5067fbfdb3..007c40af54 100644 --- a/modules.txt +++ b/modules.txt @@ -1,5 +1,4 @@ # cloud.google.com/go v0.34.0 -## explicit cloud.google.com/go/compute/metadata cloud.google.com/go/iam cloud.google.com/go/internal @@ -10,7 +9,6 @@ cloud.google.com/go/storage # github.com/Azure/azure-pipeline-go v0.1.8 github.com/Azure/azure-pipeline-go/pipeline # github.com/Azure/azure-sdk-for-go v33.4.0+incompatible -## explicit github.com/Azure/azure-sdk-for-go/profiles/latest/compute/mgmt/compute github.com/Azure/azure-sdk-for-go/profiles/latest/network/mgmt/network github.com/Azure/azure-sdk-for-go/profiles/latest/resources/mgmt/resources @@ -21,39 +19,32 @@ github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2018-06-01/subscriptio github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-05-01/resources github.com/Azure/azure-sdk-for-go/version # github.com/Azure/azure-storage-blob-go v0.0.0-20190104215108-45d0c5e3638e -## explicit github.com/Azure/azure-storage-blob-go/azblob # github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 github.com/Azure/go-ansiterm github.com/Azure/go-ansiterm/winterm # github.com/Azure/go-autorest/autorest v0.10.2 -## explicit github.com/Azure/go-autorest/autorest github.com/Azure/go-autorest/autorest/azure # github.com/Azure/go-autorest/autorest/adal v0.8.2 github.com/Azure/go-autorest/autorest/adal # github.com/Azure/go-autorest/autorest/azure/auth v0.4.2 -## explicit github.com/Azure/go-autorest/autorest/azure/auth # github.com/Azure/go-autorest/autorest/azure/cli v0.3.1 github.com/Azure/go-autorest/autorest/azure/cli # github.com/Azure/go-autorest/autorest/date v0.2.0 github.com/Azure/go-autorest/autorest/date # github.com/Azure/go-autorest/autorest/to v0.3.0 -## explicit github.com/Azure/go-autorest/autorest/to # github.com/Azure/go-autorest/autorest/validation v0.2.0 -## explicit github.com/Azure/go-autorest/autorest/validation # github.com/Azure/go-autorest/logger v0.1.0 github.com/Azure/go-autorest/logger # github.com/Azure/go-autorest/tracing v0.5.0 github.com/Azure/go-autorest/tracing # github.com/BurntSushi/toml v0.3.1 -## explicit github.com/BurntSushi/toml # github.com/DataDog/zstd v1.4.4 -## explicit github.com/DataDog/zstd # github.com/Masterminds/goutils v1.1.0 github.com/Masterminds/goutils @@ -62,7 +53,6 @@ github.com/Masterminds/semver # github.com/Masterminds/sprig v2.22.0+incompatible github.com/Masterminds/sprig # github.com/MichaelTJones/walk v0.0.0-20161122175330-4748e29d5718 -## explicit github.com/MichaelTJones/walk # github.com/Microsoft/go-winio v0.4.14 github.com/Microsoft/go-winio @@ -70,35 +60,26 @@ github.com/Microsoft/go-winio/pkg/guid # github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 github.com/Nvveen/Gotty # github.com/PuerkitoBio/goquery v1.5.0 -## explicit github.com/PuerkitoBio/goquery # github.com/Shopify/sarama v1.22.2-0.20190604114437-cd910a683f9f -## explicit github.com/Shopify/sarama # github.com/Shopify/toxiproxy v2.1.4+incompatible -## explicit github.com/Shopify/toxiproxy/client # github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 -## explicit github.com/StackExchange/wmi # github.com/VividCortex/ewma v1.1.1 -## explicit github.com/VividCortex/ewma # github.com/abourget/teamcity v0.0.0-00010101000000-000000000000 => github.com/cockroachdb/teamcity v0.0.0-20180905144921-8ca25c33eb11 -## explicit github.com/abourget/teamcity # github.com/alexbrainman/sspi v0.0.0-20180613141037-e580b900e9f5 github.com/alexbrainman/sspi github.com/alexbrainman/sspi/negotiate # github.com/andy-kimball/arenaskl v0.0.0-20200617143215-f701008588b9 -## explicit github.com/andy-kimball/arenaskl github.com/andy-kimball/arenaskl/internal/fastrand # github.com/andybalholm/cascadia v1.2.0 -## explicit github.com/andybalholm/cascadia # github.com/apache/arrow/go/arrow v0.0.0-20200610220642-670890229854 -## explicit github.com/apache/arrow/go/arrow github.com/apache/arrow/go/arrow/array github.com/apache/arrow/go/arrow/bitutil @@ -108,13 +89,10 @@ github.com/apache/arrow/go/arrow/internal/cpu github.com/apache/arrow/go/arrow/internal/debug github.com/apache/arrow/go/arrow/memory # github.com/apache/thrift v0.0.0-20181211084444-2b7365c54f82 -## explicit github.com/apache/thrift/lib/go/thrift # github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e -## explicit github.com/armon/circbuf # github.com/aws/aws-sdk-go v1.33.8 -## explicit github.com/aws/aws-sdk-go/aws github.com/aws/aws-sdk-go/aws/arn github.com/aws/aws-sdk-go/aws/awserr @@ -163,45 +141,33 @@ github.com/aws/aws-sdk-go/service/s3/s3manager github.com/aws/aws-sdk-go/service/sts github.com/aws/aws-sdk-go/service/sts/stsiface # github.com/axiomhq/hyperloglog v0.0.0-20181223111420-4b99d0c2c99e -## explicit github.com/axiomhq/hyperloglog # github.com/benesch/cgosymbolizer v0.0.0-20180702220239-70e1ee2b39d3 -## explicit github.com/benesch/cgosymbolizer # github.com/beorn7/perks v1.0.1 github.com/beorn7/perks/quantile # github.com/biogo/store v0.0.0-20160505134755-913427a1d5e8 -## explicit github.com/biogo/store/llrb # github.com/cenkalti/backoff v2.1.1+incompatible -## explicit github.com/cenkalti/backoff # github.com/client9/misspell v0.3.4 -## explicit github.com/client9/misspell github.com/client9/misspell/cmd/misspell # github.com/cockroachdb/apd/v2 v2.0.2 -## explicit github.com/cockroachdb/apd/v2 # github.com/cockroachdb/circuitbreaker v2.2.2-0.20190114160014-a614b14ccf63+incompatible -## explicit github.com/cockroachdb/circuitbreaker # github.com/cockroachdb/cmux v0.0.0-20170110192607-30d10be49292 -## explicit github.com/cockroachdb/cmux # github.com/cockroachdb/cockroach-go v0.0.0-20200504194139-73ffeee90b62 -## explicit github.com/cockroachdb/cockroach-go/crdb # github.com/cockroachdb/crlfmt v0.0.0-20200116191136-a78e1c207bc0 -## explicit github.com/cockroachdb/crlfmt github.com/cockroachdb/crlfmt/internal/parser github.com/cockroachdb/crlfmt/internal/render # github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5 -## explicit github.com/cockroachdb/datadriven # github.com/cockroachdb/errors v1.5.0 -## explicit github.com/cockroachdb/errors github.com/cockroachdb/errors/assert github.com/cockroachdb/errors/barriers @@ -221,10 +187,8 @@ github.com/cockroachdb/errors/telemetrykeys github.com/cockroachdb/errors/testutils github.com/cockroachdb/errors/withstack # github.com/cockroachdb/go-test-teamcity v0.0.0-20191211140407-cff980ad0a55 -## explicit github.com/cockroachdb/go-test-teamcity # github.com/cockroachdb/gostdlib v1.13.0 -## explicit github.com/cockroachdb/gostdlib/cmd/gofmt github.com/cockroachdb/gostdlib/go/format github.com/cockroachdb/gostdlib/go/printer @@ -240,10 +204,8 @@ github.com/cockroachdb/gostdlib/x/tools/internal/module github.com/cockroachdb/gostdlib/x/tools/internal/semver github.com/cockroachdb/gostdlib/x/tools/internal/span # github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f -## explicit github.com/cockroachdb/logtags # github.com/cockroachdb/pebble v0.0.0-20200729202514-cfd19d33cdca -## explicit github.com/cockroachdb/pebble github.com/cockroachdb/pebble/bloom github.com/cockroachdb/pebble/internal/arenaskl @@ -267,27 +229,20 @@ github.com/cockroachdb/pebble/sstable github.com/cockroachdb/pebble/tool github.com/cockroachdb/pebble/vfs # github.com/cockroachdb/redact v0.0.0-20200622112456-cd282804bbd3 -## explicit github.com/cockroachdb/redact # github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd -## explicit github.com/cockroachdb/returncheck # github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 -## explicit github.com/cockroachdb/sentry-go # github.com/cockroachdb/stress v0.0.0-20170808184505-29b5d31b4c3a -## explicit github.com/cockroachdb/stress # github.com/cockroachdb/ttycolor v0.0.0-20180709150743-a1d5aaeb377d -## explicit github.com/cockroachdb/ttycolor # github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd -## explicit github.com/codahale/hdrhistogram # github.com/cpuguy83/go-md2man v1.0.10 github.com/cpuguy83/go-md2man/md2man # github.com/dave/dst v0.24.0 -## explicit github.com/dave/dst github.com/dave/dst/decorator github.com/dave/dst/decorator/resolver @@ -299,16 +254,13 @@ github.com/davecgh/go-spew/spew # github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/dgrijalva/jwt-go # github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc -## explicit github.com/dgryski/go-metro # github.com/dimchansky/utfbom v1.1.0 github.com/dimchansky/utfbom # github.com/docker/distribution v2.7.0+incompatible -## explicit github.com/docker/distribution/digestset github.com/docker/distribution/reference # github.com/docker/docker v17.12.0-ce-rc1.0.20190115172544-0dc531243dd3+incompatible -## explicit github.com/docker/docker/api github.com/docker/docker/api/types github.com/docker/docker/api/types/blkiodev @@ -332,58 +284,42 @@ github.com/docker/docker/pkg/stdcopy github.com/docker/docker/pkg/term github.com/docker/docker/pkg/term/windows # github.com/docker/go-connections v0.4.0 -## explicit github.com/docker/go-connections/nat github.com/docker/go-connections/sockets github.com/docker/go-connections/tlsconfig # github.com/docker/go-units v0.4.0 -## explicit github.com/docker/go-units # github.com/dustin/go-humanize v1.0.0 -## explicit github.com/dustin/go-humanize # github.com/eapache/go-resiliency v1.2.0 -## explicit github.com/eapache/go-resiliency/breaker # github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 github.com/eapache/go-xerial-snappy # github.com/eapache/queue v1.1.0 github.com/eapache/queue # github.com/edsrzf/mmap-go v1.0.0 -## explicit github.com/edsrzf/mmap-go # github.com/elastic/gosigar v0.10.0 -## explicit github.com/elastic/gosigar github.com/elastic/gosigar/sys/windows # github.com/elazarl/go-bindata-assetfs v1.0.0 -## explicit github.com/elazarl/go-bindata-assetfs # github.com/envoyproxy/protoc-gen-validate v0.1.0 github.com/envoyproxy/protoc-gen-validate/validate # github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a -## explicit github.com/facebookgo/clock -# github.com/frankban/quicktest v1.7.3 -## explicit # github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9 -## explicit github.com/ghemawat/stream # github.com/ghodss/yaml v1.0.0 github.com/ghodss/yaml # github.com/go-logfmt/logfmt v0.4.0 github.com/go-logfmt/logfmt # github.com/go-ole/go-ole v1.2.2 -## explicit github.com/go-ole/go-ole github.com/go-ole/go-ole/oleutil # github.com/go-sql-driver/mysql v1.5.0 -## explicit github.com/go-sql-driver/mysql -# github.com/gofrs/uuid v3.3.0+incompatible -## explicit # github.com/gogo/protobuf v1.3.1 => github.com/cockroachdb/gogoproto v1.2.1-0.20190102194534-ca10b809dba0 -## explicit github.com/gogo/protobuf/gogoproto github.com/gogo/protobuf/jsonpb github.com/gogo/protobuf/plugin/compare @@ -413,22 +349,16 @@ github.com/gogo/protobuf/types github.com/gogo/protobuf/vanity github.com/gogo/protobuf/vanity/command # github.com/golang-commonmark/html v0.0.0-20180910111043-7d7c804e1d46 -## explicit github.com/golang-commonmark/html # github.com/golang-commonmark/linkify v0.0.0-20180910111149-f05efb453a0e -## explicit github.com/golang-commonmark/linkify # github.com/golang-commonmark/markdown v0.0.0-20180910011815-a8f139058164 -## explicit github.com/golang-commonmark/markdown # github.com/golang-commonmark/mdurl v0.0.0-20180910110917-8d018c6567d6 -## explicit github.com/golang-commonmark/mdurl # github.com/golang-commonmark/puny v0.0.0-20180910110745-050be392d8b8 -## explicit github.com/golang-commonmark/puny # github.com/golang/geo v0.0.0-20200319012246-673a6f80352d -## explicit github.com/golang/geo/r1 github.com/golang/geo/r2 github.com/golang/geo/r3 @@ -437,7 +367,6 @@ github.com/golang/geo/s2 # github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/glog # github.com/golang/protobuf v1.4.2 -## explicit github.com/golang/protobuf/descriptor github.com/golang/protobuf/jsonpb github.com/golang/protobuf/proto @@ -449,16 +378,12 @@ github.com/golang/protobuf/ptypes/duration github.com/golang/protobuf/ptypes/timestamp github.com/golang/protobuf/ptypes/wrappers # github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf -## explicit github.com/golang/snappy # github.com/google/btree v1.0.0 -## explicit github.com/google/btree # github.com/google/flatbuffers v1.11.0 -## explicit github.com/google/flatbuffers/go # github.com/google/go-cmp v0.4.0 -## explicit github.com/google/go-cmp/cmp github.com/google/go-cmp/cmp/cmpopts github.com/google/go-cmp/cmp/internal/diff @@ -466,14 +391,10 @@ github.com/google/go-cmp/cmp/internal/flags github.com/google/go-cmp/cmp/internal/function github.com/google/go-cmp/cmp/internal/value # github.com/google/go-github v17.0.0+incompatible -## explicit github.com/google/go-github/github # github.com/google/go-querystring v1.0.0 github.com/google/go-querystring/query -# github.com/google/martian v2.1.0+incompatible -## explicit # github.com/google/pprof v0.0.0-20190109223431-e84dfd68c163 -## explicit github.com/google/pprof/driver github.com/google/pprof/internal/binutils github.com/google/pprof/internal/driver @@ -492,20 +413,14 @@ github.com/google/pprof/third_party/svgpan # github.com/google/uuid v1.1.1 github.com/google/uuid # github.com/googleapis/gax-go v2.0.2+incompatible -## explicit github.com/googleapis/gax-go # github.com/gorhill/cronexpr v0.0.0-20140423231348-a557574d6c02 -## explicit github.com/gorhill/cronexpr -# github.com/gorilla/mux v1.7.4 -## explicit # github.com/gorilla/websocket v1.4.0 github.com/gorilla/websocket # github.com/goware/modvendor v0.3.0 -## explicit github.com/goware/modvendor # github.com/grpc-ecosystem/grpc-gateway v1.13.0 => github.com/cockroachdb/grpc-gateway v1.14.6-0.20200519165156-52697fc4a249 -## explicit github.com/grpc-ecosystem/grpc-gateway/codegenerator github.com/grpc-ecosystem/grpc-gateway/internal github.com/grpc-ecosystem/grpc-gateway/internal/casing @@ -517,14 +432,12 @@ github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway/internal/gengatew github.com/grpc-ecosystem/grpc-gateway/runtime github.com/grpc-ecosystem/grpc-gateway/utilities # github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 -## explicit github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc # github.com/hashicorp/go-uuid v1.0.2 github.com/hashicorp/go-uuid # github.com/huandu/xstrings v1.3.0 github.com/huandu/xstrings # github.com/ianlancetaylor/cgosymbolizer v0.0.0-20170921033129-f5072df9c550 -## explicit github.com/ianlancetaylor/cgosymbolizer # github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 github.com/ianlancetaylor/demangle @@ -534,10 +447,7 @@ github.com/imdario/mergo github.com/inconshreveable/mousetrap # github.com/jackc/chunkreader/v2 v2.0.1 github.com/jackc/chunkreader/v2 -# github.com/jackc/fake v0.0.0-20150926172116-812a484cc733 -## explicit # github.com/jackc/pgconn v1.6.1 -## explicit github.com/jackc/pgconn github.com/jackc/pgconn/internal/ctxwatch github.com/jackc/pgconn/stmtcache @@ -546,14 +456,12 @@ github.com/jackc/pgio # github.com/jackc/pgpassfile v1.0.0 github.com/jackc/pgpassfile # github.com/jackc/pgproto3/v2 v2.0.2 -## explicit github.com/jackc/pgproto3/v2 # github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8 github.com/jackc/pgservicefile # github.com/jackc/pgtype v1.3.0 github.com/jackc/pgtype # github.com/jackc/pgx v3.6.2+incompatible -## explicit github.com/jackc/pgx github.com/jackc/pgx/chunkreader github.com/jackc/pgx/internal/sanitize @@ -561,11 +469,9 @@ github.com/jackc/pgx/pgio github.com/jackc/pgx/pgproto3 github.com/jackc/pgx/pgtype # github.com/jackc/pgx/v4 v4.6.0 -## explicit github.com/jackc/pgx/v4 github.com/jackc/pgx/v4/internal/sanitize # github.com/jaegertracing/jaeger v1.17.0 -## explicit github.com/jaegertracing/jaeger/model/json # github.com/jcmturner/aescts/v2 v2.0.0 github.com/jcmturner/aescts/v2 @@ -615,53 +521,41 @@ github.com/jcmturner/rpc/v2/ndr # github.com/jmespath/go-jmespath v0.3.0 github.com/jmespath/go-jmespath # github.com/jordanlewis/gcassert v0.0.0-20200706043056-bf61eb72ee48 -## explicit github.com/jordanlewis/gcassert # github.com/kevinburke/go-bindata v3.13.0+incompatible -## explicit github.com/kevinburke/go-bindata github.com/kevinburke/go-bindata/go-bindata # github.com/kisielk/errcheck v1.2.0 -## explicit github.com/kisielk/errcheck github.com/kisielk/errcheck/internal/errcheck # github.com/kisielk/gotool v1.0.0 -## explicit github.com/kisielk/gotool github.com/kisielk/gotool/internal/load # github.com/knz/go-libedit v1.10.1 -## explicit github.com/knz/go-libedit github.com/knz/go-libedit/common github.com/knz/go-libedit/other github.com/knz/go-libedit/unix # github.com/knz/strtime v0.0.0-20200318182718-be999391ffa9 -## explicit github.com/knz/strtime # github.com/konsorten/go-windows-terminal-sequences v1.0.2 github.com/konsorten/go-windows-terminal-sequences # github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 github.com/kr/logfmt # github.com/kr/pretty v0.2.0 -## explicit github.com/kr/pretty # github.com/kr/text v0.2.0 -## explicit github.com/kr/text # github.com/leanovate/gopter v0.2.5-0.20190402064358-634a59d12406 -## explicit github.com/leanovate/gopter github.com/leanovate/gopter/prop # github.com/lib/pq v1.8.0 -## explicit github.com/lib/pq github.com/lib/pq/oid github.com/lib/pq/scram # github.com/lib/pq/auth/kerberos v0.0.0-20200720160335-984a6aa1ca46 -## explicit github.com/lib/pq/auth/kerberos # github.com/lightstep/lightstep-tracer-go v0.15.6 -## explicit github.com/lightstep/lightstep-tracer-go github.com/lightstep/lightstep-tracer-go/collectorpb github.com/lightstep/lightstep-tracer-go/lightstep/rand @@ -669,23 +563,14 @@ github.com/lightstep/lightstep-tracer-go/lightstep_thrift github.com/lightstep/lightstep-tracer-go/lightsteppb github.com/lightstep/lightstep-tracer-go/thrift_0_9_2/lib/go/thrift # github.com/linkedin/goavro/v2 v2.8.1 -## explicit github.com/linkedin/goavro/v2 # github.com/lufia/iostat v1.0.0 -## explicit github.com/lufia/iostat -# github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5 -## explicit -# github.com/lusis/slack-test v0.0.0-20190426140909-c40012f20018 -## explicit # github.com/maruel/panicparse v1.1.2-0.20180806203336-f20d4c4d746f -## explicit github.com/maruel/panicparse/stack # github.com/marusama/semaphore v0.0.0-20190110074507-6952cef993b2 -## explicit github.com/marusama/semaphore # github.com/mattn/go-isatty v0.0.12 -## explicit github.com/mattn/go-isatty # github.com/mattn/go-runewidth v0.0.7 github.com/mattn/go-runewidth @@ -693,12 +578,10 @@ github.com/mattn/go-runewidth github.com/mattn/go-zglob github.com/mattn/go-zglob/fastwalk # github.com/mattn/goveralls v0.0.2 -## explicit github.com/mattn/goveralls # github.com/matttproud/golang_protobuf_extensions v1.0.1 github.com/matttproud/golang_protobuf_extensions/pbutil # github.com/mibk/dupl v1.0.0 -## explicit github.com/mibk/dupl github.com/mibk/dupl/job github.com/mibk/dupl/output @@ -710,89 +593,65 @@ github.com/mitchellh/copystructure # github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/go-homedir # github.com/mitchellh/reflectwalk v1.0.0 -## explicit github.com/mitchellh/reflectwalk # github.com/mmatczuk/go_generics v0.0.0-20181212143635-0aaa050f9bab -## explicit github.com/mmatczuk/go_generics/cmd/go_generics github.com/mmatczuk/go_generics/globals # github.com/montanaflynn/stats v0.4.1-0.20190115100425-713f2944833c -## explicit github.com/montanaflynn/stats # github.com/mwitkow/go-proto-validators v0.0.0-20180403085117-0950a7990007 github.com/mwitkow/go-proto-validators # github.com/nlopes/slack v0.4.0 -## explicit github.com/nlopes/slack github.com/nlopes/slack/slackutilsx # github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5 => github.com/cockroachdb/tablewriter v0.0.5-0.20200105123400-bd15540e8847 -## explicit github.com/olekukonko/tablewriter # github.com/opencontainers/go-digest v1.0.0-rc1 github.com/opencontainers/go-digest # github.com/opencontainers/image-spec v1.0.1 github.com/opencontainers/image-spec/specs-go github.com/opencontainers/image-spec/specs-go/v1 -# github.com/opennota/wd v0.0.0-20180911144301-b446539ab1e7 -## explicit # github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492 -## explicit github.com/opentracing-contrib/go-observer # github.com/opentracing/opentracing-go v1.1.0 -## explicit github.com/opentracing/opentracing-go github.com/opentracing/opentracing-go/ext github.com/opentracing/opentracing-go/log # github.com/openzipkin-contrib/zipkin-go-opentracing v0.3.5 -## explicit github.com/openzipkin-contrib/zipkin-go-opentracing github.com/openzipkin-contrib/zipkin-go-opentracing/flag github.com/openzipkin-contrib/zipkin-go-opentracing/thrift/gen-go/scribe github.com/openzipkin-contrib/zipkin-go-opentracing/thrift/gen-go/zipkincore github.com/openzipkin-contrib/zipkin-go-opentracing/types github.com/openzipkin-contrib/zipkin-go-opentracing/wire -# github.com/pborman/uuid v1.2.0 -## explicit -# github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea -## explicit # github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 -## explicit github.com/petermattis/goid # github.com/pierrec/lz4 v2.4.1+incompatible -## explicit github.com/pierrec/lz4 github.com/pierrec/lz4/internal/xxh32 # github.com/pierrre/geohash v1.0.0 -## explicit github.com/pierrre/geohash # github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 -## explicit github.com/pkg/browser # github.com/pkg/errors v0.9.1 github.com/pkg/errors # github.com/pmezard/go-difflib v1.0.0 -## explicit github.com/pmezard/go-difflib/difflib # github.com/prometheus/client_golang v1.1.0 -## explicit github.com/prometheus/client_golang/prometheus github.com/prometheus/client_golang/prometheus/graphite github.com/prometheus/client_golang/prometheus/internal # github.com/prometheus/client_model v0.2.0 -## explicit github.com/prometheus/client_model/go # github.com/prometheus/common v0.9.1 -## explicit github.com/prometheus/common/expfmt github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg github.com/prometheus/common/model # github.com/prometheus/procfs v0.0.10 -## explicit github.com/prometheus/procfs github.com/prometheus/procfs/internal/fs github.com/prometheus/procfs/internal/util # github.com/pseudomuto/protoc-gen-doc v1.3.2 -## explicit github.com/pseudomuto/protoc-gen-doc github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc github.com/pseudomuto/protoc-gen-doc/extensions @@ -803,16 +662,13 @@ github.com/pseudomuto/protoc-gen-doc/extensions/validator_field # github.com/pseudomuto/protokit v0.2.0 github.com/pseudomuto/protokit # github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 -## explicit github.com/rcrowley/go-metrics github.com/rcrowley/go-metrics/exp # github.com/russross/blackfriday v1.5.2 github.com/russross/blackfriday # github.com/sasha-s/go-deadlock v0.2.0 -## explicit github.com/sasha-s/go-deadlock # github.com/shirou/gopsutil v2.20.6+incompatible -## explicit github.com/shirou/gopsutil/cpu github.com/shirou/gopsutil/disk github.com/shirou/gopsutil/host @@ -821,23 +677,17 @@ github.com/shirou/gopsutil/load github.com/shirou/gopsutil/mem github.com/shirou/gopsutil/net github.com/shirou/gopsutil/process -# github.com/shopspring/decimal v1.2.0 -## explicit # github.com/sirupsen/logrus v1.4.2 github.com/sirupsen/logrus # github.com/spf13/cobra v0.0.5 -## explicit github.com/spf13/cobra github.com/spf13/cobra/doc # github.com/spf13/pflag v1.0.5 -## explicit github.com/spf13/pflag # github.com/stretchr/testify v1.6.1 -## explicit github.com/stretchr/testify/assert github.com/stretchr/testify/require # github.com/twpayne/go-geom v1.3.2 -## explicit github.com/twpayne/go-geom github.com/twpayne/go-geom/bigxy github.com/twpayne/go-geom/encoding/ewkb @@ -857,10 +707,8 @@ github.com/twpayne/go-geom/xy/orientation # github.com/twpayne/go-kml v1.5.0 github.com/twpayne/go-kml # github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad -## explicit github.com/wadey/gocovmerge # go.etcd.io/etcd v0.0.0-00010101000000-000000000000 => github.com/cockroachdb/etcd v0.4.7-0.20200615211340-a17df30d5955 -## explicit go.etcd.io/etcd/raft go.etcd.io/etcd/raft/confchange go.etcd.io/etcd/raft/quorum @@ -882,7 +730,6 @@ go.opencensus.io/trace/internal go.opencensus.io/trace/propagation go.opencensus.io/trace/tracestate # golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 -## explicit golang.org/x/crypto/bcrypt golang.org/x/crypto/blowfish golang.org/x/crypto/chacha20 @@ -902,17 +749,13 @@ golang.org/x/crypto/ssh/internal/bcrypt_pbkdf golang.org/x/crypto/ssh/knownhosts golang.org/x/crypto/ssh/terminal # golang.org/x/exp v0.0.0-20200513190911-00229845015e -## explicit golang.org/x/exp/rand # golang.org/x/lint v0.0.0-20200130185559-910be7a94367 -## explicit golang.org/x/lint golang.org/x/lint/golint # golang.org/x/mod v0.3.0 -## explicit golang.org/x/mod/semver # golang.org/x/net v0.0.0-20200602114024-627f9648deb9 -## explicit golang.org/x/net/context golang.org/x/net/context/ctxhttp golang.org/x/net/html @@ -926,30 +769,25 @@ golang.org/x/net/internal/timeseries golang.org/x/net/proxy golang.org/x/net/trace # golang.org/x/oauth2 v0.0.0-20190115181402-5dab4167f31c -## explicit golang.org/x/oauth2 golang.org/x/oauth2/google golang.org/x/oauth2/internal golang.org/x/oauth2/jws golang.org/x/oauth2/jwt # golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852 -## explicit golang.org/x/perf/benchstat golang.org/x/perf/cmd/benchstat golang.org/x/perf/internal/stats golang.org/x/perf/storage/benchfmt # golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e -## explicit golang.org/x/sync/errgroup golang.org/x/sync/syncmap # golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1 -## explicit golang.org/x/sys/cpu golang.org/x/sys/internal/unsafeheader golang.org/x/sys/unix golang.org/x/sys/windows # golang.org/x/text v0.3.3 -## explicit golang.org/x/text/cases golang.org/x/text/collate golang.org/x/text/internal @@ -966,10 +804,8 @@ golang.org/x/text/unicode/bidi golang.org/x/text/unicode/norm golang.org/x/text/width # golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 -## explicit golang.org/x/time/rate # golang.org/x/tools v0.0.0-20200702044944-0cc1aa72b347 -## explicit golang.org/x/tools/cmd/goyacc golang.org/x/tools/cmd/stringer golang.org/x/tools/container/intsets @@ -1034,7 +870,6 @@ golang.org/x/tools/txtar golang.org/x/xerrors golang.org/x/xerrors/internal # google.golang.org/api v0.1.0 -## explicit google.golang.org/api/gensupport google.golang.org/api/googleapi google.golang.org/api/googleapi/internal/uritemplates @@ -1057,7 +892,6 @@ google.golang.org/appengine/internal/remote_api google.golang.org/appengine/internal/urlfetch google.golang.org/appengine/urlfetch # google.golang.org/genproto v0.0.0-20200218151345-dad8c97a84f5 -## explicit google.golang.org/genproto/googleapis/api/annotations google.golang.org/genproto/googleapis/api/httpbody google.golang.org/genproto/googleapis/iam/v1 @@ -1066,7 +900,6 @@ google.golang.org/genproto/googleapis/rpc/status google.golang.org/genproto/googleapis/type/expr google.golang.org/genproto/protobuf/field_mask # google.golang.org/grpc v1.29.1 -## explicit google.golang.org/grpc google.golang.org/grpc/attributes google.golang.org/grpc/backoff @@ -1149,10 +982,7 @@ google.golang.org/protobuf/types/pluginpb gopkg.in/jcmturner/aescts.v1 # gopkg.in/jcmturner/dnsutils.v1 v1.0.1 gopkg.in/jcmturner/dnsutils.v1 -# gopkg.in/jcmturner/goidentity.v3 v3.0.0 -## explicit # gopkg.in/jcmturner/gokrb5.v7 v7.5.0 -## explicit gopkg.in/jcmturner/gokrb5.v7/asn1tools gopkg.in/jcmturner/gokrb5.v7/client gopkg.in/jcmturner/gokrb5.v7/config @@ -1187,12 +1017,10 @@ gopkg.in/jcmturner/gokrb5.v7/types gopkg.in/jcmturner/rpc.v1/mstypes gopkg.in/jcmturner/rpc.v1/ndr # gopkg.in/yaml.v2 v2.3.0 => github.com/cockroachdb/yaml v0.0.0-20180705215940-0e2822948641 -## explicit gopkg.in/yaml.v2 # gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c gopkg.in/yaml.v3 # honnef.co/go/tools v0.0.0-20190530104931-1f0868a609b7 -## explicit honnef.co/go/tools/arg honnef.co/go/tools/cmd/staticcheck honnef.co/go/tools/config @@ -1219,7 +1047,6 @@ honnef.co/go/tools/stylecheck honnef.co/go/tools/unused honnef.co/go/tools/version # vitess.io/vitess v0.0.0-00010101000000-000000000000 => github.com/cockroachdb/vitess v2.2.0-rc.1.0.20180830030426-1740ce8b3188+incompatible -## explicit vitess.io/vitess/go/bytes2 vitess.io/vitess/go/hack vitess.io/vitess/go/sqltypes @@ -1229,10 +1056,3 @@ vitess.io/vitess/go/vt/proto/vtgate vitess.io/vitess/go/vt/proto/vtrpc vitess.io/vitess/go/vt/sqlparser vitess.io/vitess/go/vt/vterrors -# github.com/gogo/protobuf => github.com/cockroachdb/gogoproto v1.2.1-0.20190102194534-ca10b809dba0 -# github.com/grpc-ecosystem/grpc-gateway => github.com/cockroachdb/grpc-gateway v1.14.6-0.20200519165156-52697fc4a249 -# github.com/olekukonko/tablewriter => github.com/cockroachdb/tablewriter v0.0.5-0.20200105123400-bd15540e8847 -# github.com/abourget/teamcity => github.com/cockroachdb/teamcity v0.0.0-20180905144921-8ca25c33eb11 -# vitess.io/vitess => github.com/cockroachdb/vitess v2.2.0-rc.1.0.20180830030426-1740ce8b3188+incompatible -# gopkg.in/yaml.v2 => github.com/cockroachdb/yaml v0.0.0-20180705215940-0e2822948641 -# go.etcd.io/etcd => github.com/cockroachdb/etcd v0.4.7-0.20200615211340-a17df30d5955 From 9843123d7e0a4703223e6d2e1d64c23fd8462a4c Mon Sep 17 00:00:00 2001 From: Raphael 'kena' Poss Date: Thu, 13 Aug 2020 15:46:53 +0200 Subject: [PATCH 22/49] bump cockroachdb/errors --- github.com/cockroachdb/errors/README.md | 2 +- .../cockroachdb/errors/report/report.go | 2 +- .../cockroachdb/errors/safedetails/redact.go | 29 +++++++++++-------- .../errors/safedetails/safedetails.go | 12 +++++++- modules.txt | 2 +- 5 files changed, 31 insertions(+), 16 deletions(-) diff --git a/github.com/cockroachdb/errors/README.md b/github.com/cockroachdb/errors/README.md index 55979da394..c26dd877b2 100644 --- a/github.com/cockroachdb/errors/README.md +++ b/github.com/cockroachdb/errors/README.md @@ -45,7 +45,7 @@ Table of contents: | wrappers for user-facing hints and details | | | | ✔ | | wrappers to attach secondary causes | | | | ✔ | | wrappers to attach [`logtags`](https://github.com/cockroachdb/logtags) details from `context.Context` | | | | ✔ | -| `errors.FormatError()`, `Formatter`, `Printer` | | | (under construction) | (under construction) | +| `errors.FormatError()`, `Formatter`, `Printer` | | | (under construction) | ✔ | "Forward compatibility" above refers to the ability of this library to recognize and properly handle network communication of error types it diff --git a/github.com/cockroachdb/errors/report/report.go b/github.com/cockroachdb/errors/report/report.go index c793b234f4..9a5d984ba0 100644 --- a/github.com/cockroachdb/errors/report/report.go +++ b/github.com/cockroachdb/errors/report/report.go @@ -269,7 +269,7 @@ func BuildSentryReport(err error) (event *sentry.Event, extraDetails map[string] stKey := fmt.Sprintf("%d: details", extraNum) var extraStr bytes.Buffer for _, d := range details[i].SafeDetails { - fmt.Fprintln(&extraStr, d) + fmt.Fprintln(&extraStr, strings.ReplaceAll(d, "\n", "\n ")) } extras[stKey] = extraStr.String() fmt.Fprintf(&longMsgBuf, " (%d)", extraNum) diff --git a/github.com/cockroachdb/errors/safedetails/redact.go b/github.com/cockroachdb/errors/safedetails/redact.go index d4d4f5d127..6a20e4f266 100644 --- a/github.com/cockroachdb/errors/safedetails/redact.go +++ b/github.com/cockroachdb/errors/safedetails/redact.go @@ -42,7 +42,7 @@ func Redact(r interface{}) string { } redactErr(&buf, t) default: - typAnd(&buf, r, "") + typRedacted(&buf, r) } return buf.String() @@ -60,7 +60,7 @@ func redactErr(buf *strings.Builder, err error) { // Add any additional safe strings from the wrapper, if present. if payload := errbase.GetSafeDetails(err); len(payload.SafeDetails) > 0 { - buf.WriteString("\n(more details:)") + buf.WriteString("\n(more details about this error:)") for _, sd := range payload.SafeDetails { buf.WriteByte('\n') buf.WriteString(strings.TrimSpace(sd)) @@ -69,7 +69,7 @@ func redactErr(buf *strings.Builder, err error) { } func redactWrapper(buf *strings.Builder, err error) { - buf.WriteString("\nwrapper: ") + buf.WriteString("\n") switch t := err.(type) { case *os.SyscallError: typAnd(buf, t, t.Syscall) @@ -83,16 +83,16 @@ func redactWrapper(buf *strings.Builder, err error) { fmt.Fprintf(buf, " %s", t.Net) } if t.Source != nil { - buf.WriteString("") + buf.WriteString(" ") } if t.Addr != nil { if t.Source != nil { - buf.WriteString("->") + buf.WriteString(" ->") } - buf.WriteString("") + buf.WriteString(" ") } default: - typAnd(buf, err, "") + typRedacted(buf, err) } } @@ -122,16 +122,21 @@ func redactLeafErr(buf *strings.Builder, err error) { typAnd(buf, t, t.Error()) case syscall.Errno: typAnd(buf, t, t.Error()) + case SafeMessager: + typAnd(buf, t, t.SafeMessage()) default: // No further information about this error, simply report its type. - typAnd(buf, err, "") + typRedacted(buf, err) } } +func typRedacted(buf *strings.Builder, r interface{}) { + fmt.Fprintf(buf, "%T:", r) +} + func typAnd(buf *strings.Builder, r interface{}, msg string) { - if msg == "" { - fmt.Fprintf(buf, "<%T>", r) - } else { - fmt.Fprintf(buf, "%T: %s", r, msg) + if len(msg) == 0 { + msg = "(empty string)" } + fmt.Fprintf(buf, "%T: %s", r, msg) } diff --git a/github.com/cockroachdb/errors/safedetails/safedetails.go b/github.com/cockroachdb/errors/safedetails/safedetails.go index b3cd35495d..32ab8f8ac7 100644 --- a/github.com/cockroachdb/errors/safedetails/safedetails.go +++ b/github.com/cockroachdb/errors/safedetails/safedetails.go @@ -33,8 +33,18 @@ func WithSafeDetails(err error, format string, args ...interface{}) error { details := make([]string, 1, 1+len(args)) details[0] = format + if len(format) > 0 && len(args) > 0 { + // Call out that this string was a formatting string. + details[0] = fmt.Sprintf("format: %q", details[0]) + } for i, a := range args { - details = append(details, fmt.Sprintf("-- arg %d: %s", i+1, Redact(a))) + what := "" + if _, ok := a.(error); ok { + // Call out that this argument was an error, to facilitate + // interpretation of subsequent lines. + what = " (error)" + } + details = append(details, fmt.Sprintf("-- arg %d%s: %s", i+1, what, Redact(a))) } return &withSafeDetails{cause: err, safeDetails: details} } diff --git a/modules.txt b/modules.txt index 007c40af54..54ff1630a8 100644 --- a/modules.txt +++ b/modules.txt @@ -167,7 +167,7 @@ github.com/cockroachdb/crlfmt/internal/parser github.com/cockroachdb/crlfmt/internal/render # github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5 github.com/cockroachdb/datadriven -# github.com/cockroachdb/errors v1.5.0 +# github.com/cockroachdb/errors v1.5.1 github.com/cockroachdb/errors github.com/cockroachdb/errors/assert github.com/cockroachdb/errors/barriers From c652fce11ebbb0a3af3150b90ecb1836a3a73b26 Mon Sep 17 00:00:00 2001 From: Raphael 'kena' Poss Date: Fri, 14 Aug 2020 10:55:19 +0200 Subject: [PATCH 23/49] bump cockroachdb/errors --- .../cockroachdb/errors/safedetails/redact.go | 51 +++++++++++++++---- modules.txt | 2 +- 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/github.com/cockroachdb/errors/safedetails/redact.go b/github.com/cockroachdb/errors/safedetails/redact.go index 6a20e4f266..92507a6761 100644 --- a/github.com/cockroachdb/errors/safedetails/redact.go +++ b/github.com/cockroachdb/errors/safedetails/redact.go @@ -49,35 +49,59 @@ func Redact(r interface{}) string { } func redactErr(buf *strings.Builder, err error) { + foundDetail := false if c := errbase.UnwrapOnce(err); c == nil { // This is a leaf error. Decode the leaf and return. - redactLeafErr(buf, err) + foundDetail = redactLeafErr(buf, err) } else /* c != nil */ { // Print the inner error before the outer error. redactErr(buf, c) - redactWrapper(buf, err) + foundDetail = redactWrapper(buf, err) } // Add any additional safe strings from the wrapper, if present. if payload := errbase.GetSafeDetails(err); len(payload.SafeDetails) > 0 { - buf.WriteString("\n(more details about this error:)") - for _, sd := range payload.SafeDetails { - buf.WriteByte('\n') - buf.WriteString(strings.TrimSpace(sd)) + consumed := 0 + if !foundDetail { + firstDetail := strings.TrimSpace(payload.SafeDetails[0]) + if strings.IndexByte(firstDetail, '\n') < 0 { + firstDetail = strings.ReplaceAll(strings.TrimSpace(payload.SafeDetails[0]), "\n", "\n ") + if len(firstDetail) > 0 { + buf.WriteString(": ") + } + buf.WriteString(firstDetail) + consumed = 1 + foundDetail = true + } + } + if len(payload.SafeDetails) > consumed { + buf.WriteString("\n (more details:)") + for _, sd := range payload.SafeDetails[consumed:] { + buf.WriteString("\n ") + buf.WriteString(strings.ReplaceAll(strings.TrimSpace(sd), "\n", "\n ")) + } + foundDetail = true } } + if !foundDetail { + buf.WriteString(": ") + } } -func redactWrapper(buf *strings.Builder, err error) { +func redactWrapper(buf *strings.Builder, err error) (hasDetail bool) { buf.WriteString("\n") switch t := err.(type) { case *os.SyscallError: + hasDetail = true typAnd(buf, t, t.Syscall) case *os.PathError: + hasDetail = true typAnd(buf, t, t.Op) case *os.LinkError: + hasDetail = true fmt.Fprintf(buf, "%T: %s ", t, t.Op) case *net.OpError: + hasDetail = true typAnd(buf, t, t.Op) if t.Net != "" { fmt.Fprintf(buf, " %s", t.Net) @@ -92,11 +116,12 @@ func redactWrapper(buf *strings.Builder, err error) { buf.WriteString(" ") } default: - typRedacted(buf, err) + fmt.Fprintf(buf, "%T", err) } + return } -func redactLeafErr(buf *strings.Builder, err error) { +func redactLeafErr(buf *strings.Builder, err error) (hasDetail bool) { // Is it a sentinel error? These are safe. if markers.IsAny(err, context.DeadlineExceeded, @@ -108,26 +133,32 @@ func redactLeafErr(buf *strings.Builder, err error) { os.ErrClosed, os.ErrNoDeadline, ) { + hasDetail = true typAnd(buf, err, err.Error()) return } if redactPre113Wrappers(buf, err) { + hasDetail = true return } // The following two types are safe too. switch t := err.(type) { case runtime.Error: + hasDetail = true typAnd(buf, t, t.Error()) case syscall.Errno: + hasDetail = true typAnd(buf, t, t.Error()) case SafeMessager: + hasDetail = true typAnd(buf, t, t.SafeMessage()) default: // No further information about this error, simply report its type. - typRedacted(buf, err) + fmt.Fprintf(buf, "%T", err) } + return } func typRedacted(buf *strings.Builder, r interface{}) { diff --git a/modules.txt b/modules.txt index 54ff1630a8..6f1099171e 100644 --- a/modules.txt +++ b/modules.txt @@ -167,7 +167,7 @@ github.com/cockroachdb/crlfmt/internal/parser github.com/cockroachdb/crlfmt/internal/render # github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5 github.com/cockroachdb/datadriven -# github.com/cockroachdb/errors v1.5.1 +# github.com/cockroachdb/errors v1.5.2 github.com/cockroachdb/errors github.com/cockroachdb/errors/assert github.com/cockroachdb/errors/barriers From efcaaae16b755c118656a060120404047d8aa5cb Mon Sep 17 00:00:00 2001 From: Raphael 'kena' Poss Date: Fri, 14 Aug 2020 11:22:47 +0200 Subject: [PATCH 24/49] bump cockroachdb/errors --- github.com/cockroachdb/errors/README.md | 8 ++-- .../cockroachdb/errors/errutil/utilities.go | 37 ++++++++++++++++--- modules.txt | 2 +- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/github.com/cockroachdb/errors/README.md b/github.com/cockroachdb/errors/README.md index c26dd877b2..0320d73190 100644 --- a/github.com/cockroachdb/errors/README.md +++ b/github.com/cockroachdb/errors/README.md @@ -75,8 +75,8 @@ older version of the package. | Error detail | `Error()` and format `%s`/`%q`/`%v` | format `%+v` | `GetSafeDetails()` | Sentry report via `ReportError()` | |-----------------------------------------------------------------|-------------------------------------|--------------|-------------------------------|-----------------------------------| -| main message, eg `New()` | visible | visible | redacted | redacted | -| wrap prefix, eg `WithMessage()` | visible (as prefix) | visible | redacted | redacted | +| main message, eg `New()` | visible | visible | yes (CHANGED IN v1.6) | full (CHANGED IN v1.6) | +| wrap prefix, eg `WithMessage()` | visible (as prefix) | visible | yes (CHANGED IN v1.6) | full (CHANGED IN v1.6) | | stack trace, eg `WithStack()` | not visible | simplified | yes | full | | hint , eg `WithHint()` | not visible | visible | no | type only | | detail, eg `WithDetail()` | not visible | visible | no | type only | @@ -98,7 +98,7 @@ method. - `New(string) error`, `Newf(string, ...interface{}) error`, `Errorf(string, ...interface{}) error`: leaf errors with message - **when to use: common error cases.** - what it does: also captures the stack trace at point of call and redacts the provided message for safe reporting. - - how to access the detail: `Error()`, regular Go formatting. Details redacted in Sentry report. + - how to access the detail: `Error()`, regular Go formatting. **Details in Sentry report.** - see also: Section [Error composition](#Error-composition-summary) below. `errors.NewWithDepth()` variants to customize at which call depth the stack trace is captured. - `AssertionFailedf(string, ...interface{}) error`, `NewAssertionFailureWithWrappedErrf(error, string, ...interface{}) error`: signals an assertion failure / programming error. @@ -141,7 +141,7 @@ return errors.Wrap(foo(), "foo") - `Wrap(error, string) error`, `Wrapf(error, string, ...interface{}) error`: - **when to use: on error return paths.** - what it does: combines `WithMessage()`, `WithStack()`, `WithSafeDetails()`. - - how to access the details: `Error()`, regular Go formatting. Details redacted in Sentry report. + - how to access the details: `Error()`, regular Go formatting. **Details in Sentry report.** - see also: Section [Error composition](#Error-composition-summary) below. `WrapWithDepth()` variants to customize at which depth the stack trace is captured. - `WithSecondaryError(error, error) error`: annotate an error with a secondary error. diff --git a/github.com/cockroachdb/errors/errutil/utilities.go b/github.com/cockroachdb/errors/errutil/utilities.go index 28e1ed29ae..f6be48357d 100644 --- a/github.com/cockroachdb/errors/errutil/utilities.go +++ b/github.com/cockroachdb/errors/errutil/utilities.go @@ -25,11 +25,16 @@ import ( // New creates an error with a simple error message. // A stack trace is retained. // +// Note: the message string is assumed to not contain +// PII and is included in Sentry reports. +// Use errors.Newf("%s", ) for errors +// strings that may contain PII information. +// // Detail output: // - message via `Error()` and formatting using `%v`/`%s`/`%q`. // - everything when formatting with `%+v`. -// - stack trace (not message) via `errors.GetSafeDetails()`. -// - stack trace (not message) in Sentry reports. +// - stack trace and message via `errors.GetSafeDetails()`. +// - stack trace and message in Sentry reports. func New(msg string) error { return NewWithDepth(1, msg) } @@ -39,12 +44,21 @@ func New(msg string) error { // See the doc of `New()` for more details. func NewWithDepth(depth int, msg string) error { err := goErr.New(msg) + if len(msg) > 0 { + err = safedetails.WithSafeDetails(err, msg) + } err = withstack.WithStackDepth(err, 1+depth) return err } // Newf creates an error with a formatted error message. // A stack trace is retained. +// +// Note: the format string is assumed to not contain +// PII and is included in Sentry reports. +// Use errors.Newf("%s", ) for errors +// strings that may contain PII information. +// // See the doc of `New()` for more details. func Newf(format string, args ...interface{}) error { return NewWithDepthf(1, format, args...) @@ -65,11 +79,16 @@ func NewWithDepthf(depth int, format string, args ...interface{}) error { // Wrap wraps an error with a message prefix. // A stack trace is retained. // +// Note: the prefix string is assumed to not contain +// PII and is included in Sentry reports. +// Use errors.Wrapf(err, "%s", ) for errors +// strings that may contain PII information. +// // Detail output: // - original error message + prefix via `Error()` and formatting using `%v`/`%s`/`%q`. // - everything when formatting with `%+v`. -// - stack trace (not message) via `errors.GetSafeDetails()`. -// - stack trace (not message) in Sentry reports. +// - stack trace and message via `errors.GetSafeDetails()`. +// - stack trace and message in Sentry reports. func Wrap(err error, msg string) error { return WrapWithDepth(1, err, msg) } @@ -80,6 +99,7 @@ func Wrap(err error, msg string) error { func WrapWithDepth(depth int, err error, msg string) error { if msg != "" { err = WithMessage(err, msg) + err = safedetails.WithSafeDetails(err, msg) } err = withstack.WithStackDepth(err, depth+1) return err @@ -89,11 +109,16 @@ func WrapWithDepth(depth int, err error, msg string) error { // trace is also retained. If the format is empty, no prefix is added, // but the extra arguments are still processed for reportable strings. // +// Note: the format string is assumed to not contain +// PII and is included in Sentry reports. +// Use errors.Wrapf(err, "%s", ) for errors +// strings that may contain PII information. +// // Detail output: // - original error message + prefix via `Error()` and formatting using `%v`/`%s`/`%q`. // - everything when formatting with `%+v`. -// - stack trace (not message) and redacted details via `errors.GetSafeDetails()`. -// - stack trace (not message) and redacted details in Sentry reports. +// - stack trace, format, and redacted details via `errors.GetSafeDetails()`. +// - stack trace, format, and redacted details in Sentry reports. func Wrapf(err error, format string, args ...interface{}) error { return WrapWithDepthf(1, err, format, args...) } diff --git a/modules.txt b/modules.txt index 6f1099171e..20423f6b29 100644 --- a/modules.txt +++ b/modules.txt @@ -167,7 +167,7 @@ github.com/cockroachdb/crlfmt/internal/parser github.com/cockroachdb/crlfmt/internal/render # github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5 github.com/cockroachdb/datadriven -# github.com/cockroachdb/errors v1.5.2 +# github.com/cockroachdb/errors v1.6.1 github.com/cockroachdb/errors github.com/cockroachdb/errors/assert github.com/cockroachdb/errors/barriers From 951be40001d7985e0bd3316e9d9db77a84e737c3 Mon Sep 17 00:00:00 2001 From: Jackson Owens Date: Mon, 17 Aug 2020 13:58:59 -0400 Subject: [PATCH 25/49] bump Pebble to 77c18adb0ee3 77c18adb Update README.md (#853) a2588dc3 db: add elision-only L6 compactions 3dbbc99e db: propagate FormatKey to rangedel.Fragmenter in more places f20b433f db: add godoc link to README e7d2ee42 internal/manifest: add B-Tree implementation 3c13b0a8 go.mod: bump testify from v1.6.1 b259a65f db: minor batch.release() code cleanup 692a7cb6 db: minor change for TestIndexedBatchReset 993cd1d8 db: cleaned up Batch.Reset() related tests 3359447f db: fixed how indexed batch is reset 6ddb11f5 internal/manifest: remove LevelSlice Collect method 132ca835 db: properly reset batches 65352a3c db: mark corruption errors 1473be86 db: clear the applied flag in Batch.Reset() fdd14bd4 db: handle nil ParseHooks for cache_size option 94df1ff6 internal/manifest: use LevelMetadata in L0 sublevels --- github.com/cockroachdb/pebble/README.md | 40 +- github.com/cockroachdb/pebble/batch.go | 38 +- github.com/cockroachdb/pebble/compaction.go | 57 +- .../cockroachdb/pebble/compaction_iter.go | 12 +- .../cockroachdb/pebble/compaction_picker.go | 83 +- github.com/cockroachdb/pebble/db.go | 3 + github.com/cockroachdb/pebble/go.mod | 2 +- github.com/cockroachdb/pebble/go.sum | 8 + github.com/cockroachdb/pebble/ingest.go | 9 +- .../cockroachdb/pebble/internal/base/error.go | 22 +- .../pebble/internal/manifest/btree.go | 831 ++++++++++++++++++ .../pebble/internal/manifest/l0_sublevels.go | 55 +- .../internal/manifest/level_metadata.go | 20 +- .../pebble/internal/manifest/version.go | 36 +- .../pebble/internal/manifest/version_edit.go | 22 +- .../pebble/internal/record/record.go | 6 +- github.com/cockroachdb/pebble/iterator.go | 7 +- github.com/cockroachdb/pebble/mem_table.go | 6 +- github.com/cockroachdb/pebble/open.go | 2 +- github.com/cockroachdb/pebble/options.go | 7 +- github.com/cockroachdb/pebble/snapshot.go | 19 +- .../cockroachdb/pebble/sstable/block.go | 3 +- .../cockroachdb/pebble/sstable/properties.go | 3 +- .../cockroachdb/pebble/sstable/raw_block.go | 4 +- .../cockroachdb/pebble/sstable/reader.go | 20 +- .../cockroachdb/pebble/sstable/table.go | 18 +- github.com/cockroachdb/pebble/table_stats.go | 39 +- github.com/cockroachdb/pebble/tool/db.go | 4 +- github.com/cockroachdb/pebble/tool/lsm.go | 9 +- .../cockroachdb/pebble/tool/manifest.go | 11 +- github.com/cockroachdb/pebble/version_set.go | 15 +- golang.org/x/sys/cpu/cpu.go | 96 ++ golang.org/x/sys/cpu/cpu_aix.go | 4 +- golang.org/x/sys/cpu/cpu_arm.go | 33 + golang.org/x/sys/cpu/cpu_arm64.go | 31 +- golang.org/x/sys/cpu/cpu_linux.go | 2 +- golang.org/x/sys/cpu/cpu_linux_ppc64x.go | 2 - golang.org/x/sys/cpu/cpu_linux_s390x.go | 2 - golang.org/x/sys/cpu/cpu_mips64x.go | 6 + golang.org/x/sys/cpu/cpu_mipsx.go | 2 + golang.org/x/sys/cpu/cpu_other_arm.go | 9 + golang.org/x/sys/cpu/cpu_ppc64x.go | 16 + golang.org/x/sys/cpu/cpu_riscv64.go | 2 + golang.org/x/sys/cpu/cpu_s390x.go | 30 + golang.org/x/sys/cpu/cpu_wasm.go | 4 + golang.org/x/sys/cpu/cpu_x86.go | 31 +- golang.org/x/sys/unix/mkerrors.sh | 2 + golang.org/x/sys/unix/syscall_linux.go | 59 +- golang.org/x/sys/unix/zerrors_linux.go | 65 +- golang.org/x/sys/unix/zerrors_linux_arm64.go | 1 + golang.org/x/sys/unix/zsyscall_linux.go | 15 + golang.org/x/sys/unix/zsysnum_linux_386.go | 1 + golang.org/x/sys/unix/zsysnum_linux_amd64.go | 1 + golang.org/x/sys/unix/zsysnum_linux_arm.go | 1 + golang.org/x/sys/unix/zsysnum_linux_arm64.go | 1 + golang.org/x/sys/unix/zsysnum_linux_mips.go | 1 + golang.org/x/sys/unix/zsysnum_linux_mips64.go | 1 + .../x/sys/unix/zsysnum_linux_mips64le.go | 1 + golang.org/x/sys/unix/zsysnum_linux_mipsle.go | 1 + golang.org/x/sys/unix/zsysnum_linux_ppc64.go | 1 + .../x/sys/unix/zsysnum_linux_ppc64le.go | 1 + .../x/sys/unix/zsysnum_linux_riscv64.go | 1 + golang.org/x/sys/unix/zsysnum_linux_s390x.go | 1 + .../x/sys/unix/zsysnum_linux_sparc64.go | 1 + golang.org/x/sys/unix/ztypes_linux.go | 90 +- golang.org/x/sys/unix/ztypes_linux_386.go | 5 + golang.org/x/sys/unix/ztypes_linux_amd64.go | 7 + golang.org/x/sys/unix/ztypes_linux_arm.go | 7 + golang.org/x/sys/unix/ztypes_linux_arm64.go | 7 + golang.org/x/sys/unix/ztypes_linux_mips.go | 7 + golang.org/x/sys/unix/ztypes_linux_mips64.go | 7 + .../x/sys/unix/ztypes_linux_mips64le.go | 7 + golang.org/x/sys/unix/ztypes_linux_mipsle.go | 7 + golang.org/x/sys/unix/ztypes_linux_ppc64.go | 7 + golang.org/x/sys/unix/ztypes_linux_ppc64le.go | 7 + golang.org/x/sys/unix/ztypes_linux_riscv64.go | 7 + golang.org/x/sys/unix/ztypes_linux_s390x.go | 7 + golang.org/x/sys/unix/ztypes_linux_sparc64.go | 7 + modules.txt | 4 +- 79 files changed, 1824 insertions(+), 195 deletions(-) create mode 100644 github.com/cockroachdb/pebble/internal/manifest/btree.go create mode 100644 golang.org/x/sys/cpu/cpu_other_arm.go create mode 100644 golang.org/x/sys/cpu/cpu_ppc64x.go create mode 100644 golang.org/x/sys/cpu/cpu_s390x.go diff --git a/github.com/cockroachdb/pebble/README.md b/github.com/cockroachdb/pebble/README.md index 6c975355e1..3af5200648 100644 --- a/github.com/cockroachdb/pebble/README.md +++ b/github.com/cockroachdb/pebble/README.md @@ -1,4 +1,5 @@ -# Pebble [![Build Status](https://travis-ci.org/cockroachdb/pebble.svg?branch=master)](https://travis-ci.org/cockroachdb/pebble) +# Pebble [![Build Status](https://travis-ci.org/cockroachdb/pebble.svg?branch=master)](https://travis-ci.org/cockroachdb/pebble) [![GoDoc](https://godoc.org/github.com/cockroachdb/pebble?status.svg)](https://godoc.org/github.com/cockroachdb/pebble) + #### [Nightly benchmarks](https://cockroachdb.github.io/pebble/) Pebble is a LevelDB/RocksDB inspired key-value store focused on @@ -108,3 +109,40 @@ https://github.com/google/leveldb Optimizations and inspiration were drawn from RocksDB: https://github.com/facebook/rocksdb + +## Getting Started + +### Example Code + +```go +package main + +import ( + "fmt" + "log" + + "github.com/cockroachdb/pebble" +) + +func main() { + db, err := pebble.Open("demo", &pebble.Options{}) + if err != nil { + log.Fatal(err) + } + key := []byte("hello") + if err := db.Set(key, []byte("world"), pebble.Sync); err != nil { + log.Fatal(err) + } + value, closer, err := db.Get(key) + if err != nil { + log.Fatal(err) + } + fmt.Printf("%s %s\n", key, value) + if err := closer.Close(); err != nil { + log.Fatal(err) + } + if err := db.Close(); err != nil { + log.Fatal(err) + } +} +``` diff --git a/github.com/cockroachdb/pebble/batch.go b/github.com/cockroachdb/pebble/batch.go index 60db3725ab..c64a08e013 100644 --- a/github.com/cockroachdb/pebble/batch.go +++ b/github.com/cockroachdb/pebble/batch.go @@ -279,19 +279,10 @@ func (b *Batch) release() { b.cmp = nil b.formatKey = nil b.abbreviatedKey = nil - b.memTableSize = 0 - - b.deferredOp = DeferredBatchOp{} - b.tombstones = nil - b.flushable = nil - b.commit = sync.WaitGroup{} - b.commitErr = nil - atomic.StoreUint32(&b.applied, 0) if b.index == nil { batchPool.Put(b) } else { - b.index.Reset() b.index, b.rangeDelIndex = nil, nil indexedBatchPool.Put((*indexedBatch)(unsafe.Pointer(b))) } @@ -324,7 +315,7 @@ func (b *Batch) Apply(batch *Batch, _ *WriteOptions) error { return nil } if len(batch.data) < batchHeaderLen { - return errors.New("pebble: invalid batch") + return base.CorruptionErrorf("pebble: invalid batch") } offset := len(b.data) @@ -657,7 +648,7 @@ func (b *Batch) Repr() []byte { // Batch is no longer in use. func (b *Batch) SetRepr(data []byte) error { if len(data) < batchHeaderLen { - return errors.New("invalid batch") + return base.CorruptionErrorf("invalid batch") } b.data = data b.count = uint64(binary.LittleEndian.Uint32(b.countData())) @@ -767,6 +758,13 @@ func (b *Batch) init(cap int) { func (b *Batch) Reset() { b.count = 0 b.countRangeDels = 0 + b.memTableSize = 0 + b.deferredOp = DeferredBatchOp{} + b.tombstones = nil + b.flushable = nil + b.commit = sync.WaitGroup{} + b.commitErr = nil + atomic.StoreUint32(&b.applied, 0) if b.data != nil { if cap(b.data) > batchMaxRetainedSize { // If the capacity of the buffer is larger than our maximum @@ -780,6 +778,10 @@ func (b *Batch) Reset() { b.setSeqNum(0) } } + if b.index != nil { + b.index.Init(&b.data, b.cmp, b.abbreviatedKey) + b.rangeDelIndex = nil + } } // seqNumData returns the 8 byte little-endian sequence number. Zero means that @@ -984,7 +986,7 @@ func (i *batchIter) Value() []byte { offset, _, keyEnd := i.iter.KeyInfo() data := i.batch.data if len(data[offset:]) == 0 { - i.err = errors.New("corrupted batch") + i.err = base.CorruptionErrorf("corrupted batch") return nil } @@ -1370,12 +1372,12 @@ func (i *flushableBatchIter) Key() *InternalKey { func (i *flushableBatchIter) Value() []byte { p := i.data[i.offsets[i.index].offset:] if len(p) == 0 { - i.err = errors.New("corrupted batch") + i.err = base.CorruptionErrorf("corrupted batch") return nil } kind := InternalKeyKind(p[0]) if kind > InternalKeyKindMax { - i.err = errors.New("corrupted batch") + i.err = base.CorruptionErrorf("corrupted batch") return nil } var value []byte @@ -1385,7 +1387,7 @@ func (i *flushableBatchIter) Value() []byte { keyEnd := i.offsets[i.index].keyEnd _, value, ok = batchDecodeStr(i.data[keyEnd:]) if !ok { - i.err = errors.New("corrupted batch") + i.err = base.CorruptionErrorf("corrupted batch") return nil } } @@ -1469,12 +1471,12 @@ func (i flushFlushableBatchIter) Prev() (*InternalKey, []byte) { func (i flushFlushableBatchIter) valueSize() uint64 { p := i.data[i.offsets[i.index].offset:] if len(p) == 0 { - i.err = errors.New("corrupted batch") + i.err = base.CorruptionErrorf("corrupted batch") return 0 } kind := InternalKeyKind(p[0]) if kind > InternalKeyKindMax { - i.err = errors.New("corrupted batch") + i.err = base.CorruptionErrorf("corrupted batch") return 0 } var length uint64 @@ -1483,7 +1485,7 @@ func (i flushFlushableBatchIter) valueSize() uint64 { keyEnd := i.offsets[i.index].keyEnd v, n := binary.Uvarint(i.data[keyEnd:]) if n <= 0 { - i.err = errors.New("corrupted batch") + i.err = base.CorruptionErrorf("corrupted batch") return 0 } length = v + uint64(n) diff --git a/github.com/cockroachdb/pebble/compaction.go b/github.com/cockroachdb/pebble/compaction.go index 776f79bf90..184ec4023f 100644 --- a/github.com/cockroachdb/pebble/compaction.go +++ b/github.com/cockroachdb/pebble/compaction.go @@ -69,10 +69,11 @@ type compactionLevel struct { type compactionKind string const ( - compactionKindDefault compactionKind = "default" - compactionKindFlush = "flush" - compactionKindMove = "move" - compactionKindDeleteOnly = "delete-only" + compactionKindDefault compactionKind = "default" + compactionKindFlush = "flush" + compactionKindMove = "move" + compactionKindDeleteOnly = "delete-only" + compactionKindElisionOnly = "elision-only" ) // compaction is a table compaction from one level to the next, starting from a @@ -218,12 +219,16 @@ func newCompaction(pc *pickedCompaction, opts *Options, bytesCompacted *uint64) } c.setupInuseKeyRanges() - // Check if this compaction can be converted into a trivial move from one - // level to the next. We avoid such a move if there is lots of overlapping - // grandparent data. Otherwise, the move could create a parent file that - // will require a very expensive merge later on. - if c.outputLevel.files.Empty() && c.startLevel.files.Len() == 1 && + if c.startLevel.level == numLevels-1 { + // This compaction is an L6->L6 elision-only compaction to rewrite + // a sstable without unnecessary tombstones. + c.kind = compactionKindElisionOnly + } else if c.outputLevel.files.Empty() && c.startLevel.files.Len() == 1 && c.grandparents.SizeSum() <= c.maxOverlapBytes { + // This compaction can be converted into a trivial move from one level + // to the next. We avoid such a move if there is lots of overlapping + // grandparent data. Otherwise, the move could create a parent file + // that will require a very expensive merge later on. c.kind = compactionKindMove } return c @@ -517,13 +522,13 @@ func (c *compaction) newInputIter(newIters tableNewIters) (_ internalIterator, r err := manifest.CheckOrdering(c.cmp, c.formatKey, manifest.Level(c.startLevel.level), c.startLevel.files.Iter()) if err != nil { - c.logger.Fatalf("%s", err) + return nil, err } } err := manifest.CheckOrdering(c.cmp, c.formatKey, manifest.Level(c.outputLevel.level), c.outputLevel.files.Iter()) if err != nil { - c.logger.Fatalf("%s", err) + return nil, err } iters := make([]internalIterator, 0, 2*c.startLevel.files.Len()+1) @@ -1033,6 +1038,22 @@ func (d *DB) flush1() error { // // d.mu must be held when calling this. func (d *DB) maybeScheduleCompaction() { + d.maybeScheduleCompactionPicker(pickAuto) +} + +func pickAuto(picker compactionPicker, env compactionEnv) *pickedCompaction { + return picker.pickAuto(env) +} + +func pickElisionOnly(picker compactionPicker, env compactionEnv) *pickedCompaction { + return picker.pickElisionOnlyCompaction(env) +} + +// maybeScheduleCompactionPicker schedules a compaction if necessary, +// calling `pickFunc` to pick automatic compactions. +// +// d.mu must be held when calling this. +func (d *DB) maybeScheduleCompactionPicker(pickFunc func(compactionPicker, compactionEnv) *pickedCompaction) { if d.closed.Load() != nil || d.opts.ReadOnly { return } @@ -1059,6 +1080,7 @@ func (d *DB) maybeScheduleCompaction() { env := compactionEnv{ bytesCompacted: &d.bytesCompacted, + earliestSnapshotSeqNum: d.mu.snapshots.earliest(), earliestUnflushedSeqNum: d.getEarliestUnflushedSeqNumLocked(), } @@ -1103,7 +1125,7 @@ func (d *DB) maybeScheduleCompaction() { for !d.opts.private.disableAutomaticCompactions && d.mu.compact.compactingCount < d.opts.MaxConcurrentCompactions { env.inProgressCompactions = d.getInProgressCompactionInfoLocked(nil) - pc := d.mu.versions.picker.pickAuto(env) + pc := pickFunc(d.mu.versions.picker, env) if pc == nil { break } @@ -1475,8 +1497,8 @@ func (d *DB) runCompaction( return nil, pendingOutputs, err } c.allowedZeroSeqNum = c.allowZeroSeqNum(iiter) - iter := newCompactionIter(c.cmp, d.merge, iiter, snapshots, &c.rangeDelFrag, - c.allowedZeroSeqNum, c.elideTombstone, c.elideRangeTombstone) + iter := newCompactionIter(c.cmp, c.formatKey, d.merge, iiter, snapshots, + &c.rangeDelFrag, c.allowedZeroSeqNum, c.elideTombstone, c.elideRangeTombstone) var ( filenames []string @@ -1640,12 +1662,7 @@ func (d *DB) runCompaction( meta.MarkedForCompaction = writerMeta.MarkedForCompaction // If the file didn't contain any range deletions, we can fill its // table stats now, avoiding unnecessarily loading the table later. - if writerMeta.Properties.NumRangeDeletions == 0 { - meta.Stats = manifest.TableStats{ - Valid: true, - RangeDeletionsBytesEstimate: 0, - } - } + maybeSetStatsFromProperties(meta, &writerMeta.Properties) if c.flushing == nil { outputMetrics.TablesCompacted++ diff --git a/github.com/cockroachdb/pebble/compaction_iter.go b/github.com/cockroachdb/pebble/compaction_iter.go index 5e1c6e36e8..f35121c84e 100644 --- a/github.com/cockroachdb/pebble/compaction_iter.go +++ b/github.com/cockroachdb/pebble/compaction_iter.go @@ -9,6 +9,7 @@ import ( "sort" "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/bytealloc" "github.com/cockroachdb/pebble/internal/rangedel" ) @@ -181,6 +182,7 @@ type compactionIter struct { func newCompactionIter( cmp Compare, + formatKey base.FormatKey, merge Merge, iter internalIterator, snapshots []uint64, @@ -200,6 +202,7 @@ func newCompactionIter( elideRangeTombstone: elideRangeTombstone, } i.rangeDelFrag.Cmp = cmp + i.rangeDelFrag.Format = formatKey i.rangeDelFrag.Emit = i.emitRangeDelChunk return i } @@ -337,10 +340,13 @@ func (i *compactionIter) Next() (*InternalKey, []byte) { } return &i.key, i.value } + if i.err != nil { + i.err = base.MarkCorruptionError(i.err) + } return nil, nil default: - i.err = errors.Errorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind())) + i.err = base.CorruptionErrorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind())) return nil, nil } } @@ -492,7 +498,7 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType { } default: - i.err = errors.Errorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind())) + i.err = base.CorruptionErrorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind())) return sameStripeSkippable } } @@ -528,7 +534,7 @@ func (i *compactionIter) singleDeleteNext() bool { continue default: - i.err = errors.Errorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind())) + i.err = base.CorruptionErrorf("invalid internal key kind: %d", errors.Safe(i.iterKey.Kind())) return false } } diff --git a/github.com/cockroachdb/pebble/compaction_picker.go b/github.com/cockroachdb/pebble/compaction_picker.go index 8fd7e801ca..f7fb6cc3cb 100644 --- a/github.com/cockroachdb/pebble/compaction_picker.go +++ b/github.com/cockroachdb/pebble/compaction_picker.go @@ -22,6 +22,7 @@ const minIntraL0Count = 4 type compactionEnv struct { bytesCompacted *uint64 earliestUnflushedSeqNum uint64 + earliestSnapshotSeqNum uint64 inProgressCompactions []compactionInfo } @@ -32,6 +33,7 @@ type compactionPicker interface { estimatedCompactionDebt(l0ExtraSize uint64) uint64 pickAuto(env compactionEnv) (pc *pickedCompaction) pickManual(env compactionEnv, manual *manualCompaction) (c *pickedCompaction, retryLater bool) + pickElisionOnlyCompaction(env compactionEnv) (pc *pickedCompaction) forceBaseLevel1() } @@ -413,6 +415,14 @@ type compactionPickerByScore struct { // levelMaxBytes holds the dynamically adjusted max bytes setting for each // level. levelMaxBytes [numLevels]int64 + + // Information for facilitating L6->L6 elision-only compactions. If + // elisionThreshold is non-nil, it's the lowest LargestSeqNum of a file + // that meets the conditions for an elision-only compaction. It may be + // math.MaxUint64 if it's known that no file within L6 meets the + // conditions for an elision-only compaction. + elisionThreshold *uint64 + elisionCandidate *manifest.LevelFile } var _ compactionPicker = &compactionPickerByScore{} @@ -933,6 +943,14 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (pc *pickedCompact } } + // Check for L6 files with tombstones that may be elided. These files may + // exist if a snapshot prevented the elision of a tombstone or because of + // a move compaction. These are low-priority compactions because they + // don't help us keep up with writes, just reclaim disk space. + if pc := p.pickElisionOnlyCompaction(env); pc != nil { + return pc + } + // Check for forced compactions. These are lower priority than score-based // compactions. Note that this loop only runs if we haven't already found a // score-based compaction. @@ -956,10 +974,69 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (pc *pickedCompact } } } + return nil +} - // TODO(peter): When a snapshot is released, we may need to compact tables at - // the bottom level in order to free up entries that were pinned by the - // snapshot. +// pickElisionOnlyCompaction looks for compactions of sstables in the +// bottommost level containing obsolete records that may now be dropped. +func (p *compactionPickerByScore) pickElisionOnlyCompaction(env compactionEnv) (pc *pickedCompaction) { + if p.elisionThreshold == nil || (p.elisionCandidate != nil && p.elisionCandidate.Compacting) { + var lowestCandidateSeqNum uint64 = math.MaxUint64 + var lowestCandidate manifest.LevelFile + iter := p.vers.Levels[numLevels-1].Iter() + for f := iter.First(); f != nil; f = iter.Next() { + if f.Compacting { + continue + } + if f.LargestSeqNum >= lowestCandidateSeqNum { + continue + } + if !f.Stats.Valid { + continue + } + // Bottommost files are large and not worthwhile to compact just + // to remove a few tombstones. Consider a file ineligible if its + // own range deletions delete less than 10% of its data and its + // deletion tombstones make up less than 10% of its entries. + // + // TODO(jackson): This does not account for duplicate user keys + // which may be collapsed. Ideally, we would have 'obsolete keys' + // statistics that would include tombstones, the keys that are + // dropped by tombstones and duplicated user keys. See #847. + if f.Stats.RangeDeletionsBytesEstimate*10 < f.Size && + f.Stats.NumDeletions*10 < f.Stats.NumEntries { + continue + } + lowestCandidate = iter.Take() + lowestCandidateSeqNum = f.LargestSeqNum + } + p.elisionThreshold = &lowestCandidateSeqNum + if lowestCandidate.FileMetadata == nil { + p.elisionCandidate = nil + } else { + p.elisionCandidate = &lowestCandidate + } + } + + // We've already scanned L6 and know that we can perform an L6 + // elision-only compaction only if the sequence number + // `elisionThreshold` is in the last snapshot stripe. + if *p.elisionThreshold >= env.earliestSnapshotSeqNum { + return nil + } + + // Construct a picked compaction of the elision candidate's atomic + // compaction unit. + pc = newPickedCompaction(p.opts, p.vers, numLevels-1, numLevels-1) + pc.startLevel.files = expandToAtomicUnit(p.opts.Comparer.Compare, p.elisionCandidate.Slice()) + + p.elisionThreshold = nil + p.elisionCandidate = nil + + // Fail-safe to protect against compacting the same sstable concurrently. + if !inputAlreadyCompacting(pc) { + return pc + } return nil } diff --git a/github.com/cockroachdb/pebble/db.go b/github.com/cockroachdb/pebble/db.go index 4cc57eab49..06c254736e 100644 --- a/github.com/cockroachdb/pebble/db.go +++ b/github.com/cockroachdb/pebble/db.go @@ -523,6 +523,9 @@ func (d *DB) Apply(batch *Batch, opts *WriteOptions) error { if err := d.closed.Load(); err != nil { panic(err) } + if atomic.LoadUint32(&batch.applied) != 0 { + panic("pebble: batch already applied") + } if d.opts.ReadOnly { return ErrReadOnly } diff --git a/github.com/cockroachdb/pebble/go.mod b/github.com/cockroachdb/pebble/go.mod index e0ed903726..12f0d1d9c6 100644 --- a/github.com/cockroachdb/pebble/go.mod +++ b/github.com/cockroachdb/pebble/go.mod @@ -17,7 +17,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 // indirect - github.com/stretchr/testify v1.2.2 + github.com/stretchr/testify v1.6.1 golang.org/x/exp v0.0.0-20200513190911-00229845015e golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 ) diff --git a/github.com/cockroachdb/pebble/go.sum b/github.com/cockroachdb/pebble/go.sum index 4f9c45333b..c9b86596ec 100644 --- a/github.com/cockroachdb/pebble/go.sum +++ b/github.com/cockroachdb/pebble/go.sum @@ -10,6 +10,7 @@ github.com/cockroachdb/redact v0.0.0-20200622112456-cd282804bbd3 h1:2+dpIJzYMSbL github.com/cockroachdb/redact v0.0.0-20200622112456-cd282804bbd3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs= @@ -38,8 +39,12 @@ github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -63,3 +68,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/github.com/cockroachdb/pebble/ingest.go b/github.com/cockroachdb/pebble/ingest.go index 45ae3bfe87..fc4f1feadb 100644 --- a/github.com/cockroachdb/pebble/ingest.go +++ b/github.com/cockroachdb/pebble/ingest.go @@ -33,11 +33,11 @@ func sstableKeyCompare(userCmp Compare, a, b InternalKey) int { func ingestValidateKey(opts *Options, key *InternalKey) error { if key.Kind() == InternalKeyKindInvalid { - return errors.Errorf("pebble: external sstable has corrupted key: %s", + return base.CorruptionErrorf("pebble: external sstable has corrupted key: %s", key.Pretty(opts.Comparer.FormatKey)) } if key.SeqNum() != 0 { - return errors.Errorf("pebble: external sstable has non-zero seqnum: %s", + return base.CorruptionErrorf("pebble: external sstable has non-zero seqnum: %s", key.Pretty(opts.Comparer.FormatKey)) } return nil @@ -79,10 +79,7 @@ func ingestLoad1( // disallowing removal of an open file. Under MemFS, if we don't populate // meta.Stats here, the file will be loaded into the table cache for // calculating stats before we can remove the original link. - if r.Properties.NumRangeDeletions == 0 { - meta.Stats.Valid = true - meta.Stats.RangeDeletionsBytesEstimate = 0 - } + maybeSetStatsFromProperties(meta, &r.Properties) smallestSet, largestSet := false, false empty := true diff --git a/github.com/cockroachdb/pebble/internal/base/error.go b/github.com/cockroachdb/pebble/internal/base/error.go index 1b1937fe13..6a0303eb5e 100644 --- a/github.com/cockroachdb/pebble/internal/base/error.go +++ b/github.com/cockroachdb/pebble/internal/base/error.go @@ -4,7 +4,27 @@ package base -import "github.com/cockroachdb/errors" +import ( + "github.com/cockroachdb/errors" +) // ErrNotFound means that a get or delete call did not find the requested key. var ErrNotFound = errors.New("pebble: not found") + +// ErrCorruption is a marker to indicate that data in a file (WAL, MANIFEST, +// sstable) isn't in the expected format. +var ErrCorruption = errors.New("pebble: corruption") + +// MarkCorruptionError marks given error as a corruption error. +func MarkCorruptionError(err error) error { + if errors.Is(err, ErrCorruption) { + return err + } + return errors.Mark(err, ErrCorruption) +} + +// CorruptionErrorf formats according to a format specifier and returns +// the string as an error value that is marked as a corruption error. +func CorruptionErrorf(format string, args ...interface{}) error { + return errors.Mark(errors.Newf(format, args...), ErrCorruption) +} diff --git a/github.com/cockroachdb/pebble/internal/manifest/btree.go b/github.com/cockroachdb/pebble/internal/manifest/btree.go new file mode 100644 index 0000000000..8074a9f88e --- /dev/null +++ b/github.com/cockroachdb/pebble/internal/manifest/btree.go @@ -0,0 +1,831 @@ +// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package manifest + +import ( + "strings" + "sync" + "sync/atomic" + "unsafe" +) + +const ( + degree = 16 + maxItems = 2*degree - 1 + minItems = degree - 1 +) + +type leafNode struct { + ref int32 + count int16 + leaf bool + items [maxItems]*FileMetadata +} + +type node struct { + leafNode + children [maxItems + 1]*node +} + +//go:nocheckptr casts a ptr to a smaller struct to a ptr to a larger struct. +func leafToNode(ln *leafNode) *node { + return (*node)(unsafe.Pointer(ln)) +} + +func nodeToLeaf(n *node) *leafNode { + return (*leafNode)(unsafe.Pointer(n)) +} + +var leafPool = sync.Pool{ + New: func() interface{} { + return new(leafNode) + }, +} + +var nodePool = sync.Pool{ + New: func() interface{} { + return new(node) + }, +} + +func newLeafNode() *node { + n := leafToNode(leafPool.Get().(*leafNode)) + n.leaf = true + n.ref = 1 + return n +} + +func newNode() *node { + n := nodePool.Get().(*node) + n.ref = 1 + return n +} + +// mut creates and returns a mutable node reference. If the node is not shared +// with any other trees then it can be modified in place. Otherwise, it must be +// cloned to ensure unique ownership. In this way, we enforce a copy-on-write +// policy which transparently incorporates the idea of local mutations, like +// Clojure's transients or Haskell's ST monad, where nodes are only copied +// during the first time that they are modified between Clone operations. +// +// When a node is cloned, the provided pointer will be redirected to the new +// mutable node. +func mut(n **node) *node { + if atomic.LoadInt32(&(*n).ref) == 1 { + // Exclusive ownership. Can mutate in place. + return *n + } + // If we do not have unique ownership over the node then we + // clone it to gain unique ownership. After doing so, we can + // release our reference to the old node. We pass recursive + // as true because even though we just observed the node's + // reference count to be greater than 1, we might be racing + // with another call to decRef on this node. + c := (*n).clone() + (*n).decRef(true /* recursive */) + *n = c + return *n +} + +// incRef acquires a reference to the node. +func (n *node) incRef() { + atomic.AddInt32(&n.ref, 1) +} + +// decRef releases a reference to the node. If requested, the method +// will recurse into child nodes and decrease their refcounts as well. +func (n *node) decRef(recursive bool) { + if atomic.AddInt32(&n.ref, -1) > 0 { + // Other references remain. Can't free. + return + } + // Clear and release node into memory pool. + if n.leaf { + ln := nodeToLeaf(n) + *ln = leafNode{} + leafPool.Put(ln) + } else { + // Release child references first, if requested. + if recursive { + for i := int16(0); i <= n.count; i++ { + n.children[i].decRef(true /* recursive */) + } + } + *n = node{} + nodePool.Put(n) + } +} + +// clone creates a clone of the receiver with a single reference count. +func (n *node) clone() *node { + var c *node + if n.leaf { + c = newLeafNode() + } else { + c = newNode() + } + // NB: copy field-by-field without touching n.ref to avoid + // triggering the race detector and looking like a data race. + c.count = n.count + c.items = n.items + if !c.leaf { + // Copy children and increase each refcount. + c.children = n.children + for i := int16(0); i <= c.count; i++ { + c.children[i].incRef() + } + } + return c +} + +func (n *node) insertAt(index int, item *FileMetadata, nd *node) { + if index < int(n.count) { + copy(n.items[index+1:n.count+1], n.items[index:n.count]) + if !n.leaf { + copy(n.children[index+2:n.count+2], n.children[index+1:n.count+1]) + } + } + n.items[index] = item + if !n.leaf { + n.children[index+1] = nd + } + n.count++ +} + +func (n *node) pushBack(item *FileMetadata, nd *node) { + n.items[n.count] = item + if !n.leaf { + n.children[n.count+1] = nd + } + n.count++ +} + +func (n *node) pushFront(item *FileMetadata, nd *node) { + if !n.leaf { + copy(n.children[1:n.count+2], n.children[:n.count+1]) + n.children[0] = nd + } + copy(n.items[1:n.count+1], n.items[:n.count]) + n.items[0] = item + n.count++ +} + +// removeAt removes a value at a given index, pulling all subsequent values +// back. +func (n *node) removeAt(index int) (*FileMetadata, *node) { + var child *node + if !n.leaf { + child = n.children[index+1] + copy(n.children[index+1:n.count], n.children[index+2:n.count+1]) + n.children[n.count] = nil + } + n.count-- + out := n.items[index] + copy(n.items[index:n.count], n.items[index+1:n.count+1]) + n.items[n.count] = nil + return out, child +} + +// popBack removes and returns the last element in the list. +func (n *node) popBack() (*FileMetadata, *node) { + n.count-- + out := n.items[n.count] + n.items[n.count] = nil + if n.leaf { + return out, nil + } + child := n.children[n.count+1] + n.children[n.count+1] = nil + return out, child +} + +// popFront removes and returns the first element in the list. +func (n *node) popFront() (*FileMetadata, *node) { + n.count-- + var child *node + if !n.leaf { + child = n.children[0] + copy(n.children[:n.count+1], n.children[1:n.count+2]) + n.children[n.count+1] = nil + } + out := n.items[0] + copy(n.items[:n.count], n.items[1:n.count+1]) + n.items[n.count] = nil + return out, child +} + +// find returns the index where the given item should be inserted into this +// list. 'found' is true if the item already exists in the list at the given +// index. +func (n *node) find(cmp func(*FileMetadata, *FileMetadata) int, item *FileMetadata) (index int, found bool) { + // Logic copied from sort.Search. Inlining this gave + // an 11% speedup on BenchmarkBTreeDeleteInsert. + i, j := 0, int(n.count) + for i < j { + h := int(uint(i+j) >> 1) // avoid overflow when computing h + // i ≤ h < j + v := cmp(item, n.items[h]) + if v == 0 { + return h, true + } else if v > 0 { + i = h + 1 + } else { + j = h + } + } + return i, false +} + +// split splits the given node at the given index. The current node shrinks, +// and this function returns the item that existed at that index and a new +// node containing all items/children after it. +// +// Before: +// +// +-----------+ +// | x y z | +// +--/-/-\-\--+ +// +// After: +// +// +-----------+ +// | y | +// +----/-\----+ +// / \ +// v v +// +-----------+ +-----------+ +// | x | | z | +// +-----------+ +-----------+ +// +func (n *node) split(i int) (*FileMetadata, *node) { + out := n.items[i] + var next *node + if n.leaf { + next = newLeafNode() + } else { + next = newNode() + } + next.count = n.count - int16(i+1) + copy(next.items[:], n.items[i+1:n.count]) + for j := int16(i); j < n.count; j++ { + n.items[j] = nil + } + if !n.leaf { + copy(next.children[:], n.children[i+1:n.count+1]) + for j := int16(i + 1); j <= n.count; j++ { + n.children[j] = nil + } + } + n.count = int16(i) + return out, next +} + +// insert inserts a item into the subtree rooted at this node, making sure no +// nodes in the subtree exceed maxItems items. Returns true if an existing item +// was replaced and false if a item was inserted. +func (n *node) insert(cmp func(*FileMetadata, *FileMetadata) int, item *FileMetadata) (replaced bool) { + i, found := n.find(cmp, item) + if found { + n.items[i] = item + return true + } + if n.leaf { + n.insertAt(i, item, nil) + return false + } + if n.children[i].count >= maxItems { + splitLa, splitNode := mut(&n.children[i]).split(maxItems / 2) + n.insertAt(i, splitLa, splitNode) + + switch cmp := cmp(item, n.items[i]); { + case cmp < 0: + // no change, we want first split node + case cmp > 0: + i++ // we want second split node + default: + n.items[i] = item + return true + } + } + replaced = mut(&n.children[i]).insert(cmp, item) + return replaced +} + +// removeMax removes and returns the maximum item from the subtree rooted +// at this node. +func (n *node) removeMax() *FileMetadata { + if n.leaf { + n.count-- + out := n.items[n.count] + n.items[n.count] = nil + return out + } + child := mut(&n.children[n.count]) + if child.count <= minItems { + n.rebalanceOrMerge(int(n.count)) + return n.removeMax() + } + return child.removeMax() +} + +// remove removes a item from the subtree rooted at this node. Returns +// the item that was removed or nil if no matching item was found. +func (n *node) remove(cmp func(*FileMetadata, *FileMetadata) int, item *FileMetadata) (out *FileMetadata) { + i, found := n.find(cmp, item) + if n.leaf { + if found { + out, _ = n.removeAt(i) + return out + } + return nil + } + if n.children[i].count <= minItems { + // Child not large enough to remove from. + n.rebalanceOrMerge(i) + return n.remove(cmp, item) + } + child := mut(&n.children[i]) + if found { + // Replace the item being removed with the max item in our left child. + out = n.items[i] + n.items[i] = child.removeMax() + return out + } + // Latch is not in this node and child is large enough to remove from. + out = child.remove(cmp, item) + return out +} + +// rebalanceOrMerge grows child 'i' to ensure it has sufficient room to remove +// a item from it while keeping it at or above minItems. +func (n *node) rebalanceOrMerge(i int) { + switch { + case i > 0 && n.children[i-1].count > minItems: + // Rebalance from left sibling. + // + // +-----------+ + // | y | + // +----/-\----+ + // / \ + // v v + // +-----------+ +-----------+ + // | x | | | + // +----------\+ +-----------+ + // \ + // v + // a + // + // After: + // + // +-----------+ + // | x | + // +----/-\----+ + // / \ + // v v + // +-----------+ +-----------+ + // | | | y | + // +-----------+ +/----------+ + // / + // v + // a + // + left := mut(&n.children[i-1]) + child := mut(&n.children[i]) + xLa, grandChild := left.popBack() + yLa := n.items[i-1] + child.pushFront(yLa, grandChild) + n.items[i-1] = xLa + + case i < int(n.count) && n.children[i+1].count > minItems: + // Rebalance from right sibling. + // + // +-----------+ + // | y | + // +----/-\----+ + // / \ + // v v + // +-----------+ +-----------+ + // | | | x | + // +-----------+ +/----------+ + // / + // v + // a + // + // After: + // + // +-----------+ + // | x | + // +----/-\----+ + // / \ + // v v + // +-----------+ +-----------+ + // | y | | | + // +----------\+ +-----------+ + // \ + // v + // a + // + right := mut(&n.children[i+1]) + child := mut(&n.children[i]) + xLa, grandChild := right.popFront() + yLa := n.items[i] + child.pushBack(yLa, grandChild) + n.items[i] = xLa + + default: + // Merge with either the left or right sibling. + // + // +-----------+ + // | u y v | + // +----/-\----+ + // / \ + // v v + // +-----------+ +-----------+ + // | x | | z | + // +-----------+ +-----------+ + // + // After: + // + // +-----------+ + // | u v | + // +-----|-----+ + // | + // v + // +-----------+ + // | x y z | + // +-----------+ + // + if i >= int(n.count) { + i = int(n.count - 1) + } + child := mut(&n.children[i]) + // Make mergeChild mutable, bumping the refcounts on its children if necessary. + _ = mut(&n.children[i+1]) + mergeLa, mergeChild := n.removeAt(i) + child.items[child.count] = mergeLa + copy(child.items[child.count+1:], mergeChild.items[:mergeChild.count]) + if !child.leaf { + copy(child.children[child.count+1:], mergeChild.children[:mergeChild.count+1]) + } + child.count += mergeChild.count + 1 + + mergeChild.decRef(false /* recursive */) + } +} + +// btree is an implementation of a B-Tree. +// +// btree stores FileMetadata in an ordered structure, allowing easy insertion, +// removal, and iteration. The B-Tree stores items in order based on cmp. The +// first level of the LSM uses a cmp function that compares sequence numbers. +// All other levels compare using the FileMetadata.Smallest. +// +// Write operations are not safe for concurrent mutation by multiple +// goroutines, but Read operations are. +type btree struct { + root *node + length int + cmp func(*FileMetadata, *FileMetadata) int +} + +// Reset removes all items from the btree. In doing so, it allows memory +// held by the btree to be recycled. Failure to call this method before +// letting a btree be GCed is safe in that it won't cause a memory leak, +// but it will prevent btree nodes from being efficiently re-used. +func (t *btree) Reset() { + if t.root != nil { + t.root.decRef(true /* recursive */) + t.root = nil + } + t.length = 0 +} + +// Clone clones the btree, lazily. It does so in constant time. +func (t *btree) Clone() btree { + c := *t + if c.root != nil { + // Incrementing the reference count on the root node is sufficient to + // ensure that no node in the cloned tree can be mutated by an actor + // holding a reference to the original tree and vice versa. This + // property is upheld because the root node in the receiver btree and + // the returned btree will both necessarily have a reference count of at + // least 2 when this method returns. All tree mutations recursively + // acquire mutable node references (see mut) as they traverse down the + // tree. The act of acquiring a mutable node reference performs a clone + // if a node's reference count is greater than one. Cloning a node (see + // clone) increases the reference count on each of its children, + // ensuring that they have a reference count of at least 2. This, in + // turn, ensures that any of the child nodes that are modified will also + // be copied-on-write, recursively ensuring the immutability property + // over the entire tree. + c.root.incRef() + } + return c +} + +// Delete removes a item equal to the passed in item from the tree. +func (t *btree) Delete(item *FileMetadata) { + if t.root == nil || t.root.count == 0 { + return + } + if out := mut(&t.root).remove(t.cmp, item); out != nil { + t.length-- + } + if t.root.count == 0 { + old := t.root + if t.root.leaf { + t.root = nil + } else { + t.root = t.root.children[0] + } + old.decRef(false /* recursive */) + } +} + +// Set adds the given item to the tree. If a item in the tree already +// equals the given one, it is replaced with the new item. +func (t *btree) Set(item *FileMetadata) { + if t.root == nil { + t.root = newLeafNode() + } else if t.root.count >= maxItems { + splitLa, splitNode := mut(&t.root).split(maxItems / 2) + newRoot := newNode() + newRoot.count = 1 + newRoot.items[0] = splitLa + newRoot.children[0] = t.root + newRoot.children[1] = splitNode + t.root = newRoot + } + if replaced := mut(&t.root).insert(t.cmp, item); !replaced { + t.length++ + } +} + +// MakeIter returns a new iterator object. It is not safe to continue using an +// iterator after modifications are made to the tree. If modifications are made, +// create a new iterator. +func (t *btree) MakeIter() iterator { + return iterator{r: t.root, pos: -1, cmp: t.cmp} +} + +// Height returns the height of the tree. +func (t *btree) Height() int { + if t.root == nil { + return 0 + } + h := 1 + n := t.root + for !n.leaf { + n = n.children[0] + h++ + } + return h +} + +// Len returns the number of items currently in the tree. +func (t *btree) Len() int { + return t.length +} + +// String returns a string description of the tree. The format is +// similar to the https://en.wikipedia.org/wiki/Newick_format. +func (t *btree) String() string { + if t.length == 0 { + return ";" + } + var b strings.Builder + t.root.writeString(&b) + return b.String() +} + +func (n *node) writeString(b *strings.Builder) { + if n.leaf { + for i := int16(0); i < n.count; i++ { + if i != 0 { + b.WriteString(",") + } + b.WriteString(n.items[i].String()) + } + return + } + for i := int16(0); i <= n.count; i++ { + b.WriteString("(") + n.children[i].writeString(b) + b.WriteString(")") + if i < n.count { + b.WriteString(n.items[i].String()) + } + } +} + +// iterStack represents a stack of (node, pos) tuples, which captures +// iteration state as an iterator descends a btree. +type iterStack struct { + a iterStackArr + aLen int16 // -1 when using s + s []iterFrame +} + +// Used to avoid allocations for stacks below a certain size. +type iterStackArr [3]iterFrame + +type iterFrame struct { + n *node + pos int16 +} + +func (is *iterStack) push(f iterFrame) { + if is.aLen == -1 { + is.s = append(is.s, f) + } else if int(is.aLen) == len(is.a) { + is.s = make([]iterFrame, int(is.aLen)+1, 2*int(is.aLen)) + copy(is.s, is.a[:]) + is.s[int(is.aLen)] = f + is.aLen = -1 + } else { + is.a[is.aLen] = f + is.aLen++ + } +} + +func (is *iterStack) pop() iterFrame { + if is.aLen == -1 { + f := is.s[len(is.s)-1] + is.s = is.s[:len(is.s)-1] + return f + } + is.aLen-- + return is.a[is.aLen] +} + +func (is *iterStack) len() int { + if is.aLen == -1 { + return len(is.s) + } + return int(is.aLen) +} + +func (is *iterStack) reset() { + if is.aLen == -1 { + is.s = is.s[:0] + } else { + is.aLen = 0 + } +} + +// iterator is responsible for search and traversal within a btree. +type iterator struct { + r *node + n *node + pos int16 + cmp func(*FileMetadata, *FileMetadata) int + s iterStack +} + +func (i *iterator) reset() { + i.n = i.r + i.pos = -1 + i.s.reset() +} + +func (i *iterator) descend(n *node, pos int16) { + i.s.push(iterFrame{n: n, pos: pos}) + i.n = n.children[pos] + i.pos = 0 +} + +// ascend ascends up to the current node's parent and resets the position +// to the one previously set for this parent node. +func (i *iterator) ascend() { + f := i.s.pop() + i.n = f.n + i.pos = f.pos +} + +// SeekGE seeks to the first item greater-than or equal to the provided +// item. +func (i *iterator) SeekGE(item *FileMetadata) { + i.reset() + if i.n == nil { + return + } + for { + pos, found := i.n.find(i.cmp, item) + i.pos = int16(pos) + if found { + return + } + if i.n.leaf { + if i.pos == i.n.count { + i.Next() + } + return + } + i.descend(i.n, i.pos) + } +} + +// SeekLT seeks to the first item less-than the provided item. +func (i *iterator) SeekLT(item *FileMetadata) { + i.reset() + if i.n == nil { + return + } + for { + pos, found := i.n.find(i.cmp, item) + i.pos = int16(pos) + if found || i.n.leaf { + i.Prev() + return + } + i.descend(i.n, i.pos) + } +} + +// First seeks to the first item in the btree. +func (i *iterator) First() { + i.reset() + if i.n == nil { + return + } + for !i.n.leaf { + i.descend(i.n, 0) + } + i.pos = 0 +} + +// Last seeks to the last item in the btree. +func (i *iterator) Last() { + i.reset() + if i.n == nil { + return + } + for !i.n.leaf { + i.descend(i.n, i.n.count) + } + i.pos = i.n.count - 1 +} + +// Next positions the iterator to the item immediately following +// its current position. +func (i *iterator) Next() { + if i.n == nil { + return + } + + if i.n.leaf { + i.pos++ + if i.pos < i.n.count { + return + } + for i.s.len() > 0 && i.pos >= i.n.count { + i.ascend() + } + return + } + + i.descend(i.n, i.pos+1) + for !i.n.leaf { + i.descend(i.n, 0) + } + i.pos = 0 +} + +// Prev positions the iterator to the item immediately preceding +// its current position. +func (i *iterator) Prev() { + if i.n == nil { + return + } + + if i.n.leaf { + i.pos-- + if i.pos >= 0 { + return + } + for i.s.len() > 0 && i.pos < 0 { + i.ascend() + i.pos-- + } + return + } + + i.descend(i.n, i.pos) + for !i.n.leaf { + i.descend(i.n, i.n.count) + } + i.pos = i.n.count - 1 +} + +// Valid returns whether the iterator is positioned at a valid position. +func (i *iterator) Valid() bool { + return i.pos >= 0 && i.pos < i.n.count +} + +// Cur returns the item at the iterator's current position. It is illegal +// to call Cur if the iterator is not valid. +func (i *iterator) Cur() *FileMetadata { + return i.n.items[i.pos] +} diff --git a/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go b/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go index b174df930d..a3224229c3 100644 --- a/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go +++ b/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go @@ -94,11 +94,6 @@ func sortAndDedup(keys []intervalKey, cmp Compare) []intervalKey { return keys[:j+1] } -type subLevelAndFile struct { - subLevel int - fileIndex int -} - // A key interval of the form [start, end). The end is not represented here // since it is implicit in the start of the next interval. The last interval is // an exception but we don't need to ever lookup the end of that interval; the @@ -142,10 +137,8 @@ type fileInterval struct { fileCount int compactingFileCount int - // All files in this interval, in increasing sublevel order. Indexes map - // to s.filesByAge. Note that the indexes are not sublevel indexes; the - // sublevel lives at s.filesByAge[files[i]].subLevel. - files []int + // All files in this interval, in increasing sublevel order. + files []*FileMetadata // Interpolated from files in this interval. For files spanning multiple // intervals, we assume an equal distribution of bytes across all those @@ -195,8 +188,8 @@ type L0Sublevels struct { formatKey base.FormatKey fileBytes uint64 - // Contains all files in one slice, ordered from oldest to youngest. - filesByAge []*FileMetadata + // All the L0 files, ordered from oldest to youngest. + levelMetadata *LevelMetadata // The file intervals in increasing key order. orderedIntervals []fileInterval @@ -230,12 +223,13 @@ func insertIntoSubLevel(files []*FileMetadata, f *FileMetadata) []*FileMetadata // IsIntraL0Compacting. Those fields are accessed in InitCompactingFileInfo // instead. func NewL0Sublevels( - files []*FileMetadata, cmp Compare, formatKey base.FormatKey, flushSplitMaxBytes int64, + levelMetadata *LevelMetadata, cmp Compare, formatKey base.FormatKey, flushSplitMaxBytes int64, ) (*L0Sublevels, error) { s := &L0Sublevels{cmp: cmp, formatKey: formatKey} - s.filesByAge = files - keys := make([]intervalKey, 0, 2*len(files)) - for i, f := range s.filesByAge { + s.levelMetadata = levelMetadata + keys := make([]intervalKey, 0, 2*s.levelMetadata.Len()) + iter := levelMetadata.Iter() + for i, f := 0, iter.First(); f != nil; i, f = i+1, iter.Next() { f.l0Index = i keys = append(keys, intervalKey{key: f.Smallest.UserKey}) keys = append(keys, intervalKey{ @@ -256,7 +250,7 @@ func NewL0Sublevels( } // Initialize minIntervalIndex and maxIntervalIndex for each file, and use that // to update intervals. - for fileIndex, f := range s.filesByAge { + for f := iter.First(); f != nil; f = iter.Next() { // Set f.minIntervalIndex and f.maxIntervalIndex. f.minIntervalIndex = sort.Search(len(keys), func(index int) bool { return intervalKeyCompare(cmp, intervalKey{key: f.Smallest.UserKey}, keys[index]) <= 0 @@ -284,8 +278,8 @@ func NewL0Sublevels( for i := f.minIntervalIndex; i <= f.maxIntervalIndex; i++ { interval := &s.orderedIntervals[i] if len(interval.files) > 0 && - subLevel <= s.filesByAge[interval.files[len(interval.files)-1]].subLevel { - subLevel = s.filesByAge[interval.files[len(interval.files)-1]].subLevel + 1 + subLevel <= interval.files[len(interval.files)-1].subLevel { + subLevel = interval.files[len(interval.files)-1].subLevel + 1 } s.orderedIntervals[i].fileCount++ interval.estimatedBytes += interpolatedBytes @@ -298,7 +292,7 @@ func NewL0Sublevels( } for i := f.minIntervalIndex; i <= f.maxIntervalIndex; i++ { interval := &s.orderedIntervals[i] - interval.files = append(interval.files, fileIndex) + interval.files = append(interval.files, f) } f.subLevel = subLevel if subLevel > len(s.Levels) { @@ -337,7 +331,8 @@ func (s *L0Sublevels) InitCompactingFileInfo() { s.orderedIntervals[i].isBaseCompacting = false s.orderedIntervals[i].intervalRangeIsBaseCompacting = false } - for _, f := range s.filesByAge { + iter := s.levelMetadata.Iter() + for f := iter.First(); f != nil; f = iter.Next() { if !f.Compacting { continue } @@ -376,7 +371,7 @@ func (s *L0Sublevels) String() string { func (s *L0Sublevels) describe(verbose bool) string { var buf strings.Builder fmt.Fprintf(&buf, "file count: %d, sublevels: %d, intervals: %d\nflush split keys(%d): [", - len(s.filesByAge), len(s.Levels), len(s.orderedIntervals), len(s.flushSplitUserKeys)) + s.levelMetadata.Len(), len(s.Levels), len(s.orderedIntervals), len(s.flushSplitUserKeys)) for i := range s.flushSplitUserKeys { fmt.Fprintf(&buf, "%s", s.formatKey(s.flushSplitUserKeys[i])) if i < len(s.flushSplitUserKeys)-1 { @@ -408,7 +403,7 @@ func (s *L0Sublevels) describe(verbose bool) string { if verbose { fmt.Fprintf(&buf, "\t%s\n", f) } - if len(s.filesByAge) > 50 && intervals*3 > len(s.orderedIntervals) { + if s.levelMetadata.Len() > 50 && intervals*3 > len(s.orderedIntervals) { var intervalsBytes uint64 for k := f.minIntervalIndex; k <= f.maxIntervalIndex; k++ { intervalsBytes += s.orderedIntervals[k].estimatedBytes @@ -503,7 +498,7 @@ func (s *L0Sublevels) MaxDepthAfterOngoingCompactions() int { // TODO(bilal): Simplify away the debugging statements in this method, and make // this a pure sanity checker. func (s *L0Sublevels) checkCompaction(c *L0CompactionFiles) error { - includedFiles := newBitSet(len(s.filesByAge)) + includedFiles := newBitSet(s.levelMetadata.Len()) fileIntervalsByLevel := make([]struct { min int max int @@ -885,7 +880,7 @@ func (s *L0Sublevels) PickBaseCompaction( // Pick the seed file for the interval as the file // in the lowest sub-level. - f := s.filesByAge[interval.files[0]] + f := interval.files[0] // Don't bother considering the intervals that are // covered by the seed file since they are likely // nearby. Note that it is possible that those intervals @@ -944,7 +939,7 @@ func (s *L0Sublevels) baseCompactionUsingSeed( f *FileMetadata, intervalIndex int, minCompactionDepth int, ) *L0CompactionFiles { c := &L0CompactionFiles{ - FilesIncluded: newBitSet(len(s.filesByAge)), + FilesIncluded: newBitSet(s.levelMetadata.Len()), seedInterval: intervalIndex, seedIntervalMinLevel: 0, minIntervalIndex: f.minIntervalIndex, @@ -960,7 +955,7 @@ func (s *L0Sublevels) baseCompactionUsingSeed( var lastCandidate *L0CompactionFiles interval := &s.orderedIntervals[intervalIndex] for i := 0; i < len(interval.files); i++ { - f2 := s.filesByAge[interval.files[i]] + f2 := interval.files[i] sl := f2.subLevel c.seedIntervalStackDepthReduction++ c.seedIntervalMaxLevel = sl @@ -1097,7 +1092,7 @@ func (s *L0Sublevels) PickIntraL0Compaction( // in the highest sub-level. stackDepthReduction := scoredInterval.score for i := len(interval.files) - 1; i >= 0; i-- { - f = s.filesByAge[interval.files[i]] + f = interval.files[i] if f.Compacting { break } @@ -1146,7 +1141,7 @@ func (s *L0Sublevels) intraL0CompactionUsingSeed( // we need to exclude files >= earliestUnflushedSeqNum c := &L0CompactionFiles{ - FilesIncluded: newBitSet(len(s.filesByAge)), + FilesIncluded: newBitSet(s.levelMetadata.Len()), seedInterval: intervalIndex, seedIntervalMaxLevel: len(s.Levels) - 1, minIntervalIndex: f.minIntervalIndex, @@ -1160,7 +1155,7 @@ func (s *L0Sublevels) intraL0CompactionUsingSeed( interval := &s.orderedIntervals[intervalIndex] slIndex := len(interval.files) - 1 for { - if interval.files[slIndex] == f.l0Index { + if interval.files[slIndex] == f { break } slIndex-- @@ -1172,7 +1167,7 @@ func (s *L0Sublevels) intraL0CompactionUsingSeed( // successful candidate. The code stops adding when it can't add more, or // when fileBytes grows too large. for ; slIndex >= 0; slIndex-- { - f2 := s.filesByAge[interval.files[slIndex]] + f2 := interval.files[slIndex] sl := f2.subLevel if f2.Compacting { break diff --git a/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go b/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go index f735c08b89..52a5e88c99 100644 --- a/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go +++ b/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go @@ -12,13 +12,18 @@ type LevelMetadata struct { files []*FileMetadata } +// Len returns the number of files within the level. +func (lm *LevelMetadata) Len() int { + return len(lm.files) +} + // Iter constructs a LevelIterator over the entire level. -func (lm LevelMetadata) Iter() LevelIterator { +func (lm *LevelMetadata) Iter() LevelIterator { return LevelIterator{files: lm.files, end: len(lm.files)} } // Slice constructs a slice containing the entire level. -func (lm LevelMetadata) Slice() LevelSlice { +func (lm *LevelMetadata) Slice() LevelSlice { return LevelSlice{files: lm.files, end: len(lm.files)} } @@ -49,17 +54,6 @@ type LevelSlice struct { end int } -// Collect returns a Go slice of all files in the slice. The returned slice is -// owned by the LevelSlice. This method is intended to be a temporary adatpter -// between interfaces, and callers should prefer using the iterator directory -// when possible. -// -// TODO(jackson): Revisit once the conversion of Version.Files to a btree is -// complete. -func (ls LevelSlice) Collect() []*FileMetadata { - return ls.files[ls.start:ls.end] -} - // Each invokes fn for each element in the slice. func (ls LevelSlice) Each(fn func(*FileMetadata)) { iter := ls.Iter() diff --git a/github.com/cockroachdb/pebble/internal/manifest/version.go b/github.com/cockroachdb/pebble/internal/manifest/version.go index 393923902d..465cba556c 100644 --- a/github.com/cockroachdb/pebble/internal/manifest/version.go +++ b/github.com/cockroachdb/pebble/internal/manifest/version.go @@ -43,12 +43,20 @@ type TableStats struct { // Valid true if stats have been loaded for the table. The rest of the // structure is populated only if true. Valid bool - // Estimate of the total disk space that may be reclaimed by compacting - // this table's range deletions to the bottom of the LSM. This estimate is - // at data-block granularity and is not updated if compactions beneath the - // table reduce the amount of reclaimable disk space. It also does not - // account for overlapping data in L0 and ignores L0 sublevels, but the - // error that introduces is expected to be small. + // The total number of entries in the table. + NumEntries uint64 + // The number of point and range deletion entries in the table. + NumDeletions uint64 + // Estimate of the total disk space that may be dropped by this table's + // range deletions by compacting them. This estimate is at data-block + // granularity and is not updated if compactions beneath the table reduce + // the amount of reclaimable disk space. It also does not account for + // overlapping data in L0 and ignores L0 sublevels, but the error that + // introduces is expected to be small. + // + // Tables in the bottommost level of the LSM may have a nonzero estimate + // if snapshots or move compactions prevented the elision of their range + // tombstones. RangeDeletionsBytesEstimate uint64 } @@ -101,12 +109,12 @@ func (m *FileMetadata) String() string { // error if inconsistent. func (m *FileMetadata) Validate(cmp Compare, formatKey base.FormatKey) error { if base.InternalCompare(cmp, m.Smallest, m.Largest) > 0 { - return errors.Errorf("file %s has inconsistent bounds: %s vs %s", + return base.CorruptionErrorf("file %s has inconsistent bounds: %s vs %s", errors.Safe(m.FileNum), m.Smallest.Pretty(formatKey), m.Largest.Pretty(formatKey)) } if m.SmallestSeqNum > m.LargestSeqNum { - return errors.Errorf("file %s has inconsistent seqnum bounds: %d vs %d", + return base.CorruptionErrorf("file %s has inconsistent seqnum bounds: %d vs %d", errors.Safe(m.FileNum), m.SmallestSeqNum, m.LargestSeqNum) } return nil @@ -400,7 +408,7 @@ func (v *Version) InitL0Sublevels( cmp Compare, formatKey base.FormatKey, flushSplitBytes int64, ) error { var err error - v.L0Sublevels, err = NewL0Sublevels(v.Levels[0].Slice().Collect(), cmp, formatKey, flushSplitBytes) + v.L0Sublevels, err = NewL0Sublevels(&v.Levels[0], cmp, formatKey, flushSplitBytes) return err } @@ -505,13 +513,13 @@ func (v *Version) CheckOrdering(cmp Compare, format base.FormatKey) error { for sublevel := len(v.L0Sublevels.Levels) - 1; sublevel >= 0; sublevel-- { iter := NewLevelSlice(v.L0Sublevels.Levels[sublevel]).Iter() if err := CheckOrdering(cmp, format, L0Sublevel(sublevel), iter); err != nil { - return errors.Errorf("%s\n%s", err, v.DebugString(format)) + return base.CorruptionErrorf("%s\n%s", err, v.DebugString(format)) } } for level, lm := range v.Levels { if err := CheckOrdering(cmp, format, Level(level), lm.Iter()); err != nil { - return errors.Errorf("%s\n%s", err, v.DebugString(format)) + return base.CorruptionErrorf("%s\n%s", err, v.DebugString(format)) } } return nil @@ -669,7 +677,7 @@ func CheckOrdering(cmp Compare, format base.FormatKey, level Level, files LevelI if prev.LargestSeqNum == 0 && f.LargestSeqNum == prev.LargestSeqNum { // Multiple files satisfying case 2 mentioned above. } else if !prev.lessSeqNum(f) { - return errors.Errorf("L0 files %s and %s are not properly ordered: <#%d-#%d> vs <#%d-#%d>", + return base.CorruptionErrorf("L0 files %s and %s are not properly ordered: <#%d-#%d> vs <#%d-#%d>", errors.Safe(prev.FileNum), errors.Safe(f.FileNum), errors.Safe(prev.SmallestSeqNum), errors.Safe(prev.LargestSeqNum), errors.Safe(f.SmallestSeqNum), errors.Safe(f.LargestSeqNum)) @@ -683,13 +691,13 @@ func CheckOrdering(cmp Compare, format base.FormatKey, level Level, files LevelI } if prev != nil { if !prev.lessSmallestKey(f, cmp) { - return errors.Errorf("%s files %s and %s are not properly ordered: [%s-%s] vs [%s-%s]", + return base.CorruptionErrorf("%s files %s and %s are not properly ordered: [%s-%s] vs [%s-%s]", errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum), prev.Smallest.Pretty(format), prev.Largest.Pretty(format), f.Smallest.Pretty(format), f.Largest.Pretty(format)) } if base.InternalCompare(cmp, prev.Largest, f.Smallest) >= 0 { - return errors.Errorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]", + return base.CorruptionErrorf("%s files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]", errors.Safe(level), errors.Safe(prev.FileNum), errors.Safe(f.FileNum), prev.Smallest.Pretty(format), prev.Largest.Pretty(format), f.Smallest.Pretty(format), f.Largest.Pretty(format)) diff --git a/github.com/cockroachdb/pebble/internal/manifest/version_edit.go b/github.com/cockroachdb/pebble/internal/manifest/version_edit.go index e47f4638c0..3a992ec050 100644 --- a/github.com/cockroachdb/pebble/internal/manifest/version_edit.go +++ b/github.com/cockroachdb/pebble/internal/manifest/version_edit.go @@ -8,7 +8,6 @@ import ( "bufio" "bytes" "encoding/binary" - "fmt" "io" "sort" "sync/atomic" @@ -20,7 +19,7 @@ import ( // TODO(peter): describe the MANIFEST file format, independently of the C++ // project. -var errCorruptManifest = errors.New("pebble: corrupt manifest") +var errCorruptManifest = base.CorruptionErrorf("pebble: corrupt manifest") type byteReader interface { io.ByteReader @@ -233,7 +232,7 @@ func (v *VersionEdit) Decode(r io.Reader) error { switch customTag { case customTagNeedsCompaction: if len(field) != 1 { - return errors.New("new-file4: need-compaction field wrong size") + return base.CorruptionErrorf("new-file4: need-compaction field wrong size") } markedForCompaction = (field[0] == 1) @@ -241,15 +240,15 @@ func (v *VersionEdit) Decode(r io.Reader) error { var n int creationTime, n = binary.Uvarint(field) if n != len(field) { - return errors.New("new-file4: invalid file creation time") + return base.CorruptionErrorf("new-file4: invalid file creation time") } case customTagPathID: - return errors.New("new-file4: path-id field not supported") + return base.CorruptionErrorf("new-file4: path-id field not supported") default: if (customTag & customTagNonSafeIgnoreMask) != 0 { - return errors.Errorf("new-file4: custom field not supported: %d", customTag) + return base.CorruptionErrorf("new-file4: custom field not supported: %d", customTag) } } } @@ -276,7 +275,7 @@ func (v *VersionEdit) Decode(r io.Reader) error { v.ObsoletePrevLogNum = n case tagColumnFamily, tagColumnFamilyAdd, tagColumnFamilyDrop, tagMaxColumnFamily: - return errors.New("column families are not supported") + return base.CorruptionErrorf("column families are not supported") default: return errCorruptManifest @@ -436,7 +435,7 @@ type BulkVersionEdit struct { // Accumulate adds the file addition and deletions in the specified version // edit to the bulk edit's internal state. -func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) { +func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error { for df := range ve.DeletedFiles { dmap := b.Deleted[df.Level] if dmap == nil { @@ -451,11 +450,12 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) { // VersionEdit at the same level (though files can move across levels). if dmap := b.Deleted[nf.Level]; dmap != nil { if _, ok := dmap[nf.Meta.FileNum]; ok { - panic(fmt.Sprintf("file deleted %d before it was inserted\n", nf.Meta.FileNum)) + return base.CorruptionErrorf("file deleted %d before it was inserted\n", nf.Meta.FileNum) } } b.Added[nf.Level] = append(b.Added[nf.Level], nf.Meta) } + return nil } // Apply applies the delta b to the current version to produce a new @@ -519,7 +519,7 @@ func (b *BulkVersionEdit) Apply( deletedMap := b.Deleted[level] n := len(currFiles) + len(addedFiles) if n == 0 { - return nil, nil, errors.Errorf( + return nil, nil, base.CorruptionErrorf( "pebble: internal error: No current or added files but have deleted files: %d", errors.Safe(len(deletedMap))) } @@ -614,7 +614,7 @@ func (b *BulkVersionEdit) Apply( // addedFiles for internal consistency). if base.InternalCompare(cmp, v.Levels[level].files[numFiles-1].Largest, f.Smallest) >= 0 { cf := v.Levels[level].files[numFiles-1] - return nil, nil, errors.Errorf( + return nil, nil, base.CorruptionErrorf( "pebble: internal error: L%d files %s and %s have overlapping ranges: [%s-%s] vs [%s-%s]", errors.Safe(level), errors.Safe(cf.FileNum), errors.Safe(f.FileNum), cf.Smallest.Pretty(formatKey), cf.Largest.Pretty(formatKey), diff --git a/github.com/cockroachdb/pebble/internal/record/record.go b/github.com/cockroachdb/pebble/internal/record/record.go index e223c34ee5..2e9058f3ad 100644 --- a/github.com/cockroachdb/pebble/internal/record/record.go +++ b/github.com/cockroachdb/pebble/internal/record/record.go @@ -141,12 +141,12 @@ var ( // ErrZeroedChunk is returned if a chunk is encountered that is zeroed. This // usually occurs due to log file preallocation. - ErrZeroedChunk = errors.New("pebble/record: zeroed chunk") + ErrZeroedChunk = base.CorruptionErrorf("pebble/record: zeroed chunk") // ErrInvalidChunk is returned if a chunk is encountered with an invalid // header, length, or checksum. This usually occurs when a log is recycled, // but can also occur due to corruption. - ErrInvalidChunk = errors.New("pebble/record: invalid chunk") + ErrInvalidChunk = base.CorruptionErrorf("pebble/record: invalid chunk") ) // IsInvalidRecord returns true if the error matches one of the error types @@ -185,7 +185,7 @@ type Reader struct { // NewReader returns a new reader. If the file contains records encoded using // the recyclable record format, then the log number in those records must -// match the specifed logNum. +// match the specified logNum. func NewReader(r io.Reader, logNum base.FileNum) *Reader { return &Reader{ r: r, diff --git a/github.com/cockroachdb/pebble/iterator.go b/github.com/cockroachdb/pebble/iterator.go index 82ee35167a..f1f8694174 100644 --- a/github.com/cockroachdb/pebble/iterator.go +++ b/github.com/cockroachdb/pebble/iterator.go @@ -9,6 +9,7 @@ import ( "io" "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/internal/base" ) type iterPos int8 @@ -117,7 +118,7 @@ func (i *Iterator) findNextEntry() bool { return i.err == nil default: - i.err = errors.Errorf("pebble: invalid internal key kind: %d", errors.Safe(key.Kind())) + i.err = base.CorruptionErrorf("pebble: invalid internal key kind: %d", errors.Safe(key.Kind())) return false } } @@ -223,7 +224,7 @@ func (i *Iterator) findPrevEntry() bool { continue default: - i.err = errors.Errorf("pebble: invalid internal key kind: %d", errors.Safe(key.Kind())) + i.err = base.CorruptionErrorf("pebble: invalid internal key kind: %d", errors.Safe(key.Kind())) return false } } @@ -300,7 +301,7 @@ func (i *Iterator) mergeNext(key InternalKey, valueMerger ValueMerger) { continue default: - i.err = errors.Errorf("pebble: invalid internal key kind: %d", errors.Safe(key.Kind())) + i.err = base.CorruptionErrorf("pebble: invalid internal key kind: %d", errors.Safe(key.Kind())) return } } diff --git a/github.com/cockroachdb/pebble/mem_table.go b/github.com/cockroachdb/pebble/mem_table.go index 7f1ccac0e1..aa4df8b103 100644 --- a/github.com/cockroachdb/pebble/mem_table.go +++ b/github.com/cockroachdb/pebble/mem_table.go @@ -185,7 +185,7 @@ func (m *memTable) prepare(batch *Batch) error { func (m *memTable) apply(batch *Batch, seqNum uint64) error { if seqNum < m.logSeqNum { - return errors.Errorf("pebble: batch seqnum %d is less than memtable creation seqnum %d", + return base.CorruptionErrorf("pebble: batch seqnum %d is less than memtable creation seqnum %d", errors.Safe(seqNum), errors.Safe(m.logSeqNum)) } @@ -215,8 +215,8 @@ func (m *memTable) apply(batch *Batch, seqNum uint64) error { } } if seqNum != startSeqNum+uint64(batch.Count()) { - panic(errors.Errorf("pebble: inconsistent batch count: %d vs %d", - errors.Safe(seqNum), errors.Safe(startSeqNum+uint64(batch.Count())))) + return base.CorruptionErrorf("pebble: inconsistent batch count: %d vs %d", + errors.Safe(seqNum), errors.Safe(startSeqNum+uint64(batch.Count()))) } if tombstoneCount != 0 { m.tombstones.invalidate(tombstoneCount) diff --git a/github.com/cockroachdb/pebble/open.go b/github.com/cockroachdb/pebble/open.go index dd7fd88570..4808bd992a 100644 --- a/github.com/cockroachdb/pebble/open.go +++ b/github.com/cockroachdb/pebble/open.go @@ -502,7 +502,7 @@ func (d *DB) replayWAL( } if buf.Len() < batchHeaderLen { - return 0, errors.Errorf("pebble: corrupt log file %q (num %s)", + return 0, base.CorruptionErrorf("pebble: corrupt log file %q (num %s)", filename, errors.Safe(logNum)) } diff --git a/github.com/cockroachdb/pebble/options.go b/github.com/cockroachdb/pebble/options.go index eca814dab2..60b50607d8 100644 --- a/github.com/cockroachdb/pebble/options.go +++ b/github.com/cockroachdb/pebble/options.go @@ -729,13 +729,16 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error { case "bytes_per_sync": o.BytesPerSync, err = strconv.Atoi(value) case "cache_size": - n, err := strconv.ParseInt(value, 10, 64) - if err == nil && hooks.NewCache != nil { + var n int64 + n, err = strconv.ParseInt(value, 10, 64) + if err == nil && hooks != nil && hooks.NewCache != nil { if o.Cache != nil { o.Cache.Unref() } o.Cache = hooks.NewCache(n) } + // We avoid calling cache.New in parsing because it makes it + // too easy to leak a cache. case "cleaner": switch value { case "archive": diff --git a/github.com/cockroachdb/pebble/snapshot.go b/github.com/cockroachdb/pebble/snapshot.go index 09ffd6b010..baf9171095 100644 --- a/github.com/cockroachdb/pebble/snapshot.go +++ b/github.com/cockroachdb/pebble/snapshot.go @@ -4,7 +4,10 @@ package pebble -import "io" +import ( + "io" + "math" +) // Snapshot provides a read-only point-in-time view of the DB state. type Snapshot struct { @@ -55,6 +58,12 @@ func (s *Snapshot) Close() error { } s.db.mu.Lock() s.db.mu.snapshots.remove(s) + + // If s was the previous earliest snapshot, we might be able to reclaim + // disk space by dropping obsolete records that were pinned by s. + if e := s.db.mu.snapshots.earliest(); e > s.seqNum { + s.db.maybeScheduleCompactionPicker(pickElisionOnly) + } s.db.mu.Unlock() s.db = nil return nil @@ -73,6 +82,14 @@ func (l *snapshotList) empty() bool { return l.root.next == &l.root } +func (l *snapshotList) earliest() uint64 { + v := uint64(math.MaxUint64) + if !l.empty() { + v = l.root.next.seqNum + } + return v +} + func (l *snapshotList) toSlice() []uint64 { if l.empty() { return nil diff --git a/github.com/cockroachdb/pebble/sstable/block.go b/github.com/cockroachdb/pebble/sstable/block.go index d0a5661d61..846056b4f6 100644 --- a/github.com/cockroachdb/pebble/sstable/block.go +++ b/github.com/cockroachdb/pebble/sstable/block.go @@ -8,7 +8,6 @@ import ( "encoding/binary" "unsafe" - "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/cache" ) @@ -256,7 +255,7 @@ func (i *blockIter) String() string { func (i *blockIter) init(cmp Compare, block block, globalSeqNum uint64) error { numRestarts := int32(binary.LittleEndian.Uint32(block[len(block)-4:])) if numRestarts == 0 { - return errors.New("pebble/table: invalid table (block has no restart points)") + return base.CorruptionErrorf("pebble/table: invalid table (block has no restart points)") } i.cmp = cmp i.restarts = int32(len(block)) - 4*(1+numRestarts) diff --git a/github.com/cockroachdb/pebble/sstable/properties.go b/github.com/cockroachdb/pebble/sstable/properties.go index 611c13e7d3..3457dca97c 100644 --- a/github.com/cockroachdb/pebble/sstable/properties.go +++ b/github.com/cockroachdb/pebble/sstable/properties.go @@ -105,7 +105,8 @@ type Properties struct { MergerName string `prop:"rocksdb.merge.operator"` // The number of blocks in this table. NumDataBlocks uint64 `prop:"rocksdb.num.data.blocks"` - // The number of deletion entries in this table. + // The number of deletion entries in this table, including both point and + // range deletions. NumDeletions uint64 `prop:"rocksdb.deleted.keys"` // The number of entries in this table. NumEntries uint64 `prop:"rocksdb.num.entries"` diff --git a/github.com/cockroachdb/pebble/sstable/raw_block.go b/github.com/cockroachdb/pebble/sstable/raw_block.go index 1c9bd2cd09..82584dcbe0 100644 --- a/github.com/cockroachdb/pebble/sstable/raw_block.go +++ b/github.com/cockroachdb/pebble/sstable/raw_block.go @@ -9,7 +9,7 @@ import ( "sort" "unsafe" - "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/internal/base" ) type rawBlockWriter struct { @@ -57,7 +57,7 @@ func newRawBlockIter(cmp Compare, block block) (*rawBlockIter, error) { func (i *rawBlockIter) init(cmp Compare, block block) error { numRestarts := int32(binary.LittleEndian.Uint32(block[len(block)-4:])) if numRestarts == 0 { - return errors.New("pebble/table: invalid table (block has no restart points)") + return base.CorruptionErrorf("pebble/table: invalid table (block has no restart points)") } i.cmp = cmp i.restarts = int32(len(block)) - 4*(1+numRestarts) diff --git a/github.com/cockroachdb/pebble/sstable/reader.go b/github.com/cockroachdb/pebble/sstable/reader.go index 775aaaa664..05f741b2e9 100644 --- a/github.com/cockroachdb/pebble/sstable/reader.go +++ b/github.com/cockroachdb/pebble/sstable/reader.go @@ -26,7 +26,7 @@ import ( "github.com/golang/snappy" ) -var errCorruptIndexEntry = errors.New("pebble/table: corrupt index entry") +var errCorruptIndexEntry = base.CorruptionErrorf("pebble/table: corrupt index entry") const ( // Constants for dynamic readahead of data blocks. Note that the size values @@ -617,7 +617,7 @@ func (i *twoLevelIterator) loadIndex() bool { } h, n := decodeBlockHandle(i.topLevelIndex.Value()) if n == 0 || n != len(i.topLevelIndex.Value()) { - i.err = errors.New("pebble/table: corrupt top level index entry") + i.err = base.CorruptionErrorf("pebble/table: corrupt top level index entry") return false } indexBlock, err := i.reader.readBlock(h, nil /* transform */, nil /* readaheadState */) @@ -1463,7 +1463,7 @@ func (r *Reader) readBlock( checksum1 := crc.New(b[:bh.Length+1]).Value() if checksum0 != checksum1 { r.opts.Cache.Free(v) - return cache.Handle{}, errors.Newf( + return cache.Handle{}, base.CorruptionErrorf( "pebble/table: invalid table %s (checksum mismatch at %d/%d)", errors.Safe(r.fileNum), errors.Safe(bh.Offset), errors.Safe(bh.Length)) } @@ -1479,7 +1479,7 @@ func (r *Reader) readBlock( decodedLen, err := snappy.DecodedLen(b) if err != nil { r.opts.Cache.Free(v) - return cache.Handle{}, err + return cache.Handle{}, base.MarkCorruptionError(err) } decoded := r.opts.Cache.Alloc(decodedLen) decodedBuf := decoded.Buf() @@ -1487,18 +1487,18 @@ func (r *Reader) readBlock( r.opts.Cache.Free(v) if err != nil { r.opts.Cache.Free(decoded) - return cache.Handle{}, err + return cache.Handle{}, base.MarkCorruptionError(err) } if len(result) != 0 && (len(result) != len(decodedBuf) || &result[0] != &decodedBuf[0]) { r.opts.Cache.Free(decoded) - return cache.Handle{}, errors.Errorf("pebble/table: snappy decoded into unexpected buffer: %p != %p", + return cache.Handle{}, base.CorruptionErrorf("pebble/table: snappy decoded into unexpected buffer: %p != %p", errors.Safe(result), errors.Safe(decodedBuf)) } v, b = decoded, decodedBuf default: r.opts.Cache.Free(v) - return cache.Handle{}, errors.Errorf("pebble/table: unknown block compression: %d", errors.Safe(typ)) + return cache.Handle{}, base.CorruptionErrorf("pebble/table: unknown block compression: %d", errors.Safe(typ)) } if transform != nil { @@ -1572,7 +1572,7 @@ func (r *Reader) readMetaindex(metaindexBH BlockHandle) error { defer b.Release() if uint64(len(data)) != metaindexBH.Length { - return errors.Errorf("pebble/table: unexpected metaindex block size: %d vs %d", + return base.CorruptionErrorf("pebble/table: unexpected metaindex block size: %d vs %d", errors.Safe(len(data)), errors.Safe(metaindexBH.Length)) } @@ -1585,7 +1585,7 @@ func (r *Reader) readMetaindex(metaindexBH BlockHandle) error { for valid := i.First(); valid; valid = i.Next() { bh, n := decodeBlockHandle(i.Value()) if n == 0 { - return errors.New("pebble/table: invalid table (bad filter block handle)") + return base.CorruptionErrorf("pebble/table: invalid table (bad filter block handle)") } meta[string(i.Key().UserKey)] = bh } @@ -1631,7 +1631,7 @@ func (r *Reader) readMetaindex(metaindexBH BlockHandle) error { case TableFilter: r.tableFilter = newTableFilterReader(fp) default: - return errors.Errorf("unknown filter type: %v", errors.Safe(t.ftype)) + return base.CorruptionErrorf("unknown filter type: %v", errors.Safe(t.ftype)) } done = true diff --git a/github.com/cockroachdb/pebble/sstable/table.go b/github.com/cockroachdb/pebble/sstable/table.go index 861d1d7b80..843bf19d69 100644 --- a/github.com/cockroachdb/pebble/sstable/table.go +++ b/github.com/cockroachdb/pebble/sstable/table.go @@ -65,6 +65,7 @@ import ( "io" "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/vfs" ) @@ -201,7 +202,7 @@ func readFooter(f vfs.File) (footer, error) { return footer, errors.Wrap(err, "pebble/table: invalid table (could not stat file)") } if stat.Size() < minFooterLen { - return footer, errors.New("pebble/table: invalid table (file size is too small)") + return footer, base.CorruptionErrorf("pebble/table: invalid table (file size is too small)") } buf := make([]byte, maxFooterLen) @@ -218,7 +219,8 @@ func readFooter(f vfs.File) (footer, error) { switch string(buf[len(buf)-len(rocksDBMagic):]) { case levelDBMagic: if len(buf) < levelDBFooterLen { - return footer, errors.Errorf("pebble/table: invalid table (footer too short): %d", errors.Safe(len(buf))) + return footer, base.CorruptionErrorf( + "pebble/table: invalid table (footer too short): %d", errors.Safe(len(buf))) } footer.footerBH.Offset = uint64(off+int64(len(buf))) - levelDBFooterLen buf = buf[len(buf)-levelDBFooterLen:] @@ -228,37 +230,37 @@ func readFooter(f vfs.File) (footer, error) { case rocksDBMagic: if len(buf) < rocksDBFooterLen { - return footer, errors.Errorf("pebble/table: invalid table (footer too short): %d", errors.Safe(len(buf))) + return footer, base.CorruptionErrorf("pebble/table: invalid table (footer too short): %d", errors.Safe(len(buf))) } footer.footerBH.Offset = uint64(off+int64(len(buf))) - rocksDBFooterLen buf = buf[len(buf)-rocksDBFooterLen:] footer.footerBH.Length = uint64(len(buf)) version := binary.LittleEndian.Uint32(buf[rocksDBVersionOffset:rocksDBMagicOffset]) if version != rocksDBFormatVersion2 { - return footer, errors.Errorf("pebble/table: unsupported format version %d", errors.Safe(version)) + return footer, base.CorruptionErrorf("pebble/table: unsupported format version %d", errors.Safe(version)) } footer.format = TableFormatRocksDBv2 footer.checksum = uint8(buf[0]) if footer.checksum != checksumCRC32c { - return footer, errors.Errorf("pebble/table: unsupported checksum type %d", errors.Safe(footer.checksum)) + return footer, base.CorruptionErrorf("pebble/table: unsupported checksum type %d", errors.Safe(footer.checksum)) } buf = buf[1:] default: - return footer, errors.New("pebble/table: invalid table (bad magic number)") + return footer, base.CorruptionErrorf("pebble/table: invalid table (bad magic number)") } { var n int footer.metaindexBH, n = decodeBlockHandle(buf) if n == 0 { - return footer, errors.New("pebble/table: invalid table (bad metaindex block handle)") + return footer, base.CorruptionErrorf("pebble/table: invalid table (bad metaindex block handle)") } buf = buf[n:] footer.indexBH, n = decodeBlockHandle(buf) if n == 0 { - return footer, errors.New("pebble/table: invalid table (bad index block handle)") + return footer, base.CorruptionErrorf("pebble/table: invalid table (bad index block handle)") } } diff --git a/github.com/cockroachdb/pebble/table_stats.go b/github.com/cockroachdb/pebble/table_stats.go index c2500d0bff..b677ca5edd 100644 --- a/github.com/cockroachdb/pebble/table_stats.go +++ b/github.com/cockroachdb/pebble/table_stats.go @@ -248,9 +248,11 @@ func (d *DB) scanReadStateTableStats( func (d *DB) loadTableStats( v *version, level int, meta *fileMetadata, ) (manifest.TableStats, []deleteCompactionHint, error) { - var totalRangeDeletionEstimate uint64 + var stats manifest.TableStats var compactionHints []deleteCompactionHint err := d.tableCache.withReader(meta, func(r *sstable.Reader) (err error) { + stats.NumEntries = r.Properties.NumEntries + stats.NumDeletions = r.Properties.NumDeletions if r.Properties.NumRangeDeletions == 0 { return nil } @@ -271,11 +273,25 @@ func (d *DB) loadTableStats( d.cmp, rangeDelIter, meta.Smallest.UserKey, meta.Largest.UserKey, nil, nil) err = foreachDefragmentedTombstone(rangeDelIter, d.cmp, func(startUserKey, endUserKey []byte, smallestSeqNum, largestSeqNum uint64) error { + // If the file is in the last level of the LSM, there is no + // data beneath it. The fact that there is still a range + // tombstone in a bottomost file suggests that an open + // snapshot kept the tombstone around. Estimate disk usage + // within the file itself. + if level == numLevels-1 { + size, err := r.EstimateDiskUsage(startUserKey, endUserKey) + if err != nil { + return err + } + stats.RangeDeletionsBytesEstimate += size + return nil + } + estimate, hintSeqNum, err := d.estimateSizeBeneath(v, level, meta, startUserKey, endUserKey) if err != nil { return err } - totalRangeDeletionEstimate += estimate + stats.RangeDeletionsBytesEstimate += estimate // If any files were completely contained with the range, // hintSeqNum is the smallest sequence number contained in any @@ -299,12 +315,10 @@ func (d *DB) loadTableStats( }) return err }) - var stats manifest.TableStats if err != nil { return stats, nil, err } stats.Valid = true - stats.RangeDeletionsBytesEstimate = totalRangeDeletionEstimate return stats, compactionHints, nil } @@ -413,3 +427,20 @@ func foreachDefragmentedTombstone( } return rangeDelIter.Close() } + +func maybeSetStatsFromProperties(meta *fileMetadata, props *sstable.Properties) bool { + // If a table has range deletions, we can't calculate the + // RangeDeletionsBytesEstimate statistic and can't populate table stats + // from just the properties. The table stats collector goroutine will + // populate the stats. + if props.NumRangeDeletions != 0 { + return false + } + meta.Stats = manifest.TableStats{ + Valid: true, + NumEntries: props.NumEntries, + NumDeletions: props.NumDeletions, + RangeDeletionsBytesEstimate: 0, + } + return true +} diff --git a/github.com/cockroachdb/pebble/tool/db.go b/github.com/cockroachdb/pebble/tool/db.go index 10eb8b1484..56c2979060 100644 --- a/github.com/cockroachdb/pebble/tool/db.go +++ b/github.com/cockroachdb/pebble/tool/db.go @@ -372,7 +372,9 @@ func (d *dbT) runProperties(cmd *cobra.Command, args []string) { if err != nil { return err } - bve.Accumulate(&ve) + if err := bve.Accumulate(&ve); err != nil { + return err + } if ve.ComparerName != "" { cmp = d.comparers[ve.ComparerName] d.fmtKey.setForComparer(ve.ComparerName, d.comparers) diff --git a/github.com/cockroachdb/pebble/tool/lsm.go b/github.com/cockroachdb/pebble/tool/lsm.go index 10195747b4..7c1690e4c3 100644 --- a/github.com/cockroachdb/pebble/tool/lsm.go +++ b/github.com/cockroachdb/pebble/tool/lsm.go @@ -240,15 +240,12 @@ func (l *lsmT) buildEdits(edits []*manifest.VersionEdit) { edit.Added[nf.Level] = append(edit.Added[nf.Level], nf.Meta.FileNum) currentFiles[nf.Level] = append(currentFiles[nf.Level], nf.Meta) } - sublevels, err := manifest.NewL0Sublevels(currentFiles[0], l.cmp.Compare, l.fmtKey.fn, 0) - if err != nil { - panic(err) - } + v := manifest.NewVersion(l.cmp.Compare, l.fmtKey.fn, 0, currentFiles) edit.Sublevels = make(map[base.FileNum]int) - for sublevel, files := range sublevels.Levels { + for sublevel, files := range v.L0Sublevels.Levels { for _, f := range files { if len(l.state.Edits) > 0 { - lastEdit := l.state.Edits[len(l.state.Edits) - 1] + lastEdit := l.state.Edits[len(l.state.Edits)-1] if sublevel2, ok := lastEdit.Sublevels[f.FileNum]; ok && sublevel == sublevel2 { continue } diff --git a/github.com/cockroachdb/pebble/tool/manifest.go b/github.com/cockroachdb/pebble/tool/manifest.go index 7b3db6f5d9..654c1f1b90 100644 --- a/github.com/cockroachdb/pebble/tool/manifest.go +++ b/github.com/cockroachdb/pebble/tool/manifest.go @@ -131,7 +131,10 @@ func (m *manifestT) runDump(cmd *cobra.Command, args []string) { fmt.Fprintf(stdout, "%s\n", err) break } - bve.Accumulate(&ve) + if err := bve.Accumulate(&ve); err != nil { + fmt.Fprintf(stdout, "%s\n", err) + break + } empty := true fmt.Fprintf(stdout, "%d\n", offset) @@ -242,7 +245,11 @@ func (m *manifestT) runCheck(cmd *cobra.Command, args []string) { break } var bve manifest.BulkVersionEdit - bve.Accumulate(&ve) + if err := bve.Accumulate(&ve); err != nil { + fmt.Fprintf(stderr, "%s\n", err) + ok = false + return + } empty := true if ve.ComparerName != "" { diff --git a/github.com/cockroachdb/pebble/version_set.go b/github.com/cockroachdb/pebble/version_set.go index 6d5cfca011..6265c8eafa 100644 --- a/github.com/cockroachdb/pebble/version_set.go +++ b/github.com/cockroachdb/pebble/version_set.go @@ -38,6 +38,7 @@ type versionList = manifest.VersionList type versionSet struct { // Immutable fields. dirname string + // Set to DB.mu. mu *sync.Mutex opts *Options fs vfs.FS @@ -177,13 +178,13 @@ func (vs *versionSet) load(dirname string, opts *Options, mu *sync.Mutex) error return err } if b[n-1] != '\n' { - return errors.Errorf("pebble: CURRENT file for DB %q is malformed", dirname) + return base.CorruptionErrorf("pebble: CURRENT file for DB %q is malformed", dirname) } b = bytes.TrimSpace(b) var ok bool if _, vs.manifestFileNum, ok = base.ParseFilename(vs.fs, string(b)); !ok { - return errors.Errorf("pebble: MANIFEST name %q is malformed", errors.Safe(b)) + return base.CorruptionErrorf("pebble: MANIFEST name %q is malformed", errors.Safe(b)) } // Read the versionEdits in the manifest file. @@ -221,7 +222,9 @@ func (vs *versionSet) load(dirname string, opts *Options, mu *sync.Mutex) error errors.Safe(b), dirname, errors.Safe(ve.ComparerName), errors.Safe(vs.cmpName)) } } - bve.Accumulate(&ve) + if err := bve.Accumulate(&ve); err != nil { + return err + } if ve.MinUnflushedLogNum != 0 { vs.minUnflushedLogNum = ve.MinUnflushedLogNum } @@ -250,7 +253,7 @@ func (vs *versionSet) load(dirname string, opts *Options, mu *sync.Mutex) error // minUnflushedLogNum, even if WALs with non-zero file numbers are // present in the directory. } else { - return errors.Errorf("pebble: malformed manifest file %q for DB %q", + return base.CorruptionErrorf("pebble: malformed manifest file %q for DB %q", errors.Safe(b), dirname) } } @@ -384,7 +387,9 @@ func (vs *versionSet) logAndApply( defer vs.mu.Lock() var bve bulkVersionEdit - bve.Accumulate(ve) + if err := bve.Accumulate(ve); err != nil { + return err + } var err error newVersion, zombies, err = bve.Apply(currentVersion, vs.cmp, vs.opts.Comparer.FormatKey, vs.opts.Experimental.FlushSplitBytes) diff --git a/golang.org/x/sys/cpu/cpu.go b/golang.org/x/sys/cpu/cpu.go index e44deb7574..5cce25ed9b 100644 --- a/golang.org/x/sys/cpu/cpu.go +++ b/golang.org/x/sys/cpu/cpu.go @@ -6,6 +6,11 @@ // various CPU architectures. package cpu +import ( + "os" + "strings" +) + // Initialized reports whether the CPU features were initialized. // // For some GOOS/GOARCH combinations initialization of the CPU features depends @@ -169,3 +174,94 @@ var S390X struct { HasVXE bool // vector-enhancements facility 1 _ CacheLinePad } + +func init() { + archInit() + initOptions() + processOptions() +} + +// options contains the cpu debug options that can be used in GODEBUG. +// Options are arch dependent and are added by the arch specific initOptions functions. +// Features that are mandatory for the specific GOARCH should have the Required field set +// (e.g. SSE2 on amd64). +var options []option + +// Option names should be lower case. e.g. avx instead of AVX. +type option struct { + Name string + Feature *bool + Specified bool // whether feature value was specified in GODEBUG + Enable bool // whether feature should be enabled + Required bool // whether feature is mandatory and can not be disabled +} + +func processOptions() { + env := os.Getenv("GODEBUG") +field: + for env != "" { + field := "" + i := strings.IndexByte(env, ',') + if i < 0 { + field, env = env, "" + } else { + field, env = env[:i], env[i+1:] + } + if len(field) < 4 || field[:4] != "cpu." { + continue + } + i = strings.IndexByte(field, '=') + if i < 0 { + print("GODEBUG sys/cpu: no value specified for \"", field, "\"\n") + continue + } + key, value := field[4:i], field[i+1:] // e.g. "SSE2", "on" + + var enable bool + switch value { + case "on": + enable = true + case "off": + enable = false + default: + print("GODEBUG sys/cpu: value \"", value, "\" not supported for cpu option \"", key, "\"\n") + continue field + } + + if key == "all" { + for i := range options { + options[i].Specified = true + options[i].Enable = enable || options[i].Required + } + continue field + } + + for i := range options { + if options[i].Name == key { + options[i].Specified = true + options[i].Enable = enable + continue field + } + } + + print("GODEBUG sys/cpu: unknown cpu feature \"", key, "\"\n") + } + + for _, o := range options { + if !o.Specified { + continue + } + + if o.Enable && !*o.Feature { + print("GODEBUG sys/cpu: can not enable \"", o.Name, "\", missing CPU support\n") + continue + } + + if !o.Enable && o.Required { + print("GODEBUG sys/cpu: can not disable \"", o.Name, "\", required CPU feature\n") + continue + } + + *o.Feature = o.Enable + } +} diff --git a/golang.org/x/sys/cpu/cpu_aix.go b/golang.org/x/sys/cpu/cpu_aix.go index da29896687..464a209cf5 100644 --- a/golang.org/x/sys/cpu/cpu_aix.go +++ b/golang.org/x/sys/cpu/cpu_aix.go @@ -6,8 +6,6 @@ package cpu -const cacheLineSize = 128 - const ( // getsystemcfg constants _SC_IMPL = 2 @@ -15,7 +13,7 @@ const ( _IMPL_POWER9 = 0x20000 ) -func init() { +func archInit() { impl := getsystemcfg(_SC_IMPL) if impl&_IMPL_POWER8 != 0 { PPC64.IsPOWER8 = true diff --git a/golang.org/x/sys/cpu/cpu_arm.go b/golang.org/x/sys/cpu/cpu_arm.go index 981af6818c..301b752e9c 100644 --- a/golang.org/x/sys/cpu/cpu_arm.go +++ b/golang.org/x/sys/cpu/cpu_arm.go @@ -38,3 +38,36 @@ const ( hwcap2_SHA2 = 1 << 3 hwcap2_CRC32 = 1 << 4 ) + +func initOptions() { + options = []option{ + {Name: "pmull", Feature: &ARM.HasPMULL}, + {Name: "sha1", Feature: &ARM.HasSHA1}, + {Name: "sha2", Feature: &ARM.HasSHA2}, + {Name: "swp", Feature: &ARM.HasSWP}, + {Name: "thumb", Feature: &ARM.HasTHUMB}, + {Name: "thumbee", Feature: &ARM.HasTHUMBEE}, + {Name: "tls", Feature: &ARM.HasTLS}, + {Name: "vfp", Feature: &ARM.HasVFP}, + {Name: "vfpd32", Feature: &ARM.HasVFPD32}, + {Name: "vfpv3", Feature: &ARM.HasVFPv3}, + {Name: "vfpv3d16", Feature: &ARM.HasVFPv3D16}, + {Name: "vfpv4", Feature: &ARM.HasVFPv4}, + {Name: "half", Feature: &ARM.HasHALF}, + {Name: "26bit", Feature: &ARM.Has26BIT}, + {Name: "fastmul", Feature: &ARM.HasFASTMUL}, + {Name: "fpa", Feature: &ARM.HasFPA}, + {Name: "edsp", Feature: &ARM.HasEDSP}, + {Name: "java", Feature: &ARM.HasJAVA}, + {Name: "iwmmxt", Feature: &ARM.HasIWMMXT}, + {Name: "crunch", Feature: &ARM.HasCRUNCH}, + {Name: "neon", Feature: &ARM.HasNEON}, + {Name: "idivt", Feature: &ARM.HasIDIVT}, + {Name: "idiva", Feature: &ARM.HasIDIVA}, + {Name: "lpae", Feature: &ARM.HasLPAE}, + {Name: "evtstrm", Feature: &ARM.HasEVTSTRM}, + {Name: "aes", Feature: &ARM.HasAES}, + {Name: "crc32", Feature: &ARM.HasCRC32}, + } + +} diff --git a/golang.org/x/sys/cpu/cpu_arm64.go b/golang.org/x/sys/cpu/cpu_arm64.go index 7bcb36c7bb..2d90024387 100644 --- a/golang.org/x/sys/cpu/cpu_arm64.go +++ b/golang.org/x/sys/cpu/cpu_arm64.go @@ -8,7 +8,36 @@ import "runtime" const cacheLineSize = 64 -func init() { +func initOptions() { + options = []option{ + {Name: "fp", Feature: &ARM64.HasFP}, + {Name: "asimd", Feature: &ARM64.HasASIMD}, + {Name: "evstrm", Feature: &ARM64.HasEVTSTRM}, + {Name: "aes", Feature: &ARM64.HasAES}, + {Name: "fphp", Feature: &ARM64.HasFPHP}, + {Name: "jscvt", Feature: &ARM64.HasJSCVT}, + {Name: "lrcpc", Feature: &ARM64.HasLRCPC}, + {Name: "pmull", Feature: &ARM64.HasPMULL}, + {Name: "sha1", Feature: &ARM64.HasSHA1}, + {Name: "sha2", Feature: &ARM64.HasSHA2}, + {Name: "sha3", Feature: &ARM64.HasSHA3}, + {Name: "sha512", Feature: &ARM64.HasSHA512}, + {Name: "sm3", Feature: &ARM64.HasSM3}, + {Name: "sm4", Feature: &ARM64.HasSM4}, + {Name: "sve", Feature: &ARM64.HasSVE}, + {Name: "crc32", Feature: &ARM64.HasCRC32}, + {Name: "atomics", Feature: &ARM64.HasATOMICS}, + {Name: "asimdhp", Feature: &ARM64.HasASIMDHP}, + {Name: "cpuid", Feature: &ARM64.HasCPUID}, + {Name: "asimrdm", Feature: &ARM64.HasASIMDRDM}, + {Name: "fcma", Feature: &ARM64.HasFCMA}, + {Name: "dcpop", Feature: &ARM64.HasDCPOP}, + {Name: "asimddp", Feature: &ARM64.HasASIMDDP}, + {Name: "asimdfhm", Feature: &ARM64.HasASIMDFHM}, + } +} + +func archInit() { switch runtime.GOOS { case "android", "darwin", "netbsd": // Android and iOS don't seem to allow reading these registers. diff --git a/golang.org/x/sys/cpu/cpu_linux.go b/golang.org/x/sys/cpu/cpu_linux.go index fe139182c8..6fc874f7fe 100644 --- a/golang.org/x/sys/cpu/cpu_linux.go +++ b/golang.org/x/sys/cpu/cpu_linux.go @@ -6,7 +6,7 @@ package cpu -func init() { +func archInit() { if err := readHWCAP(); err != nil { return } diff --git a/golang.org/x/sys/cpu/cpu_linux_ppc64x.go b/golang.org/x/sys/cpu/cpu_linux_ppc64x.go index 6c8d975d40..99f8a6399e 100644 --- a/golang.org/x/sys/cpu/cpu_linux_ppc64x.go +++ b/golang.org/x/sys/cpu/cpu_linux_ppc64x.go @@ -7,8 +7,6 @@ package cpu -const cacheLineSize = 128 - // HWCAP/HWCAP2 bits. These are exposed by the kernel. const ( // ISA Level diff --git a/golang.org/x/sys/cpu/cpu_linux_s390x.go b/golang.org/x/sys/cpu/cpu_linux_s390x.go index d579eaef40..b88d6b8f66 100644 --- a/golang.org/x/sys/cpu/cpu_linux_s390x.go +++ b/golang.org/x/sys/cpu/cpu_linux_s390x.go @@ -4,8 +4,6 @@ package cpu -const cacheLineSize = 256 - const ( // bit mask values from /usr/include/bits/hwcap.h hwcap_ZARCH = 2 diff --git a/golang.org/x/sys/cpu/cpu_mips64x.go b/golang.org/x/sys/cpu/cpu_mips64x.go index 6165f12124..57b5b677de 100644 --- a/golang.org/x/sys/cpu/cpu_mips64x.go +++ b/golang.org/x/sys/cpu/cpu_mips64x.go @@ -7,3 +7,9 @@ package cpu const cacheLineSize = 32 + +func initOptions() { + options = []option{ + {Name: "msa", Feature: &MIPS64X.HasMSA}, + } +} diff --git a/golang.org/x/sys/cpu/cpu_mipsx.go b/golang.org/x/sys/cpu/cpu_mipsx.go index 1269eee88d..cfc1946b7b 100644 --- a/golang.org/x/sys/cpu/cpu_mipsx.go +++ b/golang.org/x/sys/cpu/cpu_mipsx.go @@ -7,3 +7,5 @@ package cpu const cacheLineSize = 32 + +func initOptions() {} diff --git a/golang.org/x/sys/cpu/cpu_other_arm.go b/golang.org/x/sys/cpu/cpu_other_arm.go new file mode 100644 index 0000000000..b412efc1bd --- /dev/null +++ b/golang.org/x/sys/cpu/cpu_other_arm.go @@ -0,0 +1,9 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !linux,arm + +package cpu + +func archInit() {} diff --git a/golang.org/x/sys/cpu/cpu_ppc64x.go b/golang.org/x/sys/cpu/cpu_ppc64x.go new file mode 100644 index 0000000000..d28d675b5f --- /dev/null +++ b/golang.org/x/sys/cpu/cpu_ppc64x.go @@ -0,0 +1,16 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ppc64 ppc64le + +package cpu + +const cacheLineSize = 128 + +func initOptions() { + options = []option{ + {Name: "darn", Feature: &PPC64.HasDARN}, + {Name: "scv", Feature: &PPC64.HasSCV}, + } +} diff --git a/golang.org/x/sys/cpu/cpu_riscv64.go b/golang.org/x/sys/cpu/cpu_riscv64.go index efe2b7a847..8b08de341b 100644 --- a/golang.org/x/sys/cpu/cpu_riscv64.go +++ b/golang.org/x/sys/cpu/cpu_riscv64.go @@ -7,3 +7,5 @@ package cpu const cacheLineSize = 32 + +func initOptions() {} diff --git a/golang.org/x/sys/cpu/cpu_s390x.go b/golang.org/x/sys/cpu/cpu_s390x.go new file mode 100644 index 0000000000..544cd621ce --- /dev/null +++ b/golang.org/x/sys/cpu/cpu_s390x.go @@ -0,0 +1,30 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +const cacheLineSize = 256 + +func initOptions() { + options = []option{ + {Name: "zarch", Feature: &S390X.HasZARCH}, + {Name: "stfle", Feature: &S390X.HasSTFLE}, + {Name: "ldisp", Feature: &S390X.HasLDISP}, + {Name: "eimm", Feature: &S390X.HasEIMM}, + {Name: "dfp", Feature: &S390X.HasDFP}, + {Name: "etf3eh", Feature: &S390X.HasETF3EH}, + {Name: "msa", Feature: &S390X.HasMSA}, + {Name: "aes", Feature: &S390X.HasAES}, + {Name: "aescbc", Feature: &S390X.HasAESCBC}, + {Name: "aesctr", Feature: &S390X.HasAESCTR}, + {Name: "aesgcm", Feature: &S390X.HasAESGCM}, + {Name: "ghash", Feature: &S390X.HasGHASH}, + {Name: "sha1", Feature: &S390X.HasSHA1}, + {Name: "sha256", Feature: &S390X.HasSHA256}, + {Name: "sha3", Feature: &S390X.HasSHA3}, + {Name: "sha512", Feature: &S390X.HasSHA512}, + {Name: "vx", Feature: &S390X.HasVX}, + {Name: "vxe", Feature: &S390X.HasVXE}, + } +} diff --git a/golang.org/x/sys/cpu/cpu_wasm.go b/golang.org/x/sys/cpu/cpu_wasm.go index 8681e876a9..5382f2a227 100644 --- a/golang.org/x/sys/cpu/cpu_wasm.go +++ b/golang.org/x/sys/cpu/cpu_wasm.go @@ -11,3 +11,7 @@ package cpu // rules are good enough. const cacheLineSize = 0 + +func initOptions() {} + +func archInit() {} diff --git a/golang.org/x/sys/cpu/cpu_x86.go b/golang.org/x/sys/cpu/cpu_x86.go index d70d317f5a..2ad039d40e 100644 --- a/golang.org/x/sys/cpu/cpu_x86.go +++ b/golang.org/x/sys/cpu/cpu_x86.go @@ -6,9 +6,37 @@ package cpu +import "runtime" + const cacheLineSize = 64 -func init() { +func initOptions() { + options = []option{ + {Name: "adx", Feature: &X86.HasADX}, + {Name: "aes", Feature: &X86.HasAES}, + {Name: "avx", Feature: &X86.HasAVX}, + {Name: "avx2", Feature: &X86.HasAVX2}, + {Name: "bmi1", Feature: &X86.HasBMI1}, + {Name: "bmi2", Feature: &X86.HasBMI2}, + {Name: "erms", Feature: &X86.HasERMS}, + {Name: "fma", Feature: &X86.HasFMA}, + {Name: "osxsave", Feature: &X86.HasOSXSAVE}, + {Name: "pclmulqdq", Feature: &X86.HasPCLMULQDQ}, + {Name: "popcnt", Feature: &X86.HasPOPCNT}, + {Name: "rdrand", Feature: &X86.HasRDRAND}, + {Name: "rdseed", Feature: &X86.HasRDSEED}, + {Name: "sse3", Feature: &X86.HasSSE3}, + {Name: "sse41", Feature: &X86.HasSSE41}, + {Name: "sse42", Feature: &X86.HasSSE42}, + {Name: "ssse3", Feature: &X86.HasSSSE3}, + + // These capabilities should always be enabled on amd64: + {Name: "sse2", Feature: &X86.HasSSE2, Required: runtime.GOARCH == "amd64"}, + } +} + +func archInit() { + Initialized = true maxID, _, _, _ := cpuid(0, 0) @@ -52,6 +80,7 @@ func init() { X86.HasERMS = isSet(9, ebx7) X86.HasRDSEED = isSet(18, ebx7) X86.HasADX = isSet(19, ebx7) + } func isSet(bitpos uint, value uint32) bool { diff --git a/golang.org/x/sys/unix/mkerrors.sh b/golang.org/x/sys/unix/mkerrors.sh index 53a2493126..40534f0312 100644 --- a/golang.org/x/sys/unix/mkerrors.sh +++ b/golang.org/x/sys/unix/mkerrors.sh @@ -196,6 +196,7 @@ struct ltchars { #include #include #include +#include #include #include #include @@ -517,6 +518,7 @@ ccflags="$@" $2 ~ /^FS_IOC_.*(ENCRYPTION|VERITY|[GS]ETFLAGS)/ || $2 ~ /^FS_VERITY_/ || $2 ~ /^FSCRYPT_/ || + $2 ~ /^DM_/ || $2 ~ /^GRND_/ || $2 ~ /^RND/ || $2 ~ /^KEY_(SPEC|REQKEY_DEFL)_/ || diff --git a/golang.org/x/sys/unix/syscall_linux.go b/golang.org/x/sys/unix/syscall_linux.go index fad483bb9d..c48f5ddadd 100644 --- a/golang.org/x/sys/unix/syscall_linux.go +++ b/golang.org/x/sys/unix/syscall_linux.go @@ -885,6 +885,35 @@ func (sa *SockaddrL2TPIP6) sockaddr() (unsafe.Pointer, _Socklen, error) { return unsafe.Pointer(&sa.raw), SizeofSockaddrL2TPIP6, nil } +// SockaddrIUCV implements the Sockaddr interface for AF_IUCV sockets. +type SockaddrIUCV struct { + UserID string + Name string + raw RawSockaddrIUCV +} + +func (sa *SockaddrIUCV) sockaddr() (unsafe.Pointer, _Socklen, error) { + sa.raw.Family = AF_IUCV + // These are EBCDIC encoded by the kernel, but we still need to pad them + // with blanks. Initializing with blanks allows the caller to feed in either + // a padded or an unpadded string. + for i := 0; i < 8; i++ { + sa.raw.Nodeid[i] = ' ' + sa.raw.User_id[i] = ' ' + sa.raw.Name[i] = ' ' + } + if len(sa.UserID) > 8 || len(sa.Name) > 8 { + return nil, 0, EINVAL + } + for i, b := range []byte(sa.UserID[:]) { + sa.raw.User_id[i] = int8(b) + } + for i, b := range []byte(sa.Name[:]) { + sa.raw.Name[i] = int8(b) + } + return unsafe.Pointer(&sa.raw), SizeofSockaddrIUCV, nil +} + func anyToSockaddr(fd int, rsa *RawSockaddrAny) (Sockaddr, error) { switch rsa.Addr.Family { case AF_NETLINK: @@ -1065,6 +1094,23 @@ func anyToSockaddr(fd int, rsa *RawSockaddrAny) (Sockaddr, error) { } return sa, nil + case AF_IUCV: + pp := (*RawSockaddrIUCV)(unsafe.Pointer(rsa)) + + var user [8]byte + var name [8]byte + + for i := 0; i < 8; i++ { + user[i] = byte(pp.User_id[i]) + name[i] = byte(pp.Name[i]) + } + + sa := &SockaddrIUCV{ + UserID: string(user[:]), + Name: string(name[:]), + } + return sa, nil + } return nil, EAFNOSUPPORT } @@ -1965,10 +2011,15 @@ func isGroupMember(gid int) bool { } //sys faccessat(dirfd int, path string, mode uint32) (err error) +//sys Faccessat2(dirfd int, path string, mode uint32, flags int) (err error) func Faccessat(dirfd int, path string, mode uint32, flags int) (err error) { - if flags & ^(AT_SYMLINK_NOFOLLOW|AT_EACCESS) != 0 { - return EINVAL + if flags == 0 { + return faccessat(dirfd, path, mode) + } + + if err := Faccessat2(dirfd, path, mode, flags); err != ENOSYS && err != EPERM { + return err } // The Linux kernel faccessat system call does not take any flags. @@ -1977,8 +2028,8 @@ func Faccessat(dirfd int, path string, mode uint32, flags int) (err error) { // Because people naturally expect syscall.Faccessat to act // like C faccessat, we do the same. - if flags == 0 { - return faccessat(dirfd, path, mode) + if flags & ^(AT_SYMLINK_NOFOLLOW|AT_EACCESS) != 0 { + return EINVAL } var st Stat_t diff --git a/golang.org/x/sys/unix/zerrors_linux.go b/golang.org/x/sys/unix/zerrors_linux.go index f8bd50c11b..e80d4e964b 100644 --- a/golang.org/x/sys/unix/zerrors_linux.go +++ b/golang.org/x/sys/unix/zerrors_linux.go @@ -265,6 +265,7 @@ const ( CAP_AUDIT_READ = 0x25 CAP_AUDIT_WRITE = 0x1d CAP_BLOCK_SUSPEND = 0x24 + CAP_BPF = 0x27 CAP_CHOWN = 0x0 CAP_DAC_OVERRIDE = 0x1 CAP_DAC_READ_SEARCH = 0x2 @@ -273,7 +274,7 @@ const ( CAP_IPC_LOCK = 0xe CAP_IPC_OWNER = 0xf CAP_KILL = 0x5 - CAP_LAST_CAP = 0x25 + CAP_LAST_CAP = 0x27 CAP_LEASE = 0x1c CAP_LINUX_IMMUTABLE = 0x9 CAP_MAC_ADMIN = 0x21 @@ -283,6 +284,7 @@ const ( CAP_NET_BIND_SERVICE = 0xa CAP_NET_BROADCAST = 0xb CAP_NET_RAW = 0xd + CAP_PERFMON = 0x26 CAP_SETFCAP = 0x1f CAP_SETGID = 0x6 CAP_SETPCAP = 0x8 @@ -372,8 +374,54 @@ const ( DEVLINK_GENL_NAME = "devlink" DEVLINK_GENL_VERSION = 0x1 DEVLINK_SB_THRESHOLD_TO_ALPHA_MAX = 0x14 + DEVMEM_MAGIC = 0x454d444d DEVPTS_SUPER_MAGIC = 0x1cd1 DMA_BUF_MAGIC = 0x444d4142 + DM_ACTIVE_PRESENT_FLAG = 0x20 + DM_BUFFER_FULL_FLAG = 0x100 + DM_CONTROL_NODE = "control" + DM_DATA_OUT_FLAG = 0x10000 + DM_DEFERRED_REMOVE = 0x20000 + DM_DEV_ARM_POLL = 0xc138fd10 + DM_DEV_CREATE = 0xc138fd03 + DM_DEV_REMOVE = 0xc138fd04 + DM_DEV_RENAME = 0xc138fd05 + DM_DEV_SET_GEOMETRY = 0xc138fd0f + DM_DEV_STATUS = 0xc138fd07 + DM_DEV_SUSPEND = 0xc138fd06 + DM_DEV_WAIT = 0xc138fd08 + DM_DIR = "mapper" + DM_GET_TARGET_VERSION = 0xc138fd11 + DM_INACTIVE_PRESENT_FLAG = 0x40 + DM_INTERNAL_SUSPEND_FLAG = 0x40000 + DM_IOCTL = 0xfd + DM_LIST_DEVICES = 0xc138fd02 + DM_LIST_VERSIONS = 0xc138fd0d + DM_MAX_TYPE_NAME = 0x10 + DM_NAME_LEN = 0x80 + DM_NOFLUSH_FLAG = 0x800 + DM_PERSISTENT_DEV_FLAG = 0x8 + DM_QUERY_INACTIVE_TABLE_FLAG = 0x1000 + DM_READONLY_FLAG = 0x1 + DM_REMOVE_ALL = 0xc138fd01 + DM_SECURE_DATA_FLAG = 0x8000 + DM_SKIP_BDGET_FLAG = 0x200 + DM_SKIP_LOCKFS_FLAG = 0x400 + DM_STATUS_TABLE_FLAG = 0x10 + DM_SUSPEND_FLAG = 0x2 + DM_TABLE_CLEAR = 0xc138fd0a + DM_TABLE_DEPS = 0xc138fd0b + DM_TABLE_LOAD = 0xc138fd09 + DM_TABLE_STATUS = 0xc138fd0c + DM_TARGET_MSG = 0xc138fd0e + DM_UEVENT_GENERATED_FLAG = 0x2000 + DM_UUID_FLAG = 0x4000 + DM_UUID_LEN = 0x81 + DM_VERSION = 0xc138fd00 + DM_VERSION_EXTRA = "-ioctl (2020-02-27)" + DM_VERSION_MAJOR = 0x4 + DM_VERSION_MINOR = 0x2a + DM_VERSION_PATCHLEVEL = 0x0 DT_BLK = 0x6 DT_CHR = 0x2 DT_DIR = 0x4 @@ -475,6 +523,7 @@ const ( ETH_P_MOBITEX = 0x15 ETH_P_MPLS_MC = 0x8848 ETH_P_MPLS_UC = 0x8847 + ETH_P_MRP = 0x88e3 ETH_P_MVRP = 0x88f5 ETH_P_NCSI = 0x88f8 ETH_P_NSH = 0x894f @@ -602,8 +651,9 @@ const ( FSCRYPT_POLICY_FLAGS_PAD_4 = 0x0 FSCRYPT_POLICY_FLAGS_PAD_8 = 0x1 FSCRYPT_POLICY_FLAGS_PAD_MASK = 0x3 - FSCRYPT_POLICY_FLAGS_VALID = 0xf + FSCRYPT_POLICY_FLAGS_VALID = 0x1f FSCRYPT_POLICY_FLAG_DIRECT_KEY = 0x4 + FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32 = 0x10 FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64 = 0x8 FSCRYPT_POLICY_V1 = 0x0 FSCRYPT_POLICY_V2 = 0x2 @@ -632,7 +682,7 @@ const ( FS_POLICY_FLAGS_PAD_4 = 0x0 FS_POLICY_FLAGS_PAD_8 = 0x1 FS_POLICY_FLAGS_PAD_MASK = 0x3 - FS_POLICY_FLAGS_VALID = 0xf + FS_POLICY_FLAGS_VALID = 0x1f FS_VERITY_FL = 0x100000 FS_VERITY_HASH_ALG_SHA256 = 0x1 FS_VERITY_HASH_ALG_SHA512 = 0x2 @@ -834,6 +884,7 @@ const ( IPPROTO_EGP = 0x8 IPPROTO_ENCAP = 0x62 IPPROTO_ESP = 0x32 + IPPROTO_ETHERNET = 0x8f IPPROTO_FRAGMENT = 0x2c IPPROTO_GRE = 0x2f IPPROTO_HOPOPTS = 0x0 @@ -847,6 +898,7 @@ const ( IPPROTO_L2TP = 0x73 IPPROTO_MH = 0x87 IPPROTO_MPLS = 0x89 + IPPROTO_MPTCP = 0x106 IPPROTO_MTP = 0x5c IPPROTO_NONE = 0x3b IPPROTO_PIM = 0x67 @@ -1016,6 +1068,7 @@ const ( KEYCTL_CAPS0_PERSISTENT_KEYRINGS = 0x2 KEYCTL_CAPS0_PUBLIC_KEY = 0x8 KEYCTL_CAPS0_RESTRICT_KEYRING = 0x40 + KEYCTL_CAPS1_NOTIFICATIONS = 0x4 KEYCTL_CAPS1_NS_KEYRING_NAME = 0x1 KEYCTL_CAPS1_NS_KEY_TAG = 0x2 KEYCTL_CHOWN = 0x4 @@ -1053,6 +1106,7 @@ const ( KEYCTL_SUPPORTS_VERIFY = 0x8 KEYCTL_UNLINK = 0x9 KEYCTL_UPDATE = 0x2 + KEYCTL_WATCH_KEY = 0x20 KEY_REQKEY_DEFL_DEFAULT = 0x0 KEY_REQKEY_DEFL_GROUP_KEYRING = 0x6 KEY_REQKEY_DEFL_NO_CHANGE = -0x1 @@ -1096,6 +1150,8 @@ const ( LOOP_SET_FD = 0x4c00 LOOP_SET_STATUS = 0x4c02 LOOP_SET_STATUS64 = 0x4c04 + LOOP_SET_STATUS_CLEARABLE_FLAGS = 0x4 + LOOP_SET_STATUS_SETTABLE_FLAGS = 0xc LO_KEY_SIZE = 0x20 LO_NAME_SIZE = 0x40 MADV_COLD = 0x14 @@ -1992,8 +2048,10 @@ const ( STATX_ATTR_APPEND = 0x20 STATX_ATTR_AUTOMOUNT = 0x1000 STATX_ATTR_COMPRESSED = 0x4 + STATX_ATTR_DAX = 0x2000 STATX_ATTR_ENCRYPTED = 0x800 STATX_ATTR_IMMUTABLE = 0x10 + STATX_ATTR_MOUNT_ROOT = 0x2000 STATX_ATTR_NODUMP = 0x40 STATX_ATTR_VERITY = 0x100000 STATX_BASIC_STATS = 0x7ff @@ -2002,6 +2060,7 @@ const ( STATX_CTIME = 0x80 STATX_GID = 0x10 STATX_INO = 0x100 + STATX_MNT_ID = 0x1000 STATX_MODE = 0x2 STATX_MTIME = 0x40 STATX_NLINK = 0x4 diff --git a/golang.org/x/sys/unix/zerrors_linux_arm64.go b/golang.org/x/sys/unix/zerrors_linux_arm64.go index 8b0e024b94..84f71e99fe 100644 --- a/golang.org/x/sys/unix/zerrors_linux_arm64.go +++ b/golang.org/x/sys/unix/zerrors_linux_arm64.go @@ -192,6 +192,7 @@ const ( PPPIOCSRASYNCMAP = 0x40047454 PPPIOCSXASYNCMAP = 0x4020744f PPPIOCXFERUNIT = 0x744e + PROT_BTI = 0x10 PR_SET_PTRACER_ANY = 0xffffffffffffffff PTRACE_SYSEMU = 0x1f PTRACE_SYSEMU_SINGLESTEP = 0x20 diff --git a/golang.org/x/sys/unix/zsyscall_linux.go b/golang.org/x/sys/unix/zsyscall_linux.go index f6603de4f5..4eec7a7953 100644 --- a/golang.org/x/sys/unix/zsyscall_linux.go +++ b/golang.org/x/sys/unix/zsyscall_linux.go @@ -1821,6 +1821,21 @@ func faccessat(dirfd int, path string, mode uint32) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func Faccessat2(dirfd int, path string, mode uint32, flags int) (err error) { + var _p0 *byte + _p0, err = BytePtrFromString(path) + if err != nil { + return + } + _, _, e1 := Syscall6(SYS_FACCESSAT2, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(mode), uintptr(flags), 0, 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func nameToHandleAt(dirFD int, pathname string, fh *fileHandle, mountID *_C_int, flags int) (err error) { var _p0 *byte _p0, err = BytePtrFromString(pathname) diff --git a/golang.org/x/sys/unix/zsysnum_linux_386.go b/golang.org/x/sys/unix/zsysnum_linux_386.go index 54559a8956..a597e061ca 100644 --- a/golang.org/x/sys/unix/zsysnum_linux_386.go +++ b/golang.org/x/sys/unix/zsysnum_linux_386.go @@ -433,4 +433,5 @@ const ( SYS_CLONE3 = 435 SYS_OPENAT2 = 437 SYS_PIDFD_GETFD = 438 + SYS_FACCESSAT2 = 439 ) diff --git a/golang.org/x/sys/unix/zsysnum_linux_amd64.go b/golang.org/x/sys/unix/zsysnum_linux_amd64.go index 054a741b7f..8c102e55a1 100644 --- a/golang.org/x/sys/unix/zsysnum_linux_amd64.go +++ b/golang.org/x/sys/unix/zsysnum_linux_amd64.go @@ -355,4 +355,5 @@ const ( SYS_CLONE3 = 435 SYS_OPENAT2 = 437 SYS_PIDFD_GETFD = 438 + SYS_FACCESSAT2 = 439 ) diff --git a/golang.org/x/sys/unix/zsysnum_linux_arm.go b/golang.org/x/sys/unix/zsysnum_linux_arm.go index 307f2ba12e..98f9b68fb9 100644 --- a/golang.org/x/sys/unix/zsysnum_linux_arm.go +++ b/golang.org/x/sys/unix/zsysnum_linux_arm.go @@ -397,4 +397,5 @@ const ( SYS_CLONE3 = 435 SYS_OPENAT2 = 437 SYS_PIDFD_GETFD = 438 + SYS_FACCESSAT2 = 439 ) diff --git a/golang.org/x/sys/unix/zsysnum_linux_arm64.go b/golang.org/x/sys/unix/zsysnum_linux_arm64.go index e9404dd545..4dabc33fbc 100644 --- a/golang.org/x/sys/unix/zsysnum_linux_arm64.go +++ b/golang.org/x/sys/unix/zsysnum_linux_arm64.go @@ -300,4 +300,5 @@ const ( SYS_CLONE3 = 435 SYS_OPENAT2 = 437 SYS_PIDFD_GETFD = 438 + SYS_FACCESSAT2 = 439 ) diff --git a/golang.org/x/sys/unix/zsysnum_linux_mips.go b/golang.org/x/sys/unix/zsysnum_linux_mips.go index 68bb6d29b8..d5724e5964 100644 --- a/golang.org/x/sys/unix/zsysnum_linux_mips.go +++ b/golang.org/x/sys/unix/zsysnum_linux_mips.go @@ -418,4 +418,5 @@ const ( SYS_CLONE3 = 4435 SYS_OPENAT2 = 4437 SYS_PIDFD_GETFD = 4438 + SYS_FACCESSAT2 = 4439 ) diff --git a/golang.org/x/sys/unix/zsysnum_linux_mips64.go b/golang.org/x/sys/unix/zsysnum_linux_mips64.go index 4e5251185f..c1d824a4f3 100644 --- a/golang.org/x/sys/unix/zsysnum_linux_mips64.go +++ b/golang.org/x/sys/unix/zsysnum_linux_mips64.go @@ -348,4 +348,5 @@ const ( SYS_CLONE3 = 5435 SYS_OPENAT2 = 5437 SYS_PIDFD_GETFD = 5438 + SYS_FACCESSAT2 = 5439 ) diff --git a/golang.org/x/sys/unix/zsysnum_linux_mips64le.go b/golang.org/x/sys/unix/zsysnum_linux_mips64le.go index 4d9aa3003b..598dd5d637 100644 --- a/golang.org/x/sys/unix/zsysnum_linux_mips64le.go +++ b/golang.org/x/sys/unix/zsysnum_linux_mips64le.go @@ -348,4 +348,5 @@ const ( SYS_CLONE3 = 5435 SYS_OPENAT2 = 5437 SYS_PIDFD_GETFD = 5438 + SYS_FACCESSAT2 = 5439 ) diff --git a/golang.org/x/sys/unix/zsysnum_linux_mipsle.go b/golang.org/x/sys/unix/zsysnum_linux_mipsle.go index 64af0707d5..c36782d08e 100644 --- a/golang.org/x/sys/unix/zsysnum_linux_mipsle.go +++ b/golang.org/x/sys/unix/zsysnum_linux_mipsle.go @@ -418,4 +418,5 @@ const ( SYS_CLONE3 = 4435 SYS_OPENAT2 = 4437 SYS_PIDFD_GETFD = 4438 + SYS_FACCESSAT2 = 4439 ) diff --git a/golang.org/x/sys/unix/zsysnum_linux_ppc64.go b/golang.org/x/sys/unix/zsysnum_linux_ppc64.go index cc3c067ba3..9287538d36 100644 --- a/golang.org/x/sys/unix/zsysnum_linux_ppc64.go +++ b/golang.org/x/sys/unix/zsysnum_linux_ppc64.go @@ -397,4 +397,5 @@ const ( SYS_CLONE3 = 435 SYS_OPENAT2 = 437 SYS_PIDFD_GETFD = 438 + SYS_FACCESSAT2 = 439 ) diff --git a/golang.org/x/sys/unix/zsysnum_linux_ppc64le.go b/golang.org/x/sys/unix/zsysnum_linux_ppc64le.go index 4050ff9836..4dafad8352 100644 --- a/golang.org/x/sys/unix/zsysnum_linux_ppc64le.go +++ b/golang.org/x/sys/unix/zsysnum_linux_ppc64le.go @@ -397,4 +397,5 @@ const ( SYS_CLONE3 = 435 SYS_OPENAT2 = 437 SYS_PIDFD_GETFD = 438 + SYS_FACCESSAT2 = 439 ) diff --git a/golang.org/x/sys/unix/zsysnum_linux_riscv64.go b/golang.org/x/sys/unix/zsysnum_linux_riscv64.go index 529abb6a7f..6642cfccdf 100644 --- a/golang.org/x/sys/unix/zsysnum_linux_riscv64.go +++ b/golang.org/x/sys/unix/zsysnum_linux_riscv64.go @@ -299,4 +299,5 @@ const ( SYS_CLONE3 = 435 SYS_OPENAT2 = 437 SYS_PIDFD_GETFD = 438 + SYS_FACCESSAT2 = 439 ) diff --git a/golang.org/x/sys/unix/zsysnum_linux_s390x.go b/golang.org/x/sys/unix/zsysnum_linux_s390x.go index 2766500109..23367b9467 100644 --- a/golang.org/x/sys/unix/zsysnum_linux_s390x.go +++ b/golang.org/x/sys/unix/zsysnum_linux_s390x.go @@ -362,4 +362,5 @@ const ( SYS_CLONE3 = 435 SYS_OPENAT2 = 437 SYS_PIDFD_GETFD = 438 + SYS_FACCESSAT2 = 439 ) diff --git a/golang.org/x/sys/unix/zsysnum_linux_sparc64.go b/golang.org/x/sys/unix/zsysnum_linux_sparc64.go index 4dc82bb249..083aa0204e 100644 --- a/golang.org/x/sys/unix/zsysnum_linux_sparc64.go +++ b/golang.org/x/sys/unix/zsysnum_linux_sparc64.go @@ -376,4 +376,5 @@ const ( SYS_PIDFD_OPEN = 434 SYS_OPENAT2 = 437 SYS_PIDFD_GETFD = 438 + SYS_FACCESSAT2 = 439 ) diff --git a/golang.org/x/sys/unix/ztypes_linux.go b/golang.org/x/sys/unix/ztypes_linux.go index 27d67ac8f5..77449a9e51 100644 --- a/golang.org/x/sys/unix/ztypes_linux.go +++ b/golang.org/x/sys/unix/ztypes_linux.go @@ -67,7 +67,9 @@ type Statx_t struct { Rdev_minor uint32 Dev_major uint32 Dev_minor uint32 - _ [14]uint64 + Mnt_id uint64 + _ uint64 + _ [12]uint64 } type Fsid struct { @@ -138,6 +140,48 @@ type FscryptGetKeyStatusArg struct { _ [13]uint32 } +type DmIoctl struct { + Version [3]uint32 + Data_size uint32 + Data_start uint32 + Target_count uint32 + Open_count int32 + Flags uint32 + Event_nr uint32 + _ uint32 + Dev uint64 + Name [128]byte + Uuid [129]byte + Data [7]byte +} + +type DmTargetSpec struct { + Sector_start uint64 + Length uint64 + Status int32 + Next uint32 + Target_type [16]byte +} + +type DmTargetDeps struct { + Count uint32 + _ uint32 +} + +type DmTargetVersions struct { + Next uint32 + Version [3]uint32 +} + +type DmTargetMsg struct { + Sector uint64 +} + +const ( + SizeofDmIoctl = 0x138 + SizeofDmTargetSpec = 0x28 +) + type KeyctlDHParams struct { Private int32 Prime int32 @@ -266,6 +310,15 @@ type RawSockaddrL2TPIP6 struct { Conn_id uint32 } +type RawSockaddrIUCV struct { + Family uint16 + Port uint16 + Addr uint32 + Nodeid [8]int8 + User_id [8]int8 + Name [8]int8 +} + type _Socklen uint32 type Linger struct { @@ -378,6 +431,7 @@ const ( SizeofSockaddrTIPC = 0x10 SizeofSockaddrL2TPIP = 0x10 SizeofSockaddrL2TPIP6 = 0x20 + SizeofSockaddrIUCV = 0x20 SizeofLinger = 0x8 SizeofIPMreq = 0x8 SizeofIPMreqn = 0xc @@ -671,6 +725,8 @@ type InotifyEvent struct { const SizeofInotifyEvent = 0x10 +const SI_LOAD_SHIFT = 0x10 + type Utsname struct { Sysname [65]byte Nodename [65]byte @@ -1912,6 +1968,10 @@ const ( BPF_MAP_DELETE_BATCH = 0x1b BPF_LINK_CREATE = 0x1c BPF_LINK_UPDATE = 0x1d + BPF_LINK_GET_FD_BY_ID = 0x1e + BPF_LINK_GET_NEXT_ID = 0x1f + BPF_ENABLE_STATS = 0x20 + BPF_ITER_CREATE = 0x21 BPF_MAP_TYPE_UNSPEC = 0x0 BPF_MAP_TYPE_HASH = 0x1 BPF_MAP_TYPE_ARRAY = 0x2 @@ -1939,6 +1999,7 @@ const ( BPF_MAP_TYPE_SK_STORAGE = 0x18 BPF_MAP_TYPE_DEVMAP_HASH = 0x19 BPF_MAP_TYPE_STRUCT_OPS = 0x1a + BPF_MAP_TYPE_RINGBUF = 0x1b BPF_PROG_TYPE_UNSPEC = 0x0 BPF_PROG_TYPE_SOCKET_FILTER = 0x1 BPF_PROG_TYPE_KPROBE = 0x2 @@ -1997,6 +2058,18 @@ const ( BPF_TRACE_FEXIT = 0x19 BPF_MODIFY_RETURN = 0x1a BPF_LSM_MAC = 0x1b + BPF_TRACE_ITER = 0x1c + BPF_CGROUP_INET4_GETPEERNAME = 0x1d + BPF_CGROUP_INET6_GETPEERNAME = 0x1e + BPF_CGROUP_INET4_GETSOCKNAME = 0x1f + BPF_CGROUP_INET6_GETSOCKNAME = 0x20 + BPF_XDP_DEVMAP = 0x21 + BPF_LINK_TYPE_UNSPEC = 0x0 + BPF_LINK_TYPE_RAW_TRACEPOINT = 0x1 + BPF_LINK_TYPE_TRACING = 0x2 + BPF_LINK_TYPE_CGROUP = 0x3 + BPF_LINK_TYPE_ITER = 0x4 + BPF_LINK_TYPE_NETNS = 0x5 BPF_ANY = 0x0 BPF_NOEXIST = 0x1 BPF_EXIST = 0x2 @@ -2012,6 +2085,7 @@ const ( BPF_F_WRONLY_PROG = 0x100 BPF_F_CLONE = 0x200 BPF_F_MMAPABLE = 0x400 + BPF_STATS_RUN_TIME = 0x0 BPF_STACK_BUILD_ID_EMPTY = 0x0 BPF_STACK_BUILD_ID_VALID = 0x1 BPF_STACK_BUILD_ID_IP = 0x2 @@ -2035,16 +2109,30 @@ const ( BPF_F_CURRENT_CPU = 0xffffffff BPF_F_CTXLEN_MASK = 0xfffff00000000 BPF_F_CURRENT_NETNS = -0x1 + BPF_CSUM_LEVEL_QUERY = 0x0 + BPF_CSUM_LEVEL_INC = 0x1 + BPF_CSUM_LEVEL_DEC = 0x2 + BPF_CSUM_LEVEL_RESET = 0x3 BPF_F_ADJ_ROOM_FIXED_GSO = 0x1 BPF_F_ADJ_ROOM_ENCAP_L3_IPV4 = 0x2 BPF_F_ADJ_ROOM_ENCAP_L3_IPV6 = 0x4 BPF_F_ADJ_ROOM_ENCAP_L4_GRE = 0x8 BPF_F_ADJ_ROOM_ENCAP_L4_UDP = 0x10 + BPF_F_ADJ_ROOM_NO_CSUM_RESET = 0x20 BPF_ADJ_ROOM_ENCAP_L2_MASK = 0xff BPF_ADJ_ROOM_ENCAP_L2_SHIFT = 0x38 BPF_F_SYSCTL_BASE_NAME = 0x1 BPF_SK_STORAGE_GET_F_CREATE = 0x1 BPF_F_GET_BRANCH_RECORDS_SIZE = 0x1 + BPF_RB_NO_WAKEUP = 0x1 + BPF_RB_FORCE_WAKEUP = 0x2 + BPF_RB_AVAIL_DATA = 0x0 + BPF_RB_RING_SIZE = 0x1 + BPF_RB_CONS_POS = 0x2 + BPF_RB_PROD_POS = 0x3 + BPF_RINGBUF_BUSY_BIT = 0x80000000 + BPF_RINGBUF_DISCARD_BIT = 0x40000000 + BPF_RINGBUF_HDR_SZ = 0x8 BPF_ADJ_ROOM_NET = 0x0 BPF_ADJ_ROOM_MAC = 0x1 BPF_HDR_START_MAC = 0x0 diff --git a/golang.org/x/sys/unix/ztypes_linux_386.go b/golang.org/x/sys/unix/ztypes_linux_386.go index 761b67c864..73509d896a 100644 --- a/golang.org/x/sys/unix/ztypes_linux_386.go +++ b/golang.org/x/sys/unix/ztypes_linux_386.go @@ -117,6 +117,11 @@ type Flock_t struct { Pid int32 } +type DmNameList struct { + Dev uint64 + Next uint32 +} + const ( FADV_DONTNEED = 0x4 FADV_NOREUSE = 0x5 diff --git a/golang.org/x/sys/unix/ztypes_linux_amd64.go b/golang.org/x/sys/unix/ztypes_linux_amd64.go index 201fb3482d..45eb8738b0 100644 --- a/golang.org/x/sys/unix/ztypes_linux_amd64.go +++ b/golang.org/x/sys/unix/ztypes_linux_amd64.go @@ -117,6 +117,13 @@ type Flock_t struct { _ [4]byte } +type DmNameList struct { + Dev uint64 + Next uint32 + Name [0]byte + _ [4]byte +} + const ( FADV_DONTNEED = 0x4 FADV_NOREUSE = 0x5 diff --git a/golang.org/x/sys/unix/ztypes_linux_arm.go b/golang.org/x/sys/unix/ztypes_linux_arm.go index 8051b56108..8f6b453aba 100644 --- a/golang.org/x/sys/unix/ztypes_linux_arm.go +++ b/golang.org/x/sys/unix/ztypes_linux_arm.go @@ -121,6 +121,13 @@ type Flock_t struct { _ [4]byte } +type DmNameList struct { + Dev uint64 + Next uint32 + Name [0]byte + _ [4]byte +} + const ( FADV_DONTNEED = 0x4 FADV_NOREUSE = 0x5 diff --git a/golang.org/x/sys/unix/ztypes_linux_arm64.go b/golang.org/x/sys/unix/ztypes_linux_arm64.go index a936f21692..b1e0c24f19 100644 --- a/golang.org/x/sys/unix/ztypes_linux_arm64.go +++ b/golang.org/x/sys/unix/ztypes_linux_arm64.go @@ -118,6 +118,13 @@ type Flock_t struct { _ [4]byte } +type DmNameList struct { + Dev uint64 + Next uint32 + Name [0]byte + _ [4]byte +} + const ( FADV_DONTNEED = 0x4 FADV_NOREUSE = 0x5 diff --git a/golang.org/x/sys/unix/ztypes_linux_mips.go b/golang.org/x/sys/unix/ztypes_linux_mips.go index aaca03dd7d..fb802c3ec9 100644 --- a/golang.org/x/sys/unix/ztypes_linux_mips.go +++ b/golang.org/x/sys/unix/ztypes_linux_mips.go @@ -120,6 +120,13 @@ type Flock_t struct { _ [4]byte } +type DmNameList struct { + Dev uint64 + Next uint32 + Name [0]byte + _ [4]byte +} + const ( FADV_DONTNEED = 0x4 FADV_NOREUSE = 0x5 diff --git a/golang.org/x/sys/unix/ztypes_linux_mips64.go b/golang.org/x/sys/unix/ztypes_linux_mips64.go index 2e7f3b8ca4..30abcf3bb8 100644 --- a/golang.org/x/sys/unix/ztypes_linux_mips64.go +++ b/golang.org/x/sys/unix/ztypes_linux_mips64.go @@ -118,6 +118,13 @@ type Flock_t struct { _ [4]byte } +type DmNameList struct { + Dev uint64 + Next uint32 + Name [0]byte + _ [4]byte +} + const ( FADV_DONTNEED = 0x4 FADV_NOREUSE = 0x5 diff --git a/golang.org/x/sys/unix/ztypes_linux_mips64le.go b/golang.org/x/sys/unix/ztypes_linux_mips64le.go index 16add5a257..99761aa9a7 100644 --- a/golang.org/x/sys/unix/ztypes_linux_mips64le.go +++ b/golang.org/x/sys/unix/ztypes_linux_mips64le.go @@ -118,6 +118,13 @@ type Flock_t struct { _ [4]byte } +type DmNameList struct { + Dev uint64 + Next uint32 + Name [0]byte + _ [4]byte +} + const ( FADV_DONTNEED = 0x4 FADV_NOREUSE = 0x5 diff --git a/golang.org/x/sys/unix/ztypes_linux_mipsle.go b/golang.org/x/sys/unix/ztypes_linux_mipsle.go index 4ed2c8e54c..293690348f 100644 --- a/golang.org/x/sys/unix/ztypes_linux_mipsle.go +++ b/golang.org/x/sys/unix/ztypes_linux_mipsle.go @@ -120,6 +120,13 @@ type Flock_t struct { _ [4]byte } +type DmNameList struct { + Dev uint64 + Next uint32 + Name [0]byte + _ [4]byte +} + const ( FADV_DONTNEED = 0x4 FADV_NOREUSE = 0x5 diff --git a/golang.org/x/sys/unix/ztypes_linux_ppc64.go b/golang.org/x/sys/unix/ztypes_linux_ppc64.go index 7415190997..0ca856e559 100644 --- a/golang.org/x/sys/unix/ztypes_linux_ppc64.go +++ b/golang.org/x/sys/unix/ztypes_linux_ppc64.go @@ -119,6 +119,13 @@ type Flock_t struct { _ [4]byte } +type DmNameList struct { + Dev uint64 + Next uint32 + Name [0]byte + _ [4]byte +} + const ( FADV_DONTNEED = 0x4 FADV_NOREUSE = 0x5 diff --git a/golang.org/x/sys/unix/ztypes_linux_ppc64le.go b/golang.org/x/sys/unix/ztypes_linux_ppc64le.go index 046c2debd4..f50f6482ee 100644 --- a/golang.org/x/sys/unix/ztypes_linux_ppc64le.go +++ b/golang.org/x/sys/unix/ztypes_linux_ppc64le.go @@ -119,6 +119,13 @@ type Flock_t struct { _ [4]byte } +type DmNameList struct { + Dev uint64 + Next uint32 + Name [0]byte + _ [4]byte +} + const ( FADV_DONTNEED = 0x4 FADV_NOREUSE = 0x5 diff --git a/golang.org/x/sys/unix/ztypes_linux_riscv64.go b/golang.org/x/sys/unix/ztypes_linux_riscv64.go index 0f2f61a6ad..4d3ac8d7b4 100644 --- a/golang.org/x/sys/unix/ztypes_linux_riscv64.go +++ b/golang.org/x/sys/unix/ztypes_linux_riscv64.go @@ -118,6 +118,13 @@ type Flock_t struct { _ [4]byte } +type DmNameList struct { + Dev uint64 + Next uint32 + Name [0]byte + _ [4]byte +} + const ( FADV_DONTNEED = 0x4 FADV_NOREUSE = 0x5 diff --git a/golang.org/x/sys/unix/ztypes_linux_s390x.go b/golang.org/x/sys/unix/ztypes_linux_s390x.go index cca1b6be27..349f483a80 100644 --- a/golang.org/x/sys/unix/ztypes_linux_s390x.go +++ b/golang.org/x/sys/unix/ztypes_linux_s390x.go @@ -117,6 +117,13 @@ type Flock_t struct { _ [4]byte } +type DmNameList struct { + Dev uint64 + Next uint32 + Name [0]byte + _ [4]byte +} + const ( FADV_DONTNEED = 0x6 FADV_NOREUSE = 0x7 diff --git a/golang.org/x/sys/unix/ztypes_linux_sparc64.go b/golang.org/x/sys/unix/ztypes_linux_sparc64.go index 33a73bf183..80c73beaa1 100644 --- a/golang.org/x/sys/unix/ztypes_linux_sparc64.go +++ b/golang.org/x/sys/unix/ztypes_linux_sparc64.go @@ -121,6 +121,13 @@ type Flock_t struct { _ [2]byte } +type DmNameList struct { + Dev uint64 + Next uint32 + Name [0]byte + _ [4]byte +} + const ( FADV_DONTNEED = 0x4 FADV_NOREUSE = 0x5 diff --git a/modules.txt b/modules.txt index 20423f6b29..3eaacb6a53 100644 --- a/modules.txt +++ b/modules.txt @@ -205,7 +205,7 @@ github.com/cockroachdb/gostdlib/x/tools/internal/semver github.com/cockroachdb/gostdlib/x/tools/internal/span # github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f github.com/cockroachdb/logtags -# github.com/cockroachdb/pebble v0.0.0-20200729202514-cfd19d33cdca +# github.com/cockroachdb/pebble v0.0.0-20200814004841-77c18adb0ee3 github.com/cockroachdb/pebble github.com/cockroachdb/pebble/bloom github.com/cockroachdb/pebble/internal/arenaskl @@ -782,7 +782,7 @@ golang.org/x/perf/storage/benchfmt # golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e golang.org/x/sync/errgroup golang.org/x/sync/syncmap -# golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1 +# golang.org/x/sys v0.0.0-20200817155316-9781c653f443 golang.org/x/sys/cpu golang.org/x/sys/internal/unsafeheader golang.org/x/sys/unix From a15f2178a398bd58e9717cf96d6b1a555b290d78 Mon Sep 17 00:00:00 2001 From: Oliver Tan Date: Wed, 19 Aug 2020 08:40:35 -0700 Subject: [PATCH 26/49] add github.com/zabawaba99/go-gitignore --- .../zabawaba99/go-gitignore/.travis.yml | 62 ++++++ github.com/zabawaba99/go-gitignore/LICENSE | 201 ++++++++++++++++++ github.com/zabawaba99/go-gitignore/README.md | 50 +++++ github.com/zabawaba99/go-gitignore/ignore.go | 148 +++++++++++++ modules.txt | 2 + 5 files changed, 463 insertions(+) create mode 100644 github.com/zabawaba99/go-gitignore/.travis.yml create mode 100644 github.com/zabawaba99/go-gitignore/LICENSE create mode 100644 github.com/zabawaba99/go-gitignore/README.md create mode 100644 github.com/zabawaba99/go-gitignore/ignore.go diff --git a/github.com/zabawaba99/go-gitignore/.travis.yml b/github.com/zabawaba99/go-gitignore/.travis.yml new file mode 100644 index 0000000000..61b75f8006 --- /dev/null +++ b/github.com/zabawaba99/go-gitignore/.travis.yml @@ -0,0 +1,62 @@ +language: go + +go: + - 1.9 + - 1.10 + - tip + +matrix: + allow_failures: + - go: tip + fast_finish: true + +before_install: + # linting tools + - go get github.com/golang/lint/golint + - go get github.com/fzipp/gocyclo + + # code coverage + - go get github.com/axw/gocov/gocov + - go get github.com/mattn/goveralls + - if ! go get code.google.com/p/go.tools/cmd/cover; then go get golang.org/x/tools/cmd/cover; fi + +install: + # make sure stuff actually builds + - go build + +script: + # ensure everything is formatted all pretty like + - if gofmt -l -s . | grep '**.go'; then exit 1; fi + # vet out possible issues + - go vet ./... + # run tests + - go get -t + - go test -a -race -v ./... + +after_success: + - | + echo "mode: count" > profile.cov + for dir in $(find . -maxdepth 10 -not -path 'vendor' -not -path './.git*' -not -path '*/_*' -type d); + do + if ls $dir/*.go &> /dev/null; then + go test -covermode=count -coverprofile=$dir/profile.tmp $dir + if [ -f $dir/profile.tmp ] + then + cat $dir/profile.tmp | tail -n +2 >> profile.cov + rm $dir/profile.tmp + fi + fi + done + go tool cover -func profile.cov + goveralls -coverprofile=profile.cov -service=travis-ci -repotoken=$COVERALLS -v + +after_script: + # check possible styling errors + - golint ./... + # check for potentially complex functions but don't false build + - gocyclo -over 15 . || true + # refresh godocs in case there were api changes + - | + if [[ "$TRAVIS_PULL_REQUEST" == "false" ]] && [[ "$TRAVIS_BRANCH" == "master" ]]; then + go list ./... | xargs -n 1 -I{} curl http://godoc.org/-/refresh -d path={} + fi diff --git a/github.com/zabawaba99/go-gitignore/LICENSE b/github.com/zabawaba99/go-gitignore/LICENSE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/github.com/zabawaba99/go-gitignore/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/github.com/zabawaba99/go-gitignore/README.md b/github.com/zabawaba99/go-gitignore/README.md new file mode 100644 index 0000000000..ee689faf6c --- /dev/null +++ b/github.com/zabawaba99/go-gitignore/README.md @@ -0,0 +1,50 @@ +# go-gitignore +Gitignore pattern matching in Go. + +# Usage + +```go +package main + +import ( + "fmt" + + "github.com/zabawaba99/go-gitignore" +) + +func main() { + val := gitignore.Match("hello/**/world", "hello/foo/bar/baz/world") + fmt.Printf("I am true %b\n", val) +} + +``` + +# Pattern Format + +(Taken from https://git-scm.com/docs/gitignore) + +* A blank line matches no files, so it can serve as a separator for readability. + +* A line starting with # serves as a comment. Put a backslash ("`\`") in front of the first hash for patterns that begin with a hash. + +* Trailing spaces are ignored unless they are quoted with backslash ("`\`"). + +* An optional prefix "`!`" which negates the pattern; any matching file excluded by a previous pattern will become included again. It is not possible to re-include a file if a parent directory of that file is excluded. Git doesn’t list excluded directories for performance reasons, so any patterns on contained files have no effect, no matter where they are defined. Put a backslash ("`\`") in front of the first "`!`" for patterns that begin with a literal "`!`", for example, "`\!important!.txt`". + +* If the pattern ends with a slash, it is removed for the purpose of the following description, but it would only find a match with a directory. In other words, foo/ will match a directory foo and paths underneath it, but will not match a regular file or a symbolic link foo (this is consistent with the way how pathspec works in general in Git). + +* If the pattern does not contain a slash /, Git treats it as a shell glob pattern and checks for a match against the pathname relative to the location of the .gitignore file (relative to the toplevel of the work tree if not from a .gitignore file). + +* Otherwise, Git treats the pattern as a shell glob suitable for consumption by fnmatch(3) with the FNM_PATHNAME flag: wildcards in the pattern will not match a / in the pathname. For example, "`Documentation/*.html`" matches "`Documentation/git.html`" but not "`Documentation/ppc/ppc.html`" or "`tools/perf/Documentation/perf.html`". + +* A leading slash matches the beginning of the pathname. For example, "`/*.c`" matches "`cat-file.c`" but not "`mozilla-sha1/sha1.c`". + +Two consecutive asterisks ("`**`") in patterns matched against full pathname may have special meaning: + +* A leading "`**`" followed by a slash means match in all directories. For example, "`**/foo`" matches file or directory "`foo`" anywhere, the same as pattern "`foo`". "`**/foo/bar`" matches file or directory "`bar`" anywhere that is directly under directory "`foo`". + +* A trailing "`/**`" matches everything inside. For example, "`abc/**`" matches all files inside directory "`abc`", relative to the location of the .gitignore file, with infinite depth. + +* A slash followed by two consecutive asterisks then a slash matches zero or more directories. For example, "`a/**/b`" matches "`a/b`", "`a/x/b`", "`a/x/y/b`" and so on. + +* Other consecutive asterisks are considered invalid. diff --git a/github.com/zabawaba99/go-gitignore/ignore.go b/github.com/zabawaba99/go-gitignore/ignore.go new file mode 100644 index 0000000000..aeb51cc639 --- /dev/null +++ b/github.com/zabawaba99/go-gitignore/ignore.go @@ -0,0 +1,148 @@ +package gitignore + +import ( + "log" + "os" + "path" + "path/filepath" + "strings" +) + +const dblAsterisks = "**" + +// Match matches patterns in the same manner that gitignore does. +// Reference https://git-scm.com/docs/gitignore. +func Match(pattern, value string) bool { + // A blank line matches no files, so it can serve as a separator for readability. + if pattern == "" { + return false + } + + // A line starting with # serves as a comment. Put a backslash ("\") in front of the first hash for patterns that begin with a hash. + if strings.HasPrefix(pattern, "#") { + return false + } + + // Trailing spaces are ignored unless they are quoted with backslash ("\"). + pattern = strings.TrimSuffix(pattern, " ") + + // An optional prefix "!" which negates the pattern; any matching file + // excluded by a previous pattern will become included again. It is not + // possible to re-include a file if a parent directory of that file is excluded. + // Git doesn’t list excluded directories for performance reasons, so any patterns + // on contained files have no effect, no matter where they are defined. + // Put a backslash ("\") in front of the first "!" for patterns that begin + // with a literal "!", for example, "\!important!.txt". + negate := strings.HasPrefix(pattern, "!") + if negate { + pattern = strings.TrimPrefix(pattern, "!") + } + + // If the pattern ends with a slash, it is removed for the purpose of the + // following description, but it would only find a match with a directory. + // In other words, foo/ will match a directory foo and paths underneath it, + // but will not match a regular file or a symbolic link foo (this is consistent + // with the way how pathspec works in general in Git). + pattern = strings.TrimSuffix(pattern, string(os.PathSeparator)) + + // Two consecutive asterisks ("**") in patterns matched + // against full pathname may have special meaning: + if strings.Contains(pattern, dblAsterisks) { + result := evalDblAsterisk(pattern, value) + if negate { + result = !result + } + return result + } + + // If the pattern does not contain a slash /, Git treats it as a shell glob + // pattern and checks for a match against the pathname relative to the location + // of the .gitignore file (relative to the toplevel of the work tree if not from + // a .gitignore file). + if !strings.Contains(pattern, string(os.PathSeparator)) { + m, err := filepath.Glob(pattern) + if err != nil { + // maybe log this? + log.Printf("ERROR %s\n", err) + return false + } + + var found bool + for _, v := range m { + if v == value { + found = true + break + } + } + + if negate { + return !found + } + return found + } + + // Otherwise, Git treats the pattern as a shell glob suitable for consumption by + // fnmatch(3) with the FNM_PATHNAME flag: wildcards in the pattern will not match + // a / in the pathname. For example, "Documentation/*.html" matches + // "Documentation/git.html" but not "Documentation/ppc/ppc.html" or + // "tools/perf/Documentation/perf.html". + + // A leading slash matches the beginning of the pathname. For example, "/*.c" matches "cat-file.c" but not "mozilla-sha1/sha1.c". + + matched, err := path.Match(pattern, value) + if err != nil { + // maybe log? + return false + } + + if negate { + return !matched + } + return matched +} + +func evalDblAsterisk(pattern, value string) bool { + // A leading "**" followed by a slash means match in all directories. + // For example, "**/foo" matches file or directory "foo" anywhere, + // the same as pattern "foo". "**/foo/bar" matches file or directory + // "bar" anywhere that is directly under directory "foo". + if strings.HasPrefix(pattern, dblAsterisks) { + pattern = strings.TrimPrefix(pattern, dblAsterisks) + return strings.HasSuffix(value, pattern) + } + + // A trailing "/**" matches everything inside. For example, "abc/**" + // matches all files inside directory "abc", relative to the location + // of the .gitignore file, with infinite depth. + if strings.HasSuffix(pattern, dblAsterisks) { + pattern = strings.TrimSuffix(pattern, dblAsterisks) + return strings.HasPrefix(value, pattern) + } + + // A slash followed by two consecutive asterisks then a slash matches + // zero or more directories. For example, "a/**/b" matches "a/b", + // /"a/x/b", "a/x/y/b" and so on. + parts := strings.Split(pattern, dblAsterisks) + for i, part := range parts { + switch i { + case 0: + if !strings.HasPrefix(value, part) { + return false + } + case len(parts) - 1: // last part + part = strings.TrimPrefix(part, string(os.PathSeparator)) + return strings.HasSuffix(value, part) + default: + if !strings.Contains(value, part) { + return false + } + } + + // trim evaluated text + index := strings.Index(value, part) + len(part) + value = value[index:] + } + + // Other consecutive asterisks are considered invalid. + return false +} diff --git a/modules.txt b/modules.txt index 3eaacb6a53..9fa3932af5 100644 --- a/modules.txt +++ b/modules.txt @@ -708,6 +708,8 @@ github.com/twpayne/go-geom/xy/orientation github.com/twpayne/go-kml # github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad github.com/wadey/gocovmerge +# github.com/zabawaba99/go-gitignore v0.0.0-20200117185801-39e6bddfb292 +github.com/zabawaba99/go-gitignore # go.etcd.io/etcd v0.0.0-00010101000000-000000000000 => github.com/cockroachdb/etcd v0.4.7-0.20200615211340-a17df30d5955 go.etcd.io/etcd/raft go.etcd.io/etcd/raft/confchange From 866b5c67ce250ea715aa150626d0db65cb557324 Mon Sep 17 00:00:00 2001 From: sumeerbhola Date: Fri, 21 Aug 2020 14:31:34 -0400 Subject: [PATCH 27/49] vendor: bump Pebble to a24bc6c83054489b1bb0d9f53bcdd4ad8ec75977 a24bc6c sstable: optimize seeks to use next 76afcca db: optimize levelIter for repeated seeks with narrow and monotonic bounds --- github.com/cockroachdb/datadriven/go.mod | 9 +- github.com/cockroachdb/datadriven/go.sum | 242 +++++++++++++- .../pebble/internal/base/iterator.go | 13 +- github.com/cockroachdb/pebble/iterator.go | 12 +- github.com/cockroachdb/pebble/level_iter.go | 101 ++++-- .../cockroachdb/pebble/sstable/block.go | 79 +++++ .../cockroachdb/pebble/sstable/reader.go | 306 +++++++++++++++--- github.com/cockroachdb/redact/api.go | 3 +- github.com/cockroachdb/redact/markers.go | 14 + .../redact/markers_internal_escape.go | 43 ++- .../redact/markers_internal_print.go | 47 ++- github.com/kr/pretty/formatter.go | 2 +- golang.org/x/sys/cpu/cpu_linux_mips64x.go | 1 + golang.org/x/sys/unix/syscall_linux.go | 15 + golang.org/x/sys/unix/syscall_linux_arm.go | 5 - golang.org/x/sys/unix/syscall_linux_gc_arm.go | 13 + golang.org/x/sys/unix/ztypes_linux.go | 22 ++ modules.txt | 12 +- 18 files changed, 806 insertions(+), 133 deletions(-) create mode 100644 golang.org/x/sys/unix/syscall_linux_gc_arm.go diff --git a/github.com/cockroachdb/datadriven/go.mod b/github.com/cockroachdb/datadriven/go.mod index dd62d6aa75..4460349a23 100644 --- a/github.com/cockroachdb/datadriven/go.mod +++ b/github.com/cockroachdb/datadriven/go.mod @@ -3,12 +3,7 @@ module github.com/cockroachdb/datadriven go 1.13 require ( - github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40 // indirect - github.com/cockroachdb/errors v1.2.4 - github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect - github.com/getsentry/raven-go v0.2.0 // indirect - github.com/gogo/protobuf v1.3.1 // indirect - github.com/kr/pretty v0.1.0 // indirect - github.com/pkg/errors v0.8.1 // indirect + github.com/cockroachdb/errors v1.6.1 + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 ) diff --git a/github.com/cockroachdb/datadriven/go.sum b/github.com/cockroachdb/datadriven/go.sum index 4195c27b26..70b208972f 100644 --- a/github.com/cockroachdb/datadriven/go.sum +++ b/github.com/cockroachdb/datadriven/go.sum @@ -1,22 +1,252 @@ -github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40 h1:xvUo53O5MRZhVMJAxWCJcS5HHrqAiAG9SJ1LpMu6aAI= -github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/cockroachdb/errors v1.2.4 h1:Lap807SXTH5tri2TivECb/4abUkMZC9zRoLarvcKDqs= -github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= +github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= +github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= +github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= +github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= +github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cockroachdb/errors v1.6.1 h1:GqOx5BYPdco9HPaoUuy8W53kZsSEbPu5P/d4WLhanmM= +github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs= -github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= +github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= +github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= +github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= +github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= +github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= +github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= +github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= +github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= +github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= +github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= +github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= +github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= +github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk= +github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U= +github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw= +github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= +github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= +github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= +github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= +github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= +github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= +github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= +github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/github.com/cockroachdb/pebble/internal/base/iterator.go b/github.com/cockroachdb/pebble/internal/base/iterator.go index 23612f6f79..cc27c6f193 100644 --- a/github.com/cockroachdb/pebble/internal/base/iterator.go +++ b/github.com/cockroachdb/pebble/internal/base/iterator.go @@ -52,11 +52,14 @@ import "fmt" // Last if there is an upper bound). This imposition is done in order to // elevate that enforcement to the caller (generally pebble.Iterator or // pebble.mergingIter) rather than having it duplicated in every -// InternalIterator implementation. InternalIterator implementations are -// required to respect the iterator bounds, never returning records outside of -// the bounds with one exception: an iterator may generate synthetic RANGEDEL -// marker records. See levelIter.syntheticBoundary for the sole existing -// example of this behavior. [TODO(peter): can we eliminate this exception?] +// InternalIterator implementation. Additionally, the caller needs to ensure +// that SeekGE/SeekPrefixGE are not called with a key > the upper bound, and +// SeekLT is not called with a key < the lower bound. +// InternalIterator implementations are required to respect the iterator +// bounds, never returning records outside of the bounds with one exception: +// an iterator may generate synthetic RANGEDEL marker records. See +// levelIter.syntheticBoundary for the sole existing example of this behavior. +// [TODO(peter): can we eliminate this exception?] // // An iterator must be closed after use, but it is not necessary to read an // iterator until exhaustion. diff --git a/github.com/cockroachdb/pebble/iterator.go b/github.com/cockroachdb/pebble/iterator.go index f1f8694174..3b11809aae 100644 --- a/github.com/cockroachdb/pebble/iterator.go +++ b/github.com/cockroachdb/pebble/iterator.go @@ -315,6 +315,8 @@ func (i *Iterator) SeekGE(key []byte) bool { i.prefix = nil if lowerBound := i.opts.GetLowerBound(); lowerBound != nil && i.cmp(key, lowerBound) < 0 { key = lowerBound + } else if upperBound := i.opts.GetUpperBound(); upperBound != nil && i.cmp(key, upperBound) > 0 { + key = upperBound } i.iterKey, i.iterValue = i.iter.SeekGE(key) @@ -370,6 +372,12 @@ func (i *Iterator) SeekPrefixGE(key []byte) bool { return false } key = lowerBound + } else if upperBound := i.opts.GetUpperBound(); upperBound != nil && i.cmp(key, upperBound) > 0 { + if n := i.split(upperBound); !bytes.Equal(i.prefix, upperBound[:n]) { + i.err = errors.New("pebble: SeekPrefixGE supplied with key outside of upper bound") + return false + } + key = upperBound } i.iterKey, i.iterValue = i.iter.SeekPrefixGE(i.prefix, key) @@ -382,8 +390,10 @@ func (i *Iterator) SeekPrefixGE(key []byte) bool { func (i *Iterator) SeekLT(key []byte) bool { i.err = nil // clear cached iteration error i.prefix = nil - if upperBound := i.opts.GetUpperBound(); upperBound != nil && i.cmp(key, upperBound) >= 0 { + if upperBound := i.opts.GetUpperBound(); upperBound != nil && i.cmp(key, upperBound) > 0 { key = upperBound + } else if lowerBound := i.opts.GetLowerBound(); lowerBound != nil && i.cmp(key, lowerBound) < 0 { + key = lowerBound } i.iterKey, i.iterValue = i.iter.SeekLT(key) diff --git a/github.com/cockroachdb/pebble/level_iter.go b/github.com/cockroachdb/pebble/level_iter.go index c464dee8d5..ae07ee6fe9 100644 --- a/github.com/cockroachdb/pebble/level_iter.go +++ b/github.com/cockroachdb/pebble/level_iter.go @@ -69,12 +69,22 @@ type levelIter struct { // - err != nil // - some other constraint, like the bounds in opts, caused the file at index to not // be relevant to the iteration. - iter internalIterator - iterFile *fileMetadata - newIters tableNewIters - rangeDelIter *internalIterator - files manifest.LevelIterator - err error + iter internalIterator + iterFile *fileMetadata + newIters tableNewIters + // When rangeDelIter != nil, the caller requires that *rangeDelIter must point + // to a range del iterator corresponding to the current file. When this + // iterator returns nil, *rangeDelIter should also be set to nil. Whenever + // a non-nil internalIterator is placed in rangeDelIter, a copy is placed + // in rangeDelIterCopy. This is done for the following special case: + // when this iterator returns nil because of exceeding the bounds, we don't + // close iter and *rangeDelIter since we could reuse it in the next seek. But + // we need to set *rangeDelIter to nil because of the aforementioned contract. + // This copy is used to revive the *rangeDelIter in the case of reuse. + rangeDelIter *internalIterator + rangeDelIterCopy internalIterator + files manifest.LevelIterator + err error // Pointer into this level's entry in `mergingIterLevel::smallestUserKey,largestUserKey`. // We populate it with the corresponding bounds for the currently opened file. It is used for @@ -255,6 +265,9 @@ func (l *levelIter) loadFile(file *fileMetadata, dir int) bool { // We don't bother comparing the file bounds with the iteration bounds when we have // an already open iterator. It is possible that the iter may not be relevant given the // current iteration bounds, but it knows those bounds, so it will enforce them. + if l.rangeDelIter != nil { + *l.rangeDelIter = l.rangeDelIterCopy + } return true } // We were already at file, but don't have an iterator, probably because the file was @@ -301,6 +314,7 @@ func (l *levelIter) loadFile(file *fileMetadata, dir int) bool { } if l.rangeDelIter != nil { *l.rangeDelIter = rangeDelIter + l.rangeDelIterCopy = rangeDelIter } else if rangeDelIter != nil { rangeDelIter.Close() } @@ -368,7 +382,7 @@ func (l *levelIter) SeekPrefixGE(prefix, key []byte) (*InternalKey, []byte) { // table as findFileGE found the table where key <= meta.Largest. We treat // this case the same as SeekGE where an upper-bound resides within the // sstable and generate a synthetic boundary key. - if l.rangeDelIter != nil { + if l.rangeDelIter != nil && *l.rangeDelIter != nil { l.syntheticBoundary = l.iterFile.Largest l.syntheticBoundary.SetKind(InternalKeyKindRangeDelete) l.largestBoundary = &l.syntheticBoundary @@ -426,6 +440,17 @@ func (l *levelIter) Next() (*InternalKey, []byte) { switch { case l.largestBoundary != nil: + if l.tableOpts.UpperBound != nil { + // The UpperBound was within this file, so don't load the next + // file. We leave the largestBoundary unchanged so that subsequent + // calls to Next() stay at this file. If a Seek/First/Last call is + // made and this file continues to be relevant, loadFile() will + // set the largestBoundary to nil. + if l.rangeDelIter != nil { + *l.rangeDelIter = nil + } + return nil, nil + } // We're stepping past the boundary key, so now we can load the next file. if l.loadFile(l.files.Next(), +1) { if key, val := l.iter.First(); key != nil { @@ -452,6 +477,17 @@ func (l *levelIter) Prev() (*InternalKey, []byte) { switch { case l.smallestBoundary != nil: + if l.tableOpts.LowerBound != nil { + // The LowerBound was within this file, so don't load the previous + // file. We leave the smallestBoundary unchanged so that + // subsequent calls to Prev() stay at this file. If a + // Seek/First/Last call is made and this file continues to be + // relevant, loadFile() will set the smallestBoundary to nil. + if l.rangeDelIter != nil { + *l.rangeDelIter = nil + } + return nil, nil + } // We're stepping past the boundary key, so now we can load the prev file. if l.loadFile(l.files.Prev(), -1) { if key, val := l.iter.Last(); key != nil { @@ -501,16 +537,23 @@ func (l *levelIter) skipEmptyFileForward() (*InternalKey, []byte) { // never going to look at subsequent sstables (we've reached the upper // bound). if l.tableOpts.UpperBound != nil { - // TODO(peter): Rather than using f.Largest, can we use - // l.tableOpts.UpperBound and set the seqnum to 0? We know the upper - // bound resides within the table boundaries. Not clear if this is - // kosher with respect to the invariant that only one record for a - // given user key will have seqnum 0. See Iterator.nextUserKey for an - // optimization that requires this. - l.syntheticBoundary = l.iterFile.Largest - l.syntheticBoundary.SetKind(InternalKeyKindRangeDelete) - l.largestBoundary = &l.syntheticBoundary - return l.largestBoundary, nil + if *l.rangeDelIter != nil { + // TODO(peter): Rather than using f.Largest, can we use + // l.tableOpts.UpperBound and set the seqnum to 0? We know the upper + // bound resides within the table boundaries. Not clear if this is + // kosher with respect to the invariant that only one record for a + // given user key will have seqnum 0. See Iterator.nextUserKey for an + // optimization that requires this. + l.syntheticBoundary = l.iterFile.Largest + l.syntheticBoundary.SetKind(InternalKeyKindRangeDelete) + l.largestBoundary = &l.syntheticBoundary + return l.largestBoundary, nil + } + // Else there are no range deletions in this sstable. This + // helps with performance when many levels are populated with + // sstables and most don't have any actual keys within the + // bounds. + return nil, nil } // If the boundary is a range deletion tombstone, return that key. if l.iterFile.Largest.Kind() == InternalKeyKindRangeDelete { @@ -557,13 +600,20 @@ func (l *levelIter) skipEmptyFileBackward() (*InternalKey, []byte) { // never going to look at earlier sstables (we've reached the lower // bound). if l.tableOpts.LowerBound != nil { - // TODO(peter): Rather than using f.Smallest, can we use - // l.tableOpts.LowerBound and set the seqnum to InternalKeySeqNumMax? - // We know the lower bound resides within the table boundaries. - l.syntheticBoundary = l.iterFile.Smallest - l.syntheticBoundary.SetKind(InternalKeyKindRangeDelete) - l.smallestBoundary = &l.syntheticBoundary - return l.smallestBoundary, nil + if *l.rangeDelIter != nil { + // TODO(peter): Rather than using f.Smallest, can we use + // l.tableOpts.LowerBound and set the seqnum to InternalKeySeqNumMax? + // We know the lower bound resides within the table boundaries. + l.syntheticBoundary = l.iterFile.Smallest + l.syntheticBoundary.SetKind(InternalKeyKindRangeDelete) + l.smallestBoundary = &l.syntheticBoundary + return l.smallestBoundary, nil + } + // Else there are no range deletions in this sstable. This + // helps with performance when many levels are populated with + // sstables and most don't have any actual keys within the + // bounds. + return nil, nil } // If the boundary is a range deletion tombstone, return that key. if l.iterFile.Smallest.Kind() == InternalKeyKindRangeDelete { @@ -593,10 +643,11 @@ func (l *levelIter) Close() error { l.iter = nil } if l.rangeDelIter != nil { - if t := *l.rangeDelIter; t != nil { + if t := l.rangeDelIterCopy; t != nil { l.err = firstError(l.err, t.Close()) } *l.rangeDelIter = nil + l.rangeDelIterCopy = nil } return l.err } diff --git a/github.com/cockroachdb/pebble/sstable/block.go b/github.com/cockroachdb/pebble/sstable/block.go index 846056b4f6..0be2367b78 100644 --- a/github.com/cockroachdb/pebble/sstable/block.go +++ b/github.com/cockroachdb/pebble/sstable/block.go @@ -238,6 +238,9 @@ type blockIter struct { cached []blockEntry cachedBuf []byte cacheHandle cache.Handle + // The first key in the block. This is used by the caller to set bounds + // for block iteration for already loaded blocks. + firstKey InternalKey } // blockIter implements the base.InternalIterator interface. @@ -266,6 +269,7 @@ func (i *blockIter) init(cmp Compare, block block, globalSeqNum uint64) error { i.fullKey = i.fullKey[:0] i.val = nil i.clearCache() + i.readFirstKey() return nil } @@ -284,11 +288,16 @@ func (i *blockIter) invalidate() { i.data = nil } +func (i *blockIter) isInvalid() bool { + return i.data == nil +} + func (i *blockIter) resetForReuse() blockIter { return blockIter{ fullKey: i.fullKey[:0], cached: i.cached[:0], cachedBuf: i.cachedBuf[:0], + data: nil, } } @@ -376,6 +385,76 @@ func (i *blockIter) readEntry() { i.nextOffset = int32(uintptr(ptr)-uintptr(i.ptr)) + int32(value) } +func (i *blockIter) readFirstKey() { + ptr := i.ptr + + // This is an ugly performance hack. Reading entries from blocks is one of + // the inner-most routines and decoding the 3 varints per-entry takes a + // significant time. Neither go1.11 or go1.12 will inline decodeVarint for + // us, so we do it manually. This provides a 10-15% performance improvement + // on blockIter benchmarks on both go1.11 and go1.12. + // + // TODO(peter): remove this hack if go:inline is ever supported. + + var shared uint32 + if a := *((*uint8)(ptr)); a < 128 { + shared = uint32(a) + ptr = unsafe.Pointer(uintptr(ptr) + 1) + } else { + panic("first key in block should not share") + } + if shared != 0 { + panic("first key in block should not share") + } + + var unshared uint32 + if a := *((*uint8)(ptr)); a < 128 { + unshared = uint32(a) + ptr = unsafe.Pointer(uintptr(ptr) + 1) + } else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 { + unshared = uint32(b)<<7 | uint32(a) + ptr = unsafe.Pointer(uintptr(ptr) + 2) + } else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 { + unshared = uint32(c)<<14 | uint32(b)<<7 | uint32(a) + ptr = unsafe.Pointer(uintptr(ptr) + 3) + } else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 { + unshared = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a) + ptr = unsafe.Pointer(uintptr(ptr) + 4) + } else { + d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4))) + unshared = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a) + ptr = unsafe.Pointer(uintptr(ptr) + 5) + } + + // Skip the value length. + if a := *((*uint8)(ptr)); a < 128 { + ptr = unsafe.Pointer(uintptr(ptr) + 1) + } else if a := *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); a < 128 { + ptr = unsafe.Pointer(uintptr(ptr) + 2) + } else if a := *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); a < 128 { + ptr = unsafe.Pointer(uintptr(ptr) + 3) + } else if a := *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); a < 128 { + ptr = unsafe.Pointer(uintptr(ptr) + 4) + } else { + ptr = unsafe.Pointer(uintptr(ptr) + 5) + } + + firstKey := getBytes(ptr, int(unshared)) + // Manually inlining base.DecodeInternalKey provides a 5-10% speedup on + // BlockIter benchmarks. + if n := len(firstKey) - 8; n >= 0 { + i.firstKey.Trailer = binary.LittleEndian.Uint64(firstKey[n:]) + i.firstKey.UserKey = firstKey[:n:n] + if i.globalSeqNum != 0 { + i.firstKey.SetSeqNum(i.globalSeqNum) + } + } else { + // TODO: propagate this error? + i.firstKey.Trailer = uint64(InternalKeyKindInvalid) + i.firstKey.UserKey = nil + } +} + func (i *blockIter) decodeInternalKey(key []byte) { // Manually inlining base.DecodeInternalKey provides a 5-10% speedup on // BlockIter benchmarks. diff --git a/github.com/cockroachdb/pebble/sstable/reader.go b/github.com/cockroachdb/pebble/sstable/reader.go index 05f741b2e9..34a562d978 100644 --- a/github.com/cockroachdb/pebble/sstable/reader.go +++ b/github.com/cockroachdb/pebble/sstable/reader.go @@ -87,6 +87,31 @@ type singleLevelIterator struct { dataBH BlockHandle err error closeHook func(i Iterator) error + + // boundsCmp and positionedUsingLatestBounds are for optimizing iteration + // that uses multiple adjacent bounds. The seek after setting a new bound + // can use the fact that the iterator is either within the previous bounds + // or exactly one key before or after the bounds. If the new bounds is + // after/before the previous bounds, and we are already positioned at a + // block that is relevant for the new bounds, we can try to first position + // using Next/Prev instead of doing a more expensive seek. + // + // When there are wide files at higher levels that match the bounds + // but don't have any data for the bound, we will already be + // positioned at the key beyond the bounds and won't need to do much + // work -- given that most data is in L6, such files are likely to + // dominate the performance of the mergingIter, and may be the main + // benefit of this performance optimization (of course it also helps + // when the file that has the data has successive seeks that stay in + // the same block). + // + // Specifically, boundsCmp captures the relationship between the previous + // and current bounds, if the iterator had been positioned after setting + // the previous bounds. If it was not positioned, i.e., Seek/First/Last + // were not called, we don't know where it is positioned and cannot + // optimize. + boundsCmp int + positionedUsingLatestBounds bool } // singleLevelIterator implements the base.InternalIterator interface. @@ -222,6 +247,68 @@ func (i *singleLevelIterator) loadBlock() bool { return true } +func (i *singleLevelIterator) initBoundsForAlreadyLoadedBlock() { + i.blockLower = i.lower + if i.blockLower != nil { + if i.data.firstKey.UserKey != nil && i.cmp(i.blockLower, i.data.firstKey.UserKey) < 0 { + // The lower-bound is less than the first key in the block. No need + // to check the lower-bound again for this block. + i.blockLower = nil + } + } + i.blockUpper = i.upper + if i.blockUpper != nil && i.cmp(i.blockUpper, i.index.Key().UserKey) > 0 { + // The upper-bound is greater than the index key which itself is greater + // than or equal to every key in the block. No need to check the + // upper-bound again for this block. + i.blockUpper = nil + } +} + +// The number of times to call Next/Prev in a block before giving up and seeking. +// The value of 4 is arbitrary. +const numStepsBeforeSeek = 4 + +func (i *singleLevelIterator) trySeekGEUsingNextWithinBlock( + key []byte, +) (k *InternalKey, v []byte, done bool) { + k, v = i.data.Key(), i.data.Value() + for j := 0; j < numStepsBeforeSeek; j++ { + curKeyCmp := i.cmp(k.UserKey, key) + if curKeyCmp >= 0 { + if i.blockUpper != nil && i.cmp(k.UserKey, i.blockUpper) >= 0 { + return nil, nil, true + } + return k, v, true + } + k, v = i.data.Next() + if k == nil { + break + } + } + return k, v, false +} + +func (i *singleLevelIterator) trySeekLTUsingPrevWithinBlock( + key []byte, +) (k *InternalKey, v []byte, done bool) { + k, v = i.data.Key(), i.data.Value() + for j := 0; j < numStepsBeforeSeek; j++ { + curKeyCmp := i.cmp(k.UserKey, key) + if curKeyCmp < 0 { + if i.blockLower != nil && i.cmp(k.UserKey, i.blockLower) < 0 { + return nil, nil, true + } + return k, v, true + } + k, v = i.data.Prev() + if k == nil { + break + } + } + return k, v, false +} + func (i *singleLevelIterator) recordOffset() uint64 { offset := i.dataBH.Offset if i.data.Valid() { @@ -245,22 +332,57 @@ func (i *singleLevelIterator) recordOffset() uint64 { // caller to ensure that key is greater than or equal to the lower bound. func (i *singleLevelIterator) SeekGE(key []byte) (*InternalKey, []byte) { i.err = nil // clear cached iteration error - - if ikey, _ := i.index.SeekGE(key); ikey == nil { - // The target key is greater than any key in the sstable. Invalidate the - // block iterator so that a subsequent call to Prev() will return the last - // key in the table. - i.data.invalidate() - return nil, nil - } - if !i.loadBlock() { - return nil, nil - } - if ikey, val := i.data.SeekGE(key); ikey != nil { - if i.blockUpper != nil && i.cmp(ikey.UserKey, i.blockUpper) >= 0 { + boundsCmp := i.boundsCmp + // Seek optimization only applies until iterator is first positioned after SetBounds. + i.boundsCmp = 0 + i.positionedUsingLatestBounds = true + return i.seekGEHelper(key, boundsCmp) +} + +// seekGEHelper contains the common functionality for SeekGE and SeekPrefixGE. +func (i *singleLevelIterator) seekGEHelper(key []byte, boundsCmp int) (*InternalKey, []byte) { + var dontSeekWithinBlock bool + + if !i.data.isInvalid() && !i.index.isInvalid() && i.data.Valid() && i.index.Valid() && + boundsCmp > 0 && i.cmp(key, i.index.Key().UserKey) <= 0 { + // Fast-path: The bounds have moved forward and this SeekGE is + // respecting the lower bound (guaranteed by Iterator). We know that + // the iterator must already be positioned within or just outside the + // previous bounds. Therefore it cannot be positioned at a block (or + // the position within that block) that is ahead of the seek position. + // However it can be positioned at an earlier block. This fast-path to + // use Next() on the block is only applied when we are already at the + // block that the slow-path (the else-clause) would load -- this is + // the motivation for the i.cmp(key, i.index.Key().UserKey) <= 0 + // predicate. + i.initBoundsForAlreadyLoadedBlock() + ikey, val, done := i.trySeekGEUsingNextWithinBlock(key) + if done { + return ikey, val + } + if ikey == nil { + // Done with this block. + dontSeekWithinBlock = true + } + } else { + if ikey, _ := i.index.SeekGE(key); ikey == nil { + // The target key is greater than any key in the sstable. Invalidate the + // block iterator so that a subsequent call to Prev() will return the last + // key in the table. + i.data.invalidate() + return nil, nil + } + if !i.loadBlock() { return nil, nil } - return ikey, val + } + if !dontSeekWithinBlock { + if ikey, val := i.data.SeekGE(key); ikey != nil { + if i.blockUpper != nil && i.cmp(ikey.UserKey, i.blockUpper) >= 0 { + return nil, nil + } + return ikey, val + } } return i.skipForward() } @@ -270,6 +392,9 @@ func (i *singleLevelIterator) SeekGE(key []byte) (*InternalKey, []byte) { // to the caller to ensure that key is greater than or equal to the lower bound. func (i *singleLevelIterator) SeekPrefixGE(prefix, key []byte) (*InternalKey, []byte) { i.err = nil // clear cached iteration error + boundsCmp := i.boundsCmp + // Seek optimization only applies until iterator is first positioned after SetBounds. + i.boundsCmp = 0 // Check prefix bloom filter. if i.reader.tableFilter != nil { @@ -286,21 +411,9 @@ func (i *singleLevelIterator) SeekPrefixGE(prefix, key []byte) (*InternalKey, [] return nil, nil } } - - if ikey, _ := i.index.SeekGE(key); ikey == nil { - i.data.invalidate() - return nil, nil - } - if !i.loadBlock() { - return nil, nil - } - if ikey, val := i.data.SeekGE(key); ikey != nil { - if i.blockUpper != nil && i.cmp(ikey.UserKey, i.blockUpper) >= 0 { - return nil, nil - } - return ikey, val - } - return i.skipForward() + // Bloom filter matches, so this method will position the iterator. + i.positionedUsingLatestBounds = true + return i.seekGEHelper(key, boundsCmp) } // SeekLT implements internalIterator.SeekLT, as documented in the pebble @@ -308,18 +421,47 @@ func (i *singleLevelIterator) SeekPrefixGE(prefix, key []byte) (*InternalKey, [] // caller to ensure that key is less than the upper bound. func (i *singleLevelIterator) SeekLT(key []byte) (*InternalKey, []byte) { i.err = nil // clear cached iteration error - - if ikey, _ := i.index.SeekGE(key); ikey == nil { - i.index.Last() - } - if !i.loadBlock() { - return nil, nil - } - if ikey, val := i.data.SeekLT(key); ikey != nil { - if i.blockLower != nil && i.cmp(ikey.UserKey, i.blockLower) < 0 { + boundsCmp := i.boundsCmp + // Seek optimization only applies until iterator is first positioned after SetBounds. + i.boundsCmp = 0 + i.positionedUsingLatestBounds = true + + var dontSeekWithinBlock bool + if !i.data.isInvalid() && !i.index.isInvalid() && i.data.Valid() && i.index.Valid() && + boundsCmp < 0 && i.cmp(i.data.firstKey.UserKey, key) < 0 { + // Fast-path: The bounds have moved backward, and this SeekLT is + // respecting the upper bound (guaranteed by Iterator). We know that + // the iterator must already be positioned within or just outside the + // previous bounds. Therefore it cannot be positioned at a block (or + // the position within that block) that is behind the seek position. + // However it can be positioned at a later block. This fast-path to + // use Prev() on the block is only applied when we are already at the + // block that can satisfy this seek -- this is the motivation for the + // the i.cmp(i.data.firstKey.UserKey, key) < 0 predicate. + i.initBoundsForAlreadyLoadedBlock() + ikey, val, done := i.trySeekLTUsingPrevWithinBlock(key) + if done { + return ikey, val + } + if ikey == nil { + // Done with this block. + dontSeekWithinBlock = true + } + } else { + if ikey, _ := i.index.SeekGE(key); ikey == nil { + i.index.Last() + } + if !i.loadBlock() { return nil, nil } - return ikey, val + } + if !dontSeekWithinBlock { + if ikey, val := i.data.SeekLT(key); ikey != nil { + if i.blockLower != nil && i.cmp(ikey.UserKey, i.blockLower) < 0 { + return nil, nil + } + return ikey, val + } } // The index contains separator keys which may lie between // user-keys. Consider the user-keys: @@ -341,6 +483,9 @@ func (i *singleLevelIterator) SeekLT(key []byte) (*InternalKey, []byte) { // call to SeekGE(lower)). func (i *singleLevelIterator) First() (*InternalKey, []byte) { i.err = nil // clear cached iteration error + // Seek optimization only applies until iterator is first positioned after SetBounds. + i.boundsCmp = 0 + i.positionedUsingLatestBounds = true if ikey, _ := i.index.First(); ikey == nil { i.data.invalidate() @@ -364,6 +509,9 @@ func (i *singleLevelIterator) First() (*InternalKey, []byte) { // SeekLT(upper)) func (i *singleLevelIterator) Last() (*InternalKey, []byte) { i.err = nil // clear cached iteration error + // Seek optimization only applies until iterator is first positioned after SetBounds. + i.boundsCmp = 0 + i.positionedUsingLatestBounds = true if ikey, _ := i.index.Last(); ikey == nil { i.data.invalidate() @@ -386,6 +534,9 @@ func (i *singleLevelIterator) Last() (*InternalKey, []byte) { // Note: compactionIterator.Next mirrors the implementation of Iterator.Next // due to performance. Keep the two in sync. func (i *singleLevelIterator) Next() (*InternalKey, []byte) { + // Seek optimization only applies until iterator is first positioned after SetBounds. + i.boundsCmp = 0 + if i.err != nil { return nil, nil } @@ -401,6 +552,9 @@ func (i *singleLevelIterator) Next() (*InternalKey, []byte) { // Prev implements internalIterator.Prev, as documented in the pebble // package. func (i *singleLevelIterator) Prev() (*InternalKey, []byte) { + // Seek optimization only applies until iterator is first positioned after SetBounds. + i.boundsCmp = 0 + if i.err != nil { return nil, nil } @@ -515,6 +669,15 @@ func (i *singleLevelIterator) String() string { // SetBounds implements internalIterator.SetBounds, as documented in the pebble // package. func (i *singleLevelIterator) SetBounds(lower, upper []byte) { + i.boundsCmp = 0 + if i.positionedUsingLatestBounds { + if i.upper != nil && lower != nil && i.cmp(i.upper, lower) <= 0 { + i.boundsCmp = +1 + } else if i.lower != nil && upper != nil && i.cmp(upper, i.lower) <= 0 { + i.boundsCmp = -1 + } + i.positionedUsingLatestBounds = false + } i.lower = lower i.upper = upper i.blockLower = nil @@ -661,15 +824,27 @@ func (i *twoLevelIterator) String() string { func (i *twoLevelIterator) SeekGE(key []byte) (*InternalKey, []byte) { i.err = nil // clear cached iteration error - if ikey, _ := i.topLevelIndex.SeekGE(key); ikey == nil { - i.data.invalidate() - i.index.invalidate() - return nil, nil - } + if i.topLevelIndex.isInvalid() || !i.topLevelIndex.Valid() || i.boundsCmp <= 0 || + i.cmp(key, i.topLevelIndex.Key().UserKey) > 0 { + // Slow-path: need to position the topLevelIndex. + if ikey, _ := i.topLevelIndex.SeekGE(key); ikey == nil { + i.data.invalidate() + i.index.invalidate() + return nil, nil + } - if !i.loadIndex() { - return nil, nil + if !i.loadIndex() { + return nil, nil + } } + // Else fast-path: The bounds have moved forward and this SeekGE is + // respecting the lower bound (guaranteed by Iterator). We know that + // the iterator must already be positioned within or just outside the + // previous bounds. Therefore the topLevelIndex iter cannot be + // positioned at an entry ahead of the seek position (though it can be + // positioned behind). The !i.cmp(key, i.topLevelIndex.Key().UserKey) > 0 + // confirms that it is not behind. Since it is not ahead and not behind + // it must be at the right position. if ikey, val := i.singleLevelIterator.SeekGE(key); ikey != nil { return ikey, val @@ -683,15 +858,30 @@ func (i *twoLevelIterator) SeekGE(key []byte) (*InternalKey, []byte) { func (i *twoLevelIterator) SeekPrefixGE(prefix, key []byte) (*InternalKey, []byte) { i.err = nil // clear cached iteration error - if ikey, _ := i.topLevelIndex.SeekGE(key); ikey == nil { - i.data.invalidate() - i.index.invalidate() - return nil, nil - } + // TODO(sumeer): check the bloom filter before doing other work and not + // delay it until singleLevelIterator.SeekPrefixGE. - if !i.loadIndex() { - return nil, nil + if i.topLevelIndex.isInvalid() || !i.topLevelIndex.Valid() || i.boundsCmp <= 0 || + i.cmp(key, i.topLevelIndex.Key().UserKey) > 0 { + // Slow-path: need to position the topLevelIndex. + if ikey, _ := i.topLevelIndex.SeekGE(key); ikey == nil { + i.data.invalidate() + i.index.invalidate() + return nil, nil + } + + if !i.loadIndex() { + return nil, nil + } } + // Else fast-path: The bounds have moved forward and this SeekGE is + // respecting the lower bound (guaranteed by Iterator). We know that + // the iterator must already be positioned within or just outside the + // previous bounds. Therefore the topLevelIndex iter cannot be + // positioned at an entry ahead of the seek position (though it can be + // positioned behind). The !i.cmp(key, i.topLevelIndex.Key().UserKey) > 0 + // confirms that it is not behind. Since it is not ahead and not behind + // it must be at the right position. if ikey, val := i.singleLevelIterator.SeekPrefixGE(prefix, key); ikey != nil { return ikey, val @@ -704,7 +894,13 @@ func (i *twoLevelIterator) SeekPrefixGE(prefix, key []byte) (*InternalKey, []byt // caller to ensure that key is less than the upper bound. func (i *twoLevelIterator) SeekLT(key []byte) (*InternalKey, []byte) { i.err = nil // clear cached iteration error + // Seek optimization only applies until iterator is first positioned after SetBounds. + i.boundsCmp = 0 + // NB: Unlike SeekGE, we don't have a fast-path here since we don't know + // whether the topLevelIndex is positioned after the position that would + // be returned by doing i.topLevelIndex.SeekGE(). To know this we would + // need to know the index key preceding the current one. if ikey, _ := i.topLevelIndex.SeekGE(key); ikey == nil { if ikey, _ := i.topLevelIndex.Last(); ikey == nil { i.data.invalidate() @@ -735,6 +931,8 @@ func (i *twoLevelIterator) SeekLT(key []byte) (*InternalKey, []byte) { // call to SeekGE(lower)). func (i *twoLevelIterator) First() (*InternalKey, []byte) { i.err = nil // clear cached iteration error + // Seek optimization only applies until iterator is first positioned after SetBounds. + i.boundsCmp = 0 if ikey, _ := i.topLevelIndex.First(); ikey == nil { return nil, nil @@ -756,6 +954,8 @@ func (i *twoLevelIterator) First() (*InternalKey, []byte) { // SeekLT(upper)) func (i *twoLevelIterator) Last() (*InternalKey, []byte) { i.err = nil // clear cached iteration error + // Seek optimization only applies until iterator is first positioned after SetBounds. + i.boundsCmp = 0 if ikey, _ := i.topLevelIndex.Last(); ikey == nil { return nil, nil diff --git a/github.com/cockroachdb/redact/api.go b/github.com/cockroachdb/redact/api.go index 68394b3b32..3ed2e486df 100644 --- a/github.com/cockroachdb/redact/api.go +++ b/github.com/cockroachdb/redact/api.go @@ -17,7 +17,8 @@ package redact import "fmt" // SafeFormatter is implemented by object types that want to separate -// safe and non-safe information. +// safe and non-safe information when printed out by a Printf-like +// formatter. type SafeFormatter interface { // SafeFormat is like the Format method of fmt.Formatter, except // that it operates using a SafePrinter instead of a fmt.State for diff --git a/github.com/cockroachdb/redact/markers.go b/github.com/cockroachdb/redact/markers.go index 49f7f6c015..8bf1e64bbf 100644 --- a/github.com/cockroachdb/redact/markers.go +++ b/github.com/cockroachdb/redact/markers.go @@ -42,6 +42,13 @@ func (s RedactableString) ToBytes() RedactableBytes { return RedactableBytes([]byte(string(s))) } +// SafeFormat formats the redactable safely. +func (s RedactableString) SafeFormat(sp SafePrinter, _ rune) { + // As per annotateArgs() in markers_internal_print.go, + // we consider the redactable string not further formattable. + sp.Print(s) +} + // RedactableBytes is like RedactableString but is a byte slice. // // Instances of RedactableBytes should not be constructed directly; @@ -67,6 +74,13 @@ func (s RedactableBytes) ToString() RedactableString { return RedactableString(string([]byte(s))) } +// SafeFormat formats the redactable safely. +func (s RedactableBytes) SafeFormat(sp SafePrinter, _ rune) { + // As per annotateArgs() in markers_internal_print.go, + // we consider the redactable bytes not further formattable. + sp.Print(s) +} + // EscapeBytes escapes markers inside the given byte slice and encloses // the entire byte slice between redaction markers. func EscapeBytes(s []byte) RedactableBytes { diff --git a/github.com/cockroachdb/redact/markers_internal_escape.go b/github.com/cockroachdb/redact/markers_internal_escape.go index da555058a0..ce23200dd1 100644 --- a/github.com/cockroachdb/redact/markers_internal_escape.go +++ b/github.com/cockroachdb/redact/markers_internal_escape.go @@ -87,20 +87,37 @@ func (p *escapeWriter) Write(b []byte) (int, error) { // Now write the string. k := 0 for i := 0; i < len(b); i++ { - // Ensure that occurrences of the delimiter inside the string get - // escaped. - if i+ls <= len(b) && bytes.Equal(b[i:i+ls], start) { + if b[i] == '\n' && p.enclose { + // Avoid enclosing newline characters inside redaction markers. + // This is important when redact is used to render errors, where + // sub-strings split at newline characters are rendered + // separately. st = p.doWrite(b[k:i], st, true) - st = p.doWrite(escape, st, false) - st.l += ls - k = i + ls - i += ls - 1 - } else if i+le <= len(b) && bytes.Equal(b[i:i+le], end) { - st = p.doWrite(b[k:i], st, true) - st = p.doWrite(escape, st, false) - st.l += le - k = i + le - i += le - 1 + st = p.doWrite(end, st, false) + lastNewLine := i + for b[lastNewLine] == '\n' && lastNewLine < len(b) { + lastNewLine++ + } + st = p.doWrite(b[i:lastNewLine], st, true) + st = p.doWrite(start, st, false) + k = lastNewLine + i = lastNewLine - 1 + } else { + // Ensure that occurrences of the delimiter inside the string get + // escaped. + if i+ls <= len(b) && bytes.Equal(b[i:i+ls], start) { + st = p.doWrite(b[k:i], st, true) + st = p.doWrite(escape, st, false) + st.l += ls + k = i + ls + i += ls - 1 + } else if i+le <= len(b) && bytes.Equal(b[i:i+le], end) { + st = p.doWrite(b[k:i], st, true) + st = p.doWrite(escape, st, false) + st.l += le + k = i + le + i += le - 1 + } } } st = p.doWrite(b[k:], st, true) diff --git a/github.com/cockroachdb/redact/markers_internal_print.go b/github.com/cockroachdb/redact/markers_internal_print.go index 1f9b92857d..003c9492f1 100644 --- a/github.com/cockroachdb/redact/markers_internal_print.go +++ b/github.com/cockroachdb/redact/markers_internal_print.go @@ -27,15 +27,23 @@ func annotateArgs(args []interface{}) { // We need an intermediate struct because we don't // want the fmt machinery to re-format the string // object by adding quotes, expanding the byte slice, etc. + // + // NB: keep this logic synchronized with + // (RedactableString).SafeFormat(). args[i] = &passthrough{arg: []byte(v)} + case RedactableBytes: + // NB: keep this logic synchronized with + // (RedactableBytes).SafeFormat(). args[i] = &passthrough{arg: v} case SafeFormatter: // calls to Format() by fmt.Print will be redirected to // v.SafeFormat(). This delegates the task of adding markers to // the object itself. - args[i] = &redactFormatRedirect{v} + args[i] = &redactFormatRedirect{ + func(p SafePrinter, verb rune) { v.SafeFormat(p, verb) }, + } case SafeValue: // calls to Format() by fmt.Print will be redirected to a @@ -52,19 +60,27 @@ func annotateArgs(args []interface{}) { args[i] = &escapeArg{arg: v.SafeMessage(), enclose: false} default: - // calls to Format() by fmt.Print will be redirected to a - // display of v within redaction markers if the type is - // considered unsafe, without markers otherwise. In any case, - // occurrences of delimiters within are escaped. - args[i] = &escapeArg{arg: v, enclose: !isSafeValue(v)} + if err, ok := v.(error); ok && redactErrorFn != nil { + // We place this case after the other cases above, in case + // the error object knows how to print itself safely already. + args[i] = &redactFormatRedirect{ + func(p SafePrinter, verb rune) { redactErrorFn(err, p, verb) }, + } + } else { + // calls to Format() by fmt.Print will be redirected to a + // display of v within redaction markers if the type is + // considered unsafe, without markers otherwise. In any case, + // occurrences of delimiters within are escaped. + args[i] = &escapeArg{arg: v, enclose: !isSafeValue(v)} + } } } } -// redactFormatRedirect wraps a SafeFormatter object and uses its -// SafeFormat method as-is to implement fmt.Formatter. +// redactFormatRedirect wraps a safe print callback and uses it to +// implement fmt.Formatter. type redactFormatRedirect struct { - arg SafeFormatter + printFn func(p SafePrinter, verb rune) } // Format implements fmt.Formatter. @@ -77,7 +93,7 @@ func (r *redactFormatRedirect) Format(s fmt.State, verb rune) { }() p := &printer{} p.escapeState = makeEscapeState(s, &p.buf) - r.arg.SafeFormat(p, verb) + r.printFn(p, verb) _, _ = s.Write(p.buf.Bytes()) } @@ -135,3 +151,14 @@ type printerfn struct { func (p printerfn) SafeFormat(w SafePrinter, _ rune) { p.fn(w) } + +// redactErrorFn can be injected from an error library +// to render error objects safely. +var redactErrorFn func(err error, p SafePrinter, verb rune) + +// RegisterRedactErrorFn registers an error redaction function for use +// during automatic redaction by this package. +// Provided e.g. by cockroachdb/errors. +func RegisterRedactErrorFn(fn func(err error, p SafePrinter, verb rune)) { + redactErrorFn = fn +} diff --git a/github.com/kr/pretty/formatter.go b/github.com/kr/pretty/formatter.go index df61d8d19e..bf4b598d06 100644 --- a/github.com/kr/pretty/formatter.go +++ b/github.com/kr/pretty/formatter.go @@ -37,7 +37,7 @@ func (fo formatter) passThrough(f fmt.State, c rune) { s := "%" for i := 0; i < 128; i++ { if f.Flag(i) { - s += string(i) + s += string(rune(i)) } } if w, ok := f.Width(); ok { diff --git a/golang.org/x/sys/cpu/cpu_linux_mips64x.go b/golang.org/x/sys/cpu/cpu_linux_mips64x.go index eb24e5073e..5a41890053 100644 --- a/golang.org/x/sys/cpu/cpu_linux_mips64x.go +++ b/golang.org/x/sys/cpu/cpu_linux_mips64x.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build linux // +build mips64 mips64le package cpu diff --git a/golang.org/x/sys/unix/syscall_linux.go b/golang.org/x/sys/unix/syscall_linux.go index c48f5ddadd..3e20fcf53b 100644 --- a/golang.org/x/sys/unix/syscall_linux.go +++ b/golang.org/x/sys/unix/syscall_linux.go @@ -1111,6 +1111,21 @@ func anyToSockaddr(fd int, rsa *RawSockaddrAny) (Sockaddr, error) { } return sa, nil + case AF_CAN: + pp := (*RawSockaddrCAN)(unsafe.Pointer(rsa)) + sa := &SockaddrCAN{ + Ifindex: int(pp.Ifindex), + } + rx := (*[4]byte)(unsafe.Pointer(&sa.RxID)) + for i := 0; i < 4; i++ { + rx[i] = pp.Addr[i] + } + tx := (*[4]byte)(unsafe.Pointer(&sa.TxID)) + for i := 0; i < 4; i++ { + tx[i] = pp.Addr[i+4] + } + return sa, nil + } return nil, EAFNOSUPPORT } diff --git a/golang.org/x/sys/unix/syscall_linux_arm.go b/golang.org/x/sys/unix/syscall_linux_arm.go index e1913e2c93..496837b1e3 100644 --- a/golang.org/x/sys/unix/syscall_linux_arm.go +++ b/golang.org/x/sys/unix/syscall_linux_arm.go @@ -7,7 +7,6 @@ package unix import ( - "syscall" "unsafe" ) @@ -49,10 +48,6 @@ func Pipe2(p []int, flags int) (err error) { return } -// Underlying system call writes to newoffset via pointer. -// Implemented in assembly to avoid allocation. -func seek(fd int, offset int64, whence int) (newoffset int64, err syscall.Errno) - func Seek(fd int, offset int64, whence int) (newoffset int64, err error) { newoffset, errno := seek(fd, offset, whence) if errno != 0 { diff --git a/golang.org/x/sys/unix/syscall_linux_gc_arm.go b/golang.org/x/sys/unix/syscall_linux_gc_arm.go new file mode 100644 index 0000000000..8c514c95ed --- /dev/null +++ b/golang.org/x/sys/unix/syscall_linux_gc_arm.go @@ -0,0 +1,13 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build arm,!gccgo,linux + +package unix + +import "syscall" + +// Underlying system call writes to newoffset via pointer. +// Implemented in assembly to avoid allocation. +func seek(fd int, offset int64, whence int) (newoffset int64, err syscall.Errno) diff --git a/golang.org/x/sys/unix/ztypes_linux.go b/golang.org/x/sys/unix/ztypes_linux.go index 77449a9e51..953166c73f 100644 --- a/golang.org/x/sys/unix/ztypes_linux.go +++ b/golang.org/x/sys/unix/ztypes_linux.go @@ -1016,6 +1016,13 @@ const ( PERF_SAMPLE_STREAM_ID = 0x200 PERF_SAMPLE_RAW = 0x400 PERF_SAMPLE_BRANCH_STACK = 0x800 + PERF_SAMPLE_REGS_USER = 0x1000 + PERF_SAMPLE_STACK_USER = 0x2000 + PERF_SAMPLE_WEIGHT = 0x4000 + PERF_SAMPLE_DATA_SRC = 0x8000 + PERF_SAMPLE_IDENTIFIER = 0x10000 + PERF_SAMPLE_TRANSACTION = 0x20000 + PERF_SAMPLE_REGS_INTR = 0x40000 PERF_SAMPLE_BRANCH_USER = 0x1 PERF_SAMPLE_BRANCH_KERNEL = 0x2 @@ -1745,6 +1752,21 @@ const ( NFT_NG_RANDOM = 0x1 ) +const ( + NFTA_TARGET_UNSPEC = 0x0 + NFTA_TARGET_NAME = 0x1 + NFTA_TARGET_REV = 0x2 + NFTA_TARGET_INFO = 0x3 + NFTA_MATCH_UNSPEC = 0x0 + NFTA_MATCH_NAME = 0x1 + NFTA_MATCH_REV = 0x2 + NFTA_MATCH_INFO = 0x3 + NFTA_COMPAT_UNSPEC = 0x0 + NFTA_COMPAT_NAME = 0x1 + NFTA_COMPAT_REV = 0x2 + NFTA_COMPAT_TYPE = 0x3 +) + type RTCTime struct { Sec int32 Min int32 diff --git a/modules.txt b/modules.txt index 9fa3932af5..bec1ab9317 100644 --- a/modules.txt +++ b/modules.txt @@ -165,7 +165,7 @@ github.com/cockroachdb/cockroach-go/crdb github.com/cockroachdb/crlfmt github.com/cockroachdb/crlfmt/internal/parser github.com/cockroachdb/crlfmt/internal/render -# github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5 +# github.com/cockroachdb/datadriven v1.0.0 github.com/cockroachdb/datadriven # github.com/cockroachdb/errors v1.6.1 github.com/cockroachdb/errors @@ -205,7 +205,7 @@ github.com/cockroachdb/gostdlib/x/tools/internal/semver github.com/cockroachdb/gostdlib/x/tools/internal/span # github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f github.com/cockroachdb/logtags -# github.com/cockroachdb/pebble v0.0.0-20200814004841-77c18adb0ee3 +# github.com/cockroachdb/pebble v0.0.0-20200817204059-a24bc6c83054 github.com/cockroachdb/pebble github.com/cockroachdb/pebble/bloom github.com/cockroachdb/pebble/internal/arenaskl @@ -228,7 +228,7 @@ github.com/cockroachdb/pebble/internal/record github.com/cockroachdb/pebble/sstable github.com/cockroachdb/pebble/tool github.com/cockroachdb/pebble/vfs -# github.com/cockroachdb/redact v0.0.0-20200622112456-cd282804bbd3 +# github.com/cockroachdb/redact v1.0.2 github.com/cockroachdb/redact # github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd github.com/cockroachdb/returncheck @@ -542,7 +542,7 @@ github.com/knz/strtime github.com/konsorten/go-windows-terminal-sequences # github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 github.com/kr/logfmt -# github.com/kr/pretty v0.2.0 +# github.com/kr/pretty v0.2.1 github.com/kr/pretty # github.com/kr/text v0.2.0 github.com/kr/text @@ -750,7 +750,7 @@ golang.org/x/crypto/ssh/agent golang.org/x/crypto/ssh/internal/bcrypt_pbkdf golang.org/x/crypto/ssh/knownhosts golang.org/x/crypto/ssh/terminal -# golang.org/x/exp v0.0.0-20200513190911-00229845015e +# golang.org/x/exp v0.0.0-20200819202907-27b6b2ade93b golang.org/x/exp/rand # golang.org/x/lint v0.0.0-20200130185559-910be7a94367 golang.org/x/lint @@ -784,7 +784,7 @@ golang.org/x/perf/storage/benchfmt # golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e golang.org/x/sync/errgroup golang.org/x/sync/syncmap -# golang.org/x/sys v0.0.0-20200817155316-9781c653f443 +# golang.org/x/sys v0.0.0-20200821140526-fda516888d29 golang.org/x/sys/cpu golang.org/x/sys/internal/unsafeheader golang.org/x/sys/unix From 1b1d687f4f28f4d6aac6ec8645cbc22e28c0b183 Mon Sep 17 00:00:00 2001 From: Raphael 'kena' Poss Date: Sat, 22 Aug 2020 09:28:51 +0200 Subject: [PATCH 28/49] bump cockroachdb/errors --- github.com/cockroachdb/errors/.travis.yml | 1 - github.com/cockroachdb/errors/README.md | 75 ++- .../cockroachdb/errors/assert/assert.go | 6 +- .../cockroachdb/errors/barriers/barriers.go | 6 +- .../errors/contexttags/contexttags.go | 23 +- .../errors/contexttags/with_context.go | 14 +- .../cockroachdb/errors/domains/domains.go | 2 +- .../cockroachdb/errors/domains/with_domain.go | 7 +- .../cockroachdb/errors/errbase/encode.go | 3 + .../errors/errbase/format_error.go | 537 ++++++++++++++---- .../cockroachdb/errors/errbase/formatter.go | 36 +- .../cockroachdb/errors/errbase/opaque.go | 34 +- .../errors/errbase/safe_details.go | 9 +- github.com/cockroachdb/errors/errbase_api.go | 21 + .../errors/errutil/format_error_special.go | 88 +++ .../cockroachdb/errors/errutil/message.go | 47 +- .../cockroachdb/errors/errutil/redactable.go | 110 ++++ .../cockroachdb/errors/errutil/utilities.go | 76 ++- github.com/cockroachdb/errors/errutil_api.go | 11 - github.com/cockroachdb/errors/go.mod | 4 +- github.com/cockroachdb/errors/go.sum | 8 + .../errors/issuelink/unimplemented_error.go | 14 +- .../errors/issuelink/with_issuelink.go | 11 +- .../cockroachdb/errors/markers/markers.go | 11 +- .../cockroachdb/errors/report/report.go | 24 +- .../cockroachdb/errors/safedetails/redact.go | 156 +---- .../errors/safedetails/redact_go112.go | 48 -- .../errors/safedetails/redact_go113.go | 23 - .../errors/safedetails/safedetails.go | 74 +-- .../errors/safedetails/with_safedetails.go | 23 +- .../cockroachdb/errors/safedetails_api.go | 11 +- .../errors/secondary/with_secondary.go | 7 +- .../errors/telemetrykeys/with_telemetry.go | 4 +- .../cockroachdb/errors/withstack/withstack.go | 8 +- modules.txt | 2 +- 35 files changed, 978 insertions(+), 556 deletions(-) create mode 100644 github.com/cockroachdb/errors/errutil/format_error_special.go create mode 100644 github.com/cockroachdb/errors/errutil/redactable.go delete mode 100644 github.com/cockroachdb/errors/safedetails/redact_go112.go delete mode 100644 github.com/cockroachdb/errors/safedetails/redact_go113.go diff --git a/github.com/cockroachdb/errors/.travis.yml b/github.com/cockroachdb/errors/.travis.yml index 00f925c140..a91ba657b2 100644 --- a/github.com/cockroachdb/errors/.travis.yml +++ b/github.com/cockroachdb/errors/.travis.yml @@ -1,6 +1,5 @@ language: go go: -- 1.12.x - 1.13.x - 1.14.x diff --git a/github.com/cockroachdb/errors/README.md b/github.com/cockroachdb/errors/README.md index 0320d73190..13b80ef468 100644 --- a/github.com/cockroachdb/errors/README.md +++ b/github.com/cockroachdb/errors/README.md @@ -46,6 +46,7 @@ Table of contents: | wrappers to attach secondary causes | | | | ✔ | | wrappers to attach [`logtags`](https://github.com/cockroachdb/logtags) details from `context.Context` | | | | ✔ | | `errors.FormatError()`, `Formatter`, `Printer` | | | (under construction) | ✔ | +| `errors.SafeFormatError()`, `SafeFormatter` | | | | ✔ | "Forward compatibility" above refers to the ability of this library to recognize and properly handle network communication of error types it @@ -201,17 +202,17 @@ return errors.Wrap(foo(), "foo") - `WithMessage(error, string) error`, `WithMessagef(error, string, ...interface{}) error`: message prefix. - when to use: probably never. Use `errors.Wrap()`/`errors.Wrapf()` instead. - what it does: adds a message prefix. - - how to access the detail: `Error()`, regular Go formatting. Not included in Sentry reports. + - how to access the detail: `Error()`, regular Go formatting, Sentry Report. - `WithDetail(error, string) error`, `WithDetailf(error, string, ...interface{}) error`, user-facing detail with contextual information. - **when to use: need to embark a message string to output when the error is presented to a human.** - what it does: captures detail strings. - - how to access the detail: `errors.GetAllDetails()`, `errors.FlattenDetails()` (all details are preserved), format with `%+v`. + - how to access the detail: `errors.GetAllDetails()`, `errors.FlattenDetails()` (all details are preserved), format with `%+v`. Not included in Sentry reports. - `WithHint(error, string) error`, `WithHintf(error, string, ...interface{}) error`: user-facing detail with suggestion for action to take. - **when to use: need to embark a message string to output when the error is presented to a human.** - what it does: captures hint strings. - - how to access the detail: `errors.GetAllHints()`, `errors.FlattenHints()` (hints are de-duplicated), format with `%+v`. + - how to access the detail: `errors.GetAllHints()`, `errors.FlattenHints()` (hints are de-duplicated), format with `%+v`. Not included in Sentry reports. - `WithIssueLink(error, IssueLink) error`: annotate an error with an URL and arbitrary string. - **when to use: to refer (human) users to some external resources.** @@ -243,13 +244,14 @@ return errors.Wrap(foo(), "foo") The library support PII-free strings essentially as follows: -- by default, strings included in an error object are considered to be - PII-unsafe, and are stripped out when building a Sentry report. -- some fields in the library are whitelisted by default. +- by default, many strings included in an error object are considered + to be PII-unsafe, and are stripped out when building a Sentry + report. +- some fields in the library are assumed to be PII-safe by default. - you can opt additional strings in to Sentry reports. -The following strings from this library are "whitelisted" upfront, -considered to be PII-free, and thus included in Sentry reports automatically: +The following strings from this library are considered to be PII-free, +and thus included in Sentry reports automatically: - the *type* of error objects, - stack traces (containing only file paths, line numbers, function names - arguments are not included), @@ -260,7 +262,7 @@ considered to be PII-free, and thus included in Sentry reports automatically: - the `format string` argument of `Newf`, `AssertionFailedf`, etc (the constructors ending with `...f()`), - the *type* of the additional arguments passed to the `...f()` constructors, - the *value of specific argument types* passed to the `...f()` constructors, when known to be PII-safe. - For details of which arguments are considered PII-free, see the [`Redact()` function](safedetails/redact.go). + For details of which arguments are considered PII-free, see the [`redact` package](https://github.com/cockroachdb/redact). It is possible to opt additional in to Sentry reporting, using either of the following methods: @@ -293,7 +295,7 @@ If your error type is a wrapper, you should implement a `Format()` method that redirects to `errors.FormatError()`, otherwise `%+v` will not work. Additionally, if your type has a payload not otherwise visible via `Error()`, you may want to implement -`errors.Formatter`. See [making `%+v` work with your +`errors.SafeFormatter`. See [making `%+v` work with your type](#Making-v-work-with-your-type) below for details. Finally, you may want your new error type to be portable across @@ -445,8 +447,8 @@ In short: type, this will disable the recursive application of the `%+v` flag to the causes chained from your wrapper.) -- You may optionally implement the `errors.Formatter` interface: - `FormatError(p errors.Printer) (next error)`. This is optional, but +- You may optionally implement the `errors.SafeFormatter` interface: + `SafeFormatError(p errors.Printer) (next error)`. This is optional, but should be done when some details are not included by `Error()` and should be emitted upon `%+v`. @@ -458,11 +460,11 @@ achieves this as follows: func (w *withHTTPCode) Format(s fmt.State, verb rune) { errors.FormatError(w, s, verb) } // FormatError() formats the error. -func (w *withHTTPCode) FormatError(p errors.Printer) (next error) { +func (w *withHTTPCode) SafeFormatError(p errors.Printer) (next error) { // Note: no need to print out the cause here! // FormatError() knows how to do this automatically. if p.Detail() { - p.Printf("http code: %d", w.code) + p.Printf("http code: %d", errors.Safe(w.code)) } return w.cause } @@ -492,22 +494,22 @@ proposal](https://go.googlesource.com/proposal/+/master/design/29934-error-value ## Error composition (summary) -| Constructor | Composes | -|------------------------------------|--------------------------------------------------------------------------------------------------| -| `New` | `NewWithDepth` (see below) | -| `Errorf` | = `Newf` | -| `Newf` | `NewWithDepthf` (see below) | -| `WithMessage` | = `pkgErr.WithMessage` | -| `Wrap` | `WrapWithDepth` (see below) | -| `Wrapf` | `WrapWithDepthf` (see below) | -| `AssertionFailed` | `AssertionFailedWithDepthf` (see below) | -| `NewWithDepth` | `goErr.New` + `WithStackDepth` (see below) | -| `NewWithDepthf` | `fmt.Errorf` + `WithSafeDetails` + `WithStackDepth` | -| `WithMessagef` | `pkgErr.WithMessagef` + `WithSafeDetails` | -| `WrapWithDepth` | `WithMessage` + `WithStackDepth` | -| `WrapWithDepthf` | `WithMessage` + `WithStackDepth` + `WithSafeDetails` | -| `AssertionFailedWithDepthf` | `fmt.Errorf` + `WithStackDepth` + `WithSafeDetails` + `WithAssertionFailure` | -| `NewAssertionErrorWithWrappedErrf` | `HandledWithMessagef` (barrier) + `WithStackDepth` + `WithSafeDetails` + `WithAssertionFailure` | +| Constructor | Composes | +|------------------------------------|-----------------------------------------------------------------------------------| +| `New` | `NewWithDepth` (see below) | +| `Errorf` | = `Newf` | +| `Newf` | `NewWithDepthf` (see below) | +| `WithMessage` | custom wrapper with message prefix and knowledge of safe strings | +| `Wrap` | `WrapWithDepth` (see below) | +| `Wrapf` | `WrapWithDepthf` (see below) | +| `AssertionFailed` | `AssertionFailedWithDepthf` (see below) | +| `NewWithDepth` | custom leaf with knowledge of safe strings + `WithStackDepth` (see below) | +| `NewWithDepthf` | custom leaf with knowledge of safe strings + `WithSafeDetails` + `WithStackDepth` | +| `WithMessagef` | custom wrapper with message prefix and knowledge of safe strings | +| `WrapWithDepth` | `WithMessage` + `WithStackDepth` | +| `WrapWithDepthf` | `WithMessagef` + `WithStackDepth` | +| `AssertionFailedWithDepthf` | `NewWithDepthf` + `WithAssertionFailure` | +| `NewAssertionErrorWithWrappedErrf` | `HandledWithMessagef` (barrier) + `WrapWithDepthf` + `WithAssertionFailure` | ## API (not constructing error objects) @@ -519,6 +521,13 @@ func Cause(err error) error // compatibility func Unwrap(err error) error // compatibility type Wrapper interface { ... } // compatibility +// Error formatting. +type Formatter interface { ... } // compatibility, not recommended +type SafeFormatter interface { ... } +type Printer interface { ... } +func FormatError(err error, s fmt.State, verb rune) +func Formattable(err error) fmt.Formatter + // Identify errors. func Is(err, reference error) bool func IsAny(err error, references ...error) bool @@ -553,10 +562,14 @@ func GetReportableStackTrace(err error) *ReportableStackTrace type SafeDetailPayload struct { ... } func GetAllSafeDetails(err error) []SafeDetailPayload func GetSafeDetails(err error) (payload SafeDetailPayload) + +// Obsolete APIs. type SafeMessager interface { ... } -func Safe(v interface{}) SafeMessager func Redact(r interface{}) string +// Aliases redact.Safe. +func Safe(v interface{}) SafeMessager + // Assertion failures. func HasAssertionFailure(err error) bool func IsAssertionFailure(err error) bool diff --git a/github.com/cockroachdb/errors/assert/assert.go b/github.com/cockroachdb/errors/assert/assert.go index 9c7ddd9cde..5c641a5f10 100644 --- a/github.com/cockroachdb/errors/assert/assert.go +++ b/github.com/cockroachdb/errors/assert/assert.go @@ -62,7 +62,7 @@ type withAssertionFailure struct { var _ error = (*withAssertionFailure)(nil) var _ fmt.Formatter = (*withAssertionFailure)(nil) -var _ errbase.Formatter = (*withAssertionFailure)(nil) +var _ errbase.SafeFormatter = (*withAssertionFailure)(nil) // ErrorHint implements the hintdetail.ErrorHinter interface. func (w *withAssertionFailure) ErrorHint() string { @@ -77,9 +77,9 @@ func (w *withAssertionFailure) Cause() error { return w.cause } func (w *withAssertionFailure) Unwrap() error { return w.cause } func (w *withAssertionFailure) Format(s fmt.State, verb rune) { errbase.FormatError(w, s, verb) } -func (w *withAssertionFailure) FormatError(p errbase.Printer) error { +func (w *withAssertionFailure) SafeFormatError(p errbase.Printer) error { if p.Detail() { - p.Print("assertion failure") + p.Printf("assertion failure") } return w.cause } diff --git a/github.com/cockroachdb/errors/barriers/barriers.go b/github.com/cockroachdb/errors/barriers/barriers.go index 24b03d73b0..fa4f431180 100644 --- a/github.com/cockroachdb/errors/barriers/barriers.go +++ b/github.com/cockroachdb/errors/barriers/barriers.go @@ -75,7 +75,7 @@ type barrierError struct { var _ error = (*barrierError)(nil) var _ errbase.SafeDetailer = (*barrierError)(nil) -var _ errbase.Formatter = (*barrierError)(nil) +var _ errbase.SafeFormatter = (*barrierError)(nil) var _ fmt.Formatter = (*barrierError)(nil) // barrierError is an error. @@ -94,8 +94,8 @@ func (e *barrierError) SafeDetails() []string { // Printing a barrier reveals the details. func (e *barrierError) Format(s fmt.State, verb rune) { errbase.FormatError(e, s, verb) } -func (e *barrierError) FormatError(p errbase.Printer) (next error) { - p.Printf(e.msg) +func (e *barrierError) SafeFormatError(p errbase.Printer) (next error) { + p.Print(e.msg) if p.Detail() { p.Printf("-- cause hidden behind barrier\n%+v", e.maskedErr) } diff --git a/github.com/cockroachdb/errors/contexttags/contexttags.go b/github.com/cockroachdb/errors/contexttags/contexttags.go index 6b520c7cc4..8b8277f290 100644 --- a/github.com/cockroachdb/errors/contexttags/contexttags.go +++ b/github.com/cockroachdb/errors/contexttags/contexttags.go @@ -18,8 +18,8 @@ import ( "context" "github.com/cockroachdb/errors/errbase" - "github.com/cockroachdb/errors/safedetails" "github.com/cockroachdb/logtags" + "github.com/cockroachdb/redact" ) // WithContextTags captures the k/v pairs stored in the context via the @@ -88,15 +88,26 @@ func convertToStringsOnly(b *logtags.Buffer) (res *logtags.Buffer) { func redactTags(b *logtags.Buffer) []string { res := make([]string, len(b.Get())) + redactableTagsIterate(b, func(i int, r redact.RedactableString) { + res[i] = r.Redact().StripMarkers() + }) + return res +} + +func redactableTagsIterate(b *logtags.Buffer, fn func(i int, s redact.RedactableString)) { + var empty redact.SafeString for i, t := range b.Get() { - res[i] = t.Key() + k := t.Key() v := t.Value() + eq := empty + var val interface{} = empty if v != nil { - if len(res[i]) > 1 { - res[i] += "=" + if len(k) > 1 { + eq = "=" } - res[i] += safedetails.Redact(v) + val = v } + res := redact.Sprintf("%s%s%v", redact.Safe(k), eq, val) + fn(i, res) } - return res } diff --git a/github.com/cockroachdb/errors/contexttags/with_context.go b/github.com/cockroachdb/errors/contexttags/with_context.go index 407edb2e61..f1e63c7f03 100644 --- a/github.com/cockroachdb/errors/contexttags/with_context.go +++ b/github.com/cockroachdb/errors/contexttags/with_context.go @@ -21,6 +21,7 @@ import ( "github.com/cockroachdb/errors/errbase" "github.com/cockroachdb/errors/errorspb" "github.com/cockroachdb/logtags" + "github.com/cockroachdb/redact" "github.com/gogo/protobuf/proto" ) @@ -39,8 +40,8 @@ type withContext struct { var _ error = (*withContext)(nil) var _ errbase.SafeDetailer = (*withContext)(nil) +var _ errbase.SafeFormatter = (*withContext)(nil) var _ fmt.Formatter = (*withContext)(nil) -var _ errbase.Formatter = (*withContext)(nil) // withContext is an error. The original error message is preserved. func (w *withContext) Error() string { return w.cause.Error() } @@ -52,9 +53,16 @@ func (w *withContext) Unwrap() error { return w.cause } // Printing a withContext reveals the tags. func (w *withContext) Format(s fmt.State, verb rune) { errbase.FormatError(w, s, verb) } -func (w *withContext) FormatError(p errbase.Printer) error { +func (w *withContext) SafeFormatError(p errbase.Printer) error { if p.Detail() && w.tags != nil { - p.Printf("tags: [%s]", w.tags.String()) + p.Printf("tags: [") + redactableTagsIterate(w.tags, func(i int, r redact.RedactableString) { + if i > 0 { + p.Printf(",") + } + p.Print(r) + }) + p.Printf("]") } return w.cause } diff --git a/github.com/cockroachdb/errors/domains/domains.go b/github.com/cockroachdb/errors/domains/domains.go index dbc5be8425..2cdfe039c2 100644 --- a/github.com/cockroachdb/errors/domains/domains.go +++ b/github.com/cockroachdb/errors/domains/domains.go @@ -93,7 +93,7 @@ func HandledInDomainWithMessage(err error, domain Domain, msg string) error { // Handled creates a handled error in the implicit domain (see // PackageDomain() below) of its caller. // -// See the documentation of `errors.Handled()` for details. +// See the documentation of `barriers.Handled()` for details. func Handled(err error) error { return HandledInDomain(err, PackageDomainAtDepth(1)) } diff --git a/github.com/cockroachdb/errors/domains/with_domain.go b/github.com/cockroachdb/errors/domains/with_domain.go index d881244384..fbf24e3826 100644 --- a/github.com/cockroachdb/errors/domains/with_domain.go +++ b/github.com/cockroachdb/errors/domains/with_domain.go @@ -19,6 +19,7 @@ import ( "fmt" "github.com/cockroachdb/errors/errbase" + "github.com/cockroachdb/redact" "github.com/gogo/protobuf/proto" ) @@ -36,7 +37,7 @@ var _ error = (*withDomain)(nil) var _ errbase.SafeDetailer = (*withDomain)(nil) var _ errbase.TypeKeyMarker = (*withDomain)(nil) var _ fmt.Formatter = (*withDomain)(nil) -var _ errbase.Formatter = (*withDomain)(nil) +var _ errbase.SafeFormatter = (*withDomain)(nil) // withDomain is an error. The original error message is preserved. func (e *withDomain) Error() string { return e.cause.Error() } @@ -58,9 +59,9 @@ func (e *withDomain) SafeDetails() []string { func (e *withDomain) Format(s fmt.State, verb rune) { errbase.FormatError(e, s, verb) } -func (e *withDomain) FormatError(p errbase.Printer) error { +func (e *withDomain) SafeFormatError(p errbase.Printer) error { if p.Detail() { - p.Print(e.domain) + p.Print(redact.Safe(e.domain)) } return e.cause } diff --git a/github.com/cockroachdb/errors/errbase/encode.go b/github.com/cockroachdb/errors/errbase/encode.go index ca37de7568..a4857632e5 100644 --- a/github.com/cockroachdb/errors/errbase/encode.go +++ b/github.com/cockroachdb/errors/errbase/encode.go @@ -211,6 +211,9 @@ func getTypeDetails( // TypeKeyMarker can be implemented by errors that wish to extend // their type name as seen by GetTypeKey(). +// +// Note: the key marker is considered safe for reporting and +// is included in sentry reports. type TypeKeyMarker interface { ErrorKeyMarker() string } diff --git a/github.com/cockroachdb/errors/errbase/format_error.go b/github.com/cockroachdb/errors/errbase/format_error.go index 94ccb05099..41e0442459 100644 --- a/github.com/cockroachdb/errors/errbase/format_error.go +++ b/github.com/cockroachdb/errors/errbase/format_error.go @@ -28,8 +28,10 @@ import ( "fmt" "io" "reflect" - "strconv" + "strings" + "github.com/cockroachdb/redact" + "github.com/kr/pretty" pkgErr "github.com/pkg/errors" ) @@ -44,16 +46,53 @@ import ( // // Otherwise, its Error() text is printed. func FormatError(err error, s fmt.State, verb rune) { + formatErrorInternal(err, s, verb, false /* redactableOutput */) +} + +// FormatRedactableError formats an error as a safe object. +// +// Note that certain verb/flags combinations are currently not +// supported, and result in a rendering that considers the entire +// object as unsafe. For example, %q, %#v are not yet supported. +func FormatRedactableError(err error, s redact.SafePrinter, verb rune) { + formatErrorInternal(err, s, verb, true /* redactable */) +} + +func init() { + // Also inform the redact package of how to print an error + // safely. This is used when an error is passed as argument + // to one of the redact print functions. + redact.RegisterRedactErrorFn(FormatRedactableError) +} + +// Formattable wraps an error into a fmt.Formatter which +// will provide "smart" formatting even if the outer layer +// of the error does not implement the Formatter interface. +func Formattable(err error) fmt.Formatter { + return &errorFormatter{err} +} + +// formatErrorInternal is the shared logic between FormatError +// and FormatErrorRedactable. +// +// When the redactableOutput argument is true, the fmt.State argument +// is really a redact.SafePrinter and casted down as necessary. +// +// If verb and flags are not one of the supported error formatting +// combinations (in particular, %q, %#v etc), then the redactableOutput +// argument is ignored. This limitation may be lifted in a later +// version. +func formatErrorInternal(err error, s fmt.State, verb rune, redactableOutput bool) { // Assuming this function is only called from the Format method, and given // that FormatError takes precedence over Format, it cannot be called from // any package that supports errors.Formatter. It is therefore safe to // disregard that State may be a specific printer implementation and use one // of our choice instead. - p := state{State: s} + p := state{State: s, redactableOutput: redactableOutput} switch { - case verb == 'v' && s.Flag('+'): + case verb == 'v' && s.Flag('+') && !s.Flag('#'): // Here we are going to format as per %+v, into p.buf. // // We need to start with the innermost (root cause) error first, @@ -69,17 +108,30 @@ func FormatError(err error, s fmt.State, verb rune) { // We're done formatting. Apply width/precision parameters. p.finishDisplay(verb) - case verb == 'v' && s.Flag('#'): + case !redactableOutput && verb == 'v' && s.Flag('#'): + // We only know how to process %#v if redactable output is not + // requested. This is because the structured output may emit + // arbitrary unsafe strings without redaction markers, + // or improperly balanced/escaped redaction markers. if stringer, ok := err.(fmt.GoStringer); ok { - io.WriteString(&p.buf, stringer.GoString()) - p.finishDisplay(verb) - return + io.WriteString(&p.finalBuf, stringer.GoString()) + } else { + // Not a GoStringer: delegate to the pretty library. + fmt.Fprintf(&p.finalBuf, "%# v", pretty.Formatter(err)) } - // Not a stringer. Proceed as if it were %v. - fallthrough + p.finishDisplay(verb) - case verb == 's' || verb == 'q' || - verb == 'x' || verb == 'X' || verb == 'v': + case verb == 's' || + // We only handle %v/%+v or other combinations here; %#v is unsupported. + (verb == 'v' && !s.Flag('#')) || + // If redactable output is not requested, then we also + // know how to format %x/%X (print bytes of error message in hex) + // and %q (quote the result). + // If redactable output is requested, then we don't know + // how to perform these exotic verbs, because they + // may muck with the redaction markers. In this case, + // we simply refuse the format as per the default clause below. + (!redactableOutput && (verb == 'x' || verb == 'X' || verb == 'q')): // Only the error message. // // Use an intermediate buffer because there may be alignment @@ -99,61 +151,121 @@ func FormatError(err error, s fmt.State, verb rune) { default: // Unknown verb. Do like fmt.Printf and tell the user we're // confused. - p.buf.WriteString("%!") - p.buf.WriteRune(verb) - p.buf.WriteByte('(') + // + // Note that the following logic is correct regardless of the + // value of 'redactableOutput', because the display of the verb and type + // are always safe for redaction. If/when this code is changed to + // print more details, care is to be taken to add redaction + // markers if s.redactableOutput is set. + p.finalBuf.WriteString("%!") + p.finalBuf.WriteRune(verb) + p.finalBuf.WriteByte('(') switch { case err != nil: - p.buf.WriteString(reflect.TypeOf(err).String()) + p.finalBuf.WriteString(reflect.TypeOf(err).String()) default: - p.buf.WriteString("") + p.finalBuf.WriteString("") } - p.buf.WriteByte(')') - io.Copy(s, &p.buf) + p.finalBuf.WriteByte(')') + io.Copy(s, &p.finalBuf) } } +// formatEntries reads the entries from s.entries and produces a +// detailed rendering in s.finalBuf. +// +// Note that if s.redactableOutput is true, s.finalBuf is to contain a +// RedactableBytes. However, we are not using the helper facilities +// from redact.SafePrinter to do this, so care should be taken below +// to properly escape markers, etc. func (s *state) formatEntries(err error) { // The first entry at the top is special. We format it as follows: // // // (1)

s.formatSingleLineOutput() - s.buf.WriteString("\n(1)") - printEntry(&s.buf, s.entries[len(s.entries)-1]) + s.finalBuf.WriteString("\n(1)") + + s.printEntry(s.entries[len(s.entries)-1]) // All the entries that follow are printed as follows: // // Wraps: (N)
// for i, j := len(s.entries)-2, 2; i >= 0; i, j = i-1, j+1 { - fmt.Fprintf(&s.buf, "\nWraps: (%d)", j) + fmt.Fprintf(&s.finalBuf, "\nWraps: (%d)", j) entry := s.entries[i] - printEntry(&s.buf, entry) + s.printEntry(entry) } // At the end, we link all the (N) references to the Go type of the // error. - s.buf.WriteString("\nError types:") + s.finalBuf.WriteString("\nError types:") for i, j := len(s.entries)-1, 1; i >= 0; i, j = i-1, j+1 { - fmt.Fprintf(&s.buf, " (%d) %T", j, s.entries[i].err) + fmt.Fprintf(&s.finalBuf, " (%d) %T", j, s.entries[i].err) } } -func printEntry(buf *bytes.Buffer, entry formatEntry) { +// printEntry renders the entry given as argument +// into s.finalBuf. +// +// If s.redactableOutput is set, then s.finalBuf is to contain +// a RedactableBytes, with redaction markers. In that +// case, we must be careful to escape (or not) the entry +// depending on entry.redactable. +// +// If s.redactableOutput is unset, then we are not caring about +// redactability. In that case entry.redactable is not set +// anyway and we can pass contents through. +func (s *state) printEntry(entry formatEntry) { if len(entry.head) > 0 { if entry.head[0] != '\n' { - buf.WriteByte(' ') + s.finalBuf.WriteByte(' ') + } + if len(entry.head) > 0 { + if !s.redactableOutput || entry.redactable { + // If we don't care about redaction, then we can pass the string + // through. + // + // If we do care about redaction, and entry.redactable is true, + // then entry.head is already a RedactableBytes. Then we can + // also pass it through. + s.finalBuf.Write(entry.head) + } else { + // We care about redaction, and the head is unsafe. Escape it + // and enclose the result within redaction markers. + s.finalBuf.Write([]byte(redact.EscapeBytes(entry.head))) + } } - buf.Write(entry.head) } if len(entry.details) > 0 { if len(entry.head) == 0 { if entry.details[0] != '\n' { - buf.WriteByte(' ') + s.finalBuf.WriteByte(' ') } } - buf.Write(entry.details) + if !s.redactableOutput || entry.redactable { + // If we don't care about redaction, then we can pass the string + // through. + // + // If we do care about redaction, and entry.redactable is true, + // then entry.details is already a RedactableBytes. Then we can + // also pass it through. + s.finalBuf.Write(entry.details) + } else { + // We care about redaction, and the details are unsafe. Escape + // them and enclose the result within redaction markers. + s.finalBuf.Write([]byte(redact.EscapeBytes(entry.details))) + } + } + if entry.stackTrace != nil { + s.finalBuf.WriteString("\n -- stack trace:") + s.finalBuf.WriteString(strings.ReplaceAll( + fmt.Sprintf("%+v", entry.stackTrace), + "\n", string(detailSep))) + if entry.elidedStackTrace { + fmt.Fprintf(&s.finalBuf, "%s[...repeated from below...]", detailSep) + } } } @@ -168,23 +280,53 @@ func printEntry(buf *bytes.Buffer, entry formatEntry) { // (e *myType) Format(s fmt.State, verb rune) { errors.FormatError(s, verb, e) } // // and also to print the first line in the output of a %+v format. +// +// It reads from s.entries and writes to s.finalBuf. +// s.buf is left untouched. +// +// Note that if s.redactableOutput is true, s.finalBuf is to contain a +// RedactableBytes. However, we are not using the helper facilities +// from redact.SafePrinter to do this, so care should be taken below +// to properly escape markers, etc. func (s *state) formatSingleLineOutput() { for i := len(s.entries) - 1; i >= 0; i-- { entry := &s.entries[i] if entry.elideShort { continue } - if s.buf.Len() > 0 && len(entry.head) > 0 { - s.buf.WriteString(": ") + if s.finalBuf.Len() > 0 && len(entry.head) > 0 { + s.finalBuf.WriteString(": ") + } + if len(entry.head) == 0 { + // shortcut, to avoid the copy below. + continue + } + if !s.redactableOutput || entry.redactable { + // If we don't care about redaction, then we can pass the string + // through. + // + // If we do care about redaction, and entry.redactable is true, + // then entry.head is already a RedactableBytes. Then we can + // also pass it through. + s.finalBuf.Write(entry.head) + } else { + // We do care about redaction, but entry.redactable is unset. + // This means entry.head is unsafe. We need to escape it. + s.finalBuf.Write([]byte(redact.EscapeBytes(entry.head))) } - s.buf.Write(entry.head) } } // formatRecursive performs a post-order traversal on the chain of // errors to collect error details from innermost to outermost. // +// It uses s.buf as an intermediate buffer to collect strings. // It populates s.entries as a result. +// Between each layer of error, s.buf is reset. +// +// s.finalBuf is untouched. The conversion of s.entries +// to s.finalBuf is done by formatSingleLineOutput() and/or +// formatEntries(). func (s *state) formatRecursive(err error, isOutermost, withDetail bool) { cause := UnwrapOnce(err) if cause != nil { @@ -203,68 +345,93 @@ func (s *state) formatRecursive(err error, isOutermost, withDetail bool) { seenTrace := false - switch v := err.(type) { - case Formatter: - desiredShortening := v.FormatError((*printer)(s)) - if desiredShortening == nil { - // The error wants to elide the short messages. Do it. - for i := range s.entries { - s.entries[i].elideShort = true + bufIsRedactable := false + + printDone := false + for _, fn := range specialCases { + if handled, desiredShortening := fn(err, (*safePrinter)(s), cause == nil /* leaf */); handled { + printDone = true + bufIsRedactable = true + if desiredShortening == nil { + // The error wants to elide the short messages from inner + // causes. Do it. + for i := range s.entries { + s.entries[i].elideShort = true + } } + break } - case fmt.Formatter: - // We can only use a fmt.Formatter when both the following - // conditions are true: - // - when it is the leaf error, because a fmt.Formatter - // on a wrapper also recurses. - // - when it is not the outermost wrapper, because - // the Format() method is likely to be calling FormatError() - // to do its job and we want to avoid an infinite recursion. - if !isOutermost && cause == nil { - v.Format(s, 'v') - if st, ok := err.(StackTraceProvider); ok { - // This is likely a leaf error from github/pkg/errors. - // The thing probably printed its stack trace on its own. - seenTrace = true - // We'll subsequently simplify stack traces in wrappers. - s.lastStack = st.StackTrace() + } + if !printDone { + switch v := err.(type) { + case SafeFormatter: + bufIsRedactable = true + desiredShortening := v.SafeFormatError((*safePrinter)(s)) + if desiredShortening == nil { + // The error wants to elide the short messages from inner + // causes. Do it. + for i := range s.entries { + s.entries[i].elideShort = true + } } - } else { + + case Formatter: + desiredShortening := v.FormatError((*printer)(s)) + if desiredShortening == nil { + // The error wants to elide the short messages from inner + // causes. Do it. + for i := range s.entries { + s.entries[i].elideShort = true + } + } + + case fmt.Formatter: + // We can only use a fmt.Formatter when both the following + // conditions are true: + // - when it is the leaf error, because a fmt.Formatter + // on a wrapper also recurses. + // - when it is not the outermost wrapper, because + // the Format() method is likely to be calling FormatError() + // to do its job and we want to avoid an infinite recursion. + if !isOutermost && cause == nil { + v.Format(s, 'v') + if st, ok := err.(StackTraceProvider); ok { + // This is likely a leaf error from github/pkg/errors. + // The thing probably printed its stack trace on its own. + seenTrace = true + // We'll subsequently simplify stack traces in wrappers. + s.lastStack = st.StackTrace() + } + } else { + s.formatSimple(err, cause) + } + + default: + // If the error did not implement errors.Formatter nor + // fmt.Formatter, but it is a wrapper, still attempt best effort: + // print what we can at this level. s.formatSimple(err, cause) } - default: - // If the error did not implement errors.Formatter nor - // fmt.Formatter, but it is a wrapper, still attempt best effort: - // print what we can at this level. - s.formatSimple(err, cause) } - // If there's an embedded stack trace, print it. + // Collect the result. + entry := s.collectEntry(err, bufIsRedactable) + + // If there's an embedded stack trace, also collect it. // This will get either a stack from pkg/errors, or ours. if !seenTrace { if st, ok := err.(StackTraceProvider); ok { - newStack, elided := ElideSharedStackTraceSuffix(s.lastStack, st.StackTrace()) - s.lastStack = newStack - if s.wantDetail { - s.switchOver() - if s.multiLine { - s.Write([]byte("\n-- stack trace:")) - } - newStack.Format(s, 'v') - if elided { - s.Write([]byte("\n[...repeated from below...]")) - } - } + entry.stackTrace, entry.elidedStackTrace = ElideSharedStackTraceSuffix(s.lastStack, st.StackTrace()) + s.lastStack = entry.stackTrace } } - // Collect the result. - entry := s.collectEntry(err) + // Remember the entry for later rendering. s.entries = append(s.entries, entry) s.buf = bytes.Buffer{} } -func (s *state) collectEntry(err error) formatEntry { +func (s *state) collectEntry(err error, bufIsRedactable bool) formatEntry { entry := formatEntry{err: err} if s.wantDetail { // The buffer has been populated as a result of formatting with @@ -284,9 +451,40 @@ func (s *state) collectEntry(err error) formatEntry { } entry.head = append(entry.head, s.buf.Bytes()...) } + + if bufIsRedactable { + // In this case, we've produced entry.head/entry.details using a + // SafeFormatError() invocation. The strings in + // entry.head/entry.detail contain redaction markers at this + // point. + if s.redactableOutput { + // Redaction markers desired in the final output. Keep the + // redaction markers. + entry.redactable = true + } else { + // Markers not desired in the final output: strip the markers. + entry.head = redact.RedactableBytes(entry.head).StripMarkers() + entry.details = redact.RedactableBytes(entry.details).StripMarkers() + } + } + return entry } +// safeErrorPrinterFn is the type of a function that can take +// over the safe printing of an error. This is used to inject special +// cases into the formatting in errutil. We need this machinery to +// prevent import cycles. +type safeErrorPrinterFn = func(err error, p Printer, isLeaf bool) (handled bool, next error) + +// specialCases is a list of functions to apply for special cases. +var specialCases []safeErrorPrinterFn + +// RegisterSpecialCasePrinter registers a handler. +func RegisterSpecialCasePrinter(fn safeErrorPrinterFn) { + specialCases = append(specialCases, fn) +} + // formatSimple performs a best effort at extracting the details at a // given level of wrapping when the error object does not implement // the Formatter interface. @@ -302,14 +500,23 @@ func (s *state) formatSimple(err, cause error) { } } -// finishDisplay renders the buffer in state into the fmt.State. +// finishDisplay renders s.finalBuf into s.State. func (p *state) finishDisplay(verb rune) { + if p.redactableOutput { + // If we're rendering in redactable form, then s.finalBuf contains + // a RedactableBytes. We can emit that directly. + sp := p.State.(redact.SafePrinter) + sp.Print(redact.RedactableBytes(p.finalBuf.Bytes())) + return + } + // Not redactable: render depending on flags and verb. + width, okW := p.Width() - prec, okP := p.Precision() + _, okP := p.Precision() // If `direct` is set to false, then the buffer is always // passed through fmt.Printf regardless of the width and alignment - // settings. This is important or e.g. %q where quotes must be added + // settings. This is important for e.g. %q where quotes must be added // in any case. // If `direct` is set to true, then the detour via // fmt.Printf only occurs if there is a width or alignment @@ -317,28 +524,10 @@ func (p *state) finishDisplay(verb rune) { direct := verb == 'v' || verb == 's' if !direct || (okW && width > 0) || okP { - // Construct format string from State s. - format := []byte{'%'} - if p.Flag('-') { - format = append(format, '-') - } - if p.Flag('+') { - format = append(format, '+') - } - if p.Flag(' ') { - format = append(format, ' ') - } - if okW { - format = strconv.AppendInt(format, int64(width), 10) - } - if okP { - format = append(format, '.') - format = strconv.AppendInt(format, int64(prec), 10) - } - format = append(format, string(verb)...) - fmt.Fprintf(p.State, string(format), p.buf.String()) + _, format := redact.MakeFormat(p, verb) + fmt.Fprintf(p.State, format, p.finalBuf.String()) } else { - io.Copy(p.State, &p.buf) + io.Copy(p.State, &p.finalBuf) } } @@ -346,12 +535,35 @@ var detailSep = []byte("\n | ") // state tracks error printing state. It implements fmt.State. type state struct { + // state inherits fmt.State. + // + // If we are rendering with redactableOutput=true, then fmt.State + // can be downcasted to redact.SafePrinter. fmt.State - // buf collects the details of the current error object - // at a given stage of recursion in formatRecursive(). - // At each stage of recursion (level of wrapping), buf - // contains successively: + // redactableOutput indicates whether we want the output + // to use redaction markers. When set to true, + // the fmt.State above is actually a redact.SafePrinter. + redactableOutput bool + + // finalBuf contains the final rendered string, prior to being + // copied to the fmt.State above. + // + // If redactableOutput is true, then finalBuf contains a RedactableBytes + // and safe redaction markers. Otherwise, it can be considered + // an unsafe string. + finalBuf bytes.Buffer + + // entries collect the result of formatRecursive(). They are + // consumed by formatSingleLineOutput() and formatEntries() to + // procude the contents of finalBuf. + entries []formatEntry + + // buf collects the details of the current error object at a given + // stage of recursion in formatRecursive(). + // + // At each stage of recursion (level of wrapping), buf contains + // successively: // // - at the beginning, the "simple" part of the error message -- // either the pre-Detail() string if the error implements Formatter, @@ -360,15 +572,53 @@ type state struct { // - after the first call to Detail(), buf is copied to headBuf, // then reset, then starts collecting the "advanced" part of the // error message. + // + // At the end of an error layer, the contents of buf and headBuf + // are collected into a formatEntry by collectEntry(). + // This collection does not touch finalBuf above. + // + // The entries are later consumed by formatSingleLineOutput() or + // formatEntries() to produce the contents of finalBuf. + // + // + // Notes regarding redaction markers and string safety. Throughout a + // single "level" of error, there are three cases to consider: + // + // - the error level implements SafeErrorFormatter and + // s.redactableOutput is set. In that case, the error's + // SafeErrorFormat() is used to produce a RedactableBytes in + // buf/headBuf via safePrinter{}, and an entry is collected at the + // end of that with the redactable bit set on the entry. + // + // - the error level implements SafeErrorFormatter + // and s.redactableOutput is *not* set. In this case, + // for convenience we implement non-redactable output by using + // SafeErrorFormat() to generate a RedactableBytes into + // buf/headBuf via safePrinter{}, and then stripping the redaction + // markers to produce the entry. The entry is not marked as + // redactable. + // + // - in the remaining case (s.redactableOutput is not set or the + // error only implements Formatter), then we use FormatError() + // to produce a non-redactable string into buf/headBuf, + // and mark the resulting entry as non-redactable. buf bytes.Buffer // When an error's FormatError() calls Detail(), the current // value of buf above is copied to headBuf, and a new // buf is initialized. headBuf []byte - // entries collects the result of formatRecursive(). - entries []formatEntry - // hasDetail becomes true at each level of th formatRecursive() + // lastStack tracks the last stack trace observed when looking at + // the errors from innermost to outermost. This is used to elide + // redundant stack trace entries. + lastStack StackTrace + + // --------------------------------------------------------------- + // The following attributes organize the synchronization of writes + // to buf and headBuf, during the rendering of a single error + // layer. They get reset between layers. + + // hasDetail becomes true at each level of the formatRecursive() // recursion after the first call to .Detail(). It is used to // determine how to translate buf/headBuf into a formatEntry. hasDetail bool @@ -381,10 +631,10 @@ type state struct { // errors.FormatError().) wantDetail bool - // lastStack tracks the last stack trace observed when looking at - // the errors from innermost to outermost. This is used to elide - // redundant stack trace entries. - lastStack StackTrace + // collectingRedactableString is true iff the data being accumulated + // into buf comes from a redact string. It ensures that newline + // characters are not included inside redaction markers. + collectingRedactableString bool // notEmpty tracks, at each level of recursion of formatRecursive(), // whether there were any details printed by an error's @@ -407,6 +657,10 @@ type state struct { // wrapping or the leaf error in an error chain. type formatEntry struct { err error + // redactable is true iff the data in head and details + // are RedactableBytes. See the explanatory comments + // on (state).buf for when this is set. + redactable bool // head is the part of the text that is suitable for printing in the // one-liner summary, or when producing the output of .Error(). head []byte @@ -416,6 +670,14 @@ type formatEntry struct { // elideShort, if true, elides the value of 'head' from concatenated // "short" messages produced by formatSingleLineOutput(). elideShort bool + + // stackTrace, if non-nil, reports the stack trace embedded at this + // level of error. + stackTrace StackTrace + // elidedStackTrace, if true, indicates that the stack trace was + // truncated to avoid duplication of entries. This is used to + // display a truncation indicator during verbose rendering. + elidedStackTrace bool } // String is used for debugging only. @@ -453,8 +715,6 @@ func (s *state) Write(b []byte) (n int, err error) { } else { if s.needNewline > 0 && s.notEmpty { // If newline chars were pending, display them now. - // The s.notEmpty condition ensures that we don't - // start a detail string with excess newline characters. for i := 0; i < s.needNewline-1; i++ { s.buf.Write(detailSep[:len(sep)-1]) } @@ -525,11 +785,60 @@ func (s *printer) enhanceArgs(args []interface{}) { s.lastStack = lastSeen } +// safePrinter is a variant to printer used when the current error +// level implements SafeFormatter. +// +// In any case, it uses the error's SafeFormatError() method to +// prepare a RedactableBytes into s.buf / s.headBuf. +// The the explanation for `buf` in the state struct. +type safePrinter state + +func (s *safePrinter) Detail() bool { + return ((*state)(s)).detail() +} + +func (s *safePrinter) Print(args ...interface{}) { + s.enhanceArgs(args) + redact.Fprint((*state)(s), args...) +} + +func (s *safePrinter) Printf(format string, args ...interface{}) { + s.enhanceArgs(args) + redact.Fprintf((*state)(s), format, args...) +} + +func (s *safePrinter) enhanceArgs(args []interface{}) { + prevStack := s.lastStack + lastSeen := prevStack + for i := range args { + if st, ok := args[i].(pkgErr.StackTrace); ok { + thisStack, _ := ElideSharedStackTraceSuffix(prevStack, st) + // Stack traces are safe strings. + args[i] = redact.Safe(thisStack) + lastSeen = st + } + // In contrast with (*printer).enhanceArgs(), we dont use a + // special case for `error` here, because the redact package + // already helps us recursing into a safe print for + // error objects. + } + s.lastStack = lastSeen +} + type errorFormatter struct{ err error } // Format implements the fmt.Formatter interface. func (ef *errorFormatter) Format(s fmt.State, verb rune) { FormatError(ef.err, s, verb) } +// Error implements error, so that `redact` knows what to do with it. +func (ef *errorFormatter) Error() string { return ef.err.Error() } + +// Unwrap makes it a wrapper. +func (ef *errorFormatter) Unwrap() error { return ef.err } + +// Cause makes it a wrapper. +func (ef *errorFormatter) Cause() error { return ef.err } + // ElideSharedStackTraceSuffix removes the suffix of newStack that's already // present in prevStack. The function returns true if some entries // were elided. diff --git a/github.com/cockroachdb/errors/errbase/formatter.go b/github.com/cockroachdb/errors/errbase/formatter.go index e8b8273e54..68262428a3 100644 --- a/github.com/cockroachdb/errors/errbase/formatter.go +++ b/github.com/cockroachdb/errors/errbase/formatter.go @@ -22,13 +22,16 @@ package errbase // A Formatter formats error messages. +// +// NB: Consider implementing SafeFormatter instead. This will ensure +// that error displays can distinguish bits that are PII-safe. type Formatter interface { error // FormatError prints the receiver's first error. // The return value decides what happens in the case // FormatError() is used to produce a "short" message, - // eg. when it is used to implementError(): + // eg. when it is used to implement Error(): // // - if it returns nil, then the short message // contains no more than that produced for this error, @@ -44,6 +47,37 @@ type Formatter interface { FormatError(p Printer) (next error) } +// SafeFormatter is implemented by error leaf or wrapper types that want +// to separate safe and non-safe information when printed out. +// +// When multiple errors are chained (e.g. via errors.Wrap), intermediate +// layers in the error that do not implement SafeError are considered +// “unsafe†+type SafeFormatter interface { + // SafeFormatError prints the receiver's first error. + // + // The provided Printer behaves like a redact.SafePrinter its + // Print() and Printf() methods conditionally add redaction markers + // around unsafe bits. + // + // The return value of SafeFormatError() decides what happens in the + // case the method is used to produce a "short" message, eg. when it + // is used to implement Error(): + // + // - if it returns nil, then the short message + // contains no more than that produced for this error, + // even if the error has a further causal chain. + // + // - if it returns non-nil, then the short message + // contains the value printed by this error, + // followed by that of its causal chain. + // (e.g. thiserror: itscause: furthercause) + // + // Note that all the causal chain is reported in verbose reports in + // any case. + SafeFormatError(p Printer) (next error) +} + // A Printer formats error messages. // // The most common implementation of Printer is the one provided by package fmt diff --git a/github.com/cockroachdb/errors/errbase/opaque.go b/github.com/cockroachdb/errors/errbase/opaque.go index 56ddfaf4a7..81ca032954 100644 --- a/github.com/cockroachdb/errors/errbase/opaque.go +++ b/github.com/cockroachdb/errors/errbase/opaque.go @@ -18,6 +18,7 @@ import ( "fmt" "github.com/cockroachdb/errors/errorspb" + "github.com/cockroachdb/redact" ) // opaqueLeaf is used when receiving an unknown leaf type. @@ -32,7 +33,7 @@ type opaqueLeaf struct { var _ error = (*opaqueLeaf)(nil) var _ SafeDetailer = (*opaqueLeaf)(nil) var _ fmt.Formatter = (*opaqueLeaf)(nil) -var _ Formatter = (*opaqueLeaf)(nil) +var _ SafeFormatter = (*opaqueLeaf)(nil) // opaqueWrapper is used when receiving an unknown wrapper type. // Its important property is that if it is communicated @@ -47,7 +48,7 @@ type opaqueWrapper struct { var _ error = (*opaqueWrapper)(nil) var _ SafeDetailer = (*opaqueWrapper)(nil) var _ fmt.Formatter = (*opaqueWrapper)(nil) -var _ Formatter = (*opaqueWrapper)(nil) +var _ SafeFormatter = (*opaqueWrapper)(nil) func (e *opaqueLeaf) Error() string { return e.msg } @@ -68,31 +69,38 @@ func (e *opaqueWrapper) SafeDetails() []string { return e.details.ReportablePayl func (e *opaqueLeaf) Format(s fmt.State, verb rune) { FormatError(e, s, verb) } func (e *opaqueWrapper) Format(s fmt.State, verb rune) { FormatError(e, s, verb) } -func (e *opaqueLeaf) FormatError(p Printer) (next error) { +func (e *opaqueLeaf) SafeFormatError(p Printer) (next error) { p.Print(e.msg) if p.Detail() { - p.Print("\n(opaque error leaf)") - p.Printf("\ntype name: %s", e.details.OriginalTypeName) + p.Printf("\n(opaque error leaf)") + p.Printf("\ntype name: %s", redact.Safe(e.details.OriginalTypeName)) for i, d := range e.details.ReportablePayload { - p.Printf("\nreportable %d:\n%s", i, d) + p.Printf("\nreportable %d:\n%s", redact.Safe(i), redact.Safe(d)) } if e.details.FullDetails != nil { - p.Printf("\npayload type: %s", e.details.FullDetails.TypeUrl) + p.Printf("\npayload type: %s", redact.Safe(e.details.FullDetails.TypeUrl)) } } return nil } -func (e *opaqueWrapper) FormatError(p Printer) (next error) { - p.Print(e.prefix) +func (e *opaqueWrapper) SafeFormatError(p Printer) (next error) { + if len(e.prefix) > 0 { + // We use the condition if len(msg) > 0 because + // otherwise an empty string would cause a "redactable + // empty string" to be emitted (something that looks like "<>") + // and the error formatting code only cleanly elides + // the prefix properly if the output string is completely empty. + p.Print(e.prefix) + } if p.Detail() { - p.Print("\n(opaque error wrapper)") - p.Printf("\ntype name: %s", e.details.OriginalTypeName) + p.Printf("\n(opaque error wrapper)") + p.Printf("\ntype name: %s", redact.Safe(e.details.OriginalTypeName)) for i, d := range e.details.ReportablePayload { - p.Printf("\nreportable %d:\n%s", i, d) + p.Printf("\nreportable %d:\n%s", redact.Safe(i), redact.Safe(d)) } if e.details.FullDetails != nil { - p.Printf("\npayload type: %s", e.details.FullDetails.TypeUrl) + p.Printf("\npayload type: %s", redact.Safe(e.details.FullDetails.TypeUrl)) } } return e.cause diff --git a/github.com/cockroachdb/errors/errbase/safe_details.go b/github.com/cockroachdb/errors/errbase/safe_details.go index 8eebdb30f7..0f692c4e5f 100644 --- a/github.com/cockroachdb/errors/errbase/safe_details.go +++ b/github.com/cockroachdb/errors/errbase/safe_details.go @@ -81,8 +81,13 @@ type SafeDetailPayload struct { // Fill can be used to concatenate multiple SafeDetailPayloads. func (s *SafeDetailPayload) Fill(slice []string) []string { - slice = append(slice, fmt.Sprintf("-- details for %s::%s:", + if len(s.SafeDetails) == 0 { + return slice + } + slice = append(slice, fmt.Sprintf("details for %s::%s:", s.ErrorTypeMark.FamilyName, s.ErrorTypeMark.Extension)) - slice = append(slice, s.SafeDetails...) + for _, sd := range s.SafeDetails { + slice = append(slice, " "+sd) + } return slice } diff --git a/github.com/cockroachdb/errors/errbase_api.go b/github.com/cockroachdb/errors/errbase_api.go index f1569dc21b..31d3ef147a 100644 --- a/github.com/cockroachdb/errors/errbase_api.go +++ b/github.com/cockroachdb/errors/errbase_api.go @@ -16,6 +16,7 @@ package errors import ( "context" + "fmt" "github.com/cockroachdb/errors/errbase" ) @@ -87,3 +88,23 @@ type WrapperEncoder = errbase.WrapperEncoder // SetWarningFn forwards a definition. func SetWarningFn(fn func(context.Context, string, ...interface{})) { errbase.SetWarningFn(fn) } + +// Formatter is provided for compatibility with xerrors. +// This should probably not be used directly, and +// SafeFormatter preferred instead. +type Formatter = errbase.Formatter + +// SafeFormatter is like Formatter but supports the separation +// of safe and unsafe strings. +type SafeFormatter = errbase.SafeFormatter + +// Printer is provided for compatibility with xerrors. +type Printer = errbase.Printer + +// FormatError can be used to implement the fmt.Formatter interface. +func FormatError(err error, s fmt.State, verb rune) { errbase.FormatError(err, s, verb) } + +// Formattable can be used to print an error with enhanced detail +// printout when the outer layer of wrapping may not be provided by +// this library. +func Formattable(err error) fmt.Formatter { return errbase.Formattable(err) } diff --git a/github.com/cockroachdb/errors/errutil/format_error_special.go b/github.com/cockroachdb/errors/errutil/format_error_special.go new file mode 100644 index 0000000000..fa7548859c --- /dev/null +++ b/github.com/cockroachdb/errors/errutil/format_error_special.go @@ -0,0 +1,88 @@ +// Copyright 2020 The Cockroach Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package errutil + +import ( + "context" + "net" + "os" + "runtime" + "syscall" + + "github.com/cockroachdb/errors/errbase" + "github.com/cockroachdb/errors/markers" + "github.com/cockroachdb/redact" +) + +func init() { + errbase.RegisterSpecialCasePrinter(specialCaseFormat) +} + +func specialCaseFormat(err error, p errbase.Printer, isLeaf bool) (handled bool, next error) { + if isLeaf && markers.IsAny(err, + context.DeadlineExceeded, + context.Canceled, + os.ErrInvalid, + os.ErrPermission, + os.ErrExist, + os.ErrNotExist, + os.ErrClosed, + os.ErrNoDeadline) { + p.Print(redact.Safe(err.Error())) + return true, nil + } + + switch v := err.(type) { + // The following two types are safe too. + case runtime.Error: + p.Print(redact.Safe(v.Error())) + return true, err + case syscall.Errno: + p.Print(redact.Safe(v.Error())) + return true, err + case *os.SyscallError: + p.Print(redact.Safe(v.Syscall)) + return true, err + case *os.PathError: + p.Printf("%s %s", redact.Safe(v.Op), v.Path) + return true, err + case *os.LinkError: + p.Printf("%s %s %s", redact.Safe(v.Op), v.Old, v.New) + return true, err + case *net.OpError: + p.Print(redact.Safe(v.Op)) + if v.Net != "" { + p.Printf(" %s", redact.Safe(v.Net)) + } + if v.Source != nil { + p.Printf(" %s", v.Source) + } + if v.Addr != nil { + if v.Source != nil { + p.Printf(" ->") + } + p.Printf(" %s", v.Addr) + } + return true, err + case redact.SafeMessager: + // Backward-compatibility with previous versions + // of the errors library: if an error type implements + // SafeMessage(), use that instead of its error message. + p.Print(redact.Safe(v.SafeMessage())) + // It also short-cuts any further causes. + return true, nil + } + return false, nil +} diff --git a/github.com/cockroachdb/errors/errutil/message.go b/github.com/cockroachdb/errors/errutil/message.go index 4d2599b9b7..30423e2a5c 100644 --- a/github.com/cockroachdb/errors/errutil/message.go +++ b/github.com/cockroachdb/errors/errutil/message.go @@ -14,13 +14,7 @@ package errutil -import ( - "context" - "fmt" - - "github.com/cockroachdb/errors/errbase" - "github.com/gogo/protobuf/proto" -) +import "github.com/cockroachdb/redact" // WithMessage annotates err with a new message. // If err is nil, WithMessage returns nil. @@ -28,9 +22,9 @@ func WithMessage(err error, message string) error { if err == nil { return nil } - return &withMessage{ - cause: err, - msg: message, + return &withPrefix{ + cause: err, + prefix: redact.Sprint(redact.Safe(message)), } } @@ -40,35 +34,8 @@ func WithMessagef(err error, format string, args ...interface{}) error { if err == nil { return nil } - return &withMessage{ - cause: err, - msg: fmt.Sprintf(format, args...), + return &withPrefix{ + cause: err, + prefix: redact.Sprintf(format, args...), } } - -type withMessage struct { - cause error - msg string -} - -func (w *withMessage) Error() string { return fmt.Sprintf("%s: %v", w.msg, w.cause) } -func (w *withMessage) Cause() error { return w.cause } -func (w *withMessage) Unwrap() error { return w.cause } - -func (w *withMessage) Format(s fmt.State, verb rune) { errbase.FormatError(w, s, verb) } -func (w *withMessage) FormatError(p errbase.Printer) (next error) { - p.Print(w.msg) - return w.cause -} - -// A message wrapper is decoded exactly. -func decodeMessage( - ctx context.Context, cause error, msg string, _ []string, _ proto.Message, -) error { - return &withMessage{cause: cause, msg: msg} -} - -func init() { - tn := errbase.GetTypeKey((*withMessage)(nil)) - errbase.RegisterWrapperDecoder(tn, decodeMessage) -} diff --git a/github.com/cockroachdb/errors/errutil/redactable.go b/github.com/cockroachdb/errors/errutil/redactable.go new file mode 100644 index 0000000000..a61523ac75 --- /dev/null +++ b/github.com/cockroachdb/errors/errutil/redactable.go @@ -0,0 +1,110 @@ +package errutil + +import ( + "context" + "fmt" + + "github.com/cockroachdb/errors/errbase" + "github.com/cockroachdb/errors/errorspb" + "github.com/cockroachdb/redact" + "github.com/gogo/protobuf/proto" +) + +// leafError is like the basic error string in the stdlib except the +// message can contain redactable and non-redactable parts. +type leafError struct { + msg redact.RedactableString +} + +var _ error = (*leafError)(nil) +var _ fmt.Formatter = (*leafError)(nil) +var _ errbase.SafeFormatter = (*leafError)(nil) +var _ errbase.SafeDetailer = (*leafError)(nil) + +func (l *leafError) Error() string { return l.msg.StripMarkers() } +func (l *leafError) Format(s fmt.State, verb rune) { errbase.FormatError(l, s, verb) } +func (l *leafError) SafeFormatError(p errbase.Printer) (next error) { + p.Print(l.msg) + return nil +} +func (l *leafError) SafeDetails() []string { + return []string{l.msg.Redact().StripMarkers()} +} + +func encodeLeaf(_ context.Context, err error) (string, []string, proto.Message) { + l := err.(*leafError) + return l.Error(), l.SafeDetails(), &errorspb.StringPayload{Msg: string(l.msg)} +} + +func decodeLeaf(_ context.Context, _ string, _ []string, payload proto.Message) error { + m, ok := payload.(*errorspb.StringPayload) + if !ok { + // If this ever happens, this means some version of the library + // (presumably future) changed the payload type, and we're + // receiving this here. In this case, give up and let + // DecodeError use the opaque type. + return nil + } + return &leafError{msg: redact.RedactableString(m.Msg)} +} + +func init() { + errbase.RegisterLeafEncoder(errbase.GetTypeKey((*leafError)(nil)), encodeLeaf) + errbase.RegisterLeafDecoder(errbase.GetTypeKey((*leafError)(nil)), decodeLeaf) +} + +// withPrefix is like withMessage but the +// message can contain redactable and non-redactable parts. +type withPrefix struct { + cause error + prefix redact.RedactableString +} + +var _ error = (*withPrefix)(nil) +var _ fmt.Formatter = (*withPrefix)(nil) +var _ errbase.SafeFormatter = (*withPrefix)(nil) +var _ errbase.SafeDetailer = (*withPrefix)(nil) + +func (l *withPrefix) Error() string { + if l.prefix == "" { + return l.cause.Error() + } + return fmt.Sprintf("%s: %v", l.prefix.StripMarkers(), l.cause) +} + +func (l *withPrefix) Cause() error { return l.cause } +func (l *withPrefix) Unwrap() error { return l.cause } + +func (l *withPrefix) Format(s fmt.State, verb rune) { errbase.FormatError(l, s, verb) } +func (l *withPrefix) SafeFormatError(p errbase.Printer) (next error) { + p.Print(l.prefix) + return l.cause +} + +func (l *withPrefix) SafeDetails() []string { + return []string{l.prefix.Redact().StripMarkers()} +} + +func encodeWithPrefix(_ context.Context, err error) (string, []string, proto.Message) { + l := err.(*withPrefix) + return l.Error(), l.SafeDetails(), &errorspb.StringPayload{Msg: string(l.prefix)} +} + +func decodeWithPrefix( + _ context.Context, cause error, _ string, _ []string, payload proto.Message, +) error { + m, ok := payload.(*errorspb.StringPayload) + if !ok { + // If this ever happens, this means some version of the library + // (presumably future) changed the payload type, and we're + // receiving this here. In this case, give up and let + // DecodeError use the opaque type. + return nil + } + return &withPrefix{cause: cause, prefix: redact.RedactableString(m.Msg)} +} + +func init() { + errbase.RegisterWrapperEncoder(errbase.GetTypeKey((*withPrefix)(nil)), encodeWithPrefix) + errbase.RegisterWrapperDecoder(errbase.GetTypeKey((*withPrefix)(nil)), decodeWithPrefix) +} diff --git a/github.com/cockroachdb/errors/errutil/utilities.go b/github.com/cockroachdb/errors/errutil/utilities.go index f6be48357d..7f60b05716 100644 --- a/github.com/cockroachdb/errors/errutil/utilities.go +++ b/github.com/cockroachdb/errors/errutil/utilities.go @@ -15,11 +15,13 @@ package errutil import ( - goErr "errors" "fmt" + "regexp" "github.com/cockroachdb/errors/safedetails" + "github.com/cockroachdb/errors/secondary" "github.com/cockroachdb/errors/withstack" + "github.com/cockroachdb/redact" ) // New creates an error with a simple error message. @@ -43,10 +45,7 @@ func New(msg string) error { // trace is configurable. // See the doc of `New()` for more details. func NewWithDepth(depth int, msg string) error { - err := goErr.New(msg) - if len(msg) > 0 { - err = safedetails.WithSafeDetails(err, msg) - } + err := error(&leafError{redact.Sprint(redact.Safe(msg))}) err = withstack.WithStackDepth(err, 1+depth) return err } @@ -68,14 +67,60 @@ func Newf(format string, args ...interface{}) error { // trace is configurable. // See the doc of `New()` for more details. func NewWithDepthf(depth int, format string, args ...interface{}) error { - err := fmt.Errorf(format, args...) - if format != "" || len(args) > 0 { + // If there's the verb %w in here, shortcut to fmt.Errorf() + // and store the safe details as extra payload. That's + // because we don't want to re-implement the error wrapping + // logic from 'fmt' in there. + var err error + var errRefs []error + for _, a := range args { + if e, ok := a.(error); ok { + errRefs = append(errRefs, e) + } + } + if hasVerbW.MatchString(format) { + err = fmt.Errorf(format, args...) err = safedetails.WithSafeDetails(err, format, args...) + } else { + err = error(&leafError{redact.Sprintf(format, args...)}) + } + for _, e := range errRefs { + err = secondary.WithSecondaryError(err, e) } err = withstack.WithStackDepth(err, 1+depth) return err } +var hasVerbW = regexp.MustCompile( + `^` + + // Ignore any non-%w format directives and other characters: + `([^%]+|%` + + // A format directive starts with some flags. + `[-#0+ ]*` + + // Followed by an optional argument number [n]. + `(\[[0-9]+\])?` + + // Followed by an optional width. + `(\*|[0-9]+)?` + + // followed by an optional precision. + `(\.(\*|[0-9]))?` + + // We're only interested in non-w verbs here. + `[^w]` + + // zero or more times. + `)*` + + // Followed with at least one occurrence of a format directive + // with the 'w' verb. We replicate the rules above. + `%` + + // A format directive starts with some flags. + `[-#0+ ]*` + + // Followed by an optional argument number [n]. + `(\[[0-9]+\])?` + + // Followed by an optional width. + `(\*|[0-9]+)?` + + // followed by an optional precision. + `(\.(\*|[0-9]))?` + + // Just the verb 'w'. + `w`) + // Wrap wraps an error with a message prefix. // A stack trace is retained. // @@ -97,9 +142,11 @@ func Wrap(err error, msg string) error { // trace is configurable. // The the doc of `Wrap()` for more details. func WrapWithDepth(depth int, err error, msg string) error { + if err == nil { + return nil + } if msg != "" { err = WithMessage(err, msg) - err = safedetails.WithSafeDetails(err, msg) } err = withstack.WithStackDepth(err, depth+1) return err @@ -127,9 +174,20 @@ func Wrapf(err error, format string, args ...interface{}) error { // trace is configurable. // The the doc of `Wrapf()` for more details. func WrapWithDepthf(depth int, err error, format string, args ...interface{}) error { + if err == nil { + return nil + } + var errRefs []error + for _, a := range args { + if e, ok := a.(error); ok { + errRefs = append(errRefs, e) + } + } if format != "" || len(args) > 0 { err = WithMessagef(err, format, args...) - err = safedetails.WithSafeDetails(err, format, args...) + } + for _, e := range errRefs { + err = secondary.WithSecondaryError(err, e) } err = withstack.WithStackDepth(err, depth+1) return err diff --git a/github.com/cockroachdb/errors/errutil_api.go b/github.com/cockroachdb/errors/errutil_api.go index 9deae51416..c4086228bd 100644 --- a/github.com/cockroachdb/errors/errutil_api.go +++ b/github.com/cockroachdb/errors/errutil_api.go @@ -15,8 +15,6 @@ package errors import ( - "fmt" - "github.com/cockroachdb/errors/barriers" "github.com/cockroachdb/errors/errbase" "github.com/cockroachdb/errors/errutil" @@ -52,15 +50,6 @@ type Wrapper interface { Unwrap() error } -// Formatter is provided for compatibility with xerrors. -type Formatter = errbase.Formatter - -// Printer is provided for compatibility with xerrors. -type Printer = errbase.Printer - -// FormatError can be used to implement the fmt.Formatter interface. -func FormatError(err error, s fmt.State, verb rune) { errbase.FormatError(err, s, verb) } - // Opaque is provided for compatibility with xerrors. func Opaque(err error) error { return barriers.Handled(err) } diff --git a/github.com/cockroachdb/errors/go.mod b/github.com/cockroachdb/errors/go.mod index 1eedc27ecd..02b5cbdcb6 100644 --- a/github.com/cockroachdb/errors/go.mod +++ b/github.com/cockroachdb/errors/go.mod @@ -3,13 +3,15 @@ module github.com/cockroachdb/errors go 1.13 require ( + github.com/cockroachdb/datadriven v1.0.0 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f + github.com/cockroachdb/redact v1.0.2 github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 github.com/gogo/protobuf v1.3.1 github.com/gogo/status v1.1.0 github.com/golang/protobuf v1.4.2 github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb github.com/kr/pretty v0.1.0 - github.com/pkg/errors v0.8.1 + github.com/pkg/errors v0.9.1 google.golang.org/grpc v1.29.1 ) diff --git a/github.com/cockroachdb/errors/go.sum b/github.com/cockroachdb/errors/go.sum index 556824e7b6..d4a6d98dc0 100644 --- a/github.com/cockroachdb/errors/go.sum +++ b/github.com/cockroachdb/errors/go.sum @@ -12,8 +12,13 @@ github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cockroachdb/datadriven v1.0.0 h1:uhZrAfEayBecH2w2tZmhe20HJ7hDvrrA4x2Bg9YdZKM= +github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= +github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/cockroachdb/redact v1.0.2 h1:bRktqHBXPqI+9bkOx0ikn9RS09G9k83oAdLx6rXRVTQ= +github.com/cockroachdb/redact v1.0.2/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= @@ -141,6 +146,9 @@ github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4 github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= diff --git a/github.com/cockroachdb/errors/issuelink/unimplemented_error.go b/github.com/cockroachdb/errors/issuelink/unimplemented_error.go index 2611b87978..af64df12a4 100644 --- a/github.com/cockroachdb/errors/issuelink/unimplemented_error.go +++ b/github.com/cockroachdb/errors/issuelink/unimplemented_error.go @@ -20,17 +20,19 @@ import ( "fmt" "github.com/cockroachdb/errors/errbase" + "github.com/cockroachdb/redact" "github.com/gogo/protobuf/proto" ) type unimplementedError struct { + // For now, msg is non-reportable. msg string IssueLink } var _ error = (*unimplementedError)(nil) var _ fmt.Formatter = (*unimplementedError)(nil) -var _ errbase.Formatter = (*unimplementedError)(nil) +var _ errbase.SafeFormatter = (*unimplementedError)(nil) var _ errbase.SafeDetailer = (*unimplementedError)(nil) func (w *unimplementedError) Error() string { return w.msg } @@ -51,15 +53,17 @@ const UnimplementedErrorHint = `You have attempted to use a feature that is not func (w *unimplementedError) Format(s fmt.State, verb rune) { errbase.FormatError(w, s, verb) } -func (w *unimplementedError) FormatError(p errbase.Printer) error { +func (w *unimplementedError) SafeFormatError(p errbase.Printer) error { + // For now, msg is non-reportable. p.Print(w.msg) if p.Detail() { - p.Print("unimplemented") + // But the details are. + p.Printf("unimplemented") if w.IssueURL != "" { - p.Printf("\nissue: %s", w.IssueURL) + p.Printf("\nissue: %s", redact.Safe(w.IssueURL)) } if w.Detail != "" { - p.Printf("\ndetail: %s", w.Detail) + p.Printf("\ndetail: %s", redact.Safe(w.Detail)) } } return nil diff --git a/github.com/cockroachdb/errors/issuelink/with_issuelink.go b/github.com/cockroachdb/errors/issuelink/with_issuelink.go index 83c5d56543..422333820b 100644 --- a/github.com/cockroachdb/errors/issuelink/with_issuelink.go +++ b/github.com/cockroachdb/errors/issuelink/with_issuelink.go @@ -21,6 +21,7 @@ import ( "github.com/cockroachdb/errors/errbase" "github.com/cockroachdb/errors/stdstrings" + "github.com/cockroachdb/redact" "github.com/gogo/protobuf/proto" ) @@ -32,7 +33,7 @@ type withIssueLink struct { var _ error = (*withIssueLink)(nil) var _ errbase.SafeDetailer = (*withIssueLink)(nil) var _ fmt.Formatter = (*withIssueLink)(nil) -var _ errbase.Formatter = (*withIssueLink)(nil) +var _ errbase.SafeFormatter = (*withIssueLink)(nil) func (w *withIssueLink) Error() string { return w.cause.Error() } func (w *withIssueLink) Cause() error { return w.cause } @@ -64,15 +65,15 @@ func maybeAppendReferral(buf *bytes.Buffer, link IssueLink) { func (w *withIssueLink) Format(s fmt.State, verb rune) { errbase.FormatError(w, s, verb) } -func (w *withIssueLink) FormatError(p errbase.Printer) error { +func (w *withIssueLink) SafeFormatError(p errbase.Printer) error { if p.Detail() { - sep := "" + sep := redact.SafeString("") if w.IssueURL != "" { - p.Printf("issue: %s", w.IssueURL) + p.Printf("issue: %s", redact.Safe(w.IssueURL)) sep = "\n" } if w.Detail != "" { - p.Printf("%sdetail: %s", sep, w.Detail) + p.Printf("%sdetail: %s", sep, redact.Safe(w.Detail)) } } return w.cause diff --git a/github.com/cockroachdb/errors/markers/markers.go b/github.com/cockroachdb/errors/markers/markers.go index 75139b4b13..c3b4bd3c21 100644 --- a/github.com/cockroachdb/errors/markers/markers.go +++ b/github.com/cockroachdb/errors/markers/markers.go @@ -21,6 +21,7 @@ import ( "github.com/cockroachdb/errors/errbase" "github.com/cockroachdb/errors/errorspb" + "github.com/cockroachdb/redact" "github.com/gogo/protobuf/proto" ) @@ -225,7 +226,7 @@ type withMark struct { var _ error = (*withMark)(nil) var _ fmt.Formatter = (*withMark)(nil) -var _ errbase.Formatter = (*withMark)(nil) +var _ errbase.SafeFormatter = (*withMark)(nil) func (m *withMark) Error() string { return m.cause.Error() } func (m *withMark) Cause() error { return m.cause } @@ -233,13 +234,13 @@ func (m *withMark) Unwrap() error { return m.cause } func (m *withMark) Format(s fmt.State, verb rune) { errbase.FormatError(m, s, verb) } -func (m *withMark) FormatError(p errbase.Printer) error { +func (m *withMark) SafeFormatError(p errbase.Printer) error { if p.Detail() { - p.Print("forced error mark\n") + p.Printf("forced error mark\n") p.Printf("%q\n%s::%s", m.mark.msg, - m.mark.types[0].FamilyName, - m.mark.types[0].Extension, + redact.Safe(m.mark.types[0].FamilyName), + redact.Safe(m.mark.types[0].Extension), ) } return m.cause diff --git a/github.com/cockroachdb/errors/report/report.go b/github.com/cockroachdb/errors/report/report.go index 9a5d984ba0..c30e598ced 100644 --- a/github.com/cockroachdb/errors/report/report.go +++ b/github.com/cockroachdb/errors/report/report.go @@ -22,6 +22,7 @@ import ( "github.com/cockroachdb/errors/domains" "github.com/cockroachdb/errors/errbase" "github.com/cockroachdb/errors/withstack" + "github.com/cockroachdb/redact" "github.com/cockroachdb/sentry-go" ) @@ -111,13 +112,24 @@ func BuildSentryReport(err error) (event *sentry.Event, extraDetails map[string] } module := string(domains.GetDomain(err)) - // For the summary, we collect the innermost source context. - // file, line, fn, hasOneLineSource := withstack.GetOneLineSource(err) + // firstDetailLine is the first detail string encountered. + // This is added as decoration to the first Exception + // payload (either from the error object or synthetic) + // so as to populate the Sentry report title. + var firstDetailLine string // longMsgBuf will become the Message field, which contains the full // structure of the error with cross-references to "Exception" and // "Additional data" fields. var longMsgBuf strings.Builder + redactedErrStr := redact.Sprint(err).Redact() + if redactedErrStr != redactedMarker { + if f, l, _, ok := withstack.GetOneLineSource(err); ok { + fmt.Fprintf(&longMsgBuf, "%s:%d: ", f, l) + } + firstDetailLine = redactedErrStr.StripMarkers() + fmt.Fprintf(&longMsgBuf, "%v\n--\n", firstDetailLine) + } // sep is used to separate the entries in the longMsgBuf / Message // payload. @@ -143,12 +155,6 @@ func BuildSentryReport(err error) (event *sentry.Event, extraDetails map[string] // This is used as fallback when no Exception payload is generated. var leafErrorType string - // firstDetailLine is the first detail string encountered. - // This is added as decoration to the first Exception - // payload (either from the error object or synthetic) - // so as to populate the Sentry report title. - var firstDetailLine string - // Iterate from the last (innermost) to first (outermost) error // layer. We iterate in this order because we want to describe the // error from innermost to outermost layer in longMsgBuf and @@ -348,6 +354,8 @@ func BuildSentryReport(err error) (event *sentry.Event, extraDetails map[string] return event, extras } +var redactedMarker = redact.RedactableString(redact.RedactedMarker()) + // ReportError reports the given error to Sentry. The caller is responsible for // checking whether telemetry is enabled. // Note: an empty 'eventID' can be returned which signifies that the error was diff --git a/github.com/cockroachdb/errors/safedetails/redact.go b/github.com/cockroachdb/errors/safedetails/redact.go index 92507a6761..d9ce8b6b7d 100644 --- a/github.com/cockroachdb/errors/safedetails/redact.go +++ b/github.com/cockroachdb/errors/safedetails/redact.go @@ -14,160 +14,12 @@ package safedetails -import ( - "context" - "fmt" - "net" - "os" - "runtime" - "strings" - "syscall" - - "github.com/cockroachdb/errors/errbase" - "github.com/cockroachdb/errors/markers" - "github.com/cockroachdb/errors/withstack" -) +import "github.com/cockroachdb/redact" // Redact returns a redacted version of the supplied item that is safe to use in // anonymized reporting. +// +// NB: this interface is obsolete. Use redact.Sprint() directly. func Redact(r interface{}) string { - var buf strings.Builder - - switch t := r.(type) { - case SafeMessager: - buf.WriteString(t.SafeMessage()) - case error: - if file, line, _, ok := withstack.GetOneLineSource(t); ok { - fmt.Fprintf(&buf, "%s:%d: ", file, line) - } - redactErr(&buf, t) - default: - typRedacted(&buf, r) - } - - return buf.String() -} - -func redactErr(buf *strings.Builder, err error) { - foundDetail := false - if c := errbase.UnwrapOnce(err); c == nil { - // This is a leaf error. Decode the leaf and return. - foundDetail = redactLeafErr(buf, err) - } else /* c != nil */ { - // Print the inner error before the outer error. - redactErr(buf, c) - foundDetail = redactWrapper(buf, err) - } - - // Add any additional safe strings from the wrapper, if present. - if payload := errbase.GetSafeDetails(err); len(payload.SafeDetails) > 0 { - consumed := 0 - if !foundDetail { - firstDetail := strings.TrimSpace(payload.SafeDetails[0]) - if strings.IndexByte(firstDetail, '\n') < 0 { - firstDetail = strings.ReplaceAll(strings.TrimSpace(payload.SafeDetails[0]), "\n", "\n ") - if len(firstDetail) > 0 { - buf.WriteString(": ") - } - buf.WriteString(firstDetail) - consumed = 1 - foundDetail = true - } - } - if len(payload.SafeDetails) > consumed { - buf.WriteString("\n (more details:)") - for _, sd := range payload.SafeDetails[consumed:] { - buf.WriteString("\n ") - buf.WriteString(strings.ReplaceAll(strings.TrimSpace(sd), "\n", "\n ")) - } - foundDetail = true - } - } - if !foundDetail { - buf.WriteString(": ") - } -} - -func redactWrapper(buf *strings.Builder, err error) (hasDetail bool) { - buf.WriteString("\n") - switch t := err.(type) { - case *os.SyscallError: - hasDetail = true - typAnd(buf, t, t.Syscall) - case *os.PathError: - hasDetail = true - typAnd(buf, t, t.Op) - case *os.LinkError: - hasDetail = true - fmt.Fprintf(buf, "%T: %s ", t, t.Op) - case *net.OpError: - hasDetail = true - typAnd(buf, t, t.Op) - if t.Net != "" { - fmt.Fprintf(buf, " %s", t.Net) - } - if t.Source != nil { - buf.WriteString(" ") - } - if t.Addr != nil { - if t.Source != nil { - buf.WriteString(" ->") - } - buf.WriteString(" ") - } - default: - fmt.Fprintf(buf, "%T", err) - } - return -} - -func redactLeafErr(buf *strings.Builder, err error) (hasDetail bool) { - // Is it a sentinel error? These are safe. - if markers.IsAny(err, - context.DeadlineExceeded, - context.Canceled, - os.ErrInvalid, - os.ErrPermission, - os.ErrExist, - os.ErrNotExist, - os.ErrClosed, - os.ErrNoDeadline, - ) { - hasDetail = true - typAnd(buf, err, err.Error()) - return - } - - if redactPre113Wrappers(buf, err) { - hasDetail = true - return - } - - // The following two types are safe too. - switch t := err.(type) { - case runtime.Error: - hasDetail = true - typAnd(buf, t, t.Error()) - case syscall.Errno: - hasDetail = true - typAnd(buf, t, t.Error()) - case SafeMessager: - hasDetail = true - typAnd(buf, t, t.SafeMessage()) - default: - // No further information about this error, simply report its type. - fmt.Fprintf(buf, "%T", err) - } - return -} - -func typRedacted(buf *strings.Builder, r interface{}) { - fmt.Fprintf(buf, "%T:", r) -} - -func typAnd(buf *strings.Builder, r interface{}, msg string) { - if len(msg) == 0 { - msg = "(empty string)" - } - fmt.Fprintf(buf, "%T: %s", r, msg) + return redact.Sprint(r).Redact().StripMarkers() } diff --git a/github.com/cockroachdb/errors/safedetails/redact_go112.go b/github.com/cockroachdb/errors/safedetails/redact_go112.go deleted file mode 100644 index fca1e24cb5..0000000000 --- a/github.com/cockroachdb/errors/safedetails/redact_go112.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2019 The Cockroach Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. See the License for the specific language governing -// permissions and limitations under the License. - -// +build !go1.13 - -package safedetails - -import ( - "net" - "os" - "strings" -) - -func redactPre113Wrappers(buf *strings.Builder, err error) bool { - // The following cases are needed for go 1.12 and previous - // versions. Go 1.13 instances of these errors will be recognized - // by the unwrapping performed in redactErr(). - switch t := err.(type) { - case *os.SyscallError: - redactErr(buf, t.Err) - redactWrapper(buf, err) - return true - case *os.PathError: - redactErr(buf, t.Err) - redactWrapper(buf, err) - return true - case *net.OpError: - redactErr(buf, t.Err) - redactWrapper(buf, err) - return true - case *os.LinkError: - redactErr(buf, t.Err) - redactWrapper(buf, err) - return true - } - return false -} diff --git a/github.com/cockroachdb/errors/safedetails/redact_go113.go b/github.com/cockroachdb/errors/safedetails/redact_go113.go deleted file mode 100644 index 5c4cd5ef10..0000000000 --- a/github.com/cockroachdb/errors/safedetails/redact_go113.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2019 The Cockroach Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. See the License for the specific language governing -// permissions and limitations under the License. - -// +build go1.13 - -package safedetails - -import "strings" - -func redactPre113Wrappers(buf *strings.Builder, err error) bool { - return false -} diff --git a/github.com/cockroachdb/errors/safedetails/safedetails.go b/github.com/cockroachdb/errors/safedetails/safedetails.go index 32ab8f8ac7..bf60350df8 100644 --- a/github.com/cockroachdb/errors/safedetails/safedetails.go +++ b/github.com/cockroachdb/errors/safedetails/safedetails.go @@ -14,7 +14,11 @@ package safedetails -import "fmt" +import ( + "reflect" + + "github.com/cockroachdb/redact" +) // WithSafeDetails annotates an error with the given reportable details. // The format is made available as a PII-free string, alongside @@ -22,6 +26,9 @@ import "fmt" // Arguments can be reported as-is (without redaction) by wrapping // them using the Safe() function. // +// If the format is empty and there are no arguments, the +// error argument is returned unchanged. +// // Detail is shown: // - via `errors.GetSafeDetails()` // - when formatting with `%+v`. @@ -30,63 +37,28 @@ func WithSafeDetails(err error, format string, args ...interface{}) error { if err == nil { return nil } - - details := make([]string, 1, 1+len(args)) - details[0] = format - if len(format) > 0 && len(args) > 0 { - // Call out that this string was a formatting string. - details[0] = fmt.Sprintf("format: %q", details[0]) + if len(format) == 0 && len(args) == 0 { + return err } - for i, a := range args { - what := "" - if _, ok := a.(error); ok { - // Call out that this argument was an error, to facilitate - // interpretation of subsequent lines. - what = " (error)" - } - details = append(details, fmt.Sprintf("-- arg %d%s: %s", i+1, what, Redact(a))) + details := []string{ + redact.Sprintf(format, args...).Redact().StripMarkers(), } return &withSafeDetails{cause: err, safeDetails: details} } -// SafeMessager is implemented by objects which have a way of representing -// themselves suitably redacted for anonymized reporting. -type SafeMessager interface { - SafeMessage() string -} +var refSafeType = reflect.TypeOf(redact.Safe("")) + +// SafeMessager is implemented by objects which have a way of +// representing themselves suitably redacted for anonymized reporting. +// +// NB: this interface is obsolete. Use redact.SafeFormatter instead. +type SafeMessager = redact.SafeMessager // Safe wraps the given object into an opaque struct that implements // SafeMessager: its contents can be included as-is in PII-free // strings in error objects and reports. -func Safe(v interface{}) SafeMessager { - return safeType{V: v} -} - -// A safeType panic can be reported verbatim, i.e. does not leak -// information. A nil `*safeType` is not valid for use and may cause -// panics. -type safeType struct { - V interface{} -} - -var _ SafeMessager = safeType{} - -// SafeMessage implements SafeMessager. -func (st safeType) SafeMessage() string { - return fmt.Sprintf("%+v", st.V) -} - -// safeType implements fmt.Stringer as a convenience. -func (st safeType) String() string { - return st.SafeMessage() -} - -// Format implements fmt.Formatter. -func (st safeType) Format(s fmt.State, verb rune) { - flags := "" - if s.Flag('+') { - flags += "+" - } - fmtString := fmt.Sprintf("%%%s%c", flags, verb) - fmt.Fprintf(s, fmtString, st.V) +// +// NB: this is obsolete. Use redact.Safe instead. +func Safe(v interface{}) redact.SafeValue { + return redact.Safe(v) } diff --git a/github.com/cockroachdb/errors/safedetails/with_safedetails.go b/github.com/cockroachdb/errors/safedetails/with_safedetails.go index f710925f29..03ba7e5151 100644 --- a/github.com/cockroachdb/errors/safedetails/with_safedetails.go +++ b/github.com/cockroachdb/errors/safedetails/with_safedetails.go @@ -19,6 +19,7 @@ import ( "fmt" "github.com/cockroachdb/errors/errbase" + "github.com/cockroachdb/redact" "github.com/gogo/protobuf/proto" ) @@ -33,17 +34,29 @@ func (e *withSafeDetails) SafeDetails() []string { } var _ fmt.Formatter = (*withSafeDetails)(nil) +var _ errbase.SafeFormatter = (*withSafeDetails)(nil) // Printing a withSecondary reveals the details. func (e *withSafeDetails) Format(s fmt.State, verb rune) { errbase.FormatError(e, s, verb) } -func (e *withSafeDetails) FormatError(p errbase.Printer) error { +// SafeFormatError implements errbase.SafeFormatter. +func (e *withSafeDetails) SafeFormatError(p errbase.Printer) error { if p.Detail() { - plural := "s" - if len(e.safeDetails) == 1 { - plural = "" + comma := redact.SafeString("") + if len(e.safeDetails) != 1 { + plural := redact.SafeString("s") + if len(e.safeDetails) == 1 { + plural = "" + } + p.Printf("%d safe detail%s enclosed", redact.Safe(len(e.safeDetails)), plural) + comma = "\n" + } + // We hide the details from %+v; they are included + // during Sentry reporting. + for _, s := range e.safeDetails { + p.Printf("%s%s", comma, redact.Safe(s)) + comma = "\n" } - p.Printf("%d safe detail%s enclosed", len(e.safeDetails), plural) } return e.cause } diff --git a/github.com/cockroachdb/errors/safedetails_api.go b/github.com/cockroachdb/errors/safedetails_api.go index e4c0ee1584..e45361ee52 100644 --- a/github.com/cockroachdb/errors/safedetails_api.go +++ b/github.com/cockroachdb/errors/safedetails_api.go @@ -14,7 +14,10 @@ package errors -import "github.com/cockroachdb/errors/safedetails" +import ( + "github.com/cockroachdb/errors/safedetails" + "github.com/cockroachdb/redact" +) // WithSafeDetails forwards a definition. func WithSafeDetails(err error, format string, args ...interface{}) error { @@ -22,10 +25,12 @@ func WithSafeDetails(err error, format string, args ...interface{}) error { } // SafeMessager forwards a definition. -type SafeMessager = safedetails.SafeMessager +// NB: this is obsolete. Use redact.SafeFormatter +// or errors.SafeFormatter instead. +type SafeMessager = redact.SafeMessager // Safe forwards a definition. -func Safe(v interface{}) SafeMessager { return safedetails.Safe(v) } +func Safe(v interface{}) redact.SafeValue { return safedetails.Safe(v) } // Redact returns a redacted version of the supplied item that is safe // to use in anonymized reporting. diff --git a/github.com/cockroachdb/errors/secondary/with_secondary.go b/github.com/cockroachdb/errors/secondary/with_secondary.go index e2563cd8b8..5822b2c1cb 100644 --- a/github.com/cockroachdb/errors/secondary/with_secondary.go +++ b/github.com/cockroachdb/errors/secondary/with_secondary.go @@ -33,7 +33,7 @@ type withSecondaryError struct { var _ error = (*withSecondaryError)(nil) var _ errbase.SafeDetailer = (*withSecondaryError)(nil) var _ fmt.Formatter = (*withSecondaryError)(nil) -var _ errbase.Formatter = (*withSecondaryError)(nil) +var _ errbase.SafeFormatter = (*withSecondaryError)(nil) // SafeDetails reports the PII-free details from the secondary error. func (e *withSecondaryError) SafeDetails() []string { @@ -48,10 +48,9 @@ func (e *withSecondaryError) SafeDetails() []string { // Printing a withSecondary reveals the details. func (e *withSecondaryError) Format(s fmt.State, verb rune) { errbase.FormatError(e, s, verb) } -func (e *withSecondaryError) FormatError(p errbase.Printer) (next error) { +func (e *withSecondaryError) SafeFormatError(p errbase.Printer) (next error) { if p.Detail() { - p.Print("secondary error attachment\n") - p.Printf("%+v", e.secondaryError) + p.Printf("secondary error attachment\n%+v", e.secondaryError) } return e.cause } diff --git a/github.com/cockroachdb/errors/telemetrykeys/with_telemetry.go b/github.com/cockroachdb/errors/telemetrykeys/with_telemetry.go index 20f799fb68..2f7e5f180f 100644 --- a/github.com/cockroachdb/errors/telemetrykeys/with_telemetry.go +++ b/github.com/cockroachdb/errors/telemetrykeys/with_telemetry.go @@ -17,8 +17,10 @@ package telemetrykeys import ( "context" "fmt" + "strings" "github.com/cockroachdb/errors/errbase" + "github.com/cockroachdb/redact" "github.com/gogo/protobuf/proto" ) @@ -42,7 +44,7 @@ func (w *withTelemetry) Format(s fmt.State, verb rune) { errbase.FormatError(w, func (w *withTelemetry) FormatError(p errbase.Printer) (next error) { if p.Detail() { - p.Printf("keys: %+v", w.keys) + p.Printf("keys: [%s]", redact.Safe(strings.Join(w.keys, " "))) } return w.cause } diff --git a/github.com/cockroachdb/errors/withstack/withstack.go b/github.com/cockroachdb/errors/withstack/withstack.go index 402608999d..56f5d68486 100644 --- a/github.com/cockroachdb/errors/withstack/withstack.go +++ b/github.com/cockroachdb/errors/withstack/withstack.go @@ -58,7 +58,7 @@ type withStack struct { var _ error = (*withStack)(nil) var _ fmt.Formatter = (*withStack)(nil) -var _ errbase.Formatter = (*withStack)(nil) +var _ errbase.SafeFormatter = (*withStack)(nil) var _ errbase.SafeDetailer = (*withStack)(nil) func (w *withStack) Error() string { return w.cause.Error() } @@ -68,15 +68,17 @@ func (w *withStack) Unwrap() error { return w.cause } // Format implements the fmt.Formatter interface. func (w *withStack) Format(s fmt.State, verb rune) { errbase.FormatError(w, s, verb) } -func (w *withStack) FormatError(p errbase.Printer) error { +// SafeFormatError implements the errbase.SafeFormatter interface. +func (w *withStack) SafeFormatError(p errbase.Printer) error { if p.Detail() { - p.Print("attached stack trace") + p.Printf("attached stack trace") } // We do not print the stack trace ourselves - errbase.FormatError() // does this for us. return w.cause } +// SafeDetails implements the errbase.SafeDetailer interface. func (w *withStack) SafeDetails() []string { return []string{fmt.Sprintf("%+v", w.StackTrace())} } diff --git a/modules.txt b/modules.txt index bec1ab9317..e5a1093c78 100644 --- a/modules.txt +++ b/modules.txt @@ -167,7 +167,7 @@ github.com/cockroachdb/crlfmt/internal/parser github.com/cockroachdb/crlfmt/internal/render # github.com/cockroachdb/datadriven v1.0.0 github.com/cockroachdb/datadriven -# github.com/cockroachdb/errors v1.6.1 +# github.com/cockroachdb/errors v1.7.3 github.com/cockroachdb/errors github.com/cockroachdb/errors/assert github.com/cockroachdb/errors/barriers From 33476245342344f65832a29f31999c1357cc8f1a Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sun, 23 Aug 2020 10:37:46 -0400 Subject: [PATCH 29/49] Add golang.org/x/crypto/ocsp --- golang.org/x/crypto/ocsp/ocsp.go | 784 +++++++++++++++++++++++++++++++ modules.txt | 1 + 2 files changed, 785 insertions(+) create mode 100644 golang.org/x/crypto/ocsp/ocsp.go diff --git a/golang.org/x/crypto/ocsp/ocsp.go b/golang.org/x/crypto/ocsp/ocsp.go new file mode 100644 index 0000000000..d297ac92ea --- /dev/null +++ b/golang.org/x/crypto/ocsp/ocsp.go @@ -0,0 +1,784 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package ocsp parses OCSP responses as specified in RFC 2560. OCSP responses +// are signed messages attesting to the validity of a certificate for a small +// period of time. This is used to manage revocation for X.509 certificates. +package ocsp // import "golang.org/x/crypto/ocsp" + +import ( + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/rsa" + _ "crypto/sha1" + _ "crypto/sha256" + _ "crypto/sha512" + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" + "errors" + "fmt" + "math/big" + "strconv" + "time" +) + +var idPKIXOCSPBasic = asn1.ObjectIdentifier([]int{1, 3, 6, 1, 5, 5, 7, 48, 1, 1}) + +// ResponseStatus contains the result of an OCSP request. See +// https://tools.ietf.org/html/rfc6960#section-2.3 +type ResponseStatus int + +const ( + Success ResponseStatus = 0 + Malformed ResponseStatus = 1 + InternalError ResponseStatus = 2 + TryLater ResponseStatus = 3 + // Status code four is unused in OCSP. See + // https://tools.ietf.org/html/rfc6960#section-4.2.1 + SignatureRequired ResponseStatus = 5 + Unauthorized ResponseStatus = 6 +) + +func (r ResponseStatus) String() string { + switch r { + case Success: + return "success" + case Malformed: + return "malformed" + case InternalError: + return "internal error" + case TryLater: + return "try later" + case SignatureRequired: + return "signature required" + case Unauthorized: + return "unauthorized" + default: + return "unknown OCSP status: " + strconv.Itoa(int(r)) + } +} + +// ResponseError is an error that may be returned by ParseResponse to indicate +// that the response itself is an error, not just that it's indicating that a +// certificate is revoked, unknown, etc. +type ResponseError struct { + Status ResponseStatus +} + +func (r ResponseError) Error() string { + return "ocsp: error from server: " + r.Status.String() +} + +// These are internal structures that reflect the ASN.1 structure of an OCSP +// response. See RFC 2560, section 4.2. + +type certID struct { + HashAlgorithm pkix.AlgorithmIdentifier + NameHash []byte + IssuerKeyHash []byte + SerialNumber *big.Int +} + +// https://tools.ietf.org/html/rfc2560#section-4.1.1 +type ocspRequest struct { + TBSRequest tbsRequest +} + +type tbsRequest struct { + Version int `asn1:"explicit,tag:0,default:0,optional"` + RequestorName pkix.RDNSequence `asn1:"explicit,tag:1,optional"` + RequestList []request +} + +type request struct { + Cert certID +} + +type responseASN1 struct { + Status asn1.Enumerated + Response responseBytes `asn1:"explicit,tag:0,optional"` +} + +type responseBytes struct { + ResponseType asn1.ObjectIdentifier + Response []byte +} + +type basicResponse struct { + TBSResponseData responseData + SignatureAlgorithm pkix.AlgorithmIdentifier + Signature asn1.BitString + Certificates []asn1.RawValue `asn1:"explicit,tag:0,optional"` +} + +type responseData struct { + Raw asn1.RawContent + Version int `asn1:"optional,default:0,explicit,tag:0"` + RawResponderID asn1.RawValue + ProducedAt time.Time `asn1:"generalized"` + Responses []singleResponse +} + +type singleResponse struct { + CertID certID + Good asn1.Flag `asn1:"tag:0,optional"` + Revoked revokedInfo `asn1:"tag:1,optional"` + Unknown asn1.Flag `asn1:"tag:2,optional"` + ThisUpdate time.Time `asn1:"generalized"` + NextUpdate time.Time `asn1:"generalized,explicit,tag:0,optional"` + SingleExtensions []pkix.Extension `asn1:"explicit,tag:1,optional"` +} + +type revokedInfo struct { + RevocationTime time.Time `asn1:"generalized"` + Reason asn1.Enumerated `asn1:"explicit,tag:0,optional"` +} + +var ( + oidSignatureMD2WithRSA = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 2} + oidSignatureMD5WithRSA = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 4} + oidSignatureSHA1WithRSA = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 5} + oidSignatureSHA256WithRSA = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 11} + oidSignatureSHA384WithRSA = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 12} + oidSignatureSHA512WithRSA = asn1.ObjectIdentifier{1, 2, 840, 113549, 1, 1, 13} + oidSignatureDSAWithSHA1 = asn1.ObjectIdentifier{1, 2, 840, 10040, 4, 3} + oidSignatureDSAWithSHA256 = asn1.ObjectIdentifier{2, 16, 840, 1, 101, 3, 4, 3, 2} + oidSignatureECDSAWithSHA1 = asn1.ObjectIdentifier{1, 2, 840, 10045, 4, 1} + oidSignatureECDSAWithSHA256 = asn1.ObjectIdentifier{1, 2, 840, 10045, 4, 3, 2} + oidSignatureECDSAWithSHA384 = asn1.ObjectIdentifier{1, 2, 840, 10045, 4, 3, 3} + oidSignatureECDSAWithSHA512 = asn1.ObjectIdentifier{1, 2, 840, 10045, 4, 3, 4} +) + +var hashOIDs = map[crypto.Hash]asn1.ObjectIdentifier{ + crypto.SHA1: asn1.ObjectIdentifier([]int{1, 3, 14, 3, 2, 26}), + crypto.SHA256: asn1.ObjectIdentifier([]int{2, 16, 840, 1, 101, 3, 4, 2, 1}), + crypto.SHA384: asn1.ObjectIdentifier([]int{2, 16, 840, 1, 101, 3, 4, 2, 2}), + crypto.SHA512: asn1.ObjectIdentifier([]int{2, 16, 840, 1, 101, 3, 4, 2, 3}), +} + +// TODO(rlb): This is also from crypto/x509, so same comment as AGL's below +var signatureAlgorithmDetails = []struct { + algo x509.SignatureAlgorithm + oid asn1.ObjectIdentifier + pubKeyAlgo x509.PublicKeyAlgorithm + hash crypto.Hash +}{ + {x509.MD2WithRSA, oidSignatureMD2WithRSA, x509.RSA, crypto.Hash(0) /* no value for MD2 */}, + {x509.MD5WithRSA, oidSignatureMD5WithRSA, x509.RSA, crypto.MD5}, + {x509.SHA1WithRSA, oidSignatureSHA1WithRSA, x509.RSA, crypto.SHA1}, + {x509.SHA256WithRSA, oidSignatureSHA256WithRSA, x509.RSA, crypto.SHA256}, + {x509.SHA384WithRSA, oidSignatureSHA384WithRSA, x509.RSA, crypto.SHA384}, + {x509.SHA512WithRSA, oidSignatureSHA512WithRSA, x509.RSA, crypto.SHA512}, + {x509.DSAWithSHA1, oidSignatureDSAWithSHA1, x509.DSA, crypto.SHA1}, + {x509.DSAWithSHA256, oidSignatureDSAWithSHA256, x509.DSA, crypto.SHA256}, + {x509.ECDSAWithSHA1, oidSignatureECDSAWithSHA1, x509.ECDSA, crypto.SHA1}, + {x509.ECDSAWithSHA256, oidSignatureECDSAWithSHA256, x509.ECDSA, crypto.SHA256}, + {x509.ECDSAWithSHA384, oidSignatureECDSAWithSHA384, x509.ECDSA, crypto.SHA384}, + {x509.ECDSAWithSHA512, oidSignatureECDSAWithSHA512, x509.ECDSA, crypto.SHA512}, +} + +// TODO(rlb): This is also from crypto/x509, so same comment as AGL's below +func signingParamsForPublicKey(pub interface{}, requestedSigAlgo x509.SignatureAlgorithm) (hashFunc crypto.Hash, sigAlgo pkix.AlgorithmIdentifier, err error) { + var pubType x509.PublicKeyAlgorithm + + switch pub := pub.(type) { + case *rsa.PublicKey: + pubType = x509.RSA + hashFunc = crypto.SHA256 + sigAlgo.Algorithm = oidSignatureSHA256WithRSA + sigAlgo.Parameters = asn1.RawValue{ + Tag: 5, + } + + case *ecdsa.PublicKey: + pubType = x509.ECDSA + + switch pub.Curve { + case elliptic.P224(), elliptic.P256(): + hashFunc = crypto.SHA256 + sigAlgo.Algorithm = oidSignatureECDSAWithSHA256 + case elliptic.P384(): + hashFunc = crypto.SHA384 + sigAlgo.Algorithm = oidSignatureECDSAWithSHA384 + case elliptic.P521(): + hashFunc = crypto.SHA512 + sigAlgo.Algorithm = oidSignatureECDSAWithSHA512 + default: + err = errors.New("x509: unknown elliptic curve") + } + + default: + err = errors.New("x509: only RSA and ECDSA keys supported") + } + + if err != nil { + return + } + + if requestedSigAlgo == 0 { + return + } + + found := false + for _, details := range signatureAlgorithmDetails { + if details.algo == requestedSigAlgo { + if details.pubKeyAlgo != pubType { + err = errors.New("x509: requested SignatureAlgorithm does not match private key type") + return + } + sigAlgo.Algorithm, hashFunc = details.oid, details.hash + if hashFunc == 0 { + err = errors.New("x509: cannot sign with hash function requested") + return + } + found = true + break + } + } + + if !found { + err = errors.New("x509: unknown SignatureAlgorithm") + } + + return +} + +// TODO(agl): this is taken from crypto/x509 and so should probably be exported +// from crypto/x509 or crypto/x509/pkix. +func getSignatureAlgorithmFromOID(oid asn1.ObjectIdentifier) x509.SignatureAlgorithm { + for _, details := range signatureAlgorithmDetails { + if oid.Equal(details.oid) { + return details.algo + } + } + return x509.UnknownSignatureAlgorithm +} + +// TODO(rlb): This is not taken from crypto/x509, but it's of the same general form. +func getHashAlgorithmFromOID(target asn1.ObjectIdentifier) crypto.Hash { + for hash, oid := range hashOIDs { + if oid.Equal(target) { + return hash + } + } + return crypto.Hash(0) +} + +func getOIDFromHashAlgorithm(target crypto.Hash) asn1.ObjectIdentifier { + for hash, oid := range hashOIDs { + if hash == target { + return oid + } + } + return nil +} + +// This is the exposed reflection of the internal OCSP structures. + +// The status values that can be expressed in OCSP. See RFC 6960. +const ( + // Good means that the certificate is valid. + Good = iota + // Revoked means that the certificate has been deliberately revoked. + Revoked + // Unknown means that the OCSP responder doesn't know about the certificate. + Unknown + // ServerFailed is unused and was never used (see + // https://go-review.googlesource.com/#/c/18944). ParseResponse will + // return a ResponseError when an error response is parsed. + ServerFailed +) + +// The enumerated reasons for revoking a certificate. See RFC 5280. +const ( + Unspecified = 0 + KeyCompromise = 1 + CACompromise = 2 + AffiliationChanged = 3 + Superseded = 4 + CessationOfOperation = 5 + CertificateHold = 6 + + RemoveFromCRL = 8 + PrivilegeWithdrawn = 9 + AACompromise = 10 +) + +// Request represents an OCSP request. See RFC 6960. +type Request struct { + HashAlgorithm crypto.Hash + IssuerNameHash []byte + IssuerKeyHash []byte + SerialNumber *big.Int +} + +// Marshal marshals the OCSP request to ASN.1 DER encoded form. +func (req *Request) Marshal() ([]byte, error) { + hashAlg := getOIDFromHashAlgorithm(req.HashAlgorithm) + if hashAlg == nil { + return nil, errors.New("Unknown hash algorithm") + } + return asn1.Marshal(ocspRequest{ + tbsRequest{ + Version: 0, + RequestList: []request{ + { + Cert: certID{ + pkix.AlgorithmIdentifier{ + Algorithm: hashAlg, + Parameters: asn1.RawValue{Tag: 5 /* ASN.1 NULL */}, + }, + req.IssuerNameHash, + req.IssuerKeyHash, + req.SerialNumber, + }, + }, + }, + }, + }) +} + +// Response represents an OCSP response containing a single SingleResponse. See +// RFC 6960. +type Response struct { + // Status is one of {Good, Revoked, Unknown} + Status int + SerialNumber *big.Int + ProducedAt, ThisUpdate, NextUpdate, RevokedAt time.Time + RevocationReason int + Certificate *x509.Certificate + // TBSResponseData contains the raw bytes of the signed response. If + // Certificate is nil then this can be used to verify Signature. + TBSResponseData []byte + Signature []byte + SignatureAlgorithm x509.SignatureAlgorithm + + // IssuerHash is the hash used to compute the IssuerNameHash and IssuerKeyHash. + // Valid values are crypto.SHA1, crypto.SHA256, crypto.SHA384, and crypto.SHA512. + // If zero, the default is crypto.SHA1. + IssuerHash crypto.Hash + + // RawResponderName optionally contains the DER-encoded subject of the + // responder certificate. Exactly one of RawResponderName and + // ResponderKeyHash is set. + RawResponderName []byte + // ResponderKeyHash optionally contains the SHA-1 hash of the + // responder's public key. Exactly one of RawResponderName and + // ResponderKeyHash is set. + ResponderKeyHash []byte + + // Extensions contains raw X.509 extensions from the singleExtensions field + // of the OCSP response. When parsing certificates, this can be used to + // extract non-critical extensions that are not parsed by this package. When + // marshaling OCSP responses, the Extensions field is ignored, see + // ExtraExtensions. + Extensions []pkix.Extension + + // ExtraExtensions contains extensions to be copied, raw, into any marshaled + // OCSP response (in the singleExtensions field). Values override any + // extensions that would otherwise be produced based on the other fields. The + // ExtraExtensions field is not populated when parsing certificates, see + // Extensions. + ExtraExtensions []pkix.Extension +} + +// These are pre-serialized error responses for the various non-success codes +// defined by OCSP. The Unauthorized code in particular can be used by an OCSP +// responder that supports only pre-signed responses as a response to requests +// for certificates with unknown status. See RFC 5019. +var ( + MalformedRequestErrorResponse = []byte{0x30, 0x03, 0x0A, 0x01, 0x01} + InternalErrorErrorResponse = []byte{0x30, 0x03, 0x0A, 0x01, 0x02} + TryLaterErrorResponse = []byte{0x30, 0x03, 0x0A, 0x01, 0x03} + SigRequredErrorResponse = []byte{0x30, 0x03, 0x0A, 0x01, 0x05} + UnauthorizedErrorResponse = []byte{0x30, 0x03, 0x0A, 0x01, 0x06} +) + +// CheckSignatureFrom checks that the signature in resp is a valid signature +// from issuer. This should only be used if resp.Certificate is nil. Otherwise, +// the OCSP response contained an intermediate certificate that created the +// signature. That signature is checked by ParseResponse and only +// resp.Certificate remains to be validated. +func (resp *Response) CheckSignatureFrom(issuer *x509.Certificate) error { + return issuer.CheckSignature(resp.SignatureAlgorithm, resp.TBSResponseData, resp.Signature) +} + +// ParseError results from an invalid OCSP response. +type ParseError string + +func (p ParseError) Error() string { + return string(p) +} + +// ParseRequest parses an OCSP request in DER form. It only supports +// requests for a single certificate. Signed requests are not supported. +// If a request includes a signature, it will result in a ParseError. +func ParseRequest(bytes []byte) (*Request, error) { + var req ocspRequest + rest, err := asn1.Unmarshal(bytes, &req) + if err != nil { + return nil, err + } + if len(rest) > 0 { + return nil, ParseError("trailing data in OCSP request") + } + + if len(req.TBSRequest.RequestList) == 0 { + return nil, ParseError("OCSP request contains no request body") + } + innerRequest := req.TBSRequest.RequestList[0] + + hashFunc := getHashAlgorithmFromOID(innerRequest.Cert.HashAlgorithm.Algorithm) + if hashFunc == crypto.Hash(0) { + return nil, ParseError("OCSP request uses unknown hash function") + } + + return &Request{ + HashAlgorithm: hashFunc, + IssuerNameHash: innerRequest.Cert.NameHash, + IssuerKeyHash: innerRequest.Cert.IssuerKeyHash, + SerialNumber: innerRequest.Cert.SerialNumber, + }, nil +} + +// ParseResponse parses an OCSP response in DER form. It only supports +// responses for a single certificate. If the response contains a certificate +// then the signature over the response is checked. If issuer is not nil then +// it will be used to validate the signature or embedded certificate. +// +// Invalid responses and parse failures will result in a ParseError. +// Error responses will result in a ResponseError. +func ParseResponse(bytes []byte, issuer *x509.Certificate) (*Response, error) { + return ParseResponseForCert(bytes, nil, issuer) +} + +// ParseResponseForCert parses an OCSP response in DER form and searches for a +// Response relating to cert. If such a Response is found and the OCSP response +// contains a certificate then the signature over the response is checked. If +// issuer is not nil then it will be used to validate the signature or embedded +// certificate. +// +// Invalid responses and parse failures will result in a ParseError. +// Error responses will result in a ResponseError. +func ParseResponseForCert(bytes []byte, cert, issuer *x509.Certificate) (*Response, error) { + var resp responseASN1 + rest, err := asn1.Unmarshal(bytes, &resp) + if err != nil { + return nil, err + } + if len(rest) > 0 { + return nil, ParseError("trailing data in OCSP response") + } + + if status := ResponseStatus(resp.Status); status != Success { + return nil, ResponseError{status} + } + + if !resp.Response.ResponseType.Equal(idPKIXOCSPBasic) { + return nil, ParseError("bad OCSP response type") + } + + var basicResp basicResponse + rest, err = asn1.Unmarshal(resp.Response.Response, &basicResp) + if err != nil { + return nil, err + } + if len(rest) > 0 { + return nil, ParseError("trailing data in OCSP response") + } + + if n := len(basicResp.TBSResponseData.Responses); n == 0 || cert == nil && n > 1 { + return nil, ParseError("OCSP response contains bad number of responses") + } + + var singleResp singleResponse + if cert == nil { + singleResp = basicResp.TBSResponseData.Responses[0] + } else { + match := false + for _, resp := range basicResp.TBSResponseData.Responses { + if cert.SerialNumber.Cmp(resp.CertID.SerialNumber) == 0 { + singleResp = resp + match = true + break + } + } + if !match { + return nil, ParseError("no response matching the supplied certificate") + } + } + + ret := &Response{ + TBSResponseData: basicResp.TBSResponseData.Raw, + Signature: basicResp.Signature.RightAlign(), + SignatureAlgorithm: getSignatureAlgorithmFromOID(basicResp.SignatureAlgorithm.Algorithm), + Extensions: singleResp.SingleExtensions, + SerialNumber: singleResp.CertID.SerialNumber, + ProducedAt: basicResp.TBSResponseData.ProducedAt, + ThisUpdate: singleResp.ThisUpdate, + NextUpdate: singleResp.NextUpdate, + } + + // Handle the ResponderID CHOICE tag. ResponderID can be flattened into + // TBSResponseData once https://go-review.googlesource.com/34503 has been + // released. + rawResponderID := basicResp.TBSResponseData.RawResponderID + switch rawResponderID.Tag { + case 1: // Name + var rdn pkix.RDNSequence + if rest, err := asn1.Unmarshal(rawResponderID.Bytes, &rdn); err != nil || len(rest) != 0 { + return nil, ParseError("invalid responder name") + } + ret.RawResponderName = rawResponderID.Bytes + case 2: // KeyHash + if rest, err := asn1.Unmarshal(rawResponderID.Bytes, &ret.ResponderKeyHash); err != nil || len(rest) != 0 { + return nil, ParseError("invalid responder key hash") + } + default: + return nil, ParseError("invalid responder id tag") + } + + if len(basicResp.Certificates) > 0 { + // Responders should only send a single certificate (if they + // send any) that connects the responder's certificate to the + // original issuer. We accept responses with multiple + // certificates due to a number responders sending them[1], but + // ignore all but the first. + // + // [1] https://github.com/golang/go/issues/21527 + ret.Certificate, err = x509.ParseCertificate(basicResp.Certificates[0].FullBytes) + if err != nil { + return nil, err + } + + if err := ret.CheckSignatureFrom(ret.Certificate); err != nil { + return nil, ParseError("bad signature on embedded certificate: " + err.Error()) + } + + if issuer != nil { + if err := issuer.CheckSignature(ret.Certificate.SignatureAlgorithm, ret.Certificate.RawTBSCertificate, ret.Certificate.Signature); err != nil { + return nil, ParseError("bad OCSP signature: " + err.Error()) + } + } + } else if issuer != nil { + if err := ret.CheckSignatureFrom(issuer); err != nil { + return nil, ParseError("bad OCSP signature: " + err.Error()) + } + } + + for _, ext := range singleResp.SingleExtensions { + if ext.Critical { + return nil, ParseError("unsupported critical extension") + } + } + + for h, oid := range hashOIDs { + if singleResp.CertID.HashAlgorithm.Algorithm.Equal(oid) { + ret.IssuerHash = h + break + } + } + if ret.IssuerHash == 0 { + return nil, ParseError("unsupported issuer hash algorithm") + } + + switch { + case bool(singleResp.Good): + ret.Status = Good + case bool(singleResp.Unknown): + ret.Status = Unknown + default: + ret.Status = Revoked + ret.RevokedAt = singleResp.Revoked.RevocationTime + ret.RevocationReason = int(singleResp.Revoked.Reason) + } + + return ret, nil +} + +// RequestOptions contains options for constructing OCSP requests. +type RequestOptions struct { + // Hash contains the hash function that should be used when + // constructing the OCSP request. If zero, SHA-1 will be used. + Hash crypto.Hash +} + +func (opts *RequestOptions) hash() crypto.Hash { + if opts == nil || opts.Hash == 0 { + // SHA-1 is nearly universally used in OCSP. + return crypto.SHA1 + } + return opts.Hash +} + +// CreateRequest returns a DER-encoded, OCSP request for the status of cert. If +// opts is nil then sensible defaults are used. +func CreateRequest(cert, issuer *x509.Certificate, opts *RequestOptions) ([]byte, error) { + hashFunc := opts.hash() + + // OCSP seems to be the only place where these raw hash identifiers are + // used. I took the following from + // http://msdn.microsoft.com/en-us/library/ff635603.aspx + _, ok := hashOIDs[hashFunc] + if !ok { + return nil, x509.ErrUnsupportedAlgorithm + } + + if !hashFunc.Available() { + return nil, x509.ErrUnsupportedAlgorithm + } + h := opts.hash().New() + + var publicKeyInfo struct { + Algorithm pkix.AlgorithmIdentifier + PublicKey asn1.BitString + } + if _, err := asn1.Unmarshal(issuer.RawSubjectPublicKeyInfo, &publicKeyInfo); err != nil { + return nil, err + } + + h.Write(publicKeyInfo.PublicKey.RightAlign()) + issuerKeyHash := h.Sum(nil) + + h.Reset() + h.Write(issuer.RawSubject) + issuerNameHash := h.Sum(nil) + + req := &Request{ + HashAlgorithm: hashFunc, + IssuerNameHash: issuerNameHash, + IssuerKeyHash: issuerKeyHash, + SerialNumber: cert.SerialNumber, + } + return req.Marshal() +} + +// CreateResponse returns a DER-encoded OCSP response with the specified contents. +// The fields in the response are populated as follows: +// +// The responder cert is used to populate the responder's name field, and the +// certificate itself is provided alongside the OCSP response signature. +// +// The issuer cert is used to puplate the IssuerNameHash and IssuerKeyHash fields. +// +// The template is used to populate the SerialNumber, Status, RevokedAt, +// RevocationReason, ThisUpdate, and NextUpdate fields. +// +// If template.IssuerHash is not set, SHA1 will be used. +// +// The ProducedAt date is automatically set to the current date, to the nearest minute. +func CreateResponse(issuer, responderCert *x509.Certificate, template Response, priv crypto.Signer) ([]byte, error) { + var publicKeyInfo struct { + Algorithm pkix.AlgorithmIdentifier + PublicKey asn1.BitString + } + if _, err := asn1.Unmarshal(issuer.RawSubjectPublicKeyInfo, &publicKeyInfo); err != nil { + return nil, err + } + + if template.IssuerHash == 0 { + template.IssuerHash = crypto.SHA1 + } + hashOID := getOIDFromHashAlgorithm(template.IssuerHash) + if hashOID == nil { + return nil, errors.New("unsupported issuer hash algorithm") + } + + if !template.IssuerHash.Available() { + return nil, fmt.Errorf("issuer hash algorithm %v not linked into binary", template.IssuerHash) + } + h := template.IssuerHash.New() + h.Write(publicKeyInfo.PublicKey.RightAlign()) + issuerKeyHash := h.Sum(nil) + + h.Reset() + h.Write(issuer.RawSubject) + issuerNameHash := h.Sum(nil) + + innerResponse := singleResponse{ + CertID: certID{ + HashAlgorithm: pkix.AlgorithmIdentifier{ + Algorithm: hashOID, + Parameters: asn1.RawValue{Tag: 5 /* ASN.1 NULL */}, + }, + NameHash: issuerNameHash, + IssuerKeyHash: issuerKeyHash, + SerialNumber: template.SerialNumber, + }, + ThisUpdate: template.ThisUpdate.UTC(), + NextUpdate: template.NextUpdate.UTC(), + SingleExtensions: template.ExtraExtensions, + } + + switch template.Status { + case Good: + innerResponse.Good = true + case Unknown: + innerResponse.Unknown = true + case Revoked: + innerResponse.Revoked = revokedInfo{ + RevocationTime: template.RevokedAt.UTC(), + Reason: asn1.Enumerated(template.RevocationReason), + } + } + + rawResponderID := asn1.RawValue{ + Class: 2, // context-specific + Tag: 1, // Name (explicit tag) + IsCompound: true, + Bytes: responderCert.RawSubject, + } + tbsResponseData := responseData{ + Version: 0, + RawResponderID: rawResponderID, + ProducedAt: time.Now().Truncate(time.Minute).UTC(), + Responses: []singleResponse{innerResponse}, + } + + tbsResponseDataDER, err := asn1.Marshal(tbsResponseData) + if err != nil { + return nil, err + } + + hashFunc, signatureAlgorithm, err := signingParamsForPublicKey(priv.Public(), template.SignatureAlgorithm) + if err != nil { + return nil, err + } + + responseHash := hashFunc.New() + responseHash.Write(tbsResponseDataDER) + signature, err := priv.Sign(rand.Reader, responseHash.Sum(nil), hashFunc) + if err != nil { + return nil, err + } + + response := basicResponse{ + TBSResponseData: tbsResponseData, + SignatureAlgorithm: signatureAlgorithm, + Signature: asn1.BitString{ + Bytes: signature, + BitLength: 8 * len(signature), + }, + } + if template.Certificate != nil { + response.Certificates = []asn1.RawValue{ + {FullBytes: template.Certificate.Raw}, + } + } + responseDER, err := asn1.Marshal(response) + if err != nil { + return nil, err + } + + return asn1.Marshal(responseASN1{ + Status: asn1.Enumerated(Success), + Response: responseBytes{ + ResponseType: idPKIXOCSPBasic, + Response: responseDER, + }, + }) +} diff --git a/modules.txt b/modules.txt index e5a1093c78..378387cd1c 100644 --- a/modules.txt +++ b/modules.txt @@ -740,6 +740,7 @@ golang.org/x/crypto/ed25519 golang.org/x/crypto/ed25519/internal/edwards25519 golang.org/x/crypto/internal/subtle golang.org/x/crypto/md4 +golang.org/x/crypto/ocsp golang.org/x/crypto/pbkdf2 golang.org/x/crypto/pkcs12 golang.org/x/crypto/pkcs12/internal/rc2 From ba5acda6a9eb8e41337f2c17a877da91755a818a Mon Sep 17 00:00:00 2001 From: Bilal Akhtar Date: Mon, 24 Aug 2020 16:34:08 -0400 Subject: [PATCH 30/49] vendor: Bump pebble to 875ca32696b6 --- github.com/cockroachdb/pebble/compaction_picker.go | 11 ++++++++++- golang.org/x/sys/unix/ioctl.go | 9 +++++++++ golang.org/x/sys/unix/syscall_linux.go | 9 --------- modules.txt | 6 +++--- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/github.com/cockroachdb/pebble/compaction_picker.go b/github.com/cockroachdb/pebble/compaction_picker.go index f7fb6cc3cb..de5a9ab7ae 100644 --- a/github.com/cockroachdb/pebble/compaction_picker.go +++ b/github.com/cockroachdb/pebble/compaction_picker.go @@ -964,7 +964,16 @@ func (p *compactionPickerByScore) pickAuto(env compactionEnv) (pc *pickedCompact if !f.MarkedForCompaction { continue } - info := &scores[level] + var info *candidateLevelInfo + for i := range scores { + if scores[i].level == level { + info = &scores[i] + break + } + } + if info == nil { + panic(fmt.Sprintf("could not find candidate level info for level %d", level)) + } info.file = iter.Take() pc := pickAutoHelper(env, p.opts, p.vers, *info, p.baseLevel) // Fail-safe to protect against compacting the same sstable concurrently. diff --git a/golang.org/x/sys/unix/ioctl.go b/golang.org/x/sys/unix/ioctl.go index 3559e5dcb2..5641678613 100644 --- a/golang.org/x/sys/unix/ioctl.go +++ b/golang.org/x/sys/unix/ioctl.go @@ -20,6 +20,15 @@ func IoctlSetInt(fd int, req uint, value int) error { return ioctl(fd, req, uintptr(value)) } +// IoctlSetPointerInt performs an ioctl operation which sets an +// integer value on fd, using the specified request number. The ioctl +// argument is called with a pointer to the integer value, rather than +// passing the integer value directly. +func IoctlSetPointerInt(fd int, req uint, value int) error { + v := int32(value) + return ioctl(fd, req, uintptr(unsafe.Pointer(&v))) +} + // IoctlSetWinsize performs an ioctl on fd with a *Winsize argument. // // To change fd's window size, the req argument should be TIOCSWINSZ. diff --git a/golang.org/x/sys/unix/syscall_linux.go b/golang.org/x/sys/unix/syscall_linux.go index 3e20fcf53b..5b3af2e6a2 100644 --- a/golang.org/x/sys/unix/syscall_linux.go +++ b/golang.org/x/sys/unix/syscall_linux.go @@ -82,15 +82,6 @@ func IoctlRetInt(fd int, req uint) (int, error) { return int(ret), nil } -// IoctlSetPointerInt performs an ioctl operation which sets an -// integer value on fd, using the specified request number. The ioctl -// argument is called with a pointer to the integer value, rather than -// passing the integer value directly. -func IoctlSetPointerInt(fd int, req uint, value int) error { - v := int32(value) - return ioctl(fd, req, uintptr(unsafe.Pointer(&v))) -} - func IoctlSetRTCTime(fd int, value *RTCTime) error { err := ioctl(fd, RTC_SET_TIME, uintptr(unsafe.Pointer(value))) runtime.KeepAlive(value) diff --git a/modules.txt b/modules.txt index 378387cd1c..41d27aede0 100644 --- a/modules.txt +++ b/modules.txt @@ -205,7 +205,7 @@ github.com/cockroachdb/gostdlib/x/tools/internal/semver github.com/cockroachdb/gostdlib/x/tools/internal/span # github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f github.com/cockroachdb/logtags -# github.com/cockroachdb/pebble v0.0.0-20200817204059-a24bc6c83054 +# github.com/cockroachdb/pebble v0.0.0-20200824180455-875ca32696b6 github.com/cockroachdb/pebble github.com/cockroachdb/pebble/bloom github.com/cockroachdb/pebble/internal/arenaskl @@ -751,7 +751,7 @@ golang.org/x/crypto/ssh/agent golang.org/x/crypto/ssh/internal/bcrypt_pbkdf golang.org/x/crypto/ssh/knownhosts golang.org/x/crypto/ssh/terminal -# golang.org/x/exp v0.0.0-20200819202907-27b6b2ade93b +# golang.org/x/exp v0.0.0-20200821190819-94841d0725da golang.org/x/exp/rand # golang.org/x/lint v0.0.0-20200130185559-910be7a94367 golang.org/x/lint @@ -785,7 +785,7 @@ golang.org/x/perf/storage/benchfmt # golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e golang.org/x/sync/errgroup golang.org/x/sync/syncmap -# golang.org/x/sys v0.0.0-20200821140526-fda516888d29 +# golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 golang.org/x/sys/cpu golang.org/x/sys/internal/unsafeheader golang.org/x/sys/unix From 73ff5533a4c5e074c2be490b26b8c8aab60e88a8 Mon Sep 17 00:00:00 2001 From: Matt Jibson Date: Tue, 25 Aug 2020 09:59:17 -0600 Subject: [PATCH 31/49] pgproto3 update --- github.com/jackc/pgproto3/v2/copy_done.go | 3 +++ github.com/jackc/pgproto3/v2/copy_in_response.go | 1 + modules.txt | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/github.com/jackc/pgproto3/v2/copy_done.go b/github.com/jackc/pgproto3/v2/copy_done.go index d8b6e5d7d8..0e13282bff 100644 --- a/github.com/jackc/pgproto3/v2/copy_done.go +++ b/github.com/jackc/pgproto3/v2/copy_done.go @@ -10,6 +10,9 @@ type CopyDone struct { // Backend identifies this message as sendable by the PostgreSQL backend. func (*CopyDone) Backend() {} +// Frontend identifies this message as sendable by a PostgreSQL frontend. +func (*CopyDone) Frontend() {} + // Decode decodes src into dst. src must contain the complete message with the exception of the initial 1 byte message // type identifier and 4 byte message length. func (dst *CopyDone) Decode(src []byte) error { diff --git a/github.com/jackc/pgproto3/v2/copy_in_response.go b/github.com/jackc/pgproto3/v2/copy_in_response.go index 4439a03284..5f2595b875 100644 --- a/github.com/jackc/pgproto3/v2/copy_in_response.go +++ b/github.com/jackc/pgproto3/v2/copy_in_response.go @@ -48,6 +48,7 @@ func (src *CopyInResponse) Encode(dst []byte) []byte { sp := len(dst) dst = pgio.AppendInt32(dst, -1) + dst = append(dst, src.OverallFormat) dst = pgio.AppendUint16(dst, uint16(len(src.ColumnFormatCodes))) for _, fc := range src.ColumnFormatCodes { dst = pgio.AppendUint16(dst, fc) diff --git a/modules.txt b/modules.txt index 41d27aede0..b80a7006f0 100644 --- a/modules.txt +++ b/modules.txt @@ -455,7 +455,7 @@ github.com/jackc/pgconn/stmtcache github.com/jackc/pgio # github.com/jackc/pgpassfile v1.0.0 github.com/jackc/pgpassfile -# github.com/jackc/pgproto3/v2 v2.0.2 +# github.com/jackc/pgproto3/v2 v2.0.4 github.com/jackc/pgproto3/v2 # github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8 github.com/jackc/pgservicefile From 0f7f5a4876dd15c29b8dfc9161c0ee06dd566f0e Mon Sep 17 00:00:00 2001 From: Tobias Grieger Date: Thu, 27 Aug 2020 13:22:42 +0200 Subject: [PATCH 32/49] Bump datadriven --- github.com/cockroachdb/datadriven/datadriven.go | 1 + modules.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/github.com/cockroachdb/datadriven/datadriven.go b/github.com/cockroachdb/datadriven/datadriven.go index d45cb3f614..9849b845a8 100644 --- a/github.com/cockroachdb/datadriven/datadriven.go +++ b/github.com/cockroachdb/datadriven/datadriven.go @@ -188,6 +188,7 @@ func runDirectiveOrSubTest( mandatorySubTestPrefix string, f func(*testing.T, *TestData) string, ) { + t.Helper() if subTestName, ok := isSubTestStart(t, r, mandatorySubTestPrefix); ok { runSubTest(subTestName, t, r, f) } else { diff --git a/modules.txt b/modules.txt index b80a7006f0..532373c29b 100644 --- a/modules.txt +++ b/modules.txt @@ -165,7 +165,7 @@ github.com/cockroachdb/cockroach-go/crdb github.com/cockroachdb/crlfmt github.com/cockroachdb/crlfmt/internal/parser github.com/cockroachdb/crlfmt/internal/render -# github.com/cockroachdb/datadriven v1.0.0 +# github.com/cockroachdb/datadriven v1.0.1-0.20200826112548-92602d883b11 github.com/cockroachdb/datadriven # github.com/cockroachdb/errors v1.7.3 github.com/cockroachdb/errors From 42efe9fa6bfa3354f9c62bdea44c25b34549ea4e Mon Sep 17 00:00:00 2001 From: Oliver Tan Date: Sun, 30 Aug 2020 11:09:15 -0700 Subject: [PATCH 33/49] bump twpayne/go-geom to v1.3.5 --- github.com/twpayne/go-geom/README.md | 2 +- .../go-geom/encoding/geojson/geojson.go | 2 +- .../twpayne/go-geom/encoding/wkb/sql.go | 47 +++++++++-- .../twpayne/go-geom/encoding/wkt/decode.go | 82 +++++++++++++------ .../twpayne/go-geom/encoding/wkt/encode.go | 59 ++++++++++--- github.com/twpayne/go-geom/geom.go | 12 +++ github.com/twpayne/go-geom/go.mod | 20 +---- github.com/twpayne/go-geom/go.sum | 59 ++++++++----- github.com/twpayne/go-kml/.golangci.yml | 71 ++++++++++++++-- github.com/twpayne/go-kml/README.md | 12 ++- github.com/twpayne/go-kml/go.mod | 4 +- github.com/twpayne/go-kml/go.sum | 8 +- github.com/twpayne/go-kml/kml.go | 2 +- modules.txt | 4 +- 14 files changed, 285 insertions(+), 99 deletions(-) diff --git a/github.com/twpayne/go-geom/README.md b/github.com/twpayne/go-geom/README.md index 39913c1c88..f7e2f9892a 100644 --- a/github.com/twpayne/go-geom/README.md +++ b/github.com/twpayne/go-geom/README.md @@ -1,6 +1,6 @@ # go-geom -[![GoDoc](https://pkg.go.dev/github.com/twpayne/go-geom?status.svg)](https://pkg.go.dev/github.com/twpayne/go-geom) +[![PkgGoDev](https://pkg.go.dev/badge/github.com/twpayne/go-geom)](https://pkg.go.dev/github.com/twpayne/go-geom) [![Go Report Card](https://goreportcard.com/badge/github.com/twpayne/go-geom)](https://goreportcard.com/report/github.com/twpayne/go-geom) [![Coverage Status](https://coveralls.io/repos/github/twpayne/go-geom/badge.svg)](https://coveralls.io/github/twpayne/go-geom) diff --git a/github.com/twpayne/go-geom/encoding/geojson/geojson.go b/github.com/twpayne/go-geom/encoding/geojson/geojson.go index dff51bc22c..80b5e1e8dc 100644 --- a/github.com/twpayne/go-geom/encoding/geojson/geojson.go +++ b/github.com/twpayne/go-geom/encoding/geojson/geojson.go @@ -535,7 +535,7 @@ func encodeBBox(b *geom.Bounds) ([]float64, error) { b.Max(0), b.Max(1), b.Max(2), }, nil default: - return []float64{}, ErrUnsupportedType(l) + return []float64{}, ErrUnsupportedType(rune(l)) } } diff --git a/github.com/twpayne/go-geom/encoding/wkb/sql.go b/github.com/twpayne/go-geom/encoding/wkb/sql.go index aeb18bd808..75cbad9969 100644 --- a/github.com/twpayne/go-geom/encoding/wkb/sql.go +++ b/github.com/twpayne/go-geom/encoding/wkb/sql.go @@ -18,6 +18,14 @@ func (e ErrExpectedByteSlice) Error() string { return fmt.Sprintf("wkb: want []byte, got %T", e.Value) } +// A Geom is a WKB-ecoded Geometry that implements the sql.Scanner and +// driver.Value interfaces. +// It can be used when the geometry shape is not defined. +type Geom struct { + geom.T + opts []wkbcommon.WKBOption +} + // A Point is a WKB-encoded Point that implements the sql.Scanner and // driver.Valuer interfaces. type Point struct { @@ -67,6 +75,31 @@ type GeometryCollection struct { opts []wkbcommon.WKBOption } +// Scan scans from a []byte. +func (g *Geom) Scan(src interface{}) error { + b, ok := src.([]byte) + if !ok { + return ErrExpectedByteSlice{Value: src} + } + // NOTE(tb) other Scanners do not check the len of b, is it really useful ? + if len(b) == 0 { + return nil + } + var err error + g.T, err = Unmarshal(b, g.opts...) + return err +} + +// Value returns the WKB encoding of g. +func (g *Geom) Value() (driver.Value, error) { + return value(g.T) +} + +// Geom returns the underlying geom.T. +func (g *Geom) Geom() geom.T { + return g.T +} + // Scan scans from a []byte. func (p *Point) Scan(src interface{}) error { b, ok := src.([]byte) @@ -79,7 +112,7 @@ func (p *Point) Scan(src interface{}) error { } p1, ok := got.(*geom.Point) if !ok { - return wkbcommon.ErrUnexpectedType{Got: p1, Want: p} + return wkbcommon.ErrUnexpectedType{Got: got, Want: p} } p.Point = p1 return nil @@ -102,7 +135,7 @@ func (ls *LineString) Scan(src interface{}) error { } ls1, ok := got.(*geom.LineString) if !ok { - return wkbcommon.ErrUnexpectedType{Got: ls1, Want: ls} + return wkbcommon.ErrUnexpectedType{Got: got, Want: ls} } ls.LineString = ls1 return nil @@ -125,7 +158,7 @@ func (p *Polygon) Scan(src interface{}) error { } p1, ok := got.(*geom.Polygon) if !ok { - return wkbcommon.ErrUnexpectedType{Got: p1, Want: p} + return wkbcommon.ErrUnexpectedType{Got: got, Want: p} } p.Polygon = p1 return nil @@ -148,7 +181,7 @@ func (mp *MultiPoint) Scan(src interface{}) error { } mp1, ok := got.(*geom.MultiPoint) if !ok { - return wkbcommon.ErrUnexpectedType{Got: mp1, Want: mp} + return wkbcommon.ErrUnexpectedType{Got: got, Want: mp} } mp.MultiPoint = mp1 return nil @@ -171,7 +204,7 @@ func (mls *MultiLineString) Scan(src interface{}) error { } mls1, ok := got.(*geom.MultiLineString) if !ok { - return wkbcommon.ErrUnexpectedType{Got: mls1, Want: mls} + return wkbcommon.ErrUnexpectedType{Got: got, Want: mls} } mls.MultiLineString = mls1 return nil @@ -194,7 +227,7 @@ func (mp *MultiPolygon) Scan(src interface{}) error { } mp1, ok := got.(*geom.MultiPolygon) if !ok { - return wkbcommon.ErrUnexpectedType{Got: mp1, Want: mp} + return wkbcommon.ErrUnexpectedType{Got: got, Want: mp} } mp.MultiPolygon = mp1 return nil @@ -217,7 +250,7 @@ func (gc *GeometryCollection) Scan(src interface{}) error { } gc1, ok := got.(*geom.GeometryCollection) if !ok { - return wkbcommon.ErrUnexpectedType{Got: gc1, Want: gc} + return wkbcommon.ErrUnexpectedType{Got: got, Want: gc} } gc.GeometryCollection = gc1 return nil diff --git a/github.com/twpayne/go-geom/encoding/wkt/decode.go b/github.com/twpayne/go-geom/encoding/wkt/decode.go index 669cc2e051..48d8478f6a 100644 --- a/github.com/twpayne/go-geom/encoding/wkt/decode.go +++ b/github.com/twpayne/go-geom/encoding/wkt/decode.go @@ -19,7 +19,7 @@ func decode(wkt string) (geom.T, error) { switch t { case tPoint: - coords, _, err := readCoordsDim1(l, wkt) + coords, _, err := readCoordsDim1(l, wkt, false) if err != nil { return nil, err } @@ -30,7 +30,7 @@ func decode(wkt string) (geom.T, error) { } return p, nil case tLineString: - coords, _, err := readCoordsDim1(l, wkt) + coords, _, err := readCoordsDim1(l, wkt, false) if err != nil { return nil, err } @@ -53,7 +53,7 @@ func decode(wkt string) (geom.T, error) { } return p, nil case tMultiPoint: - coords, _, err := readCoordsDim1(l, wkt) + coords, _, err := readCoordsDim1(l, wkt, true) if err != nil { return nil, err } @@ -163,18 +163,22 @@ func createGeomCollectionForWkt(wkt string) (*geom.GeometryCollection, error) { return gc, nil } -func readCoordsDim1(l geom.Layout, wkt string) ([]geom.Coord, string, error) { - isEmpty := strings.HasSuffix(wkt, tEmpty) - if isEmpty { +func readCoordsDim1( + l geom.Layout, wkt string, allowEmptyCoordsInBrace bool, +) ([]geom.Coord, string, error) { + if strings.HasPrefix(wkt, tEmpty) { + rest := strings.TrimLeft(wkt[len(tEmpty):], ", ") + return []geom.Coord{}, rest, nil + } else if strings.HasSuffix(wkt, tEmpty) { return []geom.Coord{}, "", nil } - braceContent, rest, err := braceContentAndRestStartingWithOpeningBrace(wkt) + braceContent, rest, err := braceContentAndRestAfterComma(wkt) if err != nil { return nil, rest, err } - coords, err := coordsFromBraceContent(braceContent, l) + coords, err := coordsFromBraceContent(braceContent, l, allowEmptyCoordsInBrace) if err != nil { return nil, rest, err } @@ -184,18 +188,24 @@ func readCoordsDim1(l geom.Layout, wkt string) ([]geom.Coord, string, error) { func readCoordsDim2(l geom.Layout, wkt string) ([][]geom.Coord, string, error) { coordsDim2 := [][]geom.Coord{} - isEmpty := strings.HasSuffix(wkt, tEmpty) - if isEmpty { + if strings.HasPrefix(wkt, tEmpty) { + rest := strings.TrimLeft(wkt[len(tEmpty):], ", ") + return coordsDim2, rest, nil + } else if strings.HasSuffix(wkt, tEmpty) { return coordsDim2, "", nil } - contentDim2, restDim2, err := braceContentAndRestStartingWithOpeningBrace(wkt) + if strings.HasSuffix(wkt, tEmpty) { + return coordsDim2, "", nil + } + + contentDim2, restDim2, err := braceContentAndRestAfterComma(wkt) if err != nil { return nil, restDim2, err } for { - coordsDim1, restDim1, err := readCoordsDim1(l, contentDim2) + coordsDim1, restDim1, err := readCoordsDim1(l, contentDim2, false) if err != nil { return coordsDim2, restDim2, err } @@ -213,8 +223,7 @@ func readCoordsDim2(l geom.Layout, wkt string) ([][]geom.Coord, string, error) { func readCoordsDim3(l geom.Layout, wkt string) ([][][]geom.Coord, string, error) { coordsDim3 := [][][]geom.Coord{} - isEmpty := strings.HasSuffix(wkt, tEmpty) - if isEmpty { + if strings.HasSuffix(wkt, tEmpty) { return coordsDim3, "", nil } @@ -240,12 +249,18 @@ func readCoordsDim3(l geom.Layout, wkt string) ([][][]geom.Coord, string, error) return coordsDim3, restDim3, nil } -func coordsFromBraceContent(s string, l geom.Layout) ([]geom.Coord, error) { +func coordsFromBraceContent(s string, l geom.Layout, allowEmpty bool) ([]geom.Coord, error) { coords := []geom.Coord{} coordStrings := strings.Split(s, ",") for _, coordStr := range coordStrings { - coordElems := strings.Split(strings.TrimSpace(coordStr), " ") + coordStr = strings.TrimSpace(coordStr) + if allowEmpty && coordStr == tEmpty { + coords = append(coords, nil) + continue + } + + coordElems := strings.Split(coordStr, " ") if len(coordElems) != l.Stride() { return nil, geom.ErrStrideMismatch{ Got: len(coordElems), @@ -300,6 +315,21 @@ func braceContentAndRest(s string) (string, string, error) { return braceContent, rest, nil } +func braceContentAndRestAfterComma(s string) (string, string, error) { + content, rest, err := braceContentAndRest(s) + if err != nil { + return content, rest, err + } + + nextComma := strings.Index(rest, ",") + if nextComma >= 0 { + rest = strings.TrimSpace(rest[nextComma+1:]) + } else { + rest = "" + } + return content, rest, nil +} + func braceContentAndRestStartingWithOpeningBrace(s string) (string, string, error) { content, rest, err := braceContentAndRest(s) if err != nil { @@ -316,16 +346,22 @@ func braceContentAndRestStartingWithOpeningBrace(s string) (string, string, erro } func typeContentAndRestStartingWithLetter(s string) (string, string, error) { - content, rest, err := braceContentAndRest(s) + content, _, err := findTypeAndLayout(s) if err != nil { - return content, rest, err + return "", "", err } - - t, _, err := findTypeAndLayout(s) - if err != nil { - return content, rest, err + rest := s[len(content):] + if strings.HasPrefix(rest, tEmpty) { + content += tEmpty + rest = rest[len(tEmpty):] + } else { + c, r, err := braceContentAndRest(rest) + if err != nil { + return content, rest, err + } + content = content + "(" + c + ")" + rest = r } - content = t + "(" + content + ")" nextLetterIdx := -1 for i, char := range rest { diff --git a/github.com/twpayne/go-geom/encoding/wkt/encode.go b/github.com/twpayne/go-geom/encoding/wkt/encode.go index c75a423e29..0e473b91e3 100644 --- a/github.com/twpayne/go-geom/encoding/wkt/encode.go +++ b/github.com/twpayne/go-geom/encoding/wkt/encode.go @@ -78,22 +78,22 @@ func (e *Encoder) write(sb *strings.Builder, g geom.T) error { } return e.writeFlatCoords2(sb, g.FlatCoords(), 0, g.Ends(), layout.Stride()) case *geom.MultiPoint: - if g.Empty() { + if g.NumPoints() == 0 { return e.writeEMPTY(sb) } - return e.writeFlatCoords1(sb, g.FlatCoords(), layout.Stride()) + return e.writeFlatCoords1Ends(sb, g.FlatCoords(), 0, g.Ends()) case *geom.MultiLineString: - if g.Empty() { + if g.NumLineStrings() == 0 { return e.writeEMPTY(sb) } return e.writeFlatCoords2(sb, g.FlatCoords(), 0, g.Ends(), layout.Stride()) case *geom.MultiPolygon: - if g.Empty() { + if g.NumPolygons() == 0 { return e.writeEMPTY(sb) } return e.writeFlatCoords3(sb, g.FlatCoords(), g.Endss(), layout.Stride()) case *geom.GeometryCollection: - if g.Empty() { + if g.NumGeoms() == 0 { return e.writeEMPTY(sb) } if _, err := sb.WriteRune('('); err != nil { @@ -168,6 +168,33 @@ func (e *Encoder) writeFlatCoords1(sb *strings.Builder, flatCoords []float64, st return err } +func (e *Encoder) writeFlatCoords1Ends( + sb *strings.Builder, flatCoords []float64, start int, ends []int, +) error { + if _, err := sb.WriteRune('('); err != nil { + return err + } + for i, end := range ends { + if i != 0 { + if _, err := sb.WriteString(", "); err != nil { + return err + } + } + if end <= start { + if err := e.writeEMPTY(sb); err != nil { + return err + } + } else { + if err := e.writeCoord(sb, flatCoords[start:end]); err != nil { + return err + } + } + start = end + } + _, err := sb.WriteRune(')') + return err +} + func (e *Encoder) writeFlatCoords2( sb *strings.Builder, flatCoords []float64, start int, ends []int, stride int, ) error { @@ -180,8 +207,14 @@ func (e *Encoder) writeFlatCoords2( return err } } - if err := e.writeFlatCoords1(sb, flatCoords[start:end], stride); err != nil { - return err + if end <= start { + if err := e.writeEMPTY(sb); err != nil { + return err + } + } else { + if err := e.writeFlatCoords1(sb, flatCoords[start:end], stride); err != nil { + return err + } } start = end } @@ -202,10 +235,16 @@ func (e *Encoder) writeFlatCoords3( return err } } - if err := e.writeFlatCoords2(sb, flatCoords, start, ends, stride); err != nil { - return err + if len(ends) == 0 { + if err := e.writeEMPTY(sb); err != nil { + return err + } + } else { + if err := e.writeFlatCoords2(sb, flatCoords, start, ends, stride); err != nil { + return err + } + start = ends[len(ends)-1] } - start = ends[len(ends)-1] } _, err := sb.WriteRune(')') return err diff --git a/github.com/twpayne/go-geom/geom.go b/github.com/twpayne/go-geom/geom.go index e80448113c..1c51eefb76 100644 --- a/github.com/twpayne/go-geom/geom.go +++ b/github.com/twpayne/go-geom/geom.go @@ -193,6 +193,18 @@ func (l Layout) ZIndex() int { } } +// TransformInPlace replaces all coordinates in g using f. +func TransformInPlace(g T, f func(Coord)) T { + var ( + flatCoords = g.FlatCoords() + stride = g.Stride() + ) + for i, n := 0, len(flatCoords); i < n; i += stride { + f(flatCoords[i : i+stride]) + } + return g +} + // Must panics if err is not nil, otherwise it returns g. func Must(g T, err error) T { if err != nil { diff --git a/github.com/twpayne/go-geom/go.mod b/github.com/twpayne/go-geom/go.mod index 3f10af2542..f18c57a41e 100644 --- a/github.com/twpayne/go-geom/go.mod +++ b/github.com/twpayne/go-geom/go.mod @@ -1,24 +1,12 @@ module github.com/twpayne/go-geom require ( - github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect github.com/DATA-DOG/go-sqlmock v1.3.2 - github.com/Microsoft/go-winio v0.4.14 // indirect - github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect - github.com/cenkalti/backoff v2.1.1+incompatible // indirect - github.com/containerd/continuity v0.0.0-20181203112020-004b46473808 // indirect - github.com/docker/go-connections v0.4.0 // indirect - github.com/docker/go-units v0.3.3 // indirect github.com/google/go-cmp v0.4.0 // indirect - github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect - github.com/lib/pq v1.3.0 - github.com/opencontainers/go-digest v1.0.0-rc1 // indirect - github.com/opencontainers/image-spec v1.0.1 // indirect - github.com/opencontainers/runc v0.1.1 // indirect - github.com/ory/dockertest v3.3.4+incompatible - github.com/stretchr/testify v1.5.1 - github.com/twpayne/go-kml v1.5.0 - gotest.tools v2.2.0+incompatible // indirect + github.com/lib/pq v1.8.0 + github.com/ory/dockertest/v3 v3.6.0 + github.com/stretchr/testify v1.6.1 + github.com/twpayne/go-kml v1.5.1 ) go 1.13 diff --git a/github.com/twpayne/go-geom/go.sum b/github.com/twpayne/go-geom/go.sum index 06df206554..fffd48eab5 100644 --- a/github.com/twpayne/go-geom/go.sum +++ b/github.com/twpayne/go-geom/go.sum @@ -9,69 +9,88 @@ github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+q github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= -github.com/cenkalti/backoff v2.1.1+incompatible h1:tKJnvO2kl0zmb/jA5UKAt4VoEVw1qxKWjE/Bpp46npY= -github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= -github.com/containerd/continuity v0.0.0-20181203112020-004b46473808 h1:4BX8f882bXEDKfWIf0wa8HRvpnBoPszJJXL+TVbBw4M= -github.com/containerd/continuity v0.0.0-20181203112020-004b46473808/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/cenkalti/backoff/v3 v3.0.0 h1:ske+9nBpD9qZsTBoF41nW5L+AIuFBKMeze18XQ3eG1c= +github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= +github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6 h1:NmTXa/uVnDyp0TY5MKi197+3HWcnYWfnHGyaFthlnGw= +github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk= -github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gotestyourself/gotestyourself v2.2.0+incompatible h1:AQwinXlbQR2HvPjQZOmDhRqsv5mZf+Jb1RnSLxcqZcI= -github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY= github.com/huandu/xstrings v1.3.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= -github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= +github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y= -github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/ory/dockertest v3.3.4+incompatible h1:VrpM6Gqg7CrPm3bL4Wm1skO+zFWLbh7/Xb5kGEbJRh8= -github.com/ory/dockertest v3.3.4+incompatible/go.mod h1:1vX4m9wsvi00u5bseYwXaSnhNrne+V0E6LAcBILJdPs= +github.com/opencontainers/runc v1.0.0-rc9 h1:/k06BMULKF5hidyoZymkoDCzdJzltZpz/UU4LguQVtc= +github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/ory/dockertest/v3 v3.6.0 h1:I6KNJ6izxGduLACQii2SP/g7GN0JM9Xfaik6aAVaw6Y= +github.com/ory/dockertest/v3 v3.6.0/go.mod h1:4ZOpj8qBUmh8fcBSVzkH2bws2s91JdGvHUqan4GHEuQ= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/twpayne/go-kml v1.5.0 h1:CNWBxCmyWg1158dWbfdZsuUkUtHITrkntJS3iJhazJQ= -github.com/twpayne/go-kml v1.5.0/go.mod h1:g/OG8Q8JUxqFw8LGXE44W7osn1uXDAYaVFr1Yld43yc= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/twpayne/go-kml v1.5.1 h1:RI0JKh/VzdK/d+ZxdJzt8Ar921KMYPfg9qkw7vsbAGw= +github.com/twpayne/go-kml v1.5.1/go.mod h1:kz8jAiIz6FIdU2Zjce9qGlVtgFYES9vt7BTPBHf5jl4= github.com/twpayne/go-polyline v1.0.0/go.mod h1:ICh24bcLYBX8CknfvNPKqoTbe+eg+MX1NPyJmSBo7pU= +github.com/twpayne/go-waypoint v0.0.0-20200706203930-b263a7f6e4e8/go.mod h1:qj5pHncxKhu9gxtZEYWypA/z097sxhFlbTyOyt9gcnU= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20191003171128-d98b1b443823 h1:Ypyv6BNJh07T1pUSrehkLemqPKXhus2MkfktJ91kRh4= +golang.org/x/net v0.0.0-20191003171128-d98b1b443823/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b h1:ag/x1USPSsqHud38I9BAC88qdNLDHHtQ4mlgQIZPPNA= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200121082415-34d275377bf9 h1:N19i1HjUnR7TF7rMt8O4p3dLvqvmYyzB6ifMFmrbY50= +golang.org/x/sys v0.0.0-20200121082415-34d275377bf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.2 h1:kG1BFyqVHuQoVQiR1bWGnfz/fmHvvuiSPIV7rvl360E= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= diff --git a/github.com/twpayne/go-kml/.golangci.yml b/github.com/twpayne/go-kml/.golangci.yml index 595b3d31b8..00fe88cf06 100644 --- a/github.com/twpayne/go-kml/.golangci.yml +++ b/github.com/twpayne/go-kml/.golangci.yml @@ -1,14 +1,69 @@ linters: - enable-all: true + enable: + - asciicheck + - bodyclose + - deadcode + - depguard + - dogsled + - dupl + - errcheck + - exhaustive + - exportloopref + - gocognit + - goconst + - gocritic + - gocyclo + - godot + - goerr113 + - gofmt + - gofumpt + - goimports + - golint + - gomodguard + - goprintffuncname + - gosec + - gosimple + - govet + - ineffassign + - interfacer + - maligned + - misspell + - nakedret + - nestif + - noctx + - nolintlint + - prealloc + - rowserrcheck + - scopelint + - sqlclosecheck + - staticcheck + - structcheck + - stylecheck + - typecheck + - unconvert + - unparam + - unused + - varcheck + - whitespace disable: - - funlen - - gochecknoglobals - - godox - - gomnd - - lll - - wsl + - funlen + - gochecknoglobals + - godox + - gomnd + - lll + - testpackage + - wsl + +linters-settings: + goimports: + local-prefixes: github.com/twpayne/go-kml + issues: exclude-rules: - linters: + - dupl - scopelint - path: "_test\\.go$" \ No newline at end of file + path: "_test\\.go" + - linters: + - gosec + path: "internal/" \ No newline at end of file diff --git a/github.com/twpayne/go-kml/README.md b/github.com/twpayne/go-kml/README.md index c851911763..a0267e0c0e 100644 --- a/github.com/twpayne/go-kml/README.md +++ b/github.com/twpayne/go-kml/README.md @@ -1,8 +1,6 @@ # go-kml -[![Build Status](https://travis-ci.org/twpayne/go-kml.svg?branch=master)](https://travis-ci.org/twpayne/go-kml) -[![GoDoc](https://godoc.org/github.com/twpayne/go-kml?status.svg)](https://godoc.org/github.com/twpayne/go-kml) -[![Report Card](https://goreportcard.com/badge/github.com/twpayne/go-kml)](https://goreportcard.com/report/github.com/twpayne/go-kml) +[![GoDoc](https://pkg.go.dev/badge/github.com/twpayne/go-kml?status.svg)](https://pkg.go.dev/github.com/twpayne/go-kml) [![Coverage Status](https://coveralls.io/repos/github/twpayne/go-kml/badge.svg?branch=master)](https://coveralls.io/github/twpayne/go-kml?branch=master) Package `kml` provides convenience methods for creating and writing KML documents. @@ -11,7 +9,7 @@ Package `kml` provides convenience methods for creating and writing KML document * Simple API for building arbitrarily complex KML documents. * Support for all KML elements, including Google Earth `gx:` extensions. -* Compatibilty with the standard library [`encoding/xml`](https://godoc.org/encoding/xml) package. +* Compatibilty with the standard library [`encoding/xml`](https://pkg.go.dev/encoding/xml) package. * Pretty (neatly indented) and compact (minimum size) output formats. * Support for shared `Style` and `StyleMap` elements. * Simple mapping between functions and KML elements. @@ -53,14 +51,14 @@ Output: ``` There are more [examples in the -documentation](https://godoc.org/github.com/twpayne/go-kml#pkg-examples) +documentation](https://pkg.go.dev/github.com/twpayne/go-kml#pkg-examples) corresponding to the [examples in the KML tutorial](https://developers.google.com/kml/documentation/kml_tut). ## Subpackages -* [`icon`](https://godoc.org/github.com/twpayne/go-kml/icon) Convenience functions for using standard KML icons. -* [`sphere`](https://godoc.org/github.com/twpayne/go-kml/sphere) Convenience functions for spherical geometry. +* [`icon`](https://pkg.go.dev/github.com/twpayne/go-kml/icon) Convenience functions for using standard KML icons. +* [`sphere`](https://pkg.go.dev/github.com/twpayne/go-kml/sphere) Convenience functions for spherical geometry. ## License diff --git a/github.com/twpayne/go-kml/go.mod b/github.com/twpayne/go-kml/go.mod index 938a674206..2f2aabf57b 100644 --- a/github.com/twpayne/go-kml/go.mod +++ b/github.com/twpayne/go-kml/go.mod @@ -8,9 +8,11 @@ require ( github.com/huandu/xstrings v1.3.0 // indirect github.com/imdario/mergo v0.3.9 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect - github.com/stretchr/testify v1.5.1 + github.com/stretchr/testify v1.6.1 github.com/twpayne/go-polyline v1.0.0 + github.com/twpayne/go-waypoint v0.0.0-20200706203930-b263a7f6e4e8 golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 // indirect + gopkg.in/yaml.v2 v2.2.2 // indirect ) go 1.13 diff --git a/github.com/twpayne/go-kml/go.sum b/github.com/twpayne/go-kml/go.sum index 74bf82af48..525b808b8a 100644 --- a/github.com/twpayne/go-kml/go.sum +++ b/github.com/twpayne/go-kml/go.sum @@ -19,10 +19,12 @@ github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/twpayne/go-polyline v1.0.0 h1:EA8HPg0MNS62R5D2E8B6zyz2TMkmkXVlQaVBVgY3F5A= github.com/twpayne/go-polyline v1.0.0/go.mod h1:ICh24bcLYBX8CknfvNPKqoTbe+eg+MX1NPyJmSBo7pU= +github.com/twpayne/go-waypoint v0.0.0-20200706203930-b263a7f6e4e8 h1:0DvmyIQBIEYpXJt7Wm242Wn6ePH84FR5wv2Jh/yRgUw= +github.com/twpayne/go-waypoint v0.0.0-20200706203930-b263a7f6e4e8/go.mod h1:qj5pHncxKhu9gxtZEYWypA/z097sxhFlbTyOyt9gcnU= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 h1:3zb4D3T4G8jdExgVU/95+vQXfpEPiMdCaZgmGVxjNHM= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -34,3 +36,5 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/github.com/twpayne/go-kml/kml.go b/github.com/twpayne/go-kml/kml.go index 846f4266ff..8fd0f4ab70 100644 --- a/github.com/twpayne/go-kml/kml.go +++ b/github.com/twpayne/go-kml/kml.go @@ -172,7 +172,7 @@ func newSEVec2(name string, value Vec2) *SimpleElement { } } -func newSEString(name string, value string) *SimpleElement { +func newSEString(name, value string) *SimpleElement { return &SimpleElement{ StartElement: xml.StartElement{Name: xml.Name{Local: name}}, value: value, diff --git a/modules.txt b/modules.txt index 532373c29b..f956e790b8 100644 --- a/modules.txt +++ b/modules.txt @@ -687,7 +687,7 @@ github.com/spf13/pflag # github.com/stretchr/testify v1.6.1 github.com/stretchr/testify/assert github.com/stretchr/testify/require -# github.com/twpayne/go-geom v1.3.2 +# github.com/twpayne/go-geom v1.3.5 github.com/twpayne/go-geom github.com/twpayne/go-geom/bigxy github.com/twpayne/go-geom/encoding/ewkb @@ -704,7 +704,7 @@ github.com/twpayne/go-geom/xy/internal/hcoords github.com/twpayne/go-geom/xy/lineintersection github.com/twpayne/go-geom/xy/lineintersector github.com/twpayne/go-geom/xy/orientation -# github.com/twpayne/go-kml v1.5.0 +# github.com/twpayne/go-kml v1.5.1 github.com/twpayne/go-kml # github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad github.com/wadey/gocovmerge From e45b0079e30c9a7fc384525896c9d5d4b288f793 Mon Sep 17 00:00:00 2001 From: Bilal Akhtar Date: Mon, 31 Aug 2020 10:55:09 -0400 Subject: [PATCH 34/49] Bump pebble to e6a9f9a3c936 --- github.com/cockroachdb/pebble/compaction.go | 48 +++++ github.com/cockroachdb/pebble/event.go | 30 +++ .../pebble/internal/manifest/btree.go | 111 ++++++++--- github.com/cockroachdb/pebble/options.go | 12 +- .../cockroachdb/pebble/vfs/disk_health.go | 177 ++++++++++++++++++ .../cockroachdb/pebble/vfs/syncing_file.go | 11 ++ .../pebble/vfs/syncing_file_linux.go | 16 +- modules.txt | 2 +- 8 files changed, 370 insertions(+), 37 deletions(-) create mode 100644 github.com/cockroachdb/pebble/vfs/disk_health.go diff --git a/github.com/cockroachdb/pebble/compaction.go b/github.com/cockroachdb/pebble/compaction.go index 184ec4023f..845407f12a 100644 --- a/github.com/cockroachdb/pebble/compaction.go +++ b/github.com/cockroachdb/pebble/compaction.go @@ -1841,6 +1841,54 @@ func (d *DB) runCompaction( // beginning of the first file. n := len(ve.NewFiles) limit = c.findGrandparentLimit(ve.NewFiles[n-1].Meta.Largest.UserKey) + // This conditional is necessary to maintain the invariant that + // successive calls to finishOutput() have an increasing + // limit, and that the fragmenter's *FlushTo() and Add() calls are + // always made with successively increasing user keys. It's possible + // for the last file's Meta.Largest.UserKey + // to be substantially less than the last key returned by + // the compactionIter, such as in a sequence of consecutive + // rangedel tombstones, of which some but not all get elided. + // Consider this example: + // + // Compaction input (all range tombstones in this snippet): + // a-b + // c-d + // e-f (elided) + // g-h (elided) <-- grandparent limit before this (at ff) + // i-j (elided) + // k-q <-- grandparent limit at k + // m-q + // + // Note that elided tombstones are added to the fragmenter, but + // removed before they make their way from the fragmenter onto + // iter.tombstones. They still affect the fragmenter's internal + // tracking of the key up to which all tombstones have been flushed. + // After the first output is cut with limit = g, the fragmenter + // is empty, so the next grandparent calculation happens with + // key = g, and returns k. We continue adding range tombstones to + // the fragmenter between [g,k], which includes k-q as we only + // switch outputs after the limit is exceeded. So key = m and + // limit = k when finishOutput is called. Since the start key in + // the fragmenter (k) matches the limit, and no point keys were + // added, we actually don't produce a new output (see the + // conditional in finishOutput() on why). + // + // When we try to calculate the next grandparent limit, since + // c.rangeDelFrag.Empty() == false (as k-q is sitting in it), + // we fall into this case, and use the end key of the last written + // sstable (which was [a-d]) to calculate the grandparent limit. + // That gets us g again, which gets us to call + // c.rangeDelFrag.FlushTo(g), which violates the + // invariant that the current flush-to key (g) be greater than the + // last one (k). + // + // To solve this, if the grandparent limit falls "in between" + // ve.NewFiles[n-1].Meta.Largest.UserKey and c.rangeDelFrag.Start(), + // re-calculate a higher limit ahead of that key. + if c.rangeDelFrag.Start() != nil && c.cmp(limit, c.rangeDelFrag.Start()) <= 0 { + limit = c.findGrandparentLimit(c.rangeDelFrag.Start()) + } } // Each inner loop iteration processes one key from the input iterator. diff --git a/github.com/cockroachdb/pebble/event.go b/github.com/cockroachdb/pebble/event.go index 3ed371dcbc..14de07ce24 100644 --- a/github.com/cockroachdb/pebble/event.go +++ b/github.com/cockroachdb/pebble/event.go @@ -106,6 +106,25 @@ func (i levelInfos) SafeFormat(w redact.SafePrinter, _ rune) { } } +// DiskSlowInfo contains the info for a disk slowness event when writing to a +// file. +type DiskSlowInfo struct { + // Path of file being written to. + Path string + // Duration that has elapsed since this disk operation started. + Duration time.Duration +} + +func (i DiskSlowInfo) String() string { + return redact.StringWithoutMarkers(i) +} + +// SafeFormat implements redact.SafeFormatter. +func (i DiskSlowInfo) SafeFormat(w redact.SafePrinter, _ rune) { + w.Printf("disk slowness detected: write to file %s has been ongoing for %0.1fs", + i.Path, redact.Safe(i.Duration.Seconds())) +} + // FlushInfo contains the info for a flush event. type FlushInfo struct { // JobID is the ID of the flush job. @@ -379,6 +398,11 @@ type EventListener struct { // has been installed. CompactionEnd func(CompactionInfo) + // DiskSlow is invoked after a disk write operation on a file created + // with a disk health checking vfs.FS (see vfs.DefaultWithDiskHealthChecks) + // is observed to exceed the specified disk slowness threshold duration. + DiskSlow func(DiskSlowInfo) + // FlushBegin is invoked after the inputs to a flush have been determined, // but before the flush has produced any output. FlushBegin func(FlushInfo) @@ -436,6 +460,9 @@ func (l *EventListener) EnsureDefaults(logger Logger) { if l.CompactionEnd == nil { l.CompactionEnd = func(info CompactionInfo) {} } + if l.DiskSlow == nil { + l.DiskSlow = func(info DiskSlowInfo) {} + } if l.FlushBegin == nil { l.FlushBegin = func(info FlushInfo) {} } @@ -491,6 +518,9 @@ func MakeLoggingEventListener(logger Logger) EventListener { CompactionEnd: func(info CompactionInfo) { logger.Infof("%s", info) }, + DiskSlow: func(info DiskSlowInfo) { + logger.Infof("%s", info) + }, FlushBegin: func(info FlushInfo) { logger.Infof("%s", info) }, diff --git a/github.com/cockroachdb/pebble/internal/manifest/btree.go b/github.com/cockroachdb/pebble/internal/manifest/btree.go index 8074a9f88e..601b6eb143 100644 --- a/github.com/cockroachdb/pebble/internal/manifest/btree.go +++ b/github.com/cockroachdb/pebble/internal/manifest/btree.go @@ -5,10 +5,13 @@ package manifest import ( + "fmt" "strings" "sync" "sync/atomic" "unsafe" + + "github.com/cockroachdb/pebble/internal/base" ) const ( @@ -84,7 +87,7 @@ func mut(n **node) *node { // reference count to be greater than 1, we might be racing // with another call to decRef on this node. c := (*n).clone() - (*n).decRef(true /* recursive */) + (*n).decRef(true /* recursive */, nil) *n = c return *n } @@ -96,11 +99,29 @@ func (n *node) incRef() { // decRef releases a reference to the node. If requested, the method // will recurse into child nodes and decrease their refcounts as well. -func (n *node) decRef(recursive bool) { +// When a node is released, its contained files are dereferenced. +func (n *node) decRef(recursive bool, obsolete *[]base.FileNum) { if atomic.AddInt32(&n.ref, -1) > 0 { // Other references remain. Can't free. return } + + // Dereference the node's metadata. + if recursive { + for _, f := range n.items[:n.count] { + if atomic.AddInt32(&f.refs, -1) == 0 { + // There are two sources of node dereferences: tree mutations + // and Version dereferences. Files should only be made obsolete + // during Version dereferences, during which `obsolete` will be + // non-nil. + if obsolete == nil { + panic(fmt.Sprintf("file metadata %s dereferenced to zero during tree mutation", f.FileNum)) + } + *obsolete = append(*obsolete, f.FileNum) + } + } + } + // Clear and release node into memory pool. if n.leaf { ln := nodeToLeaf(n) @@ -110,7 +131,7 @@ func (n *node) decRef(recursive bool) { // Release child references first, if requested. if recursive { for i := int16(0); i <= n.count; i++ { - n.children[i].decRef(true /* recursive */) + n.children[i].decRef(true /* recursive */, obsolete) } } *n = node{} @@ -130,6 +151,10 @@ func (n *node) clone() *node { // triggering the race detector and looking like a data race. c.count = n.count c.items = n.items + // Increase the refcount of each contained item. + for _, f := range n.items[:n.count] { + atomic.AddInt32(&f.refs, 1) + } if !c.leaf { // Copy children and increase each refcount. c.children = n.children @@ -283,17 +308,19 @@ func (n *node) split(i int) (*FileMetadata, *node) { } // insert inserts a item into the subtree rooted at this node, making sure no -// nodes in the subtree exceed maxItems items. Returns true if an existing item -// was replaced and false if a item was inserted. -func (n *node) insert(cmp func(*FileMetadata, *FileMetadata) int, item *FileMetadata) (replaced bool) { +// nodes in the subtree exceed maxItems items. +func (n *node) insert(cmp func(*FileMetadata, *FileMetadata) int, item *FileMetadata) { i, found := n.find(cmp, item) if found { - n.items[i] = item - return true + // cmp provides a total ordering of the files within a level. + // If we're inserting a metadata that's equal to an existing item + // in the tree, we're inserting a file into a level twice. + panic(fmt.Sprintf("file key collision: existing metadata %s, inserting %s", + n.items[i].FileNum, item.FileNum)) } if n.leaf { n.insertAt(i, item, nil) - return false + return } if n.children[i].count >= maxItems { splitLa, splitNode := mut(&n.children[i]).split(maxItems / 2) @@ -305,12 +332,14 @@ func (n *node) insert(cmp func(*FileMetadata, *FileMetadata) int, item *FileMeta case cmp > 0: i++ // we want second split node default: - n.items[i] = item - return true + // cmp provides a total ordering of the files within a level. + // If we're inserting a metadata that's equal to an existing item + // in the tree, we're inserting a file into a level twice. + panic(fmt.Sprintf("file key collision: existing metadata %s, inserting %s", + n.items[i].FileNum, item.FileNum)) } } - replaced = mut(&n.children[i]).insert(cmp, item) - return replaced + mut(&n.children[i]).insert(cmp, item) } // removeMax removes and returns the maximum item from the subtree rooted @@ -471,7 +500,7 @@ func (n *node) rebalanceOrMerge(i int) { } child.count += mergeChild.count + 1 - mergeChild.decRef(false /* recursive */) + mergeChild.decRef(false /* recursive */, nil) } } @@ -490,16 +519,17 @@ type btree struct { cmp func(*FileMetadata, *FileMetadata) int } -// Reset removes all items from the btree. In doing so, it allows memory -// held by the btree to be recycled. Failure to call this method before -// letting a btree be GCed is safe in that it won't cause a memory leak, -// but it will prevent btree nodes from being efficiently re-used. -func (t *btree) Reset() { +// Release dereferences and clears the root node of the btree, removing all +// items from the btree. In doing so, it allows memory held by the btree to be +// recycled. It returns a slice of file numbers of newly obsolete files, if +// any. +func (t *btree) Release() (obsolete []base.FileNum) { if t.root != nil { - t.root.decRef(true /* recursive */) + t.root.decRef(true /* recursive */, &obsolete) t.root = nil } t.length = 0 + return obsolete } // Clone clones the btree, lazily. It does so in constant time. @@ -525,13 +555,15 @@ func (t *btree) Clone() btree { return c } -// Delete removes a item equal to the passed in item from the tree. -func (t *btree) Delete(item *FileMetadata) { +// Delete removes the provided file from the tree. +// It returns true if the file now has a zero reference count. +func (t *btree) Delete(item *FileMetadata) (obsolete bool) { if t.root == nil || t.root.count == 0 { - return + return false } if out := mut(&t.root).remove(t.cmp, item); out != nil { t.length-- + obsolete = atomic.AddInt32(&out.refs, -1) == 0 } if t.root.count == 0 { old := t.root @@ -540,13 +572,14 @@ func (t *btree) Delete(item *FileMetadata) { } else { t.root = t.root.children[0] } - old.decRef(false /* recursive */) + old.decRef(false /* recursive */, nil) } + return obsolete } -// Set adds the given item to the tree. If a item in the tree already -// equals the given one, it is replaced with the new item. -func (t *btree) Set(item *FileMetadata) { +// Insert adds the given item to the tree. If a item in the tree already +// equals the given one, Insert panics. +func (t *btree) Insert(item *FileMetadata) { if t.root == nil { t.root = newLeafNode() } else if t.root.count >= maxItems { @@ -558,9 +591,9 @@ func (t *btree) Set(item *FileMetadata) { newRoot.children[1] = splitNode t.root = newRoot } - if replaced := mut(&t.root).insert(t.cmp, item); !replaced { - t.length++ - } + atomic.AddInt32(&item.refs, 1) + mut(&t.root).insert(t.cmp, item) + t.length++ } // MakeIter returns a new iterator object. It is not safe to continue using an @@ -667,6 +700,18 @@ func (is *iterStack) len() int { return int(is.aLen) } +func (is *iterStack) clone() iterStack { + // If the iterator is using the embedded iterStackArr, we only need to + // copy the struct itself. + if is.s == nil { + return *is + } + clone := *is + clone.s = make([]iterFrame, len(is.s)) + copy(clone.s, is.s) + return clone +} + func (is *iterStack) reset() { if is.aLen == -1 { is.s = is.s[:0] @@ -684,6 +729,12 @@ type iterator struct { s iterStack } +func (i *iterator) clone() iterator { + c := *i + c.s = i.s.clone() + return c +} + func (i *iterator) reset() { i.n = i.r i.pos = -1 diff --git a/github.com/cockroachdb/pebble/options.go b/github.com/cockroachdb/pebble/options.go index 60b50607d8..9ec0536e48 100644 --- a/github.com/cockroachdb/pebble/options.go +++ b/github.com/cockroachdb/pebble/options.go @@ -477,9 +477,6 @@ func (o *Options) EnsureDefaults() *Options { if o.Comparer == nil { o.Comparer = DefaultComparer } - if o.FS == nil { - o.FS = vfs.Default - } if o.Experimental.L0CompactionConcurrency <= 0 { o.Experimental.L0CompactionConcurrency = 10 } @@ -536,6 +533,15 @@ func (o *Options) EnsureDefaults() *Options { if o.MaxConcurrentCompactions <= 0 { o.MaxConcurrentCompactions = 1 } + if o.FS == nil { + o.FS = vfs.WithDiskHealthChecks(vfs.Default, 5 * time.Second, + func(name string, duration time.Duration) { + o.EventListener.DiskSlow(DiskSlowInfo{ + Path: name, + Duration: duration, + }) + }) + } o.initMaps() return o diff --git a/github.com/cockroachdb/pebble/vfs/disk_health.go b/github.com/cockroachdb/pebble/vfs/disk_health.go new file mode 100644 index 0000000000..ff8f09377a --- /dev/null +++ b/github.com/cockroachdb/pebble/vfs/disk_health.go @@ -0,0 +1,177 @@ +// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package vfs + +import ( + "sync/atomic" + "time" +) + +const ( + // defaultTickInterval is the default interval between two ticks of each + // diskHealthCheckingFile loop iteration. + defaultTickInterval = 2 * time.Second +) + +// diskHealthCheckingFile is a File wrapper to detect slow disk operations, and +// call onSlowDisk if a disk operation is seen to exceed diskSlowThreshold. +// +// This struct creates a goroutine (in startTicker()) that, at every tick +// interval, sees if there's a disk operation taking longer than the specified +// duration. This setup is preferable to creating a new timer at every disk +// operation, as it reduces overhead per disk operation. +type diskHealthCheckingFile struct { + File + + onSlowDisk func(time.Duration) + diskSlowThreshold time.Duration + tickInterval time.Duration + + stopper chan struct{} + lastWriteNanos int64 +} + +// newDiskHealthCheckingFile instantiates a new diskHealthCheckingFile, with the +// specified time threshold and event listener. +func newDiskHealthCheckingFile( + file File, diskSlowThreshold time.Duration, onSlowDisk func(time.Duration), +) *diskHealthCheckingFile { + return &diskHealthCheckingFile{ + File: file, + onSlowDisk: onSlowDisk, + diskSlowThreshold: diskSlowThreshold, + tickInterval: defaultTickInterval, + + stopper: make(chan struct{}), + } +} + +// startTicker starts a new goroutine with a ticker to monitor disk operations. +// Can only be called if the ticker goroutine isn't running already. +func (d *diskHealthCheckingFile) startTicker() { + if d.diskSlowThreshold == 0 { + return + } + + go func() { + ticker := time.NewTicker(d.tickInterval) + defer ticker.Stop() + + for { + select { + case <-d.stopper: + return + + case <-ticker.C: + lastWriteNanos := atomic.LoadInt64(&d.lastWriteNanos) + if lastWriteNanos == 0 { + continue + } + lastWrite := time.Unix(0, lastWriteNanos) + now := time.Now() + if lastWrite.Add(d.diskSlowThreshold).Before(now) { + // diskSlowThreshold was exceeded. Call the passed-in + // listener. + d.onSlowDisk(now.Sub(lastWrite)) + } + } + } + }() +} + +// stopTicker stops the goroutine started in startTicker. +func (d *diskHealthCheckingFile) stopTicker() { + close(d.stopper) +} + +// Write implements the io.Writer interface. +func (d *diskHealthCheckingFile) Write(p []byte) (n int, err error) { + d.timeDiskOp(func() { + n, err = d.File.Write(p) + }) + return n, err +} + +// Close implements the io.Closer interface. +func (d *diskHealthCheckingFile) Close() error { + d.stopTicker() + return d.File.Close() +} + +// Sync implements the io.Syncer interface. +func (d *diskHealthCheckingFile) Sync() (err error) { + d.timeDiskOp(func() { + err = d.File.Sync() + }) + return err +} + +// timeDiskOp runs the specified closure and makes its timing visible to the +// monitoring goroutine, in case it exceeds one of the slow disk durations. +func (d *diskHealthCheckingFile) timeDiskOp(op func()) { + if d == nil { + op() + return + } + + atomic.StoreInt64(&d.lastWriteNanos, time.Now().UnixNano()) + defer func() { + atomic.StoreInt64(&d.lastWriteNanos, 0) + }() + op() +} + +type diskHealthCheckingFS struct { + FS + + diskSlowThreshold time.Duration + onSlowDisk func(string, time.Duration) +} + +// WithDiskHealthChecks wraps an FS and ensures that all +// write-oriented created with that FS are wrapped with disk health detection +// checks. Disk operations that are observed to take longer than +// diskSlowThreshold trigger an onSlowDisk call. +func WithDiskHealthChecks( + fs FS, diskSlowThreshold time.Duration, onSlowDisk func(string, time.Duration), +) FS { + return diskHealthCheckingFS{ + FS: fs, + diskSlowThreshold: diskSlowThreshold, + onSlowDisk: onSlowDisk, + } +} + +// Create implements the vfs.FS interface. +func (d diskHealthCheckingFS) Create(name string) (File, error) { + f, err := d.FS.Create(name) + if err != nil { + return f, err + } + if d.diskSlowThreshold == 0 { + return f, nil + } + checkingFile := newDiskHealthCheckingFile(f, d.diskSlowThreshold, func(duration time.Duration) { + d.onSlowDisk(name, duration) + }) + checkingFile.startTicker() + return checkingFile, nil +} + +// ReuseForWrite implements the vfs.FS interface. +func (d diskHealthCheckingFS) ReuseForWrite(oldname, newname string) (File, error) { + f, err := d.FS.ReuseForWrite(oldname, newname) + if err != nil { + return f, err + } + if d.diskSlowThreshold == 0 { + return f, nil + } + checkingFile := newDiskHealthCheckingFile(f, d.diskSlowThreshold, func(duration time.Duration) { + d.onSlowDisk(newname, duration) + }) + checkingFile.startTicker() + return checkingFile, nil +} diff --git a/github.com/cockroachdb/pebble/vfs/syncing_file.go b/github.com/cockroachdb/pebble/vfs/syncing_file.go index ab337e75ad..1811de662f 100644 --- a/github.com/cockroachdb/pebble/vfs/syncing_file.go +++ b/github.com/cockroachdb/pebble/vfs/syncing_file.go @@ -31,6 +31,7 @@ type syncingFile struct { preallocatedBlocks int64 syncData func() error syncTo func(offset int64) error + timeDiskOp func(op func()) } // NewSyncingFile wraps a writable file and ensures that data is synced @@ -54,6 +55,16 @@ func NewSyncingFile(f File, opts SyncingFileOptions) File { if d, ok := f.(fd); ok { s.fd = d.Fd() } + type dhChecker interface { + timeDiskOp(op func()) + } + if d, ok := f.(dhChecker); ok { + s.timeDiskOp = d.timeDiskOp + } else { + s.timeDiskOp = func(op func()) { + op() + } + } s.init() diff --git a/github.com/cockroachdb/pebble/vfs/syncing_file_linux.go b/github.com/cockroachdb/pebble/vfs/syncing_file_linux.go index df204d891e..5393349b0d 100644 --- a/github.com/cockroachdb/pebble/vfs/syncing_file_linux.go +++ b/github.com/cockroachdb/pebble/vfs/syncing_file_linux.go @@ -42,7 +42,9 @@ func (f *syncingFile) init() { if f.fd == 0 { return } - f.useSyncRange = isSyncRangeSupported(f.fd) + f.timeDiskOp(func() { + f.useSyncRange = isSyncRangeSupported(f.fd) + }) if f.useSyncRange { f.syncTo = f.syncToRange } else { @@ -55,7 +57,11 @@ func (f *syncingFile) syncFdatasync() error { if f.fd == 0 { return f.File.Sync() } - return syscall.Fdatasync(int(f.fd)) + var err error + f.timeDiskOp(func() { + err = syscall.Fdatasync(int(f.fd)) + }) + return err } func (f *syncingFile) syncToFdatasync(_ int64) error { @@ -82,5 +88,9 @@ func (f *syncingFile) syncToRange(offset int64) error { // use of `waitBefore` is to limit how much dirty data is allowed to // accumulate. Linux sometimes behaves poorly when a large amount of dirty // data accumulates, impacting other I/O operations. - return syscall.SyncFileRange(int(f.fd), 0, offset, write|waitBefore) + var err error + f.timeDiskOp(func() { + err = syscall.SyncFileRange(int(f.fd), 0, offset, write|waitBefore) + }) + return err } diff --git a/modules.txt b/modules.txt index f956e790b8..c7c3132de9 100644 --- a/modules.txt +++ b/modules.txt @@ -205,7 +205,7 @@ github.com/cockroachdb/gostdlib/x/tools/internal/semver github.com/cockroachdb/gostdlib/x/tools/internal/span # github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f github.com/cockroachdb/logtags -# github.com/cockroachdb/pebble v0.0.0-20200824180455-875ca32696b6 +# github.com/cockroachdb/pebble v0.0.0-20200831143935-e6a9f9a3c936 github.com/cockroachdb/pebble github.com/cockroachdb/pebble/bloom github.com/cockroachdb/pebble/internal/arenaskl From 208203b24cedfb6b59dbd7b737f9c23da14d0404 Mon Sep 17 00:00:00 2001 From: Raphael 'kena' Poss Date: Tue, 1 Sep 2020 19:11:30 +0200 Subject: [PATCH 35/49] bump cockroachdb/errors, cockroachdb/redact --- github.com/cockroachdb/errors/assert_api.go | 15 +- github.com/cockroachdb/errors/barriers_api.go | 15 +- .../cockroachdb/errors/contexttags_api.go | 21 +- .../cockroachdb/errors/domains/domains.go | 2 +- github.com/cockroachdb/errors/domains_api.go | 45 +- .../errors/errbase/format_error.go | 2 + github.com/cockroachdb/errors/errbase_api.go | 129 +- .../cockroachdb/errors/errutil/message.go | 4 + .../cockroachdb/errors/errutil/redactable.go | 53 + .../cockroachdb/errors/errutil/utilities.go | 42 +- github.com/cockroachdb/errors/errutil_api.go | 128 +- github.com/cockroachdb/errors/go.mod | 2 +- github.com/cockroachdb/errors/go.sum | 6 +- .../cockroachdb/errors/hintdetail_api.go | 48 +- .../cockroachdb/errors/issuelink_api.go | 42 +- .../cockroachdb/errors/markers/markers.go | 3 +- github.com/cockroachdb/errors/markers_api.go | 35 +- .../cockroachdb/errors/report/report.go | 5 +- github.com/cockroachdb/errors/report_api.go | 79 +- .../cockroachdb/errors/safedetails_api.go | 33 +- .../cockroachdb/errors/secondary_api.go | 19 +- .../errors/telemetrykeys/with_telemetry.go | 4 +- .../cockroachdb/errors/telemetrykeys_api.go | 13 +- .../cockroachdb/errors/withstack_api.go | 42 +- github.com/cockroachdb/redact/README.md | 3 + github.com/cockroachdb/redact/builder.go | 139 ++ github.com/cockroachdb/redact/doc.go | 28 + .../cockroachdb/redact/internal/README.md | 52 + .../redact/internal/fmtsort/sort.go | 222 +++ .../cockroachdb/redact/internal/format.go | 584 ++++++++ .../cockroachdb/redact/internal/hooks.go | 83 ++ .../cockroachdb/redact/internal/print.go | 1189 +++++++++++++++++ .../cockroachdb/redact/internal/refresh.sh | 29 + github.com/cockroachdb/redact/make_format.go | 7 +- github.com/cockroachdb/redact/markers.go | 13 +- .../redact/markers_internal_escape.go | 84 +- .../redact/markers_internal_print.go | 174 ++- .../redact/markers_internal_printer.go | 9 +- .../cockroachdb/redact/markers_print.go | 67 +- modules.txt | 6 +- 40 files changed, 3231 insertions(+), 245 deletions(-) create mode 100644 github.com/cockroachdb/redact/builder.go create mode 100644 github.com/cockroachdb/redact/internal/README.md create mode 100644 github.com/cockroachdb/redact/internal/fmtsort/sort.go create mode 100644 github.com/cockroachdb/redact/internal/format.go create mode 100644 github.com/cockroachdb/redact/internal/hooks.go create mode 100644 github.com/cockroachdb/redact/internal/print.go create mode 100644 github.com/cockroachdb/redact/internal/refresh.sh diff --git a/github.com/cockroachdb/errors/assert_api.go b/github.com/cockroachdb/errors/assert_api.go index 419402f2fc..d01e0bf63f 100644 --- a/github.com/cockroachdb/errors/assert_api.go +++ b/github.com/cockroachdb/errors/assert_api.go @@ -16,11 +16,20 @@ package errors import "github.com/cockroachdb/errors/assert" -// WithAssertionFailure forwards a definition. +// WithAssertionFailure decorates the error with an assertion failure marker. +// This is not intended to be used directly (see AssertionFailed() for +// further decoration). +// +// Detail is shown: +// - when formatting with `%+v`. +// - in Sentry reports. func WithAssertionFailure(err error) error { return assert.WithAssertionFailure(err) } -// HasAssertionFailure forwards a definition. +// HasAssertionFailure returns true if the error or any of its causes +// is an assertion failure annotation. func HasAssertionFailure(err error) bool { return assert.HasAssertionFailure(err) } -// IsAssertionFailure forwards a definition. +// IsAssertionFailure returns true if the error (not its causes) is an +// assertion failure annotation. Consider using markers.If or +// HasAssertionFailure to test both the error and its causes. func IsAssertionFailure(err error) bool { return assert.IsAssertionFailure(err) } diff --git a/github.com/cockroachdb/errors/barriers_api.go b/github.com/cockroachdb/errors/barriers_api.go index 61757112e2..feeb618e5c 100644 --- a/github.com/cockroachdb/errors/barriers_api.go +++ b/github.com/cockroachdb/errors/barriers_api.go @@ -16,8 +16,19 @@ package errors import "github.com/cockroachdb/errors/barriers" -// Handled forwards a definition. +// Handled swallows the provided error and hides it from the +// Cause()/Unwrap() interface, and thus the Is() facility that +// identifies causes. However, it retains it for the purpose of +// printing the error out (e.g. for troubleshooting). The error +// message is preserved in full. +// +// Detail is shown: +// - via `errors.GetSafeDetails()`, shows details from hidden error. +// - when formatting with `%+v`. +// - in Sentry reports. func Handled(err error) error { return barriers.Handled(err) } -// HandledWithMessage forwards a definition. +// HandledWithMessage is like Handled except the message is overridden. +// This can be used e.g. to hide message details or to prevent +// downstream code to make assertions on the message's contents. func HandledWithMessage(err error, msg string) error { return barriers.HandledWithMessage(err, msg) } diff --git a/github.com/cockroachdb/errors/contexttags_api.go b/github.com/cockroachdb/errors/contexttags_api.go index 51ac6ed32b..3f77960e84 100644 --- a/github.com/cockroachdb/errors/contexttags_api.go +++ b/github.com/cockroachdb/errors/contexttags_api.go @@ -21,10 +21,27 @@ import ( "github.com/cockroachdb/logtags" ) -// WithContextTags forwards a definition. +// WithContextTags captures the k/v pairs stored in the context via the +// `logtags` package and annotates them on the error. +// +// Only the stromg representation of values remains available. This is +// because the library cannot guarantee that the underlying value is +// preserved across the network. To avoid creating a stateful interface +// (where the user code needs to know whether an error has traveled +// through the network or not), the library restricts access to the +// value part as strings. See GetContextTags() below. +// +// Detail is shown: +// - via `errors.GetSafeDetails()`. +// - via `GetContextTags()` below. +// - when formatting with `%+v`. +// - in Sentry reports. func WithContextTags(err error, ctx context.Context) error { return contexttags.WithContextTags(err, ctx) } -// GetContextTags forwards a definition. +// GetContextTags retrieves the k/v pairs stored in the error. +// The sets are returned from outermost to innermost level of cause. +// The returned logtags.Buffer only know about the string +// representation of the values originally captured by the error. func GetContextTags(err error) []*logtags.Buffer { return contexttags.GetContextTags(err) } diff --git a/github.com/cockroachdb/errors/domains/domains.go b/github.com/cockroachdb/errors/domains/domains.go index 2cdfe039c2..a7b0dde5cd 100644 --- a/github.com/cockroachdb/errors/domains/domains.go +++ b/github.com/cockroachdb/errors/domains/domains.go @@ -144,7 +144,7 @@ func PackageDomain() Domain { } // PackageDomainAtDepth returns an error domain that describes the -// package at the given depth. +// package at the given call depth. func PackageDomainAtDepth(depth int) Domain { _, f, _, _ := runtime.Caller(1 + depth) return Domain("error domain: pkg " + filepath.Dir(f)) diff --git a/github.com/cockroachdb/errors/domains_api.go b/github.com/cockroachdb/errors/domains_api.go index fea98f31e5..5e965e5494 100644 --- a/github.com/cockroachdb/errors/domains_api.go +++ b/github.com/cockroachdb/errors/domains_api.go @@ -16,42 +16,63 @@ package errors import "github.com/cockroachdb/errors/domains" -// Domain forwards a definition. +// Domain is the type of a domain annotation. type Domain = domains.Domain -// NoDomain forwards a definition. +// NoDomain is the domain of errors that don't originate +// from a barrier. const NoDomain Domain = domains.NoDomain -// NamedDomain forwards a definition. +// NamedDomain returns an error domain identified by the given string. func NamedDomain(domainName string) Domain { return domains.NamedDomain(domainName) } -// PackageDomain forwards a definition. +// PackageDomain returns an error domain that represents the +// package of its caller. func PackageDomain() Domain { return domains.PackageDomainAtDepth(1) } -// PackageDomainAtDepth forwards a definition. +// PackageDomainAtDepth returns an error domain that describes the +// package at the given call depth. func PackageDomainAtDepth(depth int) Domain { return domains.PackageDomainAtDepth(depth) } -// WithDomain forwards a definition. +// WithDomain wraps an error so that it appears to come from the given domain. +// +// Domain is shown: +// - via `errors.GetSafeDetails()`. +// - when formatting with `%+v`. +// - in Sentry reports. func WithDomain(err error, domain Domain) error { return domains.WithDomain(err, domain) } -// NotInDomain forwards a definition. +// NotInDomain returns true if and only if the error's +// domain is not one of the specified domains. func NotInDomain(err error, doms ...Domain) bool { return domains.NotInDomain(err, doms...) } -// EnsureNotInDomain forwards a definition. +// EnsureNotInDomain checks whether the error is in the given domain(s). +// If it is, the given constructor if provided is called to construct +// an alternate error. If no error constructor is provided, +// a new barrier is constructed automatically using the first +// provided domain as new domain. The original error message +// is preserved. func EnsureNotInDomain(err error, constructor DomainOverrideFn, forbiddenDomains ...Domain) error { return domains.EnsureNotInDomain(err, constructor, forbiddenDomains...) } -// DomainOverrideFn forwards a definition. +// DomainOverrideFn is the type of the callback function passed to EnsureNotInDomain(). type DomainOverrideFn = func(originalDomain Domain, err error) error -// HandledInDomain forwards a definition. +// HandledInDomain creates an error in the given domain and retains +// the details of the given original error as context for +// debugging. The original error is hidden and does not become a +// "cause" for the new error. The original's error _message_ +// is preserved. +// +// See the documentation of `WithDomain()` and `errors.Handled()` for details. func HandledInDomain(err error, domain Domain) error { return domains.HandledInDomain(err, domain) } -// HandledInDomainWithMessage forwards a definition. +// HandledInDomainWithMessage is like HandledWithMessage but with a domain. func HandledInDomainWithMessage(err error, domain Domain, msg string) error { return domains.HandledInDomainWithMessage(err, domain, msg) } -// GetDomain forwards a definition. +// GetDomain extracts the domain of the given error, or NoDomain if +// the error's cause does not have a domain annotation. func GetDomain(err error) Domain { return domains.GetDomain(err) } diff --git a/github.com/cockroachdb/errors/errbase/format_error.go b/github.com/cockroachdb/errors/errbase/format_error.go index 41e0442459..534db0809e 100644 --- a/github.com/cockroachdb/errors/errbase/format_error.go +++ b/github.com/cockroachdb/errors/errbase/format_error.go @@ -36,6 +36,8 @@ import ( ) // FormatError formats an error according to s and verb. +// This is a helper meant for use when implementing the fmt.Formatter +// interface on custom error objects. // // If the error implements errors.Formatter, FormatError calls its // FormatError method of f with an errors.Printer configured according diff --git a/github.com/cockroachdb/errors/errbase_api.go b/github.com/cockroachdb/errors/errbase_api.go index 31d3ef147a..7e5bfbff40 100644 --- a/github.com/cockroachdb/errors/errbase_api.go +++ b/github.com/cockroachdb/errors/errbase_api.go @@ -21,90 +21,159 @@ import ( "github.com/cockroachdb/errors/errbase" ) -// UnwrapOnce forwards a definition. +// UnwrapOnce accesses the direct cause of the error if any, otherwise +// returns nil. +// +// It supports both errors implementing causer (`Cause()` method, from +// github.com/pkg/errors) and `Wrapper` (`Unwrap()` method, from the +// Go 2 error proposal). func UnwrapOnce(err error) error { return errbase.UnwrapOnce(err) } -// UnwrapAll forwards a definition. +// UnwrapAll accesses the root cause object of the error. +// If the error has no cause (leaf error), it is returned directly. func UnwrapAll(err error) error { return errbase.UnwrapAll(err) } -// EncodedError forwards a definition. +// EncodedError is the type of an encoded (and protobuf-encodable) error. type EncodedError = errbase.EncodedError -// EncodeError forwards a definition. +// EncodeError encodes an error. func EncodeError(ctx context.Context, err error) EncodedError { return errbase.EncodeError(ctx, err) } -// DecodeError forwards a definition. +// DecodeError decodes an error. func DecodeError(ctx context.Context, enc EncodedError) error { return errbase.DecodeError(ctx, enc) } -// SafeDetailer forwards a definition. +// SafeDetailer is an interface that can be implemented by errors that +// can provide PII-free additional strings suitable for reporting or +// telemetry. type SafeDetailer = errbase.SafeDetailer -// GetAllSafeDetails forwards a definition. +// GetAllSafeDetails collects the safe details from the given error object +// and all its causes. +// The details are collected from outermost to innermost level of cause. func GetAllSafeDetails(err error) []SafeDetailPayload { return errbase.GetAllSafeDetails(err) } -// GetSafeDetails forwards a definition. +// GetSafeDetails collects the safe details from the given error +// object. If it is a wrapper, only the details from the wrapper are +// returned. func GetSafeDetails(err error) (payload SafeDetailPayload) { return errbase.GetSafeDetails(err) } -// SafeDetailPayload forwards a definition. +// SafeDetailPayload captures the safe strings for one +// level of wrapping. type SafeDetailPayload = errbase.SafeDetailPayload -// RegisterLeafDecoder forwards a definition. +// RegisterLeafDecoder can be used to register new leaf error types to +// the library. Registered types will be decoded using their own +// Go type when an error is decoded. Wrappers that have not been +// registered will be decoded using the opaqueLeaf type. +// +// Note: if the error type has been migrated from a previous location +// or a different type, ensure that RegisterTypeMigration() was called +// prior to RegisterLeafDecoder(). func RegisterLeafDecoder(typeName TypeKey, decoder LeafDecoder) { errbase.RegisterLeafDecoder(typeName, decoder) } -// TypeKey forwards a definition. +// TypeKey identifies an error for the purpose of looking up decoders. +// It is equivalent to the "family name" in ErrorTypeMarker. type TypeKey = errbase.TypeKey -// GetTypeKey forwards a definition. +// GetTypeKey retrieve the type key for a given error object. This +// is meant for use in combination with the Register functions. func GetTypeKey(err error) TypeKey { return errbase.GetTypeKey(err) } -// LeafDecoder forwards a definition. +// LeafDecoder is to be provided (via RegisterLeafDecoder above) +// by additional wrapper types not yet known to this library. +// A nil return indicates that decoding was not successful. type LeafDecoder = errbase.LeafDecoder -// RegisterWrapperDecoder forwards a definition. +// RegisterWrapperDecoder can be used to register new wrapper types to +// the library. Registered wrappers will be decoded using their own +// Go type when an error is decoded. Wrappers that have not been +// registered will be decoded using the opaqueWrapper type. +// +// Note: if the error type has been migrated from a previous location +// or a different type, ensure that RegisterTypeMigration() was called +// prior to RegisterWrapperDecoder(). func RegisterWrapperDecoder(typeName TypeKey, decoder WrapperDecoder) { errbase.RegisterWrapperDecoder(typeName, decoder) } -// WrapperDecoder forwards a definition. +// WrapperDecoder is to be provided (via RegisterWrapperDecoder above) +// by additional wrapper types not yet known to this library. +// A nil return indicates that decoding was not successful. type WrapperDecoder = errbase.WrapperDecoder -// RegisterLeafEncoder forwards a definition. +// RegisterLeafEncoder can be used to register new leaf error types to +// the library. Registered types will be encoded using their own +// Go type when an error is encoded. Wrappers that have not been +// registered will be encoded using the opaqueLeaf type. +// +// Note: if the error type has been migrated from a previous location +// or a different type, ensure that RegisterTypeMigration() was called +// prior to RegisterLeafEncoder(). func RegisterLeafEncoder(typeName TypeKey, encoder LeafEncoder) { errbase.RegisterLeafEncoder(typeName, encoder) } -// LeafEncoder forwards a definition. +// LeafEncoder is to be provided (via RegisterLeafEncoder above) +// by additional wrapper types not yet known to this library. type LeafEncoder = errbase.LeafEncoder -// RegisterWrapperEncoder forwards a definition. +// RegisterWrapperEncoder can be used to register new wrapper types to +// the library. Registered wrappers will be encoded using their own +// Go type when an error is encoded. Wrappers that have not been +// registered will be encoded using the opaqueWrapper type. +// +// Note: if the error type has been migrated from a previous location +// or a different type, ensure that RegisterTypeMigration() was called +// prior to RegisterWrapperEncoder(). func RegisterWrapperEncoder(typeName TypeKey, encoder WrapperEncoder) { errbase.RegisterWrapperEncoder(typeName, encoder) } -// WrapperEncoder forwards a definition. +// WrapperEncoder is to be provided (via RegisterWrapperEncoder above) +// by additional wrapper types not yet known to this library. type WrapperEncoder = errbase.WrapperEncoder -// SetWarningFn forwards a definition. +// SetWarningFn enables configuration of the warning function. func SetWarningFn(fn func(context.Context, string, ...interface{})) { errbase.SetWarningFn(fn) } -// Formatter is provided for compatibility with xerrors. -// This should probably not be used directly, and -// SafeFormatter preferred instead. +// A Formatter formats error messages. +// +// NB: Consider implementing SafeFormatter instead. This will ensure +// that error displays can distinguish bits that are PII-safe. type Formatter = errbase.Formatter -// SafeFormatter is like Formatter but supports the separation -// of safe and unsafe strings. +// SafeFormatter is implemented by error leaf or wrapper types that want +// to separate safe and non-safe information when printed out. +// +// When multiple errors are chained (e.g. via errors.Wrap), intermediate +// layers in the error that do not implement SafeError are considered +// “unsafe†type SafeFormatter = errbase.SafeFormatter -// Printer is provided for compatibility with xerrors. +// A Printer formats error messages. +// +// The most common implementation of Printer is the one provided by package fmt +// during Printf (as of Go 1.13). Localization packages such as golang.org/x/text/message +// typically provide their own implementations. type Printer = errbase.Printer -// FormatError can be used to implement the fmt.Formatter interface. +// FormatError formats an error according to s and verb. +// This is a helper meant for use when implementing the fmt.Formatter +// interface on custom error objects. +// +// If the error implements errors.Formatter, FormatError calls its +// FormatError method of f with an errors.Printer configured according +// to s and verb, and writes the result to s. +// +// Otherwise, if it is a wrapper, FormatError prints out its error prefix, +// then recurses on its cause. +// +// Otherwise, its Error() text is printed. func FormatError(err error, s fmt.State, verb rune) { errbase.FormatError(err, s, verb) } -// Formattable can be used to print an error with enhanced detail -// printout when the outer layer of wrapping may not be provided by -// this library. +// Formattable wraps an error into a fmt.Formatter which +// will provide "smart" formatting even if the outer layer +// of the error does not implement the Formatter interface. func Formattable(err error) fmt.Formatter { return errbase.Formattable(err) } diff --git a/github.com/cockroachdb/errors/errutil/message.go b/github.com/cockroachdb/errors/errutil/message.go index 30423e2a5c..2ee60b842a 100644 --- a/github.com/cockroachdb/errors/errutil/message.go +++ b/github.com/cockroachdb/errors/errutil/message.go @@ -18,6 +18,8 @@ import "github.com/cockroachdb/redact" // WithMessage annotates err with a new message. // If err is nil, WithMessage returns nil. +// The message is considered safe for reporting +// and is included in Sentry reports. func WithMessage(err error, message string) error { if err == nil { return nil @@ -30,6 +32,8 @@ func WithMessage(err error, message string) error { // WithMessagef annotates err with the format specifier. // If err is nil, WithMessagef returns nil. +// The message is formatted as per redact.Sprintf, +// to separate safe and unsafe strings for Sentry reporting. func WithMessagef(err error, format string, args ...interface{}) error { if err == nil { return nil diff --git a/github.com/cockroachdb/errors/errutil/redactable.go b/github.com/cockroachdb/errors/errutil/redactable.go index a61523ac75..6d34f250a5 100644 --- a/github.com/cockroachdb/errors/errutil/redactable.go +++ b/github.com/cockroachdb/errors/errutil/redactable.go @@ -108,3 +108,56 @@ func init() { errbase.RegisterWrapperEncoder(errbase.GetTypeKey((*withPrefix)(nil)), encodeWithPrefix) errbase.RegisterWrapperDecoder(errbase.GetTypeKey((*withPrefix)(nil)), decodeWithPrefix) } + +// withNewMessage is like withPrefix but the message completely +// overrides that of the underlying error. +type withNewMessage struct { + cause error + message redact.RedactableString +} + +var _ error = (*withNewMessage)(nil) +var _ fmt.Formatter = (*withNewMessage)(nil) +var _ errbase.SafeFormatter = (*withNewMessage)(nil) +var _ errbase.SafeDetailer = (*withNewMessage)(nil) + +func (l *withNewMessage) Error() string { + return l.message.StripMarkers() +} + +func (l *withNewMessage) Cause() error { return l.cause } +func (l *withNewMessage) Unwrap() error { return l.cause } + +func (l *withNewMessage) Format(s fmt.State, verb rune) { errbase.FormatError(l, s, verb) } +func (l *withNewMessage) SafeFormatError(p errbase.Printer) (next error) { + p.Print(l.message) + return nil /* nil here overrides the cause's message */ +} + +func (l *withNewMessage) SafeDetails() []string { + return []string{l.message.Redact().StripMarkers()} +} + +func encodeWithNewMessage(_ context.Context, err error) (string, []string, proto.Message) { + l := err.(*withNewMessage) + return l.Error(), l.SafeDetails(), &errorspb.StringPayload{Msg: string(l.message)} +} + +func decodeWithNewMessage( + _ context.Context, cause error, _ string, _ []string, payload proto.Message, +) error { + m, ok := payload.(*errorspb.StringPayload) + if !ok { + // If this ever happens, this means some version of the library + // (presumably future) changed the payload type, and we're + // receiving this here. In this case, give up and let + // DecodeError use the opaque type. + return nil + } + return &withNewMessage{cause: cause, message: redact.RedactableString(m.Msg)} +} + +func init() { + errbase.RegisterWrapperEncoder(errbase.GetTypeKey((*withNewMessage)(nil)), encodeWithNewMessage) + errbase.RegisterWrapperDecoder(errbase.GetTypeKey((*withNewMessage)(nil)), decodeWithNewMessage) +} diff --git a/github.com/cockroachdb/errors/errutil/utilities.go b/github.com/cockroachdb/errors/errutil/utilities.go index 7f60b05716..2fd126d74b 100644 --- a/github.com/cockroachdb/errors/errutil/utilities.go +++ b/github.com/cockroachdb/errors/errutil/utilities.go @@ -15,10 +15,6 @@ package errutil import ( - "fmt" - "regexp" - - "github.com/cockroachdb/errors/safedetails" "github.com/cockroachdb/errors/secondary" "github.com/cockroachdb/errors/withstack" "github.com/cockroachdb/redact" @@ -78,11 +74,11 @@ func NewWithDepthf(depth int, format string, args ...interface{}) error { errRefs = append(errRefs, e) } } - if hasVerbW.MatchString(format) { - err = fmt.Errorf(format, args...) - err = safedetails.WithSafeDetails(err, format, args...) + redactable, wrappedErr := redact.HelperForErrorf(format, args...) + if wrappedErr != nil { + err = &withNewMessage{cause: wrappedErr, message: redactable} } else { - err = error(&leafError{redact.Sprintf(format, args...)}) + err = &leafError{redactable} } for _, e := range errRefs { err = secondary.WithSecondaryError(err, e) @@ -91,36 +87,6 @@ func NewWithDepthf(depth int, format string, args ...interface{}) error { return err } -var hasVerbW = regexp.MustCompile( - `^` + - // Ignore any non-%w format directives and other characters: - `([^%]+|%` + - // A format directive starts with some flags. - `[-#0+ ]*` + - // Followed by an optional argument number [n]. - `(\[[0-9]+\])?` + - // Followed by an optional width. - `(\*|[0-9]+)?` + - // followed by an optional precision. - `(\.(\*|[0-9]))?` + - // We're only interested in non-w verbs here. - `[^w]` + - // zero or more times. - `)*` + - // Followed with at least one occurrence of a format directive - // with the 'w' verb. We replicate the rules above. - `%` + - // A format directive starts with some flags. - `[-#0+ ]*` + - // Followed by an optional argument number [n]. - `(\[[0-9]+\])?` + - // Followed by an optional width. - `(\*|[0-9]+)?` + - // followed by an optional precision. - `(\.(\*|[0-9]))?` + - // Just the verb 'w'. - `w`) - // Wrap wraps an error with a message prefix. // A stack trace is retained. // diff --git a/github.com/cockroachdb/errors/errutil_api.go b/github.com/cockroachdb/errors/errutil_api.go index c4086228bd..512bdedb9e 100644 --- a/github.com/cockroachdb/errors/errutil_api.go +++ b/github.com/cockroachdb/errors/errutil_api.go @@ -20,89 +20,175 @@ import ( "github.com/cockroachdb/errors/errutil" ) -// New forwards a definition. +// New creates an error with a simple error message. +// A stack trace is retained. +// +// Note: the message string is assumed to not contain +// PII and is included in Sentry reports. +// Use errors.Newf("%s", ) for errors +// strings that may contain PII information. +// +// Detail output: +// - message via `Error()` and formatting using `%v`/`%s`/`%q`. +// - everything when formatting with `%+v`. +// - stack trace and message via `errors.GetSafeDetails()`. +// - stack trace and message in Sentry reports. func New(msg string) error { return errutil.NewWithDepth(1, msg) } -// NewWithDepth forwards a definition. +// NewWithDepth is like New() except the depth to capture the stack +// trace is configurable. +// See the doc of `New()` for more details. func NewWithDepth(depth int, msg string) error { return errutil.NewWithDepth(depth+1, msg) } -// Newf forwards a definition. +// Newf creates an error with a formatted error message. +// A stack trace is retained. +// +// Note: the format string is assumed to not contain +// PII and is included in Sentry reports. +// Use errors.Newf("%s", ) for errors +// strings that may contain PII information. +// +// See the doc of `New()` for more details. func Newf(format string, args ...interface{}) error { return errutil.NewWithDepthf(1, format, args...) } -// NewWithDepthf forwards a definition. +// NewWithDepthf is like Newf() except the depth to capture the stack +// trace is configurable. +// See the doc of `New()` for more details. func NewWithDepthf(depth int, format string, args ...interface{}) error { return errutil.NewWithDepthf(depth+1, format, args...) } -// Errorf forwards a definition. +// Errorf aliases Newf(). func Errorf(format string, args ...interface{}) error { return errutil.NewWithDepthf(1, format, args...) } -// Cause is provided for compatibility with github.com/pkg/errors. +// Cause aliases UnwrapAll() for compatibility with github.com/pkg/errors. func Cause(err error) error { return errbase.UnwrapAll(err) } -// Unwrap is provided for compatibility with xerrors. +// Unwrap aliases UnwrapOnce() for compatibility with xerrors. func Unwrap(err error) error { return errbase.UnwrapOnce(err) } -// Wrapper is provided for compatibility with xerrors. +// Wrapper is the type of an error wrapper. type Wrapper interface { Unwrap() error } -// Opaque is provided for compatibility with xerrors. +// Opaque aliases barrier.Handled(), for compatibility with xerrors. func Opaque(err error) error { return barriers.Handled(err) } -// WithMessage forwards a definition. +// WithMessage annotates err with a new message. +// If err is nil, WithMessage returns nil. +// The message is considered safe for reporting +// and is included in Sentry reports. func WithMessage(err error, msg string) error { return errutil.WithMessage(err, msg) } -// WithMessagef forwards a definition. +// WithMessagef annotates err with the format specifier. +// If err is nil, WithMessagef returns nil. +// The message is formatted as per redact.Sprintf, +// to separate safe and unsafe strings for Sentry reporting. func WithMessagef(err error, format string, args ...interface{}) error { return errutil.WithMessagef(err, format, args...) } -// Wrap forwards a definition. +// Wrap wraps an error with a message prefix. +// A stack trace is retained. +// +// Note: the prefix string is assumed to not contain +// PII and is included in Sentry reports. +// Use errors.Wrapf(err, "%s", ) for errors +// strings that may contain PII information. +// +// Detail output: +// - original error message + prefix via `Error()` and formatting using `%v`/`%s`/`%q`. +// - everything when formatting with `%+v`. +// - stack trace and message via `errors.GetSafeDetails()`. +// - stack trace and message in Sentry reports. func Wrap(err error, msg string) error { return errutil.WrapWithDepth(1, err, msg) } -// WrapWithDepth forwards a definition. +// WrapWithDepth is like Wrap except the depth to capture the stack +// trace is configurable. +// The the doc of `Wrap()` for more details. func WrapWithDepth(depth int, err error, msg string) error { return errutil.WrapWithDepth(depth+1, err, msg) } -// Wrapf forwards a definition. +// Wrapf wraps an error with a formatted message prefix. A stack +// trace is also retained. If the format is empty, no prefix is added, +// but the extra arguments are still processed for reportable strings. +// +// Note: the format string is assumed to not contain +// PII and is included in Sentry reports. +// Use errors.Wrapf(err, "%s", ) for errors +// strings that may contain PII information. +// +// Detail output: +// - original error message + prefix via `Error()` and formatting using `%v`/`%s`/`%q`. +// - everything when formatting with `%+v`. +// - stack trace, format, and redacted details via `errors.GetSafeDetails()`. +// - stack trace, format, and redacted details in Sentry reports. func Wrapf(err error, format string, args ...interface{}) error { return errutil.WrapWithDepthf(1, err, format, args...) } -// WrapWithDepthf forwards a definition. +// WrapWithDepthf is like Wrapf except the depth to capture the stack +// trace is configurable. +// The the doc of `Wrapf()` for more details. func WrapWithDepthf(depth int, err error, format string, args ...interface{}) error { return errutil.WrapWithDepthf(depth+1, err, format, args...) } -// AssertionFailedf forwards a definition. +// AssertionFailedf creates an internal error. +// +// Detail is shown: +// - via `errors.GetSafeDetails()`, shows redacted strings. +// - when formatting with `%+v`. +// - in Sentry reports. func AssertionFailedf(format string, args ...interface{}) error { return errutil.AssertionFailedWithDepthf(1, format, args...) } -// AssertionFailedWithDepthf forwards a definition. +// AssertionFailedWithDepthf creates an internal error +// with a stack trace collected at the specified depth. +// See the doc of `AssertionFailedf()` for more details. func AssertionFailedWithDepthf(depth int, format string, args ...interface{}) error { return errutil.AssertionFailedWithDepthf(depth+1, format, args...) } -// NewAssertionErrorWithWrappedErrf forwards a definition. +// NewAssertionErrorWithWrappedErrf wraps an error and turns it into +// an assertion error. Both details from the original error and the +// context of the caller are preserved. The original error is not +// visible as cause any more. The original error message is preserved. +// See the doc of `AssertionFailedf()` for more details. func NewAssertionErrorWithWrappedErrf(origErr error, format string, args ...interface{}) error { return errutil.NewAssertionErrorWithWrappedErrDepthf(1, origErr, format, args...) } -// HandleAsAssertionFailure forwards a definition. +// HandleAsAssertionFailure hides an error and turns it into +// an assertion failure. Both details from the original error and the +// context of the caller are preserved. The original error is not +// visible as cause any more. The original error message is preserved. +// See the doc of `AssertionFailedf()` for more details. func HandleAsAssertionFailure(origErr error) error { return errutil.HandleAsAssertionFailureDepth(1, origErr) } -// HandleAsAssertionFailureDepth forwards a definition. +// HandleAsAssertionFailureDepth is like HandleAsAssertionFailure but +// the depth at which the call stack is captured can be specified. func HandleAsAssertionFailureDepth(depth int, origErr error) error { return errutil.HandleAsAssertionFailureDepth(1+depth, origErr) } -// As forwards a definition +// As finds the first error in err's chain that matches the type to which target +// points, and if so, sets the target to its value and returns true. An error +// matches a type if it is assignable to the target type, or if it has a method +// As(interface{}) bool such that As(target) returns true. As will panic if target +// is not a non-nil pointer to a type which implements error or is of interface type. +// +// The As method should set the target to its value and return true if err +// matches the type to which target points. +// +// Note: this implementation differs from that of xerrors as follows: +// - it also supports recursing through causes with Cause(). +// - if it detects an API use error, its panic object is a valid error. func As(err error, target interface{}) bool { return errutil.As(err, target) } diff --git a/github.com/cockroachdb/errors/go.mod b/github.com/cockroachdb/errors/go.mod index 02b5cbdcb6..8aad9b7c7d 100644 --- a/github.com/cockroachdb/errors/go.mod +++ b/github.com/cockroachdb/errors/go.mod @@ -5,7 +5,7 @@ go 1.13 require ( github.com/cockroachdb/datadriven v1.0.0 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f - github.com/cockroachdb/redact v1.0.2 + github.com/cockroachdb/redact v1.0.4 github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 github.com/gogo/protobuf v1.3.1 github.com/gogo/status v1.1.0 diff --git a/github.com/cockroachdb/errors/go.sum b/github.com/cockroachdb/errors/go.sum index d4a6d98dc0..4dd9db9cc5 100644 --- a/github.com/cockroachdb/errors/go.sum +++ b/github.com/cockroachdb/errors/go.sum @@ -17,8 +17,10 @@ github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlN github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/cockroachdb/redact v1.0.2 h1:bRktqHBXPqI+9bkOx0ikn9RS09G9k83oAdLx6rXRVTQ= -github.com/cockroachdb/redact v1.0.2/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/redact v1.0.4-0.20200901154857-49cd8220a922 h1:GL77i8JixDUQjDrb0bRU4DNioBRsh0t1t1g3imQYyAE= +github.com/cockroachdb/redact v1.0.4-0.20200901154857-49cd8220a922/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/redact v1.0.4 h1:UZv01KibZfgs975zjD0vf1PewwbGofROVtIfBk2Xyt0= +github.com/cockroachdb/redact v1.0.4/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= diff --git a/github.com/cockroachdb/errors/hintdetail_api.go b/github.com/cockroachdb/errors/hintdetail_api.go index 8df54e4cb7..623bf64af2 100644 --- a/github.com/cockroachdb/errors/hintdetail_api.go +++ b/github.com/cockroachdb/errors/hintdetail_api.go @@ -16,36 +16,64 @@ package errors import "github.com/cockroachdb/errors/hintdetail" -// ErrorHinter forwards a definition. +// ErrorHinter is implemented by types that can provide +// user-informing detail strings. This is implemented by withHint +// here, withIssueLink, assertionFailure and pgerror.Error. type ErrorHinter = hintdetail.ErrorHinter -// ErrorDetailer forwards a definition. +// ErrorDetailer is implemented by types that can provide +// user-informing detail strings. type ErrorDetailer = hintdetail.ErrorDetailer -// WithHint forwards a definition. +// WithHint decorates an error with a textual hint. +// The hint may contain PII and thus will not reportable. +// +// Hint is shown: +// - when formatting with `%+v`. +// - with `GetAllHints()` / `FlattenHints()` below. +// +// Note: the hint does not appear in the main error message returned +// with Error(). Use GetAllHints() or FlattenHints() to retrieve it. func WithHint(err error, msg string) error { return hintdetail.WithHint(err, msg) } -// WithHintf forwards a definition. +// WithHintf is a helper that formats the hint. +// See the documentation of WithHint() for details. func WithHintf(err error, format string, args ...interface{}) error { return hintdetail.WithHintf(err, format, args...) } -// WithDetail forwards a definition. +// WithDetail decorates an error with a textual detail. +// The detail may contain PII and thus will not reportable. +// +// Detail is shown: +// - when formatting with `%+v`. +// - with `GetAllHints()` / `FlattenHints()` below. +// +// Note: the detail does not appear in the main error message returned +// with Error(). Use GetAllDetails() or FlattenDetails() to retrieve +// it. func WithDetail(err error, msg string) error { return hintdetail.WithDetail(err, msg) } -// WithDetailf forwards a definition. +// WithDetailf is a helper that formats the detail string. +// See the documentation of WithDetail() for details. func WithDetailf(err error, format string, args ...interface{}) error { return hintdetail.WithDetailf(err, format, args...) } -// GetAllHints forwards a definition. +// GetAllHints retrieves the hints from the error using in post-order +// traversal. The hints are de-duplicated. Assertion failures, issue +// links and unimplemented errors are detected and receive standard +// hints. func GetAllHints(err error) []string { return hintdetail.GetAllHints(err) } -// FlattenHints forwards a definition. +// FlattenHints retrieves the hints as per GetAllHints() and +// concatenates them into a single string. func FlattenHints(err error) string { return hintdetail.FlattenHints(err) } -// GetAllDetails forwards a definition. +// GetAllDetails retrieves the details from the error using in post-order +// traversal. func GetAllDetails(err error) []string { return hintdetail.GetAllDetails(err) } -// FlattenDetails forwards a definition. +// FlattenDetails retrieves the details as per GetAllDetails() and +// concatenates them into a single string. func FlattenDetails(err error) string { return hintdetail.FlattenDetails(err) } diff --git a/github.com/cockroachdb/errors/issuelink_api.go b/github.com/cockroachdb/errors/issuelink_api.go index 22f6db13da..431915864b 100644 --- a/github.com/cockroachdb/errors/issuelink_api.go +++ b/github.com/cockroachdb/errors/issuelink_api.go @@ -16,36 +16,58 @@ package errors import "github.com/cockroachdb/errors/issuelink" -// WithIssueLink forwards a definition. +// WithIssueLink adds an annotation to a know issue +// on a web issue tracker. +// +// The url and detail strings may contain PII and will +// be considered reportable. +// +// Detail is shown: +// - via `errors.GetSafeDetails()` +// - when formatting with `%+v`. +// - in Sentry reports. +// - via `errors.GetAllHints()` / `errors.FlattenHints()` func WithIssueLink(err error, issue IssueLink) error { return issuelink.WithIssueLink(err, issue) } -// IssueLink forwards a definition. +// IssueLink is the payload for a linked issue annotation. type IssueLink = issuelink.IssueLink -// UnimplementedError forwards a definition. +// UnimplementedError creates a new leaf error that indicates that +// some feature was not (yet) implemented. +// +// Detail is shown: +// - via `errors.GetSafeDetails()` +// - when formatting with `%+v`. +// - in Sentry reports. +// - via `errors.GetAllHints()` / `errors.FlattenHints()` func UnimplementedError(issueLink IssueLink, msg string) error { return issuelink.UnimplementedError(issueLink, msg) } -// UnimplementedErrorf forwards a definition. +// UnimplementedErrorf creates a new leaf error that indicates that +// some feature was not (yet) implemented. The message is formatted. func UnimplementedErrorf(issueLink IssueLink, format string, args ...interface{}) error { return issuelink.UnimplementedErrorf(issueLink, format, args...) } -// GetAllIssueLinks forwards a definition. +// GetAllIssueLinks retrieves the linked issue carried +// by the error or its direct causes. func GetAllIssueLinks(err error) (issues []IssueLink) { return issuelink.GetAllIssueLinks(err) } -// HasIssueLink forwards a definition. +// HasIssueLink returns true iff the error or one of its +// causes has a linked issue payload. func HasIssueLink(err error) bool { return issuelink.HasIssueLink(err) } -// IsIssueLink forwards a definition. +// IsIssueLink returns true iff the error (not its +// causes) has a linked issue payload. func IsIssueLink(err error) bool { return issuelink.IsIssueLink(err) } -// HasUnimplementedError forwards a definition. +// HasUnimplementedError returns iff if err or its cause is an +// unimplemented error. func HasUnimplementedError(err error) bool { return issuelink.HasUnimplementedError(err) } -// IsUnimplementedError forwards a definition. +// IsUnimplementedError returns iff if err is an unimplemented error. func IsUnimplementedError(err error) bool { return issuelink.IsUnimplementedError(err) } -// UnimplementedErrorHint forwards a definition. +// UnimplementedErrorHint is the hint emitted upon unimplemented errors. const UnimplementedErrorHint = issuelink.UnimplementedErrorHint diff --git a/github.com/cockroachdb/errors/markers/markers.go b/github.com/cockroachdb/errors/markers/markers.go index c3b4bd3c21..8a2c2e62ec 100644 --- a/github.com/cockroachdb/errors/markers/markers.go +++ b/github.com/cockroachdb/errors/markers/markers.go @@ -98,7 +98,8 @@ func getInterfaceType(context string, referenceInterface interface{}) reflect.Ty return typ.Elem() } -// If returns a predicate's return value the first time the predicate returns true. +// If iterates on the error's causal chain and returns a predicate's +// return value the first time the predicate returns true. // // Note: if any of the error types has been migrated from a previous // package location or a different type, ensure that diff --git a/github.com/cockroachdb/errors/markers_api.go b/github.com/cockroachdb/errors/markers_api.go index aced76fbbb..64c8f29415 100644 --- a/github.com/cockroachdb/errors/markers_api.go +++ b/github.com/cockroachdb/errors/markers_api.go @@ -16,24 +16,47 @@ package errors import "github.com/cockroachdb/errors/markers" -// Is forwards a definition. +// Is determines whether one of the causes of the given error or any +// of its causes is equivalent to some reference error. +// +// Note: if any of the error types has been migrated from a previous +// package location or a different type, ensure that +// RegisterTypeMigration() was called prior to Is(). func Is(err, reference error) bool { return markers.Is(err, reference) } -// HasType forwards a definition. +// HasType returns true iff err contains an error whose concrete type +// matches that of referenceType. func HasType(err, referenceType error) bool { return markers.HasType(err, referenceType) } -// HasInterface forwards a definition. +// HasInterface returns true if err contains an error which implements the +// interface pointed to by referenceInterface. The type of referenceInterface +// must be a pointer to an interface type. If referenceInterface is not a +// pointer to an interface, this function will panic. func HasInterface(err error, referenceInterface interface{}) bool { return markers.HasInterface(err, referenceInterface) } -// If forwards a definition. +// If iterates on the error's causal chain and returns a predicate's +// return value the first time the predicate returns true. +// +// Note: if any of the error types has been migrated from a previous +// package location or a different type, ensure that +// RegisterTypeMigration() was called prior to If(). func If(err error, pred func(err error) (interface{}, bool)) (interface{}, bool) { return markers.If(err, pred) } -// IsAny forwards a definition. +// IsAny is like Is except that multiple references are compared. +// +// Note: if any of the error types has been migrated from a previous +// package location or a different type, ensure that +// RegisterTypeMigration() was called prior to IsAny(). func IsAny(err error, references ...error) bool { return markers.IsAny(err, references...) } -// Mark forwards a definition. +// Mark creates an explicit mark for the given error, using +// the same mark as some reference error. +// +// Note: if any of the error types has been migrated from a previous +// package location or a different type, ensure that +// RegisterTypeMigration() was called prior to Mark(). func Mark(err error, reference error) error { return markers.Mark(err, reference) } diff --git a/github.com/cockroachdb/errors/report/report.go b/github.com/cockroachdb/errors/report/report.go index c30e598ced..33d507ae7c 100644 --- a/github.com/cockroachdb/errors/report/report.go +++ b/github.com/cockroachdb/errors/report/report.go @@ -357,7 +357,10 @@ func BuildSentryReport(err error) (event *sentry.Event, extraDetails map[string] var redactedMarker = redact.RedactableString(redact.RedactedMarker()) // ReportError reports the given error to Sentry. The caller is responsible for -// checking whether telemetry is enabled. +// checking whether telemetry is enabled, and calling the sentry.Flush() +// function to wait for the report to be uploaded. (By default, +// Sentry submits reports asynchronously.) +// // Note: an empty 'eventID' can be returned which signifies that the error was // not reported. This can occur when Sentry client hasn't been properly // configured or Sentry client decided to not report the error (due to diff --git a/github.com/cockroachdb/errors/report_api.go b/github.com/cockroachdb/errors/report_api.go index 2e4d60ffc6..c346990ff4 100644 --- a/github.com/cockroachdb/errors/report_api.go +++ b/github.com/cockroachdb/errors/report_api.go @@ -19,10 +19,85 @@ import ( "github.com/cockroachdb/sentry-go" ) -// BuildSentryReport forwards a definition. +// BuildSentryReport builds the components of a sentry report. This +// can be used instead of ReportError() below to use additional custom +// conditions in the reporting or add additional reporting tags. +// +// The Sentry Event is populated for maximal utility when exploited in +// the Sentry.io web interface and database. +// +// A Sentry report is displayed visually in the Sentry UI as follows: +// +//////////////// +// Title: (1) some prefix in bold (2) one line for a stack trace +// (3) a single-line subtitle +// +// (4) the tags, as a tag soup (concatenated in a single paragrah, +// unsorted) +// +// (5) a "message" +// +// (6) zero or more "exceptions", each composed of: +// (7) a bold title +// (8) some freeform text +// (9) a stack trace +// +// (10) metadata fields: environment, arch, etc +// +// (11) "Additional data" fields +// +// (12) SDK version +///////////////// +// +// These visual items map to the Sentry Event object as follows: +// +// (1) the Type field of the 1st Exception object, if any +// otherwise the Message field +// (2) the topmost entry from the Stacktrace field of the 1st Exception object, if any +// (3) the Value field of the 1st Exception object, if any, unwrapped as a single line +// (4) the Tags field +// (5) the Message field +// (7) the Type field (same as (1) for 1st execption) +// (8) the Value field (same as (3) for 1st exception) +// (9) the Stacktrace field (input to (2) on 1st exception) +// (10) the other fields on the Event object +// (11) the Extra field +// +// (Note how the top-level title fields (1) (3) are unrelated to the +// Message field in the event, which is surprising!) +// +// Given this mapping, an error object is decomposed as follows: +// +// (1)/(7): : () +// (3)/(8): : +// (4): not populated in this function, caller is to manage this +// (5): detailed structure of the entire error object, with references to "additional data" +// and additional "exception" objects +// (9): generated from innermost stack trace +// (6): every exception object after the 1st reports additional stack trace contexts +// (11): "additional data" populated from safe detail payloads +// +// If there is no stack trace in the error, a synthetic Exception +// object is still produced to provide visual detail in the Sentry UI. +// +// Note that if a layer in the error has both a stack trace (ie +// provides the `StackTrace()` interface) and also safe details +// (`SafeDetails()`) other than the stack trace, only the stack trace +// is included in the Sentry report. This does not affect error types +// provided by the library, but could impact error types defined by +// 3rd parties. This limitation may be lifted in a later version. +// func BuildSentryReport(err error) (*sentry.Event, map[string]interface{}) { return report.BuildSentryReport(err) } -// ReportError forwards a definition. +// ReportError reports the given error to Sentry. The caller is responsible for +// checking whether telemetry is enabled, and calling the sentry.Flush() +// function to wait for the report to be uploaded. (By default, +// Sentry submits reports asynchronously.) +// +// Note: an empty 'eventID' can be returned which signifies that the error was +// not reported. This can occur when Sentry client hasn't been properly +// configured or Sentry client decided to not report the error (due to +// configured sampling rate, callbacks, Sentry's event processors, etc). func ReportError(err error) string { return report.ReportError(err) } diff --git a/github.com/cockroachdb/errors/safedetails_api.go b/github.com/cockroachdb/errors/safedetails_api.go index e45361ee52..4caac9d4f2 100644 --- a/github.com/cockroachdb/errors/safedetails_api.go +++ b/github.com/cockroachdb/errors/safedetails_api.go @@ -19,19 +19,38 @@ import ( "github.com/cockroachdb/redact" ) -// WithSafeDetails forwards a definition. +// WithSafeDetails annotates an error with the given reportable details. +// The format is made available as a PII-free string, alongside +// with a PII-free representation of every additional argument. +// Arguments can be reported as-is (without redaction) by wrapping +// them using the Safe() function. +// +// If the format is empty and there are no arguments, the +// error argument is returned unchanged. +// +// Detail is shown: +// - via `errors.GetSafeDetails()` +// - when formatting with `%+v`. +// - in Sentry reports. func WithSafeDetails(err error, format string, args ...interface{}) error { return safedetails.WithSafeDetails(err, format, args...) } -// SafeMessager forwards a definition. -// NB: this is obsolete. Use redact.SafeFormatter -// or errors.SafeFormatter instead. +// SafeMessager aliases redact.SafeMessager. +// +// NB: this is obsolete. Use redact.SafeFormatter or +// errors.SafeFormatter instead. type SafeMessager = redact.SafeMessager -// Safe forwards a definition. +// Safe wraps the given object into an opaque struct that implements +// SafeMessager: its contents can be included as-is in PII-free +// strings in error objects and reports. +// +// NB: this is obsolete. Use redact.Safe instead. func Safe(v interface{}) redact.SafeValue { return safedetails.Safe(v) } -// Redact returns a redacted version of the supplied item that is safe -// to use in anonymized reporting. +// Redact returns a redacted version of the supplied item that is safe to use in +// anonymized reporting. +// +// NB: this interface is obsolete. Use redact.Sprint() directly. func Redact(r interface{}) string { return safedetails.Redact(r) } diff --git a/github.com/cockroachdb/errors/secondary_api.go b/github.com/cockroachdb/errors/secondary_api.go index 3b74d9a832..2f5e34f39b 100644 --- a/github.com/cockroachdb/errors/secondary_api.go +++ b/github.com/cockroachdb/errors/secondary_api.go @@ -16,12 +16,27 @@ package errors import "github.com/cockroachdb/errors/secondary" -// WithSecondaryError forwards a definition. +// WithSecondaryError enhances the error given as first argument with +// an annotation that carries the error given as second argument. The +// second error does not participate in cause analysis (Is, etc) and +// is only revealed when printing out the error or collecting safe +// (PII-free) details for reporting. +// +// If additionalErr is nil, the first error is returned as-is. +// +// Tip: consider using CombineErrors() below in the general case. +// +// Detail is shown: +// - via `errors.GetSafeDetails()`, shows details from secondary error. +// - when formatting with `%+v`. +// - in Sentry reports. func WithSecondaryError(err error, additionalErr error) error { return secondary.WithSecondaryError(err, additionalErr) } -// CombineErrors forwards a definition. +// CombineErrors returns err, or, if err is nil, otherErr. +// if err is non-nil, otherErr is attached as secondary error. +// See the documentation of `WithSecondaryError()` for details. func CombineErrors(err, otherErr error) error { return secondary.CombineErrors(err, otherErr) } diff --git a/github.com/cockroachdb/errors/telemetrykeys/with_telemetry.go b/github.com/cockroachdb/errors/telemetrykeys/with_telemetry.go index 2f7e5f180f..d495329c4a 100644 --- a/github.com/cockroachdb/errors/telemetrykeys/with_telemetry.go +++ b/github.com/cockroachdb/errors/telemetrykeys/with_telemetry.go @@ -32,7 +32,7 @@ type withTelemetry struct { var _ error = (*withTelemetry)(nil) var _ errbase.SafeDetailer = (*withTelemetry)(nil) var _ fmt.Formatter = (*withTelemetry)(nil) -var _ errbase.Formatter = (*withTelemetry)(nil) +var _ errbase.SafeFormatter = (*withTelemetry)(nil) func (w *withTelemetry) Error() string { return w.cause.Error() } func (w *withTelemetry) Cause() error { return w.cause } @@ -42,7 +42,7 @@ func (w *withTelemetry) SafeDetails() []string { return w.keys } func (w *withTelemetry) Format(s fmt.State, verb rune) { errbase.FormatError(w, s, verb) } -func (w *withTelemetry) FormatError(p errbase.Printer) (next error) { +func (w *withTelemetry) SafeFormatError(p errbase.Printer) (next error) { if p.Detail() { p.Printf("keys: [%s]", redact.Safe(strings.Join(w.keys, " "))) } diff --git a/github.com/cockroachdb/errors/telemetrykeys_api.go b/github.com/cockroachdb/errors/telemetrykeys_api.go index 0ccb615a69..145d9b9974 100644 --- a/github.com/cockroachdb/errors/telemetrykeys_api.go +++ b/github.com/cockroachdb/errors/telemetrykeys_api.go @@ -16,8 +16,17 @@ package errors import "github.com/cockroachdb/errors/telemetrykeys" -// WithTelemetry forwards a definition. +// WithTelemetry annotates err with the given telemetry key(s). +// The telemetry keys must be PII-free. +// +// Detail is shown: +// - via `errors.GetSafeDetails()`. +// - via `GetTelemetryKeys()` below. +// - when formatting with `%+v`. +// - in Sentry reports. func WithTelemetry(err error, keys ...string) error { return telemetrykeys.WithTelemetry(err, keys...) } -// GetTelemetryKeys forwards a definition. +// GetTelemetryKeys retrieves the (de-duplicated) set of +// all telemetry keys present in the direct causal chain +// of the error. The keys may not be sorted. func GetTelemetryKeys(err error) []string { return telemetrykeys.GetTelemetryKeys(err) } diff --git a/github.com/cockroachdb/errors/withstack_api.go b/github.com/cockroachdb/errors/withstack_api.go index 1da56f576c..db6eb25c03 100644 --- a/github.com/cockroachdb/errors/withstack_api.go +++ b/github.com/cockroachdb/errors/withstack_api.go @@ -16,21 +16,53 @@ package errors import "github.com/cockroachdb/errors/withstack" -// WithStack forwards a definition. +// This file mirrors the WithStack functionality from +// github.com/pkg/errors. We would prefer to reuse the withStack +// struct from that package directly (the library recognizes it well) +// unfortunately github.com/pkg/errors does not enable client code to +// customize the depth at which the stack trace is captured. + +// WithStack annotates err with a stack trace at the point WithStack was called. +// +// Detail is shown: +// - via `errors.GetSafeDetails()` +// - when formatting with `%+v`. +// - in Sentry reports. +// - when innermost stack capture, with `errors.GetOneLineSource()`. func WithStack(err error) error { return withstack.WithStackDepth(err, 1) } -// WithStackDepth forwards a definition. +// WithStackDepth annotates err with a stack trace starting from the +// given call depth. The value zero identifies the caller +// of WithStackDepth itself. +// See the documentation of WithStack() for more details. func WithStackDepth(err error, depth int) error { return withstack.WithStackDepth(err, depth+1) } -// ReportableStackTrace forwards a definition. +// ReportableStackTrace aliases the type of the same name in the sentry +// package. This is used by SendReport(). type ReportableStackTrace = withstack.ReportableStackTrace -// GetOneLineSource forwards a definition. +// GetOneLineSource extracts the file/line/function information +// of the topmost caller in the innermost recorded stack trace. +// The filename is simplified to remove the path prefix. +// +// This is used e.g. to populate the "source" field in +// PostgreSQL errors in CockroachDB. func GetOneLineSource(err error) (file string, line int, fn string, ok bool) { return withstack.GetOneLineSource(err) } -// GetReportableStackTrace forwards a definition. +// GetReportableStackTrace extracts a stack trace embedded in the +// given error in the format suitable for Sentry reporting. +// +// This supports: +// - errors generated by github.com/pkg/errors (either generated +// locally or after transfer through the network), +// - errors generated with WithStack() in this package, +// - any other error that implements a StackTrace() method +// returning a StackTrace from github.com/pkg/errors. +// +// Note: Sentry wants the oldest call frame first, so +// the entries are reversed in the result. func GetReportableStackTrace(err error) *ReportableStackTrace { return withstack.GetReportableStackTrace(err) } diff --git a/github.com/cockroachdb/redact/README.md b/github.com/cockroachdb/redact/README.md index 293b96871a..38c248af36 100644 --- a/github.com/cockroachdb/redact/README.md +++ b/github.com/cockroachdb/redact/README.md @@ -1,2 +1,5 @@ # redact Utilities to redact Go strings for confidentiality + +Godoc link: https://pkg.go.dev/github.com/cockroachdb/redact?tab=doc + diff --git a/github.com/cockroachdb/redact/builder.go b/github.com/cockroachdb/redact/builder.go new file mode 100644 index 0000000000..05efd94d26 --- /dev/null +++ b/github.com/cockroachdb/redact/builder.go @@ -0,0 +1,139 @@ +// Copyright 2020 The Cockroach Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package redact + +import ( + "bytes" + "io" + "unicode/utf8" +) + +// StringBuilder accumulates strings with optional redaction markers. +// +// It implements io.Writer but marks direct writes as redactable. +// To distinguish safe and unsafe bits, it also implements the SafeWriter +// interface. +type StringBuilder struct { + // we use bytes.Buffer internally to simplify the implementation of + // the SafeWriter interface. + buf bytes.Buffer +} + +// String returns the accumulated string, with redaction markers stripped out. +// To obtain the redactable string, call RedactableString(). +func (b *StringBuilder) String() string { return b.RedactableString().StripMarkers() } + +// RedactableString returns the accumulated string, including redaction markers. +func (b *StringBuilder) RedactableString() RedactableString { return RedactableString(b.buf.String()) } + +// SafeFormat implements SafeFormatter. +func (b *StringBuilder) SafeFormat(p SafePrinter, _ rune) { + // We only support the %v / %s natural print here. + // Go supports other formatting verbs for strings: %x/%X/%q. + // + // We don't do this here, keeping in mind that the output + // of a SafeFormat must remain a redactable string. + // + // %x/%X cannot be implemented because they would turn redaction + // markers into hex codes, and the entire result string would + // appear safe for reporting, which would break the semantics + // of this package. + // + // %q cannot be implemented because it replaces non-ASCII characters + // with numeric unicode escapes, which breaks redaction + // markers too. + p.Print(b.RedactableString()) +} + +// Len returns the number of accumulated bytes, including redaction +// markers; b.Len() == len(b.RedactableString()). +func (b *StringBuilder) Len() int { return b.buf.Len() } + +// Cap returns the capacity of the builder's underlying byte slice. It is the +// total space allocated for the string being built and includes any bytes +// already written. +func (b *StringBuilder) Cap() int { return b.buf.Cap() } + +// Reset resets the Builder to be empty. +func (b *StringBuilder) Reset() { b.buf.Reset() } + +// StringBuilder implements io.Writer. +// Direct Write() calls are considered unsafe. +var _ io.Writer = (*StringBuilder)(nil) + +// Write implements the io.Writer interface. +func (b *StringBuilder) Write(s []byte) (int, error) { + b.UnsafeBytes(s) + return len(s), nil +} + +// StringBuilder implements SafeWriter. +var _ SafeWriter = (*StringBuilder)(nil) + +// Print is part of the SafeWriter interface. +func (b *StringBuilder) Print(args ...interface{}) { + _, _ = Fprint(&b.buf, args...) +} + +// Printf is part of the SafeWriter interface. +func (b *StringBuilder) Printf(format string, args ...interface{}) { + _, _ = Fprintf(&b.buf, format, args...) +} + +// SafeString is part of the SafeWriter interface. +func (b *StringBuilder) SafeString(s SafeString) { + w := escapeWriter{w: &b.buf, enclose: false} + _, _ = w.Write([]byte(s)) +} + +// SafeRune is part of the SafeWriter interface. +func (b *StringBuilder) SafeRune(s SafeRune) { + if s == startRedactable || s == endRedactable { + s = escapeMark + } + _, _ = b.buf.WriteRune(rune(s)) +} + +// UnsafeString is part of the SafeWriter interface. +func (b *StringBuilder) UnsafeString(s string) { + w := escapeWriter{w: &b.buf, enclose: true, strip: true} + _, _ = w.Write([]byte(s)) +} + +// UnsafeRune is part of the SafeWriter interface. +func (b *StringBuilder) UnsafeRune(s rune) { + _, _ = b.buf.WriteRune(startRedactable) + b.SafeRune(SafeRune(s)) + _, _ = b.buf.WriteRune(endRedactable) +} + +// UnsafeByte is part of the SafeWriter interface. +func (b *StringBuilder) UnsafeByte(s byte) { + _, _ = b.buf.WriteRune(startRedactable) + if s >= utf8.RuneSelf || + s == startRedactableBytes[0] || s == endRedactableBytes[0] { + // Unsafe byte. Escape it. + _, _ = b.buf.Write(escapeBytes) + } else { + _ = b.buf.WriteByte(s) + } + _, _ = b.buf.WriteRune(endRedactable) +} + +// UnsafeBytes is part of the SafeWriter interface. +func (b *StringBuilder) UnsafeBytes(s []byte) { + w := escapeWriter{w: &b.buf, enclose: true, strip: true} + _, _ = w.Write(s) +} diff --git a/github.com/cockroachdb/redact/doc.go b/github.com/cockroachdb/redact/doc.go index a1b4274000..ff37065e7f 100644 --- a/github.com/cockroachdb/redact/doc.go +++ b/github.com/cockroachdb/redact/doc.go @@ -20,4 +20,32 @@ // the boundaries of the current system, for example via telemetry // or crash reporting. Conversely, data is considered “unsafe†// until/unless it is known to be “safeâ€. +// +// Example use: +// +// redactable := redact.Sprintf("hello %s", "universe") +// +// // At this point, 'redactable' contains "hello ‹universe›". +// +// // This prints "hello universe": +// fmt.Println(redactable.StripMarkers()) +// +// // This reports "hello ‹×›": +// fmt.Println(redactable.Redact()) +// +// +// When defining your own custom types, you can define +// a SafeFormat method, implementing the redact.SafeFormatter +// interface in a way you'd otherwise implement fmt.Formatter. +// This is then recognized by this package's API automatically. +// +// Alternatively: +// +// - you can implement the SafeValue interface, which tells the +// redact package to always the default formatting of a type +// as safe and thus not included inside redaction markers. +// +// - you can include a value within redact.Safe() and redact.Unsafe() +// in redact.Sprintf / redact.Fprintf calls, to force +// the omission or inclusion of redaction markers. package redact diff --git a/github.com/cockroachdb/redact/internal/README.md b/github.com/cockroachdb/redact/internal/README.md new file mode 100644 index 0000000000..768a89545f --- /dev/null +++ b/github.com/cockroachdb/redact/internal/README.md @@ -0,0 +1,52 @@ +Overall code structure +====================== + +This directory imports the "printf" logic as-is from the Go standard +library and instruments it for redaction of sensitive data. + +Overall, the original Go code is structured as follows: + +- the top-level API functions (`Printf` `Fprintf` etc) instantiate a + "printer" struct called `pp`, then call the `doPrint*()` methods + on it. + +- the `pp` contains a byte slice (`pp.buf`, type `buffer`) that + accumulates the result of formatting *regardless of the final output + of the API* - i.e. a buffer is used even when using `Fprint` to an + `io.Writer`. Only after the `doPrint()` method finishes, is the + `buffer` copied to the final `io.Writer` in the `Fprint*` variants. + +- each of the `doPrint` methods does some analysis on the argument + list - quite simple for e.g. `doPrintln`, more intricate for + `doPrintf`. As part of the analysis it emits "spacing" or no-op + bytes directly on the `pp.buf`. For example `doPrint` emits spaces + between arguments directly, `doPrintf` emits the non-formatting + characters from the format string, as well as certain *constant* + error strings (e.g. `%(BADPREC)`). + +At this point **we make the following assumption**: there is no data +from the printed *arguments* that is emitted to `pp.buf` directly by a +`doPrint()` method. This is true of the implementation as of Go 1.14. + +From this point, this package contains a *custom patch* onto the +default implementation: the calls to `p.printArg` performed by the +`doPrint()` methods are redirected to a function callback that is +injected by the `redact` package. + +Refreshing the sources +====================== + +The files in this directory have been imported from + +`$GOROOT/src/fmt/{format,print}.go` + +and + +`$GOROOT/src/internal/fmtsort` + +And patched using the included `.diff` files. + +To upgrade to a newer Go implementation, import the files anew and +re-apply the patches. + +See the script `refresh.sh` for details. diff --git a/github.com/cockroachdb/redact/internal/fmtsort/sort.go b/github.com/cockroachdb/redact/internal/fmtsort/sort.go new file mode 100644 index 0000000000..9d2d580aa8 --- /dev/null +++ b/github.com/cockroachdb/redact/internal/fmtsort/sort.go @@ -0,0 +1,222 @@ +// Code generated from the Go standard library. DO NOT EDIT +// GENERATED FILE DO NOT EDIT +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package fmtsort provides a general stable ordering mechanism +// for maps, on behalf of the fmt and text/template packages. +// It is not guaranteed to be efficient and works only for types +// that are valid map keys. +package fmtsort + +import ( + "reflect" + "sort" +) + +// Note: Throughout this package we avoid calling reflect.Value.Interface as +// it is not always legal to do so and it's easier to avoid the issue than to face it. + +// SortedMap represents a map's keys and values. The keys and values are +// aligned in index order: Value[i] is the value in the map corresponding to Key[i]. +type SortedMap struct { + Key []reflect.Value + Value []reflect.Value +} + +func (o *SortedMap) Len() int { return len(o.Key) } +func (o *SortedMap) Less(i, j int) bool { return compare(o.Key[i], o.Key[j]) < 0 } +func (o *SortedMap) Swap(i, j int) { + o.Key[i], o.Key[j] = o.Key[j], o.Key[i] + o.Value[i], o.Value[j] = o.Value[j], o.Value[i] +} + +// Sort accepts a map and returns a SortedMap that has the same keys and +// values but in a stable sorted order according to the keys, modulo issues +// raised by unorderable key values such as NaNs. +// +// The ordering rules are more general than with Go's < operator: +// +// - when applicable, nil compares low +// - ints, floats, and strings order by < +// - NaN compares less than non-NaN floats +// - bool compares false before true +// - complex compares real, then imag +// - pointers compare by machine address +// - channel values compare by machine address +// - structs compare each field in turn +// - arrays compare each element in turn. +// Otherwise identical arrays compare by length. +// - interface values compare first by reflect.Type describing the concrete type +// and then by concrete value as described in the previous rules. +// +func Sort(mapValue reflect.Value) *SortedMap { + if mapValue.Type().Kind() != reflect.Map { + return nil + } + // Note: this code is arranged to not panic even in the presence + // of a concurrent map update. The runtime is responsible for + // yelling loudly if that happens. See issue 33275. + n := mapValue.Len() + key := make([]reflect.Value, 0, n) + value := make([]reflect.Value, 0, n) + iter := mapValue.MapRange() + for iter.Next() { + key = append(key, iter.Key()) + value = append(value, iter.Value()) + } + sorted := &SortedMap{ + Key: key, + Value: value, + } + sort.Stable(sorted) + return sorted +} + +// compare compares two values of the same type. It returns -1, 0, 1 +// according to whether a > b (1), a == b (0), or a < b (-1). +// If the types differ, it returns -1. +// See the comment on Sort for the comparison rules. +func compare(aVal, bVal reflect.Value) int { + aType, bType := aVal.Type(), bVal.Type() + if aType != bType { + return -1 // No good answer possible, but don't return 0: they're not equal. + } + switch aVal.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + a, b := aVal.Int(), bVal.Int() + switch { + case a < b: + return -1 + case a > b: + return 1 + default: + return 0 + } + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + a, b := aVal.Uint(), bVal.Uint() + switch { + case a < b: + return -1 + case a > b: + return 1 + default: + return 0 + } + case reflect.String: + a, b := aVal.String(), bVal.String() + switch { + case a < b: + return -1 + case a > b: + return 1 + default: + return 0 + } + case reflect.Float32, reflect.Float64: + return floatCompare(aVal.Float(), bVal.Float()) + case reflect.Complex64, reflect.Complex128: + a, b := aVal.Complex(), bVal.Complex() + if c := floatCompare(real(a), real(b)); c != 0 { + return c + } + return floatCompare(imag(a), imag(b)) + case reflect.Bool: + a, b := aVal.Bool(), bVal.Bool() + switch { + case a == b: + return 0 + case a: + return 1 + default: + return -1 + } + case reflect.Ptr: + a, b := aVal.Pointer(), bVal.Pointer() + switch { + case a < b: + return -1 + case a > b: + return 1 + default: + return 0 + } + case reflect.Chan: + if c, ok := nilCompare(aVal, bVal); ok { + return c + } + ap, bp := aVal.Pointer(), bVal.Pointer() + switch { + case ap < bp: + return -1 + case ap > bp: + return 1 + default: + return 0 + } + case reflect.Struct: + for i := 0; i < aVal.NumField(); i++ { + if c := compare(aVal.Field(i), bVal.Field(i)); c != 0 { + return c + } + } + return 0 + case reflect.Array: + for i := 0; i < aVal.Len(); i++ { + if c := compare(aVal.Index(i), bVal.Index(i)); c != 0 { + return c + } + } + return 0 + case reflect.Interface: + if c, ok := nilCompare(aVal, bVal); ok { + return c + } + c := compare(reflect.ValueOf(aVal.Elem().Type()), reflect.ValueOf(bVal.Elem().Type())) + if c != 0 { + return c + } + return compare(aVal.Elem(), bVal.Elem()) + default: + // Certain types cannot appear as keys (maps, funcs, slices), but be explicit. + panic("bad type in compare: " + aType.String()) + } +} + +// nilCompare checks whether either value is nil. If not, the boolean is false. +// If either value is nil, the boolean is true and the integer is the comparison +// value. The comparison is defined to be 0 if both are nil, otherwise the one +// nil value compares low. Both arguments must represent a chan, func, +// interface, map, pointer, or slice. +func nilCompare(aVal, bVal reflect.Value) (int, bool) { + if aVal.IsNil() { + if bVal.IsNil() { + return 0, true + } + return -1, true + } + if bVal.IsNil() { + return 1, true + } + return 0, false +} + +// floatCompare compares two floating-point values. NaNs compare low. +func floatCompare(a, b float64) int { + switch { + case isNaN(a): + return -1 // No good answer if b is a NaN so don't bother checking. + case isNaN(b): + return 1 + case a < b: + return -1 + case a > b: + return 1 + } + return 0 +} + +func isNaN(a float64) bool { + return a != a +} diff --git a/github.com/cockroachdb/redact/internal/format.go b/github.com/cockroachdb/redact/internal/format.go new file mode 100644 index 0000000000..1082879b5b --- /dev/null +++ b/github.com/cockroachdb/redact/internal/format.go @@ -0,0 +1,584 @@ +// Code generated from the Go standard library. DO NOT EDIT +// GENERATED FILE DO NOT EDIT +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fmt + +import ( + "strconv" + "unicode/utf8" +) + +const ( + ldigits = "0123456789abcdefx" + udigits = "0123456789ABCDEFX" +) + +const ( + signed = true + unsigned = false +) + +// flags placed in a separate struct for easy clearing. +type fmtFlags struct { + widPresent bool + precPresent bool + minus bool + plus bool + sharp bool + space bool + zero bool + + // For the formats %+v %#v, we set the plusV/sharpV flags + // and clear the plus/sharp flags since %+v and %#v are in effect + // different, flagless formats set at the top level. + plusV bool + sharpV bool +} + +// A fmt is the raw formatter used by Printf etc. +// It prints into a buffer that must be set up separately. +type fmt struct { + buf *buffer + + fmtFlags + + wid int // width + prec int // precision + + // intbuf is large enough to store %b of an int64 with a sign and + // avoids padding at the end of the struct on 32 bit architectures. + intbuf [68]byte +} + +func (f *fmt) clearflags() { + f.fmtFlags = fmtFlags{} +} + +func (f *fmt) init(buf *buffer) { + f.buf = buf + f.clearflags() +} + +// writePadding generates n bytes of padding. +func (f *fmt) writePadding(n int) { + if n <= 0 { // No padding bytes needed. + return + } + buf := *f.buf + oldLen := len(buf) + newLen := oldLen + n + // Make enough room for padding. + if newLen > cap(buf) { + buf = make(buffer, cap(buf)*2+n) + copy(buf, *f.buf) + } + // Decide which byte the padding should be filled with. + padByte := byte(' ') + if f.zero { + padByte = byte('0') + } + // Fill padding with padByte. + padding := buf[oldLen:newLen] + for i := range padding { + padding[i] = padByte + } + *f.buf = buf[:newLen] +} + +// pad appends b to f.buf, padded on left (!f.minus) or right (f.minus). +func (f *fmt) pad(b []byte) { + if !f.widPresent || f.wid == 0 { + f.buf.write(b) + return + } + width := f.wid - utf8.RuneCount(b) + if !f.minus { + // left padding + f.writePadding(width) + f.buf.write(b) + } else { + // right padding + f.buf.write(b) + f.writePadding(width) + } +} + +// padString appends s to f.buf, padded on left (!f.minus) or right (f.minus). +func (f *fmt) padString(s string) { + if !f.widPresent || f.wid == 0 { + f.buf.writeString(s) + return + } + width := f.wid - utf8.RuneCountInString(s) + if !f.minus { + // left padding + f.writePadding(width) + f.buf.writeString(s) + } else { + // right padding + f.buf.writeString(s) + f.writePadding(width) + } +} + +// fmtBoolean formats a boolean. +func (f *fmt) fmtBoolean(v bool) { + if v { + f.padString("true") + } else { + f.padString("false") + } +} + +// fmtUnicode formats a uint64 as "U+0078" or with f.sharp set as "U+0078 'x'". +func (f *fmt) fmtUnicode(u uint64) { + buf := f.intbuf[0:] + + // With default precision set the maximum needed buf length is 18 + // for formatting -1 with %#U ("U+FFFFFFFFFFFFFFFF") which fits + // into the already allocated intbuf with a capacity of 68 bytes. + prec := 4 + if f.precPresent && f.prec > 4 { + prec = f.prec + // Compute space needed for "U+" , number, " '", character, "'". + width := 2 + prec + 2 + utf8.UTFMax + 1 + if width > len(buf) { + buf = make([]byte, width) + } + } + + // Format into buf, ending at buf[i]. Formatting numbers is easier right-to-left. + i := len(buf) + + // For %#U we want to add a space and a quoted character at the end of the buffer. + if f.sharp && u <= utf8.MaxRune && strconv.IsPrint(rune(u)) { + i-- + buf[i] = '\'' + i -= utf8.RuneLen(rune(u)) + utf8.EncodeRune(buf[i:], rune(u)) + i-- + buf[i] = '\'' + i-- + buf[i] = ' ' + } + // Format the Unicode code point u as a hexadecimal number. + for u >= 16 { + i-- + buf[i] = udigits[u&0xF] + prec-- + u >>= 4 + } + i-- + buf[i] = udigits[u] + prec-- + // Add zeros in front of the number until requested precision is reached. + for prec > 0 { + i-- + buf[i] = '0' + prec-- + } + // Add a leading "U+". + i-- + buf[i] = '+' + i-- + buf[i] = 'U' + + oldZero := f.zero + f.zero = false + f.pad(buf[i:]) + f.zero = oldZero +} + +// fmtInteger formats signed and unsigned integers. +func (f *fmt) fmtInteger(u uint64, base int, isSigned bool, verb rune, digits string) { + negative := isSigned && int64(u) < 0 + if negative { + u = -u + } + + buf := f.intbuf[0:] + // The already allocated f.intbuf with a capacity of 68 bytes + // is large enough for integer formatting when no precision or width is set. + if f.widPresent || f.precPresent { + // Account 3 extra bytes for possible addition of a sign and "0x". + width := 3 + f.wid + f.prec // wid and prec are always positive. + if width > len(buf) { + // We're going to need a bigger boat. + buf = make([]byte, width) + } + } + + // Two ways to ask for extra leading zero digits: %.3d or %03d. + // If both are specified the f.zero flag is ignored and + // padding with spaces is used instead. + prec := 0 + if f.precPresent { + prec = f.prec + // Precision of 0 and value of 0 means "print nothing" but padding. + if prec == 0 && u == 0 { + oldZero := f.zero + f.zero = false + f.writePadding(f.wid) + f.zero = oldZero + return + } + } else if f.zero && f.widPresent { + prec = f.wid + if negative || f.plus || f.space { + prec-- // leave room for sign + } + } + + // Because printing is easier right-to-left: format u into buf, ending at buf[i]. + // We could make things marginally faster by splitting the 32-bit case out + // into a separate block but it's not worth the duplication, so u has 64 bits. + i := len(buf) + // Use constants for the division and modulo for more efficient code. + // Switch cases ordered by popularity. + switch base { + case 10: + for u >= 10 { + i-- + next := u / 10 + buf[i] = byte('0' + u - next*10) + u = next + } + case 16: + for u >= 16 { + i-- + buf[i] = digits[u&0xF] + u >>= 4 + } + case 8: + for u >= 8 { + i-- + buf[i] = byte('0' + u&7) + u >>= 3 + } + case 2: + for u >= 2 { + i-- + buf[i] = byte('0' + u&1) + u >>= 1 + } + default: + panic("fmt: unknown base; can't happen") + } + i-- + buf[i] = digits[u] + for i > 0 && prec > len(buf)-i { + i-- + buf[i] = '0' + } + + // Various prefixes: 0x, -, etc. + if f.sharp { + switch base { + case 2: + // Add a leading 0b. + i-- + buf[i] = 'b' + i-- + buf[i] = '0' + case 8: + if buf[i] != '0' { + i-- + buf[i] = '0' + } + case 16: + // Add a leading 0x or 0X. + i-- + buf[i] = digits[16] + i-- + buf[i] = '0' + } + } + if verb == 'O' { + i-- + buf[i] = 'o' + i-- + buf[i] = '0' + } + + if negative { + i-- + buf[i] = '-' + } else if f.plus { + i-- + buf[i] = '+' + } else if f.space { + i-- + buf[i] = ' ' + } + + // Left padding with zeros has already been handled like precision earlier + // or the f.zero flag is ignored due to an explicitly set precision. + oldZero := f.zero + f.zero = false + f.pad(buf[i:]) + f.zero = oldZero +} + +// truncate truncates the string s to the specified precision, if present. +func (f *fmt) truncateString(s string) string { + if f.precPresent { + n := f.prec + for i := range s { + n-- + if n < 0 { + return s[:i] + } + } + } + return s +} + +// truncate truncates the byte slice b as a string of the specified precision, if present. +func (f *fmt) truncate(b []byte) []byte { + if f.precPresent { + n := f.prec + for i := 0; i < len(b); { + n-- + if n < 0 { + return b[:i] + } + wid := 1 + if b[i] >= utf8.RuneSelf { + _, wid = utf8.DecodeRune(b[i:]) + } + i += wid + } + } + return b +} + +// fmtS formats a string. +func (f *fmt) fmtS(s string) { + s = f.truncateString(s) + f.padString(s) +} + +// fmtBs formats the byte slice b as if it was formatted as string with fmtS. +func (f *fmt) fmtBs(b []byte) { + b = f.truncate(b) + f.pad(b) +} + +// fmtSbx formats a string or byte slice as a hexadecimal encoding of its bytes. +func (f *fmt) fmtSbx(s string, b []byte, digits string) { + length := len(b) + if b == nil { + // No byte slice present. Assume string s should be encoded. + length = len(s) + } + // Set length to not process more bytes than the precision demands. + if f.precPresent && f.prec < length { + length = f.prec + } + // Compute width of the encoding taking into account the f.sharp and f.space flag. + width := 2 * length + if width > 0 { + if f.space { + // Each element encoded by two hexadecimals will get a leading 0x or 0X. + if f.sharp { + width *= 2 + } + // Elements will be separated by a space. + width += length - 1 + } else if f.sharp { + // Only a leading 0x or 0X will be added for the whole string. + width += 2 + } + } else { // The byte slice or string that should be encoded is empty. + if f.widPresent { + f.writePadding(f.wid) + } + return + } + // Handle padding to the left. + if f.widPresent && f.wid > width && !f.minus { + f.writePadding(f.wid - width) + } + // Write the encoding directly into the output buffer. + buf := *f.buf + if f.sharp { + // Add leading 0x or 0X. + buf = append(buf, '0', digits[16]) + } + var c byte + for i := 0; i < length; i++ { + if f.space && i > 0 { + // Separate elements with a space. + buf = append(buf, ' ') + if f.sharp { + // Add leading 0x or 0X for each element. + buf = append(buf, '0', digits[16]) + } + } + if b != nil { + c = b[i] // Take a byte from the input byte slice. + } else { + c = s[i] // Take a byte from the input string. + } + // Encode each byte as two hexadecimal digits. + buf = append(buf, digits[c>>4], digits[c&0xF]) + } + *f.buf = buf + // Handle padding to the right. + if f.widPresent && f.wid > width && f.minus { + f.writePadding(f.wid - width) + } +} + +// fmtSx formats a string as a hexadecimal encoding of its bytes. +func (f *fmt) fmtSx(s, digits string) { + f.fmtSbx(s, nil, digits) +} + +// fmtBx formats a byte slice as a hexadecimal encoding of its bytes. +func (f *fmt) fmtBx(b []byte, digits string) { + f.fmtSbx("", b, digits) +} + +// fmtQ formats a string as a double-quoted, escaped Go string constant. +// If f.sharp is set a raw (backquoted) string may be returned instead +// if the string does not contain any control characters other than tab. +func (f *fmt) fmtQ(s string) { + s = f.truncateString(s) + if f.sharp && strconv.CanBackquote(s) { + f.padString("`" + s + "`") + return + } + buf := f.intbuf[:0] + if f.plus { + f.pad(strconv.AppendQuoteToASCII(buf, s)) + } else { + f.pad(strconv.AppendQuote(buf, s)) + } +} + +// fmtC formats an integer as a Unicode character. +// If the character is not valid Unicode, it will print '\ufffd'. +func (f *fmt) fmtC(c uint64) { + r := rune(c) + if c > utf8.MaxRune { + r = utf8.RuneError + } + buf := f.intbuf[:0] + w := utf8.EncodeRune(buf[:utf8.UTFMax], r) + f.pad(buf[:w]) +} + +// fmtQc formats an integer as a single-quoted, escaped Go character constant. +// If the character is not valid Unicode, it will print '\ufffd'. +func (f *fmt) fmtQc(c uint64) { + r := rune(c) + if c > utf8.MaxRune { + r = utf8.RuneError + } + buf := f.intbuf[:0] + if f.plus { + f.pad(strconv.AppendQuoteRuneToASCII(buf, r)) + } else { + f.pad(strconv.AppendQuoteRune(buf, r)) + } +} + +// fmtFloat formats a float64. It assumes that verb is a valid format specifier +// for strconv.AppendFloat and therefore fits into a byte. +func (f *fmt) fmtFloat(v float64, size int, verb rune, prec int) { + // Explicit precision in format specifier overrules default precision. + if f.precPresent { + prec = f.prec + } + // Format number, reserving space for leading + sign if needed. + num := strconv.AppendFloat(f.intbuf[:1], v, byte(verb), prec, size) + if num[1] == '-' || num[1] == '+' { + num = num[1:] + } else { + num[0] = '+' + } + // f.space means to add a leading space instead of a "+" sign unless + // the sign is explicitly asked for by f.plus. + if f.space && num[0] == '+' && !f.plus { + num[0] = ' ' + } + // Special handling for infinities and NaN, + // which don't look like a number so shouldn't be padded with zeros. + if num[1] == 'I' || num[1] == 'N' { + oldZero := f.zero + f.zero = false + // Remove sign before NaN if not asked for. + if num[1] == 'N' && !f.space && !f.plus { + num = num[1:] + } + f.pad(num) + f.zero = oldZero + return + } + // The sharp flag forces printing a decimal point for non-binary formats + // and retains trailing zeros, which we may need to restore. + if f.sharp && verb != 'b' { + digits := 0 + switch verb { + case 'v', 'g', 'G', 'x': + digits = prec + // If no precision is set explicitly use a precision of 6. + if digits == -1 { + digits = 6 + } + } + + // Buffer pre-allocated with enough room for + // exponent notations of the form "e+123" or "p-1023". + var tailBuf [6]byte + tail := tailBuf[:0] + + hasDecimalPoint := false + // Starting from i = 1 to skip sign at num[0]. + for i := 1; i < len(num); i++ { + switch num[i] { + case '.': + hasDecimalPoint = true + case 'p', 'P': + tail = append(tail, num[i:]...) + num = num[:i] + case 'e', 'E': + if verb != 'x' && verb != 'X' { + tail = append(tail, num[i:]...) + num = num[:i] + break + } + fallthrough + default: + digits-- + } + } + if !hasDecimalPoint { + num = append(num, '.') + } + for digits > 0 { + num = append(num, '0') + digits-- + } + num = append(num, tail...) + } + // We want a sign if asked for and if the sign is not positive. + if f.plus || num[0] != '+' { + // If we're zero padding to the left we want the sign before the leading zeros. + // Achieve this by writing the sign out and then padding the unsigned number. + if f.zero && f.widPresent && f.wid > len(num) { + f.buf.writeByte(num[0]) + f.writePadding(f.wid - len(num)) + f.buf.write(num[1:]) + return + } + f.pad(num) + return + } + // No sign to show and the number is positive; just print the unsigned number. + f.pad(num[1:]) +} diff --git a/github.com/cockroachdb/redact/internal/hooks.go b/github.com/cockroachdb/redact/internal/hooks.go new file mode 100644 index 0000000000..246e600de6 --- /dev/null +++ b/github.com/cockroachdb/redact/internal/hooks.go @@ -0,0 +1,83 @@ +// Copyright 2020 The Cockroach Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +package fmt + +// printArg overrides the original implementation by redirecting the +// call to a substitute function in the printer, if available. +func (p *pp) printArg(a interface{}, verb rune) { + if p.printArgSubstituteFn != nil { + p.substituteState = p.printArgSubstituteFn(p, a, verb) + } else { + p.printArgOrig(a, verb) + } +} + +// InternalPrinter exposes pp to the redact package. +type InternalPrinter = pp + +// InternalBuffer exposes buffer to the redact package. +type InternalBuffer = buffer + +// NewInternalPrinter exposes pp allocation to the redact package. +func NewInternalPrinter() *InternalPrinter { return newPrinter() } + +// SetHook connects an outer printer to the inner printer. +func SetHook( + p *InternalPrinter, fn func(p *InternalPrinter, arg interface{}, verb rune) (newState int), +) { + p.printArgSubstituteFn = fn +} + +// Free exposes pp deallocation to the redact package. +func Free(p *InternalPrinter) { + p.substituteState = 0 + p.printArgSubstituteFn = nil + p.free() +} + +// Buf exposes the string buffer to the redact package. +func Buf(p *InternalPrinter) []byte { return []byte(p.buf) } + +// GetState exposes the state to the redact package. +func GetState(p *InternalPrinter) int { + return p.substituteState +} + +// SetState exposes the string buffer to the redact package. +func SetState(p *InternalPrinter, b []byte) { + p.buf = buffer(b) + p.substituteState = len(p.buf) +} + +// Append adds bytes to the buffer. +func Append(p *InternalPrinter, b []byte) { p.buf.write(b) } + +// PrintArg exposes the printArgOrig() method to the redact package. +func PrintArg(p *InternalPrinter, a interface{}, verb rune) { p.printArgOrig(a, verb) } + +// DoPrint exposes the doPrint() method to the redact package. +func DoPrint(p *InternalPrinter, a []interface{}) { p.doPrint(a) } + +// DoPrintf exposes the doPrintf() method to the redact package. +func DoPrintf(p *InternalPrinter, format string, a []interface{}) { p.doPrintf(format, a) } + +// SetCollectError enables wrapped error collection. +func SetCollectError(p *InternalPrinter) { p.wrapErrs = true } + +// CollectingError exposes wrapped error collection. +func CollectingError(p *InternalPrinter) bool { return p.wrapErrs } + +// WrappedError retrieves the wrapped error if found. +func WrappedError(p *InternalPrinter) error { return p.wrappedErr } diff --git a/github.com/cockroachdb/redact/internal/print.go b/github.com/cockroachdb/redact/internal/print.go new file mode 100644 index 0000000000..b7cdd3fa60 --- /dev/null +++ b/github.com/cockroachdb/redact/internal/print.go @@ -0,0 +1,1189 @@ +// Code generated from print.go.orig. DO NOT EDIT +// GENERATED FILE DO NOT EDIT +// +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fmt + +import ( + // CUSTOM: needed to avoid a type mismatch on Formatter. + origFmt "fmt" + "io" + "os" + "reflect" + "sync" + "unicode/utf8" + + // CUSTOM: our own import since we can't use internal from go stdlib. + "github.com/cockroachdb/redact/internal/fmtsort" +) + +// Strings for use with buffer.WriteString. +// This is less overhead than using buffer.Write with byte arrays. +const ( + commaSpaceString = ", " + nilAngleString = "" + nilParenString = "(nil)" + nilString = "nil" + mapString = "map[" + percentBangString = "%!" + missingString = "(MISSING)" + badIndexString = "(BADINDEX)" + panicString = "(PANIC=" + extraString = "%!(EXTRA " + badWidthString = "%!(BADWIDTH)" + badPrecString = "%!(BADPREC)" + noVerbString = "%!(NOVERB)" + invReflectString = "" +) + +// State represents the printer state passed to custom formatters. +// It provides access to the io.Writer interface plus information about +// the flags and options for the operand's format specifier. +type State interface { + // Write is the function to call to emit formatted output to be printed. + Write(b []byte) (n int, err error) + // Width returns the value of the width option and whether it has been set. + Width() (wid int, ok bool) + // Precision returns the value of the precision option and whether it has been set. + Precision() (prec int, ok bool) + + // Flag reports whether the flag c, a character, has been set. + Flag(c int) bool +} + +// Formatter is the interface implemented by values with a custom formatter. +// The implementation of Format may call Sprint(f) or Fprint(f) etc. +// to generate its output. +type Formatter interface { + // CUSTOM: refer to the original type, not the one defined here. + Format(f origFmt.State, c rune) +} + +// Stringer is implemented by any value that has a String method, +// which defines the ``native'' format for that value. +// The String method is used to print values passed as an operand +// to any format that accepts a string or to an unformatted printer +// such as Print. +type Stringer interface { + String() string +} + +// GoStringer is implemented by any value that has a GoString method, +// which defines the Go syntax for that value. +// The GoString method is used to print values passed as an operand +// to a %#v format. +type GoStringer interface { + GoString() string +} + +// Use simple []byte instead of bytes.Buffer to avoid large dependency. +type buffer []byte + +func (b *buffer) write(p []byte) { + *b = append(*b, p...) +} + +func (b *buffer) writeString(s string) { + *b = append(*b, s...) +} + +func (b *buffer) writeByte(c byte) { + *b = append(*b, c) +} + +func (bp *buffer) writeRune(r rune) { + if r < utf8.RuneSelf { + *bp = append(*bp, byte(r)) + return + } + + b := *bp + n := len(b) + for n+utf8.UTFMax > cap(b) { + b = append(b, 0) + } + w := utf8.EncodeRune(b[n:n+utf8.UTFMax], r) + *bp = b[:n+w] +} + +// pp is used to store a printer's state and is reused with sync.Pool to avoid allocations. +type pp struct { + buf buffer + + // CUSTOM: hook fn for the redact package. + printArgSubstituteFn func(p *pp, a interface{}, verb rune) (newState int) + substituteState int + + // arg holds the current item, as an interface{}. + arg interface{} + + // value is used instead of arg for reflect values. + value reflect.Value + + // fmt is used to format basic items such as integers or strings. + fmt fmt + + // reordered records whether the format string used argument reordering. + reordered bool + // goodArgNum records whether the most recent reordering directive was valid. + goodArgNum bool + // panicking is set by catchPanic to avoid infinite panic, recover, panic, ... recursion. + panicking bool + // erroring is set when printing an error string to guard against calling handleMethods. + erroring bool + // wrapErrs is set when the format string may contain a %w verb. + wrapErrs bool + // wrappedErr records the target of the %w verb. + wrappedErr error +} + +var ppFree = sync.Pool{ + New: func() interface{} { return new(pp) }, +} + +// newPrinter allocates a new pp struct or grabs a cached one. +func newPrinter() *pp { + p := ppFree.Get().(*pp) + p.panicking = false + p.erroring = false + p.wrapErrs = false + p.fmt.init(&p.buf) + return p +} + +// free saves used pp structs in ppFree; avoids an allocation per invocation. +func (p *pp) free() { + // Proper usage of a sync.Pool requires each entry to have approximately + // the same memory cost. To obtain this property when the stored type + // contains a variably-sized buffer, we add a hard limit on the maximum buffer + // to place back in the pool. + // + // See https://golang.org/issue/23199 + if cap(p.buf) > 64<<10 { + return + } + + p.buf = p.buf[:0] + p.arg = nil + p.value = reflect.Value{} + p.wrappedErr = nil + ppFree.Put(p) +} + +func (p *pp) Width() (wid int, ok bool) { return p.fmt.wid, p.fmt.widPresent } + +func (p *pp) Precision() (prec int, ok bool) { return p.fmt.prec, p.fmt.precPresent } + +func (p *pp) Flag(b int) bool { + switch b { + case '-': + return p.fmt.minus + case '+': + return p.fmt.plus || p.fmt.plusV + case '#': + return p.fmt.sharp || p.fmt.sharpV + case ' ': + return p.fmt.space + case '0': + return p.fmt.zero + } + return false +} + +// Implement Write so we can call Fprintf on a pp (through State), for +// recursive use in custom verbs. +func (p *pp) Write(b []byte) (ret int, err error) { + p.buf.write(b) + return len(b), nil +} + +// Implement WriteString so that we can call io.WriteString +// on a pp (through state), for efficiency. +func (p *pp) WriteString(s string) (ret int, err error) { + p.buf.writeString(s) + return len(s), nil +} + +// These routines end in 'f' and take a format string. + +// Fprintf formats according to a format specifier and writes to w. +// It returns the number of bytes written and any write error encountered. +func Fprintf(w io.Writer, format string, a ...interface{}) (n int, err error) { + p := newPrinter() + p.doPrintf(format, a) + n, err = w.Write(p.buf) + p.free() + return +} + +// Printf formats according to a format specifier and writes to standard output. +// It returns the number of bytes written and any write error encountered. +func Printf(format string, a ...interface{}) (n int, err error) { + return Fprintf(os.Stdout, format, a...) +} + +// Sprintf formats according to a format specifier and returns the resulting string. +func Sprintf(format string, a ...interface{}) string { + p := newPrinter() + p.doPrintf(format, a) + s := string(p.buf) + p.free() + return s +} + +// These routines do not take a format string + +// Fprint formats using the default formats for its operands and writes to w. +// Spaces are added between operands when neither is a string. +// It returns the number of bytes written and any write error encountered. +func Fprint(w io.Writer, a ...interface{}) (n int, err error) { + p := newPrinter() + p.doPrint(a) + n, err = w.Write(p.buf) + p.free() + return +} + +// Print formats using the default formats for its operands and writes to standard output. +// Spaces are added between operands when neither is a string. +// It returns the number of bytes written and any write error encountered. +func Print(a ...interface{}) (n int, err error) { + return Fprint(os.Stdout, a...) +} + +// Sprint formats using the default formats for its operands and returns the resulting string. +// Spaces are added between operands when neither is a string. +func Sprint(a ...interface{}) string { + p := newPrinter() + p.doPrint(a) + s := string(p.buf) + p.free() + return s +} + +// These routines end in 'ln', do not take a format string, +// always add spaces between operands, and add a newline +// after the last operand. + +// Fprintln formats using the default formats for its operands and writes to w. +// Spaces are always added between operands and a newline is appended. +// It returns the number of bytes written and any write error encountered. +func Fprintln(w io.Writer, a ...interface{}) (n int, err error) { + p := newPrinter() + p.doPrintln(a) + n, err = w.Write(p.buf) + p.free() + return +} + +// Println formats using the default formats for its operands and writes to standard output. +// Spaces are always added between operands and a newline is appended. +// It returns the number of bytes written and any write error encountered. +func Println(a ...interface{}) (n int, err error) { + return Fprintln(os.Stdout, a...) +} + +// Sprintln formats using the default formats for its operands and returns the resulting string. +// Spaces are always added between operands and a newline is appended. +func Sprintln(a ...interface{}) string { + p := newPrinter() + p.doPrintln(a) + s := string(p.buf) + p.free() + return s +} + +// getField gets the i'th field of the struct value. +// If the field is itself is an interface, return a value for +// the thing inside the interface, not the interface itself. +func getField(v reflect.Value, i int) reflect.Value { + val := v.Field(i) + if val.Kind() == reflect.Interface && !val.IsNil() { + val = val.Elem() + } + return val +} + +// tooLarge reports whether the magnitude of the integer is +// too large to be used as a formatting width or precision. +func tooLarge(x int) bool { + const max int = 1e6 + return x > max || x < -max +} + +// parsenum converts ASCII to integer. num is 0 (and isnum is false) if no number present. +func parsenum(s string, start, end int) (num int, isnum bool, newi int) { + if start >= end { + return 0, false, end + } + for newi = start; newi < end && '0' <= s[newi] && s[newi] <= '9'; newi++ { + if tooLarge(num) { + return 0, false, end // Overflow; crazy long number most likely. + } + num = num*10 + int(s[newi]-'0') + isnum = true + } + return +} + +func (p *pp) unknownType(v reflect.Value) { + if !v.IsValid() { + p.buf.writeString(nilAngleString) + return + } + p.buf.writeByte('?') + p.buf.writeString(v.Type().String()) + p.buf.writeByte('?') +} + +func (p *pp) badVerb(verb rune) { + p.erroring = true + p.buf.writeString(percentBangString) + p.buf.writeRune(verb) + p.buf.writeByte('(') + switch { + case p.arg != nil: + p.buf.writeString(reflect.TypeOf(p.arg).String()) + p.buf.writeByte('=') + p.printArg(p.arg, 'v') + case p.value.IsValid(): + p.buf.writeString(p.value.Type().String()) + p.buf.writeByte('=') + p.printValue(p.value, 'v', 0) + default: + p.buf.writeString(nilAngleString) + } + p.buf.writeByte(')') + p.erroring = false +} + +func (p *pp) fmtBool(v bool, verb rune) { + switch verb { + case 't', 'v': + p.fmt.fmtBoolean(v) + default: + p.badVerb(verb) + } +} + +// fmt0x64 formats a uint64 in hexadecimal and prefixes it with 0x or +// not, as requested, by temporarily setting the sharp flag. +func (p *pp) fmt0x64(v uint64, leading0x bool) { + sharp := p.fmt.sharp + p.fmt.sharp = leading0x + p.fmt.fmtInteger(v, 16, unsigned, 'v', ldigits) + p.fmt.sharp = sharp +} + +// fmtInteger formats a signed or unsigned integer. +func (p *pp) fmtInteger(v uint64, isSigned bool, verb rune) { + switch verb { + case 'v': + if p.fmt.sharpV && !isSigned { + p.fmt0x64(v, true) + } else { + p.fmt.fmtInteger(v, 10, isSigned, verb, ldigits) + } + case 'd': + p.fmt.fmtInteger(v, 10, isSigned, verb, ldigits) + case 'b': + p.fmt.fmtInteger(v, 2, isSigned, verb, ldigits) + case 'o', 'O': + p.fmt.fmtInteger(v, 8, isSigned, verb, ldigits) + case 'x': + p.fmt.fmtInteger(v, 16, isSigned, verb, ldigits) + case 'X': + p.fmt.fmtInteger(v, 16, isSigned, verb, udigits) + case 'c': + p.fmt.fmtC(v) + case 'q': + if v <= utf8.MaxRune { + p.fmt.fmtQc(v) + } else { + p.badVerb(verb) + } + case 'U': + p.fmt.fmtUnicode(v) + default: + p.badVerb(verb) + } +} + +// fmtFloat formats a float. The default precision for each verb +// is specified as last argument in the call to fmt_float. +func (p *pp) fmtFloat(v float64, size int, verb rune) { + switch verb { + case 'v': + p.fmt.fmtFloat(v, size, 'g', -1) + case 'b', 'g', 'G', 'x', 'X': + p.fmt.fmtFloat(v, size, verb, -1) + case 'f', 'e', 'E': + p.fmt.fmtFloat(v, size, verb, 6) + case 'F': + p.fmt.fmtFloat(v, size, 'f', 6) + default: + p.badVerb(verb) + } +} + +// fmtComplex formats a complex number v with +// r = real(v) and j = imag(v) as (r+ji) using +// fmtFloat for r and j formatting. +func (p *pp) fmtComplex(v complex128, size int, verb rune) { + // Make sure any unsupported verbs are found before the + // calls to fmtFloat to not generate an incorrect error string. + switch verb { + case 'v', 'b', 'g', 'G', 'x', 'X', 'f', 'F', 'e', 'E': + oldPlus := p.fmt.plus + p.buf.writeByte('(') + p.fmtFloat(real(v), size/2, verb) + // Imaginary part always has a sign. + p.fmt.plus = true + p.fmtFloat(imag(v), size/2, verb) + p.buf.writeString("i)") + p.fmt.plus = oldPlus + default: + p.badVerb(verb) + } +} + +func (p *pp) fmtString(v string, verb rune) { + switch verb { + case 'v': + if p.fmt.sharpV { + p.fmt.fmtQ(v) + } else { + p.fmt.fmtS(v) + } + case 's': + p.fmt.fmtS(v) + case 'x': + p.fmt.fmtSx(v, ldigits) + case 'X': + p.fmt.fmtSx(v, udigits) + case 'q': + p.fmt.fmtQ(v) + default: + p.badVerb(verb) + } +} + +func (p *pp) fmtBytes(v []byte, verb rune, typeString string) { + switch verb { + case 'v', 'd': + if p.fmt.sharpV { + p.buf.writeString(typeString) + if v == nil { + p.buf.writeString(nilParenString) + return + } + p.buf.writeByte('{') + for i, c := range v { + if i > 0 { + p.buf.writeString(commaSpaceString) + } + p.fmt0x64(uint64(c), true) + } + p.buf.writeByte('}') + } else { + p.buf.writeByte('[') + for i, c := range v { + if i > 0 { + p.buf.writeByte(' ') + } + p.fmt.fmtInteger(uint64(c), 10, unsigned, verb, ldigits) + } + p.buf.writeByte(']') + } + case 's': + p.fmt.fmtBs(v) + case 'x': + p.fmt.fmtBx(v, ldigits) + case 'X': + p.fmt.fmtBx(v, udigits) + case 'q': + p.fmt.fmtQ(string(v)) + default: + p.printValue(reflect.ValueOf(v), verb, 0) + } +} + +func (p *pp) fmtPointer(value reflect.Value, verb rune) { + var u uintptr + switch value.Kind() { + case reflect.Chan, reflect.Func, reflect.Map, reflect.Ptr, reflect.Slice, reflect.UnsafePointer: + u = value.Pointer() + default: + p.badVerb(verb) + return + } + + switch verb { + case 'v': + if p.fmt.sharpV { + p.buf.writeByte('(') + p.buf.writeString(value.Type().String()) + p.buf.writeString(")(") + if u == 0 { + p.buf.writeString(nilString) + } else { + p.fmt0x64(uint64(u), true) + } + p.buf.writeByte(')') + } else { + if u == 0 { + p.fmt.padString(nilAngleString) + } else { + p.fmt0x64(uint64(u), !p.fmt.sharp) + } + } + case 'p': + p.fmt0x64(uint64(u), !p.fmt.sharp) + case 'b', 'o', 'd', 'x', 'X': + p.fmtInteger(uint64(u), unsigned, verb) + default: + p.badVerb(verb) + } +} + +func (p *pp) catchPanic(arg interface{}, verb rune, method string) { + if err := recover(); err != nil { + // If it's a nil pointer, just say "". The likeliest causes are a + // Stringer that fails to guard against nil or a nil pointer for a + // value receiver, and in either case, "" is a nice result. + if v := reflect.ValueOf(arg); v.Kind() == reflect.Ptr && v.IsNil() { + p.buf.writeString(nilAngleString) + return + } + // Otherwise print a concise panic message. Most of the time the panic + // value will print itself nicely. + if p.panicking { + // Nested panics; the recursion in printArg cannot succeed. + panic(err) + } + + oldFlags := p.fmt.fmtFlags + // For this output we want default behavior. + p.fmt.clearflags() + + p.buf.writeString(percentBangString) + p.buf.writeRune(verb) + p.buf.writeString(panicString) + p.buf.writeString(method) + p.buf.writeString(" method: ") + p.panicking = true + p.printArg(err, 'v') + p.panicking = false + p.buf.writeByte(')') + + p.fmt.fmtFlags = oldFlags + } +} + +func (p *pp) handleMethods(verb rune) (handled bool) { + if p.erroring { + return + } + if verb == 'w' { + // It is invalid to use %w other than with Errorf, more than once, + // or with a non-error arg. + err, ok := p.arg.(error) + if !ok || !p.wrapErrs || p.wrappedErr != nil { + p.wrappedErr = nil + p.wrapErrs = false + p.badVerb(verb) + return true + } + p.wrappedErr = err + // If the arg is a Formatter, pass 'v' as the verb to it. + verb = 'v' + } + + // Is it a Formatter? + if formatter, ok := p.arg.(Formatter); ok { + handled = true + defer p.catchPanic(p.arg, verb, "Format") + formatter.Format(p, verb) + return + } + + // If we're doing Go syntax and the argument knows how to supply it, take care of it now. + if p.fmt.sharpV { + if stringer, ok := p.arg.(GoStringer); ok { + handled = true + defer p.catchPanic(p.arg, verb, "GoString") + // Print the result of GoString unadorned. + p.fmt.fmtS(stringer.GoString()) + return + } + } else { + // If a string is acceptable according to the format, see if + // the value satisfies one of the string-valued interfaces. + // Println etc. set verb to %v, which is "stringable". + switch verb { + case 'v', 's', 'x', 'X', 'q': + // Is it an error or Stringer? + // The duplication in the bodies is necessary: + // setting handled and deferring catchPanic + // must happen before calling the method. + switch v := p.arg.(type) { + case error: + handled = true + defer p.catchPanic(p.arg, verb, "Error") + p.fmtString(v.Error(), verb) + return + + case Stringer: + handled = true + defer p.catchPanic(p.arg, verb, "String") + p.fmtString(v.String(), verb) + return + } + } + } + return false +} + +// CUSTOM: printArg() renamed to printArgOrig(). +func (p *pp) printArgOrig(arg interface{}, verb rune) { + p.arg = arg + p.value = reflect.Value{} + + if arg == nil { + switch verb { + case 'T', 'v': + p.fmt.padString(nilAngleString) + default: + p.badVerb(verb) + } + return + } + + // Special processing considerations. + // %T (the value's type) and %p (its address) are special; we always do them first. + switch verb { + case 'T': + p.fmt.fmtS(reflect.TypeOf(arg).String()) + return + case 'p': + p.fmtPointer(reflect.ValueOf(arg), 'p') + return + } + + // Some types can be done without reflection. + switch f := arg.(type) { + case bool: + p.fmtBool(f, verb) + case float32: + p.fmtFloat(float64(f), 32, verb) + case float64: + p.fmtFloat(f, 64, verb) + case complex64: + p.fmtComplex(complex128(f), 64, verb) + case complex128: + p.fmtComplex(f, 128, verb) + case int: + p.fmtInteger(uint64(f), signed, verb) + case int8: + p.fmtInteger(uint64(f), signed, verb) + case int16: + p.fmtInteger(uint64(f), signed, verb) + case int32: + p.fmtInteger(uint64(f), signed, verb) + case int64: + p.fmtInteger(uint64(f), signed, verb) + case uint: + p.fmtInteger(uint64(f), unsigned, verb) + case uint8: + p.fmtInteger(uint64(f), unsigned, verb) + case uint16: + p.fmtInteger(uint64(f), unsigned, verb) + case uint32: + p.fmtInteger(uint64(f), unsigned, verb) + case uint64: + p.fmtInteger(f, unsigned, verb) + case uintptr: + p.fmtInteger(uint64(f), unsigned, verb) + case string: + p.fmtString(f, verb) + case []byte: + p.fmtBytes(f, verb, "[]byte") + case reflect.Value: + // Handle extractable values with special methods + // since printValue does not handle them at depth 0. + if f.IsValid() && f.CanInterface() { + p.arg = f.Interface() + if p.handleMethods(verb) { + return + } + } + p.printValue(f, verb, 0) + default: + // If the type is not simple, it might have methods. + if !p.handleMethods(verb) { + // Need to use reflection, since the type had no + // interface methods that could be used for formatting. + p.printValue(reflect.ValueOf(f), verb, 0) + } + } +} + +// printValue is similar to printArg but starts with a reflect value, not an interface{} value. +// It does not handle 'p' and 'T' verbs because these should have been already handled by printArg. +func (p *pp) printValue(value reflect.Value, verb rune, depth int) { + // Handle values with special methods if not already handled by printArg (depth == 0). + if depth > 0 && value.IsValid() && value.CanInterface() { + p.arg = value.Interface() + if p.handleMethods(verb) { + return + } + } + p.arg = nil + p.value = value + + switch f := value; value.Kind() { + case reflect.Invalid: + if depth == 0 { + p.buf.writeString(invReflectString) + } else { + switch verb { + case 'v': + p.buf.writeString(nilAngleString) + default: + p.badVerb(verb) + } + } + case reflect.Bool: + p.fmtBool(f.Bool(), verb) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + p.fmtInteger(uint64(f.Int()), signed, verb) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + p.fmtInteger(f.Uint(), unsigned, verb) + case reflect.Float32: + p.fmtFloat(f.Float(), 32, verb) + case reflect.Float64: + p.fmtFloat(f.Float(), 64, verb) + case reflect.Complex64: + p.fmtComplex(f.Complex(), 64, verb) + case reflect.Complex128: + p.fmtComplex(f.Complex(), 128, verb) + case reflect.String: + p.fmtString(f.String(), verb) + case reflect.Map: + if p.fmt.sharpV { + p.buf.writeString(f.Type().String()) + if f.IsNil() { + p.buf.writeString(nilParenString) + return + } + p.buf.writeByte('{') + } else { + p.buf.writeString(mapString) + } + sorted := fmtsort.Sort(f) + for i, key := range sorted.Key { + if i > 0 { + if p.fmt.sharpV { + p.buf.writeString(commaSpaceString) + } else { + p.buf.writeByte(' ') + } + } + p.printValue(key, verb, depth+1) + p.buf.writeByte(':') + p.printValue(sorted.Value[i], verb, depth+1) + } + if p.fmt.sharpV { + p.buf.writeByte('}') + } else { + p.buf.writeByte(']') + } + case reflect.Struct: + if p.fmt.sharpV { + p.buf.writeString(f.Type().String()) + } + p.buf.writeByte('{') + for i := 0; i < f.NumField(); i++ { + if i > 0 { + if p.fmt.sharpV { + p.buf.writeString(commaSpaceString) + } else { + p.buf.writeByte(' ') + } + } + if p.fmt.plusV || p.fmt.sharpV { + if name := f.Type().Field(i).Name; name != "" { + p.buf.writeString(name) + p.buf.writeByte(':') + } + } + p.printValue(getField(f, i), verb, depth+1) + } + p.buf.writeByte('}') + case reflect.Interface: + value := f.Elem() + if !value.IsValid() { + if p.fmt.sharpV { + p.buf.writeString(f.Type().String()) + p.buf.writeString(nilParenString) + } else { + p.buf.writeString(nilAngleString) + } + } else { + p.printValue(value, verb, depth+1) + } + case reflect.Array, reflect.Slice: + switch verb { + case 's', 'q', 'x', 'X': + // Handle byte and uint8 slices and arrays special for the above verbs. + t := f.Type() + if t.Elem().Kind() == reflect.Uint8 { + var bytes []byte + if f.Kind() == reflect.Slice { + bytes = f.Bytes() + } else if f.CanAddr() { + bytes = f.Slice(0, f.Len()).Bytes() + } else { + // We have an array, but we cannot Slice() a non-addressable array, + // so we build a slice by hand. This is a rare case but it would be nice + // if reflection could help a little more. + bytes = make([]byte, f.Len()) + for i := range bytes { + bytes[i] = byte(f.Index(i).Uint()) + } + } + p.fmtBytes(bytes, verb, t.String()) + return + } + } + if p.fmt.sharpV { + p.buf.writeString(f.Type().String()) + if f.Kind() == reflect.Slice && f.IsNil() { + p.buf.writeString(nilParenString) + return + } + p.buf.writeByte('{') + for i := 0; i < f.Len(); i++ { + if i > 0 { + p.buf.writeString(commaSpaceString) + } + p.printValue(f.Index(i), verb, depth+1) + } + p.buf.writeByte('}') + } else { + p.buf.writeByte('[') + for i := 0; i < f.Len(); i++ { + if i > 0 { + p.buf.writeByte(' ') + } + p.printValue(f.Index(i), verb, depth+1) + } + p.buf.writeByte(']') + } + case reflect.Ptr: + // pointer to array or slice or struct? ok at top level + // but not embedded (avoid loops) + if depth == 0 && f.Pointer() != 0 { + switch a := f.Elem(); a.Kind() { + case reflect.Array, reflect.Slice, reflect.Struct, reflect.Map: + p.buf.writeByte('&') + p.printValue(a, verb, depth+1) + return + } + } + fallthrough + case reflect.Chan, reflect.Func, reflect.UnsafePointer: + p.fmtPointer(f, verb) + default: + p.unknownType(f) + } +} + +// intFromArg gets the argNumth element of a. On return, isInt reports whether the argument has integer type. +func intFromArg(a []interface{}, argNum int) (num int, isInt bool, newArgNum int) { + newArgNum = argNum + if argNum < len(a) { + num, isInt = a[argNum].(int) // Almost always OK. + if !isInt { + // Work harder. + switch v := reflect.ValueOf(a[argNum]); v.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + n := v.Int() + if int64(int(n)) == n { + num = int(n) + isInt = true + } + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + n := v.Uint() + if int64(n) >= 0 && uint64(int(n)) == n { + num = int(n) + isInt = true + } + default: + // Already 0, false. + } + } + newArgNum = argNum + 1 + if tooLarge(num) { + num = 0 + isInt = false + } + } + return +} + +// parseArgNumber returns the value of the bracketed number, minus 1 +// (explicit argument numbers are one-indexed but we want zero-indexed). +// The opening bracket is known to be present at format[0]. +// The returned values are the index, the number of bytes to consume +// up to the closing paren, if present, and whether the number parsed +// ok. The bytes to consume will be 1 if no closing paren is present. +func parseArgNumber(format string) (index int, wid int, ok bool) { + // There must be at least 3 bytes: [n]. + if len(format) < 3 { + return 0, 1, false + } + + // Find closing bracket. + for i := 1; i < len(format); i++ { + if format[i] == ']' { + width, ok, newi := parsenum(format, 1, i) + if !ok || newi != i { + return 0, i + 1, false + } + return width - 1, i + 1, true // arg numbers are one-indexed and skip paren. + } + } + return 0, 1, false +} + +// argNumber returns the next argument to evaluate, which is either the value of the passed-in +// argNum or the value of the bracketed integer that begins format[i:]. It also returns +// the new value of i, that is, the index of the next byte of the format to process. +func (p *pp) argNumber(argNum int, format string, i int, numArgs int) (newArgNum, newi int, found bool) { + if len(format) <= i || format[i] != '[' { + return argNum, i, false + } + p.reordered = true + index, wid, ok := parseArgNumber(format[i:]) + if ok && 0 <= index && index < numArgs { + return index, i + wid, true + } + p.goodArgNum = false + return argNum, i + wid, ok +} + +func (p *pp) badArgNum(verb rune) { + p.buf.writeString(percentBangString) + p.buf.writeRune(verb) + p.buf.writeString(badIndexString) +} + +func (p *pp) missingArg(verb rune) { + p.buf.writeString(percentBangString) + p.buf.writeRune(verb) + p.buf.writeString(missingString) +} + +func (p *pp) doPrintf(format string, a []interface{}) { + end := len(format) + argNum := 0 // we process one argument per non-trivial format + afterIndex := false // previous item in format was an index like [3]. + p.reordered = false +formatLoop: + for i := 0; i < end; { + p.goodArgNum = true + lasti := i + for i < end && format[i] != '%' { + i++ + } + if i > lasti { + p.buf.writeString(format[lasti:i]) + } + if i >= end { + // done processing format string + break + } + + // Process one verb + i++ + + // Do we have flags? + p.fmt.clearflags() + simpleFormat: + for ; i < end; i++ { + c := format[i] + switch c { + case '#': + p.fmt.sharp = true + case '0': + p.fmt.zero = !p.fmt.minus // Only allow zero padding to the left. + case '+': + p.fmt.plus = true + case '-': + p.fmt.minus = true + p.fmt.zero = false // Do not pad with zeros to the right. + case ' ': + p.fmt.space = true + default: + // Fast path for common case of ascii lower case simple verbs + // without precision or width or argument indices. + if 'a' <= c && c <= 'z' && argNum < len(a) { + if c == 'v' { + // Go syntax + p.fmt.sharpV = p.fmt.sharp + p.fmt.sharp = false + // Struct-field syntax + p.fmt.plusV = p.fmt.plus + p.fmt.plus = false + } + p.printArg(a[argNum], rune(c)) + argNum++ + i++ + continue formatLoop + } + // Format is more complex than simple flags and a verb or is malformed. + break simpleFormat + } + } + + // Do we have an explicit argument index? + argNum, i, afterIndex = p.argNumber(argNum, format, i, len(a)) + + // Do we have width? + if i < end && format[i] == '*' { + i++ + p.fmt.wid, p.fmt.widPresent, argNum = intFromArg(a, argNum) + + if !p.fmt.widPresent { + p.buf.writeString(badWidthString) + } + + // We have a negative width, so take its value and ensure + // that the minus flag is set + if p.fmt.wid < 0 { + p.fmt.wid = -p.fmt.wid + p.fmt.minus = true + p.fmt.zero = false // Do not pad with zeros to the right. + } + afterIndex = false + } else { + p.fmt.wid, p.fmt.widPresent, i = parsenum(format, i, end) + if afterIndex && p.fmt.widPresent { // "%[3]2d" + p.goodArgNum = false + } + } + + // Do we have precision? + if i+1 < end && format[i] == '.' { + i++ + if afterIndex { // "%[3].2d" + p.goodArgNum = false + } + argNum, i, afterIndex = p.argNumber(argNum, format, i, len(a)) + if i < end && format[i] == '*' { + i++ + p.fmt.prec, p.fmt.precPresent, argNum = intFromArg(a, argNum) + // Negative precision arguments don't make sense + if p.fmt.prec < 0 { + p.fmt.prec = 0 + p.fmt.precPresent = false + } + if !p.fmt.precPresent { + p.buf.writeString(badPrecString) + } + afterIndex = false + } else { + p.fmt.prec, p.fmt.precPresent, i = parsenum(format, i, end) + if !p.fmt.precPresent { + p.fmt.prec = 0 + p.fmt.precPresent = true + } + } + } + + if !afterIndex { + argNum, i, afterIndex = p.argNumber(argNum, format, i, len(a)) + } + + if i >= end { + p.buf.writeString(noVerbString) + break + } + + verb, size := rune(format[i]), 1 + if verb >= utf8.RuneSelf { + verb, size = utf8.DecodeRuneInString(format[i:]) + } + i += size + + switch { + case verb == '%': // Percent does not absorb operands and ignores f.wid and f.prec. + p.buf.writeByte('%') + case !p.goodArgNum: + p.badArgNum(verb) + case argNum >= len(a): // No argument left over to print for the current verb. + p.missingArg(verb) + case verb == 'v': + // Go syntax + p.fmt.sharpV = p.fmt.sharp + p.fmt.sharp = false + // Struct-field syntax + p.fmt.plusV = p.fmt.plus + p.fmt.plus = false + fallthrough + default: + p.printArg(a[argNum], verb) + argNum++ + } + } + + // Check for extra arguments unless the call accessed the arguments + // out of order, in which case it's too expensive to detect if they've all + // been used and arguably OK if they're not. + if !p.reordered && argNum < len(a) { + p.fmt.clearflags() + p.buf.writeString(extraString) + for i, arg := range a[argNum:] { + if i > 0 { + p.buf.writeString(commaSpaceString) + } + if arg == nil { + p.buf.writeString(nilAngleString) + } else { + p.buf.writeString(reflect.TypeOf(arg).String()) + p.buf.writeByte('=') + p.printArg(arg, 'v') + } + } + p.buf.writeByte(')') + } +} + +func (p *pp) doPrint(a []interface{}) { + prevString := false + for argNum, arg := range a { + isString := arg != nil && reflect.TypeOf(arg).Kind() == reflect.String + // Add a space between two non-string arguments. + if argNum > 0 && !isString && !prevString { + p.buf.writeByte(' ') + } + p.printArg(arg, 'v') + prevString = isString + } +} + +// doPrintln is like doPrint but always adds a space between arguments +// and a newline after the last argument. +func (p *pp) doPrintln(a []interface{}) { + for argNum, arg := range a { + if argNum > 0 { + p.buf.writeByte(' ') + } + p.printArg(arg, 'v') + } + p.buf.writeByte('\n') +} diff --git a/github.com/cockroachdb/redact/internal/refresh.sh b/github.com/cockroachdb/redact/internal/refresh.sh new file mode 100644 index 0000000000..1be06b5cb3 --- /dev/null +++ b/github.com/cockroachdb/redact/internal/refresh.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# +# This file re-generates the sources in this directory from the Go +# standard library. +# +set -euxo pipefail + +( + echo "// Code generated from the Go standard library. DO NOT EDIT" + echo "// GENERATED FILE DO NOT EDIT" + cat $GOROOT/src/fmt/format.go +) >format.go + +cp $GOROOT/src/fmt/print.go print.go +patch -p0 fmtsort/"$n" +done + diff --git a/github.com/cockroachdb/redact/make_format.go b/github.com/cockroachdb/redact/make_format.go index 534300ba7d..e088b93799 100644 --- a/github.com/cockroachdb/redact/make_format.go +++ b/github.com/cockroachdb/redact/make_format.go @@ -31,9 +31,10 @@ func reproducePrintf(w io.Writer, s fmt.State, verb rune, arg interface{}) { } } -// MakeFormat reproduces the format currently active in fmt.State and -// verb. This is provided because Go's standard fmt.State does not -// make the original format string available to us. +// MakeFormat is a helper for use by implementations of the +// SafeFormatter interface. It reproduces the format currently active +// in fmt.State and verb. This is provided because Go's standard +// fmt.State does not make the original format string available to us. // // If the return value justV is true, then the current state // was found to be %v exactly; in that case the caller diff --git a/github.com/cockroachdb/redact/markers.go b/github.com/cockroachdb/redact/markers.go index 8bf1e64bbf..7b719183d7 100644 --- a/github.com/cockroachdb/redact/markers.go +++ b/github.com/cockroachdb/redact/markers.go @@ -14,6 +14,8 @@ package redact +import "bytes" + // RedactableString is a string that contains a mix of safe and unsafe // bits of data, but where it is known that unsafe bits are enclosed // by redaction markers ‹ and ›, and occurrences of the markers @@ -84,12 +86,11 @@ func (s RedactableBytes) SafeFormat(sp SafePrinter, _ rune) { // EscapeBytes escapes markers inside the given byte slice and encloses // the entire byte slice between redaction markers. func EscapeBytes(s []byte) RedactableBytes { - s = reStripMarkers.ReplaceAll(s, escapeBytes) - enclosed := make([]byte, 0, len(s)+len(startRedactableBytes)+len(endRedactableBytes)) - enclosed = append(enclosed, startRedactableBytes...) - enclosed = append(enclosed, s...) - enclosed = append(enclosed, endRedactableBytes...) - return RedactableBytes(enclosed) + var buf bytes.Buffer + buf.Grow(len(s) + len(startRedactableS) + len(endRedactableS)) + e := escapeWriter{w: &buf, enclose: true, strip: false} + _, _ = e.Write(s) + return RedactableBytes(buf.Bytes()) } // StartMarker returns the start delimiter for an unsafe string. diff --git a/github.com/cockroachdb/redact/markers_internal_escape.go b/github.com/cockroachdb/redact/markers_internal_escape.go index ce23200dd1..9d6989f407 100644 --- a/github.com/cockroachdb/redact/markers_internal_escape.go +++ b/github.com/cockroachdb/redact/markers_internal_escape.go @@ -74,10 +74,11 @@ func (p *escapeWriter) Write(b []byte) (int, error) { // entirely if there was nothing but empty space: // if len(b) == 0 { return 0, nil } - start := startRedactableBytes - ls := len(startRedactableS) - end := endRedactableBytes - le := len(endRedactableS) + // Note: we use len(...RedactableS) and not len(...RedactableBytes) + // because the ...S variant is a compile-time constant so this + // accelerates the loops below. + start, ls := startRedactableBytes, len(startRedactableS) + end, le := endRedactableBytes, len(endRedactableS) escape := escapeBytes if p.enclose { @@ -85,7 +86,11 @@ func (p *escapeWriter) Write(b []byte) (int, error) { } // Now write the string. + + // k is the index in b up to (and excluding) the byte which we've + // already copied into the output. k := 0 + for i := 0; i < len(b); i++ { if b[i] == '\n' && p.enclose { // Avoid enclosing newline characters inside redaction markers. @@ -94,29 +99,36 @@ func (p *escapeWriter) Write(b []byte) (int, error) { // separately. st = p.doWrite(b[k:i], st, true) st = p.doWrite(end, st, false) + // Advance to the last newline character. We want to forward + // them all in a single call to doWrite, for performance. lastNewLine := i for b[lastNewLine] == '\n' && lastNewLine < len(b) { lastNewLine++ } st = p.doWrite(b[i:lastNewLine], st, true) st = p.doWrite(start, st, false) + // Advance the counters by the number of newline characters. k = lastNewLine - i = lastNewLine - 1 + i = lastNewLine - 1 /* -1 because we have i++ at the end of every iteration */ } else { // Ensure that occurrences of the delimiter inside the string get // escaped. + // Reminder: ls and le are likely greater than 1, as we are scanning + // utf-8 encoded delimiters (the utf-8 encoding is multibyte). if i+ls <= len(b) && bytes.Equal(b[i:i+ls], start) { st = p.doWrite(b[k:i], st, true) st = p.doWrite(escape, st, false) + // Advance the counters by the length (in bytes) of the delimiter. st.l += ls k = i + ls - i += ls - 1 + i += ls - 1 /* -1 because we have i++ at the end of every iteration */ } else if i+le <= len(b) && bytes.Equal(b[i:i+le], end) { st = p.doWrite(b[k:i], st, true) st = p.doWrite(escape, st, false) + // Advance the counters by the length (in bytes) of the delimiter. st.l += le k = i + le - i += le - 1 + i += le - 1 /* -1 because we have i++ at the end of every iteration */ } } } @@ -145,3 +157,61 @@ func (p *escapeWriter) doWrite(b []byte, st escapeResult, count bool) escapeResu st.err = err return st } + +// internalEscapeBytes escapes redaction markers in the provided buf +// starting at the location startLoc. +// The bytes before startLoc are considered safe (already escaped). +func internalEscapeBytes(b []byte, startLoc int) (res []byte) { + // Note: we use len(...RedactableS) and not len(...RedactableBytes) + // because the ...S variant is a compile-time constant so this + // accelerates the loops below. + start, ls := startRedactableBytes, len(startRedactableS) + end, le := endRedactableBytes, len(endRedactableS) + escape := escapeBytes + + // res is the output slice. In the common case where there is + // nothing to escape, the input slice is returned directly + // and no allocation takes place. + res = b + // copied is true if and only if `res` is a copy of `b`. It only + // turns to true if the loop below finds something to escape. + copied := false + // k is the index in b up to (and excluding) the byte which we've + // already copied into res (if copied=true). + k := 0 + + for i := startLoc; i < len(b); i++ { + // Ensure that occurrences of the delimiter inside the string get + // escaped. + // Reminder: ls and le are likely greater than 1, as we are scanning + // utf-8 encoded delimiters (the utf-8 encoding is multibyte). + if i+ls <= len(b) && bytes.Equal(b[i:i+ls], start) { + if !copied { + // We only allocate an output slice when we know we definitely + // need it. + res = make([]byte, 0, len(b)+len(escape)) + copied = true + } + res = append(res, b[k:i]...) + res = append(res, escape...) + // Advance the counters by the length (in bytes) of the delimiter. + k = i + ls + i += ls - 1 /* -1 because we have i++ at the end of every iteration */ + } else if i+le <= len(b) && bytes.Equal(b[i:i+le], end) { + if !copied { + // See the comment above about res allocation. + res = make([]byte, 0, len(b)+len(escape)) + copied = true + } + res = append(res, b[k:i]...) + res = append(res, escape...) + // Advance the counters by the length (in bytes) of the delimiter. + k = i + le + i += le - 1 /* -1 because we have i++ at the end of every iteration */ + } + } + if copied { + res = append(res, b[k:]...) + } + return +} diff --git a/github.com/cockroachdb/redact/markers_internal_print.go b/github.com/cockroachdb/redact/markers_internal_print.go index 003c9492f1..672821e301 100644 --- a/github.com/cockroachdb/redact/markers_internal_print.go +++ b/github.com/cockroachdb/redact/markers_internal_print.go @@ -14,69 +14,133 @@ package redact -import "fmt" +import ( + "fmt" -// annotateArgs wraps the arguments to one of the print functions with -// an indirect formatter which either encloses the result of the -// display between redaction markers, or not. -func annotateArgs(args []interface{}) { - for i, arg := range args { + internalFmt "github.com/cockroachdb/redact/internal" +) + +// printArgFn is the hook injected into the standard fmt logic +// by the printer functions in markers_print.go. +func printArgFn(p *internalFmt.InternalPrinter, arg interface{}, verb rune) (newState int) { + redactLastWrites(p) + + if verb == 'T' { + // If the value was wrapped, reveal its original type. Anything else is not very useful. switch v := arg.(type) { - case RedactableString: - // Already formatted as redactable. Include as-is. - // We need an intermediate struct because we don't - // want the fmt machinery to re-format the string - // object by adding quotes, expanding the byte slice, etc. - // - // NB: keep this logic synchronized with - // (RedactableString).SafeFormat(). - args[i] = &passthrough{arg: []byte(v)} - - case RedactableBytes: - // NB: keep this logic synchronized with - // (RedactableBytes).SafeFormat(). - args[i] = &passthrough{arg: v} - - case SafeFormatter: - // calls to Format() by fmt.Print will be redirected to - // v.SafeFormat(). This delegates the task of adding markers to - // the object itself. - args[i] = &redactFormatRedirect{ - func(p SafePrinter, verb rune) { v.SafeFormat(p, verb) }, - } + case safeWrapper: + arg = v.a + case unsafeWrap: + arg = v.a + } - case SafeValue: - // calls to Format() by fmt.Print will be redirected to a - // display of v without redaction markers. - // - // Note that we can't let the value be displayed as-is because - // we must prevent any marker inside the value from leaking into - // the result. (We want to avoid mismatched markers.) - args[i] = &escapeArg{arg: arg, enclose: false} - - case SafeMessager: - // Obsolete interface. - // TODO(knz): Remove this. - args[i] = &escapeArg{arg: v.SafeMessage(), enclose: false} - - default: - if err, ok := v.(error); ok && redactErrorFn != nil { - // We place this case after the other cases above, in case - // the error object knows how to print itself safely already. - args[i] = &redactFormatRedirect{ - func(p SafePrinter, verb rune) { redactErrorFn(err, p, verb) }, - } - } else { - // calls to Format() by fmt.Print will be redirected to a - // display of v within redaction markers if the type is - // considered unsafe, without markers otherwise. In any case, - // occurrences of delimiters within are escaped. - args[i] = &escapeArg{arg: v, enclose: !isSafeValue(v)} + // Shortcut: %T is always safe to print as-is. + internalFmt.PrintArg(p, arg, verb) + return len(internalFmt.Buf(p)) + } + + // RedactableBytes/RedactableString are already formatted as + // redactable. Include them as-is. + // + // NB: keep this logic synchronized with + // (RedactableString/Bytes).SafeFormat(). + switch v := arg.(type) { + case RedactableString: + internalFmt.Append(p, []byte(v)) + return len(internalFmt.Buf(p)) + case RedactableBytes: + internalFmt.Append(p, []byte(v)) + return len(internalFmt.Buf(p)) + } + + arg = annotateArg(arg, internalFmt.CollectingError(p)) + internalFmt.PrintArg(p, arg, verb) + return len(internalFmt.Buf(p)) +} + +// redactLastWrites escapes any markers that were added by the +// internals of the printf functions, for example +// if markers were present in the format string. +func redactLastWrites(p *internalFmt.InternalPrinter) { + state := internalFmt.GetState(p) + newBuf := internalEscapeBytes(internalFmt.Buf(p), state) + internalFmt.SetState(p, newBuf) +} + +// annotateArg wraps the arguments to one of the print functions with +// an indirect formatter which ensures that redaction markers inside +// the representation of the object are escaped, and optionally +// encloses the result of the display between redaction markers. +// +// collectingError is true iff we are in the context of +// HelperForErrorf, where we want the %w verb to work properly. This +// adds a little overhead to the processing, but this is OK because +// typically the error path is not perf-critical. +func annotateArg(arg interface{}, collectingError bool) interface{} { + var newArg fmt.Formatter + err, isError := arg.(error) + + switch v := arg.(type) { + case SafeFormatter: + // calls to Format() by fmt.Print will be redirected to + // v.SafeFormat(). This delegates the task of adding markers to + // the object itself. + newArg = &redactFormatRedirect{ + func(p SafePrinter, verb rune) { v.SafeFormat(p, verb) }, + } + + case SafeValue: + // calls to Format() by fmt.Print will be redirected to a + // display of v without redaction markers. + // + // Note that we can't let the value be displayed as-is because + // we must prevent any marker inside the value from leaking into + // the result. (We want to avoid mismatched markers.) + newArg = &escapeArg{arg: arg, enclose: false} + + case SafeMessager: + // Obsolete interface. + // TODO(knz): Remove this. + newArg = &escapeArg{arg: v.SafeMessage(), enclose: false} + + default: + if isError && redactErrorFn != nil { + // We place this case after the other cases above, in case + // the error object knows how to print itself safely already. + newArg = &redactFormatRedirect{ + func(p SafePrinter, verb rune) { redactErrorFn(err, p, verb) }, } + } else { + // calls to Format() by fmt.Print will be redirected to a + // display of v within redaction markers if the type is + // considered unsafe, without markers otherwise. In any case, + // occurrences of delimiters within are escaped. + newArg = &escapeArg{arg: v, enclose: !isSafeValue(v)} } } + + if isError && collectingError { + // Ensure the arg still implements the `error` interface for + // detection by the handling of %w, while also implementing + // fmt.Formatter to forward the implementation to the objects + // constructed above. + newArg = &makeError{err: err, arg: newArg} + } + + return newArg +} + +type makeError struct { + err error + arg fmt.Formatter } +// Error implements error. +func (m *makeError) Error() string { return m.err.Error() } + +// Format implements fmt.Formatter. +func (m *makeError) Format(f fmt.State, verb rune) { m.arg.Format(f, verb) } + // redactFormatRedirect wraps a safe print callback and uses it to // implement fmt.Formatter. type redactFormatRedirect struct { diff --git a/github.com/cockroachdb/redact/markers_internal_printer.go b/github.com/cockroachdb/redact/markers_internal_printer.go index 17da9e36cb..7b0fd75a4e 100644 --- a/github.com/cockroachdb/redact/markers_internal_printer.go +++ b/github.com/cockroachdb/redact/markers_internal_printer.go @@ -17,6 +17,7 @@ package redact import ( "bytes" "fmt" + "unicode/utf8" ) // printer implements SafePrinter. @@ -70,7 +71,13 @@ func (b *printer) UnsafeRune(s rune) { // UnsafeByte is part of the SafeWriter interface. func (b *printer) UnsafeByte(s byte) { _, _ = b.buf.WriteRune(startRedactable) - _ = b.buf.WriteByte(s) + if s >= utf8.RuneSelf || + s == startRedactableBytes[0] || s == endRedactableBytes[0] { + // Unsafe byte. Escape it. + _, _ = b.buf.Write(escapeBytes) + } else { + _ = b.buf.WriteByte(s) + } _, _ = b.buf.WriteRune(endRedactable) } diff --git a/github.com/cockroachdb/redact/markers_print.go b/github.com/cockroachdb/redact/markers_print.go index 55ceade95b..50eea32b4b 100644 --- a/github.com/cockroachdb/redact/markers_print.go +++ b/github.com/cockroachdb/redact/markers_print.go @@ -15,8 +15,9 @@ package redact import ( - "fmt" "io" + + internalFmt "github.com/cockroachdb/redact/internal" ) // Sprint prints out the arguments and encloses unsafe bits @@ -26,8 +27,13 @@ import ( // If a RedactableString or RedactableBytes argument is passed, // it is reproduced as-is without escaping. func Sprint(args ...interface{}) RedactableString { - annotateArgs(args) - return RedactableString(fmt.Sprint(args...)) + p := internalFmt.NewInternalPrinter() + internalFmt.SetHook(p, printArgFn) + internalFmt.DoPrint(p, args) + redactLastWrites(p) + s := RedactableString(internalFmt.Buf(p)) + internalFmt.Free(p) + return s } // Sprintf formats the arguments and encloses unsafe bits @@ -38,8 +44,37 @@ func Sprint(args ...interface{}) RedactableString { // is responsible to ensure that the markers are not present // in the format string. func Sprintf(format string, args ...interface{}) RedactableString { - annotateArgs(args) - return RedactableString(fmt.Sprintf(format, args...)) + p := internalFmt.NewInternalPrinter() + internalFmt.SetHook(p, printArgFn) + internalFmt.DoPrintf(p, format, args) + redactLastWrites(p) + s := RedactableString(internalFmt.Buf(p)) + internalFmt.Free(p) + return s +} + +// HelperForErrorf is a helper to implement a redaction-aware +// fmt.Errorf-compatible function in a different package. It formats +// the string according to the given format and arguments in the same +// way as Sprintf, but in addition to this if the format contains %w +// and an error object in the proper argument position it also returns +// that error object. +// +// Note: This function only works if an error redaction function +// has been injected with RegisterRedactErrorFn(). +func HelperForErrorf(format string, args ...interface{}) (RedactableString, error) { + p := internalFmt.NewInternalPrinter() + internalFmt.SetCollectError(p) + internalFmt.SetHook(p, printArgFn) + internalFmt.DoPrintf(p, format, args) + redactLastWrites(p) + s := RedactableString(internalFmt.Buf(p)) + e := internalFmt.WrappedError(p) + internalFmt.Free(p) + if m, ok := e.(*makeError); ok { + e = m.err + } + return s, e } // Sprintfn produces a RedactableString using the provided @@ -64,14 +99,24 @@ func StringWithoutMarkers(f SafeFormatter) string { // Fprint is like Sprint but outputs the redactable // string to the provided Writer. -func Fprint(w io.Writer, args ...interface{}) (int, error) { - annotateArgs(args) - return fmt.Fprint(w, args...) +func Fprint(w io.Writer, args ...interface{}) (n int, err error) { + p := internalFmt.NewInternalPrinter() + internalFmt.SetHook(p, printArgFn) + internalFmt.DoPrint(p, args) + redactLastWrites(p) + n, err = w.Write(internalFmt.Buf(p)) + internalFmt.Free(p) + return } // Fprintf is like Sprintf but outputs the redactable string to the // provided Writer. -func Fprintf(w io.Writer, format string, args ...interface{}) (int, error) { - annotateArgs(args) - return fmt.Fprintf(w, format, args...) +func Fprintf(w io.Writer, format string, args ...interface{}) (n int, err error) { + p := internalFmt.NewInternalPrinter() + internalFmt.SetHook(p, printArgFn) + internalFmt.DoPrintf(p, format, args) + redactLastWrites(p) + n, err = w.Write(internalFmt.Buf(p)) + internalFmt.Free(p) + return } diff --git a/modules.txt b/modules.txt index c7c3132de9..3077e6cb10 100644 --- a/modules.txt +++ b/modules.txt @@ -167,7 +167,7 @@ github.com/cockroachdb/crlfmt/internal/parser github.com/cockroachdb/crlfmt/internal/render # github.com/cockroachdb/datadriven v1.0.1-0.20200826112548-92602d883b11 github.com/cockroachdb/datadriven -# github.com/cockroachdb/errors v1.7.3 +# github.com/cockroachdb/errors v1.7.4-0.20200901162659-b0f54ae0a5e8 github.com/cockroachdb/errors github.com/cockroachdb/errors/assert github.com/cockroachdb/errors/barriers @@ -228,8 +228,10 @@ github.com/cockroachdb/pebble/internal/record github.com/cockroachdb/pebble/sstable github.com/cockroachdb/pebble/tool github.com/cockroachdb/pebble/vfs -# github.com/cockroachdb/redact v1.0.2 +# github.com/cockroachdb/redact v1.0.4 github.com/cockroachdb/redact +github.com/cockroachdb/redact/internal +github.com/cockroachdb/redact/internal/fmtsort # github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd github.com/cockroachdb/returncheck # github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 From 5fae4fa05676a0583af721b028a5d388bdf27187 Mon Sep 17 00:00:00 2001 From: Raphael 'kena' Poss Date: Wed, 2 Sep 2020 11:48:53 +0200 Subject: [PATCH 36/49] Bump cockroachdb/errors and cockroachdb/redact once more --- github.com/cockroachdb/errors/go.mod | 2 +- github.com/cockroachdb/errors/go.sum | 6 +- github.com/cockroachdb/redact/builder.go | 3 + .../cockroachdb/redact/internal/print.go.diff | 58 +++++++++++++++++++ modules.txt | 4 +- 5 files changed, 66 insertions(+), 7 deletions(-) create mode 100644 github.com/cockroachdb/redact/internal/print.go.diff diff --git a/github.com/cockroachdb/errors/go.mod b/github.com/cockroachdb/errors/go.mod index 8aad9b7c7d..ca5e0412bc 100644 --- a/github.com/cockroachdb/errors/go.mod +++ b/github.com/cockroachdb/errors/go.mod @@ -5,7 +5,7 @@ go 1.13 require ( github.com/cockroachdb/datadriven v1.0.0 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f - github.com/cockroachdb/redact v1.0.4 + github.com/cockroachdb/redact v1.0.5 github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 github.com/gogo/protobuf v1.3.1 github.com/gogo/status v1.1.0 diff --git a/github.com/cockroachdb/errors/go.sum b/github.com/cockroachdb/errors/go.sum index 4dd9db9cc5..24aca82e9f 100644 --- a/github.com/cockroachdb/errors/go.sum +++ b/github.com/cockroachdb/errors/go.sum @@ -17,10 +17,8 @@ github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlN github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/cockroachdb/redact v1.0.4-0.20200901154857-49cd8220a922 h1:GL77i8JixDUQjDrb0bRU4DNioBRsh0t1t1g3imQYyAE= -github.com/cockroachdb/redact v1.0.4-0.20200901154857-49cd8220a922/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= -github.com/cockroachdb/redact v1.0.4 h1:UZv01KibZfgs975zjD0vf1PewwbGofROVtIfBk2Xyt0= -github.com/cockroachdb/redact v1.0.4/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/redact v1.0.5 h1:yxqIMS6G2Bvi6GiSHFmsrFGO3aP1rwt8cOm4pixw9eY= +github.com/cockroachdb/redact v1.0.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= diff --git a/github.com/cockroachdb/redact/builder.go b/github.com/cockroachdb/redact/builder.go index 05efd94d26..a87d89756e 100644 --- a/github.com/cockroachdb/redact/builder.go +++ b/github.com/cockroachdb/redact/builder.go @@ -38,6 +38,9 @@ func (b *StringBuilder) String() string { return b.RedactableString().StripMarke // RedactableString returns the accumulated string, including redaction markers. func (b *StringBuilder) RedactableString() RedactableString { return RedactableString(b.buf.String()) } +// RedactableString returns the accumulated bytes, including redaction markers. +func (b *StringBuilder) RedactableBytes() RedactableBytes { return RedactableBytes(b.buf.Bytes()) } + // SafeFormat implements SafeFormatter. func (b *StringBuilder) SafeFormat(p SafePrinter, _ rune) { // We only support the %v / %s natural print here. diff --git a/github.com/cockroachdb/redact/internal/print.go.diff b/github.com/cockroachdb/redact/internal/print.go.diff new file mode 100644 index 0000000000..4451b4b104 --- /dev/null +++ b/github.com/cockroachdb/redact/internal/print.go.diff @@ -0,0 +1,58 @@ +--- print.go.orig 2020-08-26 14:49:30.396184000 +0200 ++++ print.go 2020-08-26 14:50:29.330829000 +0200 +@@ -1,3 +1,6 @@ ++// Code generated from print.go.orig. DO NOT EDIT ++// GENERATED FILE DO NOT EDIT ++// + // Copyright 2009 The Go Authors. All rights reserved. + // Use of this source code is governed by a BSD-style + // license that can be found in the LICENSE file. +@@ -5,12 +8,16 @@ + package fmt + + import ( +- "internal/fmtsort" ++ // CUSTOM: needed to avoid a type mismatch on Formatter. ++ origFmt "fmt" + "io" + "os" + "reflect" + "sync" + "unicode/utf8" ++ ++ // CUSTOM: our own import since we can't use internal from go stdlib. ++ "github.com/cockroachdb/redact/internal/fmtsort" + ) + + // Strings for use with buffer.WriteString. +@@ -51,7 +58,8 @@ + // The implementation of Format may call Sprint(f) or Fprint(f) etc. + // to generate its output. + type Formatter interface { +- Format(f State, c rune) ++ // CUSTOM: refer to the original type, not the one defined here. ++ Format(f origFmt.State, c rune) + } + + // Stringer is implemented by any value that has a String method, +@@ -105,6 +113,10 @@ + type pp struct { + buf buffer + ++ // CUSTOM: hook fn for the redact package. ++ printArgSubstituteFn func(p *pp, a interface{}, verb rune) (newState int) ++ substituteState int ++ + // arg holds the current item, as an interface{}. + arg interface{} + +@@ -635,7 +647,8 @@ + return false + } + +-func (p *pp) printArg(arg interface{}, verb rune) { ++// CUSTOM: printArg() renamed to printArgOrig(). ++func (p *pp) printArgOrig(arg interface{}, verb rune) { + p.arg = arg + p.value = reflect.Value{} + diff --git a/modules.txt b/modules.txt index 3077e6cb10..2c279fd001 100644 --- a/modules.txt +++ b/modules.txt @@ -167,7 +167,7 @@ github.com/cockroachdb/crlfmt/internal/parser github.com/cockroachdb/crlfmt/internal/render # github.com/cockroachdb/datadriven v1.0.1-0.20200826112548-92602d883b11 github.com/cockroachdb/datadriven -# github.com/cockroachdb/errors v1.7.4-0.20200901162659-b0f54ae0a5e8 +# github.com/cockroachdb/errors v1.7.4-0.20200902094217-2c28eca57c8f github.com/cockroachdb/errors github.com/cockroachdb/errors/assert github.com/cockroachdb/errors/barriers @@ -228,7 +228,7 @@ github.com/cockroachdb/pebble/internal/record github.com/cockroachdb/pebble/sstable github.com/cockroachdb/pebble/tool github.com/cockroachdb/pebble/vfs -# github.com/cockroachdb/redact v1.0.4 +# github.com/cockroachdb/redact v1.0.5 github.com/cockroachdb/redact github.com/cockroachdb/redact/internal github.com/cockroachdb/redact/internal/fmtsort From 49c7fe7b4405b33447f29e1c40d7a1bb959d1bf4 Mon Sep 17 00:00:00 2001 From: Raphael 'kena' Poss Date: Wed, 2 Sep 2020 14:17:54 +0200 Subject: [PATCH 37/49] fix the errors vernum using a tag --- modules.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules.txt b/modules.txt index 2c279fd001..a5babcd5dd 100644 --- a/modules.txt +++ b/modules.txt @@ -167,7 +167,7 @@ github.com/cockroachdb/crlfmt/internal/parser github.com/cockroachdb/crlfmt/internal/render # github.com/cockroachdb/datadriven v1.0.1-0.20200826112548-92602d883b11 github.com/cockroachdb/datadriven -# github.com/cockroachdb/errors v1.7.4-0.20200902094217-2c28eca57c8f +# github.com/cockroachdb/errors v1.7.4 github.com/cockroachdb/errors github.com/cockroachdb/errors/assert github.com/cockroachdb/errors/barriers From 38594b2a17cab688dde5b26006866344b442a4aa Mon Sep 17 00:00:00 2001 From: Jackson Owens Date: Fri, 4 Sep 2020 16:51:37 -0400 Subject: [PATCH 38/49] bump Pebble to b66f9d0155ab b66f9d01 db: fix nil pointer dereference in compaction c29cc161 internal/record: write an EOF trailer on LogWriter Close --- github.com/cockroachdb/pebble/compaction.go | 6 +++++- .../pebble/internal/record/log_writer.go | 17 +++++++++++++++++ github.com/cockroachdb/pebble/open.go | 18 +++++++++++------- modules.txt | 4 ++-- 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/github.com/cockroachdb/pebble/compaction.go b/github.com/cockroachdb/pebble/compaction.go index 845407f12a..07f7885ff9 100644 --- a/github.com/cockroachdb/pebble/compaction.go +++ b/github.com/cockroachdb/pebble/compaction.go @@ -1821,7 +1821,11 @@ func (d *DB) runCompaction( // In this case, `limit` will be a larger user key than `key.UserKey`, or // nil. In either case, the inner loop will execute at least once to // process `key`, and the input iterator will be advanced. - limit = c.findGrandparentLimit(key.UserKey) + if key != nil { + limit = c.findGrandparentLimit(key.UserKey) + } else { // !c.rangeDelFrag.Empty() + limit = c.findGrandparentLimit(c.rangeDelFrag.Start()) + } } else { // There is a range tombstone spanning from the last file into the // current one. Therefore this file's smallest boundary will overlap the diff --git a/github.com/cockroachdb/pebble/internal/record/log_writer.go b/github.com/cockroachdb/pebble/internal/record/log_writer.go index e3ce92af0d..91e9409df8 100644 --- a/github.com/cockroachdb/pebble/internal/record/log_writer.go +++ b/github.com/cockroachdb/pebble/internal/record/log_writer.go @@ -534,6 +534,11 @@ func (w *LogWriter) queueBlock() { func (w *LogWriter) Close() error { f := &w.flusher + // Emit an EOF trailer signifying the end of this log. This helps readers + // differentiate between a corrupted entry in the middle of a log from + // garbage at the tail from a recycled log file. + w.emitEOFTrailer() + // Signal the flush loop to close. f.Lock() f.close = true @@ -613,6 +618,18 @@ func (w *LogWriter) Size() int64 { return w.blockNum*blockSize + int64(w.block.written) } +func (w *LogWriter) emitEOFTrailer() { + // Write a recyclable chunk header with a different log number. Readers + // will treat the header as EOF when the log number does not match. + b := w.block + i := b.written + binary.LittleEndian.PutUint32(b.buf[i+0:i+4], 0) // CRC + binary.LittleEndian.PutUint16(b.buf[i+4:i+6], 0) // Size + b.buf[i+6] = recyclableFullChunkType + binary.LittleEndian.PutUint32(b.buf[i+7:i+11], w.logNum+1) // Log number + atomic.StoreInt32(&b.written, i+int32(recyclableHeaderSize)) +} + func (w *LogWriter) emitFragment(n int, p []byte) []byte { b := w.block i := b.written diff --git a/github.com/cockroachdb/pebble/open.go b/github.com/cockroachdb/pebble/open.go index 4808bd992a..1101cd7b55 100644 --- a/github.com/cockroachdb/pebble/open.go +++ b/github.com/cockroachdb/pebble/open.go @@ -264,8 +264,9 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { }) var ve versionEdit - for _, lf := range logFiles { - maxSeqNum, err := d.replayWAL(jobID, &ve, opts.FS, opts.FS.PathJoin(d.walDirname, lf.name), lf.num) + for i, lf := range logFiles { + lastWAL := i == len(logFiles)-1 + maxSeqNum, err := d.replayWAL(jobID, &ve, opts.FS, opts.FS.PathJoin(d.walDirname, lf.name), lf.num, lastWAL) if err != nil { return nil, err } @@ -427,7 +428,7 @@ func GetVersion(dir string, fs vfs.FS) (string, error) { // d.mu must be held when calling this, but the mutex may be dropped and // re-acquired during the course of this method. func (d *DB) replayWAL( - jobID int, ve *versionEdit, fs vfs.FS, filename string, logNum FileNum, + jobID int, ve *versionEdit, fs vfs.FS, filename string, logNum FileNum, lastWAL bool, ) (maxSeqNum uint64, err error) { file, err := fs.Open(filename) if err != nil { @@ -492,10 +493,13 @@ func (d *DB) replayWAL( } if err != nil { // It is common to encounter a zeroed or invalid chunk due to WAL - // preallocation and WAL recycling. We need to distinguish these errors - // from EOF in order to recognize that the record was truncated, but want + // preallocation and WAL recycling. We need to distinguish these + // errors from EOF in order to recognize that the record was + // truncated and to avoid replaying subsequent WALs, but want // to otherwise treat them like EOF. - if err == io.EOF || record.IsInvalidRecord(err) { + if err == io.EOF { + break + } else if record.IsInvalidRecord(err) && lastWAL { break } return 0, errors.Wrap(err, "pebble: error when replaying WAL") @@ -565,7 +569,7 @@ func (d *DB) replayWAL( toFlush[i].readerUnref() } } - return maxSeqNum, nil + return maxSeqNum, err } func checkOptions(opts *Options, path string) error { diff --git a/modules.txt b/modules.txt index a5babcd5dd..e0e1479d7a 100644 --- a/modules.txt +++ b/modules.txt @@ -205,7 +205,7 @@ github.com/cockroachdb/gostdlib/x/tools/internal/semver github.com/cockroachdb/gostdlib/x/tools/internal/span # github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f github.com/cockroachdb/logtags -# github.com/cockroachdb/pebble v0.0.0-20200831143935-e6a9f9a3c936 +# github.com/cockroachdb/pebble v0.0.0-20200904203921-b66f9d0155ab github.com/cockroachdb/pebble github.com/cockroachdb/pebble/bloom github.com/cockroachdb/pebble/internal/arenaskl @@ -753,7 +753,7 @@ golang.org/x/crypto/ssh/agent golang.org/x/crypto/ssh/internal/bcrypt_pbkdf golang.org/x/crypto/ssh/knownhosts golang.org/x/crypto/ssh/terminal -# golang.org/x/exp v0.0.0-20200821190819-94841d0725da +# golang.org/x/exp v0.0.0-20200901203048-c4f52b2c50aa golang.org/x/exp/rand # golang.org/x/lint v0.0.0-20200130185559-910be7a94367 golang.org/x/lint From 048b0a468b1b01faff682ee86370d786f90f9483 Mon Sep 17 00:00:00 2001 From: Bilal Akhtar Date: Tue, 8 Sep 2020 12:00:39 -0400 Subject: [PATCH 39/49] Bump pebble to a9c6d3b6685c --- .../pebble/internal/base/iterator.go | 13 +- github.com/cockroachdb/pebble/iterator.go | 12 +- github.com/cockroachdb/pebble/level_iter.go | 101 ++---- .../cockroachdb/pebble/sstable/block.go | 79 ----- .../cockroachdb/pebble/sstable/reader.go | 306 +++--------------- .../cockroachdb/pebble/sstable/unsafe.go | 8 +- .../pebble/tool/make_test_find_db.go | 5 - modules.txt | 2 +- 8 files changed, 91 insertions(+), 435 deletions(-) diff --git a/github.com/cockroachdb/pebble/internal/base/iterator.go b/github.com/cockroachdb/pebble/internal/base/iterator.go index cc27c6f193..23612f6f79 100644 --- a/github.com/cockroachdb/pebble/internal/base/iterator.go +++ b/github.com/cockroachdb/pebble/internal/base/iterator.go @@ -52,14 +52,11 @@ import "fmt" // Last if there is an upper bound). This imposition is done in order to // elevate that enforcement to the caller (generally pebble.Iterator or // pebble.mergingIter) rather than having it duplicated in every -// InternalIterator implementation. Additionally, the caller needs to ensure -// that SeekGE/SeekPrefixGE are not called with a key > the upper bound, and -// SeekLT is not called with a key < the lower bound. -// InternalIterator implementations are required to respect the iterator -// bounds, never returning records outside of the bounds with one exception: -// an iterator may generate synthetic RANGEDEL marker records. See -// levelIter.syntheticBoundary for the sole existing example of this behavior. -// [TODO(peter): can we eliminate this exception?] +// InternalIterator implementation. InternalIterator implementations are +// required to respect the iterator bounds, never returning records outside of +// the bounds with one exception: an iterator may generate synthetic RANGEDEL +// marker records. See levelIter.syntheticBoundary for the sole existing +// example of this behavior. [TODO(peter): can we eliminate this exception?] // // An iterator must be closed after use, but it is not necessary to read an // iterator until exhaustion. diff --git a/github.com/cockroachdb/pebble/iterator.go b/github.com/cockroachdb/pebble/iterator.go index 3b11809aae..f1f8694174 100644 --- a/github.com/cockroachdb/pebble/iterator.go +++ b/github.com/cockroachdb/pebble/iterator.go @@ -315,8 +315,6 @@ func (i *Iterator) SeekGE(key []byte) bool { i.prefix = nil if lowerBound := i.opts.GetLowerBound(); lowerBound != nil && i.cmp(key, lowerBound) < 0 { key = lowerBound - } else if upperBound := i.opts.GetUpperBound(); upperBound != nil && i.cmp(key, upperBound) > 0 { - key = upperBound } i.iterKey, i.iterValue = i.iter.SeekGE(key) @@ -372,12 +370,6 @@ func (i *Iterator) SeekPrefixGE(key []byte) bool { return false } key = lowerBound - } else if upperBound := i.opts.GetUpperBound(); upperBound != nil && i.cmp(key, upperBound) > 0 { - if n := i.split(upperBound); !bytes.Equal(i.prefix, upperBound[:n]) { - i.err = errors.New("pebble: SeekPrefixGE supplied with key outside of upper bound") - return false - } - key = upperBound } i.iterKey, i.iterValue = i.iter.SeekPrefixGE(i.prefix, key) @@ -390,10 +382,8 @@ func (i *Iterator) SeekPrefixGE(key []byte) bool { func (i *Iterator) SeekLT(key []byte) bool { i.err = nil // clear cached iteration error i.prefix = nil - if upperBound := i.opts.GetUpperBound(); upperBound != nil && i.cmp(key, upperBound) > 0 { + if upperBound := i.opts.GetUpperBound(); upperBound != nil && i.cmp(key, upperBound) >= 0 { key = upperBound - } else if lowerBound := i.opts.GetLowerBound(); lowerBound != nil && i.cmp(key, lowerBound) < 0 { - key = lowerBound } i.iterKey, i.iterValue = i.iter.SeekLT(key) diff --git a/github.com/cockroachdb/pebble/level_iter.go b/github.com/cockroachdb/pebble/level_iter.go index ae07ee6fe9..c464dee8d5 100644 --- a/github.com/cockroachdb/pebble/level_iter.go +++ b/github.com/cockroachdb/pebble/level_iter.go @@ -69,22 +69,12 @@ type levelIter struct { // - err != nil // - some other constraint, like the bounds in opts, caused the file at index to not // be relevant to the iteration. - iter internalIterator - iterFile *fileMetadata - newIters tableNewIters - // When rangeDelIter != nil, the caller requires that *rangeDelIter must point - // to a range del iterator corresponding to the current file. When this - // iterator returns nil, *rangeDelIter should also be set to nil. Whenever - // a non-nil internalIterator is placed in rangeDelIter, a copy is placed - // in rangeDelIterCopy. This is done for the following special case: - // when this iterator returns nil because of exceeding the bounds, we don't - // close iter and *rangeDelIter since we could reuse it in the next seek. But - // we need to set *rangeDelIter to nil because of the aforementioned contract. - // This copy is used to revive the *rangeDelIter in the case of reuse. - rangeDelIter *internalIterator - rangeDelIterCopy internalIterator - files manifest.LevelIterator - err error + iter internalIterator + iterFile *fileMetadata + newIters tableNewIters + rangeDelIter *internalIterator + files manifest.LevelIterator + err error // Pointer into this level's entry in `mergingIterLevel::smallestUserKey,largestUserKey`. // We populate it with the corresponding bounds for the currently opened file. It is used for @@ -265,9 +255,6 @@ func (l *levelIter) loadFile(file *fileMetadata, dir int) bool { // We don't bother comparing the file bounds with the iteration bounds when we have // an already open iterator. It is possible that the iter may not be relevant given the // current iteration bounds, but it knows those bounds, so it will enforce them. - if l.rangeDelIter != nil { - *l.rangeDelIter = l.rangeDelIterCopy - } return true } // We were already at file, but don't have an iterator, probably because the file was @@ -314,7 +301,6 @@ func (l *levelIter) loadFile(file *fileMetadata, dir int) bool { } if l.rangeDelIter != nil { *l.rangeDelIter = rangeDelIter - l.rangeDelIterCopy = rangeDelIter } else if rangeDelIter != nil { rangeDelIter.Close() } @@ -382,7 +368,7 @@ func (l *levelIter) SeekPrefixGE(prefix, key []byte) (*InternalKey, []byte) { // table as findFileGE found the table where key <= meta.Largest. We treat // this case the same as SeekGE where an upper-bound resides within the // sstable and generate a synthetic boundary key. - if l.rangeDelIter != nil && *l.rangeDelIter != nil { + if l.rangeDelIter != nil { l.syntheticBoundary = l.iterFile.Largest l.syntheticBoundary.SetKind(InternalKeyKindRangeDelete) l.largestBoundary = &l.syntheticBoundary @@ -440,17 +426,6 @@ func (l *levelIter) Next() (*InternalKey, []byte) { switch { case l.largestBoundary != nil: - if l.tableOpts.UpperBound != nil { - // The UpperBound was within this file, so don't load the next - // file. We leave the largestBoundary unchanged so that subsequent - // calls to Next() stay at this file. If a Seek/First/Last call is - // made and this file continues to be relevant, loadFile() will - // set the largestBoundary to nil. - if l.rangeDelIter != nil { - *l.rangeDelIter = nil - } - return nil, nil - } // We're stepping past the boundary key, so now we can load the next file. if l.loadFile(l.files.Next(), +1) { if key, val := l.iter.First(); key != nil { @@ -477,17 +452,6 @@ func (l *levelIter) Prev() (*InternalKey, []byte) { switch { case l.smallestBoundary != nil: - if l.tableOpts.LowerBound != nil { - // The LowerBound was within this file, so don't load the previous - // file. We leave the smallestBoundary unchanged so that - // subsequent calls to Prev() stay at this file. If a - // Seek/First/Last call is made and this file continues to be - // relevant, loadFile() will set the smallestBoundary to nil. - if l.rangeDelIter != nil { - *l.rangeDelIter = nil - } - return nil, nil - } // We're stepping past the boundary key, so now we can load the prev file. if l.loadFile(l.files.Prev(), -1) { if key, val := l.iter.Last(); key != nil { @@ -537,23 +501,16 @@ func (l *levelIter) skipEmptyFileForward() (*InternalKey, []byte) { // never going to look at subsequent sstables (we've reached the upper // bound). if l.tableOpts.UpperBound != nil { - if *l.rangeDelIter != nil { - // TODO(peter): Rather than using f.Largest, can we use - // l.tableOpts.UpperBound and set the seqnum to 0? We know the upper - // bound resides within the table boundaries. Not clear if this is - // kosher with respect to the invariant that only one record for a - // given user key will have seqnum 0. See Iterator.nextUserKey for an - // optimization that requires this. - l.syntheticBoundary = l.iterFile.Largest - l.syntheticBoundary.SetKind(InternalKeyKindRangeDelete) - l.largestBoundary = &l.syntheticBoundary - return l.largestBoundary, nil - } - // Else there are no range deletions in this sstable. This - // helps with performance when many levels are populated with - // sstables and most don't have any actual keys within the - // bounds. - return nil, nil + // TODO(peter): Rather than using f.Largest, can we use + // l.tableOpts.UpperBound and set the seqnum to 0? We know the upper + // bound resides within the table boundaries. Not clear if this is + // kosher with respect to the invariant that only one record for a + // given user key will have seqnum 0. See Iterator.nextUserKey for an + // optimization that requires this. + l.syntheticBoundary = l.iterFile.Largest + l.syntheticBoundary.SetKind(InternalKeyKindRangeDelete) + l.largestBoundary = &l.syntheticBoundary + return l.largestBoundary, nil } // If the boundary is a range deletion tombstone, return that key. if l.iterFile.Largest.Kind() == InternalKeyKindRangeDelete { @@ -600,20 +557,13 @@ func (l *levelIter) skipEmptyFileBackward() (*InternalKey, []byte) { // never going to look at earlier sstables (we've reached the lower // bound). if l.tableOpts.LowerBound != nil { - if *l.rangeDelIter != nil { - // TODO(peter): Rather than using f.Smallest, can we use - // l.tableOpts.LowerBound and set the seqnum to InternalKeySeqNumMax? - // We know the lower bound resides within the table boundaries. - l.syntheticBoundary = l.iterFile.Smallest - l.syntheticBoundary.SetKind(InternalKeyKindRangeDelete) - l.smallestBoundary = &l.syntheticBoundary - return l.smallestBoundary, nil - } - // Else there are no range deletions in this sstable. This - // helps with performance when many levels are populated with - // sstables and most don't have any actual keys within the - // bounds. - return nil, nil + // TODO(peter): Rather than using f.Smallest, can we use + // l.tableOpts.LowerBound and set the seqnum to InternalKeySeqNumMax? + // We know the lower bound resides within the table boundaries. + l.syntheticBoundary = l.iterFile.Smallest + l.syntheticBoundary.SetKind(InternalKeyKindRangeDelete) + l.smallestBoundary = &l.syntheticBoundary + return l.smallestBoundary, nil } // If the boundary is a range deletion tombstone, return that key. if l.iterFile.Smallest.Kind() == InternalKeyKindRangeDelete { @@ -643,11 +593,10 @@ func (l *levelIter) Close() error { l.iter = nil } if l.rangeDelIter != nil { - if t := l.rangeDelIterCopy; t != nil { + if t := *l.rangeDelIter; t != nil { l.err = firstError(l.err, t.Close()) } *l.rangeDelIter = nil - l.rangeDelIterCopy = nil } return l.err } diff --git a/github.com/cockroachdb/pebble/sstable/block.go b/github.com/cockroachdb/pebble/sstable/block.go index 0be2367b78..846056b4f6 100644 --- a/github.com/cockroachdb/pebble/sstable/block.go +++ b/github.com/cockroachdb/pebble/sstable/block.go @@ -238,9 +238,6 @@ type blockIter struct { cached []blockEntry cachedBuf []byte cacheHandle cache.Handle - // The first key in the block. This is used by the caller to set bounds - // for block iteration for already loaded blocks. - firstKey InternalKey } // blockIter implements the base.InternalIterator interface. @@ -269,7 +266,6 @@ func (i *blockIter) init(cmp Compare, block block, globalSeqNum uint64) error { i.fullKey = i.fullKey[:0] i.val = nil i.clearCache() - i.readFirstKey() return nil } @@ -288,16 +284,11 @@ func (i *blockIter) invalidate() { i.data = nil } -func (i *blockIter) isInvalid() bool { - return i.data == nil -} - func (i *blockIter) resetForReuse() blockIter { return blockIter{ fullKey: i.fullKey[:0], cached: i.cached[:0], cachedBuf: i.cachedBuf[:0], - data: nil, } } @@ -385,76 +376,6 @@ func (i *blockIter) readEntry() { i.nextOffset = int32(uintptr(ptr)-uintptr(i.ptr)) + int32(value) } -func (i *blockIter) readFirstKey() { - ptr := i.ptr - - // This is an ugly performance hack. Reading entries from blocks is one of - // the inner-most routines and decoding the 3 varints per-entry takes a - // significant time. Neither go1.11 or go1.12 will inline decodeVarint for - // us, so we do it manually. This provides a 10-15% performance improvement - // on blockIter benchmarks on both go1.11 and go1.12. - // - // TODO(peter): remove this hack if go:inline is ever supported. - - var shared uint32 - if a := *((*uint8)(ptr)); a < 128 { - shared = uint32(a) - ptr = unsafe.Pointer(uintptr(ptr) + 1) - } else { - panic("first key in block should not share") - } - if shared != 0 { - panic("first key in block should not share") - } - - var unshared uint32 - if a := *((*uint8)(ptr)); a < 128 { - unshared = uint32(a) - ptr = unsafe.Pointer(uintptr(ptr) + 1) - } else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 { - unshared = uint32(b)<<7 | uint32(a) - ptr = unsafe.Pointer(uintptr(ptr) + 2) - } else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 { - unshared = uint32(c)<<14 | uint32(b)<<7 | uint32(a) - ptr = unsafe.Pointer(uintptr(ptr) + 3) - } else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 { - unshared = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a) - ptr = unsafe.Pointer(uintptr(ptr) + 4) - } else { - d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4))) - unshared = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a) - ptr = unsafe.Pointer(uintptr(ptr) + 5) - } - - // Skip the value length. - if a := *((*uint8)(ptr)); a < 128 { - ptr = unsafe.Pointer(uintptr(ptr) + 1) - } else if a := *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); a < 128 { - ptr = unsafe.Pointer(uintptr(ptr) + 2) - } else if a := *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); a < 128 { - ptr = unsafe.Pointer(uintptr(ptr) + 3) - } else if a := *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); a < 128 { - ptr = unsafe.Pointer(uintptr(ptr) + 4) - } else { - ptr = unsafe.Pointer(uintptr(ptr) + 5) - } - - firstKey := getBytes(ptr, int(unshared)) - // Manually inlining base.DecodeInternalKey provides a 5-10% speedup on - // BlockIter benchmarks. - if n := len(firstKey) - 8; n >= 0 { - i.firstKey.Trailer = binary.LittleEndian.Uint64(firstKey[n:]) - i.firstKey.UserKey = firstKey[:n:n] - if i.globalSeqNum != 0 { - i.firstKey.SetSeqNum(i.globalSeqNum) - } - } else { - // TODO: propagate this error? - i.firstKey.Trailer = uint64(InternalKeyKindInvalid) - i.firstKey.UserKey = nil - } -} - func (i *blockIter) decodeInternalKey(key []byte) { // Manually inlining base.DecodeInternalKey provides a 5-10% speedup on // BlockIter benchmarks. diff --git a/github.com/cockroachdb/pebble/sstable/reader.go b/github.com/cockroachdb/pebble/sstable/reader.go index 34a562d978..05f741b2e9 100644 --- a/github.com/cockroachdb/pebble/sstable/reader.go +++ b/github.com/cockroachdb/pebble/sstable/reader.go @@ -87,31 +87,6 @@ type singleLevelIterator struct { dataBH BlockHandle err error closeHook func(i Iterator) error - - // boundsCmp and positionedUsingLatestBounds are for optimizing iteration - // that uses multiple adjacent bounds. The seek after setting a new bound - // can use the fact that the iterator is either within the previous bounds - // or exactly one key before or after the bounds. If the new bounds is - // after/before the previous bounds, and we are already positioned at a - // block that is relevant for the new bounds, we can try to first position - // using Next/Prev instead of doing a more expensive seek. - // - // When there are wide files at higher levels that match the bounds - // but don't have any data for the bound, we will already be - // positioned at the key beyond the bounds and won't need to do much - // work -- given that most data is in L6, such files are likely to - // dominate the performance of the mergingIter, and may be the main - // benefit of this performance optimization (of course it also helps - // when the file that has the data has successive seeks that stay in - // the same block). - // - // Specifically, boundsCmp captures the relationship between the previous - // and current bounds, if the iterator had been positioned after setting - // the previous bounds. If it was not positioned, i.e., Seek/First/Last - // were not called, we don't know where it is positioned and cannot - // optimize. - boundsCmp int - positionedUsingLatestBounds bool } // singleLevelIterator implements the base.InternalIterator interface. @@ -247,68 +222,6 @@ func (i *singleLevelIterator) loadBlock() bool { return true } -func (i *singleLevelIterator) initBoundsForAlreadyLoadedBlock() { - i.blockLower = i.lower - if i.blockLower != nil { - if i.data.firstKey.UserKey != nil && i.cmp(i.blockLower, i.data.firstKey.UserKey) < 0 { - // The lower-bound is less than the first key in the block. No need - // to check the lower-bound again for this block. - i.blockLower = nil - } - } - i.blockUpper = i.upper - if i.blockUpper != nil && i.cmp(i.blockUpper, i.index.Key().UserKey) > 0 { - // The upper-bound is greater than the index key which itself is greater - // than or equal to every key in the block. No need to check the - // upper-bound again for this block. - i.blockUpper = nil - } -} - -// The number of times to call Next/Prev in a block before giving up and seeking. -// The value of 4 is arbitrary. -const numStepsBeforeSeek = 4 - -func (i *singleLevelIterator) trySeekGEUsingNextWithinBlock( - key []byte, -) (k *InternalKey, v []byte, done bool) { - k, v = i.data.Key(), i.data.Value() - for j := 0; j < numStepsBeforeSeek; j++ { - curKeyCmp := i.cmp(k.UserKey, key) - if curKeyCmp >= 0 { - if i.blockUpper != nil && i.cmp(k.UserKey, i.blockUpper) >= 0 { - return nil, nil, true - } - return k, v, true - } - k, v = i.data.Next() - if k == nil { - break - } - } - return k, v, false -} - -func (i *singleLevelIterator) trySeekLTUsingPrevWithinBlock( - key []byte, -) (k *InternalKey, v []byte, done bool) { - k, v = i.data.Key(), i.data.Value() - for j := 0; j < numStepsBeforeSeek; j++ { - curKeyCmp := i.cmp(k.UserKey, key) - if curKeyCmp < 0 { - if i.blockLower != nil && i.cmp(k.UserKey, i.blockLower) < 0 { - return nil, nil, true - } - return k, v, true - } - k, v = i.data.Prev() - if k == nil { - break - } - } - return k, v, false -} - func (i *singleLevelIterator) recordOffset() uint64 { offset := i.dataBH.Offset if i.data.Valid() { @@ -332,57 +245,22 @@ func (i *singleLevelIterator) recordOffset() uint64 { // caller to ensure that key is greater than or equal to the lower bound. func (i *singleLevelIterator) SeekGE(key []byte) (*InternalKey, []byte) { i.err = nil // clear cached iteration error - boundsCmp := i.boundsCmp - // Seek optimization only applies until iterator is first positioned after SetBounds. - i.boundsCmp = 0 - i.positionedUsingLatestBounds = true - return i.seekGEHelper(key, boundsCmp) -} - -// seekGEHelper contains the common functionality for SeekGE and SeekPrefixGE. -func (i *singleLevelIterator) seekGEHelper(key []byte, boundsCmp int) (*InternalKey, []byte) { - var dontSeekWithinBlock bool - - if !i.data.isInvalid() && !i.index.isInvalid() && i.data.Valid() && i.index.Valid() && - boundsCmp > 0 && i.cmp(key, i.index.Key().UserKey) <= 0 { - // Fast-path: The bounds have moved forward and this SeekGE is - // respecting the lower bound (guaranteed by Iterator). We know that - // the iterator must already be positioned within or just outside the - // previous bounds. Therefore it cannot be positioned at a block (or - // the position within that block) that is ahead of the seek position. - // However it can be positioned at an earlier block. This fast-path to - // use Next() on the block is only applied when we are already at the - // block that the slow-path (the else-clause) would load -- this is - // the motivation for the i.cmp(key, i.index.Key().UserKey) <= 0 - // predicate. - i.initBoundsForAlreadyLoadedBlock() - ikey, val, done := i.trySeekGEUsingNextWithinBlock(key) - if done { - return ikey, val - } - if ikey == nil { - // Done with this block. - dontSeekWithinBlock = true - } - } else { - if ikey, _ := i.index.SeekGE(key); ikey == nil { - // The target key is greater than any key in the sstable. Invalidate the - // block iterator so that a subsequent call to Prev() will return the last - // key in the table. - i.data.invalidate() - return nil, nil - } - if !i.loadBlock() { - return nil, nil - } + + if ikey, _ := i.index.SeekGE(key); ikey == nil { + // The target key is greater than any key in the sstable. Invalidate the + // block iterator so that a subsequent call to Prev() will return the last + // key in the table. + i.data.invalidate() + return nil, nil } - if !dontSeekWithinBlock { - if ikey, val := i.data.SeekGE(key); ikey != nil { - if i.blockUpper != nil && i.cmp(ikey.UserKey, i.blockUpper) >= 0 { - return nil, nil - } - return ikey, val + if !i.loadBlock() { + return nil, nil + } + if ikey, val := i.data.SeekGE(key); ikey != nil { + if i.blockUpper != nil && i.cmp(ikey.UserKey, i.blockUpper) >= 0 { + return nil, nil } + return ikey, val } return i.skipForward() } @@ -392,9 +270,6 @@ func (i *singleLevelIterator) seekGEHelper(key []byte, boundsCmp int) (*Internal // to the caller to ensure that key is greater than or equal to the lower bound. func (i *singleLevelIterator) SeekPrefixGE(prefix, key []byte) (*InternalKey, []byte) { i.err = nil // clear cached iteration error - boundsCmp := i.boundsCmp - // Seek optimization only applies until iterator is first positioned after SetBounds. - i.boundsCmp = 0 // Check prefix bloom filter. if i.reader.tableFilter != nil { @@ -411,9 +286,21 @@ func (i *singleLevelIterator) SeekPrefixGE(prefix, key []byte) (*InternalKey, [] return nil, nil } } - // Bloom filter matches, so this method will position the iterator. - i.positionedUsingLatestBounds = true - return i.seekGEHelper(key, boundsCmp) + + if ikey, _ := i.index.SeekGE(key); ikey == nil { + i.data.invalidate() + return nil, nil + } + if !i.loadBlock() { + return nil, nil + } + if ikey, val := i.data.SeekGE(key); ikey != nil { + if i.blockUpper != nil && i.cmp(ikey.UserKey, i.blockUpper) >= 0 { + return nil, nil + } + return ikey, val + } + return i.skipForward() } // SeekLT implements internalIterator.SeekLT, as documented in the pebble @@ -421,47 +308,18 @@ func (i *singleLevelIterator) SeekPrefixGE(prefix, key []byte) (*InternalKey, [] // caller to ensure that key is less than the upper bound. func (i *singleLevelIterator) SeekLT(key []byte) (*InternalKey, []byte) { i.err = nil // clear cached iteration error - boundsCmp := i.boundsCmp - // Seek optimization only applies until iterator is first positioned after SetBounds. - i.boundsCmp = 0 - i.positionedUsingLatestBounds = true - - var dontSeekWithinBlock bool - if !i.data.isInvalid() && !i.index.isInvalid() && i.data.Valid() && i.index.Valid() && - boundsCmp < 0 && i.cmp(i.data.firstKey.UserKey, key) < 0 { - // Fast-path: The bounds have moved backward, and this SeekLT is - // respecting the upper bound (guaranteed by Iterator). We know that - // the iterator must already be positioned within or just outside the - // previous bounds. Therefore it cannot be positioned at a block (or - // the position within that block) that is behind the seek position. - // However it can be positioned at a later block. This fast-path to - // use Prev() on the block is only applied when we are already at the - // block that can satisfy this seek -- this is the motivation for the - // the i.cmp(i.data.firstKey.UserKey, key) < 0 predicate. - i.initBoundsForAlreadyLoadedBlock() - ikey, val, done := i.trySeekLTUsingPrevWithinBlock(key) - if done { - return ikey, val - } - if ikey == nil { - // Done with this block. - dontSeekWithinBlock = true - } - } else { - if ikey, _ := i.index.SeekGE(key); ikey == nil { - i.index.Last() - } - if !i.loadBlock() { - return nil, nil - } + + if ikey, _ := i.index.SeekGE(key); ikey == nil { + i.index.Last() } - if !dontSeekWithinBlock { - if ikey, val := i.data.SeekLT(key); ikey != nil { - if i.blockLower != nil && i.cmp(ikey.UserKey, i.blockLower) < 0 { - return nil, nil - } - return ikey, val + if !i.loadBlock() { + return nil, nil + } + if ikey, val := i.data.SeekLT(key); ikey != nil { + if i.blockLower != nil && i.cmp(ikey.UserKey, i.blockLower) < 0 { + return nil, nil } + return ikey, val } // The index contains separator keys which may lie between // user-keys. Consider the user-keys: @@ -483,9 +341,6 @@ func (i *singleLevelIterator) SeekLT(key []byte) (*InternalKey, []byte) { // call to SeekGE(lower)). func (i *singleLevelIterator) First() (*InternalKey, []byte) { i.err = nil // clear cached iteration error - // Seek optimization only applies until iterator is first positioned after SetBounds. - i.boundsCmp = 0 - i.positionedUsingLatestBounds = true if ikey, _ := i.index.First(); ikey == nil { i.data.invalidate() @@ -509,9 +364,6 @@ func (i *singleLevelIterator) First() (*InternalKey, []byte) { // SeekLT(upper)) func (i *singleLevelIterator) Last() (*InternalKey, []byte) { i.err = nil // clear cached iteration error - // Seek optimization only applies until iterator is first positioned after SetBounds. - i.boundsCmp = 0 - i.positionedUsingLatestBounds = true if ikey, _ := i.index.Last(); ikey == nil { i.data.invalidate() @@ -534,9 +386,6 @@ func (i *singleLevelIterator) Last() (*InternalKey, []byte) { // Note: compactionIterator.Next mirrors the implementation of Iterator.Next // due to performance. Keep the two in sync. func (i *singleLevelIterator) Next() (*InternalKey, []byte) { - // Seek optimization only applies until iterator is first positioned after SetBounds. - i.boundsCmp = 0 - if i.err != nil { return nil, nil } @@ -552,9 +401,6 @@ func (i *singleLevelIterator) Next() (*InternalKey, []byte) { // Prev implements internalIterator.Prev, as documented in the pebble // package. func (i *singleLevelIterator) Prev() (*InternalKey, []byte) { - // Seek optimization only applies until iterator is first positioned after SetBounds. - i.boundsCmp = 0 - if i.err != nil { return nil, nil } @@ -669,15 +515,6 @@ func (i *singleLevelIterator) String() string { // SetBounds implements internalIterator.SetBounds, as documented in the pebble // package. func (i *singleLevelIterator) SetBounds(lower, upper []byte) { - i.boundsCmp = 0 - if i.positionedUsingLatestBounds { - if i.upper != nil && lower != nil && i.cmp(i.upper, lower) <= 0 { - i.boundsCmp = +1 - } else if i.lower != nil && upper != nil && i.cmp(upper, i.lower) <= 0 { - i.boundsCmp = -1 - } - i.positionedUsingLatestBounds = false - } i.lower = lower i.upper = upper i.blockLower = nil @@ -824,27 +661,15 @@ func (i *twoLevelIterator) String() string { func (i *twoLevelIterator) SeekGE(key []byte) (*InternalKey, []byte) { i.err = nil // clear cached iteration error - if i.topLevelIndex.isInvalid() || !i.topLevelIndex.Valid() || i.boundsCmp <= 0 || - i.cmp(key, i.topLevelIndex.Key().UserKey) > 0 { - // Slow-path: need to position the topLevelIndex. - if ikey, _ := i.topLevelIndex.SeekGE(key); ikey == nil { - i.data.invalidate() - i.index.invalidate() - return nil, nil - } + if ikey, _ := i.topLevelIndex.SeekGE(key); ikey == nil { + i.data.invalidate() + i.index.invalidate() + return nil, nil + } - if !i.loadIndex() { - return nil, nil - } + if !i.loadIndex() { + return nil, nil } - // Else fast-path: The bounds have moved forward and this SeekGE is - // respecting the lower bound (guaranteed by Iterator). We know that - // the iterator must already be positioned within or just outside the - // previous bounds. Therefore the topLevelIndex iter cannot be - // positioned at an entry ahead of the seek position (though it can be - // positioned behind). The !i.cmp(key, i.topLevelIndex.Key().UserKey) > 0 - // confirms that it is not behind. Since it is not ahead and not behind - // it must be at the right position. if ikey, val := i.singleLevelIterator.SeekGE(key); ikey != nil { return ikey, val @@ -858,30 +683,15 @@ func (i *twoLevelIterator) SeekGE(key []byte) (*InternalKey, []byte) { func (i *twoLevelIterator) SeekPrefixGE(prefix, key []byte) (*InternalKey, []byte) { i.err = nil // clear cached iteration error - // TODO(sumeer): check the bloom filter before doing other work and not - // delay it until singleLevelIterator.SeekPrefixGE. - - if i.topLevelIndex.isInvalid() || !i.topLevelIndex.Valid() || i.boundsCmp <= 0 || - i.cmp(key, i.topLevelIndex.Key().UserKey) > 0 { - // Slow-path: need to position the topLevelIndex. - if ikey, _ := i.topLevelIndex.SeekGE(key); ikey == nil { - i.data.invalidate() - i.index.invalidate() - return nil, nil - } + if ikey, _ := i.topLevelIndex.SeekGE(key); ikey == nil { + i.data.invalidate() + i.index.invalidate() + return nil, nil + } - if !i.loadIndex() { - return nil, nil - } + if !i.loadIndex() { + return nil, nil } - // Else fast-path: The bounds have moved forward and this SeekGE is - // respecting the lower bound (guaranteed by Iterator). We know that - // the iterator must already be positioned within or just outside the - // previous bounds. Therefore the topLevelIndex iter cannot be - // positioned at an entry ahead of the seek position (though it can be - // positioned behind). The !i.cmp(key, i.topLevelIndex.Key().UserKey) > 0 - // confirms that it is not behind. Since it is not ahead and not behind - // it must be at the right position. if ikey, val := i.singleLevelIterator.SeekPrefixGE(prefix, key); ikey != nil { return ikey, val @@ -894,13 +704,7 @@ func (i *twoLevelIterator) SeekPrefixGE(prefix, key []byte) (*InternalKey, []byt // caller to ensure that key is less than the upper bound. func (i *twoLevelIterator) SeekLT(key []byte) (*InternalKey, []byte) { i.err = nil // clear cached iteration error - // Seek optimization only applies until iterator is first positioned after SetBounds. - i.boundsCmp = 0 - // NB: Unlike SeekGE, we don't have a fast-path here since we don't know - // whether the topLevelIndex is positioned after the position that would - // be returned by doing i.topLevelIndex.SeekGE(). To know this we would - // need to know the index key preceding the current one. if ikey, _ := i.topLevelIndex.SeekGE(key); ikey == nil { if ikey, _ := i.topLevelIndex.Last(); ikey == nil { i.data.invalidate() @@ -931,8 +735,6 @@ func (i *twoLevelIterator) SeekLT(key []byte) (*InternalKey, []byte) { // call to SeekGE(lower)). func (i *twoLevelIterator) First() (*InternalKey, []byte) { i.err = nil // clear cached iteration error - // Seek optimization only applies until iterator is first positioned after SetBounds. - i.boundsCmp = 0 if ikey, _ := i.topLevelIndex.First(); ikey == nil { return nil, nil @@ -954,8 +756,6 @@ func (i *twoLevelIterator) First() (*InternalKey, []byte) { // SeekLT(upper)) func (i *twoLevelIterator) Last() (*InternalKey, []byte) { i.err = nil // clear cached iteration error - // Seek optimization only applies until iterator is first positioned after SetBounds. - i.boundsCmp = 0 if ikey, _ := i.topLevelIndex.Last(); ikey == nil { return nil, nil diff --git a/github.com/cockroachdb/pebble/sstable/unsafe.go b/github.com/cockroachdb/pebble/sstable/unsafe.go index 0030be0456..11ec068914 100644 --- a/github.com/cockroachdb/pebble/sstable/unsafe.go +++ b/github.com/cockroachdb/pebble/sstable/unsafe.go @@ -4,10 +4,14 @@ package sstable -import "unsafe" +import ( + "unsafe" + + "github.com/cockroachdb/pebble/internal/manual" +) func getBytes(ptr unsafe.Pointer, length int) []byte { - return (*[1 << 30]byte)(ptr)[:length:length] + return (*[manual.MaxArrayLen]byte)(ptr)[:length:length] } func decodeVarint(ptr unsafe.Pointer) (uint32, unsafe.Pointer) { diff --git a/github.com/cockroachdb/pebble/tool/make_test_find_db.go b/github.com/cockroachdb/pebble/tool/make_test_find_db.go index ecfc3506d4..a83e88e9de 100644 --- a/github.com/cockroachdb/pebble/tool/make_test_find_db.go +++ b/github.com/cockroachdb/pebble/tool/make_test_find_db.go @@ -98,11 +98,6 @@ func (d *db) ingest(keyVals ...string) { if err := d.db.Ingest([]string{path}); err != nil { log.Fatal(err) } - - if err := fs.Remove(path); err != nil { - // TODO(peter): why is the remove sometimes failing under CI? - log.Print(err) - } } func (d *db) flush() { diff --git a/modules.txt b/modules.txt index e0e1479d7a..37e207df60 100644 --- a/modules.txt +++ b/modules.txt @@ -205,7 +205,7 @@ github.com/cockroachdb/gostdlib/x/tools/internal/semver github.com/cockroachdb/gostdlib/x/tools/internal/span # github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f github.com/cockroachdb/logtags -# github.com/cockroachdb/pebble v0.0.0-20200904203921-b66f9d0155ab +# github.com/cockroachdb/pebble v0.0.0-20200908153531-a9c6d3b6685c github.com/cockroachdb/pebble github.com/cockroachdb/pebble/bloom github.com/cockroachdb/pebble/internal/arenaskl From 193db6c6d9f7202701f9b27947ed25bba7fe6735 Mon Sep 17 00:00:00 2001 From: Oliver Tan Date: Tue, 8 Sep 2020 11:11:47 -0700 Subject: [PATCH 40/49] bump twpayne/go-geom to v1.3.6 --- github.com/twpayne/go-geom/multipolygon.go | 29 ++++++++++++++-------- modules.txt | 2 +- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/github.com/twpayne/go-geom/multipolygon.go b/github.com/twpayne/go-geom/multipolygon.go index bfb74de464..b9c8ca6086 100644 --- a/github.com/twpayne/go-geom/multipolygon.go +++ b/github.com/twpayne/go-geom/multipolygon.go @@ -48,15 +48,19 @@ func (g *MultiPolygon) NumPolygons() int { // Polygon returns the ith Polygon. func (g *MultiPolygon) Polygon(i int) *Polygon { + if len(g.endss[i]) == 0 { + return NewPolygon(g.layout) + } + // Find the offset from the previous non-empty polygon element. offset := 0 - if i > 0 { - ends := g.endss[i-1] + lastNonEmptyIdx := i - 1 + for lastNonEmptyIdx >= 0 { + ends := g.endss[lastNonEmptyIdx] if len(ends) > 0 { offset = ends[len(ends)-1] + break } - } - if len(g.endss[i]) == 0 { - return NewPolygon(g.layout) + lastNonEmptyIdx-- } ends := make([]int, len(g.endss[i])) if offset == 0 { @@ -75,12 +79,15 @@ func (g *MultiPolygon) Push(p *Polygon) error { return ErrLayoutMismatch{Got: p.layout, Want: g.layout} } offset := len(g.flatCoords) - ends := make([]int, len(p.ends)) - if offset == 0 { - copy(ends, p.ends) - } else { - for i, end := range p.ends { - ends[i] = end + offset + var ends []int + if len(p.ends) > 0 { + ends = make([]int, len(p.ends)) + if offset == 0 { + copy(ends, p.ends) + } else { + for i, end := range p.ends { + ends[i] = end + offset + } } } g.flatCoords = append(g.flatCoords, p.flatCoords...) diff --git a/modules.txt b/modules.txt index 37e207df60..8215f7ffd5 100644 --- a/modules.txt +++ b/modules.txt @@ -689,7 +689,7 @@ github.com/spf13/pflag # github.com/stretchr/testify v1.6.1 github.com/stretchr/testify/assert github.com/stretchr/testify/require -# github.com/twpayne/go-geom v1.3.5 +# github.com/twpayne/go-geom v1.3.6 github.com/twpayne/go-geom github.com/twpayne/go-geom/bigxy github.com/twpayne/go-geom/encoding/ewkb From bccfc3e967611498b876e3e0f6839d75f17c14c5 Mon Sep 17 00:00:00 2001 From: Raphael 'kena' Poss Date: Thu, 10 Sep 2020 18:33:33 +0200 Subject: [PATCH 41/49] Bump cockroachdb/redact --- github.com/cockroachdb/errors/go.mod | 2 +- github.com/cockroachdb/errors/go.sum | 4 ++-- github.com/cockroachdb/redact/markers_internal_print.go | 9 +++++++++ modules.txt | 4 ++-- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/github.com/cockroachdb/errors/go.mod b/github.com/cockroachdb/errors/go.mod index ca5e0412bc..3e5cade00f 100644 --- a/github.com/cockroachdb/errors/go.mod +++ b/github.com/cockroachdb/errors/go.mod @@ -5,7 +5,7 @@ go 1.13 require ( github.com/cockroachdb/datadriven v1.0.0 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f - github.com/cockroachdb/redact v1.0.5 + github.com/cockroachdb/redact v1.0.6 github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 github.com/gogo/protobuf v1.3.1 github.com/gogo/status v1.1.0 diff --git a/github.com/cockroachdb/errors/go.sum b/github.com/cockroachdb/errors/go.sum index 24aca82e9f..fd9be7e02e 100644 --- a/github.com/cockroachdb/errors/go.sum +++ b/github.com/cockroachdb/errors/go.sum @@ -17,8 +17,8 @@ github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlN github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/cockroachdb/redact v1.0.5 h1:yxqIMS6G2Bvi6GiSHFmsrFGO3aP1rwt8cOm4pixw9eY= -github.com/cockroachdb/redact v1.0.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/redact v1.0.6 h1:W34uRRyNR4dlZFd0MibhNELsZSgMkl52uRV/tA1xToY= +github.com/cockroachdb/redact v1.0.6/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= diff --git a/github.com/cockroachdb/redact/markers_internal_print.go b/github.com/cockroachdb/redact/markers_internal_print.go index 672821e301..de21f8b00f 100644 --- a/github.com/cockroachdb/redact/markers_internal_print.go +++ b/github.com/cockroachdb/redact/markers_internal_print.go @@ -39,6 +39,15 @@ func printArgFn(p *internalFmt.InternalPrinter, arg interface{}, verb rune) (new return len(internalFmt.Buf(p)) } + // nil arguments are printed as-is. Note: a nil argument under + // interface{} is not the same as a nil pointer passed via a pointer + // of a concrete type. The latter kind of nil goes through + // redaction as usual, because it may have its own custom Format() method. + if arg == nil { + internalFmt.PrintArg(p, arg, verb) + return len(internalFmt.Buf(p)) + } + // RedactableBytes/RedactableString are already formatted as // redactable. Include them as-is. // diff --git a/modules.txt b/modules.txt index 8215f7ffd5..0281464844 100644 --- a/modules.txt +++ b/modules.txt @@ -167,7 +167,7 @@ github.com/cockroachdb/crlfmt/internal/parser github.com/cockroachdb/crlfmt/internal/render # github.com/cockroachdb/datadriven v1.0.1-0.20200826112548-92602d883b11 github.com/cockroachdb/datadriven -# github.com/cockroachdb/errors v1.7.4 +# github.com/cockroachdb/errors v1.7.5 github.com/cockroachdb/errors github.com/cockroachdb/errors/assert github.com/cockroachdb/errors/barriers @@ -228,7 +228,7 @@ github.com/cockroachdb/pebble/internal/record github.com/cockroachdb/pebble/sstable github.com/cockroachdb/pebble/tool github.com/cockroachdb/pebble/vfs -# github.com/cockroachdb/redact v1.0.5 +# github.com/cockroachdb/redact v1.0.6 github.com/cockroachdb/redact github.com/cockroachdb/redact/internal github.com/cockroachdb/redact/internal/fmtsort From ceb347c0ecffb04144e589b05cdd557000c95446 Mon Sep 17 00:00:00 2001 From: Jackson Owens Date: Thu, 10 Sep 2020 17:02:10 -0400 Subject: [PATCH 42/49] bump Pebble to d7a278abeed4 d7a278ab compaction: Ensure that user keys aren't split across SSTs when flushing 9af7bb12 db: close previous WAL before linking next WAL 06b3565a *: Add compactionOutputSplitter, refactor compaction output switch logic caf883db *: add includesBase argument to ValueMerger.Finish --- github.com/cockroachdb/pebble/compaction.go | 559 ++++++++++++------ .../cockroachdb/pebble/compaction_iter.go | 5 +- github.com/cockroachdb/pebble/db.go | 41 +- .../pebble/internal/base/merger.go | 11 +- github.com/cockroachdb/pebble/iterator.go | 6 +- .../cockroachdb/pebble/level_checker.go | 8 +- modules.txt | 6 +- 7 files changed, 425 insertions(+), 211 deletions(-) diff --git a/github.com/cockroachdb/pebble/compaction.go b/github.com/cockroachdb/pebble/compaction.go index 07f7885ff9..09e0cbdb67 100644 --- a/github.com/cockroachdb/pebble/compaction.go +++ b/github.com/cockroachdb/pebble/compaction.go @@ -66,6 +66,278 @@ type compactionLevel struct { files manifest.LevelSlice } +// compactionOutputSplitter is an interface for encapsulating logic around +// switching the output of a compaction to a new output file. Additional +// constraints around switching compaction outputs that are specific to that +// compaction type (eg. flush splits) are implemented in +// compactionOutputSplitters that compose other child compactionOutputSplitters. +type compactionOutputSplitter interface { + // shouldSplitBefore returns whether we should split outputs before the + // specified "current key". + shouldSplitBefore(key *InternalKey, tw *sstable.Writer) bool + // onNewOutput updates internal splitter state when the compaction switches + // to a new sstable, and returns the next limit for the new output which + // would get used to truncate range tombstones if the compaction iterator + // runs out of keys. The limit returned MUST be > key according to the + // compaction's comparator. The specified key is the first key in the new + // output, or nil if this sstable will only contain range tombstones already + // in the fragmenter. + onNewOutput(key *InternalKey) []byte +} + +// fileSizeSplitter is a compactionOutputSplitter that makes a determination +// to split outputs based on the estimated file size of the current output. +// Note that, unlike most other splitters, this splitter does not guarantee +// that it will advise splits only at user key change boundaries. +type fileSizeSplitter struct { + maxFileSize uint64 +} + +func (f *fileSizeSplitter) shouldSplitBefore(key *InternalKey, tw *sstable.Writer) bool { + // The Kind != RangeDelete part exists because EstimatedSize doesn't grow + // rightaway when a range tombstone is added to the fragmenter. It's always + // better to make a sequence of range tombstones visible to the fragmenter. + return key.Kind() != InternalKeyKindRangeDelete && tw != nil && + tw.EstimatedSize() >= f.maxFileSize +} + +func (f *fileSizeSplitter) onNewOutput(key *InternalKey) []byte { + return nil +} + +type grandparentLimitSplitter struct { + c *compaction + ve *versionEdit + limit []byte +} + +func (g *grandparentLimitSplitter) shouldSplitBefore(key *InternalKey, tw *sstable.Writer) bool { + return g.limit != nil && g.c.cmp(key.UserKey, g.limit) > 0 +} + +func (g *grandparentLimitSplitter) onNewOutput(key *InternalKey) []byte { + g.limit = nil + if g.c.rangeDelFrag.Empty() || len(g.ve.NewFiles) == 0 { + // In this case, `limit` will be a larger user key than `key.UserKey`, or + // nil. In either case, the inner loop will execute at least once to + // process `key`, and the input iterator will be advanced. + if key != nil { + g.limit = g.c.findGrandparentLimit(key.UserKey) + } else { // !c.rangeDelFrag.Empty() + g.limit = g.c.findGrandparentLimit(g.c.rangeDelFrag.Start()) + } + } else { + // There is a range tombstone spanning from the last file into the + // current one. Therefore this file's smallest boundary will overlap the + // last file's largest boundary. + // + // In this case, `limit` will be a larger user key than the previous + // file's largest key and correspond to a grandparent file's largest user + // key, or nil. Then, it is possible the inner loop executes zero times, + // and the output file contains only range tombstones. That is fine as + // long as the number of times we execute this case is bounded. Since + // `findGrandparentLimit()` returns a strictly larger user key each time + // and it corresponds to a grandparent file largest key, the number of + // times this case can execute is bounded by the number of grandparent + // files (plus one for the final time it returns nil). + // + // n > 0 since we cannot have seen range tombstones at the + // beginning of the first file. + n := len(g.ve.NewFiles) + g.limit = g.c.findGrandparentLimit(g.ve.NewFiles[n-1].Meta.Largest.UserKey) + // This conditional is necessary to maintain the invariant that + // successive calls to finishOutput() have an increasing + // limit, and that the fragmenter's *FlushTo() and Add() calls are + // always made with successively increasing user keys. It's possible + // for the last file's Meta.Largest.UserKey + // to be substantially less than the last key returned by + // the compactionIter, such as in a sequence of consecutive + // rangedel tombstones, of which some but not all get elided. + // Consider this example: + // + // Compaction input (all range tombstones in this snippet): + // a-b + // c-d + // e-f (elided) + // g-h (elided) <-- grandparent limit before this (at ff) + // i-j (elided) + // k-q <-- grandparent limit at k + // m-q + // + // Note that elided tombstones are added to the fragmenter, but + // removed before they make their way from the fragmenter onto + // iter.tombstones. They still affect the fragmenter's internal + // tracking of the key up to which all tombstones have been flushed. + // After the first output is cut with limit = g, the fragmenter + // is empty, so the next grandparent calculation happens with + // key = g, and returns k. We continue adding range tombstones to + // the fragmenter between [g,k], which includes k-q as we only + // switch outputs after the limit is exceeded. So key = m and + // limit = k when finishOutput is called. Since the start key in + // the fragmenter (k) matches the limit, and no point keys were + // added, we actually don't produce a new output (see the + // conditional in finishOutput() on why). + // + // When we try to calculate the next grandparent limit, since + // c.rangeDelFrag.Empty() == false (as k-q is sitting in it), + // we fall into this case, and use the end key of the last written + // sstable (which was [a-d]) to calculate the grandparent limit. + // That gets us g again, which gets us to call + // c.rangeDelFrag.FlushTo(g), which violates the + // invariant that the current flush-to key (g) be greater than the + // last one (k). + // + // To solve this, if the grandparent limit falls "in between" + // ve.NewFiles[n-1].Meta.Largest.UserKey and c.rangeDelFrag.Start(), + // re-calculate a higher limit ahead of that key. + if g.c.rangeDelFrag.Start() != nil && g.c.cmp(g.limit, g.c.rangeDelFrag.Start()) <= 0 { + g.limit = g.c.findGrandparentLimit(g.c.rangeDelFrag.Start()) + } + } + return g.limit +} + +type l0LimitSplitter struct { + c *compaction + ve *versionEdit + limit []byte +} + +func (l *l0LimitSplitter) shouldSplitBefore(key *InternalKey, tw *sstable.Writer) bool { + return l.limit != nil && l.c.cmp(key.UserKey, l.limit) > 0 +} + +func (l *l0LimitSplitter) onNewOutput(key *InternalKey) []byte { + l.limit = nil + // For flushes being split across multiple sstables, call + // findL0Limit to find the next L0 limit. + if key != nil { + l.limit = l.c.findL0Limit(key.UserKey) + } else { + // Use the start key of the first pending tombstone to find the + // next limit. All pending tombstones have the same start key. + // We use this as opposed to the end key of the + // last written sstable to effectively handle cases like these: + // + // a.SET.3 + // (L0 limit at b) + // d.RANGEDEL.4:f + // + // In this case, the partition after b has only range deletions, + // so if we were to find the L0 limit after the last written + // key at the split point (key a), we'd get the limit b again, + // and finishOutput() would not advance any further because + // the next range tombstone to write does not start until after + // the L0 split point. + startKey := l.c.rangeDelFrag.Start() + if startKey != nil { + l.limit = l.c.findL0Limit(startKey) + } + } + return l.limit +} + +// splitterGroup is a compactionOutputSplitter that splits whenever one of its +// child splitters advises a compaction split. +type splitterGroup struct { + cmp Compare + splitters []compactionOutputSplitter +} + +func (a *splitterGroup) shouldSplitBefore(key *InternalKey, tw *sstable.Writer) bool { + for _, splitter := range a.splitters { + if splitter.shouldSplitBefore(key, tw) { + return true + } + } + return false +} + +func (a *splitterGroup) onNewOutput(key *InternalKey) []byte { + var earliestLimit []byte + for _, splitter := range a.splitters { + limit := splitter.onNewOutput(key) + if limit == nil { + continue + } + if earliestLimit == nil || a.cmp(limit, earliestLimit) < 0 { + earliestLimit = limit + } + } + return earliestLimit +} + +// userKeyChangeSplitter is a compactionOutputSplitter that takes in a child +// splitter, and splits when 1) that child splitter has advised a split, and 2) +// the compaction output is at the boundary between two user keys (also +// the boundary between atomic compaction units). Use this splitter to wrap +// any splitters that don't guarantee user key splits (i.e. splitters that make +// their determinatino in ways other than comparing the current key against a +// limit key. +type userKeyChangeSplitter struct { + cmp Compare + splitOnNextUserKey bool + savedKey []byte + splitter compactionOutputSplitter +} + +func (u *userKeyChangeSplitter) shouldSplitBefore(key *InternalKey, tw *sstable.Writer) bool { + if u.splitOnNextUserKey && u.cmp(u.savedKey, key.UserKey) != 0 { + u.splitOnNextUserKey = false + u.savedKey = u.savedKey[:0] + return true + } + if u.splitter.shouldSplitBefore(key, tw) { + u.splitOnNextUserKey = true + u.savedKey = append(u.savedKey[:0], key.UserKey...) + } + return false +} + +func (u *userKeyChangeSplitter) onNewOutput(key *InternalKey) []byte { + return u.splitter.onNewOutput(key) +} + +// nonZeroSeqNumSplitter is a compactionOutputSplitter that takes in a child +// splitter, and advises a split when 1) that child splitter advises a split, +// and 2) the compaction output is at a point where the previous point sequence +// number is nonzero. +type nonZeroSeqNumSplitter struct { + c *compaction + splitter compactionOutputSplitter + prevPointSeqNum uint64 + splitOnNonZeroSeqNum bool +} + +func (n *nonZeroSeqNumSplitter) shouldSplitBefore(key *InternalKey, tw *sstable.Writer) bool { + curSeqNum := key.SeqNum() + keyKind := key.Kind() + prevPointSeqNum := n.prevPointSeqNum + if keyKind != InternalKeyKindRangeDelete { + n.prevPointSeqNum = curSeqNum + } + + if n.splitOnNonZeroSeqNum { + if prevPointSeqNum > 0 || n.c.rangeDelFrag.Empty() { + n.splitOnNonZeroSeqNum = false + return true + } + } else if n.splitter.shouldSplitBefore(key, tw) { + userKeyChange := curSeqNum > prevPointSeqNum + if prevPointSeqNum > 0 || n.c.rangeDelFrag.Empty() || userKeyChange { + return true + } + n.splitOnNonZeroSeqNum = true + } + return false +} + +func (n *nonZeroSeqNumSplitter) onNewOutput(key *InternalKey) []byte { + n.prevPointSeqNum = InternalKeySeqNumMax + n.splitOnNonZeroSeqNum = false + return n.splitter.onNewOutput(key) +} + type compactionKind string const ( @@ -411,21 +683,38 @@ func (c *compaction) findGrandparentLimit(start []byte) []byte { } // findL0Limit takes the start key for a table and returns the user key to which -// that table can be extended without excessively overlapping the grandparent -// level, or hitting the next l0Limit, whichever happens sooner. For non-flushes -// this function passes through to findGrandparentLimit. +// that table can be extended without hitting the next l0Limit. Having flushed +// sstables "bridging across" an l0Limit could lead to increased L0 -> LBase +// compaction sizes as well as elevated read amplification. func (c *compaction) findL0Limit(start []byte) []byte { - grandparentLimit := c.findGrandparentLimit(start) if c.startLevel.level > -1 || c.outputLevel.level != 0 || len(c.l0Limits) == 0 { - return grandparentLimit + return nil } index := sort.Search(len(c.l0Limits), func(i int) bool { return c.cmp(c.l0Limits[i], start) > 0 }) - if index < len(c.l0Limits) && (grandparentLimit == nil || c.cmp(c.l0Limits[index], grandparentLimit) < 0) { + if index < len(c.l0Limits) { return c.l0Limits[index] } - return grandparentLimit + return nil +} + +// errorOnUserKeyOverlap returns an error if the last two written sstables in +// this compaction have revisions of the same user key present in both sstables, +// when it shouldn't (eg. when splitting flushes). +func (c *compaction) errorOnUserKeyOverlap(ve *versionEdit) error { + if n := len(ve.NewFiles); n > 1 { + meta := ve.NewFiles[n-1].Meta + prevMeta := ve.NewFiles[n-2].Meta + if prevMeta.Largest.Trailer != InternalKeyRangeDeleteSentinel && + c.cmp(prevMeta.Largest.UserKey, meta.Smallest.UserKey) >= 0 { + return errors.Errorf("pebble: compaction split user key across two sstables: %s in %s and %s", + prevMeta.Largest.Pretty(c.formatKey), + prevMeta.FileNum, + meta.FileNum) + } + } + return nil } // allowZeroSeqNum returns true if seqnum's can be zeroed if there are no @@ -1775,12 +2064,79 @@ func (d *DB) runCompaction( c.largest.Pretty(d.opts.Comparer.FormatKey)) } } + // Verify that when splitting flushes, we never split different + // revisions of the same user key across two different sstables. + if splittingFlush { + if err := c.errorOnUserKeyOverlap(ve); err != nil { + return err + } + } if err := meta.Validate(d.cmp, d.opts.Comparer.FormatKey); err != nil { return err } return nil } + // compactionOutputSplitters contain all logic to determine whether the + // compaction loop should stop writing to one output sstable and switch to + // a new one. Some splitters can wrap other splitters, and + // the splitterGroup can be composed of multiple splitters. In this case, + // we start off with splitters for file sizes, grandparent limits, and (for + // flush splits) L0 limits, before wrapping them in an splitterGroup. + // + // There is a complication here: we can't split outputs where the largest + // key on the left side has a seqnum of zero. This limitation + // exists because a range tombstone which extends into the next sstable + // will cause the smallest key for the next sstable to have the same user + // key, but we need the two tables to be disjoint in key space. Consider + // the scenario: + // + // a#RANGEDEL-c,3 b#SET,0 + // + // If b#SET,0 is the last key added to an sstable, the range tombstone + // [b-c)#3 will extend into the next sstable. The boundary generation + // code in finishOutput() will compute the smallest key for that sstable + // as b#RANGEDEL,3 which sorts before b#SET,0. Normally we just adjust + // the seqnum of this key, but that isn't possible for seqnum 0. To ensure + // we only split where the previous point key has a zero seqnum, we wrap + // our splitters with a nonZeroSeqNumSplitter. + // + // Another case where we may not be able to switch SSTables right away is + // when we are splitting a flush. We do not split the same user key + // across different sstables within one flush, so the userKeyChangeSplitter + // ensures we are at a user key change boundary when doing a split. + outputSplitters := []compactionOutputSplitter{ + &fileSizeSplitter{maxFileSize: c.maxOutputFileSize}, + &grandparentLimitSplitter{c: c, ve: ve}, + } + var splitter compactionOutputSplitter + if splittingFlush { + // outputSplitters[0] is the file size splitter, which doesn't guarantee + // that all advised splits will be at user key change boundaries. Wrap + // it in a userKeyChangeSplitter. + outputSplitters[0] = &userKeyChangeSplitter{ + cmp: c.cmp, + splitter: outputSplitters[0], + } + outputSplitters = append(outputSplitters, &l0LimitSplitter{c: c, ve: ve}) + } + splitter = &splitterGroup{ + cmp: c.cmp, + splitters: outputSplitters, + } + // Flush splits don't need nonzero last-point-key seqnums at split + // boundaries because when flushing to L0, we are able to guarantee that + // the end key of tombstones will also be truncated (through the + // TruncateAndFlushTo call), and no user keys will + // be split between sstables. So a nonZeroSeqNumSplitter is unnecessary + // in that case. + if !splittingFlush { + splitter = &nonZeroSeqNumSplitter{ + c: c, + splitter: splitter, + } + } + // Each outer loop iteration produces one output file. An iteration that // produces a file containing point keys (and optionally range tombstones) // guarantees that the input iterator advanced. An iteration that produces @@ -1790,159 +2146,23 @@ func (d *DB) runCompaction( // progress guarantees ensure that eventually the input iterator will be // exhausted and the range tombstone fragments will all be flushed. for key, val := iter.First(); key != nil || !c.rangeDelFrag.Empty(); { - var limit []byte - if splittingFlush { - // For flushes being split across multiple sstables, call - // findL0Limit to find the next L0 limit. - if key != nil { - limit = c.findL0Limit(key.UserKey) - } else { - // Use the start key of the first pending tombstone to find the - // next limit. All pending tombstones have the same start key. - // We use this as opposed to the end key of the - // last written sstable to effectively handle cases like these: - // - // a.SET.3 - // (L0 limit at b) - // d.RANGEDEL.4:f - // - // In this case, the partition after b has only range deletions, - // so if we were to find the L0 limit after the last written - // key at the split point (key a), we'd get the limit b again, - // and finishOutput() would not advance any further because - // the next range tombstone to write does not start until after - // the L0 split point. - startKey := c.rangeDelFrag.Start() - if startKey != nil { - limit = c.findL0Limit(startKey) - } - } - } else if c.rangeDelFrag.Empty() || len(ve.NewFiles) == 0 { - // In this case, `limit` will be a larger user key than `key.UserKey`, or - // nil. In either case, the inner loop will execute at least once to - // process `key`, and the input iterator will be advanced. - if key != nil { - limit = c.findGrandparentLimit(key.UserKey) - } else { // !c.rangeDelFrag.Empty() - limit = c.findGrandparentLimit(c.rangeDelFrag.Start()) - } - } else { - // There is a range tombstone spanning from the last file into the - // current one. Therefore this file's smallest boundary will overlap the - // last file's largest boundary. - // - // In this case, `limit` will be a larger user key than the previous - // file's largest key and correspond to a grandparent file's largest user - // key, or nil. Then, it is possible the inner loop executes zero times, - // and the output file contains only range tombstones. That is fine as - // long as the number of times we execute this case is bounded. Since - // `findGrandparentLimit()` returns a strictly larger user key each time - // and it corresponds to a grandparent file largest key, the number of - // times this case can execute is bounded by the number of grandparent - // files (plus one for the final time it returns nil). - // - // n > 0 since we cannot have seen range tombstones at the - // beginning of the first file. - n := len(ve.NewFiles) - limit = c.findGrandparentLimit(ve.NewFiles[n-1].Meta.Largest.UserKey) - // This conditional is necessary to maintain the invariant that - // successive calls to finishOutput() have an increasing - // limit, and that the fragmenter's *FlushTo() and Add() calls are - // always made with successively increasing user keys. It's possible - // for the last file's Meta.Largest.UserKey - // to be substantially less than the last key returned by - // the compactionIter, such as in a sequence of consecutive - // rangedel tombstones, of which some but not all get elided. - // Consider this example: - // - // Compaction input (all range tombstones in this snippet): - // a-b - // c-d - // e-f (elided) - // g-h (elided) <-- grandparent limit before this (at ff) - // i-j (elided) - // k-q <-- grandparent limit at k - // m-q - // - // Note that elided tombstones are added to the fragmenter, but - // removed before they make their way from the fragmenter onto - // iter.tombstones. They still affect the fragmenter's internal - // tracking of the key up to which all tombstones have been flushed. - // After the first output is cut with limit = g, the fragmenter - // is empty, so the next grandparent calculation happens with - // key = g, and returns k. We continue adding range tombstones to - // the fragmenter between [g,k], which includes k-q as we only - // switch outputs after the limit is exceeded. So key = m and - // limit = k when finishOutput is called. Since the start key in - // the fragmenter (k) matches the limit, and no point keys were - // added, we actually don't produce a new output (see the - // conditional in finishOutput() on why). - // - // When we try to calculate the next grandparent limit, since - // c.rangeDelFrag.Empty() == false (as k-q is sitting in it), - // we fall into this case, and use the end key of the last written - // sstable (which was [a-d]) to calculate the grandparent limit. - // That gets us g again, which gets us to call - // c.rangeDelFrag.FlushTo(g), which violates the - // invariant that the current flush-to key (g) be greater than the - // last one (k). - // - // To solve this, if the grandparent limit falls "in between" - // ve.NewFiles[n-1].Meta.Largest.UserKey and c.rangeDelFrag.Start(), - // re-calculate a higher limit ahead of that key. - if c.rangeDelFrag.Start() != nil && c.cmp(limit, c.rangeDelFrag.Start()) <= 0 { - limit = c.findGrandparentLimit(c.rangeDelFrag.Start()) - } - } + limit := splitter.onNewOutput(key) // Each inner loop iteration processes one key from the input iterator. - passedGrandparentLimit := false prevPointSeqNum := InternalKeySeqNumMax for ; key != nil; key, val = iter.Next() { - // Break out of this loop and switch to a new sstable if we've reached - // the grandparent limit. There is a complication here: we can't create - // an sstable where the largest key has a seqnum of zero. This limitation - // exists because a range tombstone which extends into the next sstable - // will cause the smallest key for the next sstable to have the same user - // key, but we need the two tables to be disjoint in key space. Consider - // the scenario: - // - // a#RANGEDEL-c,3 b#SET,0 - // - // If b#SET,0 is the last key added to an sstable, the range tombstone - // [b-c)#3 will extend into the next sstable. The boundary generation - // code in finishOutput() will compute the smallest key for that sstable - // as b#RANGEDEL,3 which sorts before b#SET,0. Normally we just adjust - // the seqnum of this key, but that isn't possible for seqnum 0. - if passedGrandparentLimit || (limit != nil && c.cmp(key.UserKey, limit) > 0) { - // The flush split exception exists because, when flushing - // to L0, we are able to guarantee that the end key of - // tombstones will also be truncated (through the - // TruncateAndFlushTo call), and no user keys will - // be split between sstables. - if prevPointSeqNum != 0 || c.rangeDelFrag.Empty() || splittingFlush { - if passedGrandparentLimit || splittingFlush { - limit = key.UserKey - } - if splittingFlush { - // Flush all tombstones up until key.UserKey, and - // truncate them at that key. - // - // The fragmenter could save the passed-in key. As this - // key could live beyond the write into the current - // sstable output file, make a copy. - c.rangeDelFrag.TruncateAndFlushTo(key.Clone().UserKey) - } - break + if splitter.shouldSplitBefore(key, tw) { + limit = key.UserKey + if splittingFlush { + // Flush all tombstones up until key.UserKey, and + // truncate them at that key. + // + // The fragmenter could save the passed-in key. As this + // key could live beyond the write into the current + // sstable output file, make a copy. + c.rangeDelFrag.TruncateAndFlushTo(key.Clone().UserKey) } - // If we can't cut the sstable at limit, we need to ensure that the - // limit is at least the last key added to the table. We can't actually - // use key.UserKey here because it will be invalidated by the next call - // to iter.Next(). Instead, we set a flag indicating that we've passed - // the grandparent limit and clear the limit. If we can stop output at - // a subsequent key, we'll use that key as the new limit. - passedGrandparentLimit = true - limit = nil + break } atomic.StoreUint64(c.atomicBytesIterated, c.bytesIterated) @@ -1957,31 +2177,6 @@ func (d *DB) runCompaction( c.rangeDelFrag.Add(iter.cloneKey(*key), val) continue } - if tw != nil && tw.EstimatedSize() >= c.maxOutputFileSize { - // Use the next key as the sstable boundary. Note that we already - // checked this key against the grandparent limit above. - if !splittingFlush { - limit = key.UserKey - break - } - // We do not split user keys across different sstables when - // splitting flushes. This includes range tombstones; a range - // tombstone covering a key in a given flush is guaranteed to - // end up in the same sstable as that key. Wide range tombstones - // spanning split boundaries are split to keep this invariant - // true. This invariant exists because L0 compaction picking and - // sublevel construction does not account for atomic compaction - // units, unlike regular compactions. Atomic compaction units - // resulting from user keys being split across sstables - // are an unhelpful side-effects of regular compactions; and the - // only reason it's done for regular compactions is to maintain - // the same behaviour as RocksDB. - // - // Set the limit to a copy of the current key, as the current - // key is backed by compactionIter. The inner loop will break - // and switch outputs when it iterates past this user key. - limit = key.Clone().UserKey - } if tw == nil { if err := newOutput(); err != nil { return nil, pendingOutputs, err @@ -1998,7 +2193,9 @@ func (d *DB) runCompaction( // We ran out of keys and the last key added to the sstable has a zero // seqnum and there are buffered range tombstones, so we're unable to use // the grandparent/flush limit for the sstable boundary. See the example in the - // in the loop above with range tombstones straddling sstables. + // in the loop above with range tombstones straddling sstables. Setting + // limit to nil ensures that we flush the entirety of the rangedel + // fragmenter when writing the last output. limit = nil case key == nil && splittingFlush && !c.rangeDelFrag.Empty(): // We ran out of keys with flush splits enabled, and have remaining diff --git a/github.com/cockroachdb/pebble/compaction_iter.go b/github.com/cockroachdb/pebble/compaction_iter.go index f35121c84e..e044d6f714 100644 --- a/github.com/cockroachdb/pebble/compaction_iter.go +++ b/github.com/cockroachdb/pebble/compaction_iter.go @@ -321,7 +321,10 @@ func (i *compactionIter) Next() (*InternalKey, []byte) { change = i.mergeNext(valueMerger) } if i.err == nil { - i.value, i.valueCloser, i.err = valueMerger.Finish() + // includesBase is true whenever we've transformed the MERGE record + // into a SET. + includesBase := i.key.Kind() == InternalKeyKindSet + i.value, i.valueCloser, i.err = valueMerger.Finish(includesBase) } if i.err == nil { // A non-skippable entry does not necessarily cover later merge diff --git a/github.com/cockroachdb/pebble/db.go b/github.com/cockroachdb/pebble/db.go index 06c254736e..8182eae95a 100644 --- a/github.com/cockroachdb/pebble/db.go +++ b/github.com/cockroachdb/pebble/db.go @@ -1315,6 +1315,14 @@ func (d *DB) makeRoomForWrite(b *Batch) error { d.mu.mem.switching = true d.mu.Unlock() + // Close the previous log first. This writes an EOF trailer + // signifying the end of the file and syncs it to disk. We must + // close the previous log before linking the new log file, + // otherwise a crash could leave both logs with unclean tails, and + // Open will treat the previous log as corrupt. + prevLogSize = uint64(d.mu.log.Size()) + err = d.mu.log.Close() + newLogName := base.MakeFilename(d.opts.FS, d.walDirname, fileTypeLog, newLogNum) // Try to use a recycled log file. Recycling log files is an important @@ -1323,12 +1331,15 @@ func (d *DB) makeRoomForWrite(b *Batch) error { // time. This is due to the need to sync file metadata when a file is // being written for the first time. Note this is true even if file // preallocation is performed (e.g. fallocate). - recycleLogNum := d.logRecycler.peek() - if recycleLogNum > 0 { - recycleLogName := base.MakeFilename(d.opts.FS, d.walDirname, fileTypeLog, recycleLogNum) - newLogFile, err = d.opts.FS.ReuseForWrite(recycleLogName, newLogName) - } else { - newLogFile, err = d.opts.FS.Create(newLogName) + var recycleLogNum base.FileNum + if err == nil { + recycleLogNum = d.logRecycler.peek() + if recycleLogNum > 0 { + recycleLogName := base.MakeFilename(d.opts.FS, d.walDirname, fileTypeLog, recycleLogNum) + newLogFile, err = d.opts.FS.ReuseForWrite(recycleLogName, newLogName) + } else { + newLogFile, err = d.opts.FS.Create(newLogName) + } } if err == nil { @@ -1337,17 +1348,13 @@ func (d *DB) makeRoomForWrite(b *Batch) error { err = d.walDir.Sync() } - if err == nil { - prevLogSize = uint64(d.mu.log.Size()) - err = d.mu.log.Close() - if err != nil { - newLogFile.Close() - } else { - newLogFile = vfs.NewSyncingFile(newLogFile, vfs.SyncingFileOptions{ - BytesPerSync: d.opts.BytesPerSync, - PreallocateSize: d.walPreallocateSize(), - }) - } + if err != nil && newLogFile != nil { + newLogFile.Close() + } else if err == nil { + newLogFile = vfs.NewSyncingFile(newLogFile, vfs.SyncingFileOptions{ + BytesPerSync: d.opts.BytesPerSync, + PreallocateSize: d.walPreallocateSize(), + }) } if recycleLogNum > 0 { diff --git a/github.com/cockroachdb/pebble/internal/base/merger.go b/github.com/cockroachdb/pebble/internal/base/merger.go index 0cffed5e9c..b602925ed5 100644 --- a/github.com/cockroachdb/pebble/internal/base/merger.go +++ b/github.com/cockroachdb/pebble/internal/base/merger.go @@ -48,9 +48,16 @@ type ValueMerger interface { // Finish must be the last function called on the ValueMerger. The caller // must not call any other ValueMerger functions after calling Finish. // + // If `includesBase` is true, the oldest merge operand was part of the + // merge. This will always be the true during normal iteration, but may be + // false during compaction when only a subset of operands may be + // available. Note that `includesBase` is set to true conservatively: a false + // value means that we could not definitely determine that the base merge + // operand was included. + // // If a Closer is returned, the returned slice will remain valid until it is // closed. The caller must arrange for the closer to be eventually closed. - Finish() ([]byte, io.Closer, error) + Finish(includesBase bool) ([]byte, io.Closer, error) } // Merger defines an associative merge operation. The merge operation merges @@ -93,7 +100,7 @@ func (a *AppendValueMerger) MergeOlder(value []byte) error { } // Finish returns the buffer that was constructed on-demand in `Merge{OlderNewer}()` calls. -func (a *AppendValueMerger) Finish() ([]byte, io.Closer, error) { +func (a *AppendValueMerger) Finish(includesBase bool) ([]byte, io.Closer, error) { return a.buf, nil, nil } diff --git a/github.com/cockroachdb/pebble/iterator.go b/github.com/cockroachdb/pebble/iterator.go index f1f8694174..d05806c3bf 100644 --- a/github.com/cockroachdb/pebble/iterator.go +++ b/github.com/cockroachdb/pebble/iterator.go @@ -113,7 +113,7 @@ func (i *Iterator) findNextEntry() bool { i.mergeNext(key, valueMerger) } if i.err == nil { - i.value, i.valueCloser, i.err = valueMerger.Finish() + i.value, i.valueCloser, i.err = valueMerger.Finish(true /* includesBase */) } return i.err == nil @@ -169,7 +169,7 @@ func (i *Iterator) findPrevEntry() bool { // We've iterated to the previous user key. i.pos = iterPosPrev if valueMerger != nil { - i.value, i.valueCloser, i.err = valueMerger.Finish() + i.value, i.valueCloser, i.err = valueMerger.Finish(true /* includesBase */) } return i.err == nil } @@ -232,7 +232,7 @@ func (i *Iterator) findPrevEntry() bool { if i.valid { i.pos = iterPosPrev if valueMerger != nil { - i.value, i.valueCloser, i.err = valueMerger.Finish() + i.value, i.valueCloser, i.err = valueMerger.Finish(true /* includesBase */) } return i.err == nil } diff --git a/github.com/cockroachdb/pebble/level_checker.go b/github.com/cockroachdb/pebble/level_checker.go index 199ce2bbe2..7517dc8f65 100644 --- a/github.com/cockroachdb/pebble/level_checker.go +++ b/github.com/cockroachdb/pebble/level_checker.go @@ -151,7 +151,7 @@ func (m *simpleMergingIter) step() bool { // Ongoing series of MERGE records ends with a MERGE record. if keyChanged && m.valueMerger != nil { var closer io.Closer - _, closer, m.err = m.valueMerger.Finish() + _, closer, m.err = m.valueMerger.Finish(true /* includesBase */) if m.err == nil && closer != nil { m.err = closer.Close() } @@ -162,7 +162,7 @@ func (m *simpleMergingIter) step() bool { switch item.key.Kind() { case InternalKeyKindSingleDelete, InternalKeyKindDelete: var closer io.Closer - _, closer, m.err = m.valueMerger.Finish() + _, closer, m.err = m.valueMerger.Finish(true /* includesBase */) if m.err == nil && closer != nil { m.err = closer.Close() } @@ -171,7 +171,7 @@ func (m *simpleMergingIter) step() bool { m.err = m.valueMerger.MergeOlder(item.value) if m.err == nil { var closer io.Closer - _, closer, m.err = m.valueMerger.Finish() + _, closer, m.err = m.valueMerger.Finish(true /* includesBase */) if m.err == nil && closer != nil { m.err = closer.Close() } @@ -252,7 +252,7 @@ func (m *simpleMergingIter) step() bool { // Last record was a MERGE record. if m.valueMerger != nil { var closer io.Closer - _, closer, m.err = m.valueMerger.Finish() + _, closer, m.err = m.valueMerger.Finish(true /* includesBase */) if m.err == nil && closer != nil { m.err = closer.Close() } diff --git a/modules.txt b/modules.txt index 0281464844..a9190f25b6 100644 --- a/modules.txt +++ b/modules.txt @@ -205,7 +205,7 @@ github.com/cockroachdb/gostdlib/x/tools/internal/semver github.com/cockroachdb/gostdlib/x/tools/internal/span # github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f github.com/cockroachdb/logtags -# github.com/cockroachdb/pebble v0.0.0-20200908153531-a9c6d3b6685c +# github.com/cockroachdb/pebble v0.0.0-20200910205826-d7a278abeed4 github.com/cockroachdb/pebble github.com/cockroachdb/pebble/bloom github.com/cockroachdb/pebble/internal/arenaskl @@ -753,12 +753,12 @@ golang.org/x/crypto/ssh/agent golang.org/x/crypto/ssh/internal/bcrypt_pbkdf golang.org/x/crypto/ssh/knownhosts golang.org/x/crypto/ssh/terminal -# golang.org/x/exp v0.0.0-20200901203048-c4f52b2c50aa +# golang.org/x/exp v0.0.0-20200908183739-ae8ad444f925 golang.org/x/exp/rand # golang.org/x/lint v0.0.0-20200130185559-910be7a94367 golang.org/x/lint golang.org/x/lint/golint -# golang.org/x/mod v0.3.0 +# golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449 golang.org/x/mod/semver # golang.org/x/net v0.0.0-20200602114024-627f9648deb9 golang.org/x/net/context From 1a668723941c9f2da0f73b8e3ad3f89fd312899d Mon Sep 17 00:00:00 2001 From: Bilal Akhtar Date: Tue, 15 Sep 2020 16:58:14 -0400 Subject: [PATCH 43/49] Bump pebble to 08b545a1f5403e31a76b48f46a780c8d59432f57 --- github.com/cockroachdb/pebble/compaction.go | 157 +++++++++++++------- modules.txt | 2 +- 2 files changed, 104 insertions(+), 55 deletions(-) diff --git a/github.com/cockroachdb/pebble/compaction.go b/github.com/cockroachdb/pebble/compaction.go index 09e0cbdb67..679aa7fea1 100644 --- a/github.com/cockroachdb/pebble/compaction.go +++ b/github.com/cockroachdb/pebble/compaction.go @@ -66,6 +66,27 @@ type compactionLevel struct { files manifest.LevelSlice } +// Return output from compactionOutputSplitters. See comment on +// compactionOutputSplitter.shouldSplitBefore() on how this value is used. +type compactionSplitSuggestion int + +const( + noSplit compactionSplitSuggestion = iota + splitSoon + splitNow +) + +// String implements the Stringer interface. +func (c compactionSplitSuggestion) String() string { + switch c { + case noSplit: + return "no-split" + case splitSoon: + return "split-soon" + } + return "split-now" +} + // compactionOutputSplitter is an interface for encapsulating logic around // switching the output of a compaction to a new output file. Additional // constraints around switching compaction outputs that are specific to that @@ -73,8 +94,13 @@ type compactionLevel struct { // compactionOutputSplitters that compose other child compactionOutputSplitters. type compactionOutputSplitter interface { // shouldSplitBefore returns whether we should split outputs before the - // specified "current key". - shouldSplitBefore(key *InternalKey, tw *sstable.Writer) bool + // specified "current key". The return value is one of splitNow, splitSoon, + // or noSlit. splitNow means a split is advised before the specified key, + // splitSoon means no split is advised yet but the limit returned in + // onNewOutput can be considered invalidated and a splitNow suggestion will + // be made on an upcoming key shortly, and noSplit means no split is + // advised. + shouldSplitBefore(key *InternalKey, tw *sstable.Writer) compactionSplitSuggestion // onNewOutput updates internal splitter state when the compaction switches // to a new sstable, and returns the next limit for the new output which // would get used to truncate range tombstones if the compaction iterator @@ -93,12 +119,15 @@ type fileSizeSplitter struct { maxFileSize uint64 } -func (f *fileSizeSplitter) shouldSplitBefore(key *InternalKey, tw *sstable.Writer) bool { +func (f *fileSizeSplitter) shouldSplitBefore(key *InternalKey, tw *sstable.Writer) compactionSplitSuggestion { // The Kind != RangeDelete part exists because EstimatedSize doesn't grow // rightaway when a range tombstone is added to the fragmenter. It's always // better to make a sequence of range tombstones visible to the fragmenter. - return key.Kind() != InternalKeyKindRangeDelete && tw != nil && - tw.EstimatedSize() >= f.maxFileSize + if key.Kind() != InternalKeyKindRangeDelete && tw != nil && + tw.EstimatedSize() >= f.maxFileSize { + return splitNow + } + return noSplit } func (f *fileSizeSplitter) onNewOutput(key *InternalKey) []byte { @@ -106,13 +135,16 @@ func (f *fileSizeSplitter) onNewOutput(key *InternalKey) []byte { } type grandparentLimitSplitter struct { - c *compaction - ve *versionEdit + c *compaction + ve *versionEdit limit []byte } -func (g *grandparentLimitSplitter) shouldSplitBefore(key *InternalKey, tw *sstable.Writer) bool { - return g.limit != nil && g.c.cmp(key.UserKey, g.limit) > 0 +func (g *grandparentLimitSplitter) shouldSplitBefore(key *InternalKey, tw *sstable.Writer) compactionSplitSuggestion { + if g.limit != nil && g.c.cmp(key.UserKey, g.limit) > 0 { + return splitNow + } + return noSplit } func (g *grandparentLimitSplitter) onNewOutput(key *InternalKey) []byte { @@ -198,13 +230,16 @@ func (g *grandparentLimitSplitter) onNewOutput(key *InternalKey) []byte { } type l0LimitSplitter struct { - c *compaction - ve *versionEdit + c *compaction + ve *versionEdit limit []byte } -func (l *l0LimitSplitter) shouldSplitBefore(key *InternalKey, tw *sstable.Writer) bool { - return l.limit != nil && l.c.cmp(key.UserKey, l.limit) > 0 +func (l *l0LimitSplitter) shouldSplitBefore(key *InternalKey, tw *sstable.Writer) compactionSplitSuggestion { + if l.limit != nil && l.c.cmp(key.UserKey, l.limit) > 0 { + return splitNow + } + return noSplit } func (l *l0LimitSplitter) onNewOutput(key *InternalKey) []byte { @@ -240,17 +275,21 @@ func (l *l0LimitSplitter) onNewOutput(key *InternalKey) []byte { // splitterGroup is a compactionOutputSplitter that splits whenever one of its // child splitters advises a compaction split. type splitterGroup struct { - cmp Compare + cmp Compare splitters []compactionOutputSplitter } -func (a *splitterGroup) shouldSplitBefore(key *InternalKey, tw *sstable.Writer) bool { +func (a *splitterGroup) shouldSplitBefore(key *InternalKey, tw *sstable.Writer) (suggestion compactionSplitSuggestion) { + suggestion = noSplit for _, splitter := range a.splitters { - if splitter.shouldSplitBefore(key, tw) { - return true + switch splitter.shouldSplitBefore(key, tw) { + case splitNow: + return splitNow + case splitSoon: + suggestion = splitSoon } } - return false + return suggestion } func (a *splitterGroup) onNewOutput(key *InternalKey) []byte { @@ -275,23 +314,24 @@ func (a *splitterGroup) onNewOutput(key *InternalKey) []byte { // their determinatino in ways other than comparing the current key against a // limit key. type userKeyChangeSplitter struct { - cmp Compare + cmp Compare splitOnNextUserKey bool - savedKey []byte - splitter compactionOutputSplitter + savedKey []byte + splitter compactionOutputSplitter } -func (u *userKeyChangeSplitter) shouldSplitBefore(key *InternalKey, tw *sstable.Writer) bool { +func (u *userKeyChangeSplitter) shouldSplitBefore(key *InternalKey, tw *sstable.Writer) compactionSplitSuggestion { if u.splitOnNextUserKey && u.cmp(u.savedKey, key.UserKey) != 0 { u.splitOnNextUserKey = false u.savedKey = u.savedKey[:0] - return true + return splitNow } - if u.splitter.shouldSplitBefore(key, tw) { + if split := u.splitter.shouldSplitBefore(key, tw); split == splitNow { u.splitOnNextUserKey = true u.savedKey = append(u.savedKey[:0], key.UserKey...) + return noSplit } - return false + return noSplit } func (u *userKeyChangeSplitter) onNewOutput(key *InternalKey) []byte { @@ -303,13 +343,13 @@ func (u *userKeyChangeSplitter) onNewOutput(key *InternalKey) []byte { // and 2) the compaction output is at a point where the previous point sequence // number is nonzero. type nonZeroSeqNumSplitter struct { - c *compaction - splitter compactionOutputSplitter - prevPointSeqNum uint64 + c *compaction + splitter compactionOutputSplitter + prevPointSeqNum uint64 splitOnNonZeroSeqNum bool } -func (n *nonZeroSeqNumSplitter) shouldSplitBefore(key *InternalKey, tw *sstable.Writer) bool { +func (n *nonZeroSeqNumSplitter) shouldSplitBefore(key *InternalKey, tw *sstable.Writer) compactionSplitSuggestion { curSeqNum := key.SeqNum() keyKind := key.Kind() prevPointSeqNum := n.prevPointSeqNum @@ -320,16 +360,17 @@ func (n *nonZeroSeqNumSplitter) shouldSplitBefore(key *InternalKey, tw *sstable. if n.splitOnNonZeroSeqNum { if prevPointSeqNum > 0 || n.c.rangeDelFrag.Empty() { n.splitOnNonZeroSeqNum = false - return true + return splitNow } - } else if n.splitter.shouldSplitBefore(key, tw) { + } else if split := n.splitter.shouldSplitBefore(key, tw); split == splitNow { userKeyChange := curSeqNum > prevPointSeqNum if prevPointSeqNum > 0 || n.c.rangeDelFrag.Empty() || userKeyChange { - return true + return splitNow } n.splitOnNonZeroSeqNum = true + return splitSoon } - return false + return noSplit } func (n *nonZeroSeqNumSplitter) onNewOutput(key *InternalKey) []byte { @@ -375,10 +416,6 @@ type compaction struct { // maxOverlapBytes is the maximum number of bytes of overlap allowed for a // single output table with the tables in the grandparent level. maxOverlapBytes uint64 - // maxExpandedBytes is the maximum size of an expanded compaction. If growing - // a compaction results in a larger size, the original compaction is used - // instead. - maxExpandedBytes uint64 // disableRangeTombstoneElision disables elision of range tombstones. Used by // tests to allow range tombstones to be added to tables where they would // otherwise be elided. @@ -477,7 +514,6 @@ func newCompaction(pc *pickedCompaction, opts *Options, bytesCompacted *uint64) version: pc.version, maxOutputFileSize: pc.maxOutputFileSize, maxOverlapBytes: pc.maxOverlapBytes, - maxExpandedBytes: pc.maxOverlapBytes, atomicBytesIterated: bytesCompacted, } c.startLevel = &c.inputs[0] @@ -537,7 +573,6 @@ func newFlush( inputs: []compactionLevel{{level: -1}, {level: 0}}, maxOutputFileSize: math.MaxUint64, maxOverlapBytes: math.MaxUint64, - maxExpandedBytes: math.MaxUint64, flushing: flushing, atomicBytesIterated: bytesFlushed, } @@ -597,7 +632,6 @@ func newFlush( if opts.Experimental.FlushSplitBytes > 0 { c.maxOutputFileSize = uint64(opts.Level(0).TargetFileSize) c.maxOverlapBytes = maxGrandparentOverlapBytes(opts, 0) - c.maxExpandedBytes = expandedCompactionByteSizeLimit(opts, 0) c.grandparents = c.version.Overlaps(baseLevel, c.cmp, c.smallest.UserKey, c.largest.UserKey) } @@ -1342,7 +1376,9 @@ func pickElisionOnly(picker compactionPicker, env compactionEnv) *pickedCompacti // calling `pickFunc` to pick automatic compactions. // // d.mu must be held when calling this. -func (d *DB) maybeScheduleCompactionPicker(pickFunc func(compactionPicker, compactionEnv) *pickedCompaction) { +func (d *DB) maybeScheduleCompactionPicker( + pickFunc func(compactionPicker, compactionEnv) *pickedCompaction, +) { if d.closed.Load() != nil || d.opts.ReadOnly { return } @@ -2132,8 +2168,8 @@ func (d *DB) runCompaction( // in that case. if !splittingFlush { splitter = &nonZeroSeqNumSplitter{ - c: c, - splitter: splitter, + c: c, + splitter: splitter, } } @@ -2151,18 +2187,31 @@ func (d *DB) runCompaction( // Each inner loop iteration processes one key from the input iterator. prevPointSeqNum := InternalKeySeqNumMax for ; key != nil; key, val = iter.Next() { - if splitter.shouldSplitBefore(key, tw) { - limit = key.UserKey - if splittingFlush { - // Flush all tombstones up until key.UserKey, and - // truncate them at that key. - // - // The fragmenter could save the passed-in key. As this - // key could live beyond the write into the current - // sstable output file, make a copy. - c.rangeDelFrag.TruncateAndFlushTo(key.Clone().UserKey) + if split := splitter.shouldSplitBefore(key, tw); split != noSplit { + if split == splitNow { + limit = key.UserKey + if splittingFlush { + // Flush all tombstones up until key.UserKey, and + // truncate them at that key. + // + // The fragmenter could save the passed-in key. As this + // key could live beyond the write into the current + // sstable output file, make a copy. + c.rangeDelFrag.TruncateAndFlushTo(key.Clone().UserKey) + } + break } - break + // split == splitSoon + // + // Invalidate the limit here. It has probably been exceeded + // by the current key, but we can't split just yet, such as to + // maintain the nonzero sequence number invariant mentioned + // above. Setting limit to nil is okay as it's just a transient + // setting, as when split eventually equals splitNow, we will + // set the limit to the key after that. If the compaction were + // to run out of keys before we get to that point, limit would + // be nil as it should be for all end-of-compaction cases. + limit = nil } atomic.StoreUint64(c.atomicBytesIterated, c.bytesIterated) diff --git a/modules.txt b/modules.txt index a9190f25b6..a76fbb794e 100644 --- a/modules.txt +++ b/modules.txt @@ -205,7 +205,7 @@ github.com/cockroachdb/gostdlib/x/tools/internal/semver github.com/cockroachdb/gostdlib/x/tools/internal/span # github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f github.com/cockroachdb/logtags -# github.com/cockroachdb/pebble v0.0.0-20200910205826-d7a278abeed4 +# github.com/cockroachdb/pebble v0.0.0-20200915204653-08b545a1f540 github.com/cockroachdb/pebble github.com/cockroachdb/pebble/bloom github.com/cockroachdb/pebble/internal/arenaskl From af6227130f6ae230689f7a199b63974628cd3015 Mon Sep 17 00:00:00 2001 From: David Hartunian Date: Mon, 21 Sep 2020 17:10:52 -0400 Subject: [PATCH 44/49] Add dependencies for oidc login --- github.com/coreos/go-oidc/.gitignore | 2 + github.com/coreos/go-oidc/.travis.yml | 16 + github.com/coreos/go-oidc/CONTRIBUTING.md | 71 + github.com/coreos/go-oidc/DCO | 36 + github.com/coreos/go-oidc/LICENSE | 202 +++ github.com/coreos/go-oidc/MAINTAINERS | 3 + github.com/coreos/go-oidc/NOTICE | 5 + github.com/coreos/go-oidc/README.md | 72 + github.com/coreos/go-oidc/code-of-conduct.md | 61 + github.com/coreos/go-oidc/jose.go | 20 + github.com/coreos/go-oidc/jwks.go | 228 ++++ github.com/coreos/go-oidc/oidc.go | 409 ++++++ github.com/coreos/go-oidc/test | 16 + github.com/coreos/go-oidc/verify.go | 336 +++++ github.com/pquerna/cachecontrol/.travis.yml | 5 + github.com/pquerna/cachecontrol/LICENSE | 202 +++ github.com/pquerna/cachecontrol/README.md | 107 ++ github.com/pquerna/cachecontrol/api.go | 48 + .../cachecontrol/cacheobject/directive.go | 547 ++++++++ .../pquerna/cachecontrol/cacheobject/lex.go | 93 ++ .../cachecontrol/cacheobject/object.go | 387 ++++++ .../cachecontrol/cacheobject/reasons.go | 95 ++ .../cachecontrol/cacheobject/warning.go | 107 ++ github.com/pquerna/cachecontrol/doc.go | 25 + github.com/pquerna/cachecontrol/go.mod | 5 + github.com/pquerna/cachecontrol/go.sum | 11 + gopkg.in/square/go-jose.v2/.gitcookies.sh.enc | 1 + gopkg.in/square/go-jose.v2/.gitignore | 8 + gopkg.in/square/go-jose.v2/.travis.yml | 45 + gopkg.in/square/go-jose.v2/BUG-BOUNTY.md | 10 + gopkg.in/square/go-jose.v2/CONTRIBUTING.md | 14 + gopkg.in/square/go-jose.v2/LICENSE | 202 +++ gopkg.in/square/go-jose.v2/README.md | 118 ++ gopkg.in/square/go-jose.v2/asymmetric.go | 592 ++++++++ gopkg.in/square/go-jose.v2/cipher/cbc_hmac.go | 196 +++ .../square/go-jose.v2/cipher/concat_kdf.go | 75 ++ gopkg.in/square/go-jose.v2/cipher/ecdh_es.go | 86 ++ gopkg.in/square/go-jose.v2/cipher/key_wrap.go | 109 ++ gopkg.in/square/go-jose.v2/crypter.go | 541 ++++++++ gopkg.in/square/go-jose.v2/doc.go | 27 + gopkg.in/square/go-jose.v2/encoding.go | 185 +++ gopkg.in/square/go-jose.v2/json/LICENSE | 27 + gopkg.in/square/go-jose.v2/json/README.md | 13 + gopkg.in/square/go-jose.v2/json/decode.go | 1183 ++++++++++++++++ gopkg.in/square/go-jose.v2/json/encode.go | 1197 +++++++++++++++++ gopkg.in/square/go-jose.v2/json/indent.go | 141 ++ gopkg.in/square/go-jose.v2/json/scanner.go | 623 +++++++++ gopkg.in/square/go-jose.v2/json/stream.go | 480 +++++++ gopkg.in/square/go-jose.v2/json/tags.go | 44 + gopkg.in/square/go-jose.v2/jwe.go | 294 ++++ gopkg.in/square/go-jose.v2/jwk.go | 760 +++++++++++ gopkg.in/square/go-jose.v2/jws.go | 366 +++++ gopkg.in/square/go-jose.v2/opaque.go | 144 ++ gopkg.in/square/go-jose.v2/shared.go | 520 +++++++ gopkg.in/square/go-jose.v2/signing.go | 441 ++++++ gopkg.in/square/go-jose.v2/symmetric.go | 482 +++++++ modules.txt | 9 + 57 files changed, 12042 insertions(+) create mode 100644 github.com/coreos/go-oidc/.gitignore create mode 100644 github.com/coreos/go-oidc/.travis.yml create mode 100644 github.com/coreos/go-oidc/CONTRIBUTING.md create mode 100644 github.com/coreos/go-oidc/DCO create mode 100644 github.com/coreos/go-oidc/LICENSE create mode 100644 github.com/coreos/go-oidc/MAINTAINERS create mode 100644 github.com/coreos/go-oidc/NOTICE create mode 100644 github.com/coreos/go-oidc/README.md create mode 100644 github.com/coreos/go-oidc/code-of-conduct.md create mode 100644 github.com/coreos/go-oidc/jose.go create mode 100644 github.com/coreos/go-oidc/jwks.go create mode 100644 github.com/coreos/go-oidc/oidc.go create mode 100644 github.com/coreos/go-oidc/test create mode 100644 github.com/coreos/go-oidc/verify.go create mode 100644 github.com/pquerna/cachecontrol/.travis.yml create mode 100644 github.com/pquerna/cachecontrol/LICENSE create mode 100644 github.com/pquerna/cachecontrol/README.md create mode 100644 github.com/pquerna/cachecontrol/api.go create mode 100644 github.com/pquerna/cachecontrol/cacheobject/directive.go create mode 100644 github.com/pquerna/cachecontrol/cacheobject/lex.go create mode 100644 github.com/pquerna/cachecontrol/cacheobject/object.go create mode 100644 github.com/pquerna/cachecontrol/cacheobject/reasons.go create mode 100644 github.com/pquerna/cachecontrol/cacheobject/warning.go create mode 100644 github.com/pquerna/cachecontrol/doc.go create mode 100644 github.com/pquerna/cachecontrol/go.mod create mode 100644 github.com/pquerna/cachecontrol/go.sum create mode 100644 gopkg.in/square/go-jose.v2/.gitcookies.sh.enc create mode 100644 gopkg.in/square/go-jose.v2/.gitignore create mode 100644 gopkg.in/square/go-jose.v2/.travis.yml create mode 100644 gopkg.in/square/go-jose.v2/BUG-BOUNTY.md create mode 100644 gopkg.in/square/go-jose.v2/CONTRIBUTING.md create mode 100644 gopkg.in/square/go-jose.v2/LICENSE create mode 100644 gopkg.in/square/go-jose.v2/README.md create mode 100644 gopkg.in/square/go-jose.v2/asymmetric.go create mode 100644 gopkg.in/square/go-jose.v2/cipher/cbc_hmac.go create mode 100644 gopkg.in/square/go-jose.v2/cipher/concat_kdf.go create mode 100644 gopkg.in/square/go-jose.v2/cipher/ecdh_es.go create mode 100644 gopkg.in/square/go-jose.v2/cipher/key_wrap.go create mode 100644 gopkg.in/square/go-jose.v2/crypter.go create mode 100644 gopkg.in/square/go-jose.v2/doc.go create mode 100644 gopkg.in/square/go-jose.v2/encoding.go create mode 100644 gopkg.in/square/go-jose.v2/json/LICENSE create mode 100644 gopkg.in/square/go-jose.v2/json/README.md create mode 100644 gopkg.in/square/go-jose.v2/json/decode.go create mode 100644 gopkg.in/square/go-jose.v2/json/encode.go create mode 100644 gopkg.in/square/go-jose.v2/json/indent.go create mode 100644 gopkg.in/square/go-jose.v2/json/scanner.go create mode 100644 gopkg.in/square/go-jose.v2/json/stream.go create mode 100644 gopkg.in/square/go-jose.v2/json/tags.go create mode 100644 gopkg.in/square/go-jose.v2/jwe.go create mode 100644 gopkg.in/square/go-jose.v2/jwk.go create mode 100644 gopkg.in/square/go-jose.v2/jws.go create mode 100644 gopkg.in/square/go-jose.v2/opaque.go create mode 100644 gopkg.in/square/go-jose.v2/shared.go create mode 100644 gopkg.in/square/go-jose.v2/signing.go create mode 100644 gopkg.in/square/go-jose.v2/symmetric.go diff --git a/github.com/coreos/go-oidc/.gitignore b/github.com/coreos/go-oidc/.gitignore new file mode 100644 index 0000000000..c96f2f47bc --- /dev/null +++ b/github.com/coreos/go-oidc/.gitignore @@ -0,0 +1,2 @@ +/bin +/gopath diff --git a/github.com/coreos/go-oidc/.travis.yml b/github.com/coreos/go-oidc/.travis.yml new file mode 100644 index 0000000000..3fddaaac90 --- /dev/null +++ b/github.com/coreos/go-oidc/.travis.yml @@ -0,0 +1,16 @@ +language: go + +go: + - "1.12" + - "1.13" + +install: + - go get -v -t github.com/coreos/go-oidc/... + - go get golang.org/x/tools/cmd/cover + - go get golang.org/x/lint/golint + +script: + - ./test + +notifications: + email: false diff --git a/github.com/coreos/go-oidc/CONTRIBUTING.md b/github.com/coreos/go-oidc/CONTRIBUTING.md new file mode 100644 index 0000000000..6662073a84 --- /dev/null +++ b/github.com/coreos/go-oidc/CONTRIBUTING.md @@ -0,0 +1,71 @@ +# How to Contribute + +CoreOS projects are [Apache 2.0 licensed](LICENSE) and accept contributions via +GitHub pull requests. This document outlines some of the conventions on +development workflow, commit message formatting, contact points and other +resources to make it easier to get your contribution accepted. + +# Certificate of Origin + +By contributing to this project you agree to the Developer Certificate of +Origin (DCO). This document was created by the Linux Kernel community and is a +simple statement that you, as a contributor, have the legal right to make the +contribution. See the [DCO](DCO) file for details. + +# Email and Chat + +The project currently uses the general CoreOS email list and IRC channel: +- Email: [coreos-dev](https://groups.google.com/forum/#!forum/coreos-dev) +- IRC: #[coreos](irc://irc.freenode.org:6667/#coreos) IRC channel on freenode.org + +Please avoid emailing maintainers found in the MAINTAINERS file directly. They +are very busy and read the mailing lists. + +## Getting Started + +- Fork the repository on GitHub +- Read the [README](README.md) for build and test instructions +- Play with the project, submit bugs, submit patches! + +## Contribution Flow + +This is a rough outline of what a contributor's workflow looks like: + +- Create a topic branch from where you want to base your work (usually master). +- Make commits of logical units. +- Make sure your commit messages are in the proper format (see below). +- Push your changes to a topic branch in your fork of the repository. +- Make sure the tests pass, and add any new tests as appropriate. +- Submit a pull request to the original repository. + +Thanks for your contributions! + +### Format of the Commit Message + +We follow a rough convention for commit messages that is designed to answer two +questions: what changed and why. The subject line should feature the what and +the body of the commit should describe the why. + +``` +scripts: add the test-cluster command + +this uses tmux to setup a test cluster that you can easily kill and +start for debugging. + +Fixes #38 +``` + +The format can be described more formally as follows: + +``` +: + + + +