feat: mysql chunking optimisation

constants/constants.go

@@ -33,6 +33,15 @@ const (
 	EffectiveParquetSize        = int64(256) * 1024 * 1024 * int64(8)
 	DB2StateTimestampFormat     = "2006-01-02 15:04:05.000000"
 	DefaultStateTimestampFormat = "2006-01-02T15:04:05.000000000Z"
+	// DistributionLower and DistributionUpper define the acceptable range
+	// of the distribution factor for validating evenly distributed numeric PKs.
+	DistributionLower = 0.05
+	DistributionUpper = 1000.0
+	// UnicodeSize is the total number of valid Unicode code points (0 to 0x10FFFF)
+	UnicodeSize = 1114112
+	// MysqlChunkAcceptanceRatio defines the minimum ratio of expected chunks that must be generated
+	// for the split to be considered valid.
+	MysqlChunkAcceptanceRatio = float64(0.8)
 )
 
 type DriverType string

constants/state_version.go

@@ -29,12 +29,16 @@ package constants
 //   - Version 4: Unsigned int/integer/bigint map to Int64.
 //     * Earlier unsigned int/integer/bigint were mapped to Int32 which caused integer overflows.
 //
-//   - Version 5: (Current Version) MongoDB nested DateTime values decoded as UTC time.Time.
+//   - Version 5: MongoDB nested DateTime values decoded as UTC time.Time.
 //     * BSON DateTime at any depth is now decoded directly to time.Time (UTC) via a custom client registry, preventing json.Marshal crashes for out-of-range years ([0,9999]).
 //     * Top-level DateTime fields that previously formatted with the local machine timezone (e.g. "+05:30") now always output UTC ("Z").
+//
+//   - Version 6: (Current Version) Added []uint8 (byte slice) support in ReformatInt64
+//     * Previously, numeric values returned as byte slices (common in some SQL drivers) caused errors
+//     * Now these byte slices are parsed and converted into int64
 
 const (
-	LatestStateVersion = 5
+	LatestStateVersion = 6
 )
 
 // Used as the current version of the state when the program is running