package copier import ( "context" "crypto/sha256" "encoding/hex" "fmt" "io" "os" "os/exec" "path/filepath" "strings" "sync" "time" ) const maxRetries = 3 // FileJob represents one file to copy type FileJob struct { Source string LocalDest string S3Bucket string S3Key string AWSProfile string } // FileResult is the result of copying one file type FileResult struct { Job FileJob SHA256 string Bytes int64 Error error } // ProgressFn is called with (bytesCopied, totalBytes) during local copy type ProgressFn func(n int64, total int64) // CopyLocal copies src to dst, returns sha256 and bytes written func CopyLocal(src, dst string, progress ProgressFn) (string, int64, error) { if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { return "", 0, err } in, err := os.Open(src) if err != nil { return "", 0, err } defer in.Close() info, err := in.Stat() if err != nil { return "", 0, err } total := info.Size() out, err := os.Create(dst) if err != nil { return "", 0, err } defer out.Close() h := sha256.New() var written int64 buf := make([]byte, 1<<20) // 1MB buffer for { nr, er := in.Read(buf) if nr > 0 { nw, ew := out.Write(buf[:nr]) if nw > 0 { written += int64(nw) h.Write(buf[:nw]) if progress != nil { progress(written, total) } } if ew != nil { return "", written, ew } } if er == io.EOF { break } if er != nil { return "", written, er } } return hex.EncodeToString(h.Sum(nil)), written, nil } // CopyS3 uploads localPath to s3://bucket/key via s5cmd func CopyS3(ctx context.Context, localPath, bucket, key, awsProfile string) error { s3uri := fmt.Sprintf("s3://%s/%s", bucket, key) for attempt := 0; attempt < maxRetries; attempt++ { if attempt > 0 { wait := time.Duration(1<