diff --git a/docs/dev.md b/docs/dev.md index 1eb9591..9fff49e 100644 --- a/docs/dev.md +++ b/docs/dev.md @@ -55,6 +55,4 @@ Each runs as `go run ./tools/`. The sheet, drive, and bucket tools authent - `dumptab` / `writetab` — copy one tab to a local CSV / write a local CSV into a tab, header-checked - `createtabs` — create the directory sheet's local-layer tabs with their header rows - `setcell` — set one cell in a tab by key column, appending the row if missing -- `probeblob` — time the download of a few drive media files -- `migratemedia` — copy the drive media folders into the media bucket, generating thumbnails; retired once the bucket is the only store - `splash` — regenerate the iOS splash battery from the captured original page; see `docs/pwa.md` diff --git a/go.mod b/go.mod index 5a41a0e..cc25c47 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module heliosian go 1.26 require ( + cloud.google.com/go/storage v1.65.0 github.com/chromedp/cdproto v0.0.0-20260714215040-dc233986426f github.com/chromedp/chromedp v0.16.0 github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd @@ -18,7 +19,6 @@ require ( cloud.google.com/go/compute/metadata v0.9.0 // indirect cloud.google.com/go/iam v1.11.0 // indirect cloud.google.com/go/monitoring v1.29.0 // indirect - cloud.google.com/go/storage v1.65.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 // indirect diff --git a/go.sum b/go.sum index 1c731fe..65438fb 100644 --- a/go.sum +++ b/go.sum @@ -10,14 +10,22 @@ cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdB cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= cloud.google.com/go/iam v1.11.0 h1:KieQ9Pb+LLPak1O3Rv3GgCxhnmkYf7Xyh0P5HfF1jFM= cloud.google.com/go/iam v1.11.0/go.mod h1:KP+nKGugNJW4LcLx1uEZcq1ok5sQHFaQehQNl4QDgV4= +cloud.google.com/go/logging v1.18.0 h1:KhzZq+1cSkPH9YUaKLLhLtQxIHitVayBmk0sGfoM9+k= +cloud.google.com/go/logging v1.18.0/go.mod h1:ZGKnpBaURITh+g/uom2VhbiFoFWvejcrHPDhxFtU/gI= +cloud.google.com/go/longrunning v1.2.0 h1:WjYH3YHBGCxGJP9M4dWGHBfXr/cFIjMkNgWcJj7/iMM= +cloud.google.com/go/longrunning v1.2.0/go.mod h1:5KMQALFGOCtFoi2xSOA1u3H7WKlhmckgiyFw7+LGQp0= cloud.google.com/go/monitoring v1.29.0 h1:AHhDsFaSax1/4k+qlIDX/SDGe6hggnfXJ9dkgD9qBPY= cloud.google.com/go/monitoring v1.29.0/go.mod h1:72NOVjJXHY/HBfoLT0+qlCZBT059+9VXLeAnL2PeeVM= cloud.google.com/go/storage v1.65.0 h1:McbFt5j+hTNx+dkFuzq7teakIKcpqGp/cJZRxMyfvAc= cloud.google.com/go/storage v1.65.0/go.mod h1:UsS9OgFg/XHOSYakQ8ZtLWWeyGkk1WnmD/GsGfN0BHM= +cloud.google.com/go/trace v1.16.0 h1:GmQovzFc5F0CNfl0VLgL64aoTtu7xsM0YajW2GlG9+E= +cloud.google.com/go/trace v1.16.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjx7kM7X0= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 h1:l7+6kwRMJNwdCvYdDl7Eax+wzEYHSnNY7zrrfbhDdTA= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0/go.mod h1:pJTkW8hEUIIi3Pf65lPZOnn4Y81yCllX6IWk2jNXdkM= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 h1:jLdiS1vO+XJFyDSWRHBx56r4s/NNtcl5J6KyCcWUX/w= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0/go.mod h1:8lmpHY+1VRoteiOwyrQMDt1YGXOrFKCz+1wJW7n3ODY= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.57.0 h1:cSjUzZ7KU8hicTgzaSv9NmSyM9fTVK3y5lsBUl3wOis= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.57.0/go.mod h1:dzcEjy1WJ0Q4u9twNR3LcLhNoYMRCrMCMafpxa0TjPQ= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 h1:RoO5+d7uCmDqovLrHCr2/BuViUXvdcrNxyNM1pN9dDQ= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0/go.mod h1:YqwkQPrWSC7+byyc1VlKbWLBF5JsW5IoL6xUkemYSXk= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -33,8 +41,11 @@ github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7X github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -58,6 +69,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -84,16 +97,18 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/detectors/gcp v1.44.0 h1:NmLfL734pJhM0JKaYd2Y28+nY9dPRWYAAbxhRCrKXPw= go.opentelemetry.io/contrib/detectors/gcp v1.44.0/go.mod h1:tNAsgd8avTGke1+MndXlU5Cru4PQ9Ai/cCNWQv/ZJ/s= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 h1:0Qx7VGBacMm9ZENQ7TnNObTYI4ShC+lHI16seduaxZo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0/go.mod h1:Sje3i3MjSPKTSPvVWCaL8ugBzJwik3u4smCjUeuupqg= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.44.0 h1:hqxVTu/GtBF+vJ8d1fzW7fRxZFvgoDjWcxwwCaFDYpU= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.44.0/go.mod h1:z5fVEF4X5v0ESvlJqBrrFlBVoj5EQuefZpzsu7R+x5Q= go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/metric/x v0.66.0 h1:YkCrx1zLOChi9ZcZ6euupOcsgzbVlec7D/xoEU1+cTA= +go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk= go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= @@ -121,8 +136,6 @@ gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.293.0 h1:p9XIWOf63U4OgYx120ZwVU8+vl4XTPmWfgVPnmOAS9w= google.golang.org/api v0.293.0/go.mod h1:6n5tjEB1gzwniZTepZ0g5u+wM7Bof5GeULCx/zh8ZE0= -google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= -google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= google.golang.org/genproto v0.0.0-20260519071638-aa98bba5eb94 h1:YJjbgu+dkp5kUJLfpMyCLfBIWZb/FcJyuLeo1gVBOuo= google.golang.org/genproto v0.0.0-20260519071638-aa98bba5eb94/go.mod h1:RRHjglSYABVCWpQ7USCpdfhcd9t4PkajvVwyynZizTc= google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 h1:jQ9p21COKWjP3VwuFrNRiiOTMh3mPpN45R7SLrH/HUU= diff --git a/tools/migratemedia/main.go b/tools/migratemedia/main.go deleted file mode 100644 index d67aed8..0000000 --- a/tools/migratemedia/main.go +++ /dev/null @@ -1,180 +0,0 @@ -// Command migratemedia copies the drive media folders into the media bucket, generating thumbnails. -package main - -import ( - "context" - "fmt" - "io" - "log" - "path" - "strings" - "sync" - "time" - - "cloud.google.com/go/storage" - "google.golang.org/api/drive/v3" - "google.golang.org/api/option" - - "heliosian/internal/blob" -) - -const ( - folderMime = "application/vnd.google-apps.folder" - workers = 12 -) - -type source struct { - folder string - id string - name string - mimeType string -} - -func main() { - ctx := context.Background() - drv, err := drive.NewService(ctx, option.WithScopes(drive.DriveReadonlyScope)) - if err != nil { - log.Fatalf("[ERROR] drive client: %v", err) - } - client, err := storage.NewClient(ctx) - if err != nil { - log.Fatalf("[ERROR] storage client: %v", err) - } - bucket := client.Bucket(blob.Bucket) - - drives, err := drv.Drives.List().Do() - if err != nil { - log.Fatalf("[ERROR] list shared drives: %v", err) - } - if len(drives.Drives) != 1 { - log.Fatalf("[ERROR] expected one shared drive, found %d", len(drives.Drives)) - } - root := drives.Drives[0].Id - - sources := []source{} - for _, folder := range []string{"people", "families"} { - folderID, err := subfolder(drv, root, folder) - if err != nil { - log.Fatalf("[ERROR] %v", err) - } - token := "" - for { - call := drv.Files.List(). - Q(fmt.Sprintf("'%s' in parents and trashed = false", folderID)). - SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives"). - Fields("nextPageToken, files(id, name, mimeType)").PageSize(1000) - if token != "" { - call = call.PageToken(token) - } - list, err := call.Do() - if err != nil { - log.Fatalf("[ERROR] list %s: %v", folder, err) - } - for _, f := range list.Files { - if f.MimeType == folderMime { - continue - } - if strings.HasSuffix(strings.TrimSuffix(f.Name, path.Ext(f.Name)), "-thumb") { - continue - } - sources = append(sources, source{folder: folder, id: f.Id, name: f.Name, mimeType: f.MimeType}) - } - if list.NextPageToken == "" { - break - } - token = list.NextPageToken - } - } - log.Printf("migrating %d files to gs://%s", len(sources), blob.Bucket) - - start := time.Now() - var mu sync.Mutex - var copied, thumbed int - var bytesWritten int64 - var failed []string - var wg sync.WaitGroup - slots := make(chan struct{}, workers) - for _, src := range sources { - wg.Add(1) - slots <- struct{}{} - go func() { - defer wg.Done() - defer func() { <-slots }() - n, thumb, err := migrate(ctx, drv, bucket, src) - mu.Lock() - defer mu.Unlock() - if err != nil { - log.Printf("[ERROR] %s/%s: %v", src.folder, src.name, err) - failed = append(failed, src.folder+"/"+src.name) - return - } - copied++ - bytesWritten += n - if thumb { - thumbed++ - } - }() - } - wg.Wait() - - log.Printf("copied %d files (%d thumbnails, %.1f MB) in %s", - copied, thumbed, float64(bytesWritten)/1e6, time.Since(start).Round(time.Millisecond)) - if len(failed) > 0 { - log.Fatalf("[ERROR] %d files failed: %s", len(failed), strings.Join(failed, " ")) - } -} - -func migrate(ctx context.Context, drv *drive.Service, bucket *storage.BucketHandle, src source) (int64, bool, error) { - resp, err := drv.Files.Get(src.id).SupportsAllDrives(true).Download() - if err != nil { - return 0, false, fmt.Errorf("download: %w", err) - } - defer resp.Body.Close() - content, err := io.ReadAll(resp.Body) - if err != nil { - return 0, false, fmt.Errorf("read: %w", err) - } - written := int64(len(content)) - if err := write(ctx, bucket, src.folder+"/"+src.name, src.mimeType, content); err != nil { - return 0, false, err - } - if !strings.HasPrefix(src.mimeType, "image/") { - return written, false, nil - } - thumb, err := blob.Thumbnail(content) - if err != nil { - return 0, false, fmt.Errorf("thumbnail: %w", err) - } - base := strings.TrimSuffix(src.name, path.Ext(src.name)) - if err := write(ctx, bucket, src.folder+"/"+base+"-thumb.jpg", "image/jpeg", thumb); err != nil { - return 0, false, err - } - return written + int64(len(thumb)), true, nil -} - -func write(ctx context.Context, bucket *storage.BucketHandle, name, mimeType string, content []byte) error { - w := bucket.Object(name).NewWriter(ctx) - w.ContentType = mimeType - if _, err := w.Write(content); err != nil { - w.Close() - return fmt.Errorf("write %s: %w", name, err) - } - if err := w.Close(); err != nil { - return fmt.Errorf("write %s: %w", name, err) - } - return nil -} - -func subfolder(drv *drive.Service, parent, name string) (string, error) { - list, err := drv.Files.List(). - Q(fmt.Sprintf("name = '%s' and '%s' in parents and mimeType = '%s' and trashed = false", name, parent, folderMime)). - SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives"). - Fields("files(id)").Do() - if err != nil { - return "", fmt.Errorf("find folder %s: %w", name, err) - } - if len(list.Files) != 1 { - return "", fmt.Errorf("expected one %s folder, found %d", name, len(list.Files)) - } - return list.Files[0].Id, nil -} diff --git a/tools/probeblob/main.go b/tools/probeblob/main.go deleted file mode 100644 index d6290b6..0000000 --- a/tools/probeblob/main.go +++ /dev/null @@ -1,39 +0,0 @@ -// Command probeblob times the download of a few drive media files. -package main - -import ( - "context" - "fmt" - "io" - "log" - "time" - - "google.golang.org/api/drive/v3" - "google.golang.org/api/option" -) - -func main() { - svc, err := drive.NewService(context.Background(), - option.WithScopes(drive.DriveScope)) - if err != nil { - log.Fatalf("[ERROR] drive client: %v", err) - } - list, err := svc.Files.List(). - Q("mimeType != 'application/vnd.google-apps.folder' and trashed = false"). - SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives"). - Fields("files(id, name, size)").PageSize(5).Do() - if err != nil { - log.Fatalf("[ERROR] list: %v", err) - } - for _, f := range list.Files { - start := time.Now() - resp, err := svc.Files.Get(f.Id).SupportsAllDrives(true).Download() - if err != nil { - fmt.Printf("%s (%d bytes): request error after %s: %v\n", f.Name, f.Size, time.Since(start).Round(time.Millisecond), err) - continue - } - n, err := io.Copy(io.Discard, resp.Body) - resp.Body.Close() - fmt.Printf("%s: %d bytes in %s (err %v)\n", f.Name, n, time.Since(start).Round(time.Millisecond), err) - } -}