Drop the drive-era media tools

This commit is contained in:
Ian Gulliver
2026-08-20 12:59:44 -07:00
parent 4f346ef35e
commit 7fd198e75e
5 changed files with 18 additions and 226 deletions
-2
View File
@@ -55,6 +55,4 @@ Each runs as `go run ./tools/<name>`. The sheet, drive, and bucket tools authent
- `dumptab` / `writetab` — copy one tab to a local CSV / write a local CSV into a tab, header-checked
- `createtabs` — create the directory sheet's local-layer tabs with their header rows
- `setcell` — set one cell in a tab by key column, appending the row if missing
- `probeblob` — time the download of a few drive media files
- `migratemedia` — copy the drive media folders into the media bucket, generating thumbnails; retired once the bucket is the only store
- `splash` — regenerate the iOS splash battery from the captured original page; see `docs/pwa.md`
+1 -1
View File
@@ -3,6 +3,7 @@ module heliosian
go 1.26
require (
cloud.google.com/go/storage v1.65.0
github.com/chromedp/cdproto v0.0.0-20260714215040-dc233986426f
github.com/chromedp/chromedp v0.16.0
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd
@@ -18,7 +19,6 @@ require (
cloud.google.com/go/compute/metadata v0.9.0 // indirect
cloud.google.com/go/iam v1.11.0 // indirect
cloud.google.com/go/monitoring v1.29.0 // indirect
cloud.google.com/go/storage v1.65.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 // indirect
+17 -4
View File
@@ -10,14 +10,22 @@ cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdB
cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
cloud.google.com/go/iam v1.11.0 h1:KieQ9Pb+LLPak1O3Rv3GgCxhnmkYf7Xyh0P5HfF1jFM=
cloud.google.com/go/iam v1.11.0/go.mod h1:KP+nKGugNJW4LcLx1uEZcq1ok5sQHFaQehQNl4QDgV4=
cloud.google.com/go/logging v1.18.0 h1:KhzZq+1cSkPH9YUaKLLhLtQxIHitVayBmk0sGfoM9+k=
cloud.google.com/go/logging v1.18.0/go.mod h1:ZGKnpBaURITh+g/uom2VhbiFoFWvejcrHPDhxFtU/gI=
cloud.google.com/go/longrunning v1.2.0 h1:WjYH3YHBGCxGJP9M4dWGHBfXr/cFIjMkNgWcJj7/iMM=
cloud.google.com/go/longrunning v1.2.0/go.mod h1:5KMQALFGOCtFoi2xSOA1u3H7WKlhmckgiyFw7+LGQp0=
cloud.google.com/go/monitoring v1.29.0 h1:AHhDsFaSax1/4k+qlIDX/SDGe6hggnfXJ9dkgD9qBPY=
cloud.google.com/go/monitoring v1.29.0/go.mod h1:72NOVjJXHY/HBfoLT0+qlCZBT059+9VXLeAnL2PeeVM=
cloud.google.com/go/storage v1.65.0 h1:McbFt5j+hTNx+dkFuzq7teakIKcpqGp/cJZRxMyfvAc=
cloud.google.com/go/storage v1.65.0/go.mod h1:UsS9OgFg/XHOSYakQ8ZtLWWeyGkk1WnmD/GsGfN0BHM=
cloud.google.com/go/trace v1.16.0 h1:GmQovzFc5F0CNfl0VLgL64aoTtu7xsM0YajW2GlG9+E=
cloud.google.com/go/trace v1.16.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjx7kM7X0=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 h1:l7+6kwRMJNwdCvYdDl7Eax+wzEYHSnNY7zrrfbhDdTA=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0/go.mod h1:pJTkW8hEUIIi3Pf65lPZOnn4Y81yCllX6IWk2jNXdkM=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 h1:jLdiS1vO+XJFyDSWRHBx56r4s/NNtcl5J6KyCcWUX/w=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0/go.mod h1:8lmpHY+1VRoteiOwyrQMDt1YGXOrFKCz+1wJW7n3ODY=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.57.0 h1:cSjUzZ7KU8hicTgzaSv9NmSyM9fTVK3y5lsBUl3wOis=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.57.0/go.mod h1:dzcEjy1WJ0Q4u9twNR3LcLhNoYMRCrMCMafpxa0TjPQ=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 h1:RoO5+d7uCmDqovLrHCr2/BuViUXvdcrNxyNM1pN9dDQ=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0/go.mod h1:YqwkQPrWSC7+byyc1VlKbWLBF5JsW5IoL6xUkemYSXk=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
@@ -33,8 +41,11 @@ github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7X
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA=
github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU=
github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ=
github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A=
github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI=
github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4=
github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds=
github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
@@ -58,6 +69,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc=
github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0=
github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
@@ -84,16 +97,18 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/detectors/gcp v1.44.0 h1:NmLfL734pJhM0JKaYd2Y28+nY9dPRWYAAbxhRCrKXPw=
go.opentelemetry.io/contrib/detectors/gcp v1.44.0/go.mod h1:tNAsgd8avTGke1+MndXlU5Cru4PQ9Ai/cCNWQv/ZJ/s=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 h1:0Qx7VGBacMm9ZENQ7TnNObTYI4ShC+lHI16seduaxZo=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0/go.mod h1:Sje3i3MjSPKTSPvVWCaL8ugBzJwik3u4smCjUeuupqg=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg=
go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.44.0 h1:hqxVTu/GtBF+vJ8d1fzW7fRxZFvgoDjWcxwwCaFDYpU=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.44.0/go.mod h1:z5fVEF4X5v0ESvlJqBrrFlBVoj5EQuefZpzsu7R+x5Q=
go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
go.opentelemetry.io/otel/metric/x v0.66.0 h1:YkCrx1zLOChi9ZcZ6euupOcsgzbVlec7D/xoEU1+cTA=
go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk=
go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI=
@@ -121,8 +136,6 @@ gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
google.golang.org/api v0.293.0 h1:p9XIWOf63U4OgYx120ZwVU8+vl4XTPmWfgVPnmOAS9w=
google.golang.org/api v0.293.0/go.mod h1:6n5tjEB1gzwniZTepZ0g5u+wM7Bof5GeULCx/zh8ZE0=
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0=
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I=
google.golang.org/genproto v0.0.0-20260519071638-aa98bba5eb94 h1:YJjbgu+dkp5kUJLfpMyCLfBIWZb/FcJyuLeo1gVBOuo=
google.golang.org/genproto v0.0.0-20260519071638-aa98bba5eb94/go.mod h1:RRHjglSYABVCWpQ7USCpdfhcd9t4PkajvVwyynZizTc=
google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 h1:jQ9p21COKWjP3VwuFrNRiiOTMh3mPpN45R7SLrH/HUU=
-180
View File
@@ -1,180 +0,0 @@
// Command migratemedia copies the drive media folders into the media bucket, generating thumbnails.
package main
import (
"context"
"fmt"
"io"
"log"
"path"
"strings"
"sync"
"time"
"cloud.google.com/go/storage"
"google.golang.org/api/drive/v3"
"google.golang.org/api/option"
"heliosian/internal/blob"
)
const (
folderMime = "application/vnd.google-apps.folder"
workers = 12
)
type source struct {
folder string
id string
name string
mimeType string
}
func main() {
ctx := context.Background()
drv, err := drive.NewService(ctx, option.WithScopes(drive.DriveReadonlyScope))
if err != nil {
log.Fatalf("[ERROR] drive client: %v", err)
}
client, err := storage.NewClient(ctx)
if err != nil {
log.Fatalf("[ERROR] storage client: %v", err)
}
bucket := client.Bucket(blob.Bucket)
drives, err := drv.Drives.List().Do()
if err != nil {
log.Fatalf("[ERROR] list shared drives: %v", err)
}
if len(drives.Drives) != 1 {
log.Fatalf("[ERROR] expected one shared drive, found %d", len(drives.Drives))
}
root := drives.Drives[0].Id
sources := []source{}
for _, folder := range []string{"people", "families"} {
folderID, err := subfolder(drv, root, folder)
if err != nil {
log.Fatalf("[ERROR] %v", err)
}
token := ""
for {
call := drv.Files.List().
Q(fmt.Sprintf("'%s' in parents and trashed = false", folderID)).
SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives").
Fields("nextPageToken, files(id, name, mimeType)").PageSize(1000)
if token != "" {
call = call.PageToken(token)
}
list, err := call.Do()
if err != nil {
log.Fatalf("[ERROR] list %s: %v", folder, err)
}
for _, f := range list.Files {
if f.MimeType == folderMime {
continue
}
if strings.HasSuffix(strings.TrimSuffix(f.Name, path.Ext(f.Name)), "-thumb") {
continue
}
sources = append(sources, source{folder: folder, id: f.Id, name: f.Name, mimeType: f.MimeType})
}
if list.NextPageToken == "" {
break
}
token = list.NextPageToken
}
}
log.Printf("migrating %d files to gs://%s", len(sources), blob.Bucket)
start := time.Now()
var mu sync.Mutex
var copied, thumbed int
var bytesWritten int64
var failed []string
var wg sync.WaitGroup
slots := make(chan struct{}, workers)
for _, src := range sources {
wg.Add(1)
slots <- struct{}{}
go func() {
defer wg.Done()
defer func() { <-slots }()
n, thumb, err := migrate(ctx, drv, bucket, src)
mu.Lock()
defer mu.Unlock()
if err != nil {
log.Printf("[ERROR] %s/%s: %v", src.folder, src.name, err)
failed = append(failed, src.folder+"/"+src.name)
return
}
copied++
bytesWritten += n
if thumb {
thumbed++
}
}()
}
wg.Wait()
log.Printf("copied %d files (%d thumbnails, %.1f MB) in %s",
copied, thumbed, float64(bytesWritten)/1e6, time.Since(start).Round(time.Millisecond))
if len(failed) > 0 {
log.Fatalf("[ERROR] %d files failed: %s", len(failed), strings.Join(failed, " "))
}
}
func migrate(ctx context.Context, drv *drive.Service, bucket *storage.BucketHandle, src source) (int64, bool, error) {
resp, err := drv.Files.Get(src.id).SupportsAllDrives(true).Download()
if err != nil {
return 0, false, fmt.Errorf("download: %w", err)
}
defer resp.Body.Close()
content, err := io.ReadAll(resp.Body)
if err != nil {
return 0, false, fmt.Errorf("read: %w", err)
}
written := int64(len(content))
if err := write(ctx, bucket, src.folder+"/"+src.name, src.mimeType, content); err != nil {
return 0, false, err
}
if !strings.HasPrefix(src.mimeType, "image/") {
return written, false, nil
}
thumb, err := blob.Thumbnail(content)
if err != nil {
return 0, false, fmt.Errorf("thumbnail: %w", err)
}
base := strings.TrimSuffix(src.name, path.Ext(src.name))
if err := write(ctx, bucket, src.folder+"/"+base+"-thumb.jpg", "image/jpeg", thumb); err != nil {
return 0, false, err
}
return written + int64(len(thumb)), true, nil
}
func write(ctx context.Context, bucket *storage.BucketHandle, name, mimeType string, content []byte) error {
w := bucket.Object(name).NewWriter(ctx)
w.ContentType = mimeType
if _, err := w.Write(content); err != nil {
w.Close()
return fmt.Errorf("write %s: %w", name, err)
}
if err := w.Close(); err != nil {
return fmt.Errorf("write %s: %w", name, err)
}
return nil
}
func subfolder(drv *drive.Service, parent, name string) (string, error) {
list, err := drv.Files.List().
Q(fmt.Sprintf("name = '%s' and '%s' in parents and mimeType = '%s' and trashed = false", name, parent, folderMime)).
SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives").
Fields("files(id)").Do()
if err != nil {
return "", fmt.Errorf("find folder %s: %w", name, err)
}
if len(list.Files) != 1 {
return "", fmt.Errorf("expected one %s folder, found %d", name, len(list.Files))
}
return list.Files[0].Id, nil
}
-39
View File
@@ -1,39 +0,0 @@
// Command probeblob times the download of a few drive media files.
package main
import (
"context"
"fmt"
"io"
"log"
"time"
"google.golang.org/api/drive/v3"
"google.golang.org/api/option"
)
func main() {
svc, err := drive.NewService(context.Background(),
option.WithScopes(drive.DriveScope))
if err != nil {
log.Fatalf("[ERROR] drive client: %v", err)
}
list, err := svc.Files.List().
Q("mimeType != 'application/vnd.google-apps.folder' and trashed = false").
SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives").
Fields("files(id, name, size)").PageSize(5).Do()
if err != nil {
log.Fatalf("[ERROR] list: %v", err)
}
for _, f := range list.Files {
start := time.Now()
resp, err := svc.Files.Get(f.Id).SupportsAllDrives(true).Download()
if err != nil {
fmt.Printf("%s (%d bytes): request error after %s: %v\n", f.Name, f.Size, time.Since(start).Round(time.Millisecond), err)
continue
}
n, err := io.Copy(io.Discard, resp.Body)
resp.Body.Close()
fmt.Printf("%s: %d bytes in %s (err %v)\n", f.Name, n, time.Since(start).Round(time.Millisecond), err)
}
}