Drop the drive-era media tools
This commit is contained in:
Ian Gulliver
@@ -1,180 +0,0 @@
|
||||
// Command migratemedia copies the drive media folders into the media bucket, generating thumbnails.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"cloud.google.com/go/storage"
|
||||
"google.golang.org/api/drive/v3"
|
||||
"google.golang.org/api/option"
|
||||
|
||||
"heliosian/internal/blob"
|
||||
)
|
||||
|
||||
const (
|
||||
folderMime = "application/vnd.google-apps.folder"
|
||||
workers = 12
|
||||
)
|
||||
|
||||
type source struct {
|
||||
folder string
|
||||
id string
|
||||
name string
|
||||
mimeType string
|
||||
}
|
||||
|
||||
func main() {
|
||||
ctx := context.Background()
|
||||
drv, err := drive.NewService(ctx, option.WithScopes(drive.DriveReadonlyScope))
|
||||
if err != nil {
|
||||
log.Fatalf("[ERROR] drive client: %v", err)
|
||||
}
|
||||
client, err := storage.NewClient(ctx)
|
||||
if err != nil {
|
||||
log.Fatalf("[ERROR] storage client: %v", err)
|
||||
}
|
||||
bucket := client.Bucket(blob.Bucket)
|
||||
|
||||
drives, err := drv.Drives.List().Do()
|
||||
if err != nil {
|
||||
log.Fatalf("[ERROR] list shared drives: %v", err)
|
||||
}
|
||||
if len(drives.Drives) != 1 {
|
||||
log.Fatalf("[ERROR] expected one shared drive, found %d", len(drives.Drives))
|
||||
}
|
||||
root := drives.Drives[0].Id
|
||||
|
||||
sources := []source{}
|
||||
for _, folder := range []string{"people", "families"} {
|
||||
folderID, err := subfolder(drv, root, folder)
|
||||
if err != nil {
|
||||
log.Fatalf("[ERROR] %v", err)
|
||||
}
|
||||
token := ""
|
||||
for {
|
||||
call := drv.Files.List().
|
||||
Q(fmt.Sprintf("'%s' in parents and trashed = false", folderID)).
|
||||
SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives").
|
||||
Fields("nextPageToken, files(id, name, mimeType)").PageSize(1000)
|
||||
if token != "" {
|
||||
call = call.PageToken(token)
|
||||
}
|
||||
list, err := call.Do()
|
||||
if err != nil {
|
||||
log.Fatalf("[ERROR] list %s: %v", folder, err)
|
||||
}
|
||||
for _, f := range list.Files {
|
||||
if f.MimeType == folderMime {
|
||||
continue
|
||||
}
|
||||
if strings.HasSuffix(strings.TrimSuffix(f.Name, path.Ext(f.Name)), "-thumb") {
|
||||
continue
|
||||
}
|
||||
sources = append(sources, source{folder: folder, id: f.Id, name: f.Name, mimeType: f.MimeType})
|
||||
}
|
||||
if list.NextPageToken == "" {
|
||||
break
|
||||
}
|
||||
token = list.NextPageToken
|
||||
}
|
||||
}
|
||||
log.Printf("migrating %d files to gs://%s", len(sources), blob.Bucket)
|
||||
|
||||
start := time.Now()
|
||||
var mu sync.Mutex
|
||||
var copied, thumbed int
|
||||
var bytesWritten int64
|
||||
var failed []string
|
||||
var wg sync.WaitGroup
|
||||
slots := make(chan struct{}, workers)
|
||||
for _, src := range sources {
|
||||
wg.Add(1)
|
||||
slots <- struct{}{}
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer func() { <-slots }()
|
||||
n, thumb, err := migrate(ctx, drv, bucket, src)
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] %s/%s: %v", src.folder, src.name, err)
|
||||
failed = append(failed, src.folder+"/"+src.name)
|
||||
return
|
||||
}
|
||||
copied++
|
||||
bytesWritten += n
|
||||
if thumb {
|
||||
thumbed++
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
log.Printf("copied %d files (%d thumbnails, %.1f MB) in %s",
|
||||
copied, thumbed, float64(bytesWritten)/1e6, time.Since(start).Round(time.Millisecond))
|
||||
if len(failed) > 0 {
|
||||
log.Fatalf("[ERROR] %d files failed: %s", len(failed), strings.Join(failed, " "))
|
||||
}
|
||||
}
|
||||
|
||||
func migrate(ctx context.Context, drv *drive.Service, bucket *storage.BucketHandle, src source) (int64, bool, error) {
|
||||
resp, err := drv.Files.Get(src.id).SupportsAllDrives(true).Download()
|
||||
if err != nil {
|
||||
return 0, false, fmt.Errorf("download: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
content, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return 0, false, fmt.Errorf("read: %w", err)
|
||||
}
|
||||
written := int64(len(content))
|
||||
if err := write(ctx, bucket, src.folder+"/"+src.name, src.mimeType, content); err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
if !strings.HasPrefix(src.mimeType, "image/") {
|
||||
return written, false, nil
|
||||
}
|
||||
thumb, err := blob.Thumbnail(content)
|
||||
if err != nil {
|
||||
return 0, false, fmt.Errorf("thumbnail: %w", err)
|
||||
}
|
||||
base := strings.TrimSuffix(src.name, path.Ext(src.name))
|
||||
if err := write(ctx, bucket, src.folder+"/"+base+"-thumb.jpg", "image/jpeg", thumb); err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
return written + int64(len(thumb)), true, nil
|
||||
}
|
||||
|
||||
func write(ctx context.Context, bucket *storage.BucketHandle, name, mimeType string, content []byte) error {
|
||||
w := bucket.Object(name).NewWriter(ctx)
|
||||
w.ContentType = mimeType
|
||||
if _, err := w.Write(content); err != nil {
|
||||
w.Close()
|
||||
return fmt.Errorf("write %s: %w", name, err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
return fmt.Errorf("write %s: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func subfolder(drv *drive.Service, parent, name string) (string, error) {
|
||||
list, err := drv.Files.List().
|
||||
Q(fmt.Sprintf("name = '%s' and '%s' in parents and mimeType = '%s' and trashed = false", name, parent, folderMime)).
|
||||
SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives").
|
||||
Fields("files(id)").Do()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("find folder %s: %w", name, err)
|
||||
}
|
||||
if len(list.Files) != 1 {
|
||||
return "", fmt.Errorf("expected one %s folder, found %d", name, len(list.Files))
|
||||
}
|
||||
return list.Files[0].Id, nil
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
// Command probeblob times the download of a few drive media files.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"google.golang.org/api/drive/v3"
|
||||
"google.golang.org/api/option"
|
||||
)
|
||||
|
||||
func main() {
|
||||
svc, err := drive.NewService(context.Background(),
|
||||
option.WithScopes(drive.DriveScope))
|
||||
if err != nil {
|
||||
log.Fatalf("[ERROR] drive client: %v", err)
|
||||
}
|
||||
list, err := svc.Files.List().
|
||||
Q("mimeType != 'application/vnd.google-apps.folder' and trashed = false").
|
||||
SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Corpora("allDrives").
|
||||
Fields("files(id, name, size)").PageSize(5).Do()
|
||||
if err != nil {
|
||||
log.Fatalf("[ERROR] list: %v", err)
|
||||
}
|
||||
for _, f := range list.Files {
|
||||
start := time.Now()
|
||||
resp, err := svc.Files.Get(f.Id).SupportsAllDrives(true).Download()
|
||||
if err != nil {
|
||||
fmt.Printf("%s (%d bytes): request error after %s: %v\n", f.Name, f.Size, time.Since(start).Round(time.Millisecond), err)
|
||||
continue
|
||||
}
|
||||
n, err := io.Copy(io.Discard, resp.Body)
|
||||
resp.Body.Close()
|
||||
fmt.Printf("%s: %d bytes in %s (err %v)\n", f.Name, n, time.Since(start).Round(time.Millisecond), err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user