Storage Services

Learn how to use the storage service for secure file storage and retrieval in your Asterisms applications.

Overview

The storage service provides secure, file-based storage capabilities for your applications. Files are automatically encrypted and associated with the current user context. The service handles various file types including JSON data, images, documents, and other binary files.

Basic Usage

Getting the Storage Service

const storage = sdk.storage();

Upload Files

try {
  // Upload a JSON data file
  const userData = {
    theme: 'dark',
    language: 'en',
    notifications: true
  };

  const file = new File([JSON.stringify(userData)], 'user-preferences.json', {
    type: 'application/json'
  });

  const result = await storage.upload(file, {
    bucketId: 'user-data',
    prefix: 'preferences/'
  });

  const externalId = result.externalId;
  // save externalId to your app's DB for reference to entity it's associated

  console.log('File uploaded successfully:', result);
} catch (error) {
  console.error('Failed to upload file:', error);
}

Download Files

try {
  const response = await storage.download('file-external-id', {
    fileName: 'user-preferences.json'
  });

  const fileContent = await response.text();
  const userData = JSON.parse(fileContent);

  console.log('User preferences:', userData);
} catch (error) {
  console.error('Failed to download file:', error);
}

Get File Metadata

try {
  const fileData = await storage.getData('file-external-id');
  console.log('File info:', {
    fileName: fileData.fileName,
    size: fileData.size,
    contentType: fileData.contentType,
    lastModified: fileData.lastModified
  });
} catch (error) {
  console.error('Failed to get file metadata:', error);
}

Delete Files

try {
  await storage.delete('file-external-id');
  console.log('File deleted successfully');
} catch (error) {
  console.error('Failed to delete file:', error);
}

File Types and Operations

The storage service can handle various file types:

// JSON data files
const jsonData = { userId: 'user123', settings: { theme: 'dark' } };
const jsonFile = new File([JSON.stringify(jsonData)], 'data.json', {
  type: 'application/json'
});
await storage.upload(jsonFile);

// Text files
const textFile = new File(['Hello, World!'], 'message.txt', {
  type: 'text/plain'
});
await storage.upload(textFile);

// Image files (from HTML input)
const imageInput = document.querySelector('input[type="file"]');
const imageFile = imageInput.files[0];
if (imageFile) {
  await storage.upload(imageFile, {
    bucketId: 'images',
    prefix: 'uploads/'
  });
}

// CSV files
const csvContent = 'name,age,city\nJohn,30,New York\nJane,25,London';
const csvFile = new File([csvContent], 'data.csv', {
  type: 'text/csv'
});
await storage.upload(csvFile);

List Files

try {
  // List all files in a bucket
  const files = await storage.list({ bucketId: 'user-data' });
  console.log('Files in bucket:', files);

  // List files with specific path
  const prefixedFiles = await storage.list({
    bucketId: 'user-data',
    path: 'preferences/'
  });
  console.log('Files in preferences folder:', prefixedFiles);
} catch (error) {
  console.error('Failed to list files:', error);
}

SvelteKit Integration

Storage API Routes

// src/routes/api/storage/upload/+server.ts
import { json } from '@sveltejs/kit';
import type { RequestHandler } from './$types';

export const POST: RequestHandler = async ({ request, locals }) => {
  try {
    const sdk = locals.sdk;
    if (!sdk) {
      return json({ error: 'SDK not available' }, { status: 503 });
    }

    const formData = await request.formData();
    const file = formData.get('file') as File;
    const bucketId = formData.get('bucketId') as string;
    const prefix = formData.get('prefix') as string;

    if (!file) {
      return json({ error: 'No file provided' }, { status: 400 });
    }

    const storage = sdk.storage();
    const result = await storage.upload(file, {
      bucketId,
      prefix
    });

    return json({ success: true, file: result });
  } catch (error) {
    console.error('Storage upload error:', error);
    return json({ error: 'Failed to upload file' }, { status: 500 });
  }
};

// src/routes/api/storage/[externalId]/+server.ts
import { json } from '@sveltejs/kit';
import type { RequestHandler } from './$types';

export const GET: RequestHandler = async ({ params, locals }) => {
  try {
    const sdk = locals.sdk;
    if (!sdk) {
      return json({ error: 'SDK not available' }, { status: 503 });
    }

    const storage = sdk.storage();
    const response = await storage.download(params.externalId);

    return new Response(response.body, {
      headers: {
        'Content-Type': response.headers.get('content-type') || 'application/octet-stream',
        'Content-Disposition': response.headers.get('content-disposition') || 'attachment'
      }
    });
  } catch (error) {
    console.error('Storage download error:', error);
    return json({ error: 'Failed to download file' }, { status: 500 });
  }
};

export const DELETE: RequestHandler = async ({ params, locals }) => {
  try {
    const sdk = locals.sdk;
    if (!sdk) {
      return json({ error: 'SDK not available' }, { status: 503 });
    }

    const storage = sdk.storage();
    await storage.delete(params.externalId);

    return json({ success: true });
  } catch (error) {
    console.error('Storage delete error:', error);
    return json({ error: 'Failed to delete file' }, { status: 500 });
  }
};

File Management Store (Reactive)

// src/lib/stores/fileStorage.ts
import { writable } from 'svelte/store';
import type { AsterismsBackendSDK } from '@asterisms/sdk-backend';
import type { StorageFileData } from '@asterisms/sdk-backend/storage';

export function createFileStorageStore(sdk: AsterismsBackendSDK, bucketId: string) {
  const { subscribe, set, update } = writable<StorageFileData[]>([]);

  const storage = sdk.storage();

  // Load initial files
  const refreshFiles = async () => {
    try {
      const files = await storage.list({ bucketId });
      set(files);
    } catch (error) {
      console.error('Failed to load files:', error);
    }
  };

  // Initial load
  refreshFiles();

  return {
    subscribe,
    refresh: refreshFiles,
    upload: async (file: File, metadata?: { prefix?: string }) => {
      try {
        const result = await storage.upload(file, {
          bucketId,
          ...metadata
        });
        await refreshFiles();
        return result;
      } catch (error) {
        console.error('Failed to upload file:', error);
        throw error;
      }
    },
    download: async (externalId: string, fileName?: string) => {
      try {
        return await storage.download(externalId, { fileName });
      } catch (error) {
        console.error('Failed to download file:', error);
        throw error;
      }
    },
    delete: async (externalId: string) => {
      try {
        await storage.delete(externalId);
        await refreshFiles();
      } catch (error) {
        console.error('Failed to delete file:', error);
        throw error;
      }
    },
    getMetadata: async (externalId: string) => {
      try {
        return await storage.getData(externalId);
      } catch (error) {
        console.error('Failed to get file metadata:', error);
        throw error;
      }
    }
  };
}

Usage in Components

<!-- src/lib/components/FileUpload.svelte -->
<script lang="ts">
    import { getSDK } from '$lib/sdkInstance';
    import { createFileStorageStore } from '$lib/stores/fileStorage';
    import type { StorageFileData } from '@asterisms/sdk-backend/storage';

    const sdk = getSDK();
    const fileStore = createFileStorageStore(sdk, 'user-uploads');

    let dragOver = false;
    let uploading = false;

    async function handleFileUpload(files: FileList) {
        uploading = true;
        try {
            for (const file of files) {
                await fileStore.upload(file, {
                    prefix: 'documents/'
                });
            }
        } catch (error) {
            console.error('Upload failed:', error);
        } finally {
            uploading = false;
        }
    }

    async function handleDownload(file: StorageFileData) {
        try {
            const response = await fileStore.download(file.externalId, file.fileName);
            const blob = await response.blob();

            // Create download link
            const url = URL.createObjectURL(blob);
            const a = document.createElement('a');
            a.href = url;
            a.download = file.fileName;
            a.click();
            URL.revokeObjectURL(url);
        } catch (error) {
            console.error('Download failed:', error);
        }
    }

    async function handleDelete(file: StorageFileData) {
        if (confirm(`Delete ${file.fileName}?`)) {
            try {
                await fileStore.delete(file.externalId);
            } catch (error) {
                console.error('Delete failed:', error);
            }
        }
    }

    function handleDragOver(event: DragEvent) {
        event.preventDefault();
        dragOver = true;
    }

    function handleDragLeave(event: DragEvent) {
        event.preventDefault();
        dragOver = false;
    }

    function handleDrop(event: DragEvent) {
        event.preventDefault();
        dragOver = false;

        const files = event.dataTransfer?.files;
        if (files && files.length > 0) {
            handleFileUpload(files);
        }
    }
</script>

<div class="file-upload-container">
    <div
        class="drop-zone"
        class:drag-over={dragOver}
        class:uploading
        on:dragover={handleDragOver}
        on:dragleave={handleDragLeave}
        on:drop={handleDrop}
    >
        <input
            type="file"
            multiple
            on:change={e => handleFileUpload(e.target.files)}
            disabled={uploading}
        />
        <p>{uploading ? 'Uploading...' : 'Drop files here or click to upload'}</p>
    </div>

    <div class="file-list">
        <h3>Uploaded Files</h3>
        {#each $fileStore as file (file.externalId)}
            <div class="file-item">
                <span class="file-name">{file.fileName}</span>
                <span class="file-size">{(file.size / 1024).toFixed(1)} KB</span>
                <button on:click={() => handleDownload(file)}>Download</button>
                <button on:click={() => handleDelete(file)}>Delete</button>
            </div>
        {/each}
    </div>
</div>

<style>
    .file-upload-container {
        max-width: 600px;
        margin: 20px auto;
        padding: 20px;
    }

    .drop-zone {
        border: 2px dashed #ccc;
        border-radius: 8px;
        padding: 40px;
        text-align: center;
        cursor: pointer;
        transition: all 0.3s ease;
    }

    .drop-zone.drag-over {
        border-color: #007bff;
        background-color: #f0f8ff;
    }

    .drop-zone.uploading {
        opacity: 0.6;
        pointer-events: none;
    }

    .file-list {
        margin-top: 20px;
    }

    .file-item {
        display: flex;
        align-items: center;
        gap: 10px;
        padding: 10px;
        border: 1px solid #eee;
        border-radius: 4px;
        margin-bottom: 5px;
    }

    .file-name {
        flex: 1;
        font-weight: bold;
    }

    .file-size {
        color: #666;
        font-size: 0.9em;
    }

    button {
        padding: 5px 10px;
        border: none;
        border-radius: 4px;
        cursor: pointer;
        font-size: 0.9em;
    }

    button:first-of-type {
        background-color: #007bff;
        color: white;
    }

    button:last-of-type {
        background-color: #dc3545;
        color: white;
    }
</style>

Advanced Usage

File Operations

// Move files between buckets or paths
async function moveFile(externalId: string, fromBucket: string, toBucket: string) {
  const storage = sdk.storage();

  try {
    await storage.move(
      externalId,
      { fileName: 'moved-file.json' },
      {
        externalId,
        currentBucketId: fromBucket,
        destinationBucketId: toBucket,
        keepOriginal: false
      }
    );

    console.log('File moved successfully');
  } catch (error) {
    console.error('Failed to move file:', error);
  }
}

// Replace existing file
async function replaceFile(externalId: string, newFile: File) {
  const storage = sdk.storage();

  try {
    const result = await storage.replace(externalId, newFile, {
      bucketId: 'documents'
    });

    console.log('File replaced successfully:', result);
  } catch (error) {
    console.error('Failed to replace file:', error);
  }
}

Ephemeral Links

// Generate temporary download links
async function generateEphemeralLinks(externalIds: string[]) {
  const storage = sdk.storage();
  const ephemeral = storage.ephemeral();

  try {
    const links = await ephemeral.generate({
      externalIds,
      ttl: 3600, // 1 hour
      maxUses: 10
    });

    console.log('Ephemeral links generated:', links);
    return links;
  } catch (error) {
    console.error('Failed to generate ephemeral links:', error);
  }
}

// Upload files with ephemeral callback
async function uploadWithEphemeralCallback(files: File[]) {
  const storage = sdk.storage();
  const ephemeral = storage.ephemeral();

  try {
    const uploadPayloads = files.map((file) => ({
      fileName: file.name,
      bucketId: 'uploads',
      redirectURL: 'https://myapp.com/upload-complete'
    }));

    const responses = await ephemeral.uploadBatch(uploadPayloads);

    console.log('Ephemeral upload URLs:', responses);
    return responses;
  } catch (error) {
    console.error('Failed to create ephemeral upload URLs:', error);
  }
}

Cached Storage

class CachedFileStorage {
  private cache = new Map<string, StorageFileData>();
  private storage: StorageService;

  constructor(sdk: AsterismsBackendSDK) {
    this.storage = sdk.storage();
  }

  async getFileData(externalId: string): Promise<StorageFileData | null> {
    // Check cache first
    if (this.cache.has(externalId)) {
      return this.cache.get(externalId)!;
    }

    // Load from storage
    try {
      const fileData = await this.storage.getData(externalId);
      this.cache.set(externalId, fileData);
      return fileData;
    } catch (error) {
      console.error('Failed to get file data:', error);
      return null;
    }
  }

  async uploadFile(file: File, metadata?: any): Promise<StorageFileData> {
    const result = await this.storage.upload(file, metadata);
    this.cache.set(result.externalId, result);
    return result;
  }

  async deleteFile(externalId: string): Promise<void> {
    // Remove from cache
    this.cache.delete(externalId);

    // Delete from storage
    await this.storage.delete(externalId);
  }

  clearCache(): void {
    this.cache.clear();
  }
}

Error Handling

Storage Errors

import { AsterismsBackendError } from '@asterisms/sdk-backend';

try {
  const largeFile = new File([new ArrayBuffer(10 * 1024 * 1024)], 'large-file.bin');
  await storage.upload(largeFile);
} catch (error) {
  if (error instanceof AsterismsBackendError) {
    switch (error.code) {
      case 'STORAGE_QUOTA_EXCEEDED':
        console.error('Storage quota exceeded');
        // Handle quota exceeded
        break;
      case 'STORAGE_FILE_TOO_LARGE':
        console.error('File too large to upload');
        // Handle large file
        break;
      case 'STORAGE_PERMISSION_DENIED':
        console.error('Permission denied');
        // Handle permission error
        break;
      case 'STORAGE_INVALID_FILE_TYPE':
        console.error('Invalid file type');
        // Handle invalid file type
        break;
      default:
        console.error('Storage error:', error.message);
    }
  } else {
    console.error('Unknown storage error:', error);
  }
}

Retry Logic

async function uploadWithRetry(
  file: File,
  metadata: any,
  maxRetries: number = 3
): Promise<StorageFileData | null> {
  const storage = sdk.storage();

  for (let attempt = 1; attempt <= maxRetries; attempt++) {
    try {
      const result = await storage.upload(file, metadata);
      return result;
    } catch (error) {
      console.error(`Upload attempt ${attempt} failed:`, error);

      if (attempt === maxRetries) {
        return null;
      }

      // Wait before retry (exponential backoff)
      await new Promise((resolve) => setTimeout(resolve, 1000 * Math.pow(2, attempt)));
    }
  }

  return null;
}

Testing Storage

Unit Tests

import { describe, it, expect, beforeEach } from 'vitest';
import { getAsterismsBackendSDK } from '@asterisms/sdk-backend';
import type { StorageService } from '@asterisms/sdk-backend/storage';

describe('Storage Service', () => {
  let sdk: AsterismsBackendSDK;
  let storage: StorageService;

  beforeEach(async () => {
    sdk = getAsterismsBackendSDK(testProps);
    await sdk.boot();
    storage = sdk.storage();
  });

  it('should upload and retrieve file', async () => {
    const testData = { name: 'Test', value: 42 };
    const file = new File([JSON.stringify(testData)], 'test.json', {
      type: 'application/json'
    });

    const uploadResult = await storage.upload(file);
    expect(uploadResult.fileName).toBe('test.json');
    expect(uploadResult.contentType).toBe('application/json');

    const downloadResponse = await storage.download(uploadResult.externalId);
    const downloadedContent = await downloadResponse.text();
    const parsedData = JSON.parse(downloadedContent);

    expect(parsedData).toEqual(testData);
  });

  it('should get file metadata', async () => {
    const file = new File(['test content'], 'test.txt', {
      type: 'text/plain'
    });

    const uploadResult = await storage.upload(file);
    const metadata = await storage.getData(uploadResult.externalId);

    expect(metadata.fileName).toBe('test.txt');
    expect(metadata.contentType).toBe('text/plain');
    expect(metadata.size).toBe(12); // 'test content'.length
  });

  it('should delete file successfully', async () => {
    const file = new File(['delete me'], 'delete-test.txt', {
      type: 'text/plain'
    });

    const uploadResult = await storage.upload(file);
    await storage.delete(uploadResult.externalId);

    // Should throw error when trying to get deleted file
    await expect(storage.getData(uploadResult.externalId)).rejects.toThrow();
  });

  it('should list files in bucket', async () => {
    const bucketId = 'test-bucket';

    const file1 = new File(['content1'], 'file1.txt', { type: 'text/plain' });
    const file2 = new File(['content2'], 'file2.txt', { type: 'text/plain' });

    await storage.upload(file1, { bucketId });
    await storage.upload(file2, { bucketId });

    const files = await storage.list({ bucketId });
    expect(files.length).toBeGreaterThanOrEqual(2);

    const fileNames = files.map((f) => f.fileName);
    expect(fileNames).toContain('file1.txt');
    expect(fileNames).toContain('file2.txt');
  });
});

Integration Tests

import { test, expect } from '@playwright/test';

test('storage API endpoints work correctly', async ({ page }) => {
  // Test file upload
  const fileContent = 'Hello, World!';
  const file = new File([fileContent], 'test.txt', { type: 'text/plain' });

  const formData = new FormData();
  formData.append('file', file);
  formData.append('bucketId', 'test-bucket');

  const uploadResponse = await page.request.post('/api/storage/upload', {
    data: formData
  });
  expect(uploadResponse.ok()).toBeTruthy();

  const uploadResult = await uploadResponse.json();
  expect(uploadResult.success).toBeTruthy();
  expect(uploadResult.file.fileName).toBe('test.txt');

  // Test file download
  const downloadResponse = await page.request.get(`/api/storage/${uploadResult.file.externalId}`);
  expect(downloadResponse.ok()).toBeTruthy();

  const downloadedContent = await downloadResponse.text();
  expect(downloadedContent).toBe(fileContent);

  // Test file deletion
  const deleteResponse = await page.request.delete(`/api/storage/${uploadResult.file.externalId}`);
  expect(deleteResponse.ok()).toBeTruthy();

  // Verify deletion
  const getAfterDelete = await page.request.get(`/api/storage/${uploadResult.file.externalId}`);
  expect(getAfterDelete.ok()).toBeFalsy();
});

Security and Best Practices

File Validation

import { z } from 'zod';

const ALLOWED_FILE_TYPES = [
  'image/jpeg',
  'image/png',
  'image/gif',
  'application/pdf',
  'text/plain'
];
const MAX_FILE_SIZE = 10 * 1024 * 1024; // 10MB

async function validateAndUploadFile(file: File, bucketId: string) {
  try {
    // Validate file type
    if (!ALLOWED_FILE_TYPES.includes(file.type)) {
      throw new Error(`File type ${file.type} is not allowed`);
    }

    // Validate file size
    if (file.size > MAX_FILE_SIZE) {
      throw new Error(`File size ${file.size} exceeds maximum allowed size`);
    }

    // Validate file name
    if (!/^[a-zA-Z0-9._-]+$/.test(file.name)) {
      throw new Error('File name contains invalid characters');
    }

    const storage = sdk.storage();
    const result = await storage.upload(file, { bucketId });

    return result;
  } catch (error) {
    console.error('File validation failed:', error);
    throw error;
  }
}

Secure File Handling

// ❌ Bad: Don't upload executable files
const badFile = new File(['malicious code'], 'malware.exe', { type: 'application/exe' });

// ❌ Bad: Don't store sensitive data in plain text
const sensitiveFile = new File(['password123'], 'passwords.txt', { type: 'text/plain' });

// ✅ Good: Upload safe file types with validation
const safeFile = new File(['{"theme": "dark"}'], 'user-preferences.json', {
  type: 'application/json'
});
await validateAndUploadFile(safeFile, 'user-data');

// ✅ Good: Use structured data
const configFile = new File(
  [
    JSON.stringify({
      appSettings: {
        theme: 'dark',
        language: 'en'
      }
    })
  ],
  'app-config.json',
  { type: 'application/json' }
);

File Naming Conventions

// Use consistent naming patterns
const FILE_PATTERNS = {
  USER_PREFERENCES: (userId: string) => `user-preferences-${userId}.json`,
  DOCUMENT_UPLOAD: (userId: string, timestamp: number) => `document-${userId}-${timestamp}`,
  TEMP_FILE: (sessionId: string) => `temp-${sessionId}`,
  PROCESSED_DATA: (dataId: string) => `processed-${dataId}.json`
};

// Usage
const storage = sdk.storage();
const fileName = FILE_PATTERNS.USER_PREFERENCES(currentUserId);
const file = new File([JSON.stringify(preferences)], fileName, { type: 'application/json' });
await storage.upload(file, { bucketId: 'user-data' });

Performance Optimization

Efficient File Operations

// ❌ Bad: Multiple separate uploads
await storage.upload(file1, { bucketId: 'docs' });
await storage.upload(file2, { bucketId: 'docs' });
await storage.upload(file3, { bucketId: 'docs' });

// ✅ Good: Batch uploads with Promise.all
const uploadPromises = [file1, file2, file3].map((file) =>
  storage.upload(file, { bucketId: 'docs' })
);
const results = await Promise.all(uploadPromises);

// ✅ Good: Use ephemeral upload for large batches
const ephemeral = storage.ephemeral();
const uploadPayloads = files.map((file) => ({
  fileName: file.name,
  bucketId: 'uploads'
}));
const ephemeralResponses = await ephemeral.uploadBatch(uploadPayloads);

File Size Optimization

// Compress images before upload
async function compressAndUpload(imageFile: File, quality: number = 0.8) {
  const canvas = document.createElement('canvas');
  const ctx = canvas.getContext('2d');
  const img = new Image();

  return new Promise((resolve, reject) => {
    img.onload = async () => {
      canvas.width = img.width;
      canvas.height = img.height;
      ctx.drawImage(img, 0, 0);

      canvas.toBlob(
        async (blob) => {
          if (blob) {
            const compressedFile = new File([blob], imageFile.name, {
              type: 'image/jpeg'
            });
            const result = await storage.upload(compressedFile);
            resolve(result);
          } else {
            reject(new Error('Failed to compress image'));
          }
        },
        'image/jpeg',
        quality
      );
    };

    img.src = URL.createObjectURL(imageFile);
  });
}

Caching Strategy

// Cache frequently accessed files
const fileCache = new Map<string, { data: any; timestamp: number }>();
const CACHE_TTL = 5 * 60 * 1000; // 5 minutes

async function getCachedFile(externalId: string): Promise<any> {
  const cached = fileCache.get(externalId);

  if (cached && Date.now() - cached.timestamp < CACHE_TTL) {
    return cached.data;
  }

  const response = await storage.download(externalId);
  const data = await response.json();

  fileCache.set(externalId, {
    data,
    timestamp: Date.now()
  });

  return data;
}

Next Steps

Authentication & Authorization - User authentication
Notification Services - Sending notifications
Registration Services - App registration
Examples - Complete working examples

Storage Services#

Overview#

Basic Usage#

Getting the Storage Service#

Upload Files#

Download Files#

Get File Metadata#

Delete Files#

File Types and Operations#

List Files#

SvelteKit Integration#

Storage API Routes#

File Management Store (Reactive)#

Usage in Components#

Advanced Usage#

File Operations#

Ephemeral Links#

Cached Storage#

Error Handling#

Storage Errors#

Retry Logic#

Testing Storage#

Unit Tests#

Integration Tests#

Security and Best Practices#

File Validation#

Secure File Handling#

File Naming Conventions#

Performance Optimization#

Efficient File Operations#

File Size Optimization#

Caching Strategy#

Next Steps#