Tag Archives: python

How to manage Google AppEngine maintenance periods

Properly uploading files to Amazon S3

Here is a little script I wrote and I though ought to be shared. I use it to upload static files like images, css and javascript so that they can be served by Amazon S3 instead of the main application server (like Google App Engine).

It’s written in Python and does interesting things like compressing and minifying what needs to be. It takes 3 arguments and as 2 options:

Usage: s3uploader.py [-xm] src_folder destination_bucket_name prefix

1	Usage: s3uploader.py [-xm] src_folder destination_bucket_name prefix

src_folder: path to the local folder containing the static files to upload
destination_bucket_name: name of the S3 bucket to upload to (e.g. static.example.com)
prefix: a prefix to use for the destination key (kind of a folder on the destination bucket, I use it to specify a release version to defeat browser caching)
x: if set, the script will set a far future expiry for all files, otherwise the S3 default will be used (one day if I remember well)
m: if set, the script will minify css and javascript files

First you will have to install some dependencies, namely boto, jsmin and cssmin. Installation procedure will depend on your OS but on my Mac I do the following:

sudo easy_install boto
sudo easy_install jsmin
sudo easy_install cssmin

sudo easy_install boto

sudo easy_install jsmin

sudo easy_install cssmin

And here is the script itself:

#! /usr/bin/env python
import os, sys, boto, mimetypes, zipfile, gzip
from io import StringIO, BytesIO
from optparse import OptionParser
from jsmin import *
from cssmin import *

# Boto picks up configuration from the env.
os.environ['AWS_ACCESS_KEY_ID'] = 'Your AWS access key id goes here'
os.environ['AWS_SECRET_ACCESS_KEY'] = 'Your AWS secret access key goes here'

# The list of content types to gzip, add more if needed
COMPRESSIBLE = [ 'text/plain', 'text/csv', 'application/xml',
                'application/javascript', 'text/css' ]

def main():
    parser = OptionParser(usage='usage: {5f676304cfd4ae2259631a2f5a3ea815e87ae216a7b910a3d060a7b08502a4b2}prog [options] src_folder destination_bucket_name prefix')
    parser.add_option('-x', '--expires', action='store_true', help='set far future expiry for all files')
    parser.add_option('-m', '--minify', action='store_true', help='minify javascript files')
    (options, args) = parser.parse_args()
    if len(args) != 3:
        parser.error("incorrect number of arguments")
    src_folder = os.path.normpath(args[0])
    bucket_name = args[1]
    prefix = args[2]

    conn = boto.connect_s3()
    bucket = conn.get_bucket(bucket_name)

    namelist = []
    for root, dirs, files in os.walk(src_folder):
        if files and not '.svn' in root:
            path = os.path.relpath(root, src_folder)
            namelist += [os.path.normpath(os.path.join(path, f)) for f in files]

    print 'Uploading {5f676304cfd4ae2259631a2f5a3ea815e87ae216a7b910a3d060a7b08502a4b2}d files to bucket {5f676304cfd4ae2259631a2f5a3ea815e87ae216a7b910a3d060a7b08502a4b2}s' {5f676304cfd4ae2259631a2f5a3ea815e87ae216a7b910a3d060a7b08502a4b2} (len(namelist), bucket.name)
    for name in namelist:
        content = open(os.path.join(src_folder, name))
        key = bucket.new_key(os.path.join(prefix, name))
        type, encoding = mimetypes.guess_type(name)
        type = type or 'application/octet-stream'
        headers = { 'Content-Type': type, 'x-amz-acl': 'public-read' }
        states = [type]

        if options.expires:
            # We only use HTTP 1.1 headers because they are relative to the time of download
            # instead of being hardcoded.
            headers['Cache-Control'] = 'max-age {5f676304cfd4ae2259631a2f5a3ea815e87ae216a7b910a3d060a7b08502a4b2}d' {5f676304cfd4ae2259631a2f5a3ea815e87ae216a7b910a3d060a7b08502a4b2} (3600 * 24 * 365)

        if options.minify and type == 'application/javascript':
            outs = StringIO()
            JavascriptMinify().minify(content, outs)
            content.close()
            content = outs.getvalue()
            if len(content) > 0 and content[0] == '\n':
                content = content[1:]
            content = BytesIO(content)
            states.append('minified')

        if options.minify and type == 'text/css':
            outs = cssmin(content.read())
            content.close()
            content = outs
            if len(content) > 0 and content[0] == '\n':
                content = content[1:]
            content = BytesIO(content)
            states.append('minified')

        if type in COMPRESSIBLE:
            headers['Content-Encoding'] = 'gzip'
            compressed = StringIO()
            gz = gzip.GzipFile(filename=name, fileobj=compressed, mode='w')
            gz.writelines(content)
            gz.close()
            content.close
            content = BytesIO(compressed.getvalue())
            states.append('gzipped')

        states = ', '.join(states)
        print '- {5f676304cfd4ae2259631a2f5a3ea815e87ae216a7b910a3d060a7b08502a4b2}s => {5f676304cfd4ae2259631a2f5a3ea815e87ae216a7b910a3d060a7b08502a4b2}s ({5f676304cfd4ae2259631a2f5a3ea815e87ae216a7b910a3d060a7b08502a4b2}s)' {5f676304cfd4ae2259631a2f5a3ea815e87ae216a7b910a3d060a7b08502a4b2} (name, key.name, states)
        key.set_contents_from_file(content, headers)
        content.close();

if __name__ == '__main__':
    main()

#! /usr/bin/env python

import os, sys, boto, mimetypes, zipfile, gzip

from io import StringIO, BytesIO

from optparse import OptionParser

from jsmin import *

from cssmin import *

# Boto picks up configuration from the env.

os.environ['AWS_ACCESS_KEY_ID'] = 'Your AWS access key id goes here'

os.environ['AWS_SECRET_ACCESS_KEY'] = 'Your AWS secret access key goes here'

# The list of content types to gzip, add more if needed

COMPRESSIBLE = [ 'text/plain', 'text/csv', 'application/xml',

'application/javascript', 'text/css' ]

def main():

parser = OptionParser(usage='usage: {5f676304cfd4ae2259631a2f5a3ea815e87ae216a7b910a3d060a7b08502a4b2}prog [options] src_folder destination_bucket_name prefix')

parser.add_option('-x', '--expires', action='store_true', help='set far future expiry for all files')

parser.add_option('-m', '--minify', action='store_true', help='minify javascript files')

(options, args) = parser.parse_args()

if len(args) != 3:

parser.error("incorrect number of arguments")

src_folder = os.path.normpath(args[0])

bucket_name = args[1]

prefix = args[2]

conn = boto.connect_s3()

bucket = conn.get_bucket(bucket_name)

namelist = []

for root, dirs, files in os.walk(src_folder):

if files and not '.svn' in root:

path = os.path.relpath(root, src_folder)

namelist += [os.path.normpath(os.path.join(path, f)) for f in files]

print 'Uploading {5f676304cfd4ae2259631a2f5a3ea815e87ae216a7b910a3d060a7b08502a4b2}d files to bucket {5f676304cfd4ae2259631a2f5a3ea815e87ae216a7b910a3d060a7b08502a4b2}s' {5f676304cfd4ae2259631a2f5a3ea815e87ae216a7b910a3d060a7b08502a4b2} (len(namelist), bucket.name)

for name in namelist:

content = open(os.path.join(src_folder, name))

key = bucket.new_key(os.path.join(prefix, name))

type, encoding = mimetypes.guess_type(name)

type = type or 'application/octet-stream'

headers = { 'Content-Type': type, 'x-amz-acl': 'public-read' }

states = [type]

if options.expires:

# We only use HTTP 1.1 headers because they are relative to the time of download

# instead of being hardcoded.

headers['Cache-Control'] = 'max-age {5f676304cfd4ae2259631a2f5a3ea815e87ae216a7b910a3d060a7b08502a4b2}d' {5f676304cfd4ae2259631a2f5a3ea815e87ae216a7b910a3d060a7b08502a4b2} (3600 * 24 * 365)

if options.minify and type == 'application/javascript':

outs = StringIO()

JavascriptMinify().minify(content, outs)

content.close()

content = outs.getvalue()

if len(content) > 0 and content[0] == '\n':

content = content[1:]

content = BytesIO(content)

states.append('minified')

if options.minify and type == 'text/css':

outs = cssmin(content.read())

content.close()

content = outs

if len(content) > 0 and content[0] == '\n':

content = content[1:]

content = BytesIO(content)

states.append('minified')

if type in COMPRESSIBLE:

headers['Content-Encoding'] = 'gzip'

compressed = StringIO()

gz = gzip.GzipFile(filename=name, fileobj=compressed, mode='w')

gz.writelines(content)

gz.close()

content.close

content = BytesIO(compressed.getvalue())

states.append('gzipped')

states = ', '.join(states)

print '- {5f676304cfd4ae2259631a2f5a3ea815e87ae216a7b910a3d060a7b08502a4b2}s => {5f676304cfd4ae2259631a2f5a3ea815e87ae216a7b910a3d060a7b08502a4b2}s ({5f676304cfd4ae2259631a2f5a3ea815e87ae216a7b910a3d060a7b08502a4b2}s)' {5f676304cfd4ae2259631a2f5a3ea815e87ae216a7b910a3d060a7b08502a4b2} (name, key.name, states)

key.set_contents_from_file(content, headers)

content.close();

if __name__ == '__main__':

main()

Thanks to Nico for the expiry trick :)

vedovini.net

Founded in 1969, building software since 1985.

Tag Archives: python

How to manage Google AppEngine maintenance periods

Properly uploading files to Amazon S3

A year in review