Skip to content

Commit c9aaf53

Browse files
author
aleaxit
committed
Move to shard+index strategy for tiles (prep side done).
1 parent 36dcbad commit c9aaf53

15 files changed

+246
-12
lines changed

‎cazip.ply

1.34 MB
Binary file not shown.

‎gae/ZIPCA_1.zip

953 KB
Binary file not shown.

‎gae/ZIPCA_2.zip

961 KB
Binary file not shown.

‎gae/ZIPCA_3.zip

976 KB
Binary file not shown.

‎gae/ZIPCA_4.zip

974 KB
Binary file not shown.

‎gae/ZIPCA_5.zip

985 KB
Binary file not shown.

‎gae/ZIPCA_6.zip

986 KB
Binary file not shown.

‎gae/ZIPCA_7.zip

59.1 KB
Binary file not shown.

‎gae/ZIPCA_tiles.sdb

92 KB
Binary file not shown.

‎gae/main.py

+22-12
Original file line numberDiff line numberDiff line change
@@ -5,35 +5,45 @@
55
import logging
66
import wsgiref.handlers
77
import zipfile
8+
from google.appengine.ext import db
89
from google.appengine.ext import webapp
910
from google.appengine.api import memcache
1011

1112
import models
1213

13-
thezip = None
14-
def fromzip(name, x, y, z, unused_lastarg):
15-
global thezip
16-
if thezip is None: thezip = zipfile.ZipFile('tiles.zip', 'r')
14+
zipnames = dict(USA='tiles', ZIPCA='tiles_zipca')
15+
thezips = dict()
16+
def fromzip(png, name, x, y, z, unused_lastarg):
17+
thezip = thezips.get(png)
18+
zn = zipnames.get(png)
19+
if thezip is None:
20+
if zn is None: return None
21+
thezip = thezips[png] = zipfile.ZipFile('%s.zip'%zn, 'r')
1722
name += '.png'
18-
logging.info('From zip: %s', name)
23+
logging.info('From zip: %s in %s', name, zn)
1924
try:
2025
return thezip.read(name)
2126
except KeyError:
2227
return None
2328

2429
def persist_tile(name, data):
2530
tile = models.Tile(name=name, data=data)
26-
tile.put()
27-
logging.info('%r just made (%d)', name, len(data))
31+
try:
32+
tile.put()
33+
except db.TransactionFailedError, e:
34+
logging.info('%r not stored yet, "%s"', name, e)
35+
else:
36+
logging.info('%r just made (%d)', name, len(data))
2837

29-
def get_tile(name, x, y, z, maker=None):
38+
def get_tile(png, name, x, y, z, maker=None):
3039
""" Get from cache or store, or make and put in store and cache, a tile.
3140
3241
Args:
42+
png: name of the PNG family (USA, ZIPCA, &c)
3343
name: name of the tile (unique key)
3444
x, y, z: Google Maps coordinates of the tile
3545
maker: function that generates and returns the tile (or None)
36-
when called with args: name, x, y, z, None
46+
when called with args: png, name, x, y, z, None
3747
Returns:
3848
PNG data for the tile (None if absent and maker was or returned None)
3949
"""
@@ -54,7 +64,7 @@ def get_tile(name, x, y, z, maker=None):
5464
if maker is None:
5565
data = None
5666
else:
57-
data = maker(name, x, y, z, None)
67+
data = maker(png, name, x, y, z, None)
5868
if data is None:
5969
logging.info('%r not there', name)
6070
with open('tile_crosshairs.png') as f:
@@ -84,15 +94,15 @@ def get(self):
8494
png = queryget(query, 'png')
8595
x, y, z = (int(queryget(query, n) or -1) for n in 'xyz')
8696
# generate/produce tile on-the-fly if needed
87-
if png=='USA':
97+
if png=='USA' or png=='ZIPCA':
8898
maker = fromzip
8999
else:
90100
# unknown PNG type requested
91101
self.response.set_status(404, "PNG type %r not found" % png)
92102
return
93103
# temporarily inhibit maker functionality
94104
name = 'tile_%s_%s_%s_%s' % (png, z, x, y)
95-
data = get_tile(name, x, y, z, maker)
105+
data = get_tile(png, name, x, y, z, maker)
96106
self.response.headers['Content-Type'] = 'image/png'
97107
self.response.out.write(data)
98108

‎gae/zipca.html

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
2+
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
3+
<html xmlns="http://www.w3.org/1999/xhtml"
4+
xmlns:v="urn:schemas-microsoft-com:vml">
5+
6+
<head>
7+
<meta http-equiv="content-type" content="text/html; charset=utf-8"/>
8+
<title>GEography-PYthon project demo: zipcodes of California</title>
9+
<script src="http://maps.google.com/maps?file=api&amp;v=2&amp;key=ABQIAAAAQM4ndGoq1nYAUPZHcylyPxT-UBYwKtcgiT7MuRpDGKkK3cJM8BROAlMhLOUhpXC40GxMbXWmOa2nrg"
10+
type="text/javascript"></script>
11+
<script type="text/javascript">
12+
//<![CDATA[
13+
14+
function load() {
15+
if (GBrowserIsCompatible()) {
16+
var map = new GMap2(document.getElementById("map_canvas"));
17+
map.setCenter(new GLatLng(37.14,-120.93), 6);
18+
map.addControl(new GLargeMapControl());
19+
20+
var myCopyright = new GCopyrightCollection("© ");
21+
myCopyright.addCopyright(new GCopyright('Demo',
22+
new GLatLngBounds(new GLatLng(-90,-180), new GLatLng(90,180)),
23+
0,'©2008 [email protected]'));
24+
25+
var tilelayer = new GTileLayer(myCopyright, null, null, {
26+
tileUrlTemplate: '/tile?png=ZIPCA&x={X}&y={Y}&z={Z}',
27+
isPng: true,
28+
opacity: 1.0
29+
});
30+
31+
var myTileLayer = new GTileLayerOverlay(tilelayer);
32+
map.addOverlay(myTileLayer);
33+
}
34+
}
35+
36+
//]]>
37+
</script>
38+
</head>
39+
<body onload="load()" onunload="GUnload()">
40+
<h1>Zipcodes of California</h1>
41+
<div id="map_canvas" style="width: 600px; height: 600px"></div>
42+
</body>
43+
</html>
44+

‎prepcazip_tiles.py

+91
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
""" Prepare tiles for CA zipcode boundaries as PNG files in /tmp/.
2+
"""
3+
from __future__ import with_statement
4+
5+
import cStringIO
6+
import logging
7+
import os
8+
import sys
9+
10+
from PIL import Image, ImageDraw, ImageFont, ImagePath
11+
12+
import shp2polys
13+
import tile
14+
15+
class Converter(shp2polys.Converter):
16+
infile = 'ca/zt06_d00.shp'
17+
oufile = 'cazip.ply'
18+
nameid = 'ZCTA'
19+
@classmethod
20+
def valid(cls, id): return id.isdigit()
21+
22+
class PolyReader(shp2polys.PolyReader):
23+
infile = Converter.oufile
24+
25+
def s(aray):
26+
res = []
27+
for x in aray: res.append('%.2f' % x)
28+
return '[%s]' % ','.join(res)
29+
30+
def main():
31+
shp2polys.setlogging()
32+
33+
name_format = 'tile_ZIPCA_%s_%s_%s'
34+
MIN_ZOOM = 6
35+
MAX_ZOOM = 12
36+
m = tile.GlobalMercator()
37+
38+
if not os.path.isfile(Converter.oufile):
39+
logging.info('Building %r', Converter.oufile)
40+
c = Converter()
41+
c.doit()
42+
43+
for zoom in range(MIN_ZOOM, MAX_ZOOM+1):
44+
r = PolyReader(zoom)
45+
bb = r.get_tiles_ranges()
46+
size = 256*(bb[2]-bb[0]+1), 256*(bb[3]-bb[1]+1)
47+
logging.info('zoom %s: tiles %s, size %s', r.zoom, bb, size)
48+
white = 0
49+
try: im = Image.new('P', size, white)
50+
except MemoryError:
51+
logging.error('sorry, zoom %r takes too much memory, stopping', zoom)
52+
break
53+
palette = [255]*3 + [255, 0, 0] + [0, 255, 0]
54+
red = 1
55+
green = 2
56+
im.putpalette(palette)
57+
58+
matrix = m.getMetersToPixelsXform(zoom, bb)
59+
# font = ImageFont.truetype('/Library/Fonts/ChalkboardBold.ttf', 24)
60+
draw = ImageDraw.Draw(im)
61+
for name, bbox, starts, lengths, meters in r:
62+
for s, l in zip(starts, lengths):
63+
p = ImagePath.Path(meters[s:s+l])
64+
p.transform(matrix)
65+
draw.polygon(p, outline=red)
66+
del draw
67+
68+
# save all tiles
69+
for tx in range(bb[0], bb[2]+1):
70+
left = (tx-bb[0]) * 256
71+
right = left+255
72+
for ty in range(bb[1], bb[3]+1):
73+
top = (bb[3]-ty) * 256
74+
bottom = top+255
75+
tileim = im.crop((left, top, right, bottom))
76+
if tileim.getbbox() is None:
77+
logging.debug('Skip empty tile %s/%s', tx, ty)
78+
continue
79+
gtx, gty = m.GoogleTile(tx, ty, zoom)
80+
name = name_format % (zoom, gtx, gty)
81+
out = cStringIO.StringIO()
82+
tileim.save(out, format='PNG', transparency=white)
83+
data = out.getvalue()
84+
out.close()
85+
# c.execute('INSERT INTO tiles VALUES (:1, :2, :3)', (name, data, ''))
86+
# conn.commit()
87+
with open('/tmp/%s.png'%name, 'wb') as f:
88+
f.write(data)
89+
90+
main()
91+

‎prepzips.py

+89
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
""" Prepare ZIP files with tiles, and a text indexfile z/x/y -> zipfile.
2+
3+
Expects to find in /tmp files named tile_<theme>_z_x_y.png where:
4+
- <theme> is an all-uppercase "theme name" string
5+
- z, x, y are integers (zoom level and x/y Google tile coordinates)
6+
Writes, also in /tmp:
7+
- zipfiles named <theme>_<N>.zip for increasing integers N, each zip <1MB
8+
- a sqlite DB named <theme>_tiles.sdb with one table TILE_TO_ZIP with
9+
string z_x_y as key and N as value (when tile_<theme>_z_x_y is in
10+
zipfile <theme>_N.zip)
11+
Principles of operation:
12+
- build zipfiles sequentially (sorting filenames numerically theme-z-x-y)
13+
- keep track of the total (compressed) size of the current zipfile
14+
- ensure <1MB by checking that the next tile-file would fit UNcompressed (!),
15+
else close the current zipfile and open a fresh one for the next tile +
16+
"""
17+
import glob
18+
import logging
19+
import os
20+
import sqlite3
21+
import sys
22+
import zipfile
23+
24+
# use a prudent size as we also need space for the zipfile directory &c
25+
MAX_SIZE = 1000*1000 - 50*1000
26+
27+
def setlogging(dodebug=False):
28+
logging.basicConfig(format='%(levelname)s: %(message)s')
29+
logger = logging.getLogger()
30+
logger.setLevel(logging.DEBUG if dodebug else logging.INFO)
31+
return logger
32+
33+
def namekey(filename):
34+
try: tile, theme, z, x, y = filename[:-4].split('_')
35+
except ValueError, e:
36+
print "Can't unpack %r: %s" % (filename, e)
37+
sys.exit(1)
38+
assert tile=='tile'
39+
return theme, int(z), int(x), int(y)
40+
41+
def main(working_directory='/tmp'):
42+
setlogging(dodebug=True)
43+
44+
os.chdir(working_directory)
45+
filenames = sorted(glob.iglob('tile_*.png'), key=namekey)
46+
theme, z, x, y = namekey(filenames[0])
47+
zxy_start = len('tile_%s_' % theme)
48+
dbname = '%s_tiles.sdb' % theme
49+
logging.info('Processing %d files, creating DB %r', len(filenames), dbname)
50+
conn = sqlite3.connect(dbname)
51+
conn.isolation_level = None
52+
conn.execute("""CREATE TABLE IF NOT EXISTS tile_to_zip
53+
(z_x_y STRING PRIMARY KEY, n INTEGER)
54+
""")
55+
zipnum = 0
56+
fns = iter(filenames)
57+
fn = fns.next()
58+
while True:
59+
zipnum += 1
60+
zipfna = '%s_%s.zip' % (theme, zipnum)
61+
logging.debug('Creating zipfile %r', zipfna)
62+
num_tiles_in_zip = 0
63+
zipfil = zipfile.ZipFile(zipfna, 'w', zipfile.ZIP_DEFLATED)
64+
zipsiz = 0
65+
while True:
66+
zipfil.write(fn)
67+
num_tiles_in_zip += 1
68+
z_x_y = fn[zxy_start:-4]
69+
conn.execute("INSERT INTO tile_to_zip VALUES(?,?)", (z_x_y, zipnum))
70+
zipinfo = zipfil.getinfo(fn)
71+
zipsiz += zipinfo.compress_size
72+
try: fn = fns.next()
73+
except StopIteration:
74+
fn = None
75+
break
76+
filsiz = os.stat(fn).st_size
77+
if filsiz > MAX_SIZE:
78+
logging.error("Can never pack %r, size %s: terminating!", fn, filsiz)
79+
fn = None
80+
if zipsiz + filsiz > MAX_SIZE:
81+
break
82+
zipfil.close()
83+
logging.debug('%d tiles in zip, next tile %s', num_tiles_in_zip, fn)
84+
if fn is None:
85+
break
86+
conn.close()
87+
88+
main()
89+

‎tiles10.zip

-2.67 MB
Binary file not shown.

‎tiles_9.zip

-1.25 MB
Binary file not shown.

0 commit comments

Comments
 (0)