AppEngine Example

Contributed by @hakunin, to all AppEngine lovers.

This is roughly how I use jsonpickle on AppEngine to move my data in JSON format from local machine to appengine and back.

Notice how I use low level Entity, which while not being documented is easy to work with when you want to just work with the data itself, (think migrations).

Please note that I am not using AppEngine's keys, nor am I preserving them here. They namespace of the record is encoded each key and that would prevent you from copying records from one namespace to another. I use randomly generated keys instead. That allows me to create a copy of user's data, perform migrations on them in a separate namespace and then switch user's namespace to the migrated one. (and other cool stuff, especially with client side apps that support offline mode)

from app.base_controller import *

import os

from google.appengine.api import namespace_manager
from google.appengine.api.datastore import Query
from google.appengine.api.datastore import Put
from google.appengine.api.datastore import Entity
from google.appengine.api.datastore_types import Text
from google.appengine.ext import db as Db

import json as JSON
import jsonpickle
import base64


class TextReduceHandler(jsonpickle.handlers.BaseHandler):

  def flatten(self, obj, data):
    pickler = self._base
    if not pickler.unpicklable:
      return unicode(obj)
    _, args = obj.__reduce__()
    cls = args[1]
    args = [base64.b64encode(args[2].encode('utf-8'))]
    data['__reduce__'] = (pickler.flatten(cls), args)
    return data

  def restore(self, obj):
    cls, args = obj['__reduce__']
    value = base64.b64decode(args[0]).decode('utf-8')
    unpickler = self._base
    cls = unpickler.restore(cls)
    params = map(unpickler.restore, args[1:])
    params = (value,) + tuple(params)
    return Text(params[0])

jsonpickle.handlers.registry.register(Text, TextReduceHandler)



class ImportExport:

  def export(self, namespace):
    exported = {}
    if namespace == '<ALL>':
      exported = self.export_all()
    else:
      exported[namespace] = self.export_ns(namespace)

    return jsonpickle.encode(exported)
  
  def export_all(self):
    exported = {}
    q = Db.GqlQuery("SELECT * FROM __namespace__")
    for p in q.fetch(100):
      namespace = p.namespace_name
      if namespace != '' and namespace[0] == '_': continue
      exported[namespace] = self.export_ns(namespace)

    return exported

  def export_ns(self, namespace):
    namespace_manager.set_namespace(namespace)
    namespace_kinds = {}

    for kind in Db.GqlQuery("SELECT * FROM __kind__"):
      if kind.kind_name[0] == '_': continue
      entities = Query(kind.kind_name).Run()
      records = []
      for entity in entities:
        row = {}
        for k in entity:
          row[k] = entity[k]
        records.append(row)
      namespace_kinds[kind.kind_name] = records

    return namespace_kinds


  def import_json(self, json):
    namespaces = jsonpickle.decode(json)
    for namespace, entities in namespaces.iteritems():
      if namespace == '' or namespace[0] != '_':
        namespace_manager.set_namespace(namespace)
        for entity_kind, rows in entities.iteritems():
          for row in rows:
            entity = Entity(entity_kind)
            for k in row:
              entity[k] = row[k]
            Put(entity)


  def clear_local(self):
    out = ''
    q = Db.GqlQuery("SELECT * FROM __namespace__")
    for p in q.fetch(100):
      namespace_manager.set_namespace(p.namespace_name)
      out += '\nclear namespace: '+ p.namespace_name

      for kind in Db.GqlQuery("SELECT * FROM __kind__"):
        if kind.kind_name[0] != '_':
          keys = Db.GqlQuery("SELECT __key__ FROM %s" % kind.kind_name)
          out += '\nclear kind: '+ kind.kind_name
          Db.delete(keys)