Add rooftop geocoding to a Django app

Geocoding in Django goes wrong in a predictable way: someone calls the API from Model.save(), and six months later every bulk import is timing out and nobody knows why. This is the arrangement that avoids that — a model that stores the result, a service function that is the only thing which talks to the API, and a management command for backfill.

The model

Store the quality fields, not just the point. You will want them the first time someone asks how good the data is.

from django.db import models


class Address(models.Model):
    raw = models.CharField(max_length=255)

    # Populated by the geocoder; all nullable because geocoding is a separate
    # step from creating the row.
    standardized = models.CharField(max_length=255, blank=True, default="")
    lat = models.FloatField(null=True, blank=True)
    lon = models.FloatField(null=True, blank=True)
    census_block = models.CharField(max_length=15, blank=True, default="")
    fips = models.CharField(max_length=5, blank=True, default="")
    match_code = models.CharField(max_length=64, blank=True, default="")
    location_code = models.CharField(max_length=32, blank=True, default="")
    geocoded_at = models.DateTimeField(null=True, blank=True)

    class Meta:
        indexes = [
            # The backfill query: "what still needs geocoding?"
            models.Index(fields=["geocoded_at"]),
        ]

    @property
    def is_rooftop(self):
        return self.location_code in ("ROOFTOP", "ADDRESS_POINT")

    @property
    def census_tract(self):
        """State(2) + county(3) + tract(6) of the 15-digit block GEOID."""
        return self.census_block[:11] if self.census_block else ""

census_tract as a property rather than a column is deliberate: it is a slice of a value you already store, and a denormalised copy is one more thing to keep in sync for no gain.

The service function

One module talks to the API. Everything else calls this.

# geo/services.py
import requests
from django.conf import settings
from django.utils import timezone

API = "https://weganar.com/api/v1/geocode"


class GeocodeError(Exception):
    pass


def fetch(raw_address, timeout=10):
    """Call the geocoder. Raises GeocodeError; never returns a partial result."""
    try:
        r = requests.post(
            API,
            headers={
                "X-API-Key": settings.GEOCODER_API_KEY,
                "Content-Type": "application/json",
            },
            json={"address": raw_address},
            timeout=timeout,
        )
        r.raise_for_status()
        return r.json()
    except requests.RequestException as exc:
        raise GeocodeError(str(exc)) from exc


def apply_to(address, *, save=True):
    """Geocode `address.raw` and write the result onto the instance."""
    g = fetch(address.raw)
    address.standardized = g.get("standardizedAddress") or ""
    address.lat = g.get("lat")
    address.lon = g.get("lon")
    address.census_block = g.get("censusBlock") or ""
    address.fips = g.get("fips") or ""
    address.match_code = g.get("matchCode", "")
    address.location_code = g.get("locationCode") or ""
    address.geocoded_at = timezone.now()
    if save:
        address.save(update_fields=[
            "standardized", "lat", "lon", "census_block", "fips",
            "match_code", "location_code", "geocoded_at",
        ])
    return address

update_fields matters more than it looks. Without it, saving an address you geocoded in a background job will happily overwrite any other column that changed since you loaded the row.

Do not geocode in save()

The tempting version:

# Don't.
def save(self, *args, **kwargs):
    if not self.lat:
        apply_to(self, save=False)
    super().save(*args, **kwargs)

This puts a network call with a 10-second timeout inside every write path, including bulk_create, fixtures, tests, and the admin. It also means a geocoder outage becomes an outage in your ability to create records at all — which is a much worse failure than having a row without coordinates for a minute.

Geocode after the row exists, in a job or a command.

The backfill command

# geo/management/commands/geocode_addresses.py
import time

from django.core.management.base import BaseCommand

from geo.models import Address
from geo.services import GeocodeError, apply_to


class Command(BaseCommand):
    help = "Geocode Address rows that have not been geocoded yet."

    def add_arguments(self, parser):
        parser.add_argument("--limit", type=int, default=1000)
        parser.add_argument(
            "--retry-weak",
            action="store_true",
            help="Also retry interpolated and unmatched rows.",
        )

    def handle(self, *args, **options):
        qs = Address.objects.filter(geocoded_at__isnull=True)
        if options["retry_weak"]:
            qs = Address.objects.filter(location_code__in=["SEGMENT", "PLACE"])

        done = failed = 0
        # 300 requests/minute is the default per-key ceiling; 0.2s between
        # calls keeps a single-threaded backfill just under it.
        for address in qs.order_by("pk")[: options["limit"]].iterator():
            try:
                apply_to(address)
                done += 1
            except GeocodeError as exc:
                self.stderr.write(f"{address.pk}: {exc}")
                failed += 1
            time.sleep(0.2)

        self.stdout.write(self.style.SUCCESS(f"geocoded {done}, failed {failed}"))

Run it from cron, or call it after an import. Because it selects on geocoded_at__isnull=True, it is safely re-runnable and picks up where it stopped.

Checking your data

Once a few thousand rows are through, this is the query worth looking at:

from django.db.models import Count

Address.objects.values("location_code").annotate(n=Count("id")).order_by("-n")

If SEGMENT is more than a percent or two of your rows, look at the input rather than the geocoder — it usually means unit numbers or PO boxes are being sent as street addresses. Coverage by state shows what resolution rate to expect where your addresses actually are.

Where to go next

• Cache-first geocoding — when to re-geocode, and when not to
• Geocode a large address list — concurrency and pacing for bulk jobs
• Enrich a CSV with Census demographics