Node Graph Database

node_graph_db.c

/* ======================================================================================
 * VISGRID — High-Performance Spatial Indexing Engine
 * ======================================================================================
 * 
 * DESCRIPTION:
 *   Visgrid is a specialized, embedded-grade spatial database designed for highly 
 *   interactive node graphs, infinite canvas UI, and real-time 2D simulations.
 *   It implements a "Linearized Implicit Quadtree" using Z-Order curves (Morton Codes)
 *   backed by a memory-mapped B-Tree (LMDB).
 *
 *   Unlike traditional pointer-based Quadtrees, Visgrid stores spatial relationships
 *   purely through integer arithmetic. This results in O(1) traversal complexity for
 *   neighbor finding and ensures perfect cache locality.
 *
 * KEY FEATURES:
 *   1. Zero-Copy Persistence:
 *      Data is written directly to the OS page cache via MDB_RESERVE. No mallocs,
 *      no intermediate buffers, and zero serialization overhead.
 *
 *   2. Automatic "Visual" LOD:
 *      The query engine is density-aware. If a viewport zoom level results in 
 *      sub-pixel object density (>128 tiles/screen), fine-grained levels are 
 *      mathematically skipped. This guarantees stable frame rates regardless of 
 *      world size or zoom level.
 *
 *   3. Nearest-Neighbor (KNN) Sorting:
 *      Queries are not just area overlaps; they imply a "Closest to Center" sort.
 *      The engine maintains a Bounded Max-Heap during traversal to retain only the 
 *      K-nearest objects to the viewport center. Results are returned strictly 
 *      sorted from Closest -> Furthest.
 *
 *   4. Hardware Accelerated:
 *      Uses BMI2 (PDEP) instructions on x86 and CLZ/BSR bit-scanning for O(1) 
 *      level calculation. Fallbacks provided for generic architectures.
 *
 *   5. Crash-Proof Robustness:
 *      - Endian-safe: Database files are portable between x86 (LE) and MIPS (BE).
 *      - Alignment-safe: Handles unaligned memory-mapped IO on ARM/Apple Silicon.
 *      - Padding-safe: Structs are statically asserted to prevent uninitialized garbage.
 *
 * THEORY OF OPERATION:
 *   Objects are inserted into one of 16 hierarchical levels based on their size.
 *   Within a level, objects are bucketed into grid cells using Morton encoding.
 *   The final database key is a composite 64-bit integer: [Level : MortonCode].
 *   This clusters spatially adjacent objects of similar size onto the same memory pages.
 *
 * CONSTRAINTS & LIMITS:
 *   - Coordinate Space: Signed 32-bit Integers (±2,147,483,648).
 *   - Object ID:        128-bit UUID (via __uint128_t or struct).
 *   - Max Hierarchy:    16 Levels (Level 0 cell = 256 units, Level 15 = Global).
 *   - Storage Limit:    Default 2GB memory map (configurable via DB_MAP_SIZE).
 *   - Query Limit:      Hard cap of 16,384 items per query (DB_QRY_LIMIT).
 *                       (Pre-allocated buffer to prevent runtime mallocs during queries).
 *   - Concurrency:      Single-Writer / Multi-Reader (Thread-safe via Context).
 *
 * ======================================================================================
 */
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <math.h>
#include <assert.h>
#include <lmdb.h>

#ifdef _MSC_VER
  #include <intrin.h>
  #pragma intrinsic(_BitScanReverse)
#endif

/* --- Configuration --- */
#define MAX_LVLS        16          
#define MIN_CELL_SHIFT  8           // lvl 0 Cell = 256 width
#define MIN_CELL_SIZE   (1 << MIN_CELL_SHIFT)
#define DB_MAP_SIZE     (2ULL * 1024 * 1024 * 1024) // 2GB Map
#define BATCH_LIMIT     1000        
#define DB_QRY_LIMIT    (8*1024)   // Max items to return in a query
#define COORD_OFFSET    (2147483648ULL) 
/* LOD THRESHOLD:
 * If a viewport spans more than this many cells of a specific lvl,
 * that lvl is considered "too detailed" (sub-pixel) and is skipped.
 * Lower = More aggressive optimization (objects disappear sooner).
 * Higher = More detail preserved when zoomed out.
 */
#define LOD_TILE_LIMIT  128

typedef __uint128_t objid_t; 
struct spatial_row {
  objid_t id;          // 16 bytes
  int minx, maxx;  // 8 bytes
  int miny, maxy;  // 8 bytes
}; // Total 32 bytes
_Static_assert(sizeof(struct spatial_row) == 32, "struct spatial_row must be exactly 32 bytes");

struct index_row {
  uint64_t spatial_key;
  struct spatial_row row;
};
struct heap_elm {
  objid_t id;
  double dist_sq;
};
struct db_ctx {
  MDB_env *env;
  MDB_dbi spatial_dbi;
  MDB_dbi index_dbi;
  MDB_txn *batch_txn;
  int batch_cnt;
  struct heap_elm qry_heap[DB_QRY_LIMIT]; 
};
static inline uint64_t
morton_encode(uint32_t x, uint32_t y) {
#if defined(ARCH_X86) && defined(__BMI2__)
  return _pdep_u64(x, 0x5555555555555555ULL) | 
         (_pdep_u64(y, 0x5555555555555555ULL) << 1);
#else
  uint64_t sx = x, sy = y;
  sx = (sx | (sx << 16)) & 0x0000FFFF0000FFFFULL;
  sx = (sx | (sx << 8))  & 0x00FF00FF00FF00FFULL;
  sx = (sx | (sx << 4))  & 0x0F0F0F0F0F0F0F0FULL;
  sx = (sx | (sx << 2))  & 0x3333333333333333ULL;
  sx = (sx | (sx << 1))  & 0x5555555555555555ULL;
  
  sy = (sy | (sy << 16)) & 0x0000FFFF0000FFFFULL;
  sy = (sy | (sy << 8))  & 0x00FF00FF00FF00FFULL;
  sy = (sy | (sy << 4))  & 0x0F0F0F0F0F0F0F0FULL;
  sy = (sy | (sy << 2))  & 0x3333333333333333ULL;
  sy = (sy | (sy << 1))  & 0x5555555555555555ULL;
  return sx | (sy << 1);
#endif
}
static inline int
get_size_lvl(int w, int h) {
  unsigned max_dim = (unsigned)((w > h) ? w : h);
  if (max_dim <= MIN_CELL_SIZE) {
    return 0;
  }
  unsigned v = max_dim - 1u;
  unsigned int msb;
#if defined(_MSC_VER)
  unsigned long index;
  _BitScanReverse(&index, v);
  msb = (unsigned int)index + 1u;
#elif defined(__GNUC__) || defined(__clang__)
  msb = 32u - __builtin_clz(v);
#else
  msb = 0;
  while (v >>= 1) msb++;
  msb++;
#endif
  int lvl = (int)(msb - MIN_CELL_SHIFT);
  return (lvl < 0) ? 0 : (lvl >= MAX_LVLS) ? MAX_LVLS - 1 : lvl;
}
static int
cmp_uint128(const MDB_val *a, const MDB_val *b) {
  unsigned long long va[2], vb[2];
  memcpy(va, a->mv_data, 16);
  memcpy(vb, b->mv_data, 16);
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
  unsigned long long ha = va[1], la = va[0];
  unsigned long long hb = vb[1], lb = vb[0];
#else
  unsigned long long ha = va[0], la = va[1];
  unsigned long long hb = vb[0], lb = vb[1];
#endif
  if (ha != hb) {
    return (ha > hb) - (ha < hb);
  }
  return (la > lb) - (la < lb);
}
static inline unsigned
world_to_grid(int val, int lvl) {
  unsigned long long shifted = (long long)val + COORD_OFFSET; 
  return (unsigned)(shifted >> (MIN_CELL_SHIFT + lvl));
}
static unsigned long long
get_center_key(int x, int y, int w, int h) {
  int lvl = get_size_lvl(w, h);
  int cx = x + (w >> 1);
  int cy = y + (h >> 1);
  unsigned gx = world_to_grid(cx, lvl);
  unsigned gy = world_to_grid(cy, lvl);
  unsigned long long m_code = morton_encode(gx, gy);
  return ((unsigned long long)lvl << 56) | m_code;
}
static struct db_ctx*
db_init(const char *path) {
  struct db_ctx *ctx = calloc(1, sizeof(struct db_ctx));
  mdb_env_create(&ctx->env);
  mdb_env_set_maxdbs(ctx->env, 2);
  mdb_env_set_mapsize(ctx->env, DB_MAP_SIZE);
  mdb_env_open(ctx->env, path, MDB_NOTLS | MDB_NOSUBDIR, 0664);

  MDB_txn *txn;
  mdb_txn_begin(ctx->env, NULL, 0, &txn);
  mdb_dbi_open(txn, "spatial", MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &ctx->spatial_dbi);
  mdb_dbi_open(txn, "index", MDB_CREATE, &ctx->index_dbi);
  mdb_set_compare(txn, ctx->index_dbi, cmp_uint128);
  mdb_txn_commit(txn);
  return ctx;
}
static void
db_close(struct db_ctx *ctx) {
  if (ctx->batch_txn) {
    mdb_txn_commit(ctx->batch_txn);
  }
  mdb_dbi_close(ctx->env, ctx->spatial_dbi);
  mdb_dbi_close(ctx->env, ctx->index_dbi);
  mdb_env_close(ctx->env);
  free(ctx);
}
static MDB_txn*
get_write_txn(struct db_ctx *ctx, int *is_local) {
  if (ctx->batch_txn) {
    *is_local = 0;
    return ctx->batch_txn;
  }
  MDB_txn *txn;
  mdb_txn_begin(ctx->env, 0, 0, &txn);
  *is_local = 1;
  return txn;
}
static void
check_batch(struct db_ctx *ctx) {
  if (ctx->batch_txn) {
    ctx->batch_cnt++;
    if (ctx->batch_cnt >= BATCH_LIMIT) {
      mdb_txn_commit(ctx->batch_txn);
      mdb_txn_begin(ctx->env, 0, 0, &ctx->batch_txn);
      ctx->batch_cnt = 0;
    }
  }
}
static void
db_batch_begin(struct db_ctx *ctx) {
  if (ctx->batch_txn) {
    return;
  }
  mdb_txn_begin(ctx->env, 0, 0, &ctx->batch_txn);
  ctx->batch_cnt = 0;
}
static void
db_batch_commit(struct db_ctx *ctx) {
  if (ctx->batch_txn) {
    mdb_txn_commit(ctx->batch_txn);
    ctx->batch_txn = 0;
  }
}
static void
db_batch_abort(struct db_ctx *ctx) {
  if (ctx->batch_txn) {
    mdb_txn_abort(ctx->batch_txn);
    ctx->batch_txn = 0;
  }
}
static void
db_upsert(struct db_ctx *ctx, objid_t id, int x, int y, int w, int h) {
  int local = 0;
  MDB_txn *txn = get_write_txn(ctx, &local);
  MDB_val k_id = { sizeof(objid_t), &id };
  MDB_val v_idx;

  // 1. Lookup & Cleanup Old Position
  if (mdb_get(txn, ctx->index_dbi, &k_id, &v_idx) == 0) {
    struct index_row *old_rec = (struct index_row*)v_idx.mv_data;
    MDB_val k_sp = { sizeof(uint64_t), &old_rec->spatial_key };
    MDB_val v_sp = { sizeof(struct spatial_row), &old_rec->row };
    mdb_del(txn, ctx->spatial_dbi, &k_sp, &v_sp);
  }
  // 2. Prepare New Record
  uint64_t new_sp_key = get_center_key(x, y, w, h);

  struct spatial_row sp_row;
  memset(&sp_row, 0, sizeof(struct spatial_row)); 
  sp_row.id = id;
  sp_row.minx = x;
  sp_row.miny = y;
  sp_row.maxx = x + w;
  sp_row.maxy = y + h;

  struct index_row idx_row;
  idx_row.spatial_key = new_sp_key;
  idx_row.row = sp_row;

  // 3. Zero-Copy Insert (Spatial)
  MDB_val k_sp_new = { sizeof(uint64_t), &new_sp_key };
  MDB_val v_sp_new = { sizeof(struct spatial_row), NULL };
  
  // MDB_RESERVE writes directly to DB map -> Fastest possible path
  if (mdb_put(txn, ctx->spatial_dbi, &k_sp_new, &v_sp_new, MDB_RESERVE) == 0) {
    memcpy(v_sp_new.mv_data, &sp_row, sizeof(struct spatial_row));
  }
  // 4. Update Index
  MDB_val v_idx_new = { sizeof(struct index_row), &idx_row };
  mdb_put(txn, ctx->index_dbi, &k_id, &v_idx_new, 0);
  if (local) {
    mdb_txn_commit(txn);
  } else {
    check_batch(ctx);
  }
}
static void
db_obj_del(struct db_ctx *ctx, objid_t id) {
  int local = 0;
  MDB_txn *txn = get_write_txn(ctx, &local);
  MDB_val k_id = { sizeof(objid_t), &id };
  MDB_val v_idx;

  if (mdb_get(txn, ctx->index_dbi, &k_id, &v_idx) == 0) {
    struct index_row *rec = (struct index_row*)v_idx.mv_data;
    MDB_val k_sp = { sizeof(unsigned long long), &rec->spatial_key };
    MDB_val v_sp = { sizeof(struct spatial_row), &rec->row };
    
    mdb_del(txn, ctx->spatial_dbi, &k_sp, &v_sp);
    mdb_del(txn, ctx->index_dbi, &k_id, 0);
  }
  if (local) {
    mdb_txn_commit(txn);
  } else {
    check_batch(ctx);
  }
}
static void
heap_push(struct heap_elm *h, int *n, struct heap_elm val) {
  int i = (*n)++;
  h[i] = val;
  while (i > 0) {
    int p = (i - 1) >> 1;
    if (h[p].dist_sq >= h[i].dist_sq) break;
    struct heap_elm t = h[i]; h[i] = h[p]; h[p] = t;
    i = p;
  }
}
static inline void
heap_replace(struct heap_elm *h, int n, struct heap_elm val) {
  /* 
   * OPTIMIZATION: Branchless Child Selection 
   * Eliminates branch mispredictions during the "Bubble Down" phase.
   */
  // "Half" optimization: Don't write 'val' to memory in every loop. 
  // Just find the hole, write once at the end.
  int i = 0;
  while (1) {
    int l = (i << 1) + 1;
    int r = (i << 1) + 2;
    int swap_idx = i;
    // Find largest child
    if (l < n && h[l].dist_sq > h[swap_idx].dist_sq) swap_idx = l;
    if (r < n && h[r].dist_sq > h[swap_idx].dist_sq) swap_idx = r;
    if (swap_idx == i) break; 
    h[i] = h[swap_idx]; // Move child up
    i = swap_idx;       // Move index down
  }
  h[i] = val; // Final write
}
static void
heap_sort_results(struct heap_elm *h, int n) {
  /*
   * OPTIMIZATION: Final Sort
   * The query collects a Max-Heap (Index 0 is furthest).
   * Users usually want [Closest .... Furthest].
   * We use a standard Heap Sort to reverse it in-place.
   */
  // Pop elements one by one from the heap
  // This naturally moves the largest items to the end of the array.
  // Result: [Smallest, ..., Largest] (Exactly what we want)
  int original_count = n;
  while (n > 1) {
    struct heap_elm max_val = h[0];
    struct heap_elm last_val = h[n - 1];
    n--;
    // Move Max to end
    h[n] = max_val;
    // Restore heap property for the remaining
    heap_replace(h, n, last_val);
  }
}
static int
db_qry(struct db_ctx *ctx, int vx, int vy, int vw, int vh,
       objid_t *out_ids, int max_count) {
  
  int heap_cnt = 0;
  double cx = vx + vw * 0.5;
  double cy = vy + vh * 0.5;

  MDB_txn *txn;
  mdb_txn_begin(ctx->env, 0, MDB_RDONLY, &txn);
  MDB_cursor *cur;
  mdb_cursor_open(txn, ctx->spatial_dbi, &cur);

  // Pre-calculate squared cutoff to avoid SQRT or checks if heap is full
  double max_dist_sq = 1e300; // Infinity
  for (int lvl = MAX_LVLS - 1; lvl >= 0; lvl--) {
    int cell_size = MIN_CELL_SIZE << lvl;
    int half_cell = cell_size >> 1;

    int search_min_x = vx - half_cell;
    int search_min_y = vy - half_cell;
    int search_max_x = vx + vw + half_cell; 
    int search_max_y = vy + vh + half_cell;

    unsigned t_min_x = world_to_grid(search_min_x, lvl);
    unsigned t_min_y = world_to_grid(search_min_y, lvl);
    unsigned t_max_x = world_to_grid(search_max_x, lvl);
    unsigned t_max_y = world_to_grid(search_max_y, lvl);

    /* Granular LOD Skip */
    unsigned span_x = t_max_x - t_min_x;
    unsigned span_y = t_max_y - t_min_y;
    if (span_x > LOD_TILE_LIMIT || span_y > LOD_TILE_LIMIT) {
      break;
    }
    for (unsigned ty = t_min_y; ty <= t_max_y; ty++) {
      for (unsigned tx = t_min_x; tx <= t_max_x; tx++) {
        unsigned long long m_code = morton_encode(tx, ty);
        unsigned long long key = ((unsigned long long)lvl << 56) | m_code;
        MDB_val v, k = { sizeof(key), &key };
        if (mdb_cursor_get(cur, &k, &v, MDB_SET) == 0) {
          do {
            struct spatial_row obj;
            memcpy(&obj, v.mv_data, sizeof(struct spatial_row));
            // 1. AABB Intersection (Integer math only)
            if (obj.maxx >= vx && obj.minx <= vx + vw &&
                obj.maxy >= vy && obj.miny <= vy + vh) {
              
              // 2. Distance Calculation (Floating point)
              double ocx = (obj.minx + obj.maxx) * 0.5;
              double ocy = (obj.miny + obj.maxy) * 0.5;
              double dist_sq = (ocx - cx)*(ocx - cx) + (ocy - cy)*(ocy - cy);

              // 3. Heap Logic
              if (heap_cnt < max_count) {
                heap_push(ctx->qry_heap, &heap_cnt, (struct heap_elm){obj.id, dist_sq});
                // Track max distance if heap just got full
                if (heap_cnt == max_count) {
                   max_dist_sq = ctx->qry_heap[0].dist_sq; 
                }
              }  else if (dist_sq < max_dist_sq) {
                // Optimization: Check dist against cached max before calling function
                heap_replace(ctx->qry_heap, heap_cnt, (struct heap_elm){obj.id, dist_sq});
                // Update cached max distance (Root is always largest)
                max_dist_sq = ctx->qry_heap[0].dist_sq;
              }
            }
          } while (mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP) == 0);
        }
      }
    }
  }
  mdb_cursor_close(cur);
  mdb_txn_abort(txn);
  // Sort the result
  // Turns [Furthest ... Random] into [Closest ... Furthest]
  heap_sort_results(ctx->qry_heap, heap_cnt);
  for (int i = 0; i < heap_cnt; i++) {
    out_ids[i] = ctx->qry_heap[i].id;
  }
  return heap_cnt;
}