//- http get into memory typedef struct HttpBuffer HttpBuffer; struct HttpBuffer { Arena *arena; u8 *data; u64 size; u64 capacity; }; internal size_t http_buffer_write(char *pointer, size_t size, size_t count, void *user) { HttpBuffer *buffer = (HttpBuffer *)user; u64 bytes = (u64)size * count; if(buffer->size + bytes > buffer->capacity) { u64 new_capacity = Max(buffer->capacity * 2, buffer->size + bytes); buffer->data = (u8 *)ArenaResize(buffer->arena, buffer->data, new_capacity, buffer->capacity); buffer->capacity = new_capacity; } MemCpy(buffer->data + buffer->size, pointer, bytes); buffer->size += bytes; return bytes; } internal b32 http_get(Arena *arena, const char *url, String8 *out) { CURL *curl = curl_easy_init(); if(!curl) { return false; } HttpBuffer buffer = {}; buffer.arena = arena; buffer.capacity = KiB(128); buffer.data = PushArray(arena, u8, buffer.capacity); curl_easy_setopt(curl, CURLOPT_URL, url); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, http_buffer_write); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 10L); curl_easy_setopt(curl, CURLOPT_TIMEOUT, 30L); curl_easy_setopt(curl, CURLOPT_USERAGENT, "mp3_player/1.0"); curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, ""); // gzip the json CURLcode result = curl_easy_perform(curl); curl_easy_cleanup(curl); if(result != CURLE_OK) { Log("http get %s failed: %s", url, curl_easy_strerror(result)); return false; } *out = ToString8Size(buffer.data, buffer.size); return true; } //- catalog internal String8 catalog_section_name(u32 section_id) { switch(section_id) { case 1: return str8("Recitations"); case 2: return str8("Haramain Taraweeh"); case 3: return str8("Non-Hafs"); case 4: return str8("With Translation"); } return str8("Other"); } //- json (network only) // NOTE: quranicaudio.com's api only speaks json, so json is parsed exactly once: right after a // download. the result is written as catalog.qcat and every later start reads that instead. internal b32 qari_less(Qari *a, Qari *b) { if(a->section_id != b->section_id) { return a->section_id < b->section_id; } u64 count = Min(a->name.size, b->name.size); for(u64 index = 0; index < count; ++index) { u8 ca = a->name.data[index], cb = b->name.data[index]; if(ca >= 'A' && ca <= 'Z') ca += 32; if(cb >= 'A' && cb <= 'Z') cb += 32; if(ca != cb) { return ca < cb; } } return a->name.size < b->name.size; } internal b32 catalog_parse_json(Arena *arena, Catalog *catalog, String8 qaris_text, String8 surahs_text) { MemSetZero(catalog, sizeof(*catalog)); //- surahs JsonValue *surahs = json_parse(arena, surahs_text); if(!surahs || surahs->kind != JSON_ARRAY) { Log("surahs json is malformed"); return false; } FOR_JSON_CHILD(surahs, item) { u32 number = (u32)json_get_number(item, "id"); if(number < 1 || number > SURAH_COUNT) { continue; } Surah *surah = &catalog->surahs[number - 1]; JsonValue *name = json_get(item, "name"); surah->number = number; surah->ayah_count = (u32)json_get_number(item, "ayah"); surah->name = json_get_string(name, "simple"); surah->english = json_get_string(name, "english"); surah->makki = str8_is_match(json_get_string(item, "revelation_place"), str8("makkah")); } //- qaris JsonValue *qaris = json_parse(arena, qaris_text); if(!qaris || qaris->kind != JSON_ARRAY) { Log("qaris json is malformed"); return false; } catalog->qaris = PushArrayZero(arena, Qari, qaris->count); FOR_JSON_CHILD(qaris, item) { String8 relative_path = json_get_string(item, "relative_path"); String8 formats = json_get_string(item, "file_formats"); if(relative_path.size == 0 || !str8_is_match(formats, str8("mp3"))) { continue; } Qari *qari = &catalog->qaris[catalog->qari_count++]; qari->id = (u32)json_get_number(item, "id"); qari->section_id = (u32)json_get_number(item, "section_id"); qari->name = json_get_string(item, "name"); qari->relative_path = relative_path; } // NOTE: sorted once here, the binary file stores the sorted order so loading never sorts for(u32 index = 1; index < catalog->qari_count; ++index) { Qari key = catalog->qaris[index]; u32 slot = index; while(slot > 0 && qari_less(&key, &catalog->qaris[slot - 1])) { catalog->qaris[slot] = catalog->qaris[slot - 1]; --slot; } catalog->qaris[slot] = key; } return catalog->qari_count > 0; } //- qcat: the on disk catalog // // CatalogHeader 32 bytes // CatalogQariRecord [qari_count] 24 bytes each // CatalogSurahRecord [surah_count] 32 bytes each // strings utf8, no terminators, records point in by offset/size // // every field is a little endian u32 and every section starts 8 byte aligned, so a loaded file // is used in place: records are read straight from the buffer, names are String8s pointing into // the string blob. loading = bounds checks + one hardware crc32c pass, no parsing, no copies. static_assert(__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__, "qcat is read in place, little endian only"); static_assert(sizeof(CatalogHeader) == 32 && sizeof(CatalogQariRecord) == 24 && sizeof(CatalogSurahRecord) == 32, "qcat layout"); // NOTE: crc32c. arm64 and x64 both have an instruction that eats 8 bytes at a time, so checking // the whole file costs about a microsecond. software fallback for anything else. #if ARCH_ARM #include __attribute__((target("+crc"))) internal u32 catalog_checksum(const u8 *data, u64 size) { u32 crc = 0xFFFFFFFFu; u64 at = 0; for(; at + 8 <= size; at += 8) { u64 word; MemCpy(&word, data + at, 8); crc = __crc32cd(crc, word); } for(; at < size; ++at) crc = __crc32cb(crc, data[at]); return ~crc; } #elif ARCH_x86_64 #include __attribute__((target("sse4.2"))) internal u32 catalog_checksum(const u8 *data, u64 size) { u64 crc = 0xFFFFFFFFu; u64 at = 0; for(; at + 8 <= size; at += 8) { u64 word; MemCpy(&word, data + at, 8); crc = _mm_crc32_u64(crc, word); } for(; at < size; ++at) crc = _mm_crc32_u8((u32)crc, data[at]); return ~(u32)crc; } #else internal u32 catalog_checksum(const u8 *data, u64 size) { u32 crc = 0xFFFFFFFFu; for(u64 at = 0; at < size; ++at) { crc ^= data[at]; for(u32 bit = 0; bit < 8; ++bit) crc = (crc >> 1) ^ (0x82F63B78u & (0u - (crc & 1))); } return ~crc; } #endif internal String8 catalog_to_binary(Arena *arena, Catalog *catalog) { u64 strings = 0; for(u32 index = 0; index < catalog->qari_count; ++index) { strings += catalog->qaris[index].name.size + catalog->qaris[index].relative_path.size; } for(u32 index = 0; index < SURAH_COUNT; ++index) { strings += catalog->surahs[index].name.size + catalog->surahs[index].english.size; } u64 records = sizeof(CatalogQariRecord) * catalog->qari_count + sizeof(CatalogSurahRecord) * SURAH_COUNT; u64 size = sizeof(CatalogHeader) + records + strings; u8 *data = PushArrayZero(arena, u8, size); CatalogHeader *header = (CatalogHeader *)data; CatalogQariRecord *qaris = (CatalogQariRecord *)(header + 1); CatalogSurahRecord *surahs = (CatalogSurahRecord *)(qaris + catalog->qari_count); u8 *blob = (u8 *)(surahs + SURAH_COUNT); u32 cursor = 0; #define CatalogPutString(offset_field, size_field, string) \ do { \ (offset_field) = cursor; \ (size_field) = (u32)(string).size; \ MemCpy(blob + cursor, (string).data, (string).size); \ cursor += (u32)(string).size; \ } while(0) for(u32 index = 0; index < catalog->qari_count; ++index) { Qari *qari = &catalog->qaris[index]; qaris[index].id = qari->id; qaris[index].section_id = qari->section_id; CatalogPutString(qaris[index].name_offset, qaris[index].name_size, qari->name); CatalogPutString(qaris[index].path_offset, qaris[index].path_size, qari->relative_path); } for(u32 index = 0; index < SURAH_COUNT; ++index) { Surah *surah = &catalog->surahs[index]; surahs[index].number = surah->number; surahs[index].ayah_count = surah->ayah_count; surahs[index].makki = surah->makki ? 1 : 0; CatalogPutString(surahs[index].name_offset, surahs[index].name_size, surah->name); CatalogPutString(surahs[index].english_offset, surahs[index].english_size, surah->english); } #undef CatalogPutString header->magic = CATALOG_MAGIC; header->version = CATALOG_VERSION; header->qari_count = catalog->qari_count; header->surah_count = SURAH_COUNT; header->strings_size = cursor; header->checksum = catalog_checksum(data + sizeof(CatalogHeader), size - sizeof(CatalogHeader)); return ToString8Size(data, size); } // NOTE: `data` must outlive the catalog: the names point into it internal b32 catalog_from_binary(Arena *arena, Catalog *catalog, String8 data) { MemSetZero(catalog, sizeof(*catalog)); if(data.size < sizeof(CatalogHeader) || ((umm)data.data & 7) != 0) { return false; } CatalogHeader *header = (CatalogHeader *)data.data; if(header->magic != CATALOG_MAGIC || header->version != CATALOG_VERSION || header->surah_count != SURAH_COUNT || header->qari_count == 0 || header->qari_count > 4096) { return false; } u64 records = sizeof(CatalogQariRecord) * (u64)header->qari_count + sizeof(CatalogSurahRecord) * SURAH_COUNT; if(data.size != sizeof(CatalogHeader) + records + header->strings_size) { return false; } if(catalog_checksum(data.data + sizeof(CatalogHeader), data.size - sizeof(CatalogHeader)) != header->checksum) { Log("catalog.qcat checksum mismatch"); return false; } CatalogQariRecord *qaris = (CatalogQariRecord *)(header + 1); CatalogSurahRecord *surahs = (CatalogSurahRecord *)(qaris + header->qari_count); u8 *blob = (u8 *)(surahs + SURAH_COUNT); u64 limit = header->strings_size; #define CatalogString(offset, size) (((u64)(offset) + (size) <= limit) ? ToString8Size(blob + (offset), (size)) : str8_zero()) catalog->qaris = PushArray(arena, Qari, header->qari_count); for(u32 index = 0; index < header->qari_count; ++index) { CatalogQariRecord *record = &qaris[index]; Qari *qari = &catalog->qaris[catalog->qari_count]; qari->id = record->id; qari->section_id = record->section_id; qari->name = CatalogString(record->name_offset, record->name_size); qari->relative_path = CatalogString(record->path_offset, record->path_size); if(qari->relative_path.size) ++catalog->qari_count; } for(u32 index = 0; index < SURAH_COUNT; ++index) { CatalogSurahRecord *record = &surahs[index]; Surah *surah = &catalog->surahs[index]; surah->number = record->number; surah->ayah_count = record->ayah_count; surah->makki = record->makki != 0; surah->name = CatalogString(record->name_offset, record->name_size); surah->english = CatalogString(record->english_offset, record->english_size); } #undef CatalogString return catalog->qari_count > 0; } //- loading // NOTE: compile.sh embeds data/catalog.qcat (xxd -i), so even the very first start works offline #include "catalog_seed.h" internal String8 catalog_cache_path(Arena *arena, String8 data_dir, const char *name) { u64 size = data_dir.size + 1 + cstring8_length((u8 *)name) + 1; u8 *buffer = PushArray(arena, u8, size); int length = snprintf((char *)buffer, size, Str8Fmt "/%s", Str8Arg(data_dir), name); return ToString8Size(buffer, (u64)length); } internal void catalog_save(Arena *arena, Catalog *catalog, String8 data_dir) { String8 path = catalog_cache_path(arena, data_dir, "catalog.qcat"); String8 temp = catalog_cache_path(arena, data_dir, "catalog.qcat.tmp"); String8 data = catalog_to_binary(arena, catalog); // NOTE: write then rename, a crash mid write never leaves a broken catalog behind if(FileWrite(data, temp)) { rename((char *)temp.data, (char *)path.data); } } internal b32 catalog_fetch(Arena *arena, Catalog *catalog) { String8 qaris_text = {}; String8 surahs_text = {}; Log("fetching the catalog from quranicaudio.com"); return http_get(arena, CATALOG_API_QARIS, &qaris_text) && http_get(arena, CATALOG_API_SURAHS, &surahs_text) && catalog_parse_json(arena, catalog, qaris_text, surahs_text); } internal b32 catalog_load(Arena *arena, Catalog *catalog, String8 data_dir, b32 refresh) { u64 start = time_now_ns(); String8 path = catalog_cache_path(arena, data_dir, "catalog.qcat"); //- 1. the cached binary catalog, the normal case if(!refresh && FileExists(arena, path) && catalog_from_binary(arena, catalog, FileLoad(arena, path))) { Log("catalog: %u qaris from catalog.qcat (%.3f ms)", catalog->qari_count, (f64)(time_now_ns() - start) / 1e6); return true; } //- 2. caches from older versions were json: convert once, then they're gone String8 old_qaris = catalog_cache_path(arena, data_dir, "qaris.json"); String8 old_surahs = catalog_cache_path(arena, data_dir, "surahs.json"); if(!refresh && FileExists(arena, old_qaris) && FileExists(arena, old_surahs) && catalog_parse_json(arena, catalog, FileLoad(arena, old_qaris), FileLoad(arena, old_surahs))) { catalog_save(arena, catalog, data_dir); unlink((char *)old_qaris.data); unlink((char *)old_surahs.data); Log("catalog: converted the old json cache to catalog.qcat"); return true; } //- 3. the network if(catalog_fetch(arena, catalog)) { catalog_save(arena, catalog, data_dir); Log("catalog: %u qaris from the network, saved as catalog.qcat", catalog->qari_count); return true; } //- 4. offline: a stale cache (when refreshing), then the copy baked into the binary if(FileExists(arena, path) && catalog_from_binary(arena, catalog, FileLoad(arena, path))) { return true; } if(catalog_from_binary(arena, catalog, ToString8Size(catalog_seed, catalog_seed_len))) { Log("catalog: offline, using the built in catalog"); catalog_save(arena, catalog, data_dir); return true; } return false; } internal s32 catalog_find_qari_by_id(Catalog *catalog, u32 id) { for(u32 index = 0; index < catalog->qari_count; ++index) { if(catalog->qaris[index].id == id) { return (s32)index; } } return -1; }