//- paths internal b32 make_directories(char *path) { // NOTE: mkdir -p, modifies the buffer in place and restores it for(char *at = path + 1; *at; ++at) { if(*at == '/') { *at = 0; int result = mkdir(path, 0755); *at = '/'; if(result != 0 && errno != EEXIST) { return false; } } } return mkdir(path, 0755) == 0 || errno == EEXIST; } internal u32 store_track_path(TrackStore *store, u32 key, b32 partial, char *out, u32 out_size) { Qari *qari = &store->catalog->qaris[TrackQari(key)]; int count = snprintf(out, out_size, Str8Fmt "/" Str8Fmt "%03u.mp3%s", Str8Arg(store->audio_dir), Str8Arg(qari->relative_path), TrackSurah(key) + 1, partial ? ".part" : ""); return (u32)count; } // NOTE: every write that can enter or leave DONE goes through here, so the per qari summary is // exact. exchange, then adjust by what the state actually was. internal void store_set_state(TrackStore *store, u32 key, TrackState state) { if(key >= store->entry_count) return; u8 old = AtomicExchange(&store->states[key], (u8)state); u32 qari = TrackQari(key); if(old != TRACK_DONE && state == TRACK_DONE) { AtomicAdd(&store->qari_done[qari], 1u); AtomicAdd(&store->qari_done_bytes[qari], AtomicLoad(&store->entries[key].bytes_total)); } else if(old == TRACK_DONE && state != TRACK_DONE) { AtomicAdd(&store->qari_done[qari], (u32)-1); AtomicAdd(&store->qari_done_bytes[qari], (u64)0 - AtomicLoad(&store->entries[key].bytes_total)); } } internal TrackEntry *store_entry(TrackStore *store, u32 key) { return (key < store->entry_count) ? &store->entries[key] : NULL; } //- init and scan internal void store_init(Arena *arena, TrackStore *store, Catalog *catalog, String8 data_dir) { store->catalog = catalog; store->entry_count = catalog->qari_count * SURAH_COUNT; store->entries = PushArrayAligned(arena, TrackEntry, store->entry_count); store->states = PushArrayAligned(arena, u8, store->entry_count); store->qari_done = PushArrayAligned(arena, u32, catalog->qari_count); store->qari_done_bytes = PushArrayAligned(arena, u64, catalog->qari_count); store->qari_scanned = PushArrayZero(arena, u8, catalog->qari_count); u64 size = data_dir.size + sizeof("/audio"); u8 *buffer = PushArray(arena, u8, size); int length = snprintf((char *)buffer, size, Str8Fmt "/audio", Str8Arg(data_dir)); store->audio_dir = ToString8Size(buffer, (u64)length); make_directories((char *)buffer); // NOTE: 2^15 = 32768 > 177 * 114, every track fits in the queue at once job_queue_init(arena, &store->queue, 1u << 15); } // NOTE: 114 stat()s, called lazily the first time a qari is looked at internal void store_scan_qari(TrackStore *store, u32 qari_index) { if(qari_index >= store->catalog->qari_count || store->qari_scanned[qari_index]) { return; } store->qari_scanned[qari_index] = 1; char path[1024]; for(u32 surah = 0; surah < SURAH_COUNT; ++surah) { u32 key = TrackKey(qari_index, surah); TrackEntry *entry = &store->entries[key]; if(AtomicLoad(TrackStatePtr(store, entry)) != TRACK_REMOTE) { continue; } struct stat info; store_track_path(store, key, false, path, sizeof(path)); if(stat(path, &info) == 0) { AtomicStore(&entry->bytes_total, (u64)info.st_size); AtomicStore(&entry->bytes_done, (u64)info.st_size); store_set_state(store, key, TRACK_DONE); continue; } store_track_path(store, key, true, path, sizeof(path)); if(stat(path, &info) == 0) { AtomicStore(&entry->bytes_done, (u64)info.st_size); u8 expected = TRACK_REMOTE; AtomicCompareExchange(TrackStatePtr(store, entry), &expected, (u8)TRACK_PARTIAL); } } } // NOTE: one readdir per qari directory that exists instead of 20k stat()s. at startup this // marks everything on disk, which is what the offline view lists. internal void store_scan_disk(TrackStore *store) { Catalog *catalog = store->catalog; char path[1024]; for(u32 qari = 0; qari < catalog->qari_count; ++qari) { String8 relative = catalog->qaris[qari].relative_path; snprintf(path, sizeof(path), Str8Fmt "/" Str8Fmt, Str8Arg(store->audio_dir), Str8Arg(relative)); store->qari_scanned[qari] = 1; int dir_fd = open(path, O_RDONLY | O_DIRECTORY | O_CLOEXEC); if(dir_fd < 0) { continue; } DIR *dir = fdopendir(dir_fd); if(!dir) { close(dir_fd); continue; } struct dirent *item; while((item = readdir(dir))) { // "NNN.mp3" or "NNN.mp3.part" const char *name = item->d_name; if(!(name[0] >= '0' && name[0] <= '9' && name[1] >= '0' && name[1] <= '9' && name[2] >= '0' && name[2] <= '9')) { continue; } u32 surah = (u32)((name[0] - '0') * 100 + (name[1] - '0') * 10 + (name[2] - '0')); b32 partial = strcmp(name + 3, ".mp3.part") == 0; if(surah < 1 || surah > SURAH_COUNT || (!partial && strcmp(name + 3, ".mp3") != 0)) { continue; } struct stat info; if(fstatat(dir_fd, name, &info, 0) != 0) { continue; } TrackEntry *entry = &store->entries[TrackKey(qari, surah - 1)]; if(!partial) { AtomicStore(&entry->bytes_total, (u64)info.st_size); AtomicStore(&entry->bytes_done, (u64)info.st_size); store_set_state(store, TrackKey(qari, surah - 1), TRACK_DONE); } else if(AtomicLoad(TrackStatePtr(store, entry)) == TRACK_REMOTE) { AtomicStore(&entry->bytes_done, (u64)info.st_size); AtomicStore(TrackStatePtr(store, entry), (u32)TRACK_PARTIAL); } } closedir(dir); } } internal b32 store_on_disk(TrackStore *store, u32 key) { return key < store->entry_count && AtomicLoadRelaxed(&store->states[key]) == TRACK_DONE; } internal u32 store_disk_count(TrackStore *store, u32 qari_index) { return AtomicLoadRelaxed(&store->qari_done[qari_index]); } internal b32 store_delete(TrackStore *store, u32 key) { TrackEntry *entry = store_entry(store, key); if(!entry) { return false; } u32 state = AtomicLoad(TrackStatePtr(store, entry)); if(state == TRACK_QUEUED || state == TRACK_DOWNLOADING) { return false; // a worker owns it right now } char path[1024]; store_track_path(store, key, false, path, sizeof(path)); unlink(path); store_track_path(store, key, true, path, sizeof(path)); unlink(path); store_set_state(store, key, TRACK_REMOTE); AtomicStore(&entry->bytes_done, (u64)0); AtomicStore(&entry->bytes_total, (u64)0); signal_notify(&store->progress); return true; } //- requests internal b32 store_request(TrackStore *store, u32 key, b32 urgent) { TrackEntry *entry = store_entry(store, key); if(!entry) { return false; } store_scan_qari(store, TrackQari(key)); for(;;) { u32 state = AtomicLoad(TrackStatePtr(store, entry)); switch(state) { case TRACK_DONE: case TRACK_DOWNLOADING: { return true; } case TRACK_QUEUED: { // NOTE: already in the bulk lane. pushing it again on the urgent lane is fine, the // worker that pops the second copy sees it is no longer QUEUED and skips it. if(urgent) { job_queue_push_front(&store->queue, key); } return true; } case TRACK_REMOTE: case TRACK_PARTIAL: case TRACK_FAILED: { u8 expected = (u8)state; if(AtomicCompareExchange(TrackStatePtr(store, entry), &expected, (u8)TRACK_QUEUED)) { AtomicStoreRelaxed(&entry->retries, 0u); b32 pushed = urgent ? job_queue_push_front(&store->queue, key) : job_queue_push_back(&store->queue, key); signal_notify(&store->progress); return pushed; } // lost a race with a worker, look again } break; default: return false; } } } internal u32 store_request_qari(TrackStore *store, u32 qari_index) { u32 queued = 0; store_scan_qari(store, qari_index); for(u32 surah = 0; surah < SURAH_COUNT; ++surah) { u32 key = TrackKey(qari_index, surah); u32 state = AtomicLoad(&store->states[key]); if(state != TRACK_DONE && store_request(store, key, false)) { ++queued; } } return queued; } //- worker typedef struct DownloadContext DownloadContext; struct DownloadContext { DownloadWorker *worker; TrackEntry *entry; CURL *curl; int fd; u64 resume_from; b32 checked_headers; b32 truncated; }; internal void store_notify(TrackStore *store, DownloadWorker *worker, b32 force) { signal_notify(&store->progress); if(store->quiet) { return; } // NOTE: the decoder gets every chunk, the ui only ~15 times a second per worker u64 now = time_now_ns(); if(force || now - worker->last_wake_ns > 66000000ull) { worker->last_wake_ns = now; wake_main(); } } internal size_t store_download_write(char *pointer, size_t size, size_t count, void *user) { DownloadContext *context = (DownloadContext *)user; TrackEntry *entry = context->entry; u64 bytes = (u64)size * count; if(!context->checked_headers) { context->checked_headers = true; long status = 0; curl_easy_getinfo(context->curl, CURLINFO_RESPONSE_CODE, &status); if(status == 200 && context->resume_from > 0) { // NOTE: server ignored our range request and sent the whole file, start over if(ftruncate(context->fd, 0) != 0) { return 0; } lseek(context->fd, 0, SEEK_SET); context->resume_from = 0; AtomicStore(&entry->bytes_done, (u64)0); } curl_off_t remaining = -1; curl_easy_getinfo(context->curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &remaining); if(remaining > 0) { AtomicStore(&entry->bytes_total, context->resume_from + (u64)remaining); } } u64 written = 0; while(written < bytes) { ssize_t result = write(context->fd, pointer + written, bytes - written); if(result < 0) { if(errno == EINTR) continue; return 0; // aborts the transfer } written += (u64)result; } // NOTE: release store: the decoder reads bytes_done then pread()s up to it AtomicStore(&entry->bytes_done, AtomicLoadRelaxed(&entry->bytes_done) + bytes); AtomicAddRelaxed(&context->worker->store->bytes_downloaded, bytes); store_notify(context->worker->store, context->worker, false); return bytes; } internal void store_download(DownloadWorker *worker, CURL *curl, u32 key) { TrackStore *store = worker->store; TrackEntry *entry = &store->entries[key]; char final_path[1024]; char part_path[1024]; store_track_path(store, key, false, final_path, sizeof(final_path)); store_track_path(store, key, true, part_path, sizeof(part_path)); // NOTE: make sure