[{"data":1,"prerenderedAt":790},["ShallowReactive",2],{"content-en-async-pipeline":3},{"doc":4,"debug":787},{"id":5,"title":6,"body":7,"description":780,"extension":781,"meta":782,"navigation":521,"path":783,"seo":784,"stem":785,"__hash__":786},"content\u002Fen\u002Fasync-pipeline.md","HFT Async Architecture",{"type":8,"value":9,"toc":757},"minimark",[10,14,23,58,65,68,73,78,88,95,132,146,150,157,175,183,187,205,209,232,236,260,262,266,273,277,280,383,386,394,397,401,407,411,421,469,484,488,495,535,542,546,557,559,563,570,602,617,619,623,644,670,674,681,687,693,695,699,753],[11,12,6],"h1",{"id":13},"hft-async-architecture",[15,16,17,18,22],"p",{},"OzaLog ships two ",[19,20,21],"strong",{},"independent"," async pipelines:",[24,25,26,45],"ol",{},[27,28,29,32,33,37,38,37,41,44],"li",{},[19,30,31],{},"Main logger pipeline"," — for application logs (",[34,35,36],"code",{},"LOG.Info_Log",", ",[34,39,40],{},"Error_Log",[34,42,43],{},"CustomName_Log",", …)",[27,46,47,50,51,37,54,57],{},[19,48,49],{},"Quote pipeline"," (v3.1+) — for high-frequency market tick\u002Fquote data (",[34,52,53],{},"LOG.Quote",[34,55,56],{},"LOG.QuoteTicker",")",[15,59,60,61,64],{},"The two share ",[19,62,63],{},"no locks, no streams, and no dispatcher threads",". They differ only in default tuning (queue size, file count, batch size).",[66,67],"hr",{},[69,70,72],"h2",{"id":71},"_1-main-logger-pipeline","1. Main logger pipeline",[74,75,77],"h3",{"id":76},"_11-write-path","1.1 Write path",[79,80,85],"pre",{"className":81,"code":83,"language":84},[82],"language-text","[caller threads]\n       │  enqueue (struct LogItem, zero-alloc)\n       ▼\n[ConcurrentQueue\u003CLogItem>]\n       │  signal via SemaphoreSlim\n       ▼\n[single dispatcher thread]\n       │  drain batch → format → AppendLine\n       ▼\n[FileStreamPool: per-(level, name) persistent FileStream]\n       │  LRU bound = LogOptions.MaxOpenFileStreams (default 100)\n       │  day rollover handled inline\n       │  size-based splitting → {name}_part2_Log.{ext}\n       ▼\n{baseDir}\u002F{LogPath}\u002F{yyyyMMdd}\u002F{TypeDirectories.*}\u002F{name}_Log.{ext}\n","text",[34,86,83],{"__ignoreMap":87},"",[74,89,91,92,57],{"id":90},"_12-caller-side-cost-per-loginfo_log","1.2 Caller-side cost (per ",[34,93,94],{},"LOG.Info_Log(...)",[96,97,98,104,115,121,127],"ul",{},[27,99,100,101,57],{},"1× volatile read for cached timestamp (~5 ns; ~30 ns if ",[34,102,103],{},"HighPrecisionTimestamp=true",[27,105,106,107,110,111,114],{},"1× ",[34,108,109],{},"LogFormatter.EscapeMessage"," for ",[34,112,113],{},"{}"," escaping (skipped if message has no braces)",[27,116,117,118],{},"1× struct field copy to build ",[34,119,120],{},"LogItem",[27,122,106,123,126],{},[34,124,125],{},"ConcurrentQueue.Enqueue"," (CAS)",[27,128,106,129],{},[34,130,131],{},"SemaphoreSlim.Release",[15,133,134,145],{},[19,135,136,137,140,141,144],{},"No ",[34,138,139],{},"DateTime.Now",", no ",[34,142,143],{},"string.Format",", no heap allocation"," on the caller path. All formatting is done on the dispatcher thread.",[74,147,149],{"id":148},"_13-backpressure-drop-oldest","1.3 Backpressure: drop-oldest",[15,151,152,153,156],{},"When the queue has more items than ",[34,154,155],{},"AsyncLogOptions.MaxQueueSize"," (default 10000):",[24,158,159,166,169],{},[27,160,161,162,165],{},"The oldest item is ",[34,163,164],{},"TryDequeue","-d and discarded",[27,167,168],{},"A drop counter is incremented atomically",[27,170,171,174],{},[34,172,173],{},"LogOptions.OnDropped"," callback fires (if set)",[15,176,177,178,182],{},"This guarantees the queue never grows unbounded — OOM cannot happen from log spam, only the ",[179,180,181],"em",{},"oldest"," lines are sacrificed.",[74,184,186],{"id":185},"_14-immediate-flush","1.4 Immediate flush",[15,188,189,192,193,196,197,200,201,204],{},[34,190,191],{},"Error"," and ",[34,194,195],{},"Fatal"," levels (and any call with ",[34,198,199],{},"immediateFlush: true",") trigger synchronous write + ",[34,202,203],{},"FileStream.Flush(flushToDisk: true)"," on the caller thread, in addition to the async enqueue. This guarantees crash logs reach disk before the process dies.",[74,206,208],{"id":207},"_15-disk-flush-timer","1.5 Disk flush timer",[15,210,211,212,215,216,219,220,223,224,227,228,231],{},"A ",[34,213,214],{},"Timer"," calls ",[34,217,218],{},"FileStreamPool.FlushAll()"," every ",[34,221,222],{},"DiskFlushIntervalMs"," (default 100 ms), which calls ",[34,225,226],{},"StreamWriter.Flush()"," on all open streams. The OS decides write-back timing (no forced ",[34,229,230],{},"fsync",") — favors throughput over durability.",[74,233,235],{"id":234},"_16-shutdown-safety","1.6 Shutdown safety",[96,237,238,244,250],{},[27,239,240,243],{},[34,241,242],{},"AppDomain.CurrentDomain.ProcessExit"," → drain + flush + close all streams",[27,245,246,249],{},[34,247,248],{},"AppDomain.CurrentDomain.UnhandledException"," → same",[27,251,252,255,256,259],{},[34,253,254],{},"LOG.Configure"," can subscribe ",[34,257,258],{},"EnableGlobalExceptionCapture = true"," for additional Fatal-level logging on unhandled exceptions and unobserved Task exceptions.",[66,261],{},[69,263,265],{"id":264},"_2-quote-pipeline-v31","2. Quote pipeline (v3.1+)",[15,267,268,269,272],{},"The Quote pipeline runs in ",[19,270,271],{},"parallel"," with the main logger — same architecture, separate state.",[74,274,276],{"id":275},"_21-why-a-separate-pipeline","2.1 Why a separate pipeline?",[15,278,279],{},"Quote \u002F tick data has fundamentally different characteristics from application logs:",[281,282,283,297],"table",{},[284,285,286],"thead",{},[287,288,289,292,295],"tr",{},[290,291],"th",{},[290,293,294],{},"Main logger",[290,296,49],{},[298,299,300,312,323,337,350,361,372],"tbody",{},[287,301,302,306,309],{},[303,304,305],"td",{},"Throughput",[303,307,308],{},"~10–1000 entries\u002Fsec",[303,310,311],{},"~10,000–1,000,000 entries\u002Fsec",[287,313,314,317,320],{},[303,315,316],{},"Data type",[303,318,319],{},"Free-form string",[303,321,322],{},"Structured (Symbol, Bid, Ask, …)",[287,324,325,331,334],{},[303,326,327,328],{},"Default ",[34,329,330],{},"MaxQueueSize",[303,332,333],{},"10,000",[303,335,336],{},"50,000",[287,338,339,344,347],{},[303,340,327,341],{},[34,342,343],{},"MaxOpenStreams",[303,345,346],{},"100",[303,348,349],{},"500",[287,351,352,357,359],{},[303,353,327,354],{},[34,355,356],{},"MaxBatchSize",[303,358,346],{},[303,360,349],{},[287,362,363,366,369],{},[303,364,365],{},"Severity levels",[303,367,368],{},"yes (Trace … Fatal)",[303,370,371],{},"no — all entries are equal",[287,373,374,377,380],{},[303,375,376],{},"Immediate flush",[303,378,379],{},"Error\u002FFatal trigger flush",[303,381,382],{},"none — pure async batch",[15,384,385],{},"Putting them on the same dispatcher would cause:",[96,387,388,391],{},[27,389,390],{},"Quote burst (~1M\u002Fsec) saturating the queue and dropping Error\u002FFatal application logs",[27,392,393],{},"Quote dispatcher latency growing during application log immediate-flush",[15,395,396],{},"Separating them eliminates this contention entirely.",[74,398,400],{"id":399},"_22-write-path","2.2 Write path",[79,402,405],{"className":403,"code":404,"language":84},[82],"[caller threads — WebSocket consumer, REST poller, etc.]\n       │  validate (Symbol\u002FBucket non-empty, Extras key collisions, …)\n       │  enqueue (struct QuoteRecord, zero-alloc)\n       ▼\n[ConcurrentQueue\u003CQuoteRecord>]                 ← QuoteOptions.MaxQueueSize\n       │  signal via SemaphoreSlim\n       ▼\n[independent dispatcher thread]\n       │  drain batch → QuoteFormatter.Format → AppendLine\n       ▼\n[QuoteFileStreamPool: per-(bucket, symbol) persistent FileStream]\n       │  LRU bound = QuoteOptions.MaxOpenStreams (default 500)\n       │  filename auto-sanitization for invalid file-system chars\n       ▼\n{baseDir}\u002F{LogPath}\u002F{yyyyMMdd}\u002F{QuotePath}\u002F{Bucket}_{Symbol}_Quote.{ext}\n",[34,406,404],{"__ignoreMap":87},[74,408,410],{"id":409},"_23-synchronous-validation-on-caller-thread","2.3 Synchronous validation on caller thread",[15,412,413,416,417,420],{},[34,414,415],{},"LOG.Quote(...)"," validates the record ",[19,418,419],{},"before"," enqueueing:",[96,422,423,433,440,452],{},[27,424,425,428,429,432],{},[34,426,427],{},"Symbol"," is null or empty → ",[34,430,431],{},"ArgumentException"," thrown on caller thread",[27,434,435,428,438],{},[34,436,437],{},"Bucket",[34,439,431],{},[27,441,442,443,192,446,449,450],{},"Both ",[34,444,445],{},"Extras",[34,447,448],{},"ExtrasJson"," set → ",[34,451,431],{},[27,453,454,456,457,37,460,37,463,466,467],{},[34,455,445],{}," contains a reserved key (",[34,458,459],{},"bid",[34,461,462],{},"ask",[34,464,465],{},"last",", …) → ",[34,468,431],{},[15,470,471,472,475,476,479,480,483],{},"This lets callers wrap calls in ",[34,473,474],{},"try","\u002F",[34,477,478],{},"catch"," and catch programmer errors immediately. Validation errors ",[19,481,482],{},"never"," silently disappear into the dispatcher.",[74,485,487],{"id":486},"_24-backpressure-drop-oldest-with-batched-callback","2.4 Backpressure: drop-oldest with batched callback",[15,489,490,491,494],{},"Same drop-oldest strategy as the main logger, but the ",[34,492,493],{},"OnDropped"," callback signature differs:",[79,496,500],{"className":497,"code":498,"language":499,"meta":87,"style":87},"language-csharp shiki shiki-themes github-light","\u002F\u002F Main logger\npublic Action OnDropped { get; set; }              \u002F\u002F fires once per drop event\n\n\u002F\u002F Quote pipeline\npublic Action\u003Clong> OnDropped { get; set; }        \u002F\u002F batched: parameter = newly dropped since last callback\n","csharp",[34,501,502,510,516,523,529],{"__ignoreMap":87},[503,504,507],"span",{"class":505,"line":506},"line",1,[503,508,509],{},"\u002F\u002F Main logger\n",[503,511,513],{"class":505,"line":512},2,[503,514,515],{},"public Action OnDropped { get; set; }              \u002F\u002F fires once per drop event\n",[503,517,519],{"class":505,"line":518},3,[503,520,522],{"emptyLinePlaceholder":521},true,"\n",[503,524,526],{"class":505,"line":525},4,[503,527,528],{},"\u002F\u002F Quote pipeline\n",[503,530,532],{"class":505,"line":531},5,[503,533,534],{},"public Action\u003Clong> OnDropped { get; set; }        \u002F\u002F batched: parameter = newly dropped since last callback\n",[15,536,537,538,541],{},"The Quote callback receives the ",[19,539,540],{},"delta"," (number of records dropped since the last callback fired), allowing efficient metric reporting without per-record callback overhead during heavy bursts.",[74,543,545],{"id":544},"_25-shutdown","2.5 Shutdown",[15,547,548,549,552,553,556],{},"Same ",[34,550,551],{},"ProcessExit"," hook as the main logger — the Quote pipeline flushes and closes all its streams independently. Both pipelines flush ",[19,554,555],{},"in parallel"," (no ordering coupling), so total shutdown time is bounded by the slower of the two.",[66,558],{},[69,560,562],{"id":561},"_3-why-no-thread-pool-dispatcher","3. Why no thread-pool dispatcher?",[15,564,565,566,569],{},"Both pipelines use a single dedicated ",[34,567,568],{},"Task.Run(...)"," dispatcher per pipeline — not a thread-pool worker. Reasons:",[96,571,572,578,588],{},[27,573,574,577],{},[19,575,576],{},"Predictable latency",": a dedicated thread is never preempted by user code.",[27,579,580,583,584,587],{},[19,581,582],{},"Lock-free FileStreamPool access",": only one thread writes to the pool, so ",[34,585,586],{},"FileStream"," state needs no locks during normal write path (locks are only used during shutdown \u002F disk-flush timer \u002F immediate-flush interleaving).",[27,589,590,593,594,597,598,601],{},[19,591,592],{},"Cache locality",": the dispatcher thread keeps its ",[34,595,596],{},"FileStreamPool"," slots, ",[34,599,600],{},"StreamWriter"," buffers, and dictionary hot in CPU cache.",[15,603,604,605,608,609,612,613,616],{},"The cost: per-",[34,606,607],{},"(level, name)"," (or per-",[34,610,611],{},"(bucket, symbol)",") write ordering is preserved, but ",[19,614,615],{},"cross-key write order may be slightly reordered"," (different keys may flush to disk in batched groups). For HFT tick reconstruction this is fine — timestamps in the records are the source of truth, not file ordering.",[66,618],{},[69,620,622],{"id":621},"_4-timestampcache","4. TimestampCache",[15,624,625,626,628,629,632,633,636,637,639,640,643],{},"A background ",[34,627,214],{}," updates ",[34,630,631],{},"volatile long _currentTicks"," every 1 ms by calling ",[34,634,635],{},"DateTime.Now.Ticks",". Callers do an atomic read of this value (~5 ns) instead of paying the ",[34,638,139],{}," syscall cost (~80 ns on Windows due to ",[34,641,642],{},"GetSystemTimeAsFileTime"," + time-zone conversion) on every log call.",[15,645,646,649,650,653,654,657,658,661,662,665,666,669],{},[19,647,648],{},"1 ms precision floor",": if your ",[34,651,652],{},"TimeFormat"," uses precision finer than ",[34,655,656],{},".fff"," (e.g. ",[34,659,660],{},".ffffff"," for µs), the last digits will always be ",[34,663,664],{},"0000"," unless you opt into ",[34,667,668],{},"HighPrecisionTimestamp = true",".",[74,671,673],{"id":672},"_41-highprecisiontimestamp-mode-v31","4.1 HighPrecisionTimestamp mode (v3.1+)",[15,675,676,677,680],{},"When enabled, the cache also stores ",[34,678,679],{},"Stopwatch.GetTimestamp()"," at each 1 ms update. On read, the caller computes:",[79,682,685],{"className":683,"code":684,"language":84},[82],"actualTicks = cachedTicks + (Stopwatch.GetTimestamp() - cachedSwTimestamp) * (TimeSpan.TicksPerSecond \u002F Stopwatch.Frequency)\n",[34,686,684],{"__ignoreMap":87},[15,688,689,690,692],{},"This reconstructs sub-millisecond precision from the 1 ms cache without paying the ",[34,691,139],{}," cost. Caller-side read goes from ~5 ns to ~30 ns. Use only when you need µs-level timestamps for latency analysis or tick-level time series.",[66,694],{},[69,696,698],{"id":697},"_5-source-of-truth","5. Source of truth",[96,700,701,713,723,733,743],{},[27,702,703,712],{},[704,705,709],"a",{"href":706,"rel":707},"https:\u002F\u002Fgithub.com\u002Fozakboy\u002FOzaLog\u002Fblob\u002Fmain\u002FOzaLog\u002FOzaLog\u002FCore\u002FAsyncLogHandler.cs",[708],"nofollow",[34,710,711],{},"OzaLog\u002FOzaLog\u002FCore\u002FAsyncLogHandler.cs"," — main pipeline dispatcher",[27,714,715,722],{},[704,716,719],{"href":717,"rel":718},"https:\u002F\u002Fgithub.com\u002Fozakboy\u002FOzaLog\u002Fblob\u002Fmain\u002FOzaLog\u002FOzaLog\u002FCore\u002FFileStreamPool.cs",[708],[34,720,721],{},"OzaLog\u002FOzaLog\u002FCore\u002FFileStreamPool.cs"," — main pipeline file streams + LRU",[27,724,725,732],{},[704,726,729],{"href":727,"rel":728},"https:\u002F\u002Fgithub.com\u002Fozakboy\u002FOzaLog\u002Fblob\u002Fmain\u002FOzaLog\u002FOzaLog\u002FCore\u002FQuoteLogHandler.cs",[708],[34,730,731],{},"OzaLog\u002FOzaLog\u002FCore\u002FQuoteLogHandler.cs"," — Quote pipeline dispatcher",[27,734,735,742],{},[704,736,739],{"href":737,"rel":738},"https:\u002F\u002Fgithub.com\u002Fozakboy\u002FOzaLog\u002Fblob\u002Fmain\u002FOzaLog\u002FOzaLog\u002FCore\u002FQuoteFileStreamPool.cs",[708],[34,740,741],{},"OzaLog\u002FOzaLog\u002FCore\u002FQuoteFileStreamPool.cs"," — Quote pipeline file streams",[27,744,745,752],{},[704,746,749],{"href":747,"rel":748},"https:\u002F\u002Fgithub.com\u002Fozakboy\u002FOzaLog\u002Fblob\u002Fmain\u002FOzaLog\u002FOzaLog\u002FCore\u002FTimestampCache.cs",[708],[34,750,751],{},"OzaLog\u002FOzaLog\u002FCore\u002FTimestampCache.cs"," — cached timestamps",[754,755,756],"style",{},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}",{"title":87,"searchDepth":512,"depth":518,"links":758},[759,768,775,776,779],{"id":71,"depth":512,"text":72,"children":760},[761,762,764,765,766,767],{"id":76,"depth":518,"text":77},{"id":90,"depth":518,"text":763},"1.2 Caller-side cost (per LOG.Info_Log(...))",{"id":148,"depth":518,"text":149},{"id":185,"depth":518,"text":186},{"id":207,"depth":518,"text":208},{"id":234,"depth":518,"text":235},{"id":264,"depth":512,"text":265,"children":769},[770,771,772,773,774],{"id":275,"depth":518,"text":276},{"id":399,"depth":518,"text":400},{"id":409,"depth":518,"text":410},{"id":486,"depth":518,"text":487},{"id":544,"depth":518,"text":545},{"id":561,"depth":512,"text":562},{"id":621,"depth":512,"text":622,"children":777},[778],{"id":672,"depth":518,"text":673},{"id":697,"depth":512,"text":698},"ConcurrentQueue + persistent FileStream pool + cached timestamp + drop-oldest backpressure — and the independent Quote pipeline (v3.1+).","md",{},"\u002Fen\u002Fasync-pipeline",{"title":6,"description":780},"en\u002Fasync-pipeline","Pm4HXMycSPEilvDyXHD10ISTZiAKt4JaLRVAQoRWZ8k",{"matched":788,"target":783,"all":789},"path()",null,1778734455471]