[{"data":1,"prerenderedAt":4098},["ShallowReactive",2],{"/2026/rag":3,"surround-/2026/rag":4092},{"id":4,"title":5,"body":6,"categories":4071,"date":4073,"description":4074,"draft":4075,"extension":4076,"image":4077,"meta":4078,"navigation":1420,"path":4079,"rawbody":4080,"readingTime":4081,"recommend":4077,"references":4077,"seo":4086,"sitemap":4087,"stem":4088,"tags":4089,"type":4077,"updated":4073,"url":4077,"__hash__":4091},"content/posts/2026/RAG.md","企业级 RAG 实现详解",{"type":7,"value":8,"toc":4057},"minimark",[9,13,21,124,135,193,197,203,209,213,228,1166,1171,1180,1214,1228,1253,1256,1261,1264,1272,1275,1926,1929,1936,1941,1947,1965,1970,1977,1980,1986,1993,1999,2008,2015,2021,2043,2056,2074,2748,2753,2769,2775,2781,2795,2802,2806,2821,2827,2830,2844,2857,2861,2869,2876,2883,2886,2889,2895,2906,2909,2912,3011,3014,3073,3076,3079,3304,3307,3349,3352,3433,3436,3441,3445,3448,3505,3512,3538,3788,3793,3796,3799,3805,3808,3814,3817,3823,3826,3829,3837,3842,3847,3849,3855,3862,3868,3874,3877,3883,3886,3892,3895,3900,3906,3912,3915,3918,3921,3927,3930,3933,3939,3942,3948,3951,3954,3960,3963,3966,3971,3976,3979,3982,3988,3991,3997,4000,4005,4008,4011,4017,4023,4026,4031,4034,4040,4044,4047,4053],[10,11,12],"h3",{"id":12},"项目概览与技术栈",[14,15,16,20],"p",{},[17,18,19],"strong",{},"OcEasy"," 是一个 AI 面试准备项目：把职位描述（JD）、个人经历、面试记录转成结构化准备计划、逼真模拟面试和可执行的复盘建议。它内置了一个完整的企业级 RAG 知识库（「面试知识库」），底层就是企业级的Onyx RAG 引擎。",[22,23,24,40],"table",{},[25,26,27],"thead",{},[28,29,30,34,37],"tr",{},[31,32,33],"th",{},"组件",[31,35,36],{},"技术",[31,38,39],{},"作用",[41,42,43,55,66,80,91,102,113],"tbody",{},[28,44,45,49,52],{},[46,47,48],"td",{},"后端",[46,50,51],{},"Python 3.13 / FastAPI / SQLAlchemy / Celery",[46,53,54],{},"API、任务编排",[28,56,57,60,63],{},[46,58,59],{},"关系库",[46,61,62],{},"PostgreSQL",[46,64,65],{},"文档元数据、权限、索引状态（事实来源）",[28,67,68,71,77],{},[46,69,70],{},"向量库",[46,72,73,76],{},[17,74,75],{},"OpenSearch","（kNN + BM25）",[46,78,79],{},"混合检索的文档索引",[28,81,82,85,88],{},[46,83,84],{},"缓存/消息",[46,86,87],{},"Redis",[46,89,90],{},"查询嵌入缓存、Celery broker",[28,92,93,96,99],{},[46,94,95],{},"文件存储",[46,97,98],{},"MinIO / S3（blob）",[46,100,101],{},"上传文件的原始内容、图片",[28,103,104,107,110],{},[46,105,106],{},"LLM",[46,108,109],{},"LiteLLM 多 provider（OpenAI/Anthropic/本地 model_server）",[46,111,112],{},"生成、Contextual RAG、查询扩展",[28,114,115,118,121],{},[46,116,117],{},"嵌入模型",[46,119,120],{},"本地 model_server 或云端 API（OpenAI/Cohere/Voyage/Vertex）",[46,122,123],{},"chunk / query 向量化",[14,125,126,129,130,134],{},[17,127,128],{},"几个关键目录","（均在 ",[131,132,133],"code",{"code":133},"backend/onyx/"," 下）：",[136,137,138,145,151,157,163,169,181,187],"ul",{},[139,140,141,144],"li",{},[131,142,143],{"code":143},"connectors/"," — 从 40+ 外部系统（Notion、Confluence、Gmail、GitHub…）拉文档",[139,146,147,150],{},[131,148,149],{"code":149},"indexing/"," — 索引管道：chunker → embedder → 向量库写入",[139,152,153,156],{},[131,154,155],{"code":155},"document_index/opensearch/"," — OpenSearch 适配层、查询 DSL 构建、schema",[139,158,159,162],{},[131,160,161],{"code":161},"context/search/"," — 检索编排：pipeline、search_runner、filters",[139,164,165,168],{},[131,166,167],{"code":167},"chat/"," — 生成阶段：LLM 工具循环、上下文注入、引用处理、压缩",[139,170,171,174,175],{},[131,172,173],{"code":173},"tools/tool_implementations/search/"," — ",[17,176,177,180],{},[131,178,179],{"code":179},"internal_search"," 工具（检索的核心入口）",[139,182,183,186],{},[131,184,185],{"code":185},"secondary_llm_flows/"," — 检索前后的小型 LLM 流程（查询扩展、文档筛选）",[139,188,189,192],{},[131,190,191],{"code":191},"prompts/"," — 所有提示词模板",[10,194,196],{"id":195},"rag完整请求流","RAG完整请求流",[14,198,199],{},[200,201],"img",{"alt":200,"src":202},"https://assets.yangzy.top/rag.webp",[204,205,206],"blockquote",{},[14,207,208],{},"索引阶段把海量文档切成带向量的 chunk 存进 OpenSearch；检索阶段把「一句话提问」变成「多路查询 → 混合检索 → LLM 精筛」；生成阶段让 LLM 自主决定何时检索，并把检索结果以带编号的 JSON 形式喂回去，最终产出带引用的回答。",[10,210,212],{"id":211},"阶段一文档入库与索引构建","阶段一：文档入库与索引构建",[14,214,215,216,219,220,223,224,227],{},"入口 ",[131,217,218],{"code":218},"run_indexing_pipeline"," → ",[131,221,222],{"code":222},"index_doc_batch","（",[131,225,226],{"code":226},"indexing/indexing_pipeline.py:1267","），对一批文档依次执行。完整顺序：",[229,230,235],"pre",{"className":231,"code":232,"language":233,"meta":234,"style":234},"language-python shiki shiki-themes catppuccin-latte one-dark-pro","# indexing_pipeline.py:1314-1524（已简化）\nfiltered_docs, failures = filter_fnc(document_batch)          # 1. 过滤空/超长文档\nfiltered_docs = _apply_document_ingestion_hook(filtered_docs)  # 2. 可扩展的接入钩子\ncontext = adapter.prepare(...)                                  # 3. 元数据入库（Postgres）\nindexable_docs = process_image_sections(...)                    # 4. 图片 → LLM 视觉摘要\nchunks = chunker.chunk(indexable_docs)                          # 5. 切块\nif enable_contextual_rag:\n    chunks = add_contextual_summaries(chunks, llm, ...)         # 6. Contextual RAG（可选）\nwith embed_and_stream(chunks, embedder, ...) as (result, store):# 7. 批量嵌入\n    write_chunks_to_vector_db_with_backoff(document_index, ...) # 8. 写入 OpenSearch\n    adapter.post_index(...)                                     # 9. 更新 chunk 计数/哈希\n","python","",[131,236,237,254,339,381,528,642,762,778,874,947,1002],{"__ignoreMap":234},[238,239,242,246,251],"span",{"class":240,"data-line":241,"line":241},"line",1,[238,243,245],{"class":244},"skYY2","#",[238,247,250],{"class":248},[249],"space"," ",[238,252,253],{"class":244},"indexing_pipeline.py:1314-1524（已简化）\n",[238,255,257,261,265,268,271,274,278,281,285,289,292,295,298,301,304,307,310,313,316,319,322,325,327,330,333,336],{"class":240,"data-line":256,"line":256},2,[238,258,260],{"class":259},"sa2x1","filtered_docs",[238,262,264],{"class":263},"sgT6j",",",[238,266,250],{"class":267},[249],[238,269,270],{"class":259},"failures",[238,272,250],{"class":273},[249],[238,275,277],{"class":276},"sqgB4","=",[238,279,250],{"class":280},[249],[238,282,284],{"class":283},"s3w6o","filter_fnc",[238,286,288],{"class":287},"scfJ_","(",[238,290,291],{"class":259},"document_batch",[238,293,294],{"class":287},")",[238,296,250],{"class":297},[249],[238,299,250],{"class":300},[249],[238,302,250],{"class":303},[249],[238,305,250],{"class":306},[249],[238,308,250],{"class":309},[249],[238,311,250],{"class":312},[249],[238,314,250],{"class":315},[249],[238,317,250],{"class":318},[249],[238,320,250],{"class":321},[249],[238,323,250],{"class":324},[249],[238,326,245],{"class":244},[238,328,250],{"class":329},[249],[238,331,332],{"class":244},"1.",[238,334,250],{"class":335},[249],[238,337,338],{"class":244},"过滤空/超长文档\n",[238,340,342,344,347,349,352,355,357,359,361,364,367,369,372,375,378],{"class":240,"data-line":341,"line":341},3,[238,343,260],{"class":259},[238,345,250],{"class":346},[249],[238,348,277],{"class":276},[238,350,250],{"class":351},[249],[238,353,354],{"class":283},"_apply_document_ingestion_hook",[238,356,288],{"class":287},[238,358,260],{"class":259},[238,360,294],{"class":287},[238,362,250],{"class":363},[249],[238,365,250],{"class":366},[249],[238,368,245],{"class":244},[238,370,250],{"class":371},[249],[238,373,374],{"class":244},"2.",[238,376,250],{"class":377},[249],[238,379,380],{"class":244},"可扩展的接入钩子\n",[238,382,384,387,390,392,395,398,401,404,406,410,412,415,418,421,424,427,430,433,436,439,442,445,448,451,454,457,460,463,466,469,472,475,478,481,484,487,490,493,496,499,502,505,508,511,514,516,519,522,525],{"class":240,"data-line":383,"line":383},4,[238,385,386],{"class":259},"context",[238,388,250],{"class":389},[249],[238,391,277],{"class":276},[238,393,250],{"class":394},[249],[238,396,397],{"class":259},"adapter",[238,399,400],{"class":263},".",[238,402,403],{"class":283},"prepare",[238,405,288],{"class":287},[238,407,409],{"class":408},"sJ4Hi","...",[238,411,294],{"class":287},[238,413,250],{"class":414},[249],[238,416,250],{"class":417},[249],[238,419,250],{"class":420},[249],[238,422,250],{"class":423},[249],[238,425,250],{"class":426},[249],[238,428,250],{"class":429},[249],[238,431,250],{"class":432},[249],[238,434,250],{"class":435},[249],[238,437,250],{"class":438},[249],[238,440,250],{"class":441},[249],[238,443,250],{"class":444},[249],[238,446,250],{"class":447},[249],[238,449,250],{"class":450},[249],[238,452,250],{"class":453},[249],[238,455,250],{"class":456},[249],[238,458,250],{"class":459},[249],[238,461,250],{"class":462},[249],[238,464,250],{"class":465},[249],[238,467,250],{"class":468},[249],[238,470,250],{"class":471},[249],[238,473,250],{"class":474},[249],[238,476,250],{"class":477},[249],[238,479,250],{"class":480},[249],[238,482,250],{"class":483},[249],[238,485,250],{"class":486},[249],[238,488,250],{"class":489},[249],[238,491,250],{"class":492},[249],[238,494,250],{"class":495},[249],[238,497,250],{"class":498},[249],[238,500,250],{"class":501},[249],[238,503,250],{"class":504},[249],[238,506,250],{"class":507},[249],[238,509,250],{"class":510},[249],[238,512,250],{"class":513},[249],[238,515,245],{"class":244},[238,517,250],{"class":518},[249],[238,520,521],{"class":244},"3.",[238,523,250],{"class":524},[249],[238,526,527],{"class":244},"元数据入库（Postgres）\n",[238,529,531,534,537,539,542,545,547,549,551,554,557,560,563,566,569,572,575,578,581,584,587,590,593,596,599,602,605,608,611,613,616,619,622,625,628,631,634,636,639],{"class":240,"data-line":530,"line":530},5,[238,532,533],{"class":259},"indexable_docs",[238,535,250],{"class":536},[249],[238,538,277],{"class":276},[238,540,250],{"class":541},[249],[238,543,544],{"class":283},"process_image_sections",[238,546,288],{"class":287},[238,548,409],{"class":408},[238,550,294],{"class":287},[238,552,250],{"class":553},[249],[238,555,250],{"class":556},[249],[238,558,250],{"class":559},[249],[238,561,250],{"class":562},[249],[238,564,250],{"class":565},[249],[238,567,250],{"class":568},[249],[238,570,250],{"class":571},[249],[238,573,250],{"class":574},[249],[238,576,250],{"class":577},[249],[238,579,250],{"class":580},[249],[238,582,250],{"class":583},[249],[238,585,250],{"class":586},[249],[238,588,250],{"class":589},[249],[238,591,250],{"class":592},[249],[238,594,250],{"class":595},[249],[238,597,250],{"class":598},[249],[238,600,250],{"class":601},[249],[238,603,250],{"class":604},[249],[238,606,250],{"class":607},[249],[238,609,250],{"class":610},[249],[238,612,245],{"class":244},[238,614,250],{"class":615},[249],[238,617,618],{"class":244},"4.",[238,620,250],{"class":621},[249],[238,623,624],{"class":244},"图片",[238,626,250],{"class":627},[249],[238,629,630],{"class":244},"→",[238,632,250],{"class":633},[249],[238,635,106],{"class":244},[238,637,250],{"class":638},[249],[238,640,641],{"class":244},"视觉摘要\n",[238,643,645,648,651,653,656,659,661,664,666,668,670,673,676,679,682,685,688,691,694,697,700,703,706,709,712,715,718,721,724,727,730,733,736,739,742,745,748,750,753,756,759],{"class":240,"data-line":644,"line":644},6,[238,646,647],{"class":259},"chunks",[238,649,250],{"class":650},[249],[238,652,277],{"class":276},[238,654,250],{"class":655},[249],[238,657,658],{"class":259},"chunker",[238,660,400],{"class":263},[238,662,663],{"class":283},"chunk",[238,665,288],{"class":287},[238,667,533],{"class":259},[238,669,294],{"class":287},[238,671,250],{"class":672},[249],[238,674,250],{"class":675},[249],[238,677,250],{"class":678},[249],[238,680,250],{"class":681},[249],[238,683,250],{"class":684},[249],[238,686,250],{"class":687},[249],[238,689,250],{"class":690},[249],[238,692,250],{"class":693},[249],[238,695,250],{"class":696},[249],[238,698,250],{"class":699},[249],[238,701,250],{"class":702},[249],[238,704,250],{"class":705},[249],[238,707,250],{"class":708},[249],[238,710,250],{"class":711},[249],[238,713,250],{"class":714},[249],[238,716,250],{"class":717},[249],[238,719,250],{"class":720},[249],[238,722,250],{"class":723},[249],[238,725,250],{"class":726},[249],[238,728,250],{"class":729},[249],[238,731,250],{"class":732},[249],[238,734,250],{"class":735},[249],[238,737,250],{"class":738},[249],[238,740,250],{"class":741},[249],[238,743,250],{"class":744},[249],[238,746,250],{"class":747},[249],[238,749,245],{"class":244},[238,751,250],{"class":752},[249],[238,754,755],{"class":244},"5.",[238,757,250],{"class":758},[249],[238,760,761],{"class":244},"切块\n",[238,763,765,769,772,775],{"class":240,"data-line":764,"line":764},7,[238,766,768],{"class":767},"sSWcl","if",[238,770,250],{"class":771},[249],[238,773,774],{"class":259},"enable_contextual_rag",[238,776,777],{"class":263},":\n",[238,779,781,784,787,790,793,795,798,800,803,806,808,810,812,815,818,820,823,825,827,830,833,836,839,842,845,848,851,854,856,859,862,865,868,871],{"class":240,"data-line":780,"line":780},8,[238,782,250],{"class":783},[249],[238,785,250],{"class":786},[249],[238,788,250],{"class":789},[249],[238,791,250],{"class":792},[249],[238,794,647],{"class":259},[238,796,250],{"class":797},[249],[238,799,277],{"class":276},[238,801,250],{"class":802},[249],[238,804,805],{"class":283},"add_contextual_summaries",[238,807,288],{"class":287},[238,809,647],{"class":259},[238,811,264],{"class":263},[238,813,250],{"class":814},[249],[238,816,817],{"class":259},"llm",[238,819,264],{"class":263},[238,821,250],{"class":822},[249],[238,824,409],{"class":408},[238,826,294],{"class":287},[238,828,250],{"class":829},[249],[238,831,250],{"class":832},[249],[238,834,250],{"class":835},[249],[238,837,250],{"class":838},[249],[238,840,250],{"class":841},[249],[238,843,250],{"class":844},[249],[238,846,250],{"class":847},[249],[238,849,250],{"class":850},[249],[238,852,250],{"class":853},[249],[238,855,245],{"class":244},[238,857,250],{"class":858},[249],[238,860,861],{"class":244},"6.",[238,863,250],{"class":864},[249],[238,866,867],{"class":244},"Contextual",[238,869,250],{"class":870},[249],[238,872,873],{"class":244},"RAG（可选）\n",[238,875,877,880,883,886,888,890,892,895,898,900,903,905,907,910,913,915,917,920,922,925,928,930,933,935,938,941,944],{"class":240,"data-line":876,"line":876},9,[238,878,879],{"class":767},"with",[238,881,250],{"class":882},[249],[238,884,885],{"class":283},"embed_and_stream",[238,887,288],{"class":287},[238,889,647],{"class":259},[238,891,264],{"class":263},[238,893,250],{"class":894},[249],[238,896,897],{"class":259},"embedder",[238,899,264],{"class":263},[238,901,250],{"class":902},[249],[238,904,409],{"class":408},[238,906,294],{"class":287},[238,908,250],{"class":909},[249],[238,911,912],{"class":767},"as",[238,914,250],{"class":263},[238,916,288],{"class":287},[238,918,919],{"class":259},"result",[238,921,264],{"class":263},[238,923,250],{"class":924},[249],[238,926,927],{"class":259},"store",[238,929,294],{"class":287},[238,931,932],{"class":263},":",[238,934,245],{"class":244},[238,936,250],{"class":937},[249],[238,939,940],{"class":244},"7.",[238,942,250],{"class":943},[249],[238,945,946],{"class":244},"批量嵌入\n",[238,948,950,953,956,959,962,965,967,970,972,975,977,979,982,984,987,990,993,996,999],{"class":240,"data-line":949,"line":949},10,[238,951,250],{"class":952},[249],[238,954,250],{"class":955},[249],[238,957,250],{"class":958},[249],[238,960,250],{"class":961},[249],[238,963,964],{"class":283},"write_chunks_to_vector_db_with_backoff",[238,966,288],{"class":287},[238,968,969],{"class":259},"document_index",[238,971,264],{"class":263},[238,973,250],{"class":974},[249],[238,976,409],{"class":408},[238,978,294],{"class":287},[238,980,250],{"class":981},[249],[238,983,245],{"class":244},[238,985,250],{"class":986},[249],[238,988,989],{"class":244},"8.",[238,991,250],{"class":992},[249],[238,994,995],{"class":244},"写入",[238,997,250],{"class":998},[249],[238,1000,1001],{"class":244},"OpenSearch\n",[238,1003,1005,1008,1011,1014,1017,1019,1021,1024,1026,1028,1030,1033,1036,1039,1042,1045,1048,1051,1054,1057,1060,1063,1066,1069,1072,1075,1078,1081,1084,1087,1090,1093,1096,1099,1102,1105,1108,1111,1114,1117,1120,1123,1126,1129,1132,1135,1138,1141,1143,1146,1149,1152,1155,1158,1160,1163],{"class":240,"data-line":1004,"line":1004},11,[238,1006,250],{"class":1007},[249],[238,1009,250],{"class":1010},[249],[238,1012,250],{"class":1013},[249],[238,1015,250],{"class":1016},[249],[238,1018,397],{"class":259},[238,1020,400],{"class":263},[238,1022,1023],{"class":283},"post_index",[238,1025,288],{"class":287},[238,1027,409],{"class":408},[238,1029,294],{"class":287},[238,1031,250],{"class":1032},[249],[238,1034,250],{"class":1035},[249],[238,1037,250],{"class":1038},[249],[238,1040,250],{"class":1041},[249],[238,1043,250],{"class":1044},[249],[238,1046,250],{"class":1047},[249],[238,1049,250],{"class":1050},[249],[238,1052,250],{"class":1053},[249],[238,1055,250],{"class":1056},[249],[238,1058,250],{"class":1059},[249],[238,1061,250],{"class":1062},[249],[238,1064,250],{"class":1065},[249],[238,1067,250],{"class":1068},[249],[238,1070,250],{"class":1071},[249],[238,1073,250],{"class":1074},[249],[238,1076,250],{"class":1077},[249],[238,1079,250],{"class":1080},[249],[238,1082,250],{"class":1083},[249],[238,1085,250],{"class":1086},[249],[238,1088,250],{"class":1089},[249],[238,1091,250],{"class":1092},[249],[238,1094,250],{"class":1095},[249],[238,1097,250],{"class":1098},[249],[238,1100,250],{"class":1101},[249],[238,1103,250],{"class":1104},[249],[238,1106,250],{"class":1107},[249],[238,1109,250],{"class":1110},[249],[238,1112,250],{"class":1113},[249],[238,1115,250],{"class":1116},[249],[238,1118,250],{"class":1119},[249],[238,1121,250],{"class":1122},[249],[238,1124,250],{"class":1125},[249],[238,1127,250],{"class":1128},[249],[238,1130,250],{"class":1131},[249],[238,1133,250],{"class":1134},[249],[238,1136,250],{"class":1137},[249],[238,1139,250],{"class":1140},[249],[238,1142,245],{"class":244},[238,1144,250],{"class":1145},[249],[238,1147,1148],{"class":244},"9.",[238,1150,250],{"class":1151},[249],[238,1153,1154],{"class":244},"更新",[238,1156,250],{"class":1157},[249],[238,1159,663],{"class":244},[238,1161,250],{"class":1162},[249],[238,1164,1165],{"class":244},"计数/哈希\n",[1167,1168,1170],"h4",{"id":1169},"document-预处理进入切块前","Document 预处理（进入切块前）",[14,1172,1173,223,1176,1179],{},[131,1174,1175],{"code":1175},"filter_documents",[131,1177,1178],{"code":1178},"indexing_pipeline.py:617","）在切块前做三道门槛：",[1181,1182,1183,1189,1198],"ol",{},[139,1184,1185,1188],{},[17,1186,1187],{},"无标题无内容"," → 跳过（否则没有可检索信息）。",[139,1190,1191,223,1194,1197],{},[17,1192,1193],{},"超长文档",[131,1195,1196],{"code":1196},"MAX_DOCUMENT_CHARS","，默认 512MB 字符）→ 跳过并记录失败，防止后面内存爆掉。",[139,1199,1200,223,1203,1205,1206,1209,1210,1213],{},[17,1201,1202],{},"图片 section",[131,1204,544],{"code":544},"，",[131,1207,1208],{"code":1208},"indexing_pipeline.py:723","）：用视觉 LLM 把图片转成文字摘要存进 ",[131,1211,1212],{"code":1212},"section.text","，这样图片内容也能被检索。",[14,1215,1216,1217,1220,1221,223,1224,1227],{},"另外有一个",[17,1218,1219],{},"增量去重","机制 ",[131,1222,1223],{"code":1223},"get_docs_to_update",[131,1225,1226],{"code":1226},"indexing_pipeline.py:339","），两道闸门：",[136,1229,1230,1240,1250],{},[139,1231,1232,1235,1236,1239],{},[17,1233,1234],{},"时间戳闸门","：connector 给的 ",[131,1237,1238],{"code":1238},"doc_updated_at"," 没推进 → 直接跳过（快路径）。",[139,1241,1242,1245,1246,1249],{},[17,1243,1244],{},"内容哈希闸门","：时间没变时算 ",[131,1247,1248],{"code":1248},"content_hash()"," 比对库里存的哈希，相同则跳过。",[139,1251,1252],{},"只有真正变化的文档才走后面的切块+嵌入，极大节省算力。",[14,1254,1255],{},"每个 connector（Notion、Google Drive 等）会定期轮询——每次可能拉回几万个文档。但其中 95% 都没改动过。如果每个文档都重新走「切块 → 嵌入 → 写 OpenSearch」，浪费大量算力和 API 费用。所以需要一种方法快速判断\"这个文档和上次索引时是否完全一样\"。",[14,1257,1258],{},[17,1259,1260],{},"两道闸门如何协作",[14,1262,1263],{},"对于 connector 拉回来的每一篇文档，依次经过两道检查：",[229,1265,1270],{"className":1266,"code":1268,"language":1269},[1267],"language-text","                        ┌────────────────────┐\n                          │   这个文档带了       │\n                          │   doc_updated_at？  │\n                          └────────┬───────────┘\n                                   │\n                      ┌────────────┼────────────┐\n                      │ 没带        │            │ 带了\n                      ▼             │            ▼\n                 跳过闸门1          │   和数据库里旧的 doc_updated_at 比较\n                 （往下走）          │\n                                    │     ┌──────────┬──────────┐\n                                    │     │ 新 > 旧   │ 新 ≤ 旧  │\n                                    │     ▼           ▼\n                                    │  \"时间推进了\"   \"时间没变\"\n                                    │  认定有变化！   大概率没改\n                                    │  直接排队重索引   直接跳过\n                                    │  跳过闸门2       （0次哈希计算，\n                                    │  （不算哈希）     最快路径）\n                                    │\n                ┌───────────────────┘\n                ▼\n        ┌───────────────┐\n        │ 闸门 2：算哈希 │  ← 到这里的文档：要么没带时间戳，要么带时间戳但闸门1放行了\n        │ content_hash() │\n        └───────┬───────┘\n                │\n     ┌──────────┼──────────┐\n     │ 哈希和库里的相同     │ 哈希不同\n     ▼                     ▼\n  \"内容确实没变\"         \"内容确实变了\"\n    跳过                 排队重索引\n","text",[131,1271,1268],{"__ignoreMap":234},[14,1273,1274],{},"源码实现：",[229,1276,1278],{"className":231,"code":1277,"language":233,"meta":234,"style":234},"for doc in documents:\n    # 判断：新时间戳 > 数据库里存的旧时间戳？\n    timestamp_advanced = (新时间戳 存在 and 旧时间戳 存在 and 新 > 旧)\n\n    # 闸门1：有时间戳、时间没推进 → 跳过（0 次哈希计算，最快）\n    if doc 有时间戳 and 时间没推进:\n        continue  # ← 什么都不做，下一个文档\n\n    # 闸门2：只在\"时间戳没推进\"或\"根本没时间戳\"时启用\n    content_hash = doc.content_hash()                   # ← 算哈希\n    if 时间没推进:\n        if 数据库里的旧哈希 == content_hash:           # ← 比对\n            continue  # ← 时间也说你没变，跳\n\n    # 走到这里：时间戳或哈希说\"变了\" → 排队重索引\n    updatable_docs.append(doc)\n",[131,1279,1280,1305,1339,1416,1422,1461,1501,1550,1554,1576,1681,1704,1798,1858,1863,1897],{"__ignoreMap":234},[238,1281,1282,1285,1288,1291,1294,1297,1300,1303],{"class":240,"data-line":241,"line":241},[238,1283,1284],{"class":767},"for",[238,1286,250],{"class":1287},[249],[238,1289,1290],{"class":259},"doc",[238,1292,250],{"class":1293},[249],[238,1295,1296],{"class":767},"in",[238,1298,250],{"class":1299},[249],[238,1301,1302],{"class":259},"documents",[238,1304,777],{"class":263},[238,1306,1307,1310,1313,1316,1319,1321,1324,1327,1330,1333,1336],{"class":240,"data-line":256,"line":256},[238,1308,250],{"class":1309},[249],[238,1311,250],{"class":1312},[249],[238,1314,250],{"class":1315},[249],[238,1317,250],{"class":1318},[249],[238,1320,245],{"class":244},[238,1322,250],{"class":1323},[249],[238,1325,1326],{"class":244},"判断：新时间戳",[238,1328,250],{"class":1329},[249],[238,1331,1332],{"class":244},">",[238,1334,250],{"class":1335},[249],[238,1337,1338],{"class":244},"数据库里存的旧时间戳？\n",[238,1340,1341,1344,1347,1350,1353,1356,1359,1361,1363,1365,1368,1371,1374,1377,1380,1383,1386,1389,1391,1394,1396,1399,1402,1405,1407,1410,1413],{"class":240,"data-line":341,"line":341},[238,1342,250],{"class":1343},[249],[238,1345,250],{"class":1346},[249],[238,1348,250],{"class":1349},[249],[238,1351,250],{"class":1352},[249],[238,1354,1355],{"class":259},"timestamp_advanced",[238,1357,250],{"class":1358},[249],[238,1360,277],{"class":276},[238,1362,250],{"class":263},[238,1364,288],{"class":287},[238,1366,1367],{"class":259},"新时间戳",[238,1369,250],{"class":1370},[249],[238,1372,1373],{"class":259},"存在",[238,1375,250],{"class":1376},[249],[238,1378,1379],{"class":767},"and",[238,1381,250],{"class":1382},[249],[238,1384,1385],{"class":259},"旧时间戳",[238,1387,250],{"class":1388},[249],[238,1390,1373],{"class":259},[238,1392,250],{"class":1393},[249],[238,1395,1379],{"class":767},[238,1397,250],{"class":1398},[249],[238,1400,1401],{"class":259},"新",[238,1403,250],{"class":1404},[249],[238,1406,1332],{"class":276},[238,1408,250],{"class":1409},[249],[238,1411,1412],{"class":259},"旧",[238,1414,1415],{"class":287},")\n",[238,1417,1418],{"class":240,"data-line":383,"line":383},[238,1419,1421],{"emptyLinePlaceholder":1420},true,"\n",[238,1423,1424,1427,1430,1433,1436,1438,1441,1444,1447,1449,1452,1455,1458],{"class":240,"data-line":530,"line":530},[238,1425,250],{"class":1426},[249],[238,1428,250],{"class":1429},[249],[238,1431,250],{"class":1432},[249],[238,1434,250],{"class":1435},[249],[238,1437,245],{"class":244},[238,1439,250],{"class":1440},[249],[238,1442,1443],{"class":244},"闸门1：有时间戳、时间没推进",[238,1445,250],{"class":1446},[249],[238,1448,630],{"class":244},[238,1450,250],{"class":1451},[249],[238,1453,1454],{"class":244},"跳过（0",[238,1456,250],{"class":1457},[249],[238,1459,1460],{"class":244},"次哈希计算，最快）\n",[238,1462,1463,1466,1469,1472,1475,1477,1480,1482,1485,1488,1491,1493,1496,1499],{"class":240,"data-line":644,"line":644},[238,1464,250],{"class":1465},[249],[238,1467,250],{"class":1468},[249],[238,1470,250],{"class":1471},[249],[238,1473,250],{"class":1474},[249],[238,1476,768],{"class":767},[238,1478,250],{"class":1479},[249],[238,1481,1290],{"class":259},[238,1483,250],{"class":1484},[249],[238,1486,1487],{"class":259},"有时间戳",[238,1489,250],{"class":1490},[249],[238,1492,1379],{"class":767},[238,1494,250],{"class":1495},[249],[238,1497,1498],{"class":259},"时间没推进",[238,1500,777],{"class":263},[238,1502,1503,1506,1509,1512,1515,1518,1521,1524,1527,1530,1533,1536,1538,1541,1544,1547],{"class":240,"data-line":764,"line":764},[238,1504,250],{"class":1505},[249],[238,1507,250],{"class":1508},[249],[238,1510,250],{"class":1511},[249],[238,1513,250],{"class":1514},[249],[238,1516,250],{"class":1517},[249],[238,1519,250],{"class":1520},[249],[238,1522,250],{"class":1523},[249],[238,1525,250],{"class":1526},[249],[238,1528,1529],{"class":767},"continue",[238,1531,250],{"class":1532},[249],[238,1534,250],{"class":1535},[249],[238,1537,245],{"class":244},[238,1539,250],{"class":1540},[249],[238,1542,1543],{"class":244},"←",[238,1545,250],{"class":1546},[249],[238,1548,1549],{"class":244},"什么都不做，下一个文档\n",[238,1551,1552],{"class":240,"data-line":780,"line":780},[238,1553,1421],{"emptyLinePlaceholder":1420},[238,1555,1556,1559,1562,1565,1568,1570,1573],{"class":240,"data-line":876,"line":876},[238,1557,250],{"class":1558},[249],[238,1560,250],{"class":1561},[249],[238,1563,250],{"class":1564},[249],[238,1566,250],{"class":1567},[249],[238,1569,245],{"class":244},[238,1571,250],{"class":1572},[249],[238,1574,1575],{"class":244},"闸门2：只在\"时间戳没推进\"或\"根本没时间戳\"时启用\n",[238,1577,1578,1581,1584,1587,1590,1593,1596,1598,1601,1603,1605,1607,1609,1611,1614,1617,1620,1623,1626,1629,1632,1635,1638,1641,1644,1647,1650,1653,1656,1659,1662,1665,1668,1670,1673,1675,1678],{"class":240,"data-line":949,"line":949},[238,1579,250],{"class":1580},[249],[238,1582,250],{"class":1583},[249],[238,1585,250],{"class":1586},[249],[238,1588,250],{"class":1589},[249],[238,1591,1592],{"class":259},"content_hash",[238,1594,250],{"class":1595},[249],[238,1597,277],{"class":276},[238,1599,250],{"class":1600},[249],[238,1602,1290],{"class":259},[238,1604,400],{"class":263},[238,1606,1592],{"class":283},[238,1608,288],{"class":287},[238,1610,294],{"class":287},[238,1612,250],{"class":1613},[249],[238,1615,250],{"class":1616},[249],[238,1618,250],{"class":1619},[249],[238,1621,250],{"class":1622},[249],[238,1624,250],{"class":1625},[249],[238,1627,250],{"class":1628},[249],[238,1630,250],{"class":1631},[249],[238,1633,250],{"class":1634},[249],[238,1636,250],{"class":1637},[249],[238,1639,250],{"class":1640},[249],[238,1642,250],{"class":1643},[249],[238,1645,250],{"class":1646},[249],[238,1648,250],{"class":1649},[249],[238,1651,250],{"class":1652},[249],[238,1654,250],{"class":1655},[249],[238,1657,250],{"class":1658},[249],[238,1660,250],{"class":1661},[249],[238,1663,250],{"class":1664},[249],[238,1666,250],{"class":1667},[249],[238,1669,245],{"class":244},[238,1671,250],{"class":1672},[249],[238,1674,1543],{"class":244},[238,1676,250],{"class":1677},[249],[238,1679,1680],{"class":244},"算哈希\n",[238,1682,1683,1686,1689,1692,1695,1697,1700,1702],{"class":240,"data-line":1004,"line":1004},[238,1684,250],{"class":1685},[249],[238,1687,250],{"class":1688},[249],[238,1690,250],{"class":1691},[249],[238,1693,250],{"class":1694},[249],[238,1696,768],{"class":767},[238,1698,250],{"class":1699},[249],[238,1701,1498],{"class":259},[238,1703,777],{"class":263},[238,1705,1707,1710,1713,1716,1719,1722,1725,1728,1731,1733,1736,1739,1742,1745,1748,1750,1752,1755,1758,1761,1764,1767,1770,1773,1776,1779,1782,1785,1787,1790,1792,1795],{"class":240,"data-line":1706,"line":1706},12,[238,1708,250],{"class":1709},[249],[238,1711,250],{"class":1712},[249],[238,1714,250],{"class":1715},[249],[238,1717,250],{"class":1718},[249],[238,1720,250],{"class":1721},[249],[238,1723,250],{"class":1724},[249],[238,1726,250],{"class":1727},[249],[238,1729,250],{"class":1730},[249],[238,1732,768],{"class":767},[238,1734,250],{"class":1735},[249],[238,1737,1738],{"class":259},"数据库里的旧哈希",[238,1740,250],{"class":1741},[249],[238,1743,1744],{"class":276},"==",[238,1746,250],{"class":1747},[249],[238,1749,1592],{"class":259},[238,1751,932],{"class":263},[238,1753,250],{"class":1754},[249],[238,1756,250],{"class":1757},[249],[238,1759,250],{"class":1760},[249],[238,1762,250],{"class":1763},[249],[238,1765,250],{"class":1766},[249],[238,1768,250],{"class":1769},[249],[238,1771,250],{"class":1772},[249],[238,1774,250],{"class":1775},[249],[238,1777,250],{"class":1778},[249],[238,1780,250],{"class":1781},[249],[238,1783,250],{"class":1784},[249],[238,1786,245],{"class":244},[238,1788,250],{"class":1789},[249],[238,1791,1543],{"class":244},[238,1793,250],{"class":1794},[249],[238,1796,1797],{"class":244},"比对\n",[238,1799,1801,1804,1807,1810,1813,1816,1819,1822,1825,1828,1831,1834,1837,1839,1842,1845,1847,1850,1852,1855],{"class":240,"data-line":1800,"line":1800},13,[238,1802,250],{"class":1803},[249],[238,1805,250],{"class":1806},[249],[238,1808,250],{"class":1809},[249],[238,1811,250],{"class":1812},[249],[238,1814,250],{"class":1815},[249],[238,1817,250],{"class":1818},[249],[238,1820,250],{"class":1821},[249],[238,1823,250],{"class":1824},[249],[238,1826,250],{"class":1827},[249],[238,1829,250],{"class":1830},[249],[238,1832,250],{"class":1833},[249],[238,1835,250],{"class":1836},[249],[238,1838,1529],{"class":767},[238,1840,250],{"class":1841},[249],[238,1843,250],{"class":1844},[249],[238,1846,245],{"class":244},[238,1848,250],{"class":1849},[249],[238,1851,1543],{"class":244},[238,1853,250],{"class":1854},[249],[238,1856,1857],{"class":244},"时间也说你没变，跳\n",[238,1859,1861],{"class":240,"data-line":1860,"line":1860},14,[238,1862,1421],{"emptyLinePlaceholder":1420},[238,1864,1866,1869,1872,1875,1878,1880,1883,1886,1889,1891,1894],{"class":240,"data-line":1865,"line":1865},15,[238,1867,250],{"class":1868},[249],[238,1870,250],{"class":1871},[249],[238,1873,250],{"class":1874},[249],[238,1876,250],{"class":1877},[249],[238,1879,245],{"class":244},[238,1881,250],{"class":1882},[249],[238,1884,1885],{"class":244},"走到这里：时间戳或哈希说\"变了\"",[238,1887,250],{"class":1888},[249],[238,1890,630],{"class":244},[238,1892,250],{"class":1893},[249],[238,1895,1896],{"class":244},"排队重索引\n",[238,1898,1900,1903,1906,1909,1912,1915,1917,1920,1922,1924],{"class":240,"data-line":1899,"line":1899},16,[238,1901,250],{"class":1902},[249],[238,1904,250],{"class":1905},[249],[238,1907,250],{"class":1908},[249],[238,1910,250],{"class":1911},[249],[238,1913,1914],{"class":259},"updatable_docs",[238,1916,400],{"class":263},[238,1918,1919],{"class":283},"append",[238,1921,288],{"class":287},[238,1923,1290],{"class":259},[238,1925,1415],{"class":287},[1167,1927,1928],{"id":663},"Chunk",[14,1930,1931,1932,1935],{},"切块不是把文章按字符数硬裁，而是一套",[17,1933,1934],{},"三层协作","的流程。",[14,1937,1938],{},[17,1939,1940],{},"三层结构总览",[229,1942,1945],{"className":1943,"code":1944,"language":1269},[1267],"Chunker                          (chunker.py:124)   ← 顶层编排，算 token 预算\n  └─ DocumentChunker             (chunking/document_chunker.py:25)  ← 迭代 section，调度切块器\n       ├─ TextChunker            (text_section_chunker.py:22)       ← 正文切块（最重要）\n       ├─ ImageChunker           (image_section_chunker.py)         ← 图片 section\n       └─ TabularChunker         (tabular_section_chunker)          ← CSV/表格 section\n",[131,1946,1944],{"__ignoreMap":234},[14,1948,1949,1950,223,1953,1956,1957,1960,1961,1964],{},"核心类 ",[131,1951,1952],{"code":1952},"Chunker",[131,1954,1955],{"code":1955},"indexing/chunker.py:124","），用 ",[131,1958,1959],{"code":1959},"chonkie.SentenceChunker"," 按",[17,1962,1963],{},"句子感知","的方式切块（不是硬切字符，避免把一句话拦腰斩断）。",[14,1966,1967],{},[17,1968,1969],{},"第一步：Chunker算预算",[14,1971,1972,1973,1976],{},"拿到一个文档后，第一件事不是切，而是",[17,1974,1975],{},"算清楚有多少 token 配额分给正文","。这个决定直接影响最终的文本布局。",[14,1978,1979],{},"用一个具体例子——一篇 Notion 页面《Q3 销售总结》：",[229,1981,1984],{"className":1982,"code":1983,"language":1269},[1267],"文档标题: \"Q3 销售总结\"\n文档作者: \"张三\"\n文档标签: [\"销售\", \"Q3\", \"2025\"]\n正文 section1: \"本季度整体营收增长 20%，其中...\"  ← 这段有 300 token\n正文 section2: \"华东区表现突出，达成了...\"       ← 这段有 400 token\n正文 section3: \"总结与展望如下...\"               ← 这段有 180 token\n",[131,1985,1983],{"__ignoreMap":234},[14,1987,1988,1989,1992],{},"运算过程（",[131,1990,1991],{"code":1991},"_handle_single_document","）：",[229,1994,1997],{"className":1995,"code":1996,"language":1269},[1267],"① 标题截断为 blurb（128 token）→ title_prefix = \"Q3 销售总结\\n\\n\"\n   title_tokens = len(tokenizer(\"Q3 销售总结\\n\\n\")) = 6\n\n② 元数据拼成两版后缀（(chunker.py:209-219)）：\n   metadata_suffix_semantic = \"\\nMetadata:\\n\\t作者 - 张三\\n\\t标签 - 销售, Q3, 2025\"\n   metadata_suffix_keyword  = \"\\n张三 销售 Q3 2025\"  ← 只取值，没有键名\n   metadata_tokens = 12\n\n③ 是否启动 Contextual RAG？\n   - 全文 token = 300+400+180 = 880\n   - 880 + 6(title) + 12(metadata) = 898 > 512 ← 一整个 chunk 放不下，需要 Contextual RAG\n   - context_size = DEFAULT_CONTEXTUAL_RAG_RESERVED_TOKENS = 128\n\n④ content_token_limit = 512 - 6 - 12 - 128 = 366 token\n\n⑤ 自我兜底检查：\n   - 366 ≤ 256(CHUNK_MIN_CONTENT)？ 否，够用，通过\n   - 如果不够 → 放弃 Contextual RAG，也不要元数据前缀\n   - 最后手段 → content_token_limit 退回 512，什么都不要，只存裸正文\n",[131,1998,1996],{"__ignoreMap":234},[14,2000,2001],{},[17,2002,2003,2004,2007],{},"第二步：",[131,2005,2006],{"code":2006},"TextChunker"," 切正文",[14,2009,2010,2011,2014],{},"正文 section 不是每段独立切——",[17,2012,2013],{},"它有一个跨 section 累积器","，目的是尽量把相邻的小 section 拼进同一个 chunk，避免碎片化。",[229,2016,2019],{"className":2017,"code":2018,"language":1269},[1267],"初始化: accumulator = \"\"（空缓冲）\n\n处理 section1（300 token）:\n  300 ≤ 366 → 放得下，accumulator = \"本季度整体营收增长 20%，其中...\"\n\n处理 section2（400 token）:\n  accumulator 已有 300 token + section2 有 400 token = 700 > 366\n  → 放不下！先把 accumulator 吐出为一个 chunk：Chunk1 = \"本季度整体营收...\"\n  → 然后 section2 自己有 400 > 366 → 超大！走 _handle_oversized_section\n\n      处理超大 section2（_handle_oversized_section, line 82）：\n        用 SentenceChunker 按句子感知把这 400 token 切分：\n        → chunk2a = \"华东区表现突出，达成了目标 A。\"       (200 token)\n        → chunk2b = \"此外在 B 领域也取得突破。\"           (190 token)\n        每个子块成为一个独立的 ChunkPayload\n\n处理 section3（180 token）:\n  accumulator 现在是空的，180 ≤ 366 → accumulator = \"总结与展望如下...\"\n\n最后 flush: 出来 → Chunk3 = \"总结与展望如下...\"\n",[131,2020,2018],{"__ignoreMap":234},[14,2022,2023,2026,2027,2030,2031,2034,2035,2038,2039,2042],{},[17,2024,2025],{},"关键机制——SentenceChunker","（来自 ",[131,2028,2029],{"code":2029},"chonkie"," 库）：它不是按字符硬裁，而是按句子边界切——尽量在句号/换行处下刀。如果 ",[131,2032,2033],{"code":2033},"STRICT_CHUNK_TOKEN_LIMIT"," 开启，切完后还超限的分片会被 ",[131,2036,2037],{"code":2037},"split_text_by_tokens"," 强制硬裁（",[131,2040,2041],{"code":2041},"text_section_chunker.py:93-107","）。",[14,2044,2045],{},[17,2046,2047,2048,2051,2052,2055],{},"第三步：",[131,2049,2050],{"code":2050},"ChunkPayload"," 升级为 ",[131,2053,2054],{"code":2054},"DocAwareChunk","（section_chunker.py:41-67）",[14,2057,2058,2059,2062,2063,2066,2067,2069,2070,2073],{},"上面每个 ",[131,2060,2061],{"code":2061},"ChunkPayload(text, links, ...)"," 在 ",[131,2064,2065],{"code":2065},"DocumentChunker.chunk","（document_chunker.py:68-79）里被升级成完整的 ",[131,2068,2054],{"code":2054},"，这一行代码做的是把所有文档级别的信息",[17,2071,2072],{},"复制粘贴到每个 chunk 上","：",[229,2075,2077],{"className":231,"code":2076,"language":233,"meta":234,"style":234},"# section_chunker.py:51-67（已简化）\nreturn DocAwareChunk(\n    content=self.text,                              # ← body 正文\n    title_prefix=\"Q3 销售总结\\n\\n\",                  # ← 每个 chunk 都带同样的标题\n    metadata_suffix_semantic=\"\\nMetadata:\\n...\",    # ← 每个 chunk 都带同样的元数据\n    metadata_suffix_keyword=\"\\n张三 销售 Q3 2025\",  # ← 关键词版元数据\n    blurb=extract_blurb(self.text),                 # ← 首 128 token 做预览\n    source_links={0: \"http://...\", 200: \"http://...\"},  # ← 点击引用直达原文\n    mini_chunk_texts=[\"华东区表现...\", \"此外在B领域...\"],  # ← 15-token 微块\n)\n",[131,2078,2079,2089,2102,2241,2357,2426,2491,2608,2681,2744],{"__ignoreMap":234},[238,2080,2081,2083,2086],{"class":240,"data-line":241,"line":241},[238,2082,245],{"class":244},[238,2084,250],{"class":2085},[249],[238,2087,2088],{"class":244},"section_chunker.py:51-67（已简化）\n",[238,2090,2091,2094,2097,2099],{"class":240,"data-line":256,"line":256},[238,2092,2093],{"class":767},"return",[238,2095,250],{"class":2096},[249],[238,2098,2054],{"class":283},[238,2100,2101],{"class":287},"(\n",[238,2103,2104,2107,2110,2113,2116,2120,2122,2126,2128,2130,2132,2135,2138,2141,2144,2147,2150,2153,2156,2159,2162,2165,2168,2171,2174,2177,2180,2183,2186,2189,2192,2195,2198,2201,2204,2207,2210,2213,2216,2219,2222,2224,2227,2229,2232,2235,2238],{"class":240,"data-line":341,"line":341},[238,2105,250],{"class":2106},[249],[238,2108,250],{"class":2109},[249],[238,2111,250],{"class":2112},[249],[238,2114,250],{"class":2115},[249],[238,2117,2119],{"class":2118},"sddMY","content",[238,2121,277],{"class":276},[238,2123,2125],{"class":2124},"sQf5i","self",[238,2127,400],{"class":263},[238,2129,1269],{"class":259},[238,2131,264],{"class":263},[238,2133,250],{"class":2134},[249],[238,2136,250],{"class":2137},[249],[238,2139,250],{"class":2140},[249],[238,2142,250],{"class":2143},[249],[238,2145,250],{"class":2146},[249],[238,2148,250],{"class":2149},[249],[238,2151,250],{"class":2152},[249],[238,2154,250],{"class":2155},[249],[238,2157,250],{"class":2158},[249],[238,2160,250],{"class":2161},[249],[238,2163,250],{"class":2164},[249],[238,2166,250],{"class":2167},[249],[238,2169,250],{"class":2170},[249],[238,2172,250],{"class":2173},[249],[238,2175,250],{"class":2176},[249],[238,2178,250],{"class":2179},[249],[238,2181,250],{"class":2182},[249],[238,2184,250],{"class":2185},[249],[238,2187,250],{"class":2188},[249],[238,2190,250],{"class":2191},[249],[238,2193,250],{"class":2194},[249],[238,2196,250],{"class":2197},[249],[238,2199,250],{"class":2200},[249],[238,2202,250],{"class":2203},[249],[238,2205,250],{"class":2206},[249],[238,2208,250],{"class":2209},[249],[238,2211,250],{"class":2212},[249],[238,2214,250],{"class":2215},[249],[238,2217,250],{"class":2218},[249],[238,2220,250],{"class":2221},[249],[238,2223,245],{"class":244},[238,2225,250],{"class":2226},[249],[238,2228,1543],{"class":244},[238,2230,250],{"class":2231},[249],[238,2233,2234],{"class":244},"body",[238,2236,250],{"class":2237},[249],[238,2239,2240],{"class":244},"正文\n",[238,2242,2243,2246,2249,2252,2255,2258,2260,2264,2267,2270,2274,2277,2279,2282,2285,2288,2291,2294,2297,2300,2303,2306,2309,2312,2315,2318,2321,2324,2327,2330,2333,2335,2338,2340,2343,2346,2349,2351,2354],{"class":240,"data-line":383,"line":383},[238,2244,250],{"class":2245},[249],[238,2247,250],{"class":2248},[249],[238,2250,250],{"class":2251},[249],[238,2253,250],{"class":2254},[249],[238,2256,2257],{"class":2118},"title_prefix",[238,2259,277],{"class":276},[238,2261,2263],{"class":2262},"sw_MA","\"Q3",[238,2265,250],{"class":2266},[249],[238,2268,2269],{"class":2262},"销售总结",[238,2271,2273],{"class":2272},"sFWNk","\\n\\n",[238,2275,2276],{"class":2262},"\"",[238,2278,264],{"class":263},[238,2280,250],{"class":2281},[249],[238,2283,250],{"class":2284},[249],[238,2286,250],{"class":2287},[249],[238,2289,250],{"class":2290},[249],[238,2292,250],{"class":2293},[249],[238,2295,250],{"class":2296},[249],[238,2298,250],{"class":2299},[249],[238,2301,250],{"class":2302},[249],[238,2304,250],{"class":2305},[249],[238,2307,250],{"class":2308},[249],[238,2310,250],{"class":2311},[249],[238,2313,250],{"class":2314},[249],[238,2316,250],{"class":2317},[249],[238,2319,250],{"class":2320},[249],[238,2322,250],{"class":2323},[249],[238,2325,250],{"class":2326},[249],[238,2328,250],{"class":2329},[249],[238,2331,250],{"class":2332},[249],[238,2334,245],{"class":244},[238,2336,250],{"class":2337},[249],[238,2339,1543],{"class":244},[238,2341,250],{"class":2342},[249],[238,2344,2345],{"class":244},"每个",[238,2347,250],{"class":2348},[249],[238,2350,663],{"class":244},[238,2352,250],{"class":2353},[249],[238,2355,2356],{"class":244},"都带同样的标题\n",[238,2358,2359,2362,2365,2368,2371,2374,2376,2378,2381,2384,2386,2389,2391,2394,2397,2400,2403,2405,2408,2410,2413,2415,2418,2420,2423],{"class":240,"data-line":530,"line":530},[238,2360,250],{"class":2361},[249],[238,2363,250],{"class":2364},[249],[238,2366,250],{"class":2367},[249],[238,2369,250],{"class":2370},[249],[238,2372,2373],{"class":2118},"metadata_suffix_semantic",[238,2375,277],{"class":276},[238,2377,2276],{"class":2262},[238,2379,2380],{"class":2272},"\\n",[238,2382,2383],{"class":2262},"Metadata:",[238,2385,2380],{"class":2272},[238,2387,2388],{"class":2262},"...\"",[238,2390,264],{"class":263},[238,2392,250],{"class":2393},[249],[238,2395,250],{"class":2396},[249],[238,2398,250],{"class":2399},[249],[238,2401,250],{"class":2402},[249],[238,2404,245],{"class":244},[238,2406,250],{"class":2407},[249],[238,2409,1543],{"class":244},[238,2411,250],{"class":2412},[249],[238,2414,2345],{"class":244},[238,2416,250],{"class":2417},[249],[238,2419,663],{"class":244},[238,2421,250],{"class":2422},[249],[238,2424,2425],{"class":244},"都带同样的元数据\n",[238,2427,2428,2431,2434,2437,2440,2443,2445,2447,2449,2452,2455,2458,2461,2464,2467,2470,2472,2475,2478,2480,2483,2485,2488],{"class":240,"data-line":644,"line":644},[238,2429,250],{"class":2430},[249],[238,2432,250],{"class":2433},[249],[238,2435,250],{"class":2436},[249],[238,2438,250],{"class":2439},[249],[238,2441,2442],{"class":2118},"metadata_suffix_keyword",[238,2444,277],{"class":276},[238,2446,2276],{"class":2262},[238,2448,2380],{"class":2272},[238,2450,2451],{"class":2262},"张三",[238,2453,250],{"class":2454},[249],[238,2456,2457],{"class":2262},"销售",[238,2459,250],{"class":2460},[249],[238,2462,2463],{"class":2262},"Q3",[238,2465,250],{"class":2466},[249],[238,2468,2469],{"class":2262},"2025\"",[238,2471,264],{"class":263},[238,2473,250],{"class":2474},[249],[238,2476,250],{"class":2477},[249],[238,2479,245],{"class":244},[238,2481,250],{"class":2482},[249],[238,2484,1543],{"class":244},[238,2486,250],{"class":2487},[249],[238,2489,2490],{"class":244},"关键词版元数据\n",[238,2492,2493,2496,2499,2502,2505,2508,2510,2513,2516,2518,2520,2522,2524,2526,2529,2532,2535,2538,2541,2544,2547,2550,2553,2556,2559,2562,2565,2568,2571,2574,2577,2579,2582,2584,2587,2590,2593,2596,2599,2602,2605],{"class":240,"data-line":764,"line":764},[238,2494,250],{"class":2495},[249],[238,2497,250],{"class":2498},[249],[238,2500,250],{"class":2501},[249],[238,2503,250],{"class":2504},[249],[238,2506,2507],{"class":2118},"blurb",[238,2509,277],{"class":276},[238,2511,2512],{"class":283},"extract_blurb",[238,2514,288],{"class":2515},"si0CV",[238,2517,2125],{"class":2124},[238,2519,400],{"class":263},[238,2521,1269],{"class":259},[238,2523,294],{"class":2515},[238,2525,264],{"class":263},[238,2527,250],{"class":2528},[249],[238,2530,250],{"class":2531},[249],[238,2533,250],{"class":2534},[249],[238,2536,250],{"class":2537},[249],[238,2539,250],{"class":2540},[249],[238,2542,250],{"class":2543},[249],[238,2545,250],{"class":2546},[249],[238,2548,250],{"class":2549},[249],[238,2551,250],{"class":2552},[249],[238,2554,250],{"class":2555},[249],[238,2557,250],{"class":2558},[249],[238,2560,250],{"class":2561},[249],[238,2563,250],{"class":2564},[249],[238,2566,250],{"class":2567},[249],[238,2569,250],{"class":2570},[249],[238,2572,250],{"class":2573},[249],[238,2575,250],{"class":2576},[249],[238,2578,245],{"class":244},[238,2580,250],{"class":2581},[249],[238,2583,1543],{"class":244},[238,2585,250],{"class":2586},[249],[238,2588,2589],{"class":244},"首",[238,2591,250],{"class":2592},[249],[238,2594,2595],{"class":244},"128",[238,2597,250],{"class":2598},[249],[238,2600,2601],{"class":244},"token",[238,2603,250],{"class":2604},[249],[238,2606,2607],{"class":244},"做预览\n",[238,2609,2610,2613,2616,2619,2622,2625,2627,2630,2634,2636,2639,2642,2644,2647,2650,2652,2655,2657,2660,2662,2665,2668,2670,2673,2675,2678],{"class":240,"data-line":780,"line":780},[238,2611,250],{"class":2612},[249],[238,2614,250],{"class":2615},[249],[238,2617,250],{"class":2618},[249],[238,2620,250],{"class":2621},[249],[238,2623,2624],{"class":2118},"source_links",[238,2626,277],{"class":276},[238,2628,2629],{"class":2515},"{",[238,2631,2633],{"class":2632},"sYQis","0",[238,2635,932],{"class":263},[238,2637,250],{"class":2638},[249],[238,2640,2641],{"class":2262},"\"http://...\"",[238,2643,264],{"class":263},[238,2645,250],{"class":2646},[249],[238,2648,2649],{"class":2632},"200",[238,2651,932],{"class":263},[238,2653,250],{"class":2654},[249],[238,2656,2641],{"class":2262},[238,2658,2659],{"class":2515},"}",[238,2661,264],{"class":263},[238,2663,250],{"class":2664},[249],[238,2666,250],{"class":2667},[249],[238,2669,245],{"class":244},[238,2671,250],{"class":2672},[249],[238,2674,1543],{"class":244},[238,2676,250],{"class":2677},[249],[238,2679,2680],{"class":244},"点击引用直达原文\n",[238,2682,2683,2686,2689,2692,2695,2698,2700,2703,2706,2708,2711,2714,2717,2719,2722,2725,2727,2730,2732,2735,2738,2741],{"class":240,"data-line":876,"line":876},[238,2684,250],{"class":2685},[249],[238,2687,250],{"class":2688},[249],[238,2690,250],{"class":2691},[249],[238,2693,250],{"class":2694},[249],[238,2696,2697],{"class":2118},"mini_chunk_texts",[238,2699,277],{"class":276},[238,2701,2702],{"class":2515},"[",[238,2704,2705],{"class":2262},"\"华东区表现...\"",[238,2707,264],{"class":263},[238,2709,250],{"class":2710},[249],[238,2712,2713],{"class":2262},"\"此外在B领域...\"",[238,2715,2716],{"class":2515},"]",[238,2718,264],{"class":263},[238,2720,250],{"class":2721},[249],[238,2723,250],{"class":2724},[249],[238,2726,245],{"class":244},[238,2728,250],{"class":2729},[249],[238,2731,1543],{"class":244},[238,2733,250],{"class":2734},[249],[238,2736,2737],{"class":244},"15-token",[238,2739,250],{"class":2740},[249],[238,2742,2743],{"class":244},"微块\n",[238,2745,2746],{"class":240,"data-line":949,"line":949},[238,2747,1415],{"class":287},[14,2749,2750],{},[17,2751,2752],{},"第四步：存进 OpenSearch 时的最终拼装",[14,2754,2755,2756,2758,2759,223,2762,2765,2766,2073],{},"索引写到 ",[131,2757,2119],{"code":2119}," 字段的不是原始正文——",[131,2760,2761],{"code":2761},"generate_enriched_content_for_chunk_embedding",[131,2763,2764],{"code":2764},"document_index/chunk_content_enrichment.py:11","）把这四个部分",[17,2767,2768],{},"顺序拼接",[229,2770,2773],{"className":2771,"code":2772,"language":1269},[1267],"┌──────────────┐ ┌─────────────┐ ┌──────────────────┐ ┌────────────────┐ ┌─────────────────────┐\n│ title_prefix │ │ doc_summary │ │    content       │ │ chunk_context  │ │metadata_suffix_     │\n│ Q3销售总结\\n\\n│ │ (整篇文档摘要)│ │ 华东区表现突出... │ │ (chunk所在段落)│ │ semantic/keyword   │\n└──────────────┘ └─────────────┘ └──────────────────┘ └────────────────┘ └─────────────────────┘\n    6 token         ~50 token          200 token          ~64 token            12 token\n",[131,2774,2772],{"__ignoreMap":234},[14,2776,2777,2780],{},[17,2778,2779],{},"两版不同的后缀","（同文件:7-12）：",[136,2782,2783,2789],{},[139,2784,2785,2788],{},[131,2786,2787],{"code":2787},"_semantic"," 版（键=值格式）→ 喂给向量嵌入，语义更丰富",[139,2790,2791,2794],{},[131,2792,2793],{"code":2793},"_keyword"," 版（纯值串）→ 喂给 BM25 关键词检索，不浪费 token 在 \"Metadata:\" 这种无意义词上",[14,2796,2797,2798,2801],{},"检索命中后，",[131,2799,2800],{"code":2800},"cleanup_content_for_chunks","（同文件:15-104）反向剥离这些前缀后缀，只把原始正文交还给用户和 LLM。",[1167,2803,2805],{"id":2804},"contextual-rag","Contextual RAG",[14,2807,2808,223,2810,2813,2814,2817,2818,2073],{},[131,2809,805],{"code":805},[131,2811,2812],{"code":2812},"indexing_pipeline.py:992","）在",[17,2815,2816],{},"索引期","（不是检索期）用 LLM 给每个 chunk 补一段\"定位上下文\"。提示词在 ",[131,2819,2820],{"code":2820},"prompts/contextual_retrieval.py",[229,2822,2825],{"className":2823,"code":2824,"language":1269},[1267],"\u003Cdocument>{整篇文档}\u003C/document>\n\u003Cchunk>{当前chunk}\u003C/chunk>\nPlease give a short succinct context to situate this chunk within the overall\ndocument for the purposes of improving search retrieval of the chunk.\n",[131,2826,2824],{"__ignoreMap":234},[14,2828,2829],{},"两件产物：",[136,2831,2832,2838],{},[139,2833,2834,2837],{},[131,2835,2836],{"code":2836},"doc_summary","：整篇文档的摘要（放 chunk 开头）。",[139,2839,2840,2843],{},[131,2841,2842],{"code":2842},"chunk_context","：这个 chunk 在文档里的定位说明（放 chunk 结尾）。",[14,2845,2846,2847,2849,2850,2853,2854,2856],{},"嵌入时把「标题 + doc_summary + content + chunk_context + 元数据」一起编码（",[131,2848,2764],{"code":2764},"），",[17,2851,2852],{},"检索时把这层上下文也带进向量空间","，从而解决\"chunk 脱离上下文后语义模糊\"的经典问题。检索命中后，",[131,2855,2800],{"code":2800},"（同文件:15）再把这些增强文本剥掉，只把原始内容给用户。",[1167,2858,2860],{"id":2859},"embedding-嵌入","Embedding 嵌入",[14,2862,2863,223,2866,1992],{},[131,2864,2865],{"code":2865},"DefaultIndexingEmbedder.embed_chunks",[131,2867,2868],{"code":2868},"indexing/embedder.py:117",[14,2870,2871,2872,2875],{},"1、把每个 chunk 拼成",[17,2873,2874],{},"实际要嵌入的文本","（上面说的增强内容即Contextua RAG）。",[14,2877,2878,2879,2882],{},"2、一批调 ",[131,2880,2881],{"code":2881},"embedding_model.encode(texts, text_type=EmbedTextType.PASSAGE)","。即将多个chunk一起送给Embedded模型，批量转换为向量。",[14,2884,2885],{},"3、文档标题单独嵌入一份（缓存在 dict 里复用）。即同一篇文档的标题只Embedding一次，后续的chunk直接复用这个标题向量。",[14,2887,2888],{},"单独嵌入即：假设 chunk 的增强文本是：",[229,2890,2893],{"className":2891,"code":2892,"language":1269},[1267],"Redis 使用指南\n\n本文介绍 Redis 的基本使用方法。\n\nRedis 支持 String、Hash、List 等数据结构。\n\n当前章节介绍 Redis 数据类型。\n",[131,2894,2892],{"__ignoreMap":234},[14,2896,2897,2898,2901,2902,2905],{},"这一整个东西：",[131,2899,2900],{"code":2900},"标题 + 摘要 + 正文 + 上下文","会生成一个",[131,2903,2904],{"code":2904},"chunk_embedding","。但是系统还是会单独将标题拿出来再算一次，即再进行Embedding。",[14,2907,2908],{},"4、多通道模式：每个 chunk 还会给每个 mini-chunk 单独嵌入。",[14,2910,2911],{},"假设切出来的一个chunk是：",[229,2913,2917],{"className":2914,"code":2915,"language":2916,"meta":234,"style":234},"language-txt shiki shiki-themes catppuccin-latte one-dark-pro","Redis 是一个高性能内存数据库。\nRedis 支持 String、Hash、List、Set 等数据结构。\nRedis 支持主从复制。\nRedis 可以通过 RDB 和 AOF 实现持久化。\nRedis Cluster 可以实现分布式存储。\n","txt",[131,2918,2919,2929,2951,2961,2995],{"__ignoreMap":234},[238,2920,2921,2923,2926],{"class":240,"data-line":241,"line":241},[238,2922,87],{},[238,2924,250],{"class":2925},[249],[238,2927,2928],{},"是一个高性能内存数据库。\n",[238,2930,2931,2933,2936,2939,2942,2945,2948],{"class":240,"data-line":256,"line":256},[238,2932,87],{},[238,2934,250],{"class":2935},[249],[238,2937,2938],{},"支持",[238,2940,250],{"class":2941},[249],[238,2943,2944],{},"String、Hash、List、Set",[238,2946,250],{"class":2947},[249],[238,2949,2950],{},"等数据结构。\n",[238,2952,2953,2955,2958],{"class":240,"data-line":341,"line":341},[238,2954,87],{},[238,2956,250],{"class":2957},[249],[238,2959,2960],{},"支持主从复制。\n",[238,2962,2963,2965,2968,2971,2974,2977,2980,2983,2986,2989,2992],{"class":240,"data-line":383,"line":383},[238,2964,87],{},[238,2966,250],{"class":2967},[249],[238,2969,2970],{},"可以通过",[238,2972,250],{"class":2973},[249],[238,2975,2976],{},"RDB",[238,2978,250],{"class":2979},[249],[238,2981,2982],{},"和",[238,2984,250],{"class":2985},[249],[238,2987,2988],{},"AOF",[238,2990,250],{"class":2991},[249],[238,2993,2994],{},"实现持久化。\n",[238,2996,2997,2999,3002,3005,3008],{"class":240,"data-line":530,"line":530},[238,2998,87],{},[238,3000,250],{"class":3001},[249],[238,3003,3004],{},"Cluster",[238,3006,250],{"class":3007},[249],[238,3009,3010],{},"可以实现分布式存储。\n",[14,3012,3013],{},"正常情况下：",[229,3015,3017],{"className":2914,"code":3016,"language":2916,"meta":234,"style":234},"整个 chunk\n    ↓\nEmbedding\n    ↓\n一个向量\n",[131,3018,3019,3030,3047,3052,3068],{"__ignoreMap":234},[238,3020,3021,3024,3027],{"class":240,"data-line":241,"line":241},[238,3022,3023],{},"整个",[238,3025,250],{"class":3026},[249],[238,3028,3029],{},"chunk\n",[238,3031,3032,3035,3038,3041,3044],{"class":240,"data-line":256,"line":256},[238,3033,250],{"class":3034},[249],[238,3036,250],{"class":3037},[249],[238,3039,250],{"class":3040},[249],[238,3042,250],{"class":3043},[249],[238,3045,3046],{},"↓\n",[238,3048,3049],{"class":240,"data-line":341,"line":341},[238,3050,3051],{},"Embedding\n",[238,3053,3054,3057,3060,3063,3066],{"class":240,"data-line":383,"line":383},[238,3055,250],{"class":3056},[249],[238,3058,250],{"class":3059},[249],[238,3061,250],{"class":3062},[249],[238,3064,250],{"class":3065},[249],[238,3067,3046],{},[238,3069,3070],{"class":240,"data-line":530,"line":530},[238,3071,3072],{},"一个向量\n",[14,3074,3075],{},"但如果用户问：Redis 有哪些持久化方式？真正与该问题相关的其实只有：Redis 可以通过 RDB 和 AOF 实现持久化。但chunk里还有其他信息。",[14,3077,3078],{},"mini-chunk即将刚才的大chunk内部再分：",[229,3080,3082],{"className":2914,"code":3081,"language":2916,"meta":234,"style":234},"大 chunk\n│\n├── mini-chunk1\n│   Redis 是一个高性能内存数据库。\n│\n├── mini-chunk2\n│   Redis 支持 String、Hash、List、Set 等数据结构。\n│\n├── mini-chunk3\n│   Redis 支持主从复制。\n│\n├── mini-chunk4\n│   Redis 可以通过 RDB 和 AOF 实现持久化。\n│\n└── mini-chunk5\n    Redis Cluster 可以实现分布式存储。\n",[131,3083,3084,3094,3099,3110,3131,3135,3145,3175,3179,3189,3209,3213,3223,3263,3267,3278],{"__ignoreMap":234},[238,3085,3086,3089,3092],{"class":240,"data-line":241,"line":241},[238,3087,3088],{},"大",[238,3090,250],{"class":3091},[249],[238,3093,3029],{},[238,3095,3096],{"class":240,"data-line":256,"line":256},[238,3097,3098],{},"│\n",[238,3100,3101,3104,3107],{"class":240,"data-line":341,"line":341},[238,3102,3103],{},"├──",[238,3105,250],{"class":3106},[249],[238,3108,3109],{},"mini-chunk1\n",[238,3111,3112,3115,3118,3121,3124,3126,3129],{"class":240,"data-line":383,"line":383},[238,3113,3114],{},"│",[238,3116,250],{"class":3117},[249],[238,3119,250],{"class":3120},[249],[238,3122,250],{"class":3123},[249],[238,3125,87],{},[238,3127,250],{"class":3128},[249],[238,3130,2928],{},[238,3132,3133],{"class":240,"data-line":530,"line":530},[238,3134,3098],{},[238,3136,3137,3139,3142],{"class":240,"data-line":644,"line":644},[238,3138,3103],{},[238,3140,250],{"class":3141},[249],[238,3143,3144],{},"mini-chunk2\n",[238,3146,3147,3149,3152,3155,3158,3160,3163,3165,3168,3170,3173],{"class":240,"data-line":764,"line":764},[238,3148,3114],{},[238,3150,250],{"class":3151},[249],[238,3153,250],{"class":3154},[249],[238,3156,250],{"class":3157},[249],[238,3159,87],{},[238,3161,250],{"class":3162},[249],[238,3164,2938],{},[238,3166,250],{"class":3167},[249],[238,3169,2944],{},[238,3171,250],{"class":3172},[249],[238,3174,2950],{},[238,3176,3177],{"class":240,"data-line":780,"line":780},[238,3178,3098],{},[238,3180,3181,3183,3186],{"class":240,"data-line":876,"line":876},[238,3182,3103],{},[238,3184,250],{"class":3185},[249],[238,3187,3188],{},"mini-chunk3\n",[238,3190,3191,3193,3196,3199,3202,3204,3207],{"class":240,"data-line":949,"line":949},[238,3192,3114],{},[238,3194,250],{"class":3195},[249],[238,3197,250],{"class":3198},[249],[238,3200,250],{"class":3201},[249],[238,3203,87],{},[238,3205,250],{"class":3206},[249],[238,3208,2960],{},[238,3210,3211],{"class":240,"data-line":1004,"line":1004},[238,3212,3098],{},[238,3214,3215,3217,3220],{"class":240,"data-line":1706,"line":1706},[238,3216,3103],{},[238,3218,250],{"class":3219},[249],[238,3221,3222],{},"mini-chunk4\n",[238,3224,3225,3227,3230,3233,3236,3238,3241,3243,3246,3248,3251,3253,3256,3258,3261],{"class":240,"data-line":1800,"line":1800},[238,3226,3114],{},[238,3228,250],{"class":3229},[249],[238,3231,250],{"class":3232},[249],[238,3234,250],{"class":3235},[249],[238,3237,87],{},[238,3239,250],{"class":3240},[249],[238,3242,2970],{},[238,3244,250],{"class":3245},[249],[238,3247,2976],{},[238,3249,250],{"class":3250},[249],[238,3252,2982],{},[238,3254,250],{"class":3255},[249],[238,3257,2988],{},[238,3259,250],{"class":3260},[249],[238,3262,2994],{},[238,3264,3265],{"class":240,"data-line":1860,"line":1860},[238,3266,3098],{},[238,3268,3269,3272,3275],{"class":240,"data-line":1865,"line":1865},[238,3270,3271],{},"└──",[238,3273,250],{"class":3274},[249],[238,3276,3277],{},"mini-chunk5\n",[238,3279,3280,3283,3286,3289,3292,3294,3297,3299,3302],{"class":240,"data-line":1899,"line":1899},[238,3281,250],{"class":3282},[249],[238,3284,250],{"class":3285},[249],[238,3287,250],{"class":3288},[249],[238,3290,250],{"class":3291},[249],[238,3293,87],{},[238,3295,250],{"class":3296},[249],[238,3298,3004],{},[238,3300,250],{"class":3301},[249],[238,3303,3010],{},[14,3305,3306],{},"然后不光计算：",[229,3308,3310],{"className":2914,"code":3309,"language":2916,"meta":234,"style":234},"整个大 chunk\n      ↓\nchunk_embedding\n",[131,3311,3312,3322,3344],{"__ignoreMap":234},[238,3313,3314,3317,3320],{"class":240,"data-line":241,"line":241},[238,3315,3316],{},"整个大",[238,3318,250],{"class":3319},[249],[238,3321,3029],{},[238,3323,3324,3327,3330,3333,3336,3339,3342],{"class":240,"data-line":256,"line":256},[238,3325,250],{"class":3326},[249],[238,3328,250],{"class":3329},[249],[238,3331,250],{"class":3332},[249],[238,3334,250],{"class":3335},[249],[238,3337,250],{"class":3338},[249],[238,3340,250],{"class":3341},[249],[238,3343,3046],{},[238,3345,3346],{"class":240,"data-line":341,"line":341},[238,3347,3348],{},"chunk_embedding\n",[14,3350,3351],{},"还会计算：",[229,3353,3355],{"className":2914,"code":3354,"language":2916,"meta":234,"style":234},"mini-chunk1 → embedding\nmini-chunk2 → embedding\nmini-chunk3 → embedding\nmini-chunk4 → embedding\nmini-chunk5 → embedding\n",[131,3356,3357,3373,3388,3403,3418],{"__ignoreMap":234},[238,3358,3359,3362,3365,3367,3370],{"class":240,"data-line":241,"line":241},[238,3360,3361],{},"mini-chunk1",[238,3363,250],{"class":3364},[249],[238,3366,630],{},[238,3368,250],{"class":3369},[249],[238,3371,3372],{},"embedding\n",[238,3374,3375,3378,3381,3383,3386],{"class":240,"data-line":256,"line":256},[238,3376,3377],{},"mini-chunk2",[238,3379,250],{"class":3380},[249],[238,3382,630],{},[238,3384,250],{"class":3385},[249],[238,3387,3372],{},[238,3389,3390,3393,3396,3398,3401],{"class":240,"data-line":341,"line":341},[238,3391,3392],{},"mini-chunk3",[238,3394,250],{"class":3395},[249],[238,3397,630],{},[238,3399,250],{"class":3400},[249],[238,3402,3372],{},[238,3404,3405,3408,3411,3413,3416],{"class":240,"data-line":383,"line":383},[238,3406,3407],{},"mini-chunk4",[238,3409,250],{"class":3410},[249],[238,3412,630],{},[238,3414,250],{"class":3415},[249],[238,3417,3372],{},[238,3419,3420,3423,3426,3428,3431],{"class":240,"data-line":530,"line":530},[238,3421,3422],{},"mini-chunk5",[238,3424,250],{"class":3425},[249],[238,3427,630],{},[238,3429,250],{"class":3430},[249],[238,3432,3372],{},[14,3434,3435],{},"于是现在有：一个大向量＋多个小向量",[14,3437,3438],{},[17,3439,3440],{},"即大chunk负责整体语义，mini-chunk负责局部细节",[1167,3442,3444],{"id":3443},"写入opensearch","写入OpenSearch",[14,3446,3447],{},"关于OpenSearch：",[22,3449,3450,3460],{},[25,3451,3452],{},[28,3453,3454,3457],{},[31,3455,3456],{},"工具",[31,3458,3459],{},"定位",[41,3461,3462,3470,3478,3486,3494],{},[28,3463,3464,3467],{},[46,3465,3466],{},"MySQL",[46,3468,3469],{},"关系型数据库",[28,3471,3472,3475],{},[46,3473,3474],{},"Elasticsearch / OpenSearch",[46,3476,3477],{},"搜索引擎 + 搜索数据库",[28,3479,3480,3483],{},[46,3481,3482],{},"Milvus",[46,3484,3485],{},"专门的向量数据库",[28,3487,3488,3491],{},[46,3489,3490],{},"Pinecone",[46,3492,3493],{},"专门的托管向量数据库",[28,3495,3496,3500],{},[46,3497,3498],{},[17,3499,75],{},[46,3501,3502],{},[17,3503,3504],{},"搜索引擎，同时支持很强的向量数据库能力",[14,3506,3507,223,3509,1992],{},[131,3508,964],{"code":964},[131,3510,3511],{"code":3511},"indexing/vector_db_insertion.py:31",[136,3513,3514,3525],{},[139,3515,3516,3517,3520,3521,3524],{},"先整批写，失败再",[17,3518,3519],{},"逐文档写","（保证同一文档的 chunk 一次性写入，因为 ",[131,3522,3523],{"code":3523},"DocumentIndex.index()"," 假设一个文档的 chunk 是连续的）。",[139,3526,3527,3528,219,3531,2073,3534,3537],{},"底层 ",[131,3529,3530],{"code":3530},"OpenSearchDocumentIndex.index",[131,3532,3533],{"code":3533},"_flush_chunks",[17,3535,3536],{},"先删掉该文档的旧 chunk 再写新的","（防止文档变短后留下孤儿 chunk），用 bulk 批量写入，冲突时刷新索引重试。",[229,3539,3541],{"className":2914,"code":3540,"language":2916,"meta":234,"style":234},"整体流程可以概括为：\n\nChunks + Embeddings\n        ↓\n批量写入 OpenSearch\n        ↓\n    是否成功？\n     /     \\\n   是       否\n   ↓        ↓\n完成    按文档逐个重试\n",[131,3542,3543,3548,3552,3569,3597,3607,3635,3652,3690,3728,3768],{"__ignoreMap":234},[238,3544,3545],{"class":240,"data-line":241,"line":241},[238,3546,3547],{},"整体流程可以概括为：\n",[238,3549,3550],{"class":240,"data-line":256,"line":256},[238,3551,1421],{"emptyLinePlaceholder":1420},[238,3553,3554,3557,3560,3563,3566],{"class":240,"data-line":341,"line":341},[238,3555,3556],{},"Chunks",[238,3558,250],{"class":3559},[249],[238,3561,3562],{},"+",[238,3564,250],{"class":3565},[249],[238,3567,3568],{},"Embeddings\n",[238,3570,3571,3574,3577,3580,3583,3586,3589,3592,3595],{"class":240,"data-line":383,"line":383},[238,3572,250],{"class":3573},[249],[238,3575,250],{"class":3576},[249],[238,3578,250],{"class":3579},[249],[238,3581,250],{"class":3582},[249],[238,3584,250],{"class":3585},[249],[238,3587,250],{"class":3588},[249],[238,3590,250],{"class":3591},[249],[238,3593,250],{"class":3594},[249],[238,3596,3046],{},[238,3598,3599,3602,3605],{"class":240,"data-line":530,"line":530},[238,3600,3601],{},"批量写入",[238,3603,250],{"class":3604},[249],[238,3606,1001],{},[238,3608,3609,3612,3615,3618,3621,3624,3627,3630,3633],{"class":240,"data-line":644,"line":644},[238,3610,250],{"class":3611},[249],[238,3613,250],{"class":3614},[249],[238,3616,250],{"class":3617},[249],[238,3619,250],{"class":3620},[249],[238,3622,250],{"class":3623},[249],[238,3625,250],{"class":3626},[249],[238,3628,250],{"class":3629},[249],[238,3631,250],{"class":3632},[249],[238,3634,3046],{},[238,3636,3637,3640,3643,3646,3649],{"class":240,"data-line":764,"line":764},[238,3638,250],{"class":3639},[249],[238,3641,250],{"class":3642},[249],[238,3644,250],{"class":3645},[249],[238,3647,250],{"class":3648},[249],[238,3650,3651],{},"是否成功？\n",[238,3653,3654,3657,3660,3663,3666,3669,3672,3675,3678,3681,3684,3687],{"class":240,"data-line":780,"line":780},[238,3655,250],{"class":3656},[249],[238,3658,250],{"class":3659},[249],[238,3661,250],{"class":3662},[249],[238,3664,250],{"class":3665},[249],[238,3667,250],{"class":3668},[249],[238,3670,3671],{},"/",[238,3673,250],{"class":3674},[249],[238,3676,250],{"class":3677},[249],[238,3679,250],{"class":3680},[249],[238,3682,250],{"class":3683},[249],[238,3685,250],{"class":3686},[249],[238,3688,3689],{},"\\\n",[238,3691,3692,3695,3698,3701,3704,3707,3710,3713,3716,3719,3722,3725],{"class":240,"data-line":876,"line":876},[238,3693,250],{"class":3694},[249],[238,3696,250],{"class":3697},[249],[238,3699,250],{"class":3700},[249],[238,3702,3703],{},"是",[238,3705,250],{"class":3706},[249],[238,3708,250],{"class":3709},[249],[238,3711,250],{"class":3712},[249],[238,3714,250],{"class":3715},[249],[238,3717,250],{"class":3718},[249],[238,3720,250],{"class":3721},[249],[238,3723,250],{"class":3724},[249],[238,3726,3727],{},"否\n",[238,3729,3730,3733,3736,3739,3742,3745,3748,3751,3754,3757,3760,3763,3766],{"class":240,"data-line":949,"line":949},[238,3731,250],{"class":3732},[249],[238,3734,250],{"class":3735},[249],[238,3737,250],{"class":3738},[249],[238,3740,3741],{},"↓",[238,3743,250],{"class":3744},[249],[238,3746,250],{"class":3747},[249],[238,3749,250],{"class":3750},[249],[238,3752,250],{"class":3753},[249],[238,3755,250],{"class":3756},[249],[238,3758,250],{"class":3759},[249],[238,3761,250],{"class":3762},[249],[238,3764,250],{"class":3765},[249],[238,3767,3046],{},[238,3769,3770,3773,3776,3779,3782,3785],{"class":240,"data-line":1004,"line":1004},[238,3771,3772],{},"完成",[238,3774,250],{"class":3775},[249],[238,3777,250],{"class":3778},[249],[238,3780,250],{"class":3781},[249],[238,3783,250],{"class":3784},[249],[238,3786,3787],{},"按文档逐个重试\n",[14,3789,3790],{},[17,3791,3792],{},"1. 优先批量写入，失败后降级为逐文档写入",[14,3794,3795],{},"正常情况下，系统会尽量将多个文档的 chunks 一次性写入 OpenSearch，以减少请求次数并提升索引吞吐量。",[14,3797,3798],{},"例如：",[229,3800,3803],{"className":3801,"code":3802,"language":1269},[1267],"Document A → A1 A2 A3\nDocument B → B1 B2\nDocument C → C1 C2 C3\n",[131,3804,3802],{"__ignoreMap":234},[14,3806,3807],{},"会优先按照一整个批次执行写入：",[229,3809,3812],{"className":3810,"code":3811,"language":1269},[1267],"A1 A2 A3 B1 B2 C1 C2 C3\n              ↓\n          OpenSearch\n",[131,3813,3811],{"__ignoreMap":234},[14,3815,3816],{},"如果整批写入失败，系统不会直接将整个批次判定为失败，而是降级为按文档分别重试：",[229,3818,3821],{"className":3819,"code":3820,"language":1269},[1267],"Document A → 单独写入\nDocument B → 单独写入\nDocument C → 单独写入\n",[131,3822,3820],{"__ignoreMap":234},[14,3824,3825],{},"这样即使某一篇文档存在异常，也不会影响其他文档正常进入索引。",[14,3827,3828],{},"这一设计兼顾了两点：",[136,3830,3831,3834],{},[139,3832,3833],{},"正常情况下通过批量写入提高性能；",[139,3835,3836],{},"异常情况下通过文档级重试实现故障隔离。",[14,3838,3839],{},[17,3840,3841],{},"2. 同一文档的 Chunk 必须连续处理",[14,3843,3844,3846],{},[131,3845,3523],{"code":3523}," 在处理 chunks 时，会假设属于同一篇文档的 chunks 在输入序列中是连续排列的。",[14,3848,3798],{},[229,3850,3853],{"className":3851,"code":3852,"language":1269},[1267],"A1\nA2\nA3\nB1\nB2\nC1\nC2\n",[131,3854,3852],{"__ignoreMap":234},[14,3856,3857,3858,3861],{},"系统在检测到 ",[131,3859,3860],{"code":3860},"document_id"," 从 A 变为 B 时，就可以认为：",[229,3863,3866],{"className":3864,"code":3865,"language":1269},[1267],"Document A 的 chunks 已经全部收齐\n",[131,3867,3865],{"__ignoreMap":234},[14,3869,3870,3871,3873],{},"随后将 A 的 chunks 一次性交给 ",[131,3872,3533],{"code":3533}," 写入 OpenSearch。",[14,3875,3876],{},"因此，输入顺序必须保持：",[229,3878,3881],{"className":3879,"code":3880,"language":1269},[1267],"AAA BBB CCC\n",[131,3882,3880],{"__ignoreMap":234},[14,3884,3885],{},"而不能出现：",[229,3887,3890],{"className":3888,"code":3889,"language":1269},[1267],"A B A C B\n",[131,3891,3889],{"__ignoreMap":234},[14,3893,3894],{},"否则系统可能在尚未收齐某篇文档全部 chunks 时就提前执行写入。",[14,3896,3897],{},[17,3898,3899],{},"3. 更新文档时先删除旧 Chunk，再写入新 Chunk",[14,3901,3902,3905],{},[131,3903,3904],{"code":3904},"OpenSearchDocumentIndex._flush_chunks"," 中一个非常重要的设计是：",[229,3907,3910],{"className":3908,"code":3909,"language":1269},[1267],"先删除旧 chunks\n再写入新 chunks\n",[131,3911,3909],{"__ignoreMap":234},[14,3913,3914],{},"而不是简单覆盖已有记录。",[14,3916,3917],{},"这是为了避免文档更新后产生“孤儿 chunk”。",[14,3919,3920],{},"例如，一篇文档第一次建立索引时被切成 5 个 chunk：",[229,3922,3925],{"className":3923,"code":3924,"language":1269},[1267],"Document A\n\nA1\nA2\nA3\nA4\nA5\n",[131,3926,3924],{"__ignoreMap":234},[14,3928,3929],{},"此时 OpenSearch 中保存了 5 条记录。",[14,3931,3932],{},"后来文档内容被删除了一部分，重新切分后只剩 3 个 chunk：",[229,3934,3937],{"className":3935,"code":3936,"language":1269},[1267],"A1'\nA2'\nA3'\n",[131,3938,3936],{"__ignoreMap":234},[14,3940,3941],{},"如果只覆盖前三个 chunk，那么 OpenSearch 中最终会变成：",[229,3943,3946],{"className":3944,"code":3945,"language":1269},[1267],"A1'  新\nA2'  新\nA3'  新\nA4   旧\nA5   旧\n",[131,3947,3945],{"__ignoreMap":234},[14,3949,3950],{},"其中 A4 和 A5 已经不存在于最新文档中，却仍然可能被检索出来。",[14,3952,3953],{},"因此，系统采用的是：",[229,3955,3958],{"className":3956,"code":3957,"language":1269},[1267],"删除 Document A 的所有旧 chunks\n              ↓\n重新写入 A1' A2' A3'\n",[131,3959,3957],{"__ignoreMap":234},[14,3961,3962],{},"最终 OpenSearch 中只保留当前版本的数据。",[14,3964,3965],{},"这种策略保证了：",[204,3967,3968],{},[14,3969,3970],{},"OpenSearch 中某篇文档对应的 chunks 始终与当前最新版本保持一致。",[14,3972,3973],{},[17,3974,3975],{},"4. 使用 Bulk API 提高写入效率",[14,3977,3978],{},"真正执行插入时，系统不会逐条发送请求，而是通过 OpenSearch Bulk API 批量写入多个 chunk。",[14,3980,3981],{},"如果一个文档包含 100 个 chunk，逐条写入意味着需要大量独立请求：",[229,3983,3986],{"className":3984,"code":3985,"language":1269},[1267],"A1 → 请求一次\nA2 → 请求一次\nA3 → 请求一次\n...\nA100 → 请求一次\n",[131,3987,3985],{"__ignoreMap":234},[14,3989,3990],{},"使用 Bulk API 后，可以将多个写操作合并到一次批量请求中：",[229,3992,3995],{"className":3993,"code":3994,"language":1269},[1267],"A1\nA2\nA3\n...\nA100\n  ↓\nBulk Request\n  ↓\nOpenSearch\n",[131,3996,3994],{"__ignoreMap":234},[14,3998,3999],{},"这样可以显著减少网络请求开销，提高大规模文档索引时的吞吐量。",[14,4001,4002],{},[17,4003,4004],{},"5. 索引冲突时刷新并重试",[14,4006,4007],{},"在删除旧 chunks 并立即写入新 chunks 的过程中，OpenSearch 可能出现索引状态尚未完全同步或版本冲突等情况。",[14,4009,4010],{},"因此，当 bulk 写入出现冲突时，系统会执行：",[229,4012,4015],{"className":4013,"code":4014,"language":1269},[1267],"写入\n ↓\n发生冲突\n ↓\nrefresh index\n ↓\n重新尝试写入\n",[131,4016,4014],{"__ignoreMap":234},[14,4018,4019,4022],{},[131,4020,4021],{"code":4021},"refresh"," 可以让 OpenSearch 更快地暴露最新的索引状态，再进行后续重试。",[14,4024,4025],{},"需要注意的是，系统并不会在每次写入时都主动 refresh，而是在发生冲突时进行补救，因为频繁刷新索引本身也会带来额外性能开销。",[14,4027,4028],{},[17,4029,4030],{},"整体流程",[14,4032,4033],{},"完整的向量写入过程可以概括为：",[229,4035,4038],{"className":4036,"code":4037,"language":1269},[1267],"Embedding 完成\n      ↓\nChunks + Vectors\n      ↓\n批量写入 OpenSearch\n      ↓\n整批失败？\n  /        \\\n否          是\n↓           ↓\n继续      按文档重试\n             ↓\n      DocumentIndex.index()\n             ↓\n      收集同一文档 chunks\n             ↓\n        _flush_chunks()\n             ↓\n    删除该文档旧 chunks\n             ↓\n     Bulk 写入新 chunks\n             ↓\n       发生冲突？\n        /      \\\n      否        是\n      ↓         ↓\n    完成     refresh\n                ↓\n              retry\n",[131,4039,4037],{"__ignoreMap":234},[10,4041,4043],{"id":4042},"阶段二检索召回","阶段二：检索召回",[1167,4045,4046],{"id":4046},"混合检索",[14,4048,4049,4050,4052],{},"单个查询在 OpenSearch 层面是",[17,4051,4046],{},"：向量 kNN + BM25 关键词同时跑，再归一化融合。",[4054,4055,4056],"style",{},"html pre.shiki code .skYY2, html code.shiki .skYY2{--shiki-default:#7C7F93;--shiki-default-font-style:italic;--shiki-dark:#7F848E;--shiki-dark-font-style:italic}html pre.shiki code .sa2x1, html code.shiki .sa2x1{--shiki-default:#4C4F69;--shiki-dark:#ABB2BF}html pre.shiki code .sgT6j, html code.shiki .sgT6j{--shiki-default:#7C7F93;--shiki-dark:#ABB2BF}html pre.shiki code .sqgB4, html code.shiki .sqgB4{--shiki-default:#179299;--shiki-dark:#56B6C2}html pre.shiki code .s3w6o, html code.shiki .s3w6o{--shiki-default:#1E66F5;--shiki-dark:#61AFEF}html pre.shiki code .scfJ_, html code.shiki .scfJ_{--shiki-default:#d20f39;--shiki-dark:#d19a66}html pre.shiki code .sJ4Hi, html code.shiki .sJ4Hi{--shiki-default:#4C4F69;--shiki-dark:#D19A66}html pre.shiki code .sSWcl, html code.shiki .sSWcl{--shiki-default:#8839EF;--shiki-dark:#C678DD}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .sddMY, html code.shiki .sddMY{--shiki-default:#E64553;--shiki-default-font-style:italic;--shiki-dark:#E06C75;--shiki-dark-font-style:italic}html pre.shiki code .sQf5i, html code.shiki .sQf5i{--shiki-default:#D20F39;--shiki-default-font-style:italic;--shiki-dark:#E5C07B;--shiki-dark-font-style:inherit}html pre.shiki code .sw_MA, html code.shiki .sw_MA{--shiki-default:#40A02B;--shiki-dark:#98C379}html pre.shiki code .sFWNk, html code.shiki .sFWNk{--shiki-default:#EA76CB;--shiki-dark:#56B6C2}html pre.shiki code .si0CV, html code.shiki .si0CV{--shiki-default:#fe640b;--shiki-dark:#c678dd}html pre.shiki code .sYQis, html code.shiki .sYQis{--shiki-default:#FE640B;--shiki-dark:#D19A66}",{"title":234,"searchDepth":383,"depth":383,"links":4058},[4059,4060,4061,4068],{"id":12,"depth":341,"text":12},{"id":195,"depth":341,"text":196},{"id":211,"depth":341,"text":212,"children":4062},[4063,4064,4065,4066,4067],{"id":1169,"depth":383,"text":1170},{"id":663,"depth":383,"text":1928},{"id":2804,"depth":383,"text":2805},{"id":2859,"depth":383,"text":2860},{"id":3443,"depth":383,"text":3444},{"id":4042,"depth":341,"text":4043,"children":4069},[4070],{"id":4046,"depth":383,"text":4046},[4072],"实战项目","2026-08-03 16:39:06","OcEasy（Onyx/Danswer）RAG 实现详解。",false,"md",null,{},"/2026/rag","---\ntitle: 企业级 RAG 实现详解\ndescription: OcEasy（Onyx/Danswer）RAG 实现详解。\ndate: 2026-08-03 16:39:06\nupdated: 2026-08-03 16:39:06\nimage: # 图片\ncategories: [实战项目]\ntags: [RAG]\n---\n\n### 项目概览与技术栈\n\n**OcEasy** 是一个 AI 面试准备项目：把职位描述（JD）、个人经历、面试记录转成结构化准备计划、逼真模拟面试和可执行的复盘建议。它内置了一个完整的企业级 RAG 知识库（「面试知识库」），底层就是企业级的Onyx RAG 引擎。\n\n| 组件      | 技术                                                        | 作用                                   |\n| --------- | ----------------------------------------------------------- | -------------------------------------- |\n| 后端      | Python 3.13 / FastAPI / SQLAlchemy / Celery                 | API、任务编排                          |\n| 关系库    | PostgreSQL                                                  | 文档元数据、权限、索引状态（事实来源） |\n| 向量库    | **OpenSearch**（kNN + BM25）                                | 混合检索的文档索引                     |\n| 缓存/消息 | Redis                                                       | 查询嵌入缓存、Celery broker            |\n| 文件存储  | MinIO / S3（blob）                                          | 上传文件的原始内容、图片               |\n| LLM       | LiteLLM 多 provider（OpenAI/Anthropic/本地 model_server）   | 生成、Contextual RAG、查询扩展         |\n| 嵌入模型  | 本地 model_server 或云端 API（OpenAI/Cohere/Voyage/Vertex） | chunk / query 向量化                   |\n\n**几个关键目录**（均在 `backend/onyx/` 下）：\n\n- `connectors/` — 从 40+ 外部系统（Notion、Confluence、Gmail、GitHub…）拉文档\n- `indexing/` — 索引管道：chunker → embedder → 向量库写入\n- `document_index/opensearch/` — OpenSearch 适配层、查询 DSL 构建、schema\n- `context/search/` — 检索编排：pipeline、search_runner、filters\n- `chat/` — 生成阶段：LLM 工具循环、上下文注入、引用处理、压缩\n- `tools/tool_implementations/search/` — **`internal_search` 工具（检索的核心入口）**\n- `secondary_llm_flows/` — 检索前后的小型 LLM 流程（查询扩展、文档筛选）\n- `prompts/` — 所有提示词模板\n\n### RAG完整请求流\n\n![img](https://assets.yangzy.top/rag.webp)\n\n> 索引阶段把海量文档切成带向量的 chunk 存进 OpenSearch；检索阶段把「一句话提问」变成「多路查询 → 混合检索 → LLM 精筛」；生成阶段让 LLM 自主决定何时检索，并把检索结果以带编号的 JSON 形式喂回去，最终产出带引用的回答。\n\n### 阶段一：文档入库与索引构建\n\n入口 `run_indexing_pipeline` → `index_doc_batch`（`indexing/indexing_pipeline.py:1267`），对一批文档依次执行。完整顺序：\n\n```python\n# indexing_pipeline.py:1314-1524（已简化）\nfiltered_docs, failures = filter_fnc(document_batch)          # 1. 过滤空/超长文档\nfiltered_docs = _apply_document_ingestion_hook(filtered_docs)  # 2. 可扩展的接入钩子\ncontext = adapter.prepare(...)                                  # 3. 元数据入库（Postgres）\nindexable_docs = process_image_sections(...)                    # 4. 图片 → LLM 视觉摘要\nchunks = chunker.chunk(indexable_docs)                          # 5. 切块\nif enable_contextual_rag:\n    chunks = add_contextual_summaries(chunks, llm, ...)         # 6. Contextual RAG（可选）\nwith embed_and_stream(chunks, embedder, ...) as (result, store):# 7. 批量嵌入\n    write_chunks_to_vector_db_with_backoff(document_index, ...) # 8. 写入 OpenSearch\n    adapter.post_index(...)                                     # 9. 更新 chunk 计数/哈希\n```\n\n#### Document 预处理（进入切块前）\n\n`filter_documents`（`indexing_pipeline.py:617`）在切块前做三道门槛：\n\n1. **无标题无内容** → 跳过（否则没有可检索信息）。\n2. **超长文档**（`MAX_DOCUMENT_CHARS`，默认 512MB 字符）→ 跳过并记录失败，防止后面内存爆掉。\n3. **图片 section**（`process_image_sections`，`indexing_pipeline.py:723`）：用视觉 LLM 把图片转成文字摘要存进 `section.text`，这样图片内容也能被检索。\n\n另外有一个**增量去重**机制 `get_docs_to_update`（`indexing_pipeline.py:339`），两道闸门：\n\n- **时间戳闸门**：connector 给的 `doc_updated_at` 没推进 → 直接跳过（快路径）。\n- **内容哈希闸门**：时间没变时算 `content_hash()` 比对库里存的哈希，相同则跳过。\n- 只有真正变化的文档才走后面的切块+嵌入，极大节省算力。\n\n每个 connector（Notion、Google Drive 等）会定期轮询——每次可能拉回几万个文档。但其中 95% 都没改动过。如果每个文档都重新走「切块 → 嵌入 → 写 OpenSearch」，浪费大量算力和 API 费用。所以需要一种方法快速判断\"这个文档和上次索引时是否完全一样\"。\n\n**两道闸门如何协作**\n\n对于 connector 拉回来的每一篇文档，依次经过两道检查：\n\n                            ┌────────────────────┐\n                              │   这个文档带了       │\n                              │   doc_updated_at？  │\n                              └────────┬───────────┘\n                                       │\n                          ┌────────────┼────────────┐\n                          │ 没带        │            │ 带了\n                          ▼             │            ▼\n                     跳过闸门1          │   和数据库里旧的 doc_updated_at 比较\n                     （往下走）          │\n                                        │     ┌──────────┬──────────┐\n                                        │     │ 新 > 旧   │ 新 ≤ 旧  │\n                                        │     ▼           ▼\n                                        │  \"时间推进了\"   \"时间没变\"\n                                        │  认定有变化！   大概率没改\n                                        │  直接排队重索引   直接跳过\n                                        │  跳过闸门2       （0次哈希计算，\n                                        │  （不算哈希）     最快路径）\n                                        │\n                    ┌───────────────────┘\n                    ▼\n            ┌───────────────┐\n            │ 闸门 2：算哈希 │  ← 到这里的文档：要么没带时间戳，要么带时间戳但闸门1放行了\n            │ content_hash() │\n            └───────┬───────┘\n                    │\n         ┌──────────┼──────────┐\n         │ 哈希和库里的相同     │ 哈希不同\n         ▼                     ▼\n      \"内容确实没变\"         \"内容确实变了\"\n        跳过                 排队重索引\n\n源码实现：\n\n```python\nfor doc in documents:\n    # 判断：新时间戳 > 数据库里存的旧时间戳？\n    timestamp_advanced = (新时间戳 存在 and 旧时间戳 存在 and 新 > 旧)\n\n    # 闸门1：有时间戳、时间没推进 → 跳过（0 次哈希计算，最快）\n    if doc 有时间戳 and 时间没推进:\n        continue  # ← 什么都不做，下一个文档\n\n    # 闸门2：只在\"时间戳没推进\"或\"根本没时间戳\"时启用\n    content_hash = doc.content_hash()                   # ← 算哈希\n    if 时间没推进:\n        if 数据库里的旧哈希 == content_hash:           # ← 比对\n            continue  # ← 时间也说你没变，跳\n\n    # 走到这里：时间戳或哈希说\"变了\" → 排队重索引\n    updatable_docs.append(doc)\n```\n\n#### Chunk\n\n切块不是把文章按字符数硬裁，而是一套**三层协作**的流程。\n\n**三层结构总览**\n\n```\nChunker                          (chunker.py:124)   ← 顶层编排，算 token 预算\n  └─ DocumentChunker             (chunking/document_chunker.py:25)  ← 迭代 section，调度切块器\n       ├─ TextChunker            (text_section_chunker.py:22)       ← 正文切块（最重要）\n       ├─ ImageChunker           (image_section_chunker.py)         ← 图片 section\n       └─ TabularChunker         (tabular_section_chunker)          ← CSV/表格 section\n```\n\n核心类 `Chunker`（`indexing/chunker.py:124`），用 `chonkie.SentenceChunker` 按**句子感知**的方式切块（不是硬切字符，避免把一句话拦腰斩断）。\n\n**第一步：Chunker算预算**\n\n拿到一个文档后，第一件事不是切，而是**算清楚有多少 token 配额分给正文**。这个决定直接影响最终的文本布局。\n\n用一个具体例子——一篇 Notion 页面《Q3 销售总结》：\n\n```\n文档标题: \"Q3 销售总结\"\n文档作者: \"张三\"\n文档标签: [\"销售\", \"Q3\", \"2025\"]\n正文 section1: \"本季度整体营收增长 20%，其中...\"  ← 这段有 300 token\n正文 section2: \"华东区表现突出，达成了...\"       ← 这段有 400 token\n正文 section3: \"总结与展望如下...\"               ← 这段有 180 token\n```\n\n运算过程（`_handle_single_document`）：\n\n```\n① 标题截断为 blurb（128 token）→ title_prefix = \"Q3 销售总结\\n\\n\"\n   title_tokens = len(tokenizer(\"Q3 销售总结\\n\\n\")) = 6\n\n② 元数据拼成两版后缀（(chunker.py:209-219)）：\n   metadata_suffix_semantic = \"\\nMetadata:\\n\\t作者 - 张三\\n\\t标签 - 销售, Q3, 2025\"\n   metadata_suffix_keyword  = \"\\n张三 销售 Q3 2025\"  ← 只取值，没有键名\n   metadata_tokens = 12\n\n③ 是否启动 Contextual RAG？\n   - 全文 token = 300+400+180 = 880\n   - 880 + 6(title) + 12(metadata) = 898 > 512 ← 一整个 chunk 放不下，需要 Contextual RAG\n   - context_size = DEFAULT_CONTEXTUAL_RAG_RESERVED_TOKENS = 128\n\n④ content_token_limit = 512 - 6 - 12 - 128 = 366 token\n\n⑤ 自我兜底检查：\n   - 366 ≤ 256(CHUNK_MIN_CONTENT)？ 否，够用，通过\n   - 如果不够 → 放弃 Contextual RAG，也不要元数据前缀\n   - 最后手段 → content_token_limit 退回 512，什么都不要，只存裸正文\n```\n\n**第二步：`TextChunker` 切正文**\n\n正文 section 不是每段独立切——**它有一个跨 section 累积器**，目的是尽量把相邻的小 section 拼进同一个 chunk，避免碎片化。\n\n```\n初始化: accumulator = \"\"（空缓冲）\n\n处理 section1（300 token）:\n  300 ≤ 366 → 放得下，accumulator = \"本季度整体营收增长 20%，其中...\"\n\n处理 section2（400 token）:\n  accumulator 已有 300 token + section2 有 400 token = 700 > 366\n  → 放不下！先把 accumulator 吐出为一个 chunk：Chunk1 = \"本季度整体营收...\"\n  → 然后 section2 自己有 400 > 366 → 超大！走 _handle_oversized_section\n\n      处理超大 section2（_handle_oversized_section, line 82）：\n        用 SentenceChunker 按句子感知把这 400 token 切分：\n        → chunk2a = \"华东区表现突出，达成了目标 A。\"       (200 token)\n        → chunk2b = \"此外在 B 领域也取得突破。\"           (190 token)\n        每个子块成为一个独立的 ChunkPayload\n\n处理 section3（180 token）:\n  accumulator 现在是空的，180 ≤ 366 → accumulator = \"总结与展望如下...\"\n\n最后 flush: 出来 → Chunk3 = \"总结与展望如下...\"\n```\n\n**关键机制——SentenceChunker**（来自 `chonkie` 库）：它不是按字符硬裁，而是按句子边界切——尽量在句号/换行处下刀。如果 `STRICT_CHUNK_TOKEN_LIMIT` 开启，切完后还超限的分片会被 `split_text_by_tokens` 强制硬裁（`text_section_chunker.py:93-107`）。\n\n**第三步：`ChunkPayload` 升级为 `DocAwareChunk`（section_chunker.py:41-67）**\n\n上面每个 `ChunkPayload(text, links, ...)` 在 `DocumentChunker.chunk`（document_chunker.py:68-79）里被升级成完整的 `DocAwareChunk`，这一行代码做的是把所有文档级别的信息**复制粘贴到每个 chunk 上**：\n\n```python\n# section_chunker.py:51-67（已简化）\nreturn DocAwareChunk(\n    content=self.text,                              # ← body 正文\n    title_prefix=\"Q3 销售总结\\n\\n\",                  # ← 每个 chunk 都带同样的标题\n    metadata_suffix_semantic=\"\\nMetadata:\\n...\",    # ← 每个 chunk 都带同样的元数据\n    metadata_suffix_keyword=\"\\n张三 销售 Q3 2025\",  # ← 关键词版元数据\n    blurb=extract_blurb(self.text),                 # ← 首 128 token 做预览\n    source_links={0: \"http://...\", 200: \"http://...\"},  # ← 点击引用直达原文\n    mini_chunk_texts=[\"华东区表现...\", \"此外在B领域...\"],  # ← 15-token 微块\n)\n```\n\n**第四步：存进 OpenSearch 时的最终拼装**\n\n索引写到 `content` 字段的不是原始正文——`generate_enriched_content_for_chunk_embedding`（`document_index/chunk_content_enrichment.py:11`）把这四个部分**顺序拼接**：\n\n```\n┌──────────────┐ ┌─────────────┐ ┌──────────────────┐ ┌────────────────┐ ┌─────────────────────┐\n│ title_prefix │ │ doc_summary │ │    content       │ │ chunk_context  │ │metadata_suffix_     │\n│ Q3销售总结\\n\\n│ │ (整篇文档摘要)│ │ 华东区表现突出... │ │ (chunk所在段落)│ │ semantic/keyword   │\n└──────────────┘ └─────────────┘ └──────────────────┘ └────────────────┘ └─────────────────────┘\n    6 token         ~50 token          200 token          ~64 token            12 token\n```\n\n**两版不同的后缀**（同文件:7-12）：\n\n- `_semantic` 版（键=值格式）→ 喂给向量嵌入，语义更丰富\n- `_keyword` 版（纯值串）→ 喂给 BM25 关键词检索，不浪费 token 在 \"Metadata:\" 这种无意义词上\n\n检索命中后，`cleanup_content_for_chunks`（同文件:15-104）反向剥离这些前缀后缀，只把原始正文交还给用户和 LLM。\n\n#### Contextual RAG\n\n`add_contextual_summaries`（`indexing_pipeline.py:992`）在**索引期**（不是检索期）用 LLM 给每个 chunk 补一段\"定位上下文\"。提示词在 `prompts/contextual_retrieval.py`：\n\n```\n\u003Cdocument>{整篇文档}\u003C/document>\n\u003Cchunk>{当前chunk}\u003C/chunk>\nPlease give a short succinct context to situate this chunk within the overall\ndocument for the purposes of improving search retrieval of the chunk.\n```\n\n两件产物：\n\n- `doc_summary`：整篇文档的摘要（放 chunk 开头）。\n- `chunk_context`：这个 chunk 在文档里的定位说明（放 chunk 结尾）。\n\n嵌入时把「标题 + doc_summary + content + chunk_context + 元数据」一起编码（`document_index/chunk_content_enrichment.py:11`），**检索时把这层上下文也带进向量空间**，从而解决\"chunk 脱离上下文后语义模糊\"的经典问题。检索命中后，`cleanup_content_for_chunks`（同文件:15）再把这些增强文本剥掉，只把原始内容给用户。\n\n#### Embedding 嵌入\n\n`DefaultIndexingEmbedder.embed_chunks`（`indexing/embedder.py:117`）：\n\n1、把每个 chunk 拼成**实际要嵌入的文本**（上面说的增强内容即Contextua RAG）。\n\n2、一批调 `embedding_model.encode(texts, text_type=EmbedTextType.PASSAGE)`。即将多个chunk一起送给Embedded模型，批量转换为向量。\n\n3、文档标题单独嵌入一份（缓存在 dict 里复用）。即同一篇文档的标题只Embedding一次，后续的chunk直接复用这个标题向量。\n\n单独嵌入即：假设 chunk 的增强文本是：\n\n```\nRedis 使用指南\n\n本文介绍 Redis 的基本使用方法。\n\nRedis 支持 String、Hash、List 等数据结构。\n\n当前章节介绍 Redis 数据类型。\n```\n\n这一整个东西：`标题 + 摘要 + 正文 + 上下文`会生成一个`chunk_embedding`。但是系统还是会单独将标题拿出来再算一次，即再进行Embedding。\n\n4、多通道模式：每个 chunk 还会给每个 mini-chunk 单独嵌入。\n\n假设切出来的一个chunk是：\n\n```txt\nRedis 是一个高性能内存数据库。\nRedis 支持 String、Hash、List、Set 等数据结构。\nRedis 支持主从复制。\nRedis 可以通过 RDB 和 AOF 实现持久化。\nRedis Cluster 可以实现分布式存储。\n```\n\n正常情况下：\n\n```txt\n整个 chunk\n    ↓\nEmbedding\n    ↓\n一个向量\n```\n\n但如果用户问：Redis 有哪些持久化方式？真正与该问题相关的其实只有：Redis 可以通过 RDB 和 AOF 实现持久化。但chunk里还有其他信息。\n\nmini-chunk即将刚才的大chunk内部再分：\n\n```txt\n大 chunk\n│\n├── mini-chunk1\n│   Redis 是一个高性能内存数据库。\n│\n├── mini-chunk2\n│   Redis 支持 String、Hash、List、Set 等数据结构。\n│\n├── mini-chunk3\n│   Redis 支持主从复制。\n│\n├── mini-chunk4\n│   Redis 可以通过 RDB 和 AOF 实现持久化。\n│\n└── mini-chunk5\n    Redis Cluster 可以实现分布式存储。\n```\n\n然后不光计算：\n\n```txt\n整个大 chunk\n      ↓\nchunk_embedding\n```\n\n还会计算：\n\n```txt\nmini-chunk1 → embedding\nmini-chunk2 → embedding\nmini-chunk3 → embedding\nmini-chunk4 → embedding\nmini-chunk5 → embedding\n```\n\n于是现在有：一个大向量＋多个小向量\n\n**即大chunk负责整体语义，mini-chunk负责局部细节**\n\n#### 写入OpenSearch\n\n关于OpenSearch：\n\n| 工具                       | 定位                                       |\n| -------------------------- | ------------------------------------------ |\n| MySQL                      | 关系型数据库                               |\n| Elasticsearch / OpenSearch | 搜索引擎 + 搜索数据库                      |\n| Milvus                     | 专门的向量数据库                           |\n| Pinecone                   | 专门的托管向量数据库                       |\n| **OpenSearch**             | **搜索引擎，同时支持很强的向量数据库能力** |\n\n`write_chunks_to_vector_db_with_backoff`（`indexing/vector_db_insertion.py:31`）：\n\n- 先整批写，失败再**逐文档写**（保证同一文档的 chunk 一次性写入，因为 `DocumentIndex.index()` 假设一个文档的 chunk 是连续的）。\n\n- 底层 `OpenSearchDocumentIndex.index` → `_flush_chunks`：**先删掉该文档的旧 chunk 再写新的**（防止文档变短后留下孤儿 chunk），用 bulk 批量写入，冲突时刷新索引重试。\n\n```txt\n整体流程可以概括为：\n\nChunks + Embeddings\n        ↓\n批量写入 OpenSearch\n        ↓\n    是否成功？\n     /     \\\n   是       否\n   ↓        ↓\n完成    按文档逐个重试\n```\n\n**1. 优先批量写入，失败后降级为逐文档写入**\n\n正常情况下，系统会尽量将多个文档的 chunks 一次性写入 OpenSearch，以减少请求次数并提升索引吞吐量。\n\n例如：\n\n```\nDocument A → A1 A2 A3\nDocument B → B1 B2\nDocument C → C1 C2 C3\n```\n\n会优先按照一整个批次执行写入：\n\n```\nA1 A2 A3 B1 B2 C1 C2 C3\n              ↓\n          OpenSearch\n```\n\n如果整批写入失败，系统不会直接将整个批次判定为失败，而是降级为按文档分别重试：\n\n```\nDocument A → 单独写入\nDocument B → 单独写入\nDocument C → 单独写入\n```\n\n这样即使某一篇文档存在异常，也不会影响其他文档正常进入索引。\n\n这一设计兼顾了两点：\n\n- 正常情况下通过批量写入提高性能；\n- 异常情况下通过文档级重试实现故障隔离。\n\n**2. 同一文档的 Chunk 必须连续处理**\n\n`DocumentIndex.index()` 在处理 chunks 时，会假设属于同一篇文档的 chunks 在输入序列中是连续排列的。\n\n例如：\n\n```\nA1\nA2\nA3\nB1\nB2\nC1\nC2\n```\n\n系统在检测到 `document_id` 从 A 变为 B 时，就可以认为：\n\n```\nDocument A 的 chunks 已经全部收齐\n```\n\n随后将 A 的 chunks 一次性交给 `_flush_chunks` 写入 OpenSearch。\n\n因此，输入顺序必须保持：\n\n```\nAAA BBB CCC\n```\n\n而不能出现：\n\n```\nA B A C B\n```\n\n否则系统可能在尚未收齐某篇文档全部 chunks 时就提前执行写入。\n\n**3. 更新文档时先删除旧 Chunk，再写入新 Chunk**\n\n`OpenSearchDocumentIndex._flush_chunks` 中一个非常重要的设计是：\n\n```\n先删除旧 chunks\n再写入新 chunks\n```\n\n而不是简单覆盖已有记录。\n\n这是为了避免文档更新后产生“孤儿 chunk”。\n\n例如，一篇文档第一次建立索引时被切成 5 个 chunk：\n\n```\nDocument A\n\nA1\nA2\nA3\nA4\nA5\n```\n\n此时 OpenSearch 中保存了 5 条记录。\n\n后来文档内容被删除了一部分，重新切分后只剩 3 个 chunk：\n\n```\nA1'\nA2'\nA3'\n```\n\n如果只覆盖前三个 chunk，那么 OpenSearch 中最终会变成：\n\n```\nA1'  新\nA2'  新\nA3'  新\nA4   旧\nA5   旧\n```\n\n其中 A4 和 A5 已经不存在于最新文档中，却仍然可能被检索出来。\n\n因此，系统采用的是：\n\n```\n删除 Document A 的所有旧 chunks\n              ↓\n重新写入 A1' A2' A3'\n```\n\n最终 OpenSearch 中只保留当前版本的数据。\n\n这种策略保证了：\n\n> OpenSearch 中某篇文档对应的 chunks 始终与当前最新版本保持一致。\n\n**4. 使用 Bulk API 提高写入效率**\n\n真正执行插入时，系统不会逐条发送请求，而是通过 OpenSearch Bulk API 批量写入多个 chunk。\n\n如果一个文档包含 100 个 chunk，逐条写入意味着需要大量独立请求：\n\n```\nA1 → 请求一次\nA2 → 请求一次\nA3 → 请求一次\n...\nA100 → 请求一次\n```\n\n使用 Bulk API 后，可以将多个写操作合并到一次批量请求中：\n\n```\nA1\nA2\nA3\n...\nA100\n  ↓\nBulk Request\n  ↓\nOpenSearch\n```\n\n这样可以显著减少网络请求开销，提高大规模文档索引时的吞吐量。\n\n**5. 索引冲突时刷新并重试**\n\n在删除旧 chunks 并立即写入新 chunks 的过程中，OpenSearch 可能出现索引状态尚未完全同步或版本冲突等情况。\n\n因此，当 bulk 写入出现冲突时，系统会执行：\n\n```\n写入\n ↓\n发生冲突\n ↓\nrefresh index\n ↓\n重新尝试写入\n```\n\n`refresh` 可以让 OpenSearch 更快地暴露最新的索引状态，再进行后续重试。\n\n需要注意的是，系统并不会在每次写入时都主动 refresh，而是在发生冲突时进行补救，因为频繁刷新索引本身也会带来额外性能开销。\n\n**整体流程**\n\n完整的向量写入过程可以概括为：\n\n```\nEmbedding 完成\n      ↓\nChunks + Vectors\n      ↓\n批量写入 OpenSearch\n      ↓\n整批失败？\n  /        \\\n否          是\n↓           ↓\n继续      按文档重试\n             ↓\n      DocumentIndex.index()\n             ↓\n      收集同一文档 chunks\n             ↓\n        _flush_chunks()\n             ↓\n    删除该文档旧 chunks\n             ↓\n     Bulk 写入新 chunks\n             ↓\n       发生冲突？\n        /      \\\n      否        是\n      ↓         ↓\n    完成     refresh\n                ↓\n              retry\n```\n\n### 阶段二：检索召回\n\n#### 混合检索\n\n单个查询在 OpenSearch 层面是**混合检索**：向量 kNN + BM25 关键词同时跑，再归一化融合。\n",{"text":4082,"minutes":4083,"time":4084,"words":4085},"21 min read",20.83,1249800,4166,{"title":5,"description":4074},{"loc":4079},"posts/2026/RAG",[4090],"RAG","A-tC6UWXE_r2nvkWTJ_GSjiLAL91S9zV2lsX6ncQ9Uk",[4093,4077],{"title":4094,"path":4095,"stem":4096,"date":4097,"type":4077,"children":-1},"SSE","/2026/sse","posts/2026/SSE","2026-08-03 12:22:56",1795263893448]