{"id":8469,"date":"2025-10-07T07:11:50","date_gmt":"2025-10-07T07:11:50","guid":{"rendered":"https:\/\/pokecon.jp\/job\/?p=8469"},"modified":"2025-10-07T07:11:50","modified_gmt":"2025-10-07T07:11:50","slug":"databricks-delta%e3%83%86%e3%83%bc%e3%83%96%e3%83%ab%e3%81%8b%e3%82%89vector-search%e3%81%a8faiss%e3%81%a7%e3%82%a4%e3%83%b3%e3%83%87%e3%83%83%e3%82%af%e3%82%b9%e3%82%92%e4%bd%9c%e6%88%90%e3%81%97","status":"publish","type":"post","link":"https:\/\/pokecon.jp\/job\/8469\/","title":{"rendered":"Databricks Delta\u30c6\u30fc\u30d6\u30eb\u304b\u3089Vector Search\u3068Faiss\u3067\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u3092\u4f5c\u6210\u3057\u3066\u307f\u308b – CCCMK\u30db\u30fc\u30eb\u30c7\u30a3\u30f3\u30b0\u30b9 TECH LAB\u306e Tech Blog"},"content":{"rendered":"\n<\/p>\n
\n

<\/span><\/p>\n

\u3053\u3093\u306b\u3061\u306f\u3001CCCMK\u30db\u30fc\u30eb\u30c7\u30a3\u30f3\u30b0\u30b9AI\u30a8\u30f3\u30b8\u30cb\u30a2\u306e\u4e09\u6d66\u3067\u3059\u3002<\/p>\n

\u3060\u3093\u3060\u3093\u904e\u3054\u3057\u3084\u3059\u304f\u306a\u3063\u3066\u304d\u3066\u3001\u306a\u3093\u3060\u304b\u3069\u3053\u304b\u306b\u884c\u304d\u305f\u3044\u306a\u3001\u3068\u3044\u3046\u6c17\u6301\u3061\u306b\u306a\u308b\u3053\u3068\u304c\u5897\u3048\u3066\u304d\u307e\u3057\u305f\u3002\u81ea\u5206\u304c\u77e5\u3089\u306a\u3044\u5834\u6240\u306b\u884c\u3063\u305f\u308a\u77e5\u3089\u306a\u3044\u4eba\u306b\u51fa\u4f1a\u3063\u305f\u308a\u3059\u308b\u3053\u3068\u306f\u3001\u81ea\u5206\u306b\u3068\u3063\u3066\u3068\u3066\u3082\u3088\u3044\u7d4c\u9a13\u306a\u3093\u3060\u3068\u601d\u3063\u3066\u3044\u307e\u3059\u3002<\/p>\n

\u6700\u8fd1\u3001Databricks\u3067\u975e\u69cb\u9020\u5316\u30c7\u30fc\u30bf\u306e\u6271\u3044\u65b9\u3092\u3044\u308d\u3044\u308d\u3068\u8abf\u3079\u3066\u3044\u3066\u3001\u305d\u306e\u4e2d\u3067Delta\u30c6\u30fc\u30d6\u30eb\u306b\u683c\u7d0d\u3055\u308c\u305f\u30c9\u30ad\u30e5\u30e1\u30f3\u30c8\u30c7\u30fc\u30bf\u3092\u30d9\u30af\u30c8\u30eb\u691c\u7d22\u3059\u308b\u305f\u3081\u306b\u30012\u7a2e\u985e\u306e\u691c\u7d22\u30a8\u30f3\u30b8\u30f3\uff08Databricks Vector Search\u3068Faiss\uff09\u3092\u4f7f\u3063\u3066\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u3092\u69cb\u7bc9\u3059\u308b\u65b9\u6cd5\u3092\u8a66\u3057\u3066\u3044\u307e\u3057\u305f\u3002<\/p>\n

\u4eca\u56de\u306fDatabricks Vector Search\u3068Faiss\u305d\u308c\u305e\u308c\u3067\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u3092\u69cb\u7bc9\u3057\u3001\u30af\u30a8\u30ea\u3092\u5b9f\u884c\u3059\u308b\u307e\u3067\u306e\u6d41\u308c\u306b\u3064\u3044\u3066\u307e\u3068\u3081\u3066\u307f\u305f\u3044\u3068\u601d\u3044\u307e\u3059\u3002<\/p>\n

\u5143\u306b\u306a\u308bDelta\u30c6\u30fc\u30d6\u30eb<\/h3>\n

\u30c1\u30fc\u30e0\u5185\u306eLT\u4f1a\u5411\u3051\u306b\u4f5c\u6210\u3057\u305fPPTX\u30d5\u30a1\u30a4\u30eb\u304b\u3089\u3001MarkItDown<\/strong>\u3068\u3044\u3046Microsoft\u304c\u958b\u767a\u3057\u305f\u3001\u3055\u307e\u3056\u307e\u306a\u30d5\u30a1\u30a4\u30eb\u5f62\u5f0f\uff08PowerPoint\u3001PDF\u3001\u753b\u50cf\u306a\u3069\uff09\u3092Markdown\u5f62\u5f0f\u306b\u5909\u63db\u3059\u308b\u305f\u3081\u306e\u30c4\u30fc\u30eb\u3092\u4f7f\u3063\u3066\u30c6\u30ad\u30b9\u30c8\u60c5\u5831\u3092\u62bd\u51fa\u3057\u3001\u30b9\u30e9\u30a4\u30c9\u3054\u3068\u306b\u30ec\u30b3\u30fc\u30c9\u3092\u5206\u5272\u3057\u3066Delta\u30c6\u30fc\u30d6\u30eb\u306b\u683c\u7d0d\u3057\u307e\u3057\u305f\u3002<\/p>\n

<\/span>
pptx\u306e\u30c6\u30ad\u30b9\u30c8\u60c5\u5831\u3092\u683c\u7d0d\u3057\u305fDelta\u30c6\u30fc\u30d6\u30eb<\/figcaption><\/figure>\n<\/p>\n

\u3053\u306eDelta\u30c6\u30fc\u30d6\u30eb\u3092\u8d77\u70b9\u306b\u3057\u3001\u57cb\u3081\u8fbc\u307f\u30d9\u30af\u30c8\u30eb\u306e\u51fa\u529b\u3092\u3057\u305f\u306e\u3061\u3001Vector Search\u3068Faiss\u3067\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u3092\u69cb\u7bc9\u3057\u307e\u3059\u3002<\/p>\n

\u57cb\u3081\u8fbc\u307f\u30d9\u30af\u30c8\u30eb\u306e\u51fa\u529b<\/h3>\n

\u3069\u3061\u3089\u306e\u30a8\u30f3\u30b8\u30f3\u3082\u30d9\u30af\u30c8\u30eb\u691c\u7d22\u3092\u884c\u3046\u305f\u3081\u3001\u57cb\u3081\u8fbc\u307f\u30e2\u30c7\u30eb\u3092\u4f7f\u3063\u3066\u30c6\u30ad\u30b9\u30c8\u30c7\u30fc\u30bf\u306e\u57cb\u3081\u8fbc\u307f\u30d9\u30af\u30c8\u30eb\u3092\u51fa\u529b\u3057\u3066\u304a\u304d\u307e\u3059\u3002Databricks\u306fFoundation Model\u3067\u57cb\u3081\u8fbc\u307f\u30e2\u30c7\u30eb\u3092\u63d0\u4f9b\u3057\u3066\u3044\u307e\u3059\u3002\u305d\u308c\u3089\u306e\u30e2\u30c7\u30eb\u306fAISQL\u306e1\u3064ai_query<\/code>\u95a2\u6570\u3067\u547c\u3073\u51fa\u3059\u3053\u3068\u304c\u51fa\u6765\u308b\u305f\u3081\u3001\u4ee5\u4e0b\u306e\u3088\u3046\u306a\u30b7\u30f3\u30d7\u30eb\u306aSQL\u3067\u5b9f\u73fe\u3059\u308b\u3053\u3068\u304c\u53ef\u80fd\u3067\u3059\u3002<\/p>\n

spark.sql(f\"\"\"<\/span>\nCREATE OR REPLACE TABLE <\/span>\n    {catalog_name}.{schema_name}.{emb_table} <\/span>\nTBLPROPERTIES (delta.enableChangeDataFeed = true) AS<\/span>\nSELECT<\/span>\n    *,<\/span>\n    ai_query(<\/span>\"<\/span>databricks-bge-large-en\"<\/span>, content) as embedding<\/span>\nFROM<\/span>\n    {catalog_name}.{schema_name}.{source_table}<\/span>\n\"\"\"<\/span>)\n<\/pre>\n
\u3053\u306e\u30af\u30a8\u30ea\u3092\u5b9f\u884c\u3059\u308b\u3068\u3001\u5143\u306e\u30c6\u30fc\u30d6\u30eb\u306b\u30c6\u30ad\u30b9\u30c8\u306e\u57cb\u3081\u8fbc\u307f\u30d9\u30af\u30c8\u30eb\u304c\u683c\u7d0d\u3055\u308c\u305f”embedding”\u3068\u3044\u3046\u30ab\u30e9\u30e0\u304c\u8ffd\u52a0\u3055\u308c\u305fDelta\u30c6\u30fc\u30d6\u30eb\u304c\u65b0\u898f\u4f5c\u6210\u3055\u308c\u307e\u3059\u3002
\nTBLPROPERTIES (delta.enableChangeDataFeed = true)<\/code>\u306fDatabricks Vector Search\u3092\u4f5c\u308b\u969b\u306b\u5fc5\u8981\u306b\u306a\u308b\u8a2d\u5b9a\u3067\u3059\u3002<\/p>\n
\u307e\u305f\u3001\u4eca\u56de\u4f7f\u7528\u3057\u305fFoundation Model\u306e\u57cb\u3081\u8fbc\u307f\u30e2\u30c7\u30eb\u306f”databricks-bge-large-en”\u3067\u3001\u3053\u306e\u30e2\u30c7\u30eb\u306f1,024\u6b21\u5143\u306e\u57cb\u3081\u8fbc\u307f\u30d9\u30af\u30c8\u30eb\u3092\u751f\u6210\u3059\u308b\u3053\u3068\u304c\u51fa\u6765\u307e\u3059\u3002<\/p>\n
\u57cb\u3081\u8fbc\u307f\u30d9\u30af\u30c8\u30eb\u51fa\u529b\u7528\u95a2\u6570<\/h3>\n
\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u306b\u5165\u529b\u3059\u308b\u691c\u7d22\u30af\u30a8\u30ea\u306e\u57cb\u3081\u8fbc\u307f\u30d9\u30af\u30c8\u30eb\u3092\u51fa\u529b\u3059\u308b\u95a2\u6570\u3092\u7528\u610f\u3057\u3066\u304a\u304d\u307e\u3059\u3002api_key<\/code>\u306b\u306fDatabricks\u306e\u30c8\u30fc\u30af\u30f3\u3092\u6307\u5b9a\u3057\u307e\u3059\u3002<\/p>\n
import<\/span> os\n\nfrom<\/span> openai import<\/span> OpenAI\n\ndef<\/span> create_embedding<\/span>(text):\n    client = OpenAI(\n    api_key=os.environ.get(\"API_KEY\"<\/span>),\n    base_url=f\"{workspace_url}\/serving-endpoints\"<\/span>\n    )\n\n    embeddings = client.embeddings.create(\n        input<\/span>=text,\n        model=\"databricks-bge-large-en\"<\/span>\n    )\n\n    return<\/span> embeddings.data[0<\/span>].embedding\n<\/pre>\n
Databricks Vector Search\u306b\u3088\u308b\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u69cb\u7bc9<\/h3>\n
\u30a8\u30f3\u30c9\u30dd\u30a4\u30f3\u30c8\u306e\u4f5c\u6210<\/h4>\n
\u307e\u305a\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u3092\u683c\u7d0d\u3059\u308b\u30a8\u30f3\u30c9\u30dd\u30a4\u30f3\u30c8\u3092\u4f5c\u6210\u3057\u307e\u3059\u3002<\/p>\n
from<\/span> databricks.vector_search.client import<\/span> VectorSearchClient\n\nclient = VectorSearchClient()\nclient.create_endpoint(\n    name=endpoint,\n    endpoint_type=\"STANDARD\"<\/span> \n)\n<\/pre>\n
\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u306e\u4f5c\u6210<\/h4>\n
\u6b21\u306b\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u306e\u4f5c\u6210\u3067\u3059\u3002source_table_name<\/code>\u306f\u57cb\u3081\u8fbc\u307f\u30d9\u30af\u30c8\u30eb\u3092\u4ed8\u4e0e\u3057\u305fDelta\u30c6\u30fc\u30d6\u30eb\u540d\u3092\u6307\u5b9a\u3057\u3001index_name<\/code>\u306f\u51fa\u529b\u5148\u306e\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u540d\u3092\u6307\u5b9a\u3057\u307e\u3059\u3002<\/p>\n
index = client.create_delta_sync_index(\n  endpoint_name=endpoint,\n  source_table_name=f\"{catalog_name}.{schema_name}.{emb_table}\"<\/span>,\n  index_name=f\"{catalog_name}.{schema_name}.{vs_index_name}\"<\/span>,\n  pipeline_type=\"TRIGGERED\"<\/span>,\n  embedding_dimension=1024<\/span>,\n  primary_key=\"slide\"<\/span>, \n  embedding_vector_column=\"embedding\"<\/span> \n)\n<\/pre>\n
\u30af\u30a8\u30ea\u5b9f\u884c<\/h4>\n
\u4f5c\u6210\u3057\u305f\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u306b\u5bfe\u3057\u3001\u30af\u30a8\u30ea\u3092\u5b9f\u884c\u3057\u3066\u307f\u307e\u3059\u3002<\/p>\n
import<\/span> pandas as<\/span> pd\n\nindex = client.get_index(\n  index_name=f\"{catalog_name}.{schema_name}.{vs_index_name}\"<\/span>\n)\nall_columns = spark.table(\n  f\"{catalog_name}.{schema_name}.{emb_table}\"<\/span>\n).columns\n\nresults = index.similarity_search(\n  query_vector=create_embedding(\"multi modal\"<\/span>),\n  columns=all_columns,\n  num_results=5<\/span>\n)\n\npd.DataFrame(results[\"result\"<\/span>][\"data_array\"<\/span>])\n<\/pre>\n
\u4ee5\u4e0b\u306e\u3088\u3046\u306b”multi modal”\u3068\u3044\u3046\u5358\u8a9e\u306b\u985e\u4f3c\u3057\u305f\u30c6\u30ad\u30b9\u30c8\u304c\u542b\u307e\u308c\u308b\u30ec\u30b3\u30fc\u30c9\u304c\u53d6\u5f97\u3055\u308c\u307e\u3057\u305f\u3002<\/p>\n
<\/span>
Vector Search\u306e\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u3067”multi modal”\u3092\u691c\u7d22\u3057\u305f\u7d50\u679c<\/figcaption><\/figure>\n<\/p>\n
Faiss\u306b\u3088\u308b\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u69cb\u7bc9<\/h3>\n
\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u306e\u4f5c\u6210<\/h4>\n
\u6b21\u306bFaiss\u306e\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u3092\u4f5c\u6210\u3057\u307e\u3059\u3002\u6700\u521d\u306b\u57cb\u3081\u8fbc\u307f\u30d9\u30af\u30c8\u30eb\u3092\u4ed8\u4e0e\u6e08\u307f\u306eDelta\u30c6\u30fc\u30d6\u30eb\u3092\u30ed\u30fc\u30c9\u3057\u307e\u3059\u3002<\/p>\n
df = spark.sql(f\"\"\"<\/span>\nSELECT <\/span>\n  embedding, <\/span>\n  content <\/span>\nFROM <\/span>\n  {catalog_name}.{schema_name}.{emb_table}<\/span>\n\"\"\"<\/span>\n)\n<\/pre>\n
\u30ed\u30fc\u30c9\u3057\u305fDataFrame\u304b\u3089\u57cb\u3081\u8fbc\u307f\u30d9\u30af\u30c8\u30eb\u3092\u53d6\u5f97\u3057\u3001\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u3092\u4f5c\u6210\u3057\u307e\u3059\u3002<\/p>\n
import<\/span> faiss\nimport<\/span> numpy as<\/span> np\nimport<\/span> pandas as<\/span> pd\n\n\npdf = df.toPandas()\n\n\nembeddings = np.array(pdf['embedding'<\/span>].tolist()).astype('float32'<\/span>)\ncontents = pdf['content'<\/span>].tolist()\n\n\ndimension = embeddings.shape[1<\/span>]\n\n\nindex = faiss.IndexFlatL2(dimension)\n\n\nindex.add(embeddings)\n<\/pre>\n
\u30af\u30a8\u30ea\u5b9f\u884c<\/h4>\n
Faiss\u3067\u306e\u30af\u30a8\u30ea\u5b9f\u884c\u3092\u8a66\u3057\u3066\u307f\u307e\u3059\u3002<\/p>\n
query_vector = np.array(\n    create_embedding(\"multi modal\"<\/span>),\n    dtype=\"float32\"<\/span>\n).reshape(1<\/span>,-1<\/span>)\n\n\nk = 5<\/span>\ndistances, indices = index.search(query_vector, k)\n\nresults = []\n\nfor<\/span> i, idx in<\/span> enumerate<\/span>(indices[0<\/span>]):\n    results.append(contents[idx])\n\n\npd.DataFrame({\"results\"<\/span>:results})\n<\/pre>\n
\u3053\u3061\u3089\u3082\u6b21\u306e\u3088\u3046\u306b”multi modal”\u306b\u95a2\u9023\u3059\u308b\u30c6\u30ad\u30b9\u30c8\u3092\u691c\u7d22\u3059\u308b\u3053\u3068\u304c\u51fa\u6765\u307e\u3057\u305f\u3002<\/p>\n
<\/span>
Faiss\u306e\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u306b”multi modal”\u3067\u691c\u7d22\u3057\u305f\u7d50\u679c\u3002Vector Search\u3068\u540c\u3058\u7d50\u679c\u306b\u306a\u308a\u307e\u3057\u305f\u3002<\/figcaption><\/figure>\n<\/p>\n
\u307e\u3068\u3081<\/h3>\n
\u4eca\u56de\u306f\u540c\u3058Delta\u30c6\u30fc\u30d6\u30eb\u3092\u8d77\u70b9\u3068\u3057\u3001Databricks Vector Search\u3068Faiss\u3067\u305d\u308c\u305e\u308c\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u3092\u4f5c\u6210\u3057\u3001\u30af\u30a8\u30ea\u3092\u5b9f\u884c\u3059\u308b\u307e\u3067\u306e\u624b\u9806\u3092\u307e\u3068\u3081\u3066\u307f\u307e\u3057\u305f\u3002\u72b6\u6cc1\u306b\u5fdc\u3058\u3066\u691c\u7d22\u30a8\u30f3\u30b8\u30f3\u3092\u4f7f\u3044\u5206\u3051\u308b\u3053\u3068\u304c\u51fa\u6765\u308b\u3088\u3046\u306b\u306a\u308a\u3001\u691c\u8a3c\u3057\u3066\u307f\u3066\u3088\u304b\u3063\u305f\u3068\u601d\u3044\u307e\u3057\u305f\u3002<\/p>\n
\u305d\u308c\u305e\u308c\u306e\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u3092\u7528\u3044\u3066RAG\u3092\u5b9f\u88c5\u3057\u3066\u307f\u3066\u3001\u7cbe\u5ea6\u3084\u30d1\u30d5\u30a9\u30fc\u30de\u30f3\u30b9\u306e\u691c\u8a3c\u3082\u4eca\u5f8c\u8a66\u3057\u3066\u307f\u305f\u3044\u3068\u601d\u3063\u3066\u3044\u307e\u3059\uff01<\/p>\n<\/div>\n