Skip to content

Commit

Permalink
feat/migrate onedrive src (#3295)
Browse files Browse the repository at this point in the history 
### Description
Migrate the onedrive source connector to v2, adding in more rich content
pulled from the response of the SDK to add further metadata to the
FIleData produced by the indexer.

---------

Co-authored-by: ryannikolaidis <[email protected]>
Co-authored-by: rbiseck3 <[email protected]>
  • Loading branch information
@rbiseck3 @ryannikolaidis
3 people committed Jun 26, 2024
1 parent 6939bff commit 54ec311
Show file tree
Hide file tree
Showing 6 changed files with 742 additions and 380 deletions.
174 changes: 90 additions & 84 deletions ...d_ingest/expected-structured-output/onedrive/utic-test-ingest-fixtures/fake-text.txt.json
View file
Original file line number Diff line number Diff line change
@@ -1,128 +1,134 @@
[
{
"type": "NarrativeText",
"element_id": "fc01503614e0f12f585427cccf81cf86",
"text": "This is a test document to use for unit tests.",
"metadata": {
"data_source": {
"date_created": "2023-08-24T03:00:09+00:00",
"date_modified": "2023-08-24T03:00:09+00:00",
"record_locator": {
"server_relative_path": "utic-test-ingest-fixtures/fake-text.txt",
"user_pname": "[email protected]"
},
"url": "/drives/b!3vfYDk3GHEaRbo1pkhLPIRXZrzTLHCtCm5WV6KY1m_0-lOjrjQaAS6X30Pv_E4VX/root:/utic-test-ingest-fixtures/fake-text.txt"
},
"filename": "fake-text.txt",
"filetype": "text/plain",
"languages": [
"eng"
]
},
"text": "This is a test document to use for unit tests.",
"type": "NarrativeText"
],
"filetype": "text/plain",
"data_source": {
"url": "/drives/b!3vfYDk3GHEaRbo1pkhLPIRXZrzTLHCtCm5WV6KY1m_0-lOjrjQaAS6X30Pv_E4VX/root:/utic-test-ingest-fixtures/fake-text.txt",
"version": "\"{62330F6D-C37D-44D2-B6F6-FF191FE5A2E5},1\"",
"record_locator": {
"user_pname": "[email protected]",
"server_relative_path": "utic-test-ingest-fixtures/fake-text.txt"
},
"date_created": "1692846009.0",
"date_modified": "1692846009.0"
}
}
},
{
"type": "Address",
"element_id": "d06d10c0722ac08a2488076a48e858d5",
"text": "Doylestown, PA 18901",
"metadata": {
"data_source": {
"date_created": "2023-08-24T03:00:09+00:00",
"date_modified": "2023-08-24T03:00:09+00:00",
"record_locator": {
"server_relative_path": "utic-test-ingest-fixtures/fake-text.txt",
"user_pname": "[email protected]"
},
"url": "/drives/b!3vfYDk3GHEaRbo1pkhLPIRXZrzTLHCtCm5WV6KY1m_0-lOjrjQaAS6X30Pv_E4VX/root:/utic-test-ingest-fixtures/fake-text.txt"
},
"filename": "fake-text.txt",
"filetype": "text/plain",
"languages": [
"eng"
]
},
"text": "Doylestown, PA 18901",
"type": "Address"
],
"filetype": "text/plain",
"data_source": {
"url": "/drives/b!3vfYDk3GHEaRbo1pkhLPIRXZrzTLHCtCm5WV6KY1m_0-lOjrjQaAS6X30Pv_E4VX/root:/utic-test-ingest-fixtures/fake-text.txt",
"version": "\"{62330F6D-C37D-44D2-B6F6-FF191FE5A2E5},1\"",
"record_locator": {
"user_pname": "[email protected]",
"server_relative_path": "utic-test-ingest-fixtures/fake-text.txt"
},
"date_created": "1692846009.0",
"date_modified": "1692846009.0"
}
}
},
{
"type": "Title",
"element_id": "a190164de573571375ecf759a5027a3a",
"text": "Important points:",
"metadata": {
"data_source": {
"date_created": "2023-08-24T03:00:09+00:00",
"date_modified": "2023-08-24T03:00:09+00:00",
"record_locator": {
"server_relative_path": "utic-test-ingest-fixtures/fake-text.txt",
"user_pname": "[email protected]"
},
"url": "/drives/b!3vfYDk3GHEaRbo1pkhLPIRXZrzTLHCtCm5WV6KY1m_0-lOjrjQaAS6X30Pv_E4VX/root:/utic-test-ingest-fixtures/fake-text.txt"
},
"filename": "fake-text.txt",
"filetype": "text/plain",
"languages": [
"eng"
]
},
"text": "Important points:",
"type": "Title"
],
"filetype": "text/plain",
"data_source": {
"url": "/drives/b!3vfYDk3GHEaRbo1pkhLPIRXZrzTLHCtCm5WV6KY1m_0-lOjrjQaAS6X30Pv_E4VX/root:/utic-test-ingest-fixtures/fake-text.txt",
"version": "\"{62330F6D-C37D-44D2-B6F6-FF191FE5A2E5},1\"",
"record_locator": {
"user_pname": "[email protected]",
"server_relative_path": "utic-test-ingest-fixtures/fake-text.txt"
},
"date_created": "1692846009.0",
"date_modified": "1692846009.0"
}
}
},
{
"type": "ListItem",
"element_id": "64c58eb106608ad05424e47bbcdef7dc",
"text": "Hamburgers are delicious",
"metadata": {
"data_source": {
"date_created": "2023-08-24T03:00:09+00:00",
"date_modified": "2023-08-24T03:00:09+00:00",
"record_locator": {
"server_relative_path": "utic-test-ingest-fixtures/fake-text.txt",
"user_pname": "[email protected]"
},
"url": "/drives/b!3vfYDk3GHEaRbo1pkhLPIRXZrzTLHCtCm5WV6KY1m_0-lOjrjQaAS6X30Pv_E4VX/root:/utic-test-ingest-fixtures/fake-text.txt"
},
"filename": "fake-text.txt",
"filetype": "text/plain",
"languages": [
"eng"
]
},
"text": "Hamburgers are delicious",
"type": "ListItem"
],
"filetype": "text/plain",
"data_source": {
"url": "/drives/b!3vfYDk3GHEaRbo1pkhLPIRXZrzTLHCtCm5WV6KY1m_0-lOjrjQaAS6X30Pv_E4VX/root:/utic-test-ingest-fixtures/fake-text.txt",
"version": "\"{62330F6D-C37D-44D2-B6F6-FF191FE5A2E5},1\"",
"record_locator": {
"user_pname": "[email protected]",
"server_relative_path": "utic-test-ingest-fixtures/fake-text.txt"
},
"date_created": "1692846009.0",
"date_modified": "1692846009.0"
}
}
},
{
"type": "ListItem",
"element_id": "9210d7882755a60fc82272a0e93df94f",
"text": "Dogs are the best",
"metadata": {
"data_source": {
"date_created": "2023-08-24T03:00:09+00:00",
"date_modified": "2023-08-24T03:00:09+00:00",
"record_locator": {
"server_relative_path": "utic-test-ingest-fixtures/fake-text.txt",
"user_pname": "[email protected]"
},
"url": "/drives/b!3vfYDk3GHEaRbo1pkhLPIRXZrzTLHCtCm5WV6KY1m_0-lOjrjQaAS6X30Pv_E4VX/root:/utic-test-ingest-fixtures/fake-text.txt"
},
"filename": "fake-text.txt",
"filetype": "text/plain",
"languages": [
"eng"
]
},
"text": "Dogs are the best",
"type": "ListItem"
],
"filetype": "text/plain",
"data_source": {
"url": "/drives/b!3vfYDk3GHEaRbo1pkhLPIRXZrzTLHCtCm5WV6KY1m_0-lOjrjQaAS6X30Pv_E4VX/root:/utic-test-ingest-fixtures/fake-text.txt",
"version": "\"{62330F6D-C37D-44D2-B6F6-FF191FE5A2E5},1\"",
"record_locator": {
"user_pname": "[email protected]",
"server_relative_path": "utic-test-ingest-fixtures/fake-text.txt"
},
"date_created": "1692846009.0",
"date_modified": "1692846009.0"
}
}
},
{
"type": "ListItem",
"element_id": "970dd4ac687529e2d2126e0a51cb7c27",
"text": "I love fuzzy blankets",
"metadata": {
"data_source": {
"date_created": "2023-08-24T03:00:09+00:00",
"date_modified": "2023-08-24T03:00:09+00:00",
"record_locator": {
"server_relative_path": "utic-test-ingest-fixtures/fake-text.txt",
"user_pname": "[email protected]"
},
"url": "/drives/b!3vfYDk3GHEaRbo1pkhLPIRXZrzTLHCtCm5WV6KY1m_0-lOjrjQaAS6X30Pv_E4VX/root:/utic-test-ingest-fixtures/fake-text.txt"
},
"filename": "fake-text.txt",
"filetype": "text/plain",
"languages": [
"eng"
]
},
"text": "I love fuzzy blankets",
"type": "ListItem"
],
"filetype": "text/plain",
"data_source": {
"url": "/drives/b!3vfYDk3GHEaRbo1pkhLPIRXZrzTLHCtCm5WV6KY1m_0-lOjrjQaAS6X30Pv_E4VX/root:/utic-test-ingest-fixtures/fake-text.txt",
"version": "\"{62330F6D-C37D-44D2-B6F6-FF191FE5A2E5},1\"",
"record_locator": {
"user_pname": "[email protected]",
"server_relative_path": "utic-test-ingest-fixtures/fake-text.txt"
},
"date_created": "1692846009.0",
"date_modified": "1692846009.0"
}
}
}
]
Loading

0 comments on commit 54ec311

Please sign in to comment.