Java getting millions files from SharePoint online
2024-12-08
This function is using SharePoint-Java-API library.
public static HashSet<String> getAllFileOnSPO(Pair<String, String> token, String domain, String formDigestValue) throws UnsupportedEncodingException {
int pageSize = 5000;
int p_ID = 0;
HashSet<String> filenames = new HashSet();
int LastRow = 0;
String NextHref = null;
for (int z = 0; z < 1000; z++) {
String data = "{ \n"
+ " \"parameters\": {\n"
+ " \"RenderOptions\": 2,\n"
+ " \"ViewXml\": \"<View>"
+ "<Query>"
+ "<OrderBy>"
+ "<FieldRef Name=\\\"FileLeafRef\\\"/>"
+ "</OrderBy>"
+ "</Query>"
+ "<ViewFields>"
+ " <FieldRef Name='LinkFilename'/>"
+ "</ViewFields>"
+ "<RowLimit Paged=\\\"TRUE\\\">" + pageSize + "</RowLimit>"
+ "</View>\"\n"
+ " "
+ " }"
+ "}";
// System.out.println(data);
JSONObject json2 = new JSONObject(data);
String url;
if (NextHref == null) {
url = "/sites/DMS/_api/web/GetListUsingPath(DecodedUrl=%27%2Fsites%2FDMS%2FAccount%27)/RenderListDataAsStream?RootFolder=" + URLEncoder.encode("/sites/DMS/", "utf-8") + "&PageFirstRow=" + (z * pageSize) + "&p_ID=" + p_ID + "&Paged=TRUE&ix_Paged=TRUE&ix_ID=" + p_ID;
} else {
url = "/sites/DMS/_api/web/GetListUsingPath(DecodedUrl=%27%2Fsites%2FDMS%27)/RenderListDataAsStream" + NextHref;
}
System.out.println("url=" + url);
String jsonString = SPOnline.post(token, domain, url, data, formDigestValue);
json2 = new JSONObject(jsonString);
System.out.println("length=" + json2.getJSONArray("Row").length());
System.out.println(" RowLimit=" + json2.getInt("RowLimit"));
System.out.println(" LastRow=" + json2.getInt("LastRow"));
LastRow = json2.getInt("LastRow");
JSONArray arr = json2.getJSONArray("Row");
if (arr.length() == 0) {
break;
}
for (int x = 0; x < arr.length(); x++) {
JSONObject obj = arr.getJSONObject(x);
filenames.add(obj.getString("FileLeafRef"));
}
p_ID = arr.getJSONObject(arr.length() - 1).getInt("ID");
if (!json2.has("NextHref")) {
break;
}
System.out.println(" NextHref=" + json2.getString("NextHref"));
NextHref = json2.getString("NextHref");
}
return filenames;
}