w10n_XYZ-extractor-020.html

<!DOCTYPE html>
<html>

<head>
  <style>
	body {
	  font-family: monospace;
	}

	#btnGo {
	  margin-top: 20px;
	  line-height: 25px;
	}
  </style>
<script type='text/javascript' src='https://code.jquery.com/jquery-3.5.1.min.js'></script>

 </head>
  <body>

<script>

// 0.2.0 Implementata lettura 3 file, ma da sistemare intepretazione dati RGB (una banda sola?!? forse è b/n?)) e da verificare validità dati RGB e Normals con sito JSON
// 0.0.3/0.1.0: First version successullying creating PLY file
// 0.0.2: First version properly creating output string in debug mode
// 0.0.1: First version with "float decoding" implemented

//
// Online PLY viewers:
//  http://lidarview.com/
// Offline PLY viewers:
//  Meshlab (can add mesh)
//

var PLY_HEADER_BASE = 'ply\n'+
	'format ascii 1.0\n'+
	'element vertex VVVVVV\n'+
	'property float x\n'+
	'property float y\n'+
	'property float z\n';

var PLY_HEADER_VISIBLE = "" +
	'property uchar red\n'+
	'property uchar green\n'+
	'property uchar blue\n';


var PLY_HEADER_NORMALS = "" +
	'property float nx\n'+
	'property float ny\n'+
	'property float nz\n';

//	'property list uchar float vertex_indices\n'+
var PLY_HEADER_END= 	'element face 0\n'+
						'end_header\n';


	const FACTOR = 100000;
	const TOSKIP = 1;
	const START_MARKER = '<A HREF="';
	const END_MARKER = '">';
	const PROXY = "http://win98.altervista.org/space/exploration/myp.php?pass=miapass&mode=native&url=";
	const SERVER_URL = "https://pds-imaging.jpl.nasa.gov";
	var PAGE_URL = "/w10n/mer/spirit/mer2no_0xxx/data/sol1866/rdr/2n292022791xylb0ozf0006l0m1.img/0/raster/data[]?output=json"; // provare con THUMBNAIL  (DEBUG)
	var toolsList = [];
	var toolsListTable = [];
    fileDataXYZ = null;
    fileDataVisible = null;
    fileDataNormals = null;

function startProcessing() {
    processData(/*fileDataXYZ, fileDataVisible, fileDataNormals*/);
	console.log("Processing ended.");
}


function loadFile(fileHandler, dataType) {
console.log(1);
console.log("DataType in LOADFILE:", dataType);
	const reader = new FileReader();
	reader.addEventListener('load', (event) => {
		stringContents = event.target.result;
		console.log("Loaded " + dataType + ": ", stringContents.length);
		if (dataType === "XYZ") {
			fileDataXYZ = processFile(stringContents, dataType); // processFile returns data from file converted to JSON format
			console.log("Returned from XYZ processing: ", fileDataXYZ);
		}

		if (dataType === "visible") {
			fileDataVisible = processFile(stringContents, dataType); // processFile returns data from file converted to JSON format
			console.log("Returned from VISIBLE processing", fileDataVisible);
		}

		if (dataType === "normals") {
			fileDataNormals = processFile(stringContents, dataType); // processFile returns data from file converted to JSON format
			console.log("Returned from NORMALS processing", fileDataNormals);
		}

	});
	reader.readAsBinaryString(fileHandler);

console.log(2);
}


function saveFile(fileContent) {
	myBlob = new Blob([fileContent], {type: "text/plain"});

	var url = window.URL.createObjectURL(myBlob);
	var anchor = document.createElement("a");
	anchor.href = url;
	anchor.download = "mytest.ply";

	anchor.click();
	window.URL.revokeObjectURL(url);
	//document.removeChild(anchor);
}


function handleEvent(e) {
	log.textContent = `${e.type}: ${e.loaded} bytes transferred\n` + log.textContent;
}

function addListeners(xhr) {
	xhr.addEventListener('loadstart', handleEvent);
	xhr.addEventListener('load', handleEvent);
	xhr.addEventListener('loadend', handleEvent);
	xhr.addEventListener('progress', handleEvent);
	xhr.addEventListener('error', handleEvent);
	xhr.addEventListener('abort', handleEvent);
console.log("Progress listeners added.");
}




function writeCanvas(x,y,r,g,b) {
	//var x = Math.floor(Math.random() * canvasWidth);
	//var y = Math.floor(Math.random() * canvasHeight);
	//var r = Math.floor(Math.random() * 256);
	//var g = Math.floor(Math.random() * 256);
	//var b = Math.floor(Math.random() * 256);

  ctx.fillStyle = 'rgb(' + r + ',' + g + ',' + b + ')';
  ctx.fillRect(x,y,x+1,y+1);
/*	var off = (y * id.width + x) * 4;
	pixels[off] = r;
	pixels[off + 1] = g;
	pixels[off + 2] = b;
	pixels[off + 3] = 255;
	ctx.putImageData(id, 0, 0);*/
}


  httpRequest = (PROXY, rawurl, method = 'GET', type="XML") => {
		url = PROXY + rawurl;
console.log(">>>FILE:",url);
	return new Promise((resolve, reject) => {
	  const xhr = new XMLHttpRequest();
	  addListeners(xhr);
	  xhr.open(method, url);
	  xhr.onload = () => {
		if (xhr.status === 200) {
			console.log("OK!");
				rawResponse = xhr.responseText;
				try {
					jsonResponse = JSON.parse(rawResponse);
				} catch (e) {
					console.log("JSON error:" , e);
					console.log(rawResponse);
				}
				console.log(jsonResponse);
				processData(jsonResponse);
		}  else {
				console.log("ERROR!");

				reject(new Error(xhr.responseText));
		}
	  };
	xhr.progress = (a) => {
		console.log("PROG:",a);
		document.getElementById("status").innerHTML = "Progress: " + a;
	};
	  xhr.send();
	});
  }


function processFile(fileContents, dataType) {
console.log("Start processing...", fileContents.length);
testami = fileContents;


	const IMAGE_MARKER = "/* IMAGE DATA ELEMENTS */";
	const START_OBJECT_MARKER = "OBJECT";
	const END_OBJECT_MARKER = "END_OBJECT";
	const BYTE_LENGTH = 8;

	/////// Read PDS label data:
	recordBytesLineStartPos = fileContents.indexOf("RECORD_BYTES");
	recordBytesLineEndPos = fileContents.indexOf("\n",recordBytesLineStartPos);
	recordBytesLine = fileContents.substr(recordBytesLineStartPos, recordBytesLineEndPos - recordBytesLineStartPos + 1);
	recordBytesArr = recordBytesLine.split("=");
	recordBytes = parseInt(recordBytesArr[1]);
console.log("recordBytes=",recordBytes);

	fileRecordsLineStartPos = fileContents.indexOf("FILE_RECORDS");
	fileRecordsLineEndPos = fileContents.indexOf("\n",fileRecordsLineStartPos);
	fileRecordsLine = fileContents.substr(fileRecordsLineStartPos, fileRecordsLineEndPos - fileRecordsLineStartPos + 1);
	fileRecordsArr = fileRecordsLine.split("=");
	fileRecords = parseInt(fileRecordsArr[1]);
console.log("fileRecords=",fileRecords);

	labelRecordsLineStartPos = fileContents.indexOf("LABEL_RECORDS");
	labelRecordsLineEndPos = fileContents.indexOf("\n",labelRecordsLineStartPos);
	labelRecordsLine = fileContents.substr(labelRecordsLineStartPos, labelRecordsLineEndPos - labelRecordsLineStartPos + 1);
	labelRecordsArr = labelRecordsLine.split("=");
	labelRecords = parseInt(labelRecordsArr[1]);
	console.log("labelRecords=",labelRecords);
	PDSlabelLength = recordBytes * labelRecords;
	PDSlabel = fileContents.substr(0,PDSlabelLength);
//console.log("PDS label:", PDSlabelLength, PDSlabel);

	// "The total of the label (in bytes) is an integral multiple (x LABEL_RECORDS) of the record length (RECORD_BYTES) of the data: PDS_length = RECORD_BYTES * LABEL_RECORDS"

	////// Read VICAR label data:
	vicarLabelLengthLineStartPos = fileContents.indexOf("LBLSIZE");
	vicarLabelLengthLineEndPos = fileContents.indexOf("\n",vicarLabelLengthLineStartPos);
	vicarLabelLengthLine = fileContents.substr(vicarLabelLengthLineStartPos, vicarLabelLengthLineEndPos - vicarLabelLengthLineStartPos + 1);
	vicarLabelLengthArr = vicarLabelLengthLine.split("=");
	vicarLabelLength = parseInt(vicarLabelLengthArr[1]);
	vicarLabel = fileContents.substr(PDSlabelLength, vicarLabelLength).replace('\x00',"");
	vicarLabel=vicarLabel.split("	").join("=========");
	vicarLabel=vicarLabel.split("   ").join("---");
	vicarLabel=vicarLabel.split("  ").join("\r\n");
//console.log("vicarLabel:", vicarLabelLength, vicarLabel);

	totalLabelLength = PDSlabelLength + vicarLabelLength;
	onlyData = fileContents.substr(totalLabelLength , fileContents.length -  totalLabelLength + 1);
console.log("onlyData length=", onlyData.length, onlyData.length/3);

	///// Read image/band metadata:
	imageDataStartPos = fileContents.indexOf(IMAGE_MARKER);
	imageDataStartPos = fileContents.indexOf(START_OBJECT_MARKER,imageDataStartPos);
	imageDataEndPos = fileContents.indexOf(END_OBJECT_MARKER,imageDataStartPos);
	imageDataLabel = fileContents.substr(imageDataStartPos, imageDataEndPos - imageDataStartPos);
	imageDataLabel = imageDataLabel.split(" ").join("");
	imageDataLabelArr = imageDataLabel.split("\r\n");
	imageDataLabelObj = {}
	imageDataLabelArr.forEach((elem) => {
		var pair = elem.split("=");
		imageDataLabelObj[pair[0]] = pair[1];
	});
	imageLines = imageDataLabelObj["LINES"];
	imageSamplesPerLine = imageDataLabelObj["LINE_SAMPLES"];
	bytesPerSample = imageDataLabelObj["SAMPLE_BITS"]/BYTE_LENGTH;
	lineLength = imageSamplesPerLine * bytesPerSample;
	bandsNum = imageDataLabelObj["BANDS"]
	bandLength = imageLines * lineLength;
	imageLength = bandLength * bandsNum;


console.log("PDSlabelLength", "vicarLabelLength", "totalLabelLength");
console.log(PDSlabelLength, vicarLabelLength, totalLabelLength);
console.log(PDSlabelLength.toString(16), vicarLabelLength.toString(16), totalLabelLength.toString(16));
console.log("imageLength",imageLength, "0x"+ imageLength.toString(16));
console.log("bandLength",bandLength, "0x"+ bandLength.toString(16));
console.log("file length",fileContents.length, "0x"+ fileContents.length.toString(16));

console.log("imageLines",imageLines);
console.log("imageSamplesPerLine",imageSamplesPerLine);
console.log("bytesPerSample",bytesPerSample);
console.log("lineLength",lineLength);
console.log("bandsNum",bandsNum);

console.log("fileContents:", fileContents.length, "0x"+ fileContents.length.toString(16));

console.log("Processing data...");
	imageTable = [null, null, null]; // Source data (bytes)
	bandArray = [null, null, null]; // Procesed data (floats)
	for (var band = 0; band < bandsNum*1; band++) {
console.log("	Processing band " , band, "...");
console.log("Leggo banda " + band + " da " + (totalLabelLength + band * bandLength) + " a " +  (totalLabelLength  + band * bandLength + bandLength ) + " (escluso ultimo)");
		imageTable[band] = fileContents.slice(totalLabelLength + band * bandLength, totalLabelLength  + band * bandLength + bandLength);
console.log( "imageTable[" + band + "]:" , imageTable[band].length, "0x"+ imageTable[band].length.toString(16));
		bandArray[band] = [];
console.log("Elaboro elementi di imageTable[" + band + "]  da 0 a " + (imageTable[band].length-1));
		for (let i = 0; i < imageTable[band].length; i += lineLength) { // lines
			const imageLine = /*Array.from(*/imageTable[band].slice(i, i + lineLength)/*)*/;
			samplesArray = [];
			for (let j = 0; j < imageLine.length; j += bytesPerSample) { // rows
				const sample = imageLine.slice(j, j + bytesPerSample);
				samplesArray.push(sample);
			}
			bandArray[band].push(samplesArray);
		}
console.log("Length of calculated array for band " + band + ":" + bandArray[band].length);
	}


console.log("Data processed, converting to string...");
	totalStr = '{ "data":[';
	///// user configurable (to do):
	startRow = 0;
	endRow = imageLines; 
	startCol = 0;
	endCol = imageSamplesPerLine;
	//////
	for (var row = startRow; row < endRow ; row++) {
		rowStr = "[";
//console.log("Processing row " + row + " (" + (100*row/endRow).toFixed(0) + "%)");
document.getElementById("status").innerHTML = "Processing row " + row + " (" + (100*row/endRow).toFixed(0) + "%)";
		for (var col = startCol ; col < endCol;  col++) {
			element = "[";
			for (var band = 0; band < bandsNum*1; band++) {
				if ((dataType === "XYZ") || (dataType === "normals")){ // XYZ and normals are float
					element += arrayToFloat(bandArray[band][row][col]) + ",";
				} else { // visible data are integer (16 bit)
					element += arrayToInt(bandArray[band][row][col]) + ",";
				}
			}
			element = element.substr(0,element.length-1) + "],"; // remove final comma and add "]"
			rowStr += element;
		}
		rowStr = rowStr.substr(0,rowStr.length-1) + "],"; // remove final comma and add "]"
		totalStr += rowStr;
	}
	totalStr = totalStr.substr(0,totalStr.length-1) + "]}"; // remove final comma and add "]}"

//console.log(totalStr);

console.log("Converted to string. Now converting to json...");

	jsonResponse = null;
	try {
		jsonResponse = JSON.parse(totalStr);
	} catch (e) {
		console.log("JSON error:" , e);
		console.log(totalStr);
	}

console.log("JSON result:", jsonResponse);
	//processData(jsonResponse);

	return jsonResponse; // debug
}

function arrayToFloat(data) {
	// Create a buffer
	 buf = new ArrayBuffer(4);
	// Create a data view of it
	 view = new DataView(buf);

	// set bytes
	arr=Array.from(data);
	arr.forEach(function (b, i) {
		view.setUint8(i, b.charCodeAt());
	});

	// Read the bits as a float; note that by doing this, we're implicitly
	// converting it from a 32-bit float into JavaScript's native 64-bit double
	 num = view.getFloat32(0);
	// Done
	if (num*1 === 0){
		return ("0.0");
	} else {
		return ((num*FACTOR).toFixed(0));
	}
}


function arrayToInt(data) {
	// Create a buffer
	 buf = new ArrayBuffer(2);
	// Create a data view of it
	 view = new DataView(buf);

	num = data.charCodeAt(0) *256 +  datadata.charCodeAt(1) * 1; // debug test
/*
	// set bytes
	arr=Array.from(data);
	arr.forEach(function (b, i) {
		view.setUint8(i, b.charCodeAt());
	});

	// Read the bits as a float; note that by doing this, we're implicitly
	// converting it from a 32-bit float into JavaScript's native 64-bit double
	 num = view.getInt16(0);

*/
	// Done
	if (num*1 === 0){
		return ("000");
	} else {
		return ((num*1).toFixed(0));
	}
}


function stringToArray(str) {
	arr = [];
	for (var i=0; i<str.length; i++) {
		arr.push(str[i].charCodeAt())
	}
	console.log("STR ", str , "= ARR ", arr);
}


function testDebug() {
	console.log(arrayToFloat(bandArray[15][32]));
}



function processData(/*objXYZ, objVisible, objNormals*/) {
console.log("Creating .PLY 3d file...");
	objXYZ = fileDataXYZ;
	objVisible = fileDataVisible;
	objNormals = fileDataNormals;
console.log(objXYZ, objVisible, objNormals);

// Setup XYZ data:
	rows = objXYZ.data;
	rowsNum = rows.length;
	cols = rows[0];
	colsNum = cols.length;
	bands = cols[0];
	bandsNum = bands.length;

// Setup Visible data if available:
    if (objVisible) {
		visRows = objVisible.data;
		visRowsNum = visRows.length;
		visCols = visRows[0];
		visColsNum = visCols.length;
		visBands = visCols[0];
		visBandsNum = visBands.length;
	}

// Setup Normals data if available:
    if (objNormals) {
		normRows = objNormals.data;
		normRowsNum = normRows.length;
		normCols = normRows[0];
		normColsNum = normCols.length;
		normBands = normCols[0];
		normBandsNum = normBands.length;
	}

/*	if (bandsNum) {*/
console.log("XYZ Rows x Columns =",rowsNum + "x" + colsNum + ", Bands: ", bandsNum);
if (objVisible) console.log("Vis Rows x Columns =",visRowsNum + "x" + visColsNum + ", Bands: ", visBandsNum);
if (objNormals) console.log("Norm Rows x Columns =",normRowsNum + "x" + normColsNum + ", Bands: ", normBandsNum);

		PLY_HEADER = PLY_HEADER_BASE;

		if (objVisible) {
            PLY_HEADER += PLY_HEADER_VISIBLE;
		}

		if (objNormals) {
            PLY_HEADER += PLY_HEADER_NORMALS;
		}

        PLY_HEADER += PLY_HEADER_END;

console.log("PLY_HEADER:",PLY_HEADER);

		rowsCount = 0;
		pixelsCount = 0;
		finalText = "";
		rows.forEach( (row) => {
//console.log("RIGA ",rowsCount);
			colsCount = 0;
			rowElements = row
			rowElements.forEach( (XYZelement) => {
//console.log("COLONNA ",colsCount);
			if ( (XYZelement[0] != 0) || (XYZelement[1] != 0) || (XYZelement[2] != 0)) { // Debug / to do: values (0,0,0) mean "no value", but this can vary, it's specified in label
//console.log("" + XYZelement[0] + "," + XYZelement[1] + "," + XYZelement[2]);
				XYZpart = XYZelement[0]*1 + " " + XYZelement[1]*1 + " " + XYZelement[2]*1;
        finalText += XYZpart;
				if (objVisible) {
					visRow = visRows[rowsCount];
					visElement = visRow[colsCount];
					colorLevel = visElement[0];
					visPart = colorLevel + " " + colorLevel + " " + colorLevel;// + " DEBUG " + "0x" + colorLevel.toString(6);
					finalText += " " + visPart;
					writeCanvas(colsCount, rowsCount, colorLevel, colorLevel, colorLevel);
				}
				if (objNormals) {
					normRow = normRows[rowsCount];
					normElement = normRow[colsCount];
					UVpart = normElement[0] + " " + normElement[1] + " " + normElement[2];
					finalText += " " + UVpart;
				}
				finalText +=  "\n"; // Final PLY line
				pixelsCount++;
//console.log("Processed rows: " + (100*(rowsCount / rowsNum)).toFixed(0));
				}
                colsCount++;
			});
		rowsCount++;
		});
console.log("Processed pixels:",pixelsCount);
		PLY_FINAL_HEADER = PLY_HEADER.replace("VVVVVV",pixelsCount)
		output.textContent = "Ready for download...";
		saveFile(PLY_FINAL_HEADER + finalText);
/*	} else {
		// Single band file, create depthmap image
	rowsCount = 0;
	skipper = 0;
		rows.forEach( (row) => {
		values = row
		valCount = 0;
		values.forEach( (value) => {
			skipper++;
			if (skipper === TOSKIP) {
				x = valCount;
				y = rowsCount;
					writeCanvas(x,y,value,value,value);
				valCount++;
				skipper = 0;
//	console.log("Skipped at ", valCount);
			} else {
				valCount++;
			}
		});
rowsCount++;
		});
	}*/
	console.log("PLY creation completed.");
}

function resolve(a) {
console.log("OK",a);
}


function reject(a) {
console.log("ERROR",a);
}



  $(document).ready(function () {
	  log = document.querySelector('.event-log');
		canvas = document.getElementById('depthmap');
		ctx = canvas.getContext('2d');
		canvasWidth = canvas.width;
		canvasHeight = canvas.height;
		ctx.clearRect(0, 0, canvasWidth, canvasHeight);
		id = ctx.getImageData(0, 0, canvasWidth, canvasHeight);
		pixels = id.data;

		const fileSelectorXYZ = document.getElementById('inpFileXYZ');
		fileSelectorXYZ.addEventListener('change', (event) => loadFile(event.target.files[0], "XYZ"));

		const fileSelectorVisible = document.getElementById('inpFileVisible');
		fileSelectorVisible.addEventListener('change', (event) => loadFile(event.target.files[0], "visible"));

		const fileSelectorNormals = document.getElementById('inpFileNormals');
		fileSelectorNormals.addEventListener('change', (event) => loadFile(event.target.files[0], "normals"));



		console.log("Ready.");
		document.getElementById("status").innerHTML = "READY.";
	$("#btnGo").click(function () {
		var MY_URL_1=SERVER_URL + PAGE_URL;
		var url_1 = PROXY + encodeURIComponent(MY_URL_1);
		console.log("1");
		httpRequest(PROXY, MY_URL_1,"GET","list");
		console.log("2");
	});
  });

</script>


  </body>
</html>




<body>
- <a href="https://an.rsl.wustl.edu/mera/merxbrowser/an3.aspx">Notebook</a><br>
- <a href="https://pds-imaging.jpl.nasa.gov/w10n/mer/spirit/mer2no_0xxx/data/sol1866/rdr/">JSON converter</a><br>
- <a href="https://pds-imaging.jpl.nasa.gov/data/mer/spirit/mer2no_0xxx/data/sol1866/rdr/">Raw download</a><br>
- <a href="http://lidarview.com/">Online PLY viewer</a><br>

  <div>
	<textarea id="output" cols="55" rows="15"></textarea>
  </div>
  <button id="btnGo">Go</button><br>
	Status: <span id="status" name="status">-</span><br>
	<br>
XYZ: <input type="file" id="inpFileXYZ" width="100"><br>
FFL/RAD: <input type="file" id="inpFileVisible"><br>
UV: <input type="file" id="inpFileNormals"><br>
<button id="btnStart" name="btnStart" onclick="startProcessing();">Start processing</button>
<br>
	<textarea readonly class="event-log" cols=150 rows=3></textarea><br>
	<canvas id="depthmap" name="depthmap" width = "1024" height="1024"></canvas><br>

</body>

</html>