View Single Post
  #1  
Old 13.10.2019, 03:10
Demongornot Demongornot is offline
JD Beta
 
Join Date: Sep 2019
Location: Universe, Local group, Milky Way, Solar System, Earth, France
Posts: 50
Default Anti duplicates history Script

In answer to this :
Quote:
Originally Posted by Jiaz View Post
@mgpai/Demongornot: I'll suggest to create a new thread for the discussion about the development/ideas/questions for the dupe/history support. I can then move the posts to the new thread.
Here is the post, and here I'll post the prototype code for the first part of the script which is to save finished downloads in files.

CAUTION : I've done writing it and I have fixed the few typo so it compile but I haven't tested it yet, actually I tested none if it (I usually test bits of the code while writing it) except the function where the code come from mgpai and I wanted to make sure my modified version as a function worked.
I'll test it tomorrow, so only test run this code if you know what you are doing because I doubt I wrote it first time without any flaws, compiling in JDownloader only check a small part of the possible bugs and errors.

Code:
const MAX_URLS_PER_FILE = 20; //0 = no limits, I advise setting it to 0 or between 1000 (100 if you don't download a lot) and 100 000.
const FILE_SAVE_MODE = 0; //0 = Short URL with host per file(s); 1 = Long URL with host per file(s); 2 = Long URL on a single file.
const BASEPATH = JD_HOME + '\\History';

if (link.isFinished()) {
	var myDownloadLink = link;
	var path = getPath(BASEPATH);
	var nl = getEnvironment().getNewLine();
	var separator = ' ';
	if (path.exists()) {
		var linkHost = myDownloadLink.getDownloadHost();
		var linkURL = getShortURL(myDownloadLink);
		var fileText = linkURL;
		var filePath;
		var arrayText = [];
		var pass = true;
		if (FILE_SAVE_MODE > 0 || (FILE_SAVE_MODE == 0 && MAX_URLS_PER_FILE == 0)) {
			if (FILE_SAVE_MODE == 2) filePath = getPath(path + '\\History.txt');
			if (FILE_SAVE_MODE < 2) filePath = getPath(path + '\\' + linkHost + separator + '0.txt');
			if (filePath.exists()) {
				arrayText = readFile(filePath).split(nl);
				if (arrayText != null) {
					if (arrayText.indexOf(linkURL) >= 0) pass = false;
					fileText = nl + linkURL;
				}
			}
		} else {
			var fileCounter = 0;
			var previousLength = 0;
			var loopContinue = true;
			while (loopContinue) {
				filePath = getPath(path + '\\' + linkHost + separator + fileCounter + '.txt');
				if (filePath.exists()) {
					arrayText = readFile(filePath).split(nl);
					previousLength = arrayText.length;
					if (arrayText.indexOf(linkURL) >= 0) {
						loopContinue = false;
						pass = false;
					}
					fileCounter++;
				} else {
					if (fileCounter > 0 && previousLength < MAX_URLS_PER_FILE) {
						fileCounter--;
						fileText = nl + linkURL;
					}
					loopContinue = false;
				}
			}
			if (pass) {
				if (filePath.exists()) linkURL = nl + linkURL;
				writeFile(filePath, fileText, true);
			}
		}
	} else {
		var AllDownloadLinks = getAllDownloadLinks();
		if (AllDownloadLinks.length > 0) {
			var fileText;
			var hostsList = [];
			var urlsList = [];
			var linkURL;
			var linkHost;
			var hostIndex;
			var counter;
			for (counter = 0; counter < AllDownloadLinks.length; counter++) {
				linkURL = getShortURL(AllDownloadLinks[counter]);
				if (!(linkURL.isFinished())) continue;
				if (FILE_SAVE_MODE > 1) {
					urlsList.push(linkURL);
				} else {
					linkHost = AllDownloadLinks[counter].getDownloadHost();
					hostIndex = hostsList.indexOf(linkHost);
					if (hostIndex < 0) {
						hostsList.push(linkHost);
						urlsList.push([]);
					}
					urlsList[hostIndex].push(linkURL);
				}
			}
			if (urlsList.length > 0) {
				var filePath;
				if (FILE_SAVE_MODE > 1) {
					fileText = urlsList.join(nl);
					filePath = getPath(path + '\\History.txt');
					writeFile(filePath, fileText, true);
				} else {
					var counter2;
					var counter3;
					var fileCounter = 0;
					var interUrl = '';
					for (counter = 0; counter < urlsList.length; counter++) {
						if (FILE_SAVE_MODE == 0 && MAX_URLS_PER_FILE > 0) {
							counter3 = 0;
							for (counter2 = 0; counter2 < urlsList[counter].length; counter2++) {
								if (counter3 >= MAX_URLS_PER_FILE) {
									counter3 = 0;
									fileCounter++;
									filePath = getPath(path + '\\' + hostsList[counter] + separator + fileCounter + '.txt');
									writeFile(filePath, fileText, true);
									fileText = '';
									interUrl = '';
								}
								fileText += interUrl + urlsList[counter][counter2];
								interUrl = nl;
							}
						} else {
							fileText = urlsList[counter].join(nl);
							filePath = getPath(path + '\\' + hostsList[counter] + separator + fileCounter + '.txt');
							writeFile(filePath, fileText, true);
						}
					}
				}
			}
		}
	}
}

function getShortURL(DownloadLink) {
	if (FILE_SAVE_MODE > 0) return DownloadLink.getContentURL() || DownloadLink.getPluginURL();
	var url = DownloadLink.getProperty("LINKDUPEID") || link.getPluginURL();
	return url.replace(/(^(https?|ftp):\/\/[^\/]+\/)/, '').replace(/.+:\/\//, '');
}
@mgpai what you think about it ?
(I'll test and comment it tomorrow, it's past 2am now, my bed is calling)
Reply With Quote