diff --git a/TwitToMast.js b/TwitToMast.js index f3debfa..9590493 100644 --- a/TwitToMast.js +++ b/TwitToMast.js @@ -72,6 +72,7 @@ driver.executeScript("document.body.style.zoom='35%'"); }); debuglog("opened csv",2); var processedTweets = [];//DEFINE ARRAY THAT WILL BE POPULATED WITH TWEETS PROCESSED DURING THIS SESSION + var allTweetsArray = [];//INITIALIZE THE POST QUEUE for (var t = 1; t < (parseInt(args.tweetCount) + 1); t++) {//LOOP THE NUMBER OF TIMES SPECIFIED IN ARGS debuglog(format.notice(`Processing tweet #${t} of ${args.tweetCount}...`),1); @@ -109,6 +110,16 @@ driver.executeScript("document.body.style.zoom='35%'"); if (!homeTweet.processed && !csvOutput.includes(homeTweet.url)) { //IF CSV DOES NOT CONTAIN THE TWEET URL debuglog(`Tweet #${homeTweet.no} has not been processed.`, 1); + await homeTweet.getElementProperties(driver); + + homeTweet.compileText();//COMPILE TEXT FOR CROSS-POST + + homeTweet.printPreview();//PRINT TWEET PREVIEW + + await homeTweet.downloadImages(driver,imgSavePath);//DOWNLOAD IMAGES FROM TWITTER + + await homeTweet.uploadImages(imgSavePath);//UPLOAD IMAGES TO MASTODON + if (homeTweet.isThread){ //IF TWEET IS A THREAD, RUN TWEET THREAD STUFF var threadTweet = new Tweets("thread",1); //CREATE NEW THREAD TWEET OBJECT var threadTweetArray = []; //ARRAY OF THREAD TWEET OBJECTS @@ -123,7 +134,6 @@ driver.executeScript("document.body.style.zoom='35%'"); await driver.executeScript("document.body.style.zoom='20%'"); await driver.executeScript("window.scrollTo(0, 0)"); //await driver.executeScript("window.scrollTo(0, -document.body.scrollHeight)"); - await driver.sleep(1*5000) //WAIT 5 SECONDS TO GIVE BROWSER TIME TO SET ITSELF UP await elements.waitFor(driver,threadTweet.x.containsDivs,args.timeOut); //WAIT UNTIL THREAD IS POPULATED WITH DIVS @@ -133,19 +143,23 @@ driver.executeScript("document.body.style.zoom='35%'"); debuglog(threadTweet.x.path,2); //PRINTS XPATH TO CURRENT ITERATE DIV threadTweet.entryIsOpen = await elements.doesExist(driver,threadTweet.x.entryTweet) // CHECKS IF THE CURRENT ITERATE DIV IS THE ONE USED TO OPEN THE THREAD if (!threadTweet.entryIsOpen){ //CURRENT ITERATE DIV DOES NOT CONTAIN THE TWEET USED TO OPEN THE THREAD + debuglog(`current tweet #${threadTweet.no} is not entry to thread`,2); await threadTweet.identifyElements(driver); //IDENTIFIES WHAT THE TWEET CONTAINS - debuglog(`current tweet #${threadTweet.no} is not entry to thread`,2); + processedTweets.forEach(function(u, uindex) { //CHECK IF THREAD TWEET HAS BEEN PROCESSED IN THIS SESSION + debuglog(`${u.url} exists at index ${uindex} ${(u.url == threadTweet.url)}`); + if (u.url == threadTweet.url) { //IF TWEET HAS BEEN PROCESSED AS PART OF ANOTHER THREAD, PASS ON THE IMAGE ID ARRAY AND FLAG + threadTweet.processed = true; + threadTweet.imgArray = u.imgArray; + debuglog(`thread tweet (${threadTweet.url}) adopted image array from matching tweet at url (${u.url})`,2) + } + }) debuglog(csvOutput); - - if (processedTweets.some(e => e.url == processedTweets.url)) { - debuglog("TWEET EXISTS IN PROCESSED ARRAY!!",2); - } if (!csvOutput.includes(threadTweet.url)) {//CODE TO RUN IF TWEET IS NOT IN CSV - debuglog(`Thread tweet #${threadTweet.no} has not been processed.`, 1); + debuglog(`Thread tweet #${threadTweet.no} is not in CSV.`, 1); await threadTweet.getElementProperties(driver); //COMPILE HEADER, BODY, AND FOOTER @@ -153,63 +167,37 @@ driver.executeScript("document.body.style.zoom='35%'"); threadTweet.printPreview()//PRINT TWEET PREVIEW - await threadTweet.downloadImages(driver,imgSavePath); + if (!threadTweet.processed) {//IF IMAGES HAVE ALREADY BEEN TRANSFERRED FROM TWITTER TO MASTODON, SKIP TO PREVENT SLOWDOWN & AVOID RATE LIMIT + await threadTweet.downloadImages(driver,imgSavePath); - await threadTweet.uploadImages(imgSavePath); + await threadTweet.uploadImages(imgSavePath); + } } - threadTweetArray.push(threadTweet); - processedTweets.push(threadTweet); + threadTweetArray.push(threadTweet);//SENDS THREAD TWEET TO LIST OF CURRENT THREAD'S TWEETS + processedTweets.push(threadTweet);//SENDS THREAD TWEET TO LIST OF ALL PROCESSED TWEETS WITHOUT REGARD TO ORDER OR DUPLICATES } } - - var csvArray = csvOutput.split(/[\r\n]+/); - for (var a = 0;a < threadTweetArray.length; a++) {//SET TWEET OBJECT ID TO SAVED ID IF IT EXISTS IN CSV - debuglog(`CSV ARRAY: ${csvArray}`,2); - csvArray.forEach(function(row, csvIndex) { - debuglog(`csv row: ${row}`); - rowArr = row.split(","); - debuglog(`searching for '${threadTweetArray[a].url}' in '${row}'`,2) - if (row.includes(threadTweetArray[a].url)){ - debuglog(`URL Exists at index ${csvIndex} of csv`,2); - threadTweetArray[a].id = rowArr[1]; - threadTweetArray[a].posted = true; - } - }) - } - - threadTweetArray.forEach(twt =>{//LIST IDs THAT WERE DERIVED FROM CSV - debuglog(`${twt.no} id: ${twt.id}`,2) - }) - - for (var a = 0;a < threadTweetArray.length; a++) {//POST TO MASTODON REFERENCING ID OF PRIOR OBJECT AS PROMPT - if (a != 0) {threadTweetArray[a].prompt = threadTweetArray[a - 1].id} - if (!threadTweetArray[a].posted){ - debuglog(`posting tweet: ${threadTweetArray[a].no} to mastodon in reply to id: ${threadTweetArray[a].prompt}`, 2); - threadTweetArray[a].id = await mastodon.postStatus(threadTweetArray[a],csvFileName,csvOutput) - debuglog(`POSTED TO MASTODON AND GOT BACK AN ID OF: ${threadTweetArray[a].id}`,2) + + for (var a = 0;a < threadTweetArray.length; a++) { //FOR EVERY THREAD TWEET, LINK TO PARENT IF NEEDED & MARK AS PARENT + if (a != 0) { + threadTweetArray[a].prompturl = threadTweetArray[a - 1].url + threadTweetArray[a].isReply = true; } } - - await driver.close(); - await driver.switchTo().window(parent); + homeTweet.prompturl = threadTweetArray[threadTweetArray.length-1].url; //SET HOME (ENTRY) TWEET'S PARENT URL TO THAT OF LAST TWEET IN THREAD + homeTweet.isReply = true; //MARK HOME (ENTRY) TWEET AS PARENT + allTweetsArray.push(homeTweet); //HOME TWEET NEEDS TO BE SENT TO POST QUEUE BEFORE THREAD TWEETS + for (var aa = threadTweetArray.length;aa > 0; aa--) { + allTweetsArray.push(threadTweetArray[aa-1]); //SEND THREAD TWEETS TO POST QUEUE + } + await driver.close(); //CLOSE THREAD TAB + await driver.switchTo().window(parent); //MAKE SURE MAIN TIMELINE TAB IS OPEN + } else { + allTweetsArray.push(homeTweet); //SEND HOME TWEET TO POST QUEUE EVEN IF THERE ARE NO THREAD TWEETS } - await homeTweet.getElementProperties(driver); - - homeTweet.compileText();//COMPILE TEXT FOR CROSS-POST - - homeTweet.printPreview();//PRINT TWEET PREVIEW - - await homeTweet.downloadImages(driver,imgSavePath);//DOWNLOAD IMAGES FROM TWITTER - - await homeTweet.uploadImages(imgSavePath);//UPLOAD IMAGES TO MASTODON - - if (threadTweetArray.length>0) {homeTweet.prompt = threadTweetArray[threadTweetArray.length-1].id;} - debuglog(`Publishing post ${homeTweet.no} to Mastodon...`,2); - homeTweet.id = await mastodon.postStatus(homeTweet,csvFileName,csvOutput); - - processedTweets.push(homeTweet); + processedTweets.push(homeTweet); //SEND HOME TWEET TO LIST OF ALL PROCESSED TWEETS WITHOUT REGARD TO ORDER OR DUPLICATES } else { //HOME TWEET EXISTS IN CSV debuglog(`Tweet #${homeTweet.no} has already been processed.`,1); //HOME TWEET EXISTS IN CSV } @@ -218,6 +206,52 @@ driver.executeScript("document.body.style.zoom='35%'"); } + var csvArray = csvOutput.split(/[\r\n]+/); + for (var b = 0;b < (allTweetsArray.length); b++) {//SCANS CSV FOR TWEET CORSSPOST DATA TO ADOPT + + allTweetsArray[b].posted = csvArray.some((row,rowindex) => { //LOOKS THROUGH CSV FOR POST'S ID IN CASE IT HAS BEEN PROCESSED + rowArr = row.split(","); + if (rowArr[0] == allTweetsArray[b].url){ + debuglog(`URL Exists at index ${rowindex} of csv`,2); + allTweetsArray[b].id = rowArr[1]; + return true; + } + }) + csvArray.some((row,rowindex) => {//LOOKS THROUGH CSV FOR TWEET'S PARENT + rowArr = row.split(","); + if (rowArr[0] === allTweetsArray[b].prompturl){ + console.log(rowArr[0]); + console.log(allTweetsArray[b].prompturl); + debuglog(`URL Exists at index ${rowindex} of csv`,2); + allTweetsArray[b].prompt = rowArr[1]; + return true; + } + }) + } + + allTweetsArray = allTweetsArray.filter((v,i,a)=>a.findLastIndex(v2=>(v2.url === v.url))===i) //FILTERS OUR DUPLICATE TWEETS WITH PREFERENCE FOR MOST RECENTLY ADDED + allTweetsArray = [... allTweetsArray.reverse()] //REVERSE TWEET ARRAY ORDER SO THEY CAN BE NEATLY PROCESSED IN REVERSE ORDER + for (t = 0;t < allTweetsArray.length;t++) { //ITERATES THROUGH LIST OF PROCESSED TWEETS, SENDS UNPOSTED TWEETS TO MASTODON, AND REMEMBERS ITS ID FOR FUTURE REPLIES + twt = allTweetsArray[t] + if (!twt.posted) { + if (twt.isReply){ + if ((twt.prompt == 0) && (twt.prompturl != 0)){ //ONLY LOOKS FOR PARENT ID IF PROMPT IS UNFULFILLED BUT TWEET EXPECTS A PROMPT + allTweetsArray.some((cue,cueindex) =>{ + if (twt.prompturl == cue.url){ + twt.prompt = cue.id; + return true; + } + }) + } + } + twt.id = await mastodon.postStatus(twt,csvFileName,csvOutput); + } + } + allTweetsArray.forEach((twt,index) => {//DEBUGGING, PRINTS INFO ON TWEETS + debuglog(`${String(index).padStart(2,0)}: ${twt.body.substring(0,20)}..., ${twt.url}, ${twt.prompturl}, ${twt.id}, ${twt.prompt}, is reply: ${twt.isReply}, posted: ${twt.posted}`,2) + }); + + debuglog("Cleaning up...",1); //REMOVE SAVED IMAGES fs.rm(imgSavePath, { recursive: true, force: true }, (error) => { debuglog(error,2);