Google script says - Exceeded maximum execution time

Question

I am using the below script to delete duplicate rows from the google spreadsheet. The script was working good but as the data in the spreadsheet is being added daily, now the script is throwing "Exceeded maximum execution time" error. As I am new to scripting I don't understand what is my problem.

Could someone help me in solving this problem of mine.

    function Deleteduplicates() {
  var SpreadSheetKey = "My key";
  var sheetD = SpreadsheetApp.openById(SpreadSheetKey).getSheetByName("Daily");
  var sheetW = SpreadsheetApp.openById(SpreadSheetKey).getSheetByName("Weekly");
  var dataD = sheetD.getDataRange().getValues();
  var dataW = sheetW.getDataRange().getValues();
  //Daily
  var newDataD = new Array();
  for(i in dataD){
    var row = dataD[i];
    var duplicate = false;
    for(j in newDataD){
      if(row.join() == newDataD[j].join()){
        duplicate = true;
      }
    }
    if(!duplicate){
      newDataD.push(row);
    }
  }
  //weekly
  var newDataW = new Array();
  for(i in dataW){
    var row = dataW[i];
    var duplicate = false;
    for(j in newDataW){
      if(row.join() == newDataW[j].join()){
        duplicate = true;
      }
    }
    if(!duplicate){
      newDataW.push(row);
    }
  }
  sheetD.clearContents();
  sheetW.clearContents();
  sheetD.getRange(1, 1, newDataD.length, newDataD[0].length).setValues(newDataD);
  sheetW.getRange(1, 1, newDataW.length, newDataW[0].length).setValues(newDataW);
}

Answer 1

Conceptually, this should be quite a bit faster. I have not tried it on a large data set. The first version will leave the rows sorted as they were originally. The second version will be faster but will leave the rows sorted according to the columns from first to last on first text.

function Deleteduplicates() {
  var SpreadSheetKey = "My key";
  var ss = SpreadsheetApp.openById(SpreadSheetKey);
  var sheetD = ss.getSheetByName("Daily");
  var sheetW = ss.getSheetByName("Weekly");
  var sheets = [sheetD, sheetW];
  var toSs = {};
  for(s in sheets) {
    var data = sheets[s].getDataRange().getValues();
    for(i in data){
      // EDIT: remove commas from join("") for blank test
      data[i].unshift(data[i].join(""),(1000000 + i).toString());
      }
    data.sort();
    // remove blank rows -- Edit
    var blank = 0;
    while(data[blank][0].trim().length == 0) {blank++};
    if(blank > 0) data.splice(0, blank);
    // end Edit
    var len = data.length - 1;
    for(var x = len; x > 0; x-- ) {
      if(data[x][0] == data[x-1][0]) {
        data.splice(x, 1);
        };
      };
    for(i in data) {
      data[i].splice( 0, 1);
      };
    data.sort();
    for(i in data) {
      data[i].splice(0, 1);
      };
    toSs[sheets[s].getSheetName()] = data;
  };
  for(s in sheets) {
    var data = toSs[sheets[s].getSheetName()];
    sheets[s].clearContents();
    sheets[s].getRange(1, 1, data.length, data[0].length).setValues(data);
  }
}

Faster leaving rows sorted by join() created to test for duplicates

function Deleteduplicates() {
  var SpreadSheetKey = "My key";
  var ss = SpreadsheetApp.openById(SpreadSheetKey);
  var sheetD = ss.getSheetByName("Daily");
  var sheetW = ss.getSheetByName("Weekly");
  var sheets = [sheetD, sheetW];
  var toSs = {};
  for(s in sheets) {
    var data = sheets[s].getDataRange().getValues();
    for(i in data){
      // EDIT: remove commas from join("") for blank test
      data[i].unshift(data[i].join(""));
      }
    data.sort();
    // remove blank rows -- Edit
    var blank = 0;
    while(data[blank][0].trim().length == 0) {blank++};
    if(blank > 0) data.splice(0, blank);
    // end Edit
    var len = data.length - 1;
    for(var x = len; x > 0; x-- ) {
      if(data[x][0] == data[x-1][0]) {
        data.splice(x, 1);
        };
      };
    for(i in data) {
      data[i].splice( 0, 1);
      };
    toSs[sheets[s].getSheetName()] = data;
    };
  for(s in sheets) {
    var data = toSs[sheets[s].getSheetName()];
    sheets[s].clearContents();
    sheets[s].getRange(1, 1, data.length, data[0].length).setValues(data);
  }
}

Edited per Henrique's comment.

Edited 5/8: Remove blank rows(2 edited areas marked)