How to make array distinct

Question

I have an array of integer and string fields. To make it distinct I currently copy line by line into new array and with each record I check if the record already exists in new, if not I copy it. At the end I copy new array back to original.

It works, but is slow. Is there any faster, easier way to do this?

TArrayMixed = record
    Field1: integer;
    Field2: integer;
    Field3: integer;
    Field4: string;
    Field5: string;
    Field6: string;
 end;

procedure TForm1.Button10Click(Sender: TObject);
var
  ArrayMixed, ArrayMixed_tmp: array of TArrayMixed;
  i, j, vIdx: integer;
  vExists: boolean;
begin
  SetLength(ArrayMixed, 100000);
  for i := 0 to 99999 do
  begin
    ArrayMixed[i].Field1 := 1 + Random(5);
    ArrayMixed[i].Field2 := 1 + Random(5);
    ArrayMixed[i].Field3 := 1 + Random(5);
    ArrayMixed[i].Field4 := 'String';
    ArrayMixed[i].Field5 := 'Another string';
    ArrayMixed[i].Field6 := 'New string';
  end;

  // Sort
  TArray.Sort<TArrayMixed > (ArrayMixed, TComparer<TArrayMixed > .Construct(function(const Left, Right: TArrayMixed): Integer
    begin
      Result := MyCompareAMixed(Left, Right);
    end
    ));

  // Distinct
  SetLength(ArrayMixed_tmp, Length(ArrayMixed));
  vIdx := 0;
  for i := Low(ArrayMixed) to High(ArrayMixed) do
  begin
    vExists := False;
    for j := Low(ArrayMixed_tmp) to vIdx - 1 do
      if (ArrayMixed_tmp[j].Field1 = ArrayMixed[i].Field1) and
        (ArrayMixed_tmp[j].Field2 = ArrayMixed[i].Field2) and
        (ArrayMixed_tmp[j].Field3 = ArrayMixed[i].Field3) and
        (ArrayMixed_tmp[j].Field4 = ArrayMixed[i].Field4) and
        (ArrayMixed_tmp[j].Field5 = ArrayMixed[i].Field5) and
        (ArrayMixed_tmp[j].Field6 = ArrayMixed[i].Field6) then
      begin
        vExists := True;
        Break;
      end;

    if not vExists then
    begin
      ArrayMixed_tmp[vIdx] := ArrayMixed[i];
      Inc(vIdx);
    end;
  end;
  SetLength(ArrayMixed_tmp, vIdx);

  // now copy back to original array
  SetLength(ArrayMixed, 0);
  SetLength(ArrayMixed, Length(ArrayMixed_tmp));
  for i := Low(ArrayMixed_tmp) to High(ArrayMixed_tmp) do
    ArrayMixed[i] := ArrayMixed_tmp[i];

  sleep(0);

end;

Edit:

Since in real data the strings are not all the same, the part where it makes distinct array is slower when original array is filled like this:

Edit #2: (copied wrong code in Edit #1)

for i := 0 to 999999 do
  begin
    ArrayMixed[i].Field1 := 1 + Random(5);
    ArrayMixed[i].Field2 := 1 + Random(5);
    ArrayMixed[i].Field3 := 1 + Random(5);
    ArrayMixed[i].Field4 := 'String'+IntToStr(i mod 5);
    ArrayMixed[i].Field5 := 'Another string'+IntToStr(i mod 5);
    ArrayMixed[i].Field6 := 'New string'+IntToStr( i mod 5);
  end;

Edit #3: Publishing code for sorting - only first 3 fields are sorted!

TMyArray3 = array[1..3] of Integer;

    function CompareIntegerArray3(const lhs, rhs: TMyArray3): Integer;
    var
      i: Integer;
    begin
      Assert(Length(lhs) = Length(rhs));
      for i := low(lhs) to high(lhs) do
        if lhs[i] < rhs[i] then
          exit(-1)
        else if lhs[i] > rhs[i] then
          exit(1);

      exit(0);
    end;

    function GetMyArrayAMixed(const Value: TArrayMixed): TMyArray3;
    begin
      Result[1] := Value.Field1;
      Result[2] := Value.Field2;
      Result[3] := Value.Field3;
    end;

    function MyCompareAMixed(const lhs, rhs: TArrayMixed): Integer;
    begin
      Result := CompareIntegerArray3(GetMyArrayAMixed(lhs), GetMyArrayAMixed(rhs));
    end;

Answer 1

• Implement some methods (equality check, hashcode) for the record to get away with a lot of boilerplate code

---

type
  TArrayMixed = record
    Field1: integer;
    Field2: integer;
    Field3: integer;
    Field4: string;
    Field5: string;
    Field6: string;
    class operator Equal( const a, b: TArrayMixed ): Boolean;
    class function Compare( const L, R: TArrayMixed ): integer; overload; static;
    function Compare( const Other: TArrayMixed ): integer; overload;
    function GetHashCode( ): integer;
  end;

  { TArrayMixed }

class function TArrayMixed.Compare( const L, R: TArrayMixed ): integer;
begin
  Result := L.Compare( R );
end;

function TArrayMixed.Compare( const Other: TArrayMixed ): integer;
begin
  Result := Field1 - Other.Field1;
  if Result = 0
  then
    begin
      Result := Field2 - Other.Field2;
      if Result = 0
      then
        begin
          Result := Field3 - Other.Field3;
          if Result = 0
          then
            begin
              Result := CompareStr( Field4, Other.Field4 );
              if Result = 0
              then
                begin
                  Result := CompareStr( Field5, Other.Field5 );
                  if Result = 0
                  then
                    begin
                      Result := CompareStr( Field6, Other.Field6 );
                    end;
                end;
            end;
        end;
    end;
end;

class operator TArrayMixed.Equal( const a, b: TArrayMixed ): Boolean;
begin
  Result := true
  {} and ( a.Field1 = b.Field1 )
  {} and ( a.Field2 = b.Field2 )
  {} and ( a.Field3 = b.Field3 )
  {} and ( a.Field4 = b.Field4 )
  {} and ( a.Field5 = b.Field5 )
  {} and ( a.Field6 = b.Field6 );
end;

function TArrayMixed.GetHashCode: integer;
begin
{$IFOPT Q+}
{$Q-}
{$DEFINE SET_Q_ON}
{$ENDIF}
  Result := 17;
  Result := Result * 31 + Field1;
  Result := Result * 31 + Field2;
  Result := Result * 31 + Field3;
  Result := Result * 31 + Field4.GetHashCode;
  Result := Result * 31 + Field5.GetHashCode;
  Result := Result * 31 + Field6.GetHashCode;
{$IFDEF SET_Q_ON}
{$Q+}
{$UNDEF SET_Q_ON}
{$ENDIF}
end;

• Use a TDictionary as a HashSet to check for duplicates

---

procedure Test;
var
  arr1, arr2: TArray<TArrayMixed>;
  idx       : integer;
  lst       : TDictionary<TArrayMixed, integer>;
begin
  // fill the array
  SetLength( arr1, 100000 );
  for idx := low( arr1 ) to high( arr1 ) do
    begin
      arr1[ idx ].Field1 := 1 + Random( 5 );
      arr1[ idx ].Field2 := 1 + Random( 5 );
      arr1[ idx ].Field3 := 1 + Random( 5 );
      arr1[ idx ].Field4 := 'String' + IntToStr( idx mod 5 );
      arr1[ idx ].Field5 := 'Another string' + IntToStr( idx mod 5 );
      arr1[ idx ].Field6 := 'New string' + IntToStr( idx mod 5 );
    end;

  // distinct
  lst := TDictionary<TArrayMixed, integer>.Create( TEqualityComparer<TArrayMixed>.Construct(
    function( const L, R: TArrayMixed ): Boolean
    begin
      Result := ( L = R );
    end,
    function( const i: TArrayMixed ): integer
    begin
      Result := i.GetHashCode( );
    end ) );
  try
    for idx := low( arr1 ) to high( arr1 ) do
      begin
        lst.AddOrSetValue( arr1[ idx ], 0 );
      end;

    arr2 := lst.Keys.ToArray;

  finally
    lst.Free;
  end;
end;

Answer 2

I would do something like this. You basically create the result on the fly and sort at the same time using a binary search to remove the duplicates.

function RemoveDuplicates(aSourceArray: TArray<TArrayMixed>): TArray<TArrayMixed>;
var
  i: Integer;
  index: Integer;
  sortList: TList<TArrayMixed>;
begin
  sortList := TList<TArrayMixed>.Create;
  try
    for i := Low(aSourceArray) to High(aSourceArray) do
    begin
      if not sortList.BinarySearch(aSourceArray[i], index,
        TDelegatedComparer<TArrayMixed>.Construct(
        function(const L, R: TArrayMixed): integer
        begin
          Result := L.Field1 - R.Field1;
          if Result <> 0 then Exit;
          Result := L.Field2 - R.Field2;
          if Result <> 0 then Exit;
          Result := L.Field3 - R.Field3;
          if Result <> 0 then Exit;
          Result := CompareStr(L.Field4, R.Field4);
          if Result <> 0 then Exit;
          Result := CompareStr(L.Field5, R.Field5);
          if Result <> 0 then Exit;
          Result := CompareStr(L.Field6, R.Field6);
        end)) then
      begin
        sortList.Insert(index, aSourceArray[i]);
      end;
    end;
    Result := sortList.ToArray;
  finally
    sortList.Free;
  end;
end;

To use this code you could do something like this:

procedure TForm1.Button10Click(Sender: TObject);
var
  ArrayMixed, ArrayMixed_tmp: TArray<TArrayMixed>;
  i: Integer;
begin
  SetLength(ArrayMixed, 100000);
  for i := 0 to 999999 do
  begin
    ArrayMixed[i].Field1 := 1 + Random(5);
    ArrayMixed[i].Field2 := 1 + Random(5);
    ArrayMixed[i].Field3 := 1 + Random(5);
    ArrayMixed[i].Field4 := 'String'+IntToStr(i mod 5);
    ArrayMixed[i].Field5 := 'Another string'+IntToStr(i mod 5);
    ArrayMixed[i].Field6 := 'New string'+IntToStr( i mod 5);
  end;
  ArrayMixed_tmp := RemoveDuplicates(ArrayMixed);
end;