Add array::zip and Related Macros (#2370)

* zip / zip_into
* Deprecate `add_array` in favour of `push_all` on lists.
* Add support for generic lists for zip.

---------

Co-authored-by: Christoffer Lerno <christoffer@aegik.com>

lib/std/collections/elastic_array.c3

@@ -121,7 +121,24 @@ fn usz ElasticArray.add_all_to_limit(&self, ElasticArray* other_list)
 
  @param [in] array
 *>
-fn usz ElasticArray.add_array_to_limit(&self, Type[] array)
+fn usz ElasticArray.add_array_to_limit(&self, Type[] array) @deprecated("Use push_all_to_limit")
+{
+	if (!array.len) return 0;
+	foreach (i, &value : array)
+	{
+		if (self.size == MAX_SIZE) return array.len - i;
+		self.entries[self.size++] = *value;
+	}
+	return 0;
+}
+
+<*
+ Add as many values from this array as possible, returning the
+ number of elements that didn't fit.
+
+ @param [in] array
+*>
+fn usz ElasticArray.push_all_to_limit(&self, Type[] array)
 {
 	if (!array.len) return 0;
 	foreach (i, &value : array)
@@ -139,7 +156,7 @@ fn usz ElasticArray.add_array_to_limit(&self, Type[] array)
  @require array.len + self.size <= MAX_SIZE : `Size would exceed max.`
  @ensure self.size >= array.len
 *>
-fn void ElasticArray.add_array(&self, Type[] array)
+fn void ElasticArray.add_array(&self, Type[] array) @deprecated("Use push_all")
 {
 	if (!array.len) return;
 	foreach (&value : array)
@@ -148,6 +165,21 @@ fn void ElasticArray.add_array(&self, Type[] array)
 	}
 }
 
+<*
+ Add the values of an array to this list.
+
+ @param [in] array
+ @require array.len + self.size <= MAX_SIZE : `Size would exceed max.`
+ @ensure self.size >= array.len
+*>
+fn void ElasticArray.push_all(&self, Type[] array)
+{
+	if (!array.len) return;
+	foreach (&value : array)
+	{
+		self.entries[self.size++] = *value;
+	}
+}
 
 
 <*

lib/std/collections/list.c3

@@ -57,7 +57,7 @@ fn List* List.tinit(&self, usz initial_capacity = 16)
 fn List* List.init_with_array(&self, Allocator allocator, Type[] values)
 {
 	self.init(allocator, values.len) @inline;
-	self.add_array(values) @inline;
+	self.push_all(values) @inline;
 	return self;
 }
 
@@ -70,7 +70,7 @@ fn List* List.init_with_array(&self, Allocator allocator, Type[] values)
 fn List* List.tinit_with_array(&self, Type[] values)
 {
 	self.tinit(values.len) @inline;
-	self.add_array(values) @inline;
+	self.push_all(values) @inline;
 	return self;
 }
 
@@ -199,7 +199,21 @@ fn Type[] List.array_view(&self)
  @param [in] array
  @ensure self.size >= array.len
 *>
-fn void List.add_array(&self, Type[] array)
+fn void List.add_array(&self, Type[] array) @deprecated("Use push_all")
+{
+	if (!array.len) return;
+	self.reserve(array.len);
+	usz index = self.set_size(self.size + array.len);
+	self.entries[index : array.len] = array[..];
+}
+
+<*
+ Add the values of an array to this list.
+
+ @param [in] array
+ @ensure self.size >= array.len
+*>
+fn void List.push_all(&self, Type[] array)
 {
 	if (!array.len) return;
 	self.reserve(array.len);

lib/std/core/array.c3

@@ -1,5 +1,7 @@
 module std::core::array;
+
 import std::core::array::slice;
+import std::collections::pair;
 
 <*
  Returns true if the array contains at least one element, else false
@@ -18,6 +20,7 @@ macro bool contains(array, element)
 	return false;	
 }
 
+
 <*
  Return the first index of element found in the array, searching from the start.
  
@@ -35,6 +38,7 @@ macro index_of(array, element)
 	return NOT_FOUND?;
 }
 
+
 <*
  Slice a 2d array and create a Slice2d from it.
 
@@ -74,6 +78,7 @@ macro rindex_of(array, element)
 	return NOT_FOUND?;
 }
 
+
 <*
  Concatenate two arrays or slices, returning a slice containing the concatenation of them.
 
@@ -99,6 +104,7 @@ macro concat(Allocator allocator, arr1, arr2) @nodiscard
 	}
 	return result;
 }
+
 <*
  Concatenate two arrays or slices, returning a slice containing the concatenation of them,
  allocated using the temp allocator.
@@ -110,4 +116,225 @@ macro concat(Allocator allocator, arr1, arr2) @nodiscard
  @require @typeis(arr1[0], $typeof(arr2[0])) : "Arrays must have the same type"
  @ensure return.len == arr1.len + arr2.len
 *>
-macro tconcat(arr1, arr2) @nodiscard => concat(tmem, arr1, arr2);
+macro tconcat(arr1, arr2) @nodiscard => concat(tmem, arr1, arr2);
+
+
+<*
+ Zip together two separate arrays/slices into a single array of Pairs or return values. Values will
+ be collected up to the length of the shorter array if `fill_with` is left undefined; otherwise, they
+ will be collected up to the length of the LONGER array, with missing values in the shorter array being
+ assigned to the value of `fill_with`. Return array elements do not have to be of the same type.
+
+ For example:
+ ```c3
+ uint[] chosen_session_ids = server::get_random_sessions(instance)[:128];
+ String[200] refreshed_session_keys = prng::new_keys_batch();
+
+ Pair { uint, String }[] sessions_meta = array::zip(mem, chosen_session_ids, refreshed_session_keys);
+ // The resulting Pair{}[] slice is then length of the shortest of the two arrays, so 128.
+
+ foreach (i, &sess : sessions:meta) {
+     // distribute new session keys to associated instance IDs
+ }
+ ```
+
+ Or:
+ ```c3
+ String[] client_names = server::online_usernames(instance);
+ uint128[] session_ids = server::user_keys();
+
+ // in this example, we 'know' ahead of time that 'session_ids' can only ever be SHORTER
+ //   than 'client_names', but never longer, because it's possible new users have logged
+ //   in without getting whatever this 'session ID' is delegated to them.
+ Pair { String, uint128 }[] zipped = array::tzip(client_names, session_ids, fill_with: uint128.max);
+
+ server::refresh_session_keys_by_pair(zipped)!;
+ ```
+
+ ### When an `operation` is supplied...
+ Apply an operation to each element of two slices or arrays and return the results of
+ each operation into a newly allocated array.
+
+ This essentially combines Iterable1 with Iterable2 using the `operation` functor.
+
+ See the functional `zipWith` construct, which has a more appropriate name than, e.g., `map`;
+   a la: https://hackage.haskell.org/package/base-4.21.0.0/docs/Prelude.html#v:zipWith
+
+ Similar to "normal" `zip`, this macro pads the shorter input array with a given `fill_with`, or
+ an empty value if one isn't supplied. This `fill_with` is supplied to the `operation` functor
+ _BEFORE_ calculating its result while zipping.
+
+ For example: a functor of `fn char (char a, char b) => a + b` with a `fill_with` of 7,
+ where the `left` array is the shorter iterable, will put 7 into that lambda in each place
+ where `left` is being filled in during the zip operation.
+
+ @param [&inout] allocator : "The allocator to use; default is the heap allocator."
+ @param [in] left : "The left-side array. These items will be placed as the First in each Pair"
+ @param [in] right : "The right-side array. These items will be placed as the Second in each Pair"
+ @param #operation : "The function to apply. Must have a signature of `$typeof(a) (a, b)`, where the type of 'a' and 'b' is the element type of left/right respectively."
+ @param fill_with : "The value used to fill or pad the shorter iterable to the length of the longer one while zipping."
+
+ @require @is_valid_list(left) &&& @is_valid_list(right) : "Left and right sides must be integer indexable"
+ @require @is_valid_operation(#operation, left, right) : "The operator must take two parameters matching the elements of the left and right side"
+ @require @is_valid_fill(left, right, fill_with) : "The specified fill value does not match either the left or the right array's underlying type."
+
+*>
+macro @zip(Allocator allocator, left, right, #operation = EMPTY_MACRO_SLOT, fill_with = EMPTY_MACRO_SLOT) @nodiscard
+{
+	var $LeftType = $typeof(left[0]);
+	var $RightType = $typeof(right[0]);
+
+	var $Type = Pair { $LeftType, $RightType };
+	bool $is_op = @is_valid_macro_slot(#operation);
+	$if $is_op:
+		$Type = $typeof(#operation).returns;
+	$endif
+
+
+	usz left_len = find_len(left);
+	usz right_len = find_len(right);
+
+	$LeftType left_fill;
+	$RightType right_fill;
+	usz result_len = min(left_len, right_len);
+	bool $has_fill = @is_valid_macro_slot(fill_with);
+	$if $has_fill:
+		switch
+		{
+			case left_len > right_len:
+				$if !$defined(($RightType)fill_with):
+					unreachable();
+				$else
+					right_fill = ($RightType)fill_with;
+					result_len = left_len;
+				$endif
+			case left_len < right_len:
+				$if !$defined(($LeftType)fill_with):
+					unreachable();
+				$else
+					left_fill = ($LeftType)fill_with;
+					result_len = right_len;
+				$endif
+		}
+	$endif
+
+	if (result_len == 0) return ($Type[]){};
+
+	$Type[] result = allocator::alloc_array(allocator, $Type, result_len);
+
+	foreach (idx, &item : result)
+	{
+		$if $is_op:
+			var $LambdaType = $typeof(fn $Type ($LeftType a, $RightType b) => ($Type){});
+			$LambdaType $operation = ($LambdaType)#operation;
+			$LeftType lval = idx >= left_len ? left_fill : left[idx];
+			$RightType rval = idx >= right_len ? right_fill : right[idx];
+			*item = $operation(lval, rval);
+		$else
+			*item = {
+				idx >= left_len ? left_fill : left[idx],
+				idx >= right_len ? right_fill : right[idx]
+			};
+		$endif
+	}
+
+	return result;
+}
+
+<*
+ Array 'zip' using the temp allocator.
+
+ @param [in] left : "The left-side array. These items will be placed as the First in each Pair"
+ @param [in] right : "The right-side array. These items will be placed as the Second in each Pair"
+ @param #operation : "The function to apply. Must have a signature of `$typeof(a) (a, b)`, where the type of 'a' and 'b' is the element type of left/right respectively."
+ @param fill_with : "The value used to fill or pad the shorter iterable to the length of the longer one while zipping."
+
+ @require @is_valid_list(left) &&& @is_valid_list(right) : "Left and right sides must be integer indexable"
+ @require @is_valid_operation(#operation, left, right) : "The operator must take two parameters matching the elements of the left and right side"
+ @require @is_valid_fill(left, right, fill_with) : "The specified fill value does not match either the left or the right array's underlying type."
+
+*>
+macro @tzip(left, right, #operation = EMPTY_MACRO_SLOT, fill_with = EMPTY_MACRO_SLOT) @nodiscard
+{
+	return @zip(tmem, left, right, #operation, fill_with);
+}
+
+
+<*
+ Apply an operation to each element of two slices or arrays and store the results of
+ each operation into the 'left' value.
+
+ This is useful because no memory allocations are required in order to perform the operation.
+
+ A good example of using this might be using algorithmic transformations on data in-place:
+ ```
+ char[] partial_cipher = get_next_plaintext_block();
+
+ array::@zip_into(
+     partial_cipher[ENCRYPT_OFFSET:BASE_KEY.len],
+	 BASE_KEY,
+	 fn char (char a, char b) => a ^ (b * 5) % 37
+ );
+ ```
+
+ This parameterizes the lambda function with left (`partial_cipher`) and right (`BASE_KEY`) slice
+ elements and stores the end result in-place within the left slice. This is in contrast to a
+ regular `zip_with` which will create a cloned final result and return it.
+
+ @param [inout] left : `Slice to store results of applied functor/operation.`
+ @param [in] right : `Slice to apply in the functor/operation.`
+ @param #operation : "The function to apply. Must have a signature of `$typeof(a) (a, b)`, where the type of 'a' and 'b' is the element type of left/right respectively."
+
+ @require @is_valid_list(left) : "Expected a valid list"
+ @require @is_valid_list(right) : "Expected a valid list"
+ @require find_len(right) >= find_len(left) : `Right side length must be >= the destination (left) side length; consider using a sub-array of data for the assignment.`
+ @require @assignable_to(#operation, $typefrom(@zip_into_fn(left, right))) : "The functor must use the same types as the `left` and `right` inputs, and return a value of the `left` type."
+*>
+macro @zip_into(left, right, #operation)
+{
+	$typefrom(@zip_into_fn(left, right)) $operation = #operation;
+
+	foreach (i, &v : left) *v = $operation(left[i], right[i]);
+}
+
+
+// --- helper functions
+
+macro typeid @zip_into_fn(#left, #right) @private @const
+{
+	return @typeid(fn $typeof(#left[0]) ($typeof(#left[0]) l, $typeof(#right[0]) r) => l);
+}
+macro bool @is_valid_operation(#operation, #left, #right) @const @private
+{
+	$switch:
+		$case @is_empty_macro_slot(#operation):
+			return true;
+		$case @typekind(#operation) != FUNC:
+			return false;
+		$default:
+			return $defined(#operation(#left[0], #right[0]));
+	$endswitch
+}
+
+macro bool @is_valid_list(#expr) @const @private
+{
+	return $defined(#expr[0]) &&& ($defined(#expr.len) ||| $defined(#expr.len()));
+}
+
+macro bool @is_valid_fill(left, right, fill_with) @private
+{
+	if (@is_empty_macro_slot(fill_with)) return true;
+	usz left_len = @select($defined(left.len()), left.len(), left.len);
+	usz right_len = @select($defined(right.len()), right.len(), right.len);
+	if (left_len == right_len) return true;
+	return left_len > right_len ? $defined(($typeof(right[0]))fill_with) : $defined(($typeof(left[0]))fill_with);
+}
+
+macro usz find_len(list)
+{
+	$if $defined(list.len()):
+		return list.len();
+	$else
+		return list.len;
+	$endif
+}

lib/std/core/mem.c3

@@ -684,6 +684,12 @@ macro @clone(value) @builtin @nodiscard
 	return allocator::clone(mem, value);
 }
 
+<*
+ @param value : "The value to clone"
+ @return "A pointer to the cloned value"
+*>
+macro @clone_slice(value) @builtin @nodiscard => allocator::clone_slice(mem, value);
+
 <*
  @param value : "The value to clone"
  @return "A pointer to the cloned value, which must be released using free_aligned"
@@ -706,6 +712,12 @@ macro @tclone(value) @builtin @nodiscard
 	$endif
 }
 
+<*
+ @param value : "The value to clone"
+ @return "A pointer to the cloned value"
+*>
+macro @tclone_slice(value) @builtin @nodiscard => allocator::clone_slice(tmem, value);
+
 fn void* malloc(usz size) @builtin @inline @nodiscard
 {
 	return allocator::malloc(mem, size);

lib/std/core/mem_allocator.c3

@@ -317,6 +317,23 @@ macro clone(Allocator allocator, value) @nodiscard
 	return new(allocator, $typeof(value), value);
 }
 
+<*
+ @param [&inout] allocator : "The allocator used to clone"
+ @param slice : "The slice to clone"
+ @return "A pointer to the cloned slice"
+
+ @require @typekind(slice) == SLICE || @typekind(slice) == ARRAY
+*>
+macro clone_slice(Allocator allocator, slice) @nodiscard
+{
+	var $Type = $typeof(slice[0]);
+
+	$Type[] new_arr = new_array(allocator, $Type, slice.len);
+	mem::copy(new_arr.ptr, &slice[0], slice.len * $Type.sizeof);
+
+	return new_arr;
+}
+
 <*
  Clone overaligned values. Must be released using free_aligned.
 

lib/std/core/values.c3

@@ -51,6 +51,7 @@ macro @select(bool $bool, #value_1, #value_2) @builtin
 		return #value_2;
 	$endif
 }
+
 macro promote_int_same(x, y)
 {
 	$if @is_int(x):