[git commit] libbb/yescrypt: code shrink

[Denys Vlasenko]

commit: https://git.busybox.net/busybox/commit/?id=d18ac080e4bb7d63e0ec0dea16bacc6ac455f390
branch: https://git.busybox.net/busybox/commit/?id=refs/heads/master

Setting EINVAL in errno is not necessary, just error return works.

function                                             old     new   delta
yescrypt_kdf32_body                                 1434    1423     -11
yescrypt_r                                          1029     990     -39
------------------------------------------------------------------------------
(add/remove: 0/0 grow/shrink: 0/2 up/down: 0/-50)             Total: -50 bytes

Signed-off-by: Denys Vlasenko <vda.linux at googlemail.com>
---
 libbb/yescrypt/alg-yescrypt-common.c |  4 ++-
 libbb/yescrypt/alg-yescrypt-kdf.c    | 64 ++++++++++++++++++++++++++----------
 2 files changed, 49 insertions(+), 19 deletions(-)

diff --git a/libbb/yescrypt/alg-yescrypt-common.c b/libbb/yescrypt/alg-yescrypt-common.c
index 5d8be587a..e5d045058 100644
--- a/libbb/yescrypt/alg-yescrypt-common.c
+++ b/libbb/yescrypt/alg-yescrypt-common.c
@@ -288,8 +288,10 @@ char *yescrypt_r(
 	if (need > buflen || need < prefixlen)
 		goto fail;
 
-	if (yescrypt_kdf32(yctx, passwd, passwdlen, hashbin32))
+	if (yescrypt_kdf32(yctx, passwd, passwdlen, hashbin32)) {
+		dbg("error in yescrypt_kdf32");
 		goto fail;
+	}
 
 	dst = mempcpy(buf, setting, prefixlen);
 	*dst++ = '$';
diff --git a/libbb/yescrypt/alg-yescrypt-kdf.c b/libbb/yescrypt/alg-yescrypt-kdf.c
index 4c2cfe849..c998de51d 100644
--- a/libbb/yescrypt/alg-yescrypt-kdf.c
+++ b/libbb/yescrypt/alg-yescrypt-kdf.c
@@ -927,22 +927,42 @@ static int yescrypt_kdf32_body(
 
 	r = YCTX_param_r;
 	p = YCTX_param_p;
-	if ((uint64_t)r * (uint64_t)p >= 1 << 30)
+	if ((uint64_t)r * (uint64_t)p >= 1 << 30) {
+		dbg("r * n >= 2^30");
 		goto out_EINVAL;
-	if (N > UINT32_MAX)
+	}
+	if (N > UINT32_MAX) {
+		dbg("N > 0x%lx", (long)UINT32_MAX);
 		goto out_EINVAL;
-	if ((N & (N - 1)) != 0 || N <= 3 || r < 1 || p < 1)
+	}
+	if ((N & (N - 1)) != 0
+//TODO: ^^^^^^^^^^^^^^^^^^^^^^ do not check this, code guarantees power-of-2
+	 || N <= 3
+	 || r < 1
+	 || p < 1
+	) {
+		dbg("bad N, r or p");
 		goto out_EINVAL;
-	if (r > SIZE_MAX / 256 / p ||
-	    N > SIZE_MAX / 128 / r)
+	}
+	if (r > SIZE_MAX / 256 / p
+	 || N > SIZE_MAX / 128 / r
+	) {
+		/* 32-bit testcase: mkpasswd qweRTY123 at -+ '$y$jHT$123'
+		 * (works on 64-bit, needs buffer > 4Gbytes)
+		 */
+		dbg("r > SIZE_MAX / 256 / p? %c", "NY"[r > SIZE_MAX / 256 / p]);
+		dbg("N > SIZE_MAX / 128 / r? %c", "NY"[N > SIZE_MAX / 128 / r]);
 		goto out_EINVAL;
+	}
 	if (flags___YESCRYPT_RW) {
 		/* p cannot be greater than SIZE_MAX/Salloc on 64-bit systems,
 		   but it can on 32-bit systems.  */
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wtype-limits"
-		if (N / p <= 3 || p > SIZE_MAX / Salloc)
+		if (N / p <= 3 || p > SIZE_MAX / Salloc) {
+			dbg("bad p:%ld", (long)p);
 			goto out_EINVAL;
+		}
 #pragma GCC diagnostic pop
 	}
 
@@ -956,17 +976,23 @@ static int yescrypt_kdf32_body(
 	need = V_size;
 	B_size = (size_t)128 * r * p;
 	need += B_size;
-	if (need < B_size)
+	if (need < B_size) {
+		dbg("integer overflow at += B_size(%lu)", (long)B_size);
 		goto out_EINVAL;
+	}
 	XY_size = (size_t)256 * r;
 	need += XY_size;
-	if (need < XY_size)
+	if (need < XY_size) {
+		dbg("integer overflow at += XY_size(%lu)", (long)XY_size);
 		goto out_EINVAL;
+	}
 	if (flags___YESCRYPT_RW) {
 		size_t S_size = (size_t)Salloc * p;
 		need += S_size;
-		if (need < S_size)
+		if (need < S_size) {
+			dbg("integer overflow at += S_size(%lu)", (long)S_size);
 			goto out_EINVAL;
+		}
 	}
 	if (yctx->local->aligned_size < need) {
 		free_region(yctx->local);
@@ -1050,8 +1076,8 @@ static int yescrypt_kdf32_body(
 	/* Success! */
 	return 0;
 
-out_EINVAL:
-	errno = EINVAL;
+ out_EINVAL:
+	//bbox does not need this: errno = EINVAL;
 	return -1;
 }
 
@@ -1079,7 +1105,7 @@ int yescrypt_kdf32(
 
 	/* Support for hash upgrades has been temporarily removed */
 	if (g) {
-		errno = EINVAL;
+		//bbox does not need this: errno = EINVAL;
 		return -1;
 	}
 
@@ -1093,15 +1119,17 @@ int yescrypt_kdf32(
 		    flags | YESCRYPT_ALLOC_ONLY, N, t,
 		    buf32) != -3
 		) {
-			errno = EINVAL;
+			dbg("yescrypt_kdf32_body: not -3");
 			return -1;
 		}
 		retval = yescrypt_kdf32_body(yctx,
 				passwd, passwdlen,
 				flags | YESCRYPT_PREHASH, N >> 6, 0,
 				dk32);
-		if (retval)
+		if (retval) {
+			dbg("yescrypt_kdf32_body(PREHASH):%d", retval);
 			return retval;
+		}
 		passwd = dk32;
 		passwdlen = sizeof(dk32);
 	}
@@ -1109,9 +1137,9 @@ int yescrypt_kdf32(
 	retval = yescrypt_kdf32_body(yctx,
 			passwd, passwdlen,
 			flags, N, t, buf32);
-#ifndef SKIP_MEMZERO
-	if (passwd == dk32)
-		explicit_bzero(dk32, sizeof(dk32));
-#endif
+
+	explicit_bzero(dk32, sizeof(dk32));
+
+	dbg("yescrypt_kdf32_body:%d", retval);
 	return retval;
 }