[git commit master 1/1] wget: shrink progress meter code; strink wget and add debug logging

Denys Vlasenko vda.linux at googlemail.com
Thu Feb 10 22:02:28 UTC 2011


commit: http://git.busybox.net/busybox/commit/?id=f836f01cc3eb39e5e4c7186f50b456b57fae2010
branch: http://git.busybox.net/busybox/commit/?id=refs/heads/master

function                                             old     new   delta
fgets_and_trim                                         -      73     +73
retrieve_file_data                                   367     349     -18
bb_progress_update                                   723     699     -24
wget_main                                           2220    2190     -30
ftpcmd                                               133      87     -46
gethdr                                               200     153     -47
------------------------------------------------------------------------------
(add/remove: 1/0 grow/shrink: 0/5 up/down: 73/-165)           Total: -92 bytes

Signed-off-by: Denys Vlasenko <vda.linux at googlemail.com>
---
 libbb/progress.c  |   95 ++++++++++++++++++++++++++++++++++++-----------------
 networking/wget.c |   74 ++++++++++++++++++++++-------------------
 2 files changed, 105 insertions(+), 64 deletions(-)

diff --git a/libbb/progress.c b/libbb/progress.c
index ced04ac..a490b83 100644
--- a/libbb/progress.c
+++ b/libbb/progress.c
@@ -60,9 +60,16 @@ void FAST_FUNC bb_progress_init(bb_progress_t *p)
 	p->inited = 1;
 }
 
+/* File already had beg_size bytes.
+ * Then we started downloading.
+ * We downloaded "transferred" bytes so far.
+ * Download is expected to stop when total size (beg_size + transferred)
+ * will be "totalsize" bytes.
+ * If totalsize == 0, then it is unknown.
+ */
 void FAST_FUNC bb_progress_update(bb_progress_t *p,
 		const char *curfile,
-		uoff_t beg_range,
+		uoff_t beg_size,
 		uoff_t transferred,
 		uoff_t totalsize)
 {
@@ -72,32 +79,53 @@ void FAST_FUNC bb_progress_update(bb_progress_t *p,
 	int barlength;
 	int kiloscale;
 
-	/* totalsize == 0 if it is unknown */
-
-	beg_and_transferred = beg_range + transferred;
+	beg_and_transferred = beg_size + transferred;
 
 	elapsed = monotonic_sec();
 	since_last_update = elapsed - p->lastupdate_sec;
-	/* Do not update on every call
-	 * (we can be called on every network read!) */
+	/*
+	 * Do not update on every call
+	 * (we can be called on every network read!)
+	 */
 	if (since_last_update == 0 && beg_and_transferred < totalsize)
 		return;
 
-	/* Scale sizes down if they are close to overflowing.
-	 * If off_t is only 32 bits, this allows calculations
-	 * like (100 * transferred / totalsize) without risking overflow.
-	 * Introduced error is < 0.1%
-	 */
 	kiloscale = 0;
-	if (totalsize >= (1 << 20)) {
-		totalsize >>= 10;
-		beg_range >>= 10;
-		transferred >>= 10;
-		beg_and_transferred >>= 10;
-		kiloscale++;
+	/*
+	 * Scale sizes down if they are close to overflowing.
+	 * This allows calculations like (100 * transferred / totalsize)
+	 * without risking overflow: we guarantee 10 highest bits to be 0.
+	 * Introduced error is less than 1 / 2^12 ~= 0.025%
+	 */
+	if (ULONG_MAX > 0xffffffff || sizeof(off_t) == 4 || sizeof(off_t) != 8) {
+		/*
+		 * 64-bit CPU || small off_t: in either case,
+		 * >> is cheap, single-word operation.
+		 * ... || strange off_t: also use this code (it is safe,
+		 * even if suboptimal), because 32/64 optimized one
+		 * works only for 64-bit off_t.
+		 */
+		if (totalsize >= (1 << 22)) {
+			totalsize >>= 10;
+			beg_size >>= 10;
+			transferred >>= 10;
+			beg_and_transferred >>= 10;
+			kiloscale = 1;
+		}
+	} else {
+		/* 32-bit CPU and 64-bit off_t.
+		 * Pick a shift (40 bits) which is easier to do on 32-bit CPU.
+		 */
+		if (totalsize >= (uoff_t)(1ULL << 54)) {
+			totalsize = (uint32_t)(totalsize >> 32) >> 8;
+			beg_size = (uint32_t)(beg_size >> 32) >> 8;
+			transferred = (uint32_t)(transferred >> 32) >> 8;
+			beg_and_transferred = (uint32_t)(beg_and_transferred >> 32) >> 8;
+			kiloscale = 4;
+		}
 	}
 
-	if (beg_and_transferred >= totalsize)
+	if (beg_and_transferred > totalsize)
 		beg_and_transferred = totalsize;
 
 	ratio = 100 * beg_and_transferred / totalsize;
@@ -124,14 +152,14 @@ void FAST_FUNC bb_progress_update(bb_progress_t *p,
 	}
 
 	while (beg_and_transferred >= 100000) {
-		kiloscale++;
 		beg_and_transferred >>= 10;
+		kiloscale++;
 	}
 	/* see http://en.wikipedia.org/wiki/Tera */
 	fprintf(stderr, "%6u%c ", (unsigned)beg_and_transferred, " kMGTPEZY"[kiloscale]);
 #define beg_and_transferred dont_use_beg_and_transferred_below()
 
-	if (transferred > p->lastsize) {
+	if (transferred != p->lastsize) {
 		p->lastupdate_sec = elapsed;
 		p->lastsize = transferred;
 		if (since_last_update >= STALLTIME) {
@@ -141,20 +169,27 @@ void FAST_FUNC bb_progress_update(bb_progress_t *p,
 		}
 		since_last_update = 0; /* we are un-stalled now */
 	}
+
 	elapsed -= p->start_sec; /* now it's "elapsed since start" */
 
 	if (since_last_update >= STALLTIME) {
 		fprintf(stderr, " - stalled -");
+	} else if (!totalsize || !transferred || (int)elapsed <= 0) {
+		fprintf(stderr, "--:--:-- ETA");
 	} else {
-		uoff_t to_download = totalsize - beg_range;
-		if (!totalsize || (int)elapsed <= 0 || transferred > to_download) {
-			fprintf(stderr, "--:--:-- ETA");
-		} else {
-			/* to_download / (transferred/elapsed) - elapsed: */
-			unsigned eta = to_download * elapsed / transferred - elapsed;
-			unsigned secs = eta % 3600;
-			unsigned hours = eta / 3600;
-			fprintf(stderr, "%02u:%02u:%02u ETA", hours, secs / 60, secs % 60);
-		}
+		unsigned eta, secs, hours;
+
+		totalsize -= beg_size; /* now it's "total to upload" */
+
+		/* Estimated remaining time =
+		 * estimated_sec_to_dl_totalsize_bytes - elapsed_sec =
+		 * totalsize / average_bytes_sec_so_far - elapsed =
+		 * totalsize / (transferred/elapsed) - elapsed =
+		 * totalsize * elapsed / transferred - elapsed
+		 */
+		eta = totalsize * elapsed / transferred - elapsed;
+		secs = eta % 3600;
+		hours = eta / 3600;
+		fprintf(stderr, "%02u:%02u:%02u ETA", hours, secs / 60, secs % 60);
 	}
 }
diff --git a/networking/wget.c b/networking/wget.c
index 931882f..d81426e 100644
--- a/networking/wget.c
+++ b/networking/wget.c
@@ -10,6 +10,10 @@
  */
 #include "libbb.h"
 
+//#define log_io(...) bb_error_msg(__VA_ARGS__)
+#define log_io(...) ((void)0)
+
+
 struct host_info {
 	// May be used if we ever will want to free() all xstrdup()s...
 	/* char *allocated; */
@@ -197,25 +201,39 @@ static FILE *open_socket(len_and_sockaddr *lsa)
 	return fp;
 }
 
+/* Returns '\n' if it was seen, else '\0'. Trims at first '\r' or '\n' */
+static char fgets_and_trim(FILE *fp)
+{
+	char c;
+	char *buf_ptr;
+
+	if (fgets(G.wget_buf, sizeof(G.wget_buf) - 1, fp) == NULL)
+		bb_perror_msg_and_die("error getting response");
+
+	buf_ptr = strchrnul(G.wget_buf, '\n');
+	c = *buf_ptr;
+	*buf_ptr = '\0';
+	buf_ptr = strchrnul(G.wget_buf, '\r');
+	*buf_ptr = '\0';
+
+	log_io("< %s", G.wget_buf);
+
+	return c;
+}
+
 static int ftpcmd(const char *s1, const char *s2, FILE *fp)
 {
 	int result;
 	if (s1) {
-		if (!s2) s2 = "";
+		if (!s2)
+			s2 = "";
 		fprintf(fp, "%s%s\r\n", s1, s2);
 		fflush(fp);
+		log_io("> %s%s", s1, s2);
 	}
 
 	do {
-		char *buf_ptr;
-
-		if (fgets(G.wget_buf, sizeof(G.wget_buf)-2, fp) == NULL) {
-			bb_perror_msg_and_die("error getting response");
-		}
-		buf_ptr = strstr(G.wget_buf, "\r\n");
-		if (buf_ptr) {
-			*buf_ptr = '\0';
-		}
+		fgets_and_trim(fp);
 	} while (!isdigit(G.wget_buf[0]) || G.wget_buf[3] != ' ');
 
 	G.wget_buf[3] = '\0';
@@ -284,7 +302,7 @@ static void parse_url(char *src_url, struct host_info *h)
 	sp = h->host;
 }
 
-static char *gethdr(FILE *fp /*, int *istrunc*/)
+static char *gethdr(FILE *fp)
 {
 	char *s, *hdrval;
 	int c;
@@ -292,19 +310,16 @@ static char *gethdr(FILE *fp /*, int *istrunc*/)
 	/* *istrunc = 0; */
 
 	/* retrieve header line */
-	if (fgets(G.wget_buf, sizeof(G.wget_buf), fp) == NULL)
-		return NULL;
+	c = fgets_and_trim(fp);
 
-	/* see if we are at the end of the headers */
-	for (s = G.wget_buf; *s == '\r'; ++s)
-		continue;
-	if (*s == '\n')
+	/* end of the headers? */
+	if (G.wget_buf[0] == '\0')
 		return NULL;
 
 	/* convert the header name to lower case */
 	for (s = G.wget_buf; isalnum(*s) || *s == '-' || *s == '.'; ++s) {
 		/* tolower for "A-Z", no-op for "0-9a-z-." */
-		*s = (*s | 0x20);
+		*s |= 0x20;
 	}
 
 	/* verify we are at the end of the header name */
@@ -315,20 +330,12 @@ static char *gethdr(FILE *fp /*, int *istrunc*/)
 	*s++ = '\0';
 	hdrval = skip_whitespace(s);
 
-	/* locate the end of header */
-	while (*s && *s != '\r' && *s != '\n')
-		++s;
-
-	/* end of header found */
-	if (*s) {
-		*s = '\0';
-		return hdrval;
+	if (c != '\n') {
+		/* Rats! The buffer isn't big enough to hold the entire header value */
+		while (c = getc(fp), c != EOF && c != '\n')
+			continue;
 	}
 
-	/* Rats! The buffer isn't big enough to hold the entire header value */
-	while (c = getc(fp), c != EOF && c != '\n')
-		continue;
-	/* *istrunc = 1; */
 	return hdrval;
 }
 
@@ -520,9 +527,9 @@ static void NOINLINE retrieve_file_data(FILE *dfp, int output_fd)
 		if (!G.chunked)
 			break;
 
-		fgets(G.wget_buf, sizeof(G.wget_buf), dfp); /* This is a newline */
+		fgets_and_trim(dfp); /* This is a newline */
  get_clen:
-		fgets(G.wget_buf, sizeof(G.wget_buf), dfp);
+		fgets_and_trim(dfp);
 		G.content_len = STRTOOFF(G.wget_buf, NULL, 16);
 		/* FIXME: error check? */
 		if (G.content_len == 0)
@@ -757,8 +764,7 @@ int wget_main(int argc UNUSED_PARAM, char **argv)
 		 * Retrieve HTTP response line and check for "200" status code.
 		 */
  read_response:
-		if (fgets(G.wget_buf, sizeof(G.wget_buf), sfp) == NULL)
-			bb_error_msg_and_die("no response from server");
+		fgets_and_trim(sfp);
 
 		str = G.wget_buf;
 		str = skip_non_whitespace(str);
-- 
1.7.3.4



More information about the busybox-cvs mailing list