Fixes for strtoul/strtoull. Fixes Issue #1

libc/stdlib/lib_checkbase.c

@@ -63,6 +63,11 @@
  * Assumptions:
  *   *ptr points to the first, non-whitespace character in the string.
  *
+ * Returns:
+ *   - if base is valid, the actual base to use, and pptr is updated to point
+ *     at the first digit.
+ *   - if base is invalid (<2 or >36), return -1.
+ *
  ****************************************************************************/
 
 int lib_checkbase(int base, FAR const char **pptr)
@@ -107,6 +112,12 @@ int lib_checkbase(int base, FAR const char **pptr)
         }
     }
 
+  /* Check for incorrect bases. */
+  else if (base < 2 || base > 26)
+    {
+    return -1; /* Means incorrect base */
+    }
+
   /* Return the updated pointer and base */
 
   *pptr = ptr;

libc/stdlib/lib_strtoul.c

@@ -59,13 +59,17 @@
  *   nptr to a long unsigned integer value according to the given base, which
  *   must be between 2 and 36 inclusive, or be the special value 0.
  *
- * Warning: does not check for integer overflow!
+ * Returns:
+ *   - The converted value, if the base and number are valid
+ *   - 0 if an error occurs, and seterrno to:
+ *     * EINVAL if base < 2 or base > 36
+ *     * ERANGE if the number cannot be represented using unsigned long
  *
  ****************************************************************************/
 
 unsigned long strtoul(FAR const char *nptr, FAR char **endptr, int base)
 {
-  unsigned long accum = 0;
+  unsigned long prev, accum = 0;
   int value;
 
   if (nptr)
@@ -74,16 +78,32 @@ unsigned long strtoul(FAR const char *nptr, FAR char **endptr, int base)
 
       lib_skipspace(&nptr);
 
-      /* Check for unspecified base */
+      /* Check for unspecified or incorrect base */
 
       base = lib_checkbase(base, &nptr);
 
+      if (base < 0)
+        {
+          set_errno(EINVAL);
+          return 0;
+        }
+
       /* Accumulate each "digit" */
 
       while (lib_isbasedigit(*nptr, base, &value))
         {
-            accum = accum*base + value;
+          prev  = accum;
-            nptr++;
+          accum = accum*base + value;
+          nptr++;
+
+          /* Check for overflow */
+
+          if (accum < prev)
+            {
+            set_errno(ERANGE);
+            accum = 0;
+            break;
+            }
         }
 
       /* Return the final pointer to the unused value */

libc/stdlib/lib_strtoull.c

@@ -62,11 +62,17 @@
  *   nptr to a long unsigned integer value according to the given base, which
  *   must be between 2 and 36 inclusive, or be the special value 0.
  *
+ * Returns:
+ *   - The converted value, if the base and number are valid
+ *   - 0 if an error occurs, and seterrno to:
+ *     * EINVAL if base < 2 or base > 36
+ *     * ERANGE if the number cannot be represented using unsigned long long
+ *
  ****************************************************************************/
 
 unsigned long long strtoull(FAR const char *nptr, FAR char **endptr, int base)
 {
-  unsigned long long accum = 0;
+  unsigned long long prev, accum = 0;
   int value;
 
   if (nptr)
@@ -79,12 +85,28 @@ unsigned long long strtoull(FAR const char *nptr, FAR char **endptr, int base)
 
       base = lib_checkbase(base, &nptr);
 
+      if (base < 0)
+        {
+          set_errno(EINVAL);
+          return 0;
+        }
+
       /* Accumulate each "digit" */
 
       while (lib_isbasedigit(*nptr, base, &value))
         {
-            accum = accum*base + value;
+          prev  = accum;
-            nptr++;
+          accum = accum*base + value;
+          nptr++;
+
+          /* Check for overflow */
+
+          if (accum < prev)
+            {
+            set_errno(ERANGE);
+            accum = 0;
+            break;
+            }
         }
 
       /* Return the final pointer to the unused value */