libc/textdomain: Fix error behaviour

- textdomain should set errno to ENOMEM in case if it is not
  possible to store domainname
- fix buffer overflow in textdomain if domainname is a string
  of NAME_MAX length
- improve textdomain error detection in case if domainname
  points to non null terminated buffer

Signed-off-by: Petro Karashchenko <petro.karashchenko@gmail.com>

libs/libc/locale/lib_gettext.c

@@ -373,10 +373,10 @@ FAR char *textdomain(FAR const char *domainname)
       return *domain ? domain : "messages";
     }
 
-  domainlen = strlen(domainname);
-  if (domainlen > NAME_MAX)
+  domainlen = strnlen(domainname, NAME_MAX);
+  if (domainlen == NAME_MAX)
     {
-      set_errno(EINVAL);
+      set_errno(ENOMEM);
       return NULL;
     }