__swsetup: set error flag and errno on error.

Previously, we set errno to EBADF if the cantwrite() macro (which calls
__swsetup()) returns true for POSIX compliance.  However, we neglected
to also set the error flag, __SERR.  Rather than set the error flag in
all callers of cantwrite(), set both errno and the error flag in
__swsetup().  This matches what FreeBSD does and makes it possible
to choose a proper errno value for the second error condition in
__swsetup().  OK deraadt@

Author: millert

lib/libc/stdio/fvwrite.c

@@ -1,4 +1,4 @@
-/*	$OpenBSD: fvwrite.c,v 1.20 2017/03/17 16:06:33 millert Exp $ */
+/*	$OpenBSD: fvwrite.c,v 1.21 2023/10/06 16:41:02 millert Exp $ */
 /*-
  * Copyright (c) 1990, 1993
  *	The Regents of the University of California.  All rights reserved.
@@ -34,7 +34,6 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <errno.h>
 #include <unistd.h>
 #include "local.h"
 #include "fvwrite.h"
@@ -58,10 +57,8 @@ __sfvwrite(FILE *fp, struct __suio *uio)
 	if ((len = uio->uio_resid) == 0)
 		return (0);
 	/* make sure we can write */
-	if (cantwrite(fp)) {
-		errno = EBADF;
+	if (cantwrite(fp))
 		return (EOF);
-	}
 
 #define	MIN(a, b) ((a) < (b) ? (a) : (b))
 #define	COPY(n)	  (void)memcpy(fp->_p, p, n)

lib/libc/stdio/putc.c

@@ -1,4 +1,4 @@
-/*	$OpenBSD: putc.c,v 1.13 2015/08/31 02:53:57 guenther Exp $ */
+/*	$OpenBSD: putc.c,v 1.14 2023/10/06 16:41:02 millert Exp $ */
 /*-
  * Copyright (c) 1990, 1993
  *	The Regents of the University of California.  All rights reserved.
@@ -32,7 +32,6 @@
  */
 
 #include <stdio.h>
-#include <errno.h>
 #include "local.h"
 
 /*
@@ -43,10 +42,8 @@
 int
 putc_unlocked(int c, FILE *fp)
 {
-	if (cantwrite(fp)) {
-		errno = EBADF;
+	if (cantwrite(fp))
 		return (EOF);
-	}
 	_SET_ORIENTATION(fp, -1);
 	return (__sputc(c, fp));
 }

lib/libc/stdio/vfprintf.c

@@ -1,4 +1,4 @@
-/*	$OpenBSD: vfprintf.c,v 1.81 2021/09/08 15:57:27 jca Exp $	*/
+/*	$OpenBSD: vfprintf.c,v 1.82 2023/10/06 16:41:02 millert Exp $	*/
 /*-
  * Copyright (c) 1990 The Regents of the University of California.
  * All rights reserved.
@@ -457,10 +457,8 @@ __vfprintf(FILE *fp, const char *fmt0, __va_list ap)
 
 	_SET_ORIENTATION(fp, -1);
 	/* sorry, fprintf(read_only_file, "") returns EOF, not 0 */
-	if (cantwrite(fp)) {
-		errno = EBADF;
+	if (cantwrite(fp))
 		return (EOF);
-	}
 
 	/* optimise fprintf(stderr) (and other unbuffered Unix files) */
 	if ((fp->_flags & (__SNBF|__SWR|__SRW)) == (__SNBF|__SWR) &&

lib/libc/stdio/vfwprintf.c

@@ -1,4 +1,4 @@
-/*	$OpenBSD: vfwprintf.c,v 1.22 2021/09/08 15:57:27 jca Exp $ */
+/*	$OpenBSD: vfwprintf.c,v 1.23 2023/10/06 16:41:02 millert Exp $ */
 /*-
  * Copyright (c) 1990 The Regents of the University of California.
  * All rights reserved.
@@ -451,10 +451,8 @@ __vfwprintf(FILE * __restrict fp, const wchar_t * __restrict fmt0, __va_list ap)
 
 	_SET_ORIENTATION(fp, 1);
 	/* sorry, fwprintf(read_only_file, "") returns EOF, not 0 */
-	if (cantwrite(fp)) {
-		errno = EBADF;
+	if (cantwrite(fp))
 		return (EOF);
-	}
 
 	/* optimise fwprintf(stderr) (and other unbuffered Unix files) */
 	if ((fp->_flags & (__SNBF|__SWR|__SRW)) == (__SNBF|__SWR) &&

lib/libc/stdio/wbuf.c

@@ -1,4 +1,4 @@
-/*	$OpenBSD: wbuf.c,v 1.13 2015/08/31 02:53:57 guenther Exp $ */
+/*	$OpenBSD: wbuf.c,v 1.14 2023/10/06 16:41:02 millert Exp $ */
 /*-
  * Copyright (c) 1990, 1993
  *	The Regents of the University of California.  All rights reserved.
@@ -32,7 +32,6 @@
  */
 
 #include <stdio.h>
-#include <errno.h>
 #include "local.h"
 
 /*
@@ -54,10 +53,8 @@ __swbuf(int c, FILE *fp)
 	 * calls might wrap _w from negative to positive.
 	 */
 	fp->_w = fp->_lbfsize;
-	if (cantwrite(fp)) {
-		errno = EBADF;
+	if (cantwrite(fp))
 		return (EOF);
-	}
 	c = (unsigned char)c;
 
 	/*

lib/libc/stdio/wsetup.c

@@ -1,4 +1,4 @@
-/*	$OpenBSD: wsetup.c,v 1.7 2005/08/08 08:05:36 espie Exp $ */
+/*	$OpenBSD: wsetup.c,v 1.8 2023/10/06 16:41:02 millert Exp $ */
 /*-
  * Copyright (c) 1990, 1993
  *	The Regents of the University of California.  All rights reserved.
@@ -31,14 +31,15 @@
  * SUCH DAMAGE.
  */
 
+#include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include "local.h"
 
 /*
  * Various output routines call wsetup to be sure it is safe to write,
  * because either _flags does not include __SWR, or _buf is NULL.
- * _wsetup returns 0 if OK to write, nonzero otherwise.
+ * __swsetup returns 0 if OK to write, nonzero otherwise, setting errno.
  */
 int
 __swsetup(FILE *fp)
@@ -51,8 +52,11 @@ __swsetup(FILE *fp)
 	 * If we are not writing, we had better be reading and writing.
 	 */
 	if ((fp->_flags & __SWR) == 0) {
-		if ((fp->_flags & __SRW) == 0)
+		if ((fp->_flags & __SRW) == 0) {
+			errno = EBADF;
+			fp->_flags |= __SERR;
 			return (EOF);
+		}
 		if (fp->_flags & __SRD) {
 			/* clobber any ungetc data */
 			if (HASUB(fp))
@@ -68,8 +72,11 @@ __swsetup(FILE *fp)
 	 * Make a buffer if necessary, then set _w.
 	 */
 	if (fp->_bf._base == NULL) {
-		if ((fp->_flags & (__SSTR | __SALC)) == __SSTR)
+		if ((fp->_flags & (__SSTR | __SALC)) == __SSTR) {
+			errno = EINVAL;
+			fp->_flags |= __SERR;
 			return (EOF);
+		}
 		__smakebuf(fp);
 	}
 	if (fp->_flags & __SLBF) {