rpc: check fopen return code in dumptxoutset

This change improves the usability of the `dumptxoutset` RPC in two ways,
in the case that an invalid path is passed:
  1. return from the RPC immediately, rather then when the file is first
     tried to be written (which is _after_ calculating the UTXO set hash)
  2. return a proper return code and error message instead of the cryptic
     "CAutoFile::operator<<: file handle is nullptr: unspecified
      iostream_category error" (-1)

src/rpc/blockchain.cpp

@@ -2281,6 +2281,12 @@ static RPCHelpMan dumptxoutset()
 
     FILE* file{fsbridge::fopen(temppath, "wb")};
     CAutoFile afile{file, SER_DISK, CLIENT_VERSION};
+    if (afile.IsNull()) {
+        throw JSONRPCError(
+            RPC_INVALID_PARAMETER,
+            "Couldn't open file " + temppath.u8string() + " for writing.");
+    }
+
     NodeContext& node = EnsureAnyNodeContext(request.context);
     UniValue result = CreateUTXOSnapshot(
         node, node.chainman->ActiveChainstate(), afile, path, temppath);

test/functional/rpc_dumptxoutset.py

@@ -49,9 +49,13 @@ class DumptxoutsetTest(BitcoinTestFramework):
             out['txoutset_hash'], '1f7e3befd45dc13ae198dfbb22869a9c5c4196f8e9ef9735831af1288033f890')
         assert_equal(out['nchaintx'], 101)
 
-        # Specifying a path to an existing file will fail.
+        # Specifying a path to an existing or invalid file will fail.
         assert_raises_rpc_error(
             -8, '{} already exists'.format(FILENAME),  node.dumptxoutset, FILENAME)
+        invalid_path = str(Path(node.datadir) / "invalid" / "path")
+        assert_raises_rpc_error(
+            -8, "Couldn't open file {}.incomplete for writing".format(invalid_path), node.dumptxoutset, invalid_path)
+
 
 if __name__ == '__main__':
     DumptxoutsetTest().main()