I have a WebHdfs API that successfully CREATES a file, but then when I try to APPEND a byte array of data to the same file I run into issues.
I have no issues creating a file, so I know it is not a permissions or data issue.
- I run a POST request to the server, receive a 307 response, and then send another POST request to the Header.Location with the data to be appended.
- The response from the second request is 200, indicating success, and the file I appended to grows larger, seemingly indicating the data was appended successfully.
- When I try to open the file (parquet) the reader informs me that the amount of rows is equal to the rows that have been appended, but I cannot see the data. This indicates that the APPEND function is not working and the file has been corrupted somehow.
- I have tested the same operation to a local file using FS and the data is successfully appended and the file is correct.
- I have also tested sending the data that I want to append in a CREATE operation and this also works, indicating further that the data I send in the APPEND operation is valid.
I expect that the file would grow larger after the append, and I would be able to open the file and see the appended data as well as the original data.
I have also confirmed that "&datanode=true" in the request URI.
At this point I am left to conclude that there might be something wrong with the WebHdfs API.
Has anyone here successfully appended data to a file in the fashion before?
If there are any other ideas please let me know.
uri = _strURI + (strFullFileName + "?op=APPEND&user.name=" + _webHdfsUserName);
method = "POST";
private async Task<bool> RunPutBinaryFileRequest(string strURI, string strMethod, byte[] data)
{
bool blnResult = false;
try
{
var handler = new HttpClientHandler
{
AllowAutoRedirect = false,
PreAuthenticate = true,
};
var hClient = new HttpClient(handler);
var request = new HttpRequestMessage
{
Method = new HttpMethod(strMethod), //PUT or POST: PUT = CREATE , POST REQUIRED FOR APPEND, determined by calling function.
RequestUri = new Uri(strURI),
};
hClient.DefaultRequestHeaders.Referrer = new Uri(strURI);
hClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes(_webHdfsUserName + ":" + _webHdfsPassword)));
var response = await hClient.SendAsync(request);
//response.EnsureSuccessStatusCode();
blnResult = await UseBinaryHTTP(data, response.Headers.Location, strMethod);
hClient.Dispose();
}
catch (HttpRequestException ex)
{
_logger.LogError("[RunBinaryPutFileRequest]request}::WebException{ex}", ex.Message);
}
return blnResult;
}
private async Task<bool> UseBinaryHTTP(byte[] data, Uri url, string strMethod)
{
var success = false;
try
{
var hClient = new HttpClient();
var sContent = new ByteArrayContent(data);
var aValue = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes(_webHdfsUserName + ":" + _webHdfsPassword)));
hClient.DefaultRequestHeaders.Authorization = aValue;
var request = new HttpRequestMessage
{
Method = new HttpMethod(strMethod),
RequestUri = url,
Content = sContent
};
request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
var response = await hClient.SendAsync(request);
response.EnsureSuccessStatusCode();
hClient.Dispose();
success = true;
}
catch (HttpRequestException ex)
{
_logger.LogError("[UseBinaryHTTP]request}::WebException{ex}", ex.Message);
}
return success;
}