📂Data Acquisition

How to Download Ocean Data from HYCOM via HTTP in Julia

Overview

On HiCom, manually clicking to download and organize data is not easy. This introduces a method to retrieve data by supplying a URL that specifies the selection criteria.

Code

HTTP.jl ¹

for y = 2019:2024
    url = replace("https://ncss.hycom.org/thredds/ncss/GLBy0.08/expt_93.0/ts3z/$y?var=water_temp
    &north=36.88
    &west=129.52
    &east=130.24
    &south=36.32
    &disableProjSubset=on
    &horizStride=1
    &time_start=$y-01-01T00%3A00%3A00Z
    &time_end=$y-12-31T21%3A00%3A00Z
    &timeStride=1
    &vertCoord=0
    &accept=netcdf4", '\n' => "", " " => "")
    @time response = HTTP.get(url)

    open("data_$y.nc4", "w") do f
        write(f, response.body)
    end
end

For example, the above code fetches sea surface temperature (SST, Sea Surface Temperature) at three-hour intervals for experiment number 93 of GLBy0.08, over the period 2019–2024, within the latitude range 36.32–36.88 and longitude range 129.52–130.24.

NCDatasets.jl ²

data_ = []
for y = 2019:2024
    ds  = NCDataset("data_$y.nc4")
    push!(data_, DataFrame([ds["time"][:] reshape(ds["water_temp"][:, :, 1, :], 150, :)'], ["t"; repeat("i" .* string.(1:10), outer = 15) .* repeat("j" .* string.(1:15), inner = 10)]))
    close(ds)
end

The downloaded file is a *.nc4 file, i.e., NetCDF-4 (Network Common Data Form version 4). Although it is widely used for handling high-dimensional data, in this guide we create a DataFrame using DataFrames.jl.

Saving

_data_ = vcat(data_...)
_data_.t = Date.(_data_.t)
_data_ = combine(groupby(_data_, :t), names(_data_, Not(:t)) .=> mean .=> names(_data_, Not(:t)))
CSV.write("data_GLBy0.08_expt_93.0.csv", _data_)
tnsr = reshape(Matrix(_data_[:, 2:end])', 10, 15, :)

@save "data_tnsr.jld2" tnsr
# @load "data_tnsr.jld2"; tnsr

You can save tensors as-is using JLD2.jl, or save them in an easy-to-write format using CSV.jl.

Full code

using HTTP, NCDatasets, DataFrames, Dates, CSV, JLD2, StatsBase
for y = 2019:2024
    url = replace("https://ncss.hycom.org/thredds/ncss/GLBy0.08/expt_93.0/ts3z/$y?var=water_temp
    &north=36.88
    &west=129.52
    &east=130.24
    &south=36.32
    &disableProjSubset=on
    &horizStride=1
    &time_start=$y-01-01T00%3A00%3A00Z
    &time_end=$y-12-31T21%3A00%3A00Z
    &timeStride=1
    &vertCoord=0
    &accept=netcdf4", '\n' => "", " " => "")
    @time response = HTTP.get(url)

    open("data_$y.nc4", "w") do f
        write(f, response.body)
    end
end

# ds  = NCDataset("data_$y.nc4")
# # ds.attrib
# # keys(ds)
# # ds["water_temp"][:, :, 1, :]
# CSV.write("lon.csv", DataFrame(i = 1:10, x = round.(ds["lon"][:], digits = 2)))
# CSV.write("lat.csv", DataFrame(j = 1:15, y = round.(ds["lat"][:], digits = 2)))
# close(ds)

data_ = []
for y = 2019:2024
    ds  = NCDataset("data_$y.nc4")
    push!(data_, DataFrame([ds["time"][:] reshape(ds["water_temp"][:, :, 1, :], 150, :)'], ["t"; repeat("i" .* string.(1:10), outer = 15) .* repeat("j" .* string.(1:15), inner = 10)]))
    close(ds)
end
_data_ = vcat(data_...)
_data_.t = Date.(_data_.t)
_data_ = combine(groupby(_data_, :t), names(_data_, Not(:t)) .=> mean .=> names(_data_, Not(:t)))
CSV.write("data_GLBy0.08_expt_93.0.csv", _data_)
tnsr = reshape(Matrix(_data_[:, 2:end])', 10, 15, :)

@save "data_tnsr.jld2" tnsr
# @load "data_tnsr.jld2"; tnsr

Niño 3.4

The following code retrieves the average sea surface temperature for the Niño 3.4 region³, defined as latitude 5°S–5°N and longitude 170°W–120°W.

using HTTP, NCDatasets, DataFrames, Dates, CSV, JLD2, StatsBase, ProgressMeter
using Base.Threads

@showprogress @threads for day in Date(2019, 1, 1):Date(2024, 9, 5)
try
    str_day = string(day)
    ("data_$str_day.nc4" |> isfile) && continue
    url = replace("https://ncss.hycom.org/thredds/ncss/GLBy0.08/expt_93.0/ts3z/$(Year(day).value)?var=water_temp
    &north=5
    &west=120
    &east=170
    &south=-5
    &disableProjSubset=on
    &horizStride=1
    &time_start=$(day)T00%3A00%3A00Z
    &time_end=$(day)T21%3A00%3A00Z
    &timeStride=1
    &vertCoord=0
    &accept=netcdf4", '\n' => "", " " => "")
    # @time response = HTTP.get(url)
    response = HTTP.get(url)

    open("data_$str_day.nc4", "w") do f
        write(f, response.body)
    end
catch e
    @warn e
end
end

nc4_ = filter(endswith(".nc4"), readdir())
@time data_ = CSV.read("data_form.csv", DataFrame)
@showprogress for k in eachindex(nc4_)
    NCDataset(nc4_[k]) do ds
        df = DataFrame([ds["time"][:] reshape(ds["water_temp"][:, :, 1, :], 251*626, :)'], ["t"; repeat("i" .* string.(1:626), outer = 251) .* repeat("j" .* string.(1:251), inner = 626)])
        df.t = Date.(df.t)
        data_[k, :] = combine(groupby(df, :t), names(df, Not(:t)) .=> mean .=> names(df, Not(:t)))[1, :]
    end
end

CSV.write("data_GLBy0.08_expt_93.0.csv", data_)
tnsr = reshape(Matrix(data_[:, 2:end])', 626, 251, :)

@save "data_tnsr.jld2" tnsr