char file_name[STR_PATH] = "";

ulong selected = show_file_dialog (file_name, &dlg),
{
  dlg = {
          path : ...
          file_mask : ...
          frame_window : ...
          file_dialog_type : FDLG_OPEN
        }
};

if (selected == MB_OPEN && file_name->ends_with ("*.txt"))  
{
  ...
}