Error parsing response

[rbetancor]

Hi all, I'm trying to use Hyperclient to consume the API of a server that it's build with perl+Catalyst and exposes outside as JSON+HAL, but I'm stuck with a parsing error that Hyperclient throw me and I'm unnable to figure what's happen.

This is the full response when I query the resource 'subscribers':

{
   "_embedded" : {
      "ngcp:subscribers" : [
         {
            "_links" : {
               "collection" : {
                  "href" : "/api/subscribers/"
               },
               "curies" : {
                  "href" : "http://purl.org/sipwise/ngcp-api/#rel-{rel}",
                  "name" : "ngcp",
                  "templated" : true
               },
               "ngcp:callforwards" : {
                  "href" : "/api/callforwards/52"
               },
               "ngcp:calls" : {
                  "href" : "/api/calls/?subscriber_id=52"
               },
               "ngcp:customers" : {
                  "href" : "/api/customers/3"
               },
               "ngcp:domains" : {
                  "href" : "/api/domains/3"
               },
               "ngcp:journal" : {
                  "href" : "/api/subscribers/52/journal/"
               },
               "ngcp:reminders" : {
                  "href" : "/api/reminders/?subscriber_id=52"
               },
               "ngcp:subscriberpreferences" : {
                  "href" : "/api/subscriberpreferences/52"
               },
               "ngcp:subscriberregistrations" : {
                  "href" : "/api/subscriberregistrations/?subscriber_id=52"
               },
               "ngcp:voicemailsettings" : {
                  "href" : "/api/voicemailsettings/52"
               },
               "profile" : {
                  "href" : "http://purl.org/sipwise/ngcp-api/"
               },
               "self" : {
                  "href" : "/api/subscribers/52"
               }
            },
            "administrative" : false,
            "alias_numbers" : [],
            "customer_id" : 3,
            "domain" : "proxy.testserver.org",
            "domain_id" : 3,
            "email" : null,
            "external_id" : null,
            "id" : 52,
            "password" : "testtest",
            "primary_number" : {
               "ac" : "949",
               "cc" : "1",
               "sn" : "3465301"
            },
            "profile_id" : null,
            "profile_set_id" : null,
            "status" : "active",
            "username" : "test.fax",
            "uuid" : "45cdcf6f-3f39-4245-bccb-3db775622701",
            "webpassword" : null,
            "webusername" : "test.fax"
         },
         {
            "_links" : {
               "collection" : {
                  "href" : "/api/subscribers/"
               },
               "curies" : {
                  "href" : "http://purl.org/sipwise/ngcp-api/#rel-{rel}",
                  "name" : "ngcp",
                  "templated" : true
               },
               "ngcp:callforwards" : {
                  "href" : "/api/callforwards/55"
               },
"ngcp:calls" : {
                  "href" : "/api/calls/?subscriber_id=55"
               },
               "ngcp:customers" : {
                  "href" : "/api/customers/3"
               },
               "ngcp:domains" : {
                  "href" : "/api/domains/3"
               },
               "ngcp:journal" : {
                  "href" : "/api/subscribers/55/journal/"
               },
               "ngcp:reminders" : {
                  "href" : "/api/reminders/?subscriber_id=55"
               },
               "ngcp:subscriberpreferences" : {
                  "href" : "/api/subscriberpreferences/55"
               },
               "ngcp:subscriberregistrations" : {
                  "href" : "/api/subscriberregistrations/?subscriber_id=55"
               },
               "ngcp:voicemailsettings" : {
                  "href" : "/api/voicemailsettings/55"
               },
               "profile" : {
                  "href" : "http://purl.org/sipwise/ngcp-api/"
               },
               "self" : {
                  "href" : "/api/subscribers/55"
               }
            },
            "administrative" : false,
            "alias_numbers" : [],
            "customer_id" : 3,
            "domain" : "proxy.testserver.org",
            "domain_id" : 3,
            "email" : null,
            "external_id" : null,
            "id" : 55,
            "password" : "tfp6LzOxTFa2OWHF",
            "primary_number" : {
               "ac" : "919",
               "cc" : "1",
               "sn" : "8675959"
            },
            "profile_id" : null,
            "profile_set_id" : null,
            "status" : "active",
            "username" : "7WOkIWgU",
            "uuid" : "df0ed22e-d0db-406b-b1cf-0f068f5778d4",
            "webpassword" : null,
            "webusername" : "7WOkIWgU"
         }
      ]
   },
   "_links" : {
      "curies" : {
         "href" : "http://purl.org/sipwise/ngcp-api/#rel-{rel}",
         "name" : "ngcp",
         "templated" : true
      },
      "ngcp:subscribers" : [
         {
            "href" : "/api/subscribers/52"
         },
         {
            "href" : "/api/subscribers/55"
         }
      ],
      "profile" : {
         "href" : "http://purl.org/sipwise/ngcp-api/"
      },
      "self" : {
         "href" : "/api/subscribers/?page=1&rows=10"
      }
   },
   "total_count" : 2
}

And this is the error Hyperclient throw me when I try to do a simple iteration over api.subscribers.each to just puts the subscriber.id:

RuntimeError: Invalid response for LinkCollection. The response was: "_links"
from /usr/local/rvm/gems/ruby-2.3.3/gems/hyperclient-0.8.5/lib/hyperclient/link_collection.rb:18:in `initialize'

Why it's unable the full parse the response and let my iterate over the subscribers collection ? ... I've checked that the response is ok.

[dblock]

Is it ok to have a _links under _embedded?

You should build a spec for this and lets figure out if it's legit per JSON+HAL spec?

[ivoanjo]

Maybe the issue is with the missing Content-Type: application/hal+json header? I can reproduce this exact same error message by having a server return a hal response but not include any Content-Type (and just this week a colleague was bit by this exact issue).

@dblock would you think it's useful to improve the error in these cases? I can try to whip up a PR to improve error handling when we don't get what we expect from faraday after a request.

[rbetancor]

The isssue is with the API server not returning the root doc, instead it returns a html page with the manual of the API.
The reply I posted was from asking to one of the resources URL.

I'm beeing in contact with Oriol about this.

Either way, improving the error reporting on this cases, for example trowing a exception about not valid Content-Type will help a lot.

On a side note, is there any way to use the gem pointing directly to the resources URL ?, I'm trying to build the root doc myself and then changes on nginx so it returns the root doc instead of the html one to json+hal query over the same root URL, but in the mean time I whant to test how the gem works with the API

[rbetancor]

@dblock As far as I know, it's a valid json+hal reply, because the _links section is outside _embedded one, then inside the _embedded you have another_links section, that point to the collection

[dblock]

Okay, so back to the original issue it sounds like what's described in this bug is not the case, ie. that's not the data being returned (different content-type, an HTML page), right @rbetancor? In that case this should be closed?

Improving the error message would be great, if @ivoanjo wants to take it, go for it!

[rbetancor]

@dblock Yes, you are wrigth ... the response I pasted, is the one you receive whenever you query directely the resource, the root API URL returns an html doc, no a json+hal root one, either way ... is there any way of passing to the client instance a root_doc build by myself?

[dblock]

Hyperclient.new(url) is what you want, but you just need to pass the right URL in there. Or am I missing something?

I'll close this, but feel free to ask more questions!

[rbetancor]

@dblock no ... Hyperclient.new(url) is not valid for this ... because this wrongly developed api does not return a valid ROOT document, never ever ... so I whant to 'inyect' my own root_document to the Hyperclient instance and from there it follow the _links to the resources.

[dblock]

Oh, the API should be fixed :)

That said, I looked at how you would "load" a document instead of requesting and parsing it, but there's a bunch of stuff going on there - all requests are made with Faraday.

A clean implementation is to write a middleware for Faraday that "fixes" this content. There's a bunch of examples of middleware in https://github.com/lostisland/faraday_middleware to look at. We document faraday default setup and middleware in README.

Otherwise a hack ... Hyperclient.new, is actually Hyperclient::Entrypoint#new, which then makes HTTP requests, ultimately calling Hyperclient::Link#get, so I think you could declare Hyperclient::Entrypoint#get and return an instance of Resource, so Resource.new(<the new body>, @entry_point, response).

Finally, you could add a Hyperclient#load method that takes a body. I am not 100% convinced we want this in the library, but I would consider a pull request.

[rbetancor]

@dblock finaly I found how to tweak nginx setup so for queries against the root URL /api/ ... to return a static json file I build, instead of the html help docs of the api.
But now stuck on who to query resrouces that are named in an 'prefixed' way ... I mean .. how to query for api.subscribers.each ... when on the api the href for subscribers really said ngcp:subscribers ... ? Any clue?

[rbetancor]

@dblock wear ...

This is the index.json file I'm returing for a query over the root of the api:

{
        "_links": {
                "self": {
                        "href": "/api"
                },
                "curies": [
                        {
                                "name": "ngcp",
                                "href": "https://proxy.testserver.tld:1443/rels/{rel}",
                                "templated": true
                        }
                ],
                "subscribers": {
                        "href": "/api/subscribers/"
                },
                "subscriber": {
                        "href": "/api/subscribers/{id}",
                        "templated": true
                }
        }
}

Now from rails I do:

@create_connection.subscribers.each { |sub| puts sub.id.to_s }

And get this:

TypeError: no implicit conversion of String into Integer
from /usr/local/rvm/gems/ruby-2.4.0/bundler/gems/hyperclient-9d8bd0451390/lib/hyperclient/curie.rb:25:in `[]'

Either if I ask for specific resource:

@create_connection.subscriber(id: 52).id

I get the same error, the json reply for the first query (iterate over the whole list of resources), is the one I published ad the begining of this issue ... the reply to the individual query to resource id:52 is:

{
   "_links" : {
      "collection" : {
         "href" : "/api/subscribers/"
      },
      "curies" : {
         "href" : "http://purl.org/sipwise/ngcp-api/#rel-{rel}",
         "name" : "ngcp",
         "templated" : true
      },
      "ngcp:callforwards" : {
         "href" : "/api/callforwards/52"
      },
      "ngcp:calls" : {
         "href" : "/api/calls/?subscriber_id=52"
      },
      "ngcp:customers" : {
         "href" : "/api/customers/3"
      },
      "ngcp:domains" : {
         "href" : "/api/domains/3"
      },
      "ngcp:journal" : {
         "href" : "/api/subscribers/52/journal/"
      },
      "ngcp:reminders" : {
         "href" : "/api/reminders/?subscriber_id=52"
      },
      "ngcp:subscriberpreferences" : {
         "href" : "/api/subscriberpreferences/52"
      },
      "ngcp:subscriberregistrations" : {
         "href" : "/api/subscriberregistrations/?subscriber_id=52"
      },
      "ngcp:voicemailsettings" : {
         "href" : "/api/voicemailsettings/52"
      },
      "profile" : {
         "href" : "http://purl.org/sipwise/ngcp-api/"
      },
      "self" : {
         "href" : "/api/subscribers/52"
      }
   },
   "administrative" : false,
   "alias_numbers" : [],
   "customer_id" : 3,
   "domain" : "proxy.testserver.tld",
   "domain_id" : 3,
   "email" : null,
   "external_id" : null,
   "id" : 52,
   "password" : "testtest",
   "primary_number" : {
      "ac" : "949",
      "cc" : "1",
      "sn" : "3465301"
   },
   "profile_id" : null,
   "profile_set_id" : null,
   "status" : "active",
   "username" : "test.fax",
   "uuid" : "45cdcf6f-3f39-4245-bccb-3db775622701",
   "webpassword" : null,
   "webusername" : "test.fax"
}

And I checked on the nginx server logs, the Hyperclient is asking for the right URL on both cases and get back the json reply.