Developer Reference

http://localhost:8000/swirl/api/docs/

{
    "name": "Company Funding Records - BigQuery",
    "active": false,
    "default": false,
    "connector": "BigQuery",
    "query_template": "select {fields} from `{table}` where search({field1}, '{query_string}') or search({field2}, '{query_string}');",
    "query_processors": [
        "AdaptiveQueryProcessor"
    ],
    "query_mappings": "fields=*,sort_by_date=fundedDate,table=funding.funding,field1=company,field2=city",
    "result_processors": [
        "MappingResultProcessor",
        "CosineRelevancyResultProcessor"
    ],
    "result_mappings": "title='{company}',body='{company} raised ${raisedamt} series {round} on {fundeddate}. The company is located in {city} {state} and has {numemps} employees.',url=id,date_published=fundeddate,NO_PAYLOAD",
    "credentials": "/path/to/bigquery/token.json",
    "tags": [
        "Company",
        "BigQuery",
        "Internal"
    ]
}

{
    "name": "GenAI - OpenAI",
    "active": false,
    "default": true,
    "connector": "GenAI",
    "url": "",
    "query_template": "",
    "query_processors": [
        "AdaptiveQueryProcessor"
    ],
    "query_mappings": "PROMPT='Tell me about: {query_to_provider}'",
    "result_processors": [
        "GenericResultProcessor",
        "CosineRelevancyResultProcessor"
    ],
    "response_mappings": "",
    "result_mappings": "",
    "results_per_query": 10,
    "credentials": "your-openai-API-key-here",
    "tags": [
        "GenAI",
        "Question"
    ]
}

"CHAT_QUERY_REWRITE_PROMPT: Write a more precise query of similar length to this: {query_to_provider}"

"CHAT_QUERY_REWRITE_GUIDE: You are a helpful assistant that responds like a pirate captain"

"CHAT_QUERY_DO_FILTER: false"

"query_mappings": "CHAT_QUERY_REWRITE_PROMPT='Tell me about: {query_to_provider}',CHAT_QUERY_REWRITE_GUIDE='You are a helpful assistant that talks like a pirate captain but keeps it clean!'",

{
    "name": "ENRON Email - Elastic Cloud",
    "connector": "Elastic",
    "url": "your-cloud-id-here",
    "query_template": "index='{index_name}', query={'query_string': {'query': '{query_string}', 'default_field': '{default_field}'}}",
    "query_processors": [
        "AdaptiveQueryProcessor"
    ],
    "query_mappings": "index_name=email,default_field=content,sort_by_date=date_published.keyword,NOT=true,NOT_CHAR=-",
    "result_processors": [
        "MappingResultProcessor",
        "CosineRelevancyResultProcessor"
    ],
    "result_mappings": "url=_source.url,date_published=_source.date_published,author=_source.author,title=_source.subject,body=_source.content,_source.to,NO_PAYLOAD",
    "credentials": "verify_certs=[True|False],ca_certs=/path/to/cert/file.crt,username:password",
    "tags": [
        "Enron",
        "Elastic",
        "Internal"
    ]
}

{
    "name": "ENRON Email - OpenSearch",
    "active": false,
    "default": false,
    "connector": "OpenSearch",
    "url": "https://localhost:9200/",
    "query_template": "{"query":{"query_string":{"query":"{query_string}","default_field":"{default_field}","default_operator":"and"}}}",
    "query_processors": [
        "AdaptiveQueryProcessor"
    ],
    "query_mappings": "index_name=email,default_field=content,sort_by_date=date_published.keyword,NOT=true,NOT_CHAR=-",
    "result_processors": [
        "MappingResultProcessor",
        "CosineRelevancyResultProcessor"
    ],
    "result_mappings": "url=_source.url,date_published=_source.date_published,author=_source.author,title=_source.subject,body=_source.content,_source.to,NO_PAYLOAD",
    "credentials": "verify_certs=[True|False],ca_certs=/path/to/cert/file.crt,username:password",
    "tags": [
        "Enron",
        "OpenSearch",
        "Internal"
    ]
}

{
    "name": "Files - FESS",
    "description": "SearchProvider description",
    "active": true,
    "default": false,
    "authenticator": null,
    "connector": "RequestsGet",
    "url": "http://localhost:8080/api/v1/documents",
    "query_template": "{url}?x=x&q={query_string}",
    "query_template_json": {},
    "post_query_template": {},
    "http_request_headers": {
        "Content-Type": "application/json"
    },
    "page_fetch_config_json": {
        "cache": "false",
        "headers": {
            "User-Agent": "Swirlbot/1.0 (+http://swirlaiconnect.com)"
        },
        "timeout": 10
    },
    "query_processors": [
        "AdaptiveQueryProcessor"
    ],
    "query_mappings": "DATE_SORT=sort=created.desc,RELEVANCY_SORT=sort=score",
    "result_grouping_field": "",
    "result_processors": [
        "MappingResultProcessor",
        "CosineRelevancyResultProcessor"
    ],
    "response_mappings": "RESULTS=data,FOUND=record_count",
    "result_mappings": "title=title,body=content_description,date_published=created,url=url",
    "results_per_query": 10,
    "credentials": "",
    "eval_credentials": "",
    "tags": [
        "Files",
        "FESS"
    ],
    "ephemeral_store_config_json": {
        "ephemeral": false
    },
    "query_language": "Generic_Keyword",
    "config": {}
}

{
    "name": "Outlook Messages - Microsoft 365",
    "active": false,
    "default": true,
    "connector": "M365OutlookMessages",
    "url": "",
    "query_template": "{url}",
    "query_processors": [
        "AdaptiveQueryProcessor"
    ],
    "query_mappings": "NOT=true,NOT_CHAR=-",
    "result_grouping_field": "conversationId",
    "result_processors": [
        "MappingResultProcessor",
        "DedupeByFieldResultProcessor",
        "CosineRelevancyResultProcessor"
    ],
    "response_mappings": "",
    "result_mappings": "title=resource.subject,body=summary,date_published=resource.createdDateTime,author=resource.sender.emailAddress.name,url=resource.webLink,resource.isDraft,resource.importance,resource.hasAttachments,resource.ccRecipients[*].emailAddress[*].name,resource.replyTo[*].emailAddress[*].name,NO_PAYLOAD",
    "results_per_query": 10,
    "credentials": "",
    "eval_credentials": "",
    "tags": [
        "Microsoft",
        "Email"
    ]
}

{
    "name": "IMDB Movie Samples - MongoDB",
    "active": false,
    "default": false,
    "authenticator": "",
    "connector": "MongoDB",
    "url": "sample_mflix:movies",
    "query_template": "{'$text': {'$search': '{query_string}'}}",
    "query_processors": [
        "AdaptiveQueryProcessor"
    ],
    "query_mappings": "DATE_SORT=year,NOT_CHAR=-",
    "result_processors": [
        "MappingResultProcessor",
        "CosineRelevancyResultProcessor"
    ],
    "result_mappings": "title=name,body=fullplot,date_published=released,date_published_display=year,author=directors[*],url=poster,lastupdated,genres[*],rated,runtime,languages[*],cast[*],writers[*],awards.text,imdb.rating,tomatoes.viewer.rating,tomatoes.critic.rating,NO_PAYLOAD",
    "results_per_query": 10,
    "credentials": "mongodb+srv://<mongodb-username>:<mongodb-password>@<mongdb-cluster>.mongodb.net/?retryWrites=true&w=majority",
    "eval_credentials": "",
    "tags": [
        "Movies",
        "MongoDB",
        "Internal"
    ]
}

"query_mappings": "DATE_SORT=year,NOT_CHAR=-,MATCH_ANY"

{
    "name": "Company Funding Records - PostgreSQL",
    "default": false,
    "connector": "PostgreSQL",
    "url": "host:port:database:username:password",
    "query_template": "select {fields} from {table} where {field1} ilike '%{query_string}%' or {field2} ilike '%{query_string}%';",
    "query_processors": [
        "AdaptiveQueryProcessor"
    ],
    "query_mappings": "fields=*,sort_by_date=fundedDate,table=funding,field1=city,field2=company",
    "result_processors": [
        "MappingResultProcessor",
        "CosineRelevancyResultProcessor"
    ],
    "result_mappings": "title='{company} series {round}',body='{city} {fundeddate}: {company} raised usd ${raisedamt}\nThe company is headquartered in {city} and employs {numemps}',date_published=fundeddate,NO_PAYLOAD",
    "tags": [
        "Company",
        "PostgreSQL",
        "Internal"
    ]
}

{
    "name": "Sample HTTP GET Endpoint",
    "connector": "RequestsGet",
    "url": "http://hostname/site/endpoint",
    "query_template": "{url}&textQuery={query_string}",
    "query_processors": [
        "AdaptiveQueryProcessor"
    ],
    "query_mappings": "PAGE=start=RESULT_INDEX,DATE_SORT=sort=date",
    "result_processors": [
        "MappingResultProcessor",
        "CosineRelevancyResultProcessor"
    ],
    "response_mappings": "FOUND=count,RESULTS=results",
    "result_mappings": "body=content,date_published=date,author=creator",
    "credentials": "HTTPBasicAuth('your-username-here', 'your-password-here')",
    "tags": [
        "News"
    ]
}

{
    "name": "Enterprise Search Engines - Google PSE",
    "active": true,
    "default": true,
    "connector": "RequestsGet",
    "url": "https://www.googleapis.com/customsearch/v1",
    "query_template": "{url}?cx={cx}&key={key}&q={query_string}",
    "page_fetch_config_json": {
        "cache": "false",
        "headers": {
            "User-Agent": "Swirlbot/1.0 (+http://swirl.today)"
        },
        "timeout": 30
    },
    "query_processors": [
        "AdaptiveQueryProcessor"
    ],
    "query_mappings": "cx=<your-Google-PSE-id-here>,DATE_SORT=sort=date,PAGE=start=RESULT_INDEX,NOT_CHAR=-",
    "result_processors": [
        "MappingResultProcessor",
        "DateFinderResultProcessor",
        "CosineRelevancyResultProcessor"
    ],
    "response_mappings": "FOUND=searchInformation.totalResults,RETRIEVED=queries.request[0].count,RESULTS=items",
    "result_mappings": "url=link,body=snippet,author=displayLink,cacheId,NO_PAYLOAD",
    "credentials": "key=<your-Google-API-key-here>",
    "tags": [
        "Web"
    ]
}

{
    "name": "techproducts - Apache Solr",
    "active": false,
    "default": false,
    "connector": "RequestsGet",
    "url": "http://localhost:8983/solr/{collection}/select?wt=json",
    "query_template": "{url}&q={query_string}",
    "query_processors": [
        "AdaptiveQueryProcessor"
    ],
    "query_mappings": "collection=techproducts,PAGE=start=RESULT_ZERO_INDEX,NOT=True,NOT_CHAR=-",
    "result_processors": [
        "MappingResultProcessor",
        "CosineRelevancyResultProcessor"
    ],
    "response_mappings": "FOUND=numFound,RESULTS=response.docs",
    "result_mappings": "title=name,body=features,response",
    "tags": [
        "TechProducts",
        "Solr"
    ]
}

{
    "name": "Sample HTTP POST Endpoint",
    "active": false,
    "default": false,
    "connector": "RequestsPost",
    "url": "https://xx.apis.it.h.edu/ats/person/v3/search",
    "query_template": "{url}?Query={query_string}",
    "post_query_template": {
        "fields": [
            "names.personNameKey",
            "names.firstName",
            "names.lastName"
        ],
        "conditions": {
            "names.name": "*{query_string}*"
        }
    },
    "query_processors": [
        "AdaptiveQueryProcessor"
    ],
    "query_mappings": "NOT=true,NOT_CHAR=-",
    "result_processors": [
        "MappingResultProcessor",
        "CosineRelevancyResultProcessor"
    ],
    "response_mappings": "FOUND=count,RESULTS=results",
    "result_mappings": "titles=names[0].name,url=names[0].personNameKey,body='{names[0].name} ID#: {names[*].personNameKey}'",
    "results_per_query": 10,
    "credentials": "X-Api-Key=<your-api-key>",
    "tags": [
        "People"
    ]
}

{
    "name": "Free Company Records - Snowflake",
    "connector": "Snowflake",
    "url": "<snowflake-instance-address>",
    "query_template": "SELECT {fields} FROM {table} WHERE {field1} ILIKE '%{query_string}%' AND NULLIF(TRIM(founded), '') IS NOT NULL ORDER BY TRY_TO_NUMBER(REGEXP_REPLACE(SPLIT_PART(size, '-', 1), '[^0-9]', '')) DESC;",
    "query_processors": [
        "AdaptiveQueryProcessor"
    ],
    "query_mappings": "fields=*,sort_by_date=founded,table=FREECOMPANYDATASET,field1=name",
    "result_processors": [
        "MappingResultProcessor",
        "CosineRelevancyResultProcessor"
    ],
    "response_mappings": "",
    "result_mappings": "title='{name}  ({founded})',body='{name} was founded in {founded} in {country}. It has {size} employees and operates in the {industry} industry.',url='https://{linkedin_url}',date_published=founded,NO_PAYLOAD",
    "results_per_query": 10,
    "credentials": "<username>:<password>:FREE_COMPANY_DATASET:COMPUTE_WH",
    "tags": [
        "Company",
        "Snowflake"
    ]
}

{
    "name": "Company Funding Records - SQLite3",
    "active": false,
    "default": false,
    "connector": "Sqlite3",
    "url": "db.sqlite3",
    "query_template": "select {fields} from {table} where {field1} like '%%{query_string}%%' or {field2} like '%%{query_string}%%';",
    "query_processors": [
        "AdaptiveQueryProcessor"
    ],
    "query_mappings": "fields=*,sort_by_date=fundeddate,table=funding,field1=city,field2=company",
    "result_processors": [
        "MappingResultProcessor",
        "CosineRelevancyResultProcessor"
    ],
    "result_mappings": "title='{company} series {round}',body='{city} {fundeddate}: {company} raised usd ${raisedamt}\nThe company is headquartered in {city} and employs {numemps}',date_published=fundeddate,NO_PAYLOAD",
    "tags": [
        "Company",
        "SQLite3"
    ]
}

[
    "CosineRelevancyPostResultProcessor",
    "DedupeByFieldPostResultProcessor",
    "DedupeBySimilarityPostResultProcessor",
    "VersionClusteringPostResultProcessor"
]

    "query_processors": [
        "AdaptiveQueryProcessor",
        "GenAIQueryProcessor"
    ]

{
    "query_string": "<user-query-string>",
    "pre_query_processors": ["GenAIQueryProcessor"]
}

"tags": [
    "CHAT_QUERY_REWRITE_PROMPT:If this query is in Japanese, tokenize it, and output ONLY the tokenized japanese, NOTHING ELSE -> {query_string}",
    "CHAT_QUERY_REWRITE_GUIDE:You are a speedy tokenizer!",
    "CHAT_QUERY_DO_FILTER:False"
],

"result_processors": [
    "MappingResultProcessor",
    "DateFinderResultProcessor",
    "CosineRelevancyResultProcessor"
]

http://localhost:8000/swirl/results/?search_id=1&result_mixer=DateNewItemsMixer&mark_all_as_read=1

http://localhost:8000/swirl/results/?search_id=1

http://localhost:8000/swirl/results/?search_id=1&result_mixer=Stack1Mixer

sqlite_web db.sqlite3

CREATE TABLE funding (
    id INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
    permalink TEXT,
    company TEXT,
    numemps TEXT DEFAULT '0',
    category TEXT,
    city TEXT,
    state TEXT,
    fundeddate DATE,
    raisedamt NUMERIC DEFAULT '0',
    raisedcurrency TEXT,
    round TEXT
);

CREATE UNIQUE INDEX funding_pkey ON funding(id);

COPY funding(permalink,company,numemps,category,city,state,fundeddate,raisedamt,raisedcurrency,round) 
FROM '/path/to/Data/funding_db.csv' 
DELIMITER ',' CSV HEADER;

PUT /email

## Insert path to Elastic cert below
ca_certs = "<PATH-TO-CERT>"

python DevUtils/index_email_elastic.py emails.csv -p elastic-password

python DevUtils/index_email_opensearch.py emails.csv -p admin-password

GET _search
{
  "query": {
    "match_all": {}
  }
}

{
  "took": 38,
  "timed_out": false,
  "_shards": {
    "total": 6,
    "successful": 6,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": {
      "value": 6799,
      "relation": "eq"
    },
    "max_score": 1.0,
    "hits": [
      {
        "_index": "email",
        "_id": "c13MwoUBmlIzd81ioZ7H",
        "_score": 1.0,
        "_source": {
          "url": "allen-p/_sent_mail/118.",
          "date_published": "2000-09-26 09:26:00.000000",
          "author": "<name>",
          "to": "<email-address>",
          "subject": "Investment Structure",
          "content": "---------------------- Forwarded by ..."
        }
      }
    ]
  }
}