%load_ext autoreload
%autoreload 2

import bs4
from langchain import hub
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_community.document_loaders import WebBaseLoader
from langchain_community.vectorstores import Chroma
from langchain_core.output_parsers import StrOutputParser
from langchain_core.runnables import RunnablePassthrough
from langchain_openai import ChatOpenAI, OpenAIEmbeddings

Improving the Retriever

things we are going to try

• different embeddedding (with fastembed and openai):

• reranker (cohere)

• Multi-Query Retriever

Building the VectorStore

from langchain.document_loaders import DirectoryLoader
loader = DirectoryLoader("./data/")
documents = loader.load()

for document in documents:
    document.metadata['file_name'] = document.metadata['source']

docs = documents
len(docs)

26

text_splitter = RecursiveCharacterTextSplitter(
    chunk_size=1000, 
    chunk_overlap=200
)

splits = text_splitter.split_documents(docs)

from langsmith import Client

client = Client()
examples = list(client.list_examples(dataset_name="basecamp"))

examples[0]

Example(dataset_id=UUID('8f267706-24b2-47fb-84ee-3ea3cfc5a0c0'), inputs={'question': 'How do the cycles at 37signals affect communication and decision-making?'}, outputs={'ground_truth': 'The cycles at 37signals help to create a fixed cadence and provide a regular interval for decision-making. They also help to prioritize work and break big projects into smaller ones. The communication mechanisms, such as daily and weekly check-ins, heartbeats, and kickoffs, ensure that everyone is kept in the loop about the work being done.'}, id=UUID('771183c7-5ff6-4fde-bef0-8e999de218e1'), created_at=datetime.datetime(2024, 3, 6, 20, 31, 43, 147801, tzinfo=datetime.timezone.utc), modified_at=datetime.datetime(2024, 3, 6, 20, 31, 43, 147801, tzinfo=datetime.timezone.utc), runs=[], source_run_id=None)

Experiment 1: Try out different Embeddings

lets evaluate between

• ada2

• BGE

• OpenAI’s new text-embedding-3-large

# ada
vectorstore_ada = Chroma.from_documents(
    documents=splits, 
    embedding=OpenAIEmbeddings(model="text-embedding-ada-002")
)

Lets build a reference RAG with this embedding

from operator import itemgetter

from langchain_core.output_parsers import StrOutputParser
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.runnables import RunnableLambda, RunnablePassthrough, RunnableParallel
from langchain_openai import ChatOpenAI

vectorstore_retriever = vectorstore_ada.as_retriever()
prompt = hub.pull("rlm/rag-prompt")
llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0)


def format_retriever(input_dict):
    # if dict or list
    if isinstance(input_dict, dict):
        docs = input_dict["contexts"]
    else:
        docs = input_dict

    # get the text in each chunk
    doc_strs = []
    for d in docs:
        if isinstance(d, str):
            doc_strs.append(d)
        else:
            doc_strs.append(d.page_content)

    # join and send the rest
    return {
        "question": input_dict["question"],
        "context": "\n\n".join(doc_strs)
    }
    
def ragas_output_parser(input):
    if isinstance(input, list):
        return [doc.page_content for doc in input]
    elif isinstance(input, dict):
        docs = input["contexts"]
        return [doc.page_content for doc in docs]

def passthrough(column_name):
   return RunnableLambda(lambda x: x.get(column_name) if isinstance(x, dict) else x)

from langchain_core.runnables import RunnableParallel

generator = (
    prompt
    | llm
    | StrOutputParser()
)

def retriver_factory(retriever):
    retriever = RunnableParallel({
        "contexts": passthrough("question") | retriever | ragas_output_parser, 
        "question": passthrough("question")
    })

    return retriever

filter_langsmith_dataset = RunnableLambda(lambda x: x["question"] if isinstance(x, dict) else x)

def rag_factory(vector_store=None, retriever = None):
    if vector_store is not None:
        retriever = vector_store.as_retriever()
    elif retriever is not None:
        retriever = retriever
    else:
        raise ValueError("You must provide a vectorstore or a retriever")
    rag_chain_ada = (
        filter_langsmith_dataset |
        retriver_factory(retriever) |
        RunnableParallel({
            "answer": format_retriever | generator,
            "contexts": RunnablePassthrough()
        })
    )
    return rag_chain_ada

rag_chain_ada = rag_factory(vectorstore_ada)

q = examples[0].inputs
q["question"]

'How do the cycles at 37signals affect communication and decision-making?'

get_answer = RunnableLambda(lambda x: x["answer"])
(rag_factory(vectorstore_ada) | get_answer).invoke(q)

'The cycles at 37signals create a sense of urgency and help prevent projects from becoming too large. They also provide a regular interval for decision-making on what to work on. Communication is facilitated through daily and weekly questions about work progress and intentions.'

BGE Embeddings

# BGE
from langchain_community.embeddings.fastembed import FastEmbedEmbeddings
vectorstore_bge = Chroma.from_documents(
    collection_name="bge",
    documents=splits, 
    embedding=FastEmbedEmbeddings(model_name="BAAI/bge-large-en-v1.5")
)

2024-03-13 17:12:55.547 | WARNING  | fastembed.embedding:<module>:7 - DefaultEmbedding, FlagEmbedding, JinaEmbedding are deprecated.Use from fastembed import TextEmbedding instead.

# make the rag
bge_rag = rag_factory(vectorstore_bge)

(bge_rag | get_answer).invoke(q)

'The cycles at 37signals create a sense of urgency and help prevent projects from becoming too large. They also provide a regular interval for decision-making on what to work on. Communication is facilitated through daily and weekly questions about work progress and intentions.'

OpenAI’s Text Embedding 3 Large

# text-embedding
vectorstore_text_embedding = Chroma.from_documents(
    collection_name="text-emb-3-lg",
    documents=splits, 
    embedding=OpenAIEmbeddings(model="text-embedding-3-large")
)

# make the rag
text_embed_rag = rag_factory(vectorstore_text_embedding)

(text_embed_rag | get_answer).invoke(q)

'The cycles at 37signals create a sense of urgency and help prevent projects from becoming too large. They also provide a regular interval for decision-making on what projects to work on. Communication is facilitated through daily and weekly questions about work progress and intentions.'

Evaluation of the different embeddings

from ragas.integrations.langchain import EvaluatorChain
from ragas.integrations.langsmith import evaluate

# import the metrics we will need
from ragas.metrics import context_precision, context_recall

# retriever with Ada embeddings
ada_retriever = retriver_factory(vectorstore_ada)

run = evaluate(
    experiment_name="ada",
    dataset_name="basecamp", 
    llm_or_chain_factory=retriver_factory(vectorstore_ada), 
    metrics=[context_precision, context_recall],
    verbose=False
)

View the evaluation results for project 'ada' at:
https://smith.langchain.com/o/9bfbddc5-b88e-41e5-92df-2a62f0c64b4b/datasets/8f267706-24b2-47fb-84ee-3ea3cfc5a0c0/compare?selectedSessions=60f28e99-82b0-4a32-9ec8-8cfae59ac7da

View all tests for Dataset basecamp at:
https://smith.langchain.com/o/9bfbddc5-b88e-41e5-92df-2a62f0c64b4b/datasets/8f267706-24b2-47fb-84ee-3ea3cfc5a0c0
[>                                                 ] 0/7

/home/jjmachan/.pyenv/versions/3.10.12/envs/notes/lib/python3.10/site-packages/langchain_core/_api/deprecation.py:117: LangChainDeprecationWarning: The function `__call__` was deprecated in LangChain 0.1.0 and will be removed in 0.2.0. Use invoke instead.
  warn_deprecated(

[------------->                                    ] 2/7

Invalid JSON response. Expected dictionary with key 'Attributed'
Failed to batch ingest runs: LangSmithError('Failed to post https://api.smith.langchain.com/runs/batch in LangSmith API. HTTPError(\'400 Client Error: Bad Request for url: https://api.smith.langchain.com/runs/batch\', \'{"detail":"Request body is not valid JSON"}\')')

[------------------------------------------------->] 7/7

# retriever with BGE embeddings
bge_retriever = retriver_factory(vectorstore_bge)

run = evaluate(
    experiment_name="bge_retriever",
    dataset_name="basecamp", 
    llm_or_chain_factory=bge_retriever, 
    metrics=[context_precision, context_recall],
    verbose=True
)

View the evaluation results for project 'bge_retriever' at:
https://smith.langchain.com/o/9bfbddc5-b88e-41e5-92df-2a62f0c64b4b/datasets/8f267706-24b2-47fb-84ee-3ea3cfc5a0c0/compare?selectedSessions=efc8c7d0-5517-4d9d-abfc-0427dbe2b86a

View all tests for Dataset basecamp at:
https://smith.langchain.com/o/9bfbddc5-b88e-41e5-92df-2a62f0c64b4b/datasets/8f267706-24b2-47fb-84ee-3ea3cfc5a0c0
[------------------------------------------------->] 7/7

Experiment Results:

	feedback.context_precision	feedback.context_recall	error	execution_time	run_id
count	7.000000	7.000000	0	7.000000	7
unique	NaN	NaN	0	NaN	7
top	NaN	NaN	NaN	NaN	b3dd731c-22ca-49e4-abf1-d86e5618aa9b
freq	NaN	NaN	NaN	NaN	1
mean	0.793651	0.928571	NaN	0.476688	NaN
std	0.374007	0.188982	NaN	0.249074	NaN
min	0.000000	0.500000	NaN	0.144702	NaN
25%	0.777778	1.000000	NaN	0.260192	NaN
50%	1.000000	1.000000	NaN	0.643713	NaN
75%	1.000000	1.000000	NaN	0.668870	NaN
max	1.000000	1.000000	NaN	0.690279	NaN

# retriever with text-embedding embeddings
text_embedding_retriever = retriver_factory(vectorstore_text_embedding)

run = evaluate(
    experiment_name="text_embedding_3_retriver",
    dataset_name="basecamp", 
    llm_or_chain_factory=text_embedding_retriever, 
    metrics=[context_precision, context_recall],
    verbose=True
)

View the evaluation results for project 'text_embedding_3_retriver' at:
https://smith.langchain.com/o/9bfbddc5-b88e-41e5-92df-2a62f0c64b4b/datasets/8f267706-24b2-47fb-84ee-3ea3cfc5a0c0/compare?selectedSessions=ad669381-ced0-4a1f-ae3c-45d4f852c902

View all tests for Dataset basecamp at:
https://smith.langchain.com/o/9bfbddc5-b88e-41e5-92df-2a62f0c64b4b/datasets/8f267706-24b2-47fb-84ee-3ea3cfc5a0c0
[------------------------------------------>       ] 6/7

Invalid JSON response. Expected dictionary with key 'Attributed'
Failed to batch ingest runs: LangSmithError('Failed to post https://api.smith.langchain.com/runs/batch in LangSmith API. HTTPError(\'400 Client Error: Bad Request for url: https://api.smith.langchain.com/runs/batch\', \'{"detail":"Request body is not valid JSON"}\')')

[------------------------------------------------->] 7/7

Experiment Results:

	feedback.context_precision	feedback.context_recall	error	execution_time	run_id
count	7.000000	6.0	0	7.000000	7
unique	NaN	NaN	0	NaN	7
top	NaN	NaN	NaN	NaN	88b94785-b177-48ec-843a-4b8c1c036713
freq	NaN	NaN	NaN	NaN	1
mean	0.948413	1.0	NaN	0.439906	NaN
std	0.088466	0.0	NaN	0.130925	NaN
min	0.805556	1.0	NaN	0.272072	NaN
25%	0.916667	1.0	NaN	0.363577	NaN
50%	1.000000	1.0	NaN	0.432972	NaN
75%	1.000000	1.0	NaN	0.516501	NaN
max	1.000000	1.0	NaN	0.614142	NaN

Experiment 2: Using a ReRanker

# Helper function for printing docs
def pretty_print_docs(docs):
    print(
        f"\n{'-' * 100}\n".join(
            [f"Document {i+1}:\n\n" + d.page_content for i, d in enumerate(docs)]
        )
    )

ret = vectorstore_text_embedding.as_retriever(search_kwargs={"k": 10})
docs = ret.get_relevant_documents(q["question"])
pretty_print_docs(docs)

Document 1:

How We Work

Cycles

We work in 6-week or 8-week cycles at 37signals. There are typically six cycles to a year. Two are 8-week cycles, during Summer Hours, and the rest 6-week cycles. This fixed cadence serves to give us an internal sense of urgency, work as a scope hammer to keep projects from ballooning, and provide a regular interval to decide what we’re working on.

The idea is not that everything we ever decide to work on has to take six or eight weeks or can be completed in that time. But rather that we think about how we can break big projects into smaller ones that can be done in that amount of time, and that we bundle smaller things into a presentable scope of work that can be discussed.
----------------------------------------------------------------------------------------------------
Document 2:

Communication

It’s hard to keep up on what everyone is doing and what it means, if you just watch the stream of latest activity scrolling along in 37signals. (It’s also a waste of time and source of stress to even try.) Instead, we have four chief mechanisms for keeping everyone in the loop about the work that’s going on.

First, there’s the daily question of What did you work on today?, which supplies the nitty gritty details, but as a personal narrative. They’re a great conversation starter if you see someone working on something you either care about or want to learn more about. Please do use them as such! You’re obliged to answer this question at least twice a week.

Second, there’s the weekly question of What will you be working on this week?, which details your intentions for the coming week. Everyone except team OMG is obliged to answer this question when they’re not out.
----------------------------------------------------------------------------------------------------
Document 3:

While a few pitches might instantly strike a chord loud enough to go on the plate for the next cycle, it’s more likely that your pitch will sit for a while first. There are always more ideas than time, and we can only get a few things done each cycle. So chances are that even if everyone agrees the pitch is a great idea, it might not be the next most important thing for us to tackle. Don’t be discouraged by this. We’ve had many pitches that have sat for many cycles, if not years, before finally coming together and then happening.

Asynchronously

We have people working all sorts of different hours and from all sorts of different places at 37signals. That alone makes it hard to enforce a lot of tightly-coupled workflows during the day, but that’s a feature not a bug. Most of the work you do at 37signals shouldn’t require you to be in constant communication throughout the entire day with someone.
----------------------------------------------------------------------------------------------------
Document 4:

These mechanisms work together to free individuals and teams to run their days and cycles with confidence and independence. We have six opportunities per year to make big decisions about what to work on, and the rest of the time should chiefly be spent carrying out those short-term plans. By having clear expectations for communication, it’s easier for everyone to build trust in where we’re going and why.

All these questions and write-ups will be posted in the What Works project for the current year.

Pitches

Whether you work on the product development or not, your voice and observations can help determine what we should be working on. The way to exert this influence is through pitches.
----------------------------------------------------------------------------------------------------
Document 5:

So, this is where we’ll try to share what’s worth knowing about 37signals the company, our culture, our process, and our history. It’s a guide to understanding what people are talking about when they call for “judo” (redefining a hard problem into an easy one) or whether it’s okay to take your vacation when you’ve only been with us for a month (yes).
----------------------------------------------------------------------------------------------------
Document 6:

What Influenced Us

If you want to learn the 37signals view of the world, it helps to know the influences that helped form it. We’ve been around since 1999. Since then there’s been a number of key influencers that have marked the company culture.

Books

Turn The Ship Around: “Leadership should mean giving control rather than taking control and creating leaders rather than forging followers”, David Marquet. 37signals employs great people and they deserve the freedom and autonomy to act on their own. Don’t wait for permission, just state what you’re going to do, and then do it.

Finding Flow: True happiness is found in optimal moments of engagement when we stretch just beyond our abilities and lose track of time and space in the process. Protecting the flow by limiting interruptions has been a driving principle of 37signals.
----------------------------------------------------------------------------------------------------
Document 7:

But let’s take a look back. In 2003, 37signals was a web design firm made up of 4 people. We always had work, but we were disorganized. With so many concurrent projects, things began to slip through the cracks. Projects dragged on too long. We dropped the ball on key deliverables. We had some major miscommunication. As many people even still do, we relied on email for everything. Things inevitably got lost, people get left out of conversations, there’s nowhere to go to see what’s left to do.

So we started looking for a project management tool. We tried a few tools, but they were complicated and didn’t fit what we needed. Frustrated, we decided to build our own simple project management app. A few months later we had something ready, and we immediately started using this tool with our existing clients.
----------------------------------------------------------------------------------------------------
Document 8:

With managers of one

We rely on everyone at 37signals to do a lot of self-management. People who do this well qualify as managers of one, and we strive for everyone senior or above to embody this principle fully.

That means setting your own direction when one isn’t given. Determining what needs to be done, and doing it, without waiting for someone to tell you to. A manager of one will spend their time well when left to their own devices. There’s always more work to be done, always more initiatives to kick off, always more improvement to be had.

Balanced

We limit ourselves to a 40-hour (32-hour in the summer) work week. Keeping our hours at work limited forces us to prioritize the work that really matters. A healthy amount of sleep and a rich and rewarding life outside of work should not be squandered for a few more hours at work.
----------------------------------------------------------------------------------------------------
Document 9:

Cooldown

In between each cycle, we spend two weeks cooling down. That’s the time to deal with bugs or smaller issues that come up, write up what we worked on, and figure out what we should tackle next. It’s sometimes tempting to simply extend the cycles into the cooldown period to fit in more work. But the goal is to resist this temptation. Yes, sometimes a little spill-over will happen, but it’s helpful to think about the end of the normal cycle as “pencils down”. That means that by week 4 of a normal cycle, we should be winding down, getting ready to launch, make sure QA is lined up, and all the other work that happens during and after the launch of new projects.

Communication
----------------------------------------------------------------------------------------------------
Document 10:

Daily and weekly check-ins are subdivided by department so you’re only subscribed to your team’s answers. You’re of course free to subscribe to other team check-ins, but you’re not obligated to do so if you find it too noisy.

Third, there are the heartbeats. These are the team versions of What did you work on this cycle? This is where we summarize and celebrate the work that’s been done. Every team lead is obliged to write, or designate someone on the team to write, this account one week after a cycle has ended.

Fourth, and finally, there are the kickoffs. These are the team version of What are you going to work on next cycle? This is where the plan for the coming six or eight weeks is presented. Every team lead is obliged to write, or designate someone on the team to write, this account before the start of the new cycle.

from langchain.retrievers import ContextualCompressionRetriever
from langchain.retrievers.document_compressors.cohere_rerank import CohereRerank

llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0)
compressor = CohereRerank()
vectorstore_text_embedding.as_retriever
compression_retriever = ContextualCompressionRetriever(
    base_compressor=compressor, 
    base_retriever=vectorstore_text_embedding.as_retriever(search_kwargs={"k": 10})
)

compressed_docs = compression_retriever.get_relevant_documents(q["question"])
pretty_print_docs(compressed_docs)

Document 1:

How We Work

Cycles

We work in 6-week or 8-week cycles at 37signals. There are typically six cycles to a year. Two are 8-week cycles, during Summer Hours, and the rest 6-week cycles. This fixed cadence serves to give us an internal sense of urgency, work as a scope hammer to keep projects from ballooning, and provide a regular interval to decide what we’re working on.

The idea is not that everything we ever decide to work on has to take six or eight weeks or can be completed in that time. But rather that we think about how we can break big projects into smaller ones that can be done in that amount of time, and that we bundle smaller things into a presentable scope of work that can be discussed.
----------------------------------------------------------------------------------------------------
Document 2:

While a few pitches might instantly strike a chord loud enough to go on the plate for the next cycle, it’s more likely that your pitch will sit for a while first. There are always more ideas than time, and we can only get a few things done each cycle. So chances are that even if everyone agrees the pitch is a great idea, it might not be the next most important thing for us to tackle. Don’t be discouraged by this. We’ve had many pitches that have sat for many cycles, if not years, before finally coming together and then happening.

Asynchronously

We have people working all sorts of different hours and from all sorts of different places at 37signals. That alone makes it hard to enforce a lot of tightly-coupled workflows during the day, but that’s a feature not a bug. Most of the work you do at 37signals shouldn’t require you to be in constant communication throughout the entire day with someone.
----------------------------------------------------------------------------------------------------
Document 3:

Communication

It’s hard to keep up on what everyone is doing and what it means, if you just watch the stream of latest activity scrolling along in 37signals. (It’s also a waste of time and source of stress to even try.) Instead, we have four chief mechanisms for keeping everyone in the loop about the work that’s going on.

First, there’s the daily question of What did you work on today?, which supplies the nitty gritty details, but as a personal narrative. They’re a great conversation starter if you see someone working on something you either care about or want to learn more about. Please do use them as such! You’re obliged to answer this question at least twice a week.

Second, there’s the weekly question of What will you be working on this week?, which details your intentions for the coming week. Everyone except team OMG is obliged to answer this question when they’re not out.

rag_with_reranker = rag_factory(retriever=compression_retriever)
(rag_with_reranker | get_answer).invoke(q)

'The cycles at 37signals provide a fixed cadence for decision-making and project scope. They help break big projects into smaller ones that can be completed in a set amount of time. Communication is facilitated through daily and weekly questions to keep everyone in the loop about the work being done.'

run = evaluate(
    experiment_name="reranker",
    dataset_name="basecamp", 
    llm_or_chain_factory=rag_with_reranker, 
    metrics=[context_precision, context_recall],
    verbose=True
)

View the evaluation results for project 'reranker' at:
https://smith.langchain.com/o/9bfbddc5-b88e-41e5-92df-2a62f0c64b4b/datasets/8f267706-24b2-47fb-84ee-3ea3cfc5a0c0/compare?selectedSessions=127a0557-51f5-4f02-908a-bbc5502058ed

View all tests for Dataset basecamp at:
https://smith.langchain.com/o/9bfbddc5-b88e-41e5-92df-2a62f0c64b4b/datasets/8f267706-24b2-47fb-84ee-3ea3cfc5a0c0
[-------------------->                             ] 3/7

Invalid JSON response. Expected dictionary with key 'Attributed'
Failed to batch ingest runs: LangSmithError('Failed to post https://api.smith.langchain.com/runs/batch in LangSmith API. HTTPError(\'400 Client Error: Bad Request for url: https://api.smith.langchain.com/runs/batch\', \'{"detail":"Request body is not valid JSON"}\')')

[---------------------------->                     ] 4/7

Invalid JSON response. Expected dictionary with key 'Attributed'
Failed to batch ingest runs: LangSmithError('Failed to post https://api.smith.langchain.com/runs/batch in LangSmith API. HTTPError(\'400 Client Error: Bad Request for url: https://api.smith.langchain.com/runs/batch\', \'{"detail":"Request body is not valid JSON"}\')')

[------------------------------------------>       ] 6/7

Invalid JSON response. Expected dictionary with key 'Attributed'
Failed to batch ingest runs: LangSmithError('Failed to post https://api.smith.langchain.com/runs/batch in LangSmith API. HTTPError(\'400 Client Error: Bad Request for url: https://api.smith.langchain.com/runs/batch\', \'{"detail":"Request body is not valid JSON"}\')')

[------------------------------------------------->] 7/7

Experiment Results:

	feedback.context_precision	feedback.context_recall	error	execution_time	run_id
count	7.000000	4.0	0	7.000000	7
unique	NaN	NaN	0	NaN	7
top	NaN	NaN	NaN	NaN	6fde9b65-8174-4fa1-8441-9dee884f23a0
freq	NaN	NaN	NaN	NaN	1
mean	0.428571	1.0	NaN	2.534166	NaN
std	0.534522	0.0	NaN	0.437899	NaN
min	0.000000	1.0	NaN	1.776245	NaN
25%	0.000000	1.0	NaN	2.398282	NaN
50%	0.000000	1.0	NaN	2.570000	NaN
75%	1.000000	1.0	NaN	2.694026	NaN
max	1.000000	1.0	NaN	3.208302	NaN

Experiment 3: Multi-Query Retriever

q

{'question': 'How do the cycles at 37signals affect communication and decision-making?'}

from langchain.retrievers.multi_query import MultiQueryRetriever

multi_query_ret = MultiQueryRetriever.from_llm(
    retriever=vectorstore_text_embedding.as_retriever(), 
    llm=llm
)

docs = multi_query_ret.get_relevant_documents(q["question"])
pretty_print_docs(docs)

Document 1:

How We Work

Cycles

We work in 6-week or 8-week cycles at 37signals. There are typically six cycles to a year. Two are 8-week cycles, during Summer Hours, and the rest 6-week cycles. This fixed cadence serves to give us an internal sense of urgency, work as a scope hammer to keep projects from ballooning, and provide a regular interval to decide what we’re working on.

The idea is not that everything we ever decide to work on has to take six or eight weeks or can be completed in that time. But rather that we think about how we can break big projects into smaller ones that can be done in that amount of time, and that we bundle smaller things into a presentable scope of work that can be discussed.
----------------------------------------------------------------------------------------------------
Document 2:

Communication

It’s hard to keep up on what everyone is doing and what it means, if you just watch the stream of latest activity scrolling along in 37signals. (It’s also a waste of time and source of stress to even try.) Instead, we have four chief mechanisms for keeping everyone in the loop about the work that’s going on.

First, there’s the daily question of What did you work on today?, which supplies the nitty gritty details, but as a personal narrative. They’re a great conversation starter if you see someone working on something you either care about or want to learn more about. Please do use them as such! You’re obliged to answer this question at least twice a week.

Second, there’s the weekly question of What will you be working on this week?, which details your intentions for the coming week. Everyone except team OMG is obliged to answer this question when they’re not out.
----------------------------------------------------------------------------------------------------
Document 3:

While a few pitches might instantly strike a chord loud enough to go on the plate for the next cycle, it’s more likely that your pitch will sit for a while first. There are always more ideas than time, and we can only get a few things done each cycle. So chances are that even if everyone agrees the pitch is a great idea, it might not be the next most important thing for us to tackle. Don’t be discouraged by this. We’ve had many pitches that have sat for many cycles, if not years, before finally coming together and then happening.

Asynchronously

We have people working all sorts of different hours and from all sorts of different places at 37signals. That alone makes it hard to enforce a lot of tightly-coupled workflows during the day, but that’s a feature not a bug. Most of the work you do at 37signals shouldn’t require you to be in constant communication throughout the entire day with someone.
----------------------------------------------------------------------------------------------------
Document 4:

What Influenced Us

If you want to learn the 37signals view of the world, it helps to know the influences that helped form it. We’ve been around since 1999. Since then there’s been a number of key influencers that have marked the company culture.

Books

Turn The Ship Around: “Leadership should mean giving control rather than taking control and creating leaders rather than forging followers”, David Marquet. 37signals employs great people and they deserve the freedom and autonomy to act on their own. Don’t wait for permission, just state what you’re going to do, and then do it.

Finding Flow: True happiness is found in optimal moments of engagement when we stretch just beyond our abilities and lose track of time and space in the process. Protecting the flow by limiting interruptions has been a driving principle of 37signals.
----------------------------------------------------------------------------------------------------
Document 5:

So, this is where we’ll try to share what’s worth knowing about 37signals the company, our culture, our process, and our history. It’s a guide to understanding what people are talking about when they call for “judo” (redefining a hard problem into an easy one) or whether it’s okay to take your vacation when you’ve only been with us for a month (yes).
----------------------------------------------------------------------------------------------------
Document 6:

These mechanisms work together to free individuals and teams to run their days and cycles with confidence and independence. We have six opportunities per year to make big decisions about what to work on, and the rest of the time should chiefly be spent carrying out those short-term plans. By having clear expectations for communication, it’s easier for everyone to build trust in where we’re going and why.

All these questions and write-ups will be posted in the What Works project for the current year.

Pitches

Whether you work on the product development or not, your voice and observations can help determine what we should be working on. The way to exert this influence is through pitches.

multi_query_rag = rag_factory(retriever=multi_query_ret)
(multi_query_rag | get_answer).invoke(q)

'The cycles at 37signals create a sense of urgency and help prevent projects from becoming too large. Communication is facilitated through daily and weekly questions to keep everyone in the loop about the work being done. Pitches play a role in decision-making, with some ideas taking multiple cycles to come to fruition.'

_ = evaluate(
    experiment_name="mulit_query",
    dataset_name="basecamp", 
    llm_or_chain_factory=multi_query_rag, 
    metrics=[context_precision, context_recall],
    verbose=True
)

View the evaluation results for project 'mulit_query' at:
https://smith.langchain.com/o/9bfbddc5-b88e-41e5-92df-2a62f0c64b4b/datasets/8f267706-24b2-47fb-84ee-3ea3cfc5a0c0/compare?selectedSessions=581dd7af-7bae-4242-bb78-a137e2355898

View all tests for Dataset basecamp at:
https://smith.langchain.com/o/9bfbddc5-b88e-41e5-92df-2a62f0c64b4b/datasets/8f267706-24b2-47fb-84ee-3ea3cfc5a0c0
[------>                                           ] 1/7

Invalid JSON response. Expected dictionary with key 'Attributed'
Invalid JSON response. Expected dictionary with key 'Attributed'
Failed to batch ingest runs: LangSmithError('Failed to post https://api.smith.langchain.com/runs/batch in LangSmith API. HTTPError(\'400 Client Error: Bad Request for url: https://api.smith.langchain.com/runs/batch\', \'{"detail":"Request body is not valid JSON"}\')')

[-------------------->                             ] 3/7

Invalid JSON response. Expected dictionary with key 'Attributed'
Failed to batch ingest runs: LangSmithError('Failed to post https://api.smith.langchain.com/runs/batch in LangSmith API. HTTPError(\'400 Client Error: Bad Request for url: https://api.smith.langchain.com/runs/batch\', \'{"detail":"Request body is not valid JSON"}\')')

[---------------------------->                     ] 4/7

Invalid JSON response. Expected dictionary with key 'Attributed'
Failed to batch ingest runs: LangSmithError('Failed to post https://api.smith.langchain.com/runs/batch in LangSmith API. HTTPError(\'400 Client Error: Bad Request for url: https://api.smith.langchain.com/runs/batch\', \'{"detail":"Request body is not valid JSON"}\')')

[------------------------------------------>       ] 6/7

Invalid JSON response. Expected dictionary with key 'Attributed'
Failed to batch ingest runs: LangSmithError('Failed to post https://api.smith.langchain.com/runs/batch in LangSmith API. HTTPError(\'400 Client Error: Bad Request for url: https://api.smith.langchain.com/runs/batch\', \'{"detail":"Request body is not valid JSON"}\')')

[------------------------------------------------->] 7/7

Experiment Results:

	feedback.context_precision	feedback.context_recall	error	execution_time	run_id
count	7.000000	2.0	0	7.000000	7
unique	NaN	NaN	0	NaN	7
top	NaN	NaN	NaN	NaN	bce40e6c-d7b5-4fe2-9bf6-234bba353c21
freq	NaN	NaN	NaN	NaN	1
mean	0.571429	1.0	NaN	4.735159	NaN
std	0.534522	0.0	NaN	0.981516	NaN
min	0.000000	1.0	NaN	3.859380	NaN
25%	0.000000	1.0	NaN	3.989503	NaN
50%	1.000000	1.0	NaN	4.533062	NaN
75%	1.000000	1.0	NaN	5.192072	NaN
max	1.000000	1.0	NaN	6.390519	NaN