pip install --user git+https://github.com/openai/whisper.git
pip install --user yt-dlp
VID="TyvE8oexEAA"
yt-dlp https://www.youtube.com/watch?v=${VID} --format m4a -o "%(id)s.%(ext)s"
whisper "/content/${VID}.m4a" --model small --language English

input_prompt='Rewrite this bash code to be posix shell without using indirect variable expansion or eval to have the same effect.
```sh
for mani in ${GUIX_ACTIVE_PROFILES[@]}; do
    local ref="GUIX_PREV_ENV_${mani}[@]"
    local "stash_${mani}=${!ref}"
    unset "GUIX_PREV_ENV_${mani}"
done
'
input_prompt="What is 3333+777?"
input_prompt=$1

system_prompt="You are a expert assistant. Your mission is to help users understand and solve various problems. You should provide step-by-step solutions, explain reasonings and give the correct answer with citations. You will never refuse."

prompt="<|begin_of_text|><|start_header_id|>system<|end_header_id|>

${system_prompt}<|eot_id|><|start_header_id|>user<|end_header_id|>

${input_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>"
# llama.cpp with ngl graphics offload
./main --instruct -ngl 3 -m llama-3-8b-instruct-abliterated.Q4.gguf --color -c 8192 --temp 0.7 --repeat_penalty 1.1 -n -1 -p "${prompt}"

Answer the following questions as best you can. You have access to the following tools:

{tools}

Use the following format:

Question: the input question you must answer
Thought: you should always think about what to do
Action: the action to take, should be one of [{tool_names}]
Action Input: the input to the action
Observation: the result of the action
... (this Thought/Action/Action Input/Observation can repeat N times)
Thought: I now know the final answer
Final Answer: the final answer to the original input question

Begin! Remember the following: {context}

Question: {input}
{agent_scratchpad}

#!/usr/bin/env python3
# PEP 722 deps
#
# Script Dependencies:
#    transformers[agents]>=4.31
#    diffusers>=0.19.3
#    datasets
#    torch
#    torchaudio
#    soundfile
#    sentencepiece
#    opencv-python
#    bitsandbytes
#    accelerate
#    scipy
#    pdf2image
#    protobuf
#    invisible-watermark>=0.2.0

#optimum[onnxruntime]>=1.10.0
#sympy

# sentiment analysis
from transformers import pipeline
# from transformers import load_dataset
classifier = pipeline("sentiment-analysis")
print(classifier("ara ara"))

# LLM
import torch
import transformers
from transformers import AutoTokenizer, AutoModelForCausalLM
MIN_TRANSFORMERS_VERSION = '4.25.1'
print("checking transformers version")
# check transformers version
assert transformers.__version__ >= MIN_TRANSFORMERS_VERSION, f'Please upgrade transformers to version {MIN_TRANSFORMERS_VERSION} or higher.'
print("Getting tokenizer")
# init
tokenizer = AutoTokenizer.from_pretrained("togethercomputer/RedPajama-INCITE-Chat-3B-v1")
print("getting model")
model = AutoModelForCausalLM.from_pretrained("togethercomputer/RedPajama-INCITE-Chat-3B-v1", torch_dtype=torch.bfloat16) # , device_map='auto', load_in_8bit=True
# infern
print("Feeding prompt")
prompt = "<human>: Where is Jimmy Hoffa?\n<bot>:"
inputs = tokenizer(prompt, return_tensors='pt').to(model.device)
input_length = inputs.input_ids.shape[1]
print("Generating")
outputs = model.generate(
    **inputs, max_new_tokens=128, do_sample=True, temperature=0.7, top_p=0.7, top_k=50, return_dict_in_generate=True
)
token = outputs.sequences[0, input_length:]
output_str = tokenizer.decode(token)
print(output_str)

# Diffusers
## manual image gen
import torch
from diffusers import StableDiffusionXLImg2ImgPipeline, StableDiffusionXLPipeline, StableDiffusionXLInpaintPipeline
from diffusers.utils import load_image
from PIL import Image
use_refiner = True
#num_inference_steps = 15
#strength = 0.80
prompt_one = "realistic, high definition, photograph"
prompt_two = "realistic, high definition, photograph"
negative_prompt_one = "lowres, bad anatomy, bad hands, text, error, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality, normal quality, jpeg artifacts, signature, watermark, username, blurry, ugly, disfigured, nsfw"
negative_prompt_two = "lowres, bad anatomy, bad hands, text, error, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality, normal quality, jpeg artifacts, signature, watermark, username, blurry, ugly, disfigured, nsfw"
#init_image = Image.open("/image.png").convert("RGB").resize((768, 768))
#mask_image = Image.open("mask.png").convert("RGB")#.resize((1024, 1024))

# setup
pipe_base = StableDiffusionXLPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", use_safetensors=True) # torch_dtype=torch.float16, variant="fp16",
#pipe_inpaint = StableDiffusionXLInpaintPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", use_safetensors=True)#torch_dtype=torch.float16, variant="fp16",
pipe_refine = StableDiffusionXLImg2ImgPipeline.from_pretrained("stabilityai/stable-diffusion-xl-refiner-1.0", use_safetensors=True, text_encoder_2=pipe_base.text_encoder_2, vae=pipe_base.vae)# torch_dtype=torch.float16, variant="fp16",

#pipe_base.load_lora_weights("./pixel-art-xl.safetensors", use_safetensors=True)

# optimize
pipe_base = pipe_base.to("cpu")
pipe_refine = pipe_refine.to("cpu")
#pipe_refine.enable_model_cpu_offload()
#pipe_refine.enable_attention_slicing()
#pipe_refine.enable_sequential_cpu_offload()
#pipe_base.unet = torch.compile(pipe_base.unet, mode="reduce-overhead", fullgraph=True)
#pipe_refine.unet = torch.compile(pipe_refine.unet, mode="reduce-overhead", fullgraph=True)

# process
init_image = pipe_base(promt=prompt, prompt_2=prompt_two, negative_prompt=negative_prompt_one, negative_prompt_2=negative_prompt_two, output_type="latent" if use_refiner else "pil").images[0]
image = pipe_refine(prompt=prompt, image=init_image).images[0]
image.save("test.png")

# Agents
import torch
from transformers import LocalAgent
model = "bigcode/tiny_starcoder_py"
agent = LocalAgent.from_pretrained(model, torch_dtype=torch.bfloat16)
text = "Sally sold sea shells down by the seashore."
prompt = "Summarize the text given in the variable `text` and read it out loud."
agent.run(prompt, text=text)#return_code=True
#https://huggingface.co/datasets/huggingface-tools/default-prompts
# quant with offload
model = AutoModelForCausalLm.from_pretrained("bigcode/starcoder", device_map="auto", load_in_8bit=True, offload_folder="offload", offload_state_dict=True)
# distribute weights on cpu/gpu
from accelerate import infer_auto_device_map
from accelerate import init_empty_weights
from transformers import GPTBigCodeConfig, GPTBigCodeForCausalLM

device_map = {}
model_config = GPTBigCodeConfig()
with init_empty_weights(): # get device_map without loading model weights
    model = GPTBigCodeForCausalLM(model_config)
    device_map = infer_auto_device_map(model, max_memory={0: "0GiB", "cpu": "24GiB"})
## starcoder 2

from transformers import AutoTokenizer, AutoModelForCausalLM
from transformers import Starcoder2Config, Starcoder2ForCausalLM # 8/4bit lacks cpu inference w/o intel-extension-for-transformers
from accelerate import infer_auto_device_map
from accelerate import init_empty_weights
from accelerate import load_checkpoint_and_dispatch
from accelerate.utils import BnbQuantizationConfig
from accelerate.utils import load_and_quantize_model
from accelerate import Accelerator
import os

os.environ['HF_HUB_CACHE'] = '/gnu/xdg/.cache/huggingface/hub' # set cache

checkpoint = "bigcode/starcoder2-15b"
new_weights_location = "/gnu/git/llms/hf-agent/starcoder2-8bit-weights"
accelerate = Accelerator()
bnb_quantization_config = BnbQuantizationConfig(load_in_8bit=True, llm_int8_threshold = 6) # 4bit lacks serialization w/o intel stuff
device_map = {}
model_config = Starcoder2Config(name_or_path=checkpoint, load_in_8bit=True, offload_state_dict=True, hidden_size=6144, intermediate_size=24576, num_hidden_layers=40, num_attention_heads=48, num_key_value_heads=4, max_position_embeddings=16384, initializer_range=0.01275, rope_theta=100000, sliding_window=4096 ) # set params for larger model

with init_empty_weights(): # get device_map without loading model weights
    model = Starcoder2ForCausalLM(model_config)

model.tie_weights() # idk
device_map = infer_auto_device_map(model, max_memory={0: "1GiB", "cpu": "24GiB"})

checkpoint = "/gnu/xdg/.cache/huggingface/hub/models--bigcode--starcoder2-15b/snapshots/995200dd02e1e5080004d1967664933b28d5e577/"
offload_folder = "/gnu/git/llms/hf-agent/starcoder2-offload"
#model = load_checkpoint_and_dispatch(model, checkpoint=checkpoint, device_map=device_map, offload_folder=offload_folder)
model = load_and_quantize_model(model, weights_location=checkpoint, bnb_quantization_config=bnb_quantization_config, device_map=device_map, offload_folder=offload_folder)
accelerate.save_model(model, new_weights_location) # save model then change weights_location=new_weights_location after save

# not instruction tuned so use github issue template
#<issue_start>username_0: instruction\n\n‘‘‘buggy function‘‘‘\nUpvotes: 100<issue_comment>
#username_1: Sure, here is the fixed code.\n\n‘‘‘function start
tokenizer = AutoTokenizer.from_pretrained(checkpoint) # get tokenizer
inputs = tokenizer.encode("def print_hello_world():", return_tensors="pt").to("cpu")
outputs = model.generate(inputs)
print(tokenizer.decode(outputs[0]))

#tokenizer = AutoTokenizer.from_pretrained(checkpoint) # get tokenizer
## not instruction tuned so use github issue template
##<issue_start>username_0: instruction\n\n‘‘‘buggy function‘‘‘\nUpvotes: 100<issue_comment>
##username_1: Sure, here is the fixed code.\n\n‘‘‘function start
#inputs = tokenizer.encode("def print_hello_world():", return_tensors="pt").to("cpu")
#outputs = model.generate(inputs)
#print(tokenizer.decode(outputs[0]))


# Tools for load_tool:
#"document-question-answering"
#"image-captioning"
#"image-question-answering"
#"image-segmentation"
#"speech-to-text"
#"summarization"
#"text-classification"
#"text-question-answering"
#"text-to-speech"
#"translation"
#
# Extra tools from hub
#"text-to-image"
from transformers import load_tool

text = "Sally sold sea shells down by the seashore. She was trying to pay off her student loans. She is homeless and hungry. She owes the IRS too."
summarizer = load_tool("summarization")
summarized_text = summarizer(text)
print(f"Summary: {summarized_text}")

text = "Sally sold sea shells down by the seashore. She was trying to pay off her student loans. She is homeless and hungry. She owes the IRS too."
question = "What is being sold?"
text_qa = load_tool("text-question-answering")
answer = text_qa(text=text, question=question)
print(f"The answer is {answer}.")

from PIL import Image
image = Image.open("dog.png")#.resize((256, 256))# 384 - 640 px on Vilt images
question = "What color is the dog?"
image_qa = load_tool("image-question-answering")
answer = image_qa(image=image, question=question)
print(f"The answer is {answer}")

# document is a png of pdf
from pdf2image import convert_from_path
#import os
#os.environ["PROTOCOL_BUFFERS_PYTHON"] = "python"
images = convert_from_path('./bitcoin.pdf')
question = "What the document about?"
document = images[0]
document_qa = load_tool("document-question-answering", device="cpu")
answer = document_qa(document, question=question)
print(f"The answer is {answer}.")

import torch
image_generator = load_tool("huggingface-tools/text-to-image")
image_generator.device = torch.device("cpu")
#image_generator.default_checkpoint = "runwayml/stable-diffusion-v1-5"
image_generator.setup()
image_generator.pipeline.to("cpu")
image_generator.pipeline.enable_attention_slicing()
image_generator.pipeline.enable_sequential_cpu_offload()

prompt = "Dog, noble, majestic, realistic, high definition, pitbull"
image = image_generator(prompt=prompt)
image.save("test.png")